Commit d8c349e5 authored by Klaus Zimmermann's avatar Klaus Zimmermann
Browse files

Improve logging in datahandling (closes #238)

parent 63bd7922
......@@ -11,6 +11,9 @@ import numpy as np
from .dask_setup import progress
logger = logging.getLogger(__name__)
MISSVAL = 1.0e20
......@@ -36,10 +39,10 @@ def prepare_input_data(datafiles):
for var_name, cubes in cubes_per_var_name.items():
if len(cubes) > 1:
logging.error('Found more than one cube for var_name "{}".\n'
'\n'.join(map(lambda c: str(c), cubes))))
logger.error('Found more than one cube for var_name "{}".\n'
'\n'.join(map(lambda c: str(c), cubes))))
raise ValueError('Found too many cubes for var_names {}. '
'See log for details.'.format(inconsistent_var_names))
for c in cubes:
......@@ -52,13 +55,13 @@ def prepare_input_data(datafiles):
def save(result, output_filename, iterative_storage=False, client=None):
data = result.core_data()
if iterative_storage:'Storing iteratively')'Creating empty data')'Storing iteratively')'Creating empty data') = np.zeros(data.shape, data.dtype)'Saving empty cube')'Saving empty cube'), output_filename, fill_value=MISSVAL,
local_keys=['proposed_standard_name'])'Reopening output file and beginning storage')'Reopening output file and beginning storage') = data
with netCDF4.Dataset(output_filename, 'a') as ds:
var = ds[result.var_name]
......@@ -69,27 +72,27 @@ def save(result, output_filename, iterative_storage=False, client=None):
cumulative = 0.
for i, result_data in enumerate(result.core_data()):
result_id = f'{i+1}/{no_slices}''Computing partial result {result_id}')
r = result_data.persist()
print()'Storing result {result_id}')
var[i, ...] = r'Storing partial result {result_id}')
start = end
end = time.time()
last = end - start
cumulative += last
eta = cumulative/(i+1)*no_slices'Finished {result_id} in (last cum eta): '
f'{last:4.0f} {cumulative:4.0f} {eta:4.0f}')'Finished {result_id} in (last cum eta): '
f'{last:4.0f} {cumulative:4.0f} {eta:4.0f}')
else:'Storing non-iteratively')'Computing result')'Storing non-iteratively')'Computing result')
r = client.compute(data)
print() = r.result()'Storing result')'Storing result'), output_filename, fill_value=MISSVAL,
local_keys=['proposed_standard_name'])'Calculation complete')'Calculation complete')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment