Commit d8c349e5 authored by Klaus Zimmermann's avatar Klaus Zimmermann
Browse files

Improve logging in datahandling (closes #238)

parent 63bd7922
......@@ -11,6 +11,9 @@ import numpy as np
from .dask_setup import progress
logger = logging.getLogger(__name__)
MISSVAL = 1.0e20
......@@ -36,10 +39,10 @@ def prepare_input_data(datafiles):
for var_name, cubes in cubes_per_var_name.items():
if len(cubes) > 1:
inconsistent_var_names.append(var_name)
logging.error('Found more than one cube for var_name "{}".\n'
'{}'.format(
var_name,
'\n'.join(map(lambda c: str(c), cubes))))
logger.error('Found more than one cube for var_name "{}".\n'
'{}'.format(
var_name,
'\n'.join(map(lambda c: str(c), cubes))))
raise ValueError('Found too many cubes for var_names {}. '
'See log for details.'.format(inconsistent_var_names))
for c in cubes:
......@@ -52,13 +55,13 @@ def prepare_input_data(datafiles):
def save(result, output_filename, iterative_storage=False, client=None):
data = result.core_data()
if iterative_storage:
logging.info('Storing iteratively')
logging.info('Creating empty data')
logger.info('Storing iteratively')
logger.info('Creating empty data')
result.data = np.zeros(data.shape, data.dtype)
logging.info('Saving empty cube')
logger.info('Saving empty cube')
iris.save(result, output_filename, fill_value=MISSVAL,
local_keys=['proposed_standard_name'])
logging.info('Reopening output file and beginning storage')
logger.info('Reopening output file and beginning storage')
result.data = data
with netCDF4.Dataset(output_filename, 'a') as ds:
var = ds[result.var_name]
......@@ -69,27 +72,27 @@ def save(result, output_filename, iterative_storage=False, client=None):
cumulative = 0.
for i, result_data in enumerate(result.core_data()):
result_id = f'{i+1}/{no_slices}'
logging.info(f'Computing partial result {result_id}')
r = result_data.persist()
progress(r)
print()
logging.info(f'Storing result {result_id}')
var[i, ...] = r
logger.info(f'Storing partial result {result_id}')
start = end
end = time.time()
last = end - start
cumulative += last
eta = cumulative/(i+1)*no_slices
logging.info(f'Finished {result_id} in (last cum eta): '
f'{last:4.0f} {cumulative:4.0f} {eta:4.0f}')
logger.info(f'Finished {result_id} in (last cum eta): '
f'{last:4.0f} {cumulative:4.0f} {eta:4.0f}')
else:
logging.info('Storing non-iteratively')
logging.info('Computing result')
logger.info('Storing non-iteratively')
logger.info('Computing result')
r = client.compute(data)
progress(r)
print()
result.data = r.result()
logging.info(f'Storing result')
logger.info('Storing result')
iris.save(result, output_filename, fill_value=MISSVAL,
local_keys=['proposed_standard_name'])
logging.info('Calculation complete')
logger.info('Calculation complete')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment