Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
climix
climix
Commits
dcd5951e
Commit
dcd5951e
authored
May 04, 2021
by
Klaus Zimmermann
Browse files
Improve iterative storage (closes #239)
parent
d8c349e5
Changes
1
Hide whitespace changes
Inline
Side-by-side
climix/datahandling.py
View file @
dcd5951e
...
...
@@ -53,7 +53,7 @@ def prepare_input_data(datafiles):
def
save
(
result
,
output_filename
,
iterative_storage
=
False
,
client
=
None
):
data
=
result
.
core_data
()
data
=
result
.
core_data
()
.
rechunk
()
if
iterative_storage
:
logger
.
info
(
'Storing iteratively'
)
logger
.
info
(
'Creating empty data'
)
...
...
@@ -62,7 +62,6 @@ def save(result, output_filename, iterative_storage=False, client=None):
iris
.
save
(
result
,
output_filename
,
fill_value
=
MISSVAL
,
local_keys
=
[
'proposed_standard_name'
])
logger
.
info
(
'Reopening output file and beginning storage'
)
result
.
data
=
data
with
netCDF4
.
Dataset
(
output_filename
,
'a'
)
as
ds
:
var
=
ds
[
result
.
var_name
]
time_dim
=
result
.
coord_dims
(
'time'
)[
0
]
...
...
@@ -70,19 +69,20 @@ def save(result, output_filename, iterative_storage=False, client=None):
end
=
time
.
time
()
cumulative
=
0.
for
i
,
result_data
in
enumerate
(
result
.
core_data
()):
result_id
=
f
'
{
i
+
1
}
/
{
no_slices
}
'
r
=
result_data
.
persist
()
progress
(
r
)
print
()
logging
.
info
(
f
'Storing result
{
result_id
}
'
)
var
[
i
,
...]
=
r
start_index
=
0
for
result_data
in
data
.
blocks
:
result_id
=
f
'
{
start_index
+
1
}
/
{
no_slices
}
'
logger
.
info
(
f
'Storing partial result
{
result_id
}
'
)
end_index
=
start_index
+
result_data
.
shape
[
0
]
logger
.
debug
(
f
'
{
start_index
}
:
{
end_index
}
'
)
logger
.
debug
(
f
'
{
result_data
.
shape
}
'
)
var
[
start_index
:
end_index
,
...]
=
result_data
start_index
=
end_index
start
=
end
end
=
time
.
time
()
last
=
end
-
start
cumulative
+=
last
eta
=
cumulative
/
(
i
+
1
)
*
no_slices
eta
=
cumulative
/
(
start_index
+
1
)
*
no_slices
logger
.
info
(
f
'Finished
{
result_id
}
in (last cum eta): '
f
'
{
last
:
4.0
f
}
{
cumulative
:
4.0
f
}
{
eta
:
4.0
f
}
'
)
else
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment