.gitmodules

+[submodule "cf-index-meta"]
+	path = cf-index-meta
+	url = https://bitbucket.org/cf-index-meta/cf-index-meta.git

CHANGELOG.md

@@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.12.0] - 2020-02-20
+
+### Added
+
+- Added master table as submodule (closes #169)
+- Added the following index function
+  - extreme temperature range, etr (closes #166)
+  - diurnal temperature range (closes #153)
+  - interday diurnal temperature range (closes #154)
+  - percentile (closes #80)
+  - thresholded percentile (closes #85)
+- Added changelog utility scripts (closes #172)
+
+### Changed
+
+- Update metadata to master table version 0.1.0 (closes #171)
+
+### Fixed
+
+- Remove formatting for all `standard_name`s (fixes #167)
+
+
 ## [0.11.0] - 2020-02-12
 
 ### Added
@@ -249,6 +271,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 
 [unreleased]: https://git.smhi.se/climix/climix/compare/0.11.0...HEAD
+[0.12.0]: https://git.smhi.se/climix/climix/compare/0.11.0...0.12.0
 [0.11.0]: https://git.smhi.se/climix/climix/compare/0.10.0...0.11.0
 [0.10.0]: https://git.smhi.se/climix/climix/compare/0.9.0...0.10.0
 [0.9.0]: https://git.smhi.se/climix/climix/compare/0.8.0...0.9.0

cf-index-meta @ 0646d702

+Subproject commit 0646d702614f512bb10a6b83a70a110a0e0119a7

climix/editor/main.py

@@ -36,6 +36,7 @@ def parse_args():
         description=(f'An editor for a climate index thing, '
                      f'version {climix.__version__}.'))
     parser.add_argument('-f', '--force', action='store_true')
+    parser.add_argument('-t', '--table-version', default='<unknown>')
     parser.add_argument('document')
     return parser.parse_args()
 
@@ -62,11 +63,13 @@ def main():
     env = prepare_environment(args)
     var_definition_template = env.get_template('variables.yml')
     var_definitions = build_variables(args.document)
-    var_output = var_definition_template.render(variables=var_definitions)
+    var_output = var_definition_template.render(variables=var_definitions,
+                                                version=args.table_version)
 
     idx_definition_template = env.get_template('index_definitions.yml')
     idx_definitions = build_index_definitions(args.document)
-    idx_output = idx_definition_template.render(indices=idx_definitions)
+    idx_output = idx_definition_template.render(indices=idx_definitions,
+                                                version=args.table_version)
 
     with opened_w_force('variables.yml', args.force) as outfile:
         outfile.write(var_output)

climix/editor/templates/index_definitions.yml

+# These index definitions are auto-generated from the master table at
+# https://bitbucket.org/cf-index-meta/cf-index-meta
+
+# This is based on version {{ version }}.
+---
 indices:
 {% for idx in indices %}
   {{ idx.var_name }}:

climix/editor/templates/variables.yml

+# These variables are auto-generated from the master table at
+# https://bitbucket.org/cf-index-meta/cf-index-meta
+
+# This is based on version {{ version }}.
+---
 variables:
 {% for var in variables %}
   {{ var.var_name }}:

climix/etc/index_definitions.yml

+# These index definitions are auto-generated from the master table at
+# https://bitbucket.org/cf-index-meta/cf-index-meta
+
+# This is based on version 0.1.0.
+---
 indices:
   fd:
     reference: ETCCDI
@@ -1467,7 +1472,8 @@ indices:
       default: monthly
     output:
       var_name: "dtr"
-      standard_name: air_temperature
+      standard_name:
+      proposed_standard_name: air_temperature_difference
       long_name: "Mean Diurnal Temperature Range"
       units: "degree_Celsius"
       cell_methods:
@@ -1477,7 +1483,7 @@ indices:
       low_data: tasmin
       high_data: tasmax
     index_function:
-      name: dtr
+      name: diurnal_temperature_range
       parameters:
     ET:
       short_name: "dtr"
@@ -1485,37 +1491,6 @@ indices:
       definition: "Monthly mean difference between TX and TN"
       comment: "mean of daily temperature range"
 
-  tx95t:
-    reference: ET-SCI
-    period:
-      allowed:
-        annual:
-        seasonal:
-        monthly:
-      default: annual
-    output:
-      var_name: "tx95t"
-      standard_name: air_temperature
-      long_name: "Very Warm Days Threshold (95th percentile of Tmax)"
-      units: "degree_Celsius"
-      cell_methods:
-    input:
-      data: tasmax
-    index_function:
-      name: percentile
-      parameters:
-        percentile:
-          kind: quantity
-          standard_name:
-          proposed_standard_name: quantile
-          data: 95
-          units: "%"
-    ET:
-      short_name: "tx95t"
-      long_name: "Very warm day threshold"
-      definition: "Value of 95th percentile of TX"
-      comment: "95th percentile of daily maximum temperature"
-
   tx{PRC}pctl:
     reference: CLIPC
     period:
@@ -1535,7 +1510,7 @@ indices:
     index_function:
       name: percentile
       parameters:
-        percentile:
+        percentiles:
           kind: quantity
           standard_name:
           proposed_standard_name: quantile
@@ -1566,7 +1541,7 @@ indices:
     index_function:
       name: percentile
       parameters:
-        percentile:
+        percentiles:
           kind: quantity
           standard_name:
           proposed_standard_name: quantile
@@ -1597,7 +1572,7 @@ indices:
     index_function:
       name: percentile
       parameters:
-        percentile:
+        percentiles:
           kind: quantity
           standard_name:
           proposed_standard_name: quantile
@@ -1915,6 +1890,7 @@ indices:
         threshold:
           kind: quantity
           standard_name: lwe_precipitation_rate
+          long_name: "Wet day threshold"
           data: 1
           units: "mm day-1"
         condition:
@@ -1951,6 +1927,7 @@ indices:
         threshold:
           kind: quantity
           standard_name: lwe_precipitation_rate
+          long_name: "Wet day threshold"
           data: 1
           units: "mm day-1"
         condition:
@@ -1990,6 +1967,7 @@ indices:
         threshold:
           kind: quantity
           standard_name: lwe_precipitation_rate
+          long_name: "Wet day threshold"
           data: 1
           units: "mm day-1"
         condition:
@@ -2028,6 +2006,7 @@ indices:
         threshold:
           kind: quantity
           standard_name: lwe_precipitation_rate
+          long_name: "Wet day threshold"
           data: 1
           units: "mm day-1"
         condition:
@@ -2057,7 +2036,7 @@ indices:
       units: "mm day-1"
       cell_methods:
         - time: sum within days
-        - time: sum over days
+        - time: mean over days
     input:
       data: pr
     index_function:
@@ -2066,6 +2045,7 @@ indices:
         threshold:
           kind: quantity
           standard_name: lwe_precipitation_rate
+          long_name: "Wet day threshold"
           data: 1
           units: "mm day-1"
         condition:
@@ -2080,6 +2060,46 @@ indices:
       definition: "PRCPTOT / Nwetdays"
       comment: "mean daily total precipitation during days having at least 1 mm"
 
+  r{PRC}pctl:
+    reference: CLIPC
+    period:
+      allowed:
+        annual:
+        seasonal:
+        monthly:
+      default: annual
+    output:
+      var_name: "r{PRC}pctl"
+      standard_name: lwe_thickness_of_precipitation_amount
+      long_name: "{PRC}th percentile of precipitation during wet days (Precip >= 1mm)"
+      units: "mm"
+      cell_methods:
+    input:
+      data: pr
+    index_function:
+      name: thresholded_percentile
+      parameters:
+        threshold:
+          kind: quantity
+          standard_name: lwe_precipitation_rate
+          long_name: "Wet day threshold"
+          data: 1
+          units: "mm day-1"
+        condition:
+          kind: operator
+          operator: ">"
+        percentiles:
+          kind: quantity
+          standard_name:
+          proposed_standard_name: quantile
+          data: {PRC}
+          units: "%"
+    ET:
+      short_name:
+      long_name:
+      definition:
+      comment:
+
   rx1day:
     reference: ETCCDI
     period:
@@ -2104,6 +2124,7 @@ indices:
         threshold:
           kind: quantity
           standard_name: lwe_precipitation_rate
+          long_name: "Wet day threshold"
           data: 1
           units: "mm day-1"
         condition:
@@ -2313,6 +2334,7 @@ indices:
         threshold:
           kind: quantity
           standard_name: lwe_precipitation_rate
+          long_name: "Wet day threshold"
           data: 1
           units: "mm day-1"
         condition:
@@ -2468,6 +2490,7 @@ indices:
         threshold:
           kind: quantity
           standard_name: lwe_precipitation_rate
+          long_name: "Wet day threshold"
           data: 1
           units: "mm day-1"
         condition:
@@ -2687,10 +2710,11 @@ indices:
         annual:
         seasonal:
         monthly:
-      default: annual
+      default: monthly
     output:
       var_name: "etr"
-      standard_name: air_temperature
+      standard_name:
+      proposed_standard_name: air_temperature_difference
       long_name: "Intra-period extreme temperature range"
       units: "degree_Celsius"
       cell_methods:
@@ -2699,14 +2723,8 @@ indices:
       low_data: tasmin
       high_data: tasmax
     index_function:
-      name: statistics
+      name: extreme_temperature_range
       parameters:
-        low_reducer:
-          kind: reducer
-          reducer: min
-        high_reducer:
-          kind: reducer
-          reducer: max
     ET:
       short_name:
       long_name:
@@ -2843,6 +2861,33 @@ indices:
       definition:
       comment:
 
+  vdtr:
+    reference: ECA&D
+    period:
+      allowed:
+        annual:
+        seasonal:
+        monthly:
+      default: monthly
+    output:
+      var_name: "vdtr"
+      standard_name:
+      proposed_standard_name: air_temperature_difference
+      long_name: "Mean day-to-day variation in Diurnal Temperature Range"
+      units: "degree_Celsius"
+      cell_methods:
+    input:
+      low_data: tasmin
+      high_data: tasmax
+    index_function:
+      name: interday_diurnal_temperature_range
+      parameters:
+    ET:
+      short_name:
+      long_name:
+      definition:
+      comment:
+
   nzero:
     reference: SMHI
     period:

climix/etc/metadata.yml

@@ -33,6 +33,27 @@ index_functions:
       condition:
         kind: operator
 
+  diurnal_temperature_range:
+    description: |
+      This index function takes two inputs, low_data and high_data, i.e.
+      daily minimum and maximum temperature.
+      From this it calculated the average diurnal temperature range.
+
+  interday_diurnal_temperature_range:
+    description: |
+      This index function takes two inputs, low_data and high_data, i.e.
+      daily minimum and maximum temperature.
+      From this it calculates the average day-to-day difference in
+      diurnal temperature range.
+
+  extreme_temperature_range:
+    description: |
+      This index function takes two inputs, low_data and high_data, i.e.
+      daily minimum and maximum temperature.
+      From this it calculates the extreme temperature range as the
+      maximum of daily maximum temperature minus the minimum of daily
+      minimum temperature.
+
   first_occurrence:
     description: |
       Calculates the first time some condition is met.
@@ -102,6 +123,30 @@ index_functions:
       reducer:
         kind: reducer
 
+  percentile:
+    description: |
+      Calculates percentiles, default method is "linear" option of numpy and dask.
+    parameters:
+      percentiles:
+        kind: quantity
+
+  thresholded_percentile:
+    description: |
+      Calculates percentiles of data for which some condition is met.
+      First, the threshold is transformed to the same standard_name and units as
+      the input data.
+      Then the thresholding is performed as condition(data, threshold), ie
+      if condition is <, data < threshold.
+      Finally, the percentiles is calculated for those data values that fulfill
+      the condition. The default percentile method is "linear" option of numpy.
+    parameters:
+      threshold:
+        kind: quantity
+      condition:
+        kind: operator
+      percentiles:
+        kind: quantity
+
   temperature_sum:
     description: |
       Calculates the temperature sum above/below a threshold. First, the threshold

climix/etc/variables.yml

+# These variables are auto-generated from the master table at
+# https://bitbucket.org/cf-index-meta/cf-index-meta
+
+# This is based on version 0.1.0.
+---
 variables:
   pr:
     standard_name: precipitation_flux
@@ -30,6 +35,7 @@ variables:
       - TG
       - MEANT
       - meanT
+      - tasmidpoint
 
   tasmax:
     standard_name: air_temperature

climix/index_functions/__init__.py

@@ -3,10 +3,15 @@
 from .index_functions import (  # noqa: F401
     CountLevelCrossings,
     CountOccurrences,
+    DiurnalTemperatureRange,
+    InterdayDiurnalTemperatureRange,
+    ExtremeTemperatureRange,
     FirstOccurrence,
     LastOccurrence,
+    Percentile,
     SpellLength,
     Statistics,
     ThresholdedStatistics,
+    ThresholdedPercentile,    
     TemperatureSum,
 )

climix/index_functions/index_functions.py

@@ -56,6 +56,68 @@ class CountOccurrences(ThresholdMixin, IndexFunction):
         return res.astype('float32')
 
 
+class DiurnalTemperatureRange(IndexFunction):
+    def __init__(self):
+        super().__init__(units=Unit('degree_Celsius'))
+
+    def prepare(self, input_cubes):
+        props = {(cube.dtype, cube.units, cube.standard_name)
+                 for cube in input_cubes.values()}
+        assert len(props) == 1
+        dtype, units, standard_name = props.pop()
+        assert units in [Unit('Kelvin'), Unit('degree_Celsius')]
+        super().prepare(input_cubes)
+
+    def call_func(self, data, axis, **kwargs):
+        res = (data['high_data'] - data['low_data']).mean(axis=axis)
+        return res.astype('float32')
+
+    lazy_func = call_func
+
+
+class ExtremeTemperatureRange(IndexFunction):
+    def __init__(self):
+        super().__init__(units=Unit('degree_Celsius'))
+
+    def prepare(self, input_cubes):
+        props = {(cube.dtype, cube.units, cube.standard_name)
+                 for cube in input_cubes.values()}
+        assert len(props) == 1
+        dtype, units, standard_name = props.pop()
+        assert (units == Unit('Kelvin')) or (units == Unit('degree_Celsius'))
+        super().prepare(input_cubes)
+
+    def call_func(self, data, axis, **kwargs):
+        res = (data['high_data'].max(axis=axis) -
+               data['low_data'].min(axis=axis))
+        return res.astype('float32')
+
+    lazy_func = call_func
+
+
+class InterdayDiurnalTemperatureRange(IndexFunction):
+    def __init__(self):
+        super().__init__(units=Unit('degree_Celsius'))
+
+    def prepare(self, input_cubes):
+        props = {(cube.dtype, cube.units, cube.standard_name)
+                 for cube in input_cubes.values()}
+        assert len(props) == 1
+        dtype, units, standard_name = props.pop()
+        assert units in [Unit('Kelvin'), Unit('degree_Celsius')]
+        super().prepare(input_cubes)
+
+    def call_func(self, data, axis, **kwargs):
+        res = np.absolute(np.diff(data['high_data'] -
+                                  data['low_data'], axis=axis)).mean(axis=axis)
+        return res.astype('float32')
+
+    def lazy_func(self, data, axis, **kwargs):
+        res = da.absolute(da.diff(data['high_data'] -
+                                  data['low_data'], axis=axis)).mean(axis=axis)
+        return res.astype('float32')
+
+
 class FirstOccurrence(ThresholdMixin, IndexFunction):
     def __init__(self, threshold, condition):
         super().__init__(threshold, condition, units=Unit('days'))
@@ -224,6 +286,81 @@ class ThresholdedStatistics(ThresholdMixin, ReducerMixin, IndexFunction):
         return res.astype('float32')
 
 
+class Percentile(IndexFunction):
+    def __init__(self, percentiles, interpolation='linear'):
+        super().__init__(units=Unit('days'))
+        points = percentiles.points
+        assert np.all(points > 0)
+        assert np.all(points < 100)
+        self.percentiles = percentiles
+        self.interpolation = interpolation
+        self.units = '%'
+
+    def prepare(self, input_cubes):
+        super().prepare(input_cubes)
+        ref_cube = next(iter(input_cubes.values()))
+        self.standard_name = ref_cube.standard_name
+        self.units = ref_cube.units
+
+    def call_func(self, data, axis, **kwargs):
+        axis = normalize_axis(axis, data.ndim)
+        res = np.percentile(data, q=self.percentiles, axis=axis,
+                            interpolation=self.interpolation)
+        return res.astype('float32')
+
+    def lazy_func(self, data, axis, **kwargs):
+        axis = normalize_axis(axis, data.ndim)
+
+        def percentile(arr):
+            return np.percentile(arr,
+                                 q=self.percentiles.points,
+                                 interpolation=self.interpolation)
+        res = da.apply_along_axis(percentile, axis=axis, arr=data).squeeze()
+        return res.astype('float32')
+
+
+class ThresholdedPercentile(ThresholdMixin, IndexFunction):
+    def __init__(self, threshold, condition,
+                 percentiles, interpolation='linear'):
+        super().__init__(threshold, condition)
+        points = percentiles.points
+        assert np.all(points > 0)
+        assert np.all(points < 100)
+        self.percentiles = percentiles
+        self.interpolation = interpolation
+
+    def prepare(self, input_cubes):
+        super().prepare(input_cubes)
+        ref_cube = next(iter(input_cubes.values()))
+        self.standard_name = ref_cube.standard_name
+        self.units = ref_cube.units
+
+    def call_func(self, data, axis, **kwargs):
+        axis = normalize_axis(axis, data.ndim)
+        mask = np.ma.getmaskarray(data).any(axis=axis)
+        comb = self.condition(data, self.threshold.points)
+        res = np.percentile(np.ma.masked_where(~comb, data),
+                            q=self.percentiles.points, axis=axis,
+                            interpolation=self.interpolation)
+        res = np.ma.masked_array(da.ma.getdata(res), mask)
+        return res.astype('float32')
+
+    def lazy_func(self, data, axis, **kwargs):
+        axis = normalize_axis(axis, data.ndim)
+        mask = da.ma.getmaskarray(data).any(axis=axis)
+        comb = self.condition(data, self.threshold.points)
+
+        def percentile(arr):
+            return np.percentile(arr,
+                                 q=self.percentiles.points,
+                                 interpolation=self.interpolation)
+        res = da.apply_along_axis(
+            percentile, axis=axis,
+            arr=np.ma.masked_where(~comb, data)).squeeze()
+        res = da.ma.masked_array(da.ma.getdata(res), mask)
+        return res.astype('float32')
+
+
 class TemperatureSum(ThresholdMixin, IndexFunction):
     def __init__(self, threshold, condition):
         super().__init__(threshold, condition, units=Unit('days'))

climix/metadata.py

@@ -54,8 +54,8 @@ class OutputVariable:
     def instantiate(self, parameters):
         return OutputVariable(
             format_var_name(self.var_name, parameters),
-            self.standard_name.format(**parameters),
-            self.proposed_standard_name.format(**parameters),
+            self.standard_name,
+            self.proposed_standard_name,
             self.long_name.format(**parameters),
             self.units,
             self.cell_methods)
@@ -71,8 +71,9 @@ class InputVariable:
     def instantiate(self, parameters):
         return InputVariable(
             format_var_name(self.var_name, parameters),
-            self.standard_name.format(**parameters),
-            self.cell_methods, self.aliases)
+            self.standard_name,
+            self.cell_methods,
+            self.aliases)
 
 
 def build_variable(name, variable, path):
@@ -121,7 +122,7 @@ class ParameterQuantity:
             ln = ln.format(**parameters)
         param = ParameterQuantity(
             format_var_name(self.var_name, parameters),
-            self.standard_name.format(**parameters),
+            self.standard_name,
             data,
             self.units,
             ln)

setup.py

@@ -44,7 +44,7 @@ setuptools.setup(
         'scitools-iris>=2.2.0',
     ],
     extras_require={
-        'editor': ['pyexcel', 'pyexcel-odsr', 'jinja2']
+        'editor': ['pyexcel', 'pyexcel-xls', 'jinja2']
     },
     entry_points={
         'console_scripts': [
@@ -58,8 +58,13 @@ setuptools.setup(
             'last_occurrence=climix.index_functions:LastOccurrence',
             'spell_length=climix.index_functions:SpellLength',
             'statistics=climix.index_functions:Statistics',
+            'percentile=climix.index_functions:Percentile',
             'thresholded_statistics=climix.index_functions:ThresholdedStatistics',
+            'thresholded_percentile=climix.index_functions:ThresholdedPercentile',            
             'temperature_sum=climix.index_functions:TemperatureSum',
+            'diurnal_temperature_range=climix.index_functions:DiurnalTemperatureRange',
+            'interday_diurnal_temperature_range=climix.index_functions:InterdayDiurnalTemperatureRange',
+            'extreme_temperature_range=climix.index_functions:ExtremeTemperatureRange',
         ],
     },
     project_urls={

utils/generate-changelog.sh

+#!/bin/bash
+
+for tag in $(git tag -l --sort=-version:refname)
+do
+    pre=$(git describe --abbrev=0 $tag^ 2>/dev/null)
+    tag_date=$(git log -1 --format=%ad --date=short $tag)
+    echo "## [$tag] - $tag_date"
+    echo "### Added"
+    echo "### Changed"
+    echo "### Deprecated"
+    echo "### Removed"
+    echo "### Fixed"
+    echo "### Security"
+    git shortlog $pre..$tag
+done

utils/update-cf-index-meta.sh

+#!/bin/bash
+
+SCRIPT_DIR="$(dirname $(readlink -f "$0"))"
+BASE_DIR="$SCRIPT_DIR/.."
+TABLE_DIR="$BASE_DIR/cf-index-meta"
+OUTPUT_DIR="$BASE_DIR/climix/etc"
+
+(cd $BASE_DIR && git submodule update --remote)
+
+TABLE_VERSION=$(cd $TABLE_DIR && git describe --dirty)
+
+(cd $OUTPUT_DIR &&
+     climix-editor -t "$TABLE_VERSION" -f $TABLE_DIR/master_table.xls)

utils/update-changelog.sh

+#!/bin/bash
+
+tag=$1
+
+pre=$(git describe --abbrev=0 2>/dev/null)
+tag_date=$(git log -1 --format=%ad --date=short)
+echo -e "## [$tag] - $tag_date\n"
+echo -e "### Added\n"
+echo -e "### Changed\n"
+echo -e "### Deprecated\n"
+echo -e "### Removed\n"
+echo -e "### Fixed\n"
+echo -e "### Security\n\n"
+git shortlog $pre..