Merge pull request #169 from ecmwf/feature/ECC-1691-DestineClimateDT

Feature/ECC-1691 destinE climateDT

definitions/grib2/destineLocalVersion.table

@@ -0,0 +1 @@
+1 1 MARS labeling

definitions/grib2/destine_activity.table

@@ -0,0 +1,4 @@
+0 unknown unknown
+1 CMIP6 Coupled Model Intercomparison Project Phase 6
+2 ScenarioMIP Scenario Model Intercomparison Project
+65535 65535 Missing

definitions/grib2/destine_dataset.table

@@ -0,0 +1,6 @@
+0 unknown unknown
+1 climate-dt Climate change adaptation digital twin
+2 extremes-dt Weather and geophysical extremes digital twin
+3 on-demand-climate-dt On-demand climate change adaptation digital twin
+4 on-demand-extremes-dt On-demand weather and geophysical extremes digital twin
+65535 65535 Missing

definitions/grib2/destine_experiment.table

@@ -0,0 +1,9 @@
+0 unknown unknown
+1 hist Historical
+2 amip Atmospheric Model Intercomparison Project
+3 SSP1-1.9 Shared Socio-economic Pathways 1-1.9
+4 SSP1-2.6 Shared Socio-economic Pathways 1-2.6
+5 SSP2-4.5 Shared Socio-economic Pathways 2-4.5
+6 SSP3-7.0 Shared Socio-economic Pathways 3-7.0
+7 SSP5-8.5 Shared Socio-economic Pathways 5-8.5
+65535 65535 Missing

definitions/grib2/destine_model.table

@@ -0,0 +1,6 @@
+0 unknown unknown
+1 IFS IFS with no ocean model
+2 IFS-NEMO IFS with NEMO ocean model
+3 IFS-FESOM IFS with FESOM ocean model
+4 ICON ICON with no ocean model
+65535 65535 Missing

definitions/grib2/destine_resolution.table

@@ -0,0 +1,4 @@
+0 unknown unknown
+1 standard Standard resolution model output with longer availability
+2 highres High resolution model output with limited availability
+65535 65535 Missing

definitions/grib2/local.destine.1.def

@@ -0,0 +1,7 @@
+# DestinE MARS layout
+
+# Base keywords for all datasets
+include "grib2/local.destine.base.def";
+
+# Keywords based on dataset
+template_nofail datasetTemplate "grib2/local.destine.[dataset:s].def";

definitions/grib2/local.destine.base.def

@@ -0,0 +1,4 @@
+# DestinE MARS base keywords
+
+codetable[2] dataset "grib2/destine_dataset.table" : dump;
+alias mars.dataset = dataset;

definitions/grib2/local.destine.climate-dt.def

@@ -0,0 +1,19 @@
+# DestinE MARS ClimateDT keywords
+
+codetable[2] activity "grib2/destine_activity.table" : dump;
+alias mars.activity = activity;
+
+codetable[2] experiment "grib2/destine_experiment.table" : dump;
+alias mars.experiment = experiment;
+
+unsigned[1] generation = 255 : dump;
+alias mars.generation = generation;
+
+codetable[2] model "grib2/destine_model.table" : dump;
+alias mars.model = model;
+
+unsigned[1] realization = 255 : dump;
+alias mars.realization = realization;
+
+codetable[2] resolution "grib2/destine_resolution.table" : dump;
+alias mars.resolution = resolution;

definitions/grib2/localConcepts/destine/destineOriginConcept.def

@@ -0,0 +1,10 @@
+# Concept destineOrigin
+"DWD"                                       = { centre=98; subCentre=78; }
+"Meteo-France"                              = { centre=98; subCentre=84; }
+"Meteo-France"                              = { centre=98; subCentre=85; }
+"MPI-M"                                     = { centre=98; subCentre=252; }
+"AWI"                                       = { centre=98; subCentre=253; }
+"BSC"                                       = { centre=98; subCentre=1000; }
+"CSC"                                       = { centre=98; subCentre=1001; }
+"DKRZ"                                      = { centre=98; subCentre=1002; }
+"Climate DT consortium (CSC,AWI,BSC,MPI-M)" = { centre=98; subCentre=1003; }

definitions/grib2/localConcepts/destine/gridSpecificationConcept.def

@@ -0,0 +1,13 @@
+# Concept gridSpecification
+'H16' = {numberOfDataPoints=3072; gridDefinitionTemplateNumber=150; Nside=16;}
+'H32' = {numberOfDataPoints=12288; gridDefinitionTemplateNumber=150; Nside=32;}
+'H64' = {numberOfDataPoints=49152; gridDefinitionTemplateNumber=150; Nside=64;}
+'H128' = {numberOfDataPoints=196608; gridDefinitionTemplateNumber=150; Nside=128;}
+'H256' = {numberOfDataPoints=786432; gridDefinitionTemplateNumber=150; Nside=256;}
+'H512' = {numberOfDataPoints=3145728; gridDefinitionTemplateNumber=150; Nside=512;}
+'H1024' = {numberOfDataPoints=12582912; gridDefinitionTemplateNumber=150; Nside=1024;}
+'O1280' = {numberOfDataPoints=6599680; gridDefinitionTemplateNumber=40; Nj=2560;}
+'O2560' = {numberOfDataPoints=26306560; gridDefinitionTemplateNumber=40; Nj=5120;}
+'O8000' = {numberOfDataPoints=256288000; gridDefinitionTemplateNumber=40; Nj=16000;}
+'F90' = {numberOfDataPoints=65160; gridDefinitionTemplateNumber=0; Ni=360; Nj=181; }
+'F360' = {numberOfDataPoints=1038240; gridDefinitionTemplateNumber=0; Ni=1440; Nj=721; }

definitions/grib2/post_meta_data.hook.products_12.def

@@ -0,0 +1,8 @@
+# (C) Copyright 2005- ECMWF.
+# Hooks post meta-data for DestinE (productionStatusOfProcessedData=12)
+
+# conceptsDir2 --> datasetForLocal
+concept gridSpecification(unknown, "gridSpecificationConcept.def",conceptsDir2,conceptsDir1): no_copy, read_only, dump;
+
+concept destineOrigin (unknown, "destineOriginConcept.def",conceptsDir2,conceptsDir1): no_copy, dump;
+

definitions/grib2/products_12.def

@@ -0,0 +1,5 @@
+# (C) Copyright 2005- ECMWF.
+
+# Destination Earth (DestinE)
+
+include "grib2/products_destine.def"

definitions/grib2/products_destine.def

@@ -0,0 +1,21 @@
+# (C) Copyright 2005- ECMWF.
+
+alias mars.expver = experimentVersionNumber;
+alias parameter.paramId=paramId;
+alias parameter.shortName=shortName;
+alias parameter.units=units;
+alias parameter.name=name;
+
+alias mars.class  = marsClass;
+alias mars.param = paramId;
+
+# Special rule for level type 103 'Specified height level above ground (m)'
+if(typeOfFirstFixedSurface == 103) {
+    # only the parameters above 10m
+    if (level > 10) {
+        constant heightLevelName = 'hl';
+        alias mars.levtype = heightLevelName;
+        # levelist was unaliased in template.4.horizontal.def so we must have it back
+        alias mars.levelist = level;
+    }
+}

definitions/grib2/section.2.def

@@ -34,6 +34,12 @@ if ( addEmptySection2 == 0 ) {
             template  crraSection "grib2/local.crra.[crraLocalVersion:l].def";
         }
 
+        if(productionStatusOfProcessedData == 12 || productionStatusOfProcessedData == 13) {
+            # destine = Destination Earth
+            codetable[2] destineLocalVersion 'grib2/destineLocalVersion.table' = 1 : dump;
+            template  destineSection "grib2/local.destine.[destineLocalVersion:l].def";
+        }
+
         codetable[2] grib2LocalSectionNumber 'grib2/grib2LocalSectionNumber.[centreForLocal:l].table' = 1 : dump;
 
         if (grib2LocalSectionNumber!=0) {

definitions/grib2/sections.def

@@ -44,6 +44,8 @@ meta md5Headers md5(startOfHeaders,lengthOfHeaders);
 
 lookup[1] sectionNumber(4) ;
 
+template_nofail hook_post_meta_data "grib2/post_meta_data.hook.products_[productionStatusOfProcessedData].def";
+
 if (!headersOnly) {
   if (sectionNumber == 5 or new() ) {
     position sectionPosition;

definitions/mars/grib.clmn.fc.def

@@ -0,0 +1,6 @@
+alias mars.year = year;
+alias mars.month = month;
+unalias mars.date;
+unalias mars.time;
+unalias mars.step;
+unalias mars.domain;

definitions/mars/grib.clte.fc.def

@@ -0,0 +1,5 @@
+alias mars.date = validityDate;
+meta padded_vtime sprintf("%.4d", validityTime): no_copy;
+alias mars.time = padded_vtime;
+unalias mars.step;
+unalias mars.domain;

definitions/mars/stream.table

@@ -69,6 +69,8 @@
 1095 wamf Wave monthly forecast
 1096 wmfm Wave monthly forecast means
 1097 smma Seasonal monthly means anomalies
+1098 clte Climate run output
+1099 clmn Climate run monthly means output
 1110 seap Sensitive area prediction
 1120 eefh Extended ensemble forecast hindcast
 1121 eehs Extended ensemble forecast hindcast statistics

tests/CMakeLists.txt

@@ -127,6 +127,7 @@ if( HAVE_BUILD_TOOLS )
         grib_ecc-1560
         grib_ecc-1571
         grib_ecc-1654
+        grib_ecc-1691
         bufr_ecc-1028
         bufr_ecc-1195
         bufr_ecc-1259

tests/grib_ecc-1691.sh

@@ -0,0 +1,95 @@
+#!/bin/sh
+# (C) Copyright 2005- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# 
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.ctest.sh
+
+REDIRECT=/dev/null
+
+# ECC-1691: Destination Earth ClimateDT metadata support
+
+label="grib_ecc-1691_test"
+temp_grib_a=temp.$label.a.grib
+temp_grib_b=temp.$label.b.grib
+destine_sample=temp.$label.destine.grib
+sample_grib2=$ECCODES_SAMPLES_PATH/GRIB2.tmpl
+
+tablesVersionLatest=$( ${tools_dir}/grib_get -p tablesVersionLatest $sample_grib2 )
+
+# Setup Destine pseudo-centre GRIB message
+# First latest tables version and add local section with MARS labeling
+${tools_dir}/grib_set -s tablesVersion=$tablesVersionLatest,setLocalDefinition=1,grib2LocalSectionNumber=1 $sample_grib2 $temp_grib_a
+
+# Then change to processed data = 12 --> DestinE
+${tools_dir}/grib_set -s productionStatusOfProcessedData=12,class=d1 $temp_grib_a $destine_sample
+
+# Check DestinE base related keys are present and correct
+grib_check_key_exists $destine_sample destineLocalVersion,dataset,marsClass,marsType,marsStream,experimentVersionNumber
+grib_check_key_equals $destine_sample "marsClass,dataset,destineLocalVersion" "d1 0 1"
+
+# Check setting dataset to climate-dt (1). Check keys are present and equal defaults
+${tools_dir}/grib_set -s dataset=1 $destine_sample $temp_grib_a
+
+grib_check_key_exists $temp_grib_a dataset,generation,activity,experiment,model,realization,resolution
+grib_check_key_equals $temp_grib_a "dataset,generation,activity,experiment,model,realization,resolution" "1 255 0 0 0 255 0"
+
+# Check an example where a few additional things are set in climate-dt
+${tools_dir}/grib_set -s dataset=1,generation=2,activity=1,experiment=1,model=2,realization=1,resolution=1 $destine_sample $temp_grib_a
+
+grib_check_key_equals $temp_grib_a "dataset,generation,activity,experiment,model,realization,resolution" "1 2 1 1 2 1 1"
+grib_check_key_equals $temp_grib_a "dataset:s,activity:s,experiment:s,model:s,resolution:s" "climate-dt CMIP6 hist IFS-NEMO standard"
+
+# Check setting dataset to extremes-dt (2). This time we will check string also
+${tools_dir}/grib_set -s dataset=2 $destine_sample $temp_grib_a
+
+grib_check_key_equals $temp_grib_a "dataset,dataset:s" "2 extremes-dt"
+
+# Check concepts. First check default value and then example of mapping
+grib_check_key_exists $destine_sample destineOrigin,gridSpecification
+grib_check_key_equals $destine_sample "destineOrigin,gridSpecification" "unknown unknown"
+
+# Below mapping is for MeteoFrance
+${tools_dir}/grib_set -s centre=98,subCentre=85 $destine_sample $temp_grib_a
+
+grib_check_key_equals $temp_grib_a "destineOrigin" "Meteo-France"
+
+# Below mapping is for H1024
+${tools_dir}/grib_set -s numberOfDataPoints=12582912,gridDefinitionTemplateNumber=150,Nside=1024 $destine_sample $temp_grib_a
+
+grib_check_key_equals $temp_grib_a "gridSpecification" "H1024"
+
+# Now check streams.
+# Setting stream clte and type fc should set mars.date and mars.time to validityDate and validityTime,
+# and mars.step should be unaliased
+
+${tools_dir}/grib_set -s stream=clte,type=fc $destine_sample $temp_grib_a
+
+result1=$( ${tools_dir}/grib_get -p mars.date,mars.time $temp_grib_a )
+result2=$( ${tools_dir}/grib_get -p validityDate,validityTime $temp_grib_a )
+[ "$result1" = "$result2" ]
+
+${tools_dir}/grib_ls -jm $temp_grib_a
+[ $( ${tools_dir}/grib_get -f -p mars.step $temp_grib_a ) = "not_found" ]
+
+# Setting stream clmn and type fc should set mars.year and mars.month to year and month,
+# and mars.date, mars.time, mars.step should be unaliased
+
+${tools_dir}/grib_set -s stream=clmn,type=fc $destine_sample $temp_grib_a
+
+result1=$( ${tools_dir}/grib_get -p mars.year,mars.month $temp_grib_a )
+result2=$( ${tools_dir}/grib_get -p year,month $temp_grib_a )
+[ "$result1" = "$result2" ]
+
+${tools_dir}/grib_ls -jm $temp_grib_a
+[ $( ${tools_dir}/grib_get -f -p mars.date $temp_grib_a ) = "not_found" ]
+[ $( ${tools_dir}/grib_get -f -p mars.time $temp_grib_a ) = "not_found" ]
+[ $( ${tools_dir}/grib_get -f -p mars.step $temp_grib_a ) = "not_found" ]
+
+# Clean up
+rm -f $temp_grib_a $temp_grib_b $destine_sample