diff --git a/definitions/grib2/destineLocalVersion.table b/definitions/grib2/destineLocalVersion.table new file mode 100644 index 000000000..5d488d2ae --- /dev/null +++ b/definitions/grib2/destineLocalVersion.table @@ -0,0 +1 @@ +1 1 MARS labeling diff --git a/definitions/grib2/destine_activity.table b/definitions/grib2/destine_activity.table new file mode 100644 index 000000000..852b275df --- /dev/null +++ b/definitions/grib2/destine_activity.table @@ -0,0 +1,4 @@ +0 unknown unknown +1 CMIP6 Coupled Model Intercomparison Project Phase 6 +2 ScenarioMIP Scenario Model Intercomparison Project +65535 65535 Missing diff --git a/definitions/grib2/destine_dataset.table b/definitions/grib2/destine_dataset.table new file mode 100644 index 000000000..58b82fc41 --- /dev/null +++ b/definitions/grib2/destine_dataset.table @@ -0,0 +1,6 @@ +0 unknown unknown +1 climate-dt Climate change adaptation digital twin +2 extremes-dt Weather and geophysical extremes digital twin +3 on-demand-climate-dt On-demand climate change adaptation digital twin +4 on-demand-extremes-dt On-demand weather and geophysical extremes digital twin +65535 65535 Missing diff --git a/definitions/grib2/destine_experiment.table b/definitions/grib2/destine_experiment.table new file mode 100644 index 000000000..1429f71ac --- /dev/null +++ b/definitions/grib2/destine_experiment.table @@ -0,0 +1,9 @@ +0 unknown unknown +1 hist Historical +2 amip Atmospheric Model Intercomparison Project +3 SSP1-1.9 Shared Socio-economic Pathways 1-1.9 +4 SSP1-2.6 Shared Socio-economic Pathways 1-2.6 +5 SSP2-4.5 Shared Socio-economic Pathways 2-4.5 +6 SSP3-7.0 Shared Socio-economic Pathways 3-7.0 +7 SSP5-8.5 Shared Socio-economic Pathways 5-8.5 +65535 65535 Missing diff --git a/definitions/grib2/destine_model.table b/definitions/grib2/destine_model.table new file mode 100644 index 000000000..1e33fbac1 --- /dev/null +++ b/definitions/grib2/destine_model.table @@ -0,0 +1,6 @@ +0 unknown unknown +1 IFS IFS with no ocean model +2 IFS-NEMO IFS with NEMO ocean model +3 IFS-FESOM IFS with FESOM ocean model +4 ICON ICON with no ocean model +65535 65535 Missing diff --git a/definitions/grib2/destine_resolution.table b/definitions/grib2/destine_resolution.table new file mode 100644 index 000000000..48d816018 --- /dev/null +++ b/definitions/grib2/destine_resolution.table @@ -0,0 +1,4 @@ +0 unknown unknown +1 standard Standard resolution model output with longer availability +2 highres High resolution model output with limited availability +65535 65535 Missing \ No newline at end of file diff --git a/definitions/grib2/local.destine.1.def b/definitions/grib2/local.destine.1.def new file mode 100644 index 000000000..db1f2f389 --- /dev/null +++ b/definitions/grib2/local.destine.1.def @@ -0,0 +1,7 @@ +# DestinE MARS layout + +# Base keywords for all datasets +include "grib2/local.destine.base.def"; + +# Keywords based on dataset +template_nofail datasetTemplate "grib2/local.destine.[dataset:s].def"; diff --git a/definitions/grib2/local.destine.base.def b/definitions/grib2/local.destine.base.def new file mode 100644 index 000000000..9a87b6b9a --- /dev/null +++ b/definitions/grib2/local.destine.base.def @@ -0,0 +1,4 @@ +# DestinE MARS base keywords + +codetable[2] dataset "grib2/destine_dataset.table" : dump; +alias mars.dataset = dataset; \ No newline at end of file diff --git a/definitions/grib2/local.destine.climate-dt.def b/definitions/grib2/local.destine.climate-dt.def new file mode 100644 index 000000000..61df737fc --- /dev/null +++ b/definitions/grib2/local.destine.climate-dt.def @@ -0,0 +1,19 @@ +# DestinE MARS ClimateDT keywords + +codetable[2] activity "grib2/destine_activity.table" : dump; +alias mars.activity = activity; + +codetable[2] experiment "grib2/destine_experiment.table" : dump; +alias mars.experiment = experiment; + +unsigned[1] generation = 255 : dump; +alias mars.generation = generation; + +codetable[2] model "grib2/destine_model.table" : dump; +alias mars.model = model; + +unsigned[1] realization = 255 : dump; +alias mars.realization = realization; + +codetable[2] resolution "grib2/destine_resolution.table" : dump; +alias mars.resolution = resolution; diff --git a/definitions/grib2/localConcepts/destine/destineOriginConcept.def b/definitions/grib2/localConcepts/destine/destineOriginConcept.def new file mode 100644 index 000000000..aabe1dca6 --- /dev/null +++ b/definitions/grib2/localConcepts/destine/destineOriginConcept.def @@ -0,0 +1,10 @@ +# Concept destineOrigin +"DWD" = { centre=98; subCentre=78; } +"Meteo-France" = { centre=98; subCentre=84; } +"Meteo-France" = { centre=98; subCentre=85; } +"MPI-M" = { centre=98; subCentre=252; } +"AWI" = { centre=98; subCentre=253; } +"BSC" = { centre=98; subCentre=1000; } +"CSC" = { centre=98; subCentre=1001; } +"DKRZ" = { centre=98; subCentre=1002; } +"Climate DT consortium (CSC,AWI,BSC,MPI-M)" = { centre=98; subCentre=1003; } diff --git a/definitions/grib2/localConcepts/destine/gridSpecificationConcept.def b/definitions/grib2/localConcepts/destine/gridSpecificationConcept.def new file mode 100644 index 000000000..1ae35446d --- /dev/null +++ b/definitions/grib2/localConcepts/destine/gridSpecificationConcept.def @@ -0,0 +1,13 @@ +# Concept gridSpecification +'H16' = {numberOfDataPoints=3072; gridDefinitionTemplateNumber=150; Nside=16;} +'H32' = {numberOfDataPoints=12288; gridDefinitionTemplateNumber=150; Nside=32;} +'H64' = {numberOfDataPoints=49152; gridDefinitionTemplateNumber=150; Nside=64;} +'H128' = {numberOfDataPoints=196608; gridDefinitionTemplateNumber=150; Nside=128;} +'H256' = {numberOfDataPoints=786432; gridDefinitionTemplateNumber=150; Nside=256;} +'H512' = {numberOfDataPoints=3145728; gridDefinitionTemplateNumber=150; Nside=512;} +'H1024' = {numberOfDataPoints=12582912; gridDefinitionTemplateNumber=150; Nside=1024;} +'O1280' = {numberOfDataPoints=6599680; gridDefinitionTemplateNumber=40; Nj=2560;} +'O2560' = {numberOfDataPoints=26306560; gridDefinitionTemplateNumber=40; Nj=5120;} +'O8000' = {numberOfDataPoints=256288000; gridDefinitionTemplateNumber=40; Nj=16000;} +'F90' = {numberOfDataPoints=65160; gridDefinitionTemplateNumber=0; Ni=360; Nj=181; } +'F360' = {numberOfDataPoints=1038240; gridDefinitionTemplateNumber=0; Ni=1440; Nj=721; } diff --git a/definitions/grib2/post_meta_data.hook.products_12.def b/definitions/grib2/post_meta_data.hook.products_12.def new file mode 100644 index 000000000..17a4c2001 --- /dev/null +++ b/definitions/grib2/post_meta_data.hook.products_12.def @@ -0,0 +1,8 @@ +# (C) Copyright 2005- ECMWF. +# Hooks post meta-data for DestinE (productionStatusOfProcessedData=12) + +# conceptsDir2 --> datasetForLocal +concept gridSpecification(unknown, "gridSpecificationConcept.def",conceptsDir2,conceptsDir1): no_copy, read_only, dump; + +concept destineOrigin (unknown, "destineOriginConcept.def",conceptsDir2,conceptsDir1): no_copy, dump; + diff --git a/definitions/grib2/products_12.def b/definitions/grib2/products_12.def new file mode 100644 index 000000000..35c5f909c --- /dev/null +++ b/definitions/grib2/products_12.def @@ -0,0 +1,5 @@ +# (C) Copyright 2005- ECMWF. + +# Destination Earth (DestinE) + +include "grib2/products_destine.def" diff --git a/definitions/grib2/products_destine.def b/definitions/grib2/products_destine.def new file mode 100644 index 000000000..b1d0c4aee --- /dev/null +++ b/definitions/grib2/products_destine.def @@ -0,0 +1,21 @@ +# (C) Copyright 2005- ECMWF. + +alias mars.expver = experimentVersionNumber; +alias parameter.paramId=paramId; +alias parameter.shortName=shortName; +alias parameter.units=units; +alias parameter.name=name; + +alias mars.class = marsClass; +alias mars.param = paramId; + +# Special rule for level type 103 'Specified height level above ground (m)' +if(typeOfFirstFixedSurface == 103) { + # only the parameters above 10m + if (level > 10) { + constant heightLevelName = 'hl'; + alias mars.levtype = heightLevelName; + # levelist was unaliased in template.4.horizontal.def so we must have it back + alias mars.levelist = level; + } +} \ No newline at end of file diff --git a/definitions/grib2/section.2.def b/definitions/grib2/section.2.def index f7da1537a..f6081b10b 100644 --- a/definitions/grib2/section.2.def +++ b/definitions/grib2/section.2.def @@ -34,6 +34,12 @@ if ( addEmptySection2 == 0 ) { template crraSection "grib2/local.crra.[crraLocalVersion:l].def"; } + if(productionStatusOfProcessedData == 12 || productionStatusOfProcessedData == 13) { + # destine = Destination Earth + codetable[2] destineLocalVersion 'grib2/destineLocalVersion.table' = 1 : dump; + template destineSection "grib2/local.destine.[destineLocalVersion:l].def"; + } + codetable[2] grib2LocalSectionNumber 'grib2/grib2LocalSectionNumber.[centreForLocal:l].table' = 1 : dump; if (grib2LocalSectionNumber!=0) { diff --git a/definitions/grib2/sections.def b/definitions/grib2/sections.def index ed998ca61..19f6b61db 100644 --- a/definitions/grib2/sections.def +++ b/definitions/grib2/sections.def @@ -44,6 +44,8 @@ meta md5Headers md5(startOfHeaders,lengthOfHeaders); lookup[1] sectionNumber(4) ; +template_nofail hook_post_meta_data "grib2/post_meta_data.hook.products_[productionStatusOfProcessedData].def"; + if (!headersOnly) { if (sectionNumber == 5 or new() ) { position sectionPosition; diff --git a/definitions/mars/grib.clmn.fc.def b/definitions/mars/grib.clmn.fc.def new file mode 100644 index 000000000..34723b605 --- /dev/null +++ b/definitions/mars/grib.clmn.fc.def @@ -0,0 +1,6 @@ +alias mars.year = year; +alias mars.month = month; +unalias mars.date; +unalias mars.time; +unalias mars.step; +unalias mars.domain; \ No newline at end of file diff --git a/definitions/mars/grib.clte.fc.def b/definitions/mars/grib.clte.fc.def new file mode 100644 index 000000000..f75dd3faf --- /dev/null +++ b/definitions/mars/grib.clte.fc.def @@ -0,0 +1,5 @@ +alias mars.date = validityDate; +meta padded_vtime sprintf("%.4d", validityTime): no_copy; +alias mars.time = padded_vtime; +unalias mars.step; +unalias mars.domain; \ No newline at end of file diff --git a/definitions/mars/stream.table b/definitions/mars/stream.table index a5d90c167..446d5e6b0 100644 --- a/definitions/mars/stream.table +++ b/definitions/mars/stream.table @@ -69,6 +69,8 @@ 1095 wamf Wave monthly forecast 1096 wmfm Wave monthly forecast means 1097 smma Seasonal monthly means anomalies +1098 clte Climate run output +1099 clmn Climate run monthly means output 1110 seap Sensitive area prediction 1120 eefh Extended ensemble forecast hindcast 1121 eehs Extended ensemble forecast hindcast statistics diff --git a/samples/destine_grib2.tmpl b/samples/destine_grib2.tmpl new file mode 100644 index 000000000..95154453c Binary files /dev/null and b/samples/destine_grib2.tmpl differ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a2abbfdc3..b5074a3ae 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -127,6 +127,7 @@ if( HAVE_BUILD_TOOLS ) grib_ecc-1560 grib_ecc-1571 grib_ecc-1654 + grib_ecc-1691 bufr_ecc-1028 bufr_ecc-1195 bufr_ecc-1259 diff --git a/tests/grib_ecc-1691.sh b/tests/grib_ecc-1691.sh new file mode 100755 index 000000000..38a9a1603 --- /dev/null +++ b/tests/grib_ecc-1691.sh @@ -0,0 +1,95 @@ +#!/bin/sh +# (C) Copyright 2005- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +# + +. ./include.ctest.sh + +REDIRECT=/dev/null + +# ECC-1691: Destination Earth ClimateDT metadata support + +label="grib_ecc-1691_test" +temp_grib_a=temp.$label.a.grib +temp_grib_b=temp.$label.b.grib +destine_sample=temp.$label.destine.grib +sample_grib2=$ECCODES_SAMPLES_PATH/GRIB2.tmpl + +tablesVersionLatest=$( ${tools_dir}/grib_get -p tablesVersionLatest $sample_grib2 ) + +# Setup Destine pseudo-centre GRIB message +# First latest tables version and add local section with MARS labeling +${tools_dir}/grib_set -s tablesVersion=$tablesVersionLatest,setLocalDefinition=1,grib2LocalSectionNumber=1 $sample_grib2 $temp_grib_a + +# Then change to processed data = 12 --> DestinE +${tools_dir}/grib_set -s productionStatusOfProcessedData=12,class=d1 $temp_grib_a $destine_sample + +# Check DestinE base related keys are present and correct +grib_check_key_exists $destine_sample destineLocalVersion,dataset,marsClass,marsType,marsStream,experimentVersionNumber +grib_check_key_equals $destine_sample "marsClass,dataset,destineLocalVersion" "d1 0 1" + +# Check setting dataset to climate-dt (1). Check keys are present and equal defaults +${tools_dir}/grib_set -s dataset=1 $destine_sample $temp_grib_a + +grib_check_key_exists $temp_grib_a dataset,generation,activity,experiment,model,realization,resolution +grib_check_key_equals $temp_grib_a "dataset,generation,activity,experiment,model,realization,resolution" "1 255 0 0 0 255 0" + +# Check an example where a few additional things are set in climate-dt +${tools_dir}/grib_set -s dataset=1,generation=2,activity=1,experiment=1,model=2,realization=1,resolution=1 $destine_sample $temp_grib_a + +grib_check_key_equals $temp_grib_a "dataset,generation,activity,experiment,model,realization,resolution" "1 2 1 1 2 1 1" +grib_check_key_equals $temp_grib_a "dataset:s,activity:s,experiment:s,model:s,resolution:s" "climate-dt CMIP6 hist IFS-NEMO standard" + +# Check setting dataset to extremes-dt (2). This time we will check string also +${tools_dir}/grib_set -s dataset=2 $destine_sample $temp_grib_a + +grib_check_key_equals $temp_grib_a "dataset,dataset:s" "2 extremes-dt" + +# Check concepts. First check default value and then example of mapping +grib_check_key_exists $destine_sample destineOrigin,gridSpecification +grib_check_key_equals $destine_sample "destineOrigin,gridSpecification" "unknown unknown" + +# Below mapping is for MeteoFrance +${tools_dir}/grib_set -s centre=98,subCentre=85 $destine_sample $temp_grib_a + +grib_check_key_equals $temp_grib_a "destineOrigin" "Meteo-France" + +# Below mapping is for H1024 +${tools_dir}/grib_set -s numberOfDataPoints=12582912,gridDefinitionTemplateNumber=150,Nside=1024 $destine_sample $temp_grib_a + +grib_check_key_equals $temp_grib_a "gridSpecification" "H1024" + +# Now check streams. +# Setting stream clte and type fc should set mars.date and mars.time to validityDate and validityTime, +# and mars.step should be unaliased + +${tools_dir}/grib_set -s stream=clte,type=fc $destine_sample $temp_grib_a + +result1=$( ${tools_dir}/grib_get -p mars.date,mars.time $temp_grib_a ) +result2=$( ${tools_dir}/grib_get -p validityDate,validityTime $temp_grib_a ) +[ "$result1" = "$result2" ] + +${tools_dir}/grib_ls -jm $temp_grib_a +[ $( ${tools_dir}/grib_get -f -p mars.step $temp_grib_a ) = "not_found" ] + +# Setting stream clmn and type fc should set mars.year and mars.month to year and month, +# and mars.date, mars.time, mars.step should be unaliased + +${tools_dir}/grib_set -s stream=clmn,type=fc $destine_sample $temp_grib_a + +result1=$( ${tools_dir}/grib_get -p mars.year,mars.month $temp_grib_a ) +result2=$( ${tools_dir}/grib_get -p year,month $temp_grib_a ) +[ "$result1" = "$result2" ] + +${tools_dir}/grib_ls -jm $temp_grib_a +[ $( ${tools_dir}/grib_get -f -p mars.date $temp_grib_a ) = "not_found" ] +[ $( ${tools_dir}/grib_get -f -p mars.time $temp_grib_a ) = "not_found" ] +[ $( ${tools_dir}/grib_get -f -p mars.step $temp_grib_a ) = "not_found" ] + +# Clean up +rm -f $temp_grib_a $temp_grib_b $destine_sample