From ae2f8c58209bec5bee45c83966ca2be303f3358e Mon Sep 17 00:00:00 2001 From: shahramn Date: Thu, 13 Jun 2024 12:17:28 +0100 Subject: [PATCH 1/6] Testing: chemId/paramId split --- tests/grib_ecc-1212.sh | 1 + tests/grib_local.sh | 6 ++++-- tests/grib_mars_keys1.sh | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/grib_ecc-1212.sh b/tests/grib_ecc-1212.sh index 7ff7c2ead..d2454d2b2 100755 --- a/tests/grib_ecc-1212.sh +++ b/tests/grib_ecc-1212.sh @@ -58,6 +58,7 @@ cat > $tempRef < $tempRef << EOF "class": "od", "type": "an", "stream": "oper", + "paramtype": "base", "step": 0, "levelist": 1000, "levtype": "pl", @@ -73,6 +74,7 @@ cat > $tempRef << EOF "class": "od", "type": "pf", "stream": "enfo", + "paramtype": "base", "step": 0, "levelist": 1000, "levtype": "pl", From b450d346c234133d57c09c73b0e4b6a4b3b055fc Mon Sep 17 00:00:00 2001 From: Robert Osinski Date: Thu, 13 Jun 2024 16:04:27 +0000 Subject: [PATCH 2/6] Test for paramId / chemId splitting and legacy behaviour added --- tests/CMakeLists.txt | 1 + tests/grib_paramid_chemid_split.sh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100755 tests/grib_paramid_chemid_split.sh diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0169e4986..faa588a6b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -170,6 +170,7 @@ if( HAVE_BUILD_TOOLS ) grib_grid_polar_stereographic grib_grid_healpix grib_g1day_of_the_year_date + grib_paramid_chemid_split ) # These tests require data downloads diff --git a/tests/grib_paramid_chemid_split.sh b/tests/grib_paramid_chemid_split.sh new file mode 100755 index 000000000..0bfa7b47c --- /dev/null +++ b/tests/grib_paramid_chemid_split.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# (C) Copyright 2005- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +# + +. ./include.ctest.sh + +REDIRECT=/dev/null + +label="grib__test" # Change prod to bufr or grib etc +tempGrib1=temp1.$label.grib +tempGrib2=temp2.$label.grib + +sample_grib2=$ECCODES_SAMPLES_PATH/GRIB2.tmpl + +# we expect paramId 210121 +${tools_dir}/grib_set -s productDefinitionTemplateNumber=40,discipline=0,parameterCategory=20,parameterNumber=2,constituentType=5 $sample_grib2 $tempGrib1 +# we expect paramId 402000 and chemId 17 +${tools_dir}/grib_set -s tablesVersion=32 $tempGrib1 $tempGrib2 + +grib_check_key_equals $tempGrib1 paramId "210121" +grib_check_key_equals $tempGrib2 paramId,chemId "402000 17" + +# Clean up +rm -f $tempGrib1 $tempGrib2 From ef5252723aa7253b35b996ab2b59151014614bad Mon Sep 17 00:00:00 2001 From: shahramn Date: Thu, 13 Jun 2024 17:26:33 +0100 Subject: [PATCH 3/6] Testing: chemId/paramId split --- tests/grib_paramid_chemid_split.sh | 34 ++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/grib_paramid_chemid_split.sh b/tests/grib_paramid_chemid_split.sh index 0bfa7b47c..07a0bd63b 100755 --- a/tests/grib_paramid_chemid_split.sh +++ b/tests/grib_paramid_chemid_split.sh @@ -10,21 +10,33 @@ . ./include.ctest.sh -REDIRECT=/dev/null -label="grib__test" # Change prod to bufr or grib etc -tempGrib1=temp1.$label.grib -tempGrib2=temp2.$label.grib +label="grib_paramid_chemid_split_test" +tempGribA=temp1.$label.grib +tempGribB=temp2.$label.grib sample_grib2=$ECCODES_SAMPLES_PATH/GRIB2.tmpl -# we expect paramId 210121 -${tools_dir}/grib_set -s productDefinitionTemplateNumber=40,discipline=0,parameterCategory=20,parameterNumber=2,constituentType=5 $sample_grib2 $tempGrib1 -# we expect paramId 402000 and chemId 17 -${tools_dir}/grib_set -s tablesVersion=32 $tempGrib1 $tempGrib2 +# Legacy using older tablesVersion +# We expect paramId 210121 +${tools_dir}/grib_set -s productDefinitionTemplateNumber=40,discipline=0,parameterCategory=20,parameterNumber=2,constituentType=5 \ + $sample_grib2 $tempGribA +grib_check_key_equals $tempGribA tablesVersion 4 +grib_check_key_equals $tempGribA paramId,is_chemical "210121 1" + + +# Switch to the newer tablesVersion so now the chemId/paramId split is activated +# Now expect paramId 402000 and chemId 17 +${tools_dir}/grib_set -s tablesVersion=32 $tempGribA $tempGribB + +grib_check_key_equals $tempGribB paramId,chemId "402000 17" +grib_check_key_equals $tempGribB name "Mass mixing ratio" +set +e +${tools_dir}/grib_get -p is_chemical $tempGribB 2> /dev/null +status=$? +set -e +[ $status -ne 0 ] -grib_check_key_equals $tempGrib1 paramId "210121" -grib_check_key_equals $tempGrib2 paramId,chemId "402000 17" # Clean up -rm -f $tempGrib1 $tempGrib2 +rm -f $tempGribA $tempGribB From a0f879d5822bfdee01c43cc9d5bc91745a5016fa Mon Sep 17 00:00:00 2001 From: shahramn Date: Thu, 13 Jun 2024 17:50:20 +0100 Subject: [PATCH 4/6] chemId/paramId split: chemName etc should not be present for non-chemicals --- definitions/grib2/parameters.def | 20 +++++++++++--------- tests/grib_paramid_chemid_split.sh | 14 ++++++++------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/definitions/grib2/parameters.def b/definitions/grib2/parameters.def index 07e9947ce..8096f96f6 100644 --- a/definitions/grib2/parameters.def +++ b/definitions/grib2/parameters.def @@ -44,16 +44,18 @@ else { alias cfVarName=cfVarNameFallback; } -concept chemId (unknown,"chemId.def",conceptsDir2,conceptsDir1): no_copy,long_type,dump; -concept chemName (unknown,"chemName.def",conceptsDir2,conceptsDir1): no_copy,dump; -concept chemShortName (unknown,"chemShortName.def",conceptsDir2,conceptsDir1): no_copy,dump; -concept chemFormula (unknown,"chemFormula.def",conceptsDir2,conceptsDir1): no_copy,dump; +if (tablesVersion >= 31 ) { + concept chemId (unknown,"chemId.def",conceptsDir2,conceptsDir1): no_copy,long_type,dump; + concept chemName (unknown,"chemName.def",conceptsDir2,conceptsDir1): no_copy,dump; + concept chemShortName (unknown,"chemShortName.def",conceptsDir2,conceptsDir1): no_copy,dump; + concept chemFormula (unknown,"chemFormula.def",conceptsDir2,conceptsDir1): no_copy,dump; -if ( !(chemId is "unknown") ) { - alias mars.chem = chemId; - alias parameter.chemId = chemId; - alias ls.chemShortName = chemShortName; - alias parameter.chemShortName = chemShortName; + if ( !(chemId is "unknown") ) { + alias mars.chem = chemId; + alias parameter.chemId = chemId; + alias ls.chemShortName = chemShortName; + alias parameter.chemShortName = chemShortName; + } } template_nofail names "grib2/products_[productionStatusOfProcessedData].def"; diff --git a/tests/grib_paramid_chemid_split.sh b/tests/grib_paramid_chemid_split.sh index 07a0bd63b..c36ddc769 100755 --- a/tests/grib_paramid_chemid_split.sh +++ b/tests/grib_paramid_chemid_split.sh @@ -10,7 +10,6 @@ . ./include.ctest.sh - label="grib_paramid_chemid_split_test" tempGribA=temp1.$label.grib tempGribB=temp2.$label.grib @@ -23,6 +22,9 @@ ${tools_dir}/grib_set -s productDefinitionTemplateNumber=40,discipline=0,paramet $sample_grib2 $tempGribA grib_check_key_equals $tempGribA tablesVersion 4 grib_check_key_equals $tempGribA paramId,is_chemical "210121 1" +# The new keys should not be present +result=$( ${tools_dir}/grib_get -f -p chemName,chemId $tempGribA ) +[ "$result" = "not_found not_found" ] # Switch to the newer tablesVersion so now the chemId/paramId split is activated @@ -30,12 +32,12 @@ grib_check_key_equals $tempGribA paramId,is_chemical "210121 1" ${tools_dir}/grib_set -s tablesVersion=32 $tempGribA $tempGribB grib_check_key_equals $tempGribB paramId,chemId "402000 17" +grib_check_key_equals $tempGribB chemName "Nitrogen dioxide" grib_check_key_equals $tempGribB name "Mass mixing ratio" -set +e -${tools_dir}/grib_get -p is_chemical $tempGribB 2> /dev/null -status=$? -set -e -[ $status -ne 0 ] + +# The is_chemical key should not be present +result=$( ${tools_dir}/grib_get -f -p is_chemical $tempGribB ) +[ "$result" = "not_found" ] # Clean up From 13783417c2c0571fba9a77c0877ad91107ec984d Mon Sep 17 00:00:00 2001 From: shahramn Date: Thu, 13 Jun 2024 20:22:22 +0100 Subject: [PATCH 5/6] chemId/paramId split: Use zero for chemId when it's unknown --- definitions/grib2/parameters.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/definitions/grib2/parameters.def b/definitions/grib2/parameters.def index 8096f96f6..8e01ea06d 100644 --- a/definitions/grib2/parameters.def +++ b/definitions/grib2/parameters.def @@ -45,7 +45,7 @@ else { } if (tablesVersion >= 31 ) { - concept chemId (unknown,"chemId.def",conceptsDir2,conceptsDir1): no_copy,long_type,dump; + concept chemId (zero,"chemId.def",conceptsDir2,conceptsDir1): no_copy,long_type,dump; concept chemName (unknown,"chemName.def",conceptsDir2,conceptsDir1): no_copy,dump; concept chemShortName (unknown,"chemShortName.def",conceptsDir2,conceptsDir1): no_copy,dump; concept chemFormula (unknown,"chemFormula.def",conceptsDir2,conceptsDir1): no_copy,dump; From 5cf77efff50419bf99702587469c287ab21d91d3 Mon Sep 17 00:00:00 2001 From: shahramn Date: Thu, 13 Jun 2024 20:22:42 +0100 Subject: [PATCH 6/6] chemId/paramId split: further tests --- tests/grib_paramid_chemid_split.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/grib_paramid_chemid_split.sh b/tests/grib_paramid_chemid_split.sh index c36ddc769..aff0aca2b 100755 --- a/tests/grib_paramid_chemid_split.sh +++ b/tests/grib_paramid_chemid_split.sh @@ -40,5 +40,10 @@ result=$( ${tools_dir}/grib_get -f -p is_chemical $tempGribB ) [ "$result" = "not_found" ] +# Test non-chemical e.g. temperature. Now chem keys are present but unknown +${tools_dir}/grib_set -s tablesVersion=32,discipline=0,parameterCategory=0,parameterNumber=0 $sample_grib2 $tempGribA +grib_check_key_equals $tempGribA "chemId,chemName,chemShortName" "0 unknown unknown" + + # Clean up rm -f $tempGribA $tempGribB