From df0c1a7d6d046fa8854f26c0cdf65ec485aa4096 Mon Sep 17 00:00:00 2001
From: shahramn <shahram.najm@ecmwf.int>
Date: Thu, 18 Jul 2024 19:47:38 +0100
Subject: [PATCH] chemId/paramId split: Add mechanism to detect changes to
 tablesVersion

---
 definitions/grib2/boot.def          |  1 +
 definitions/grib2/parameters.def    |  2 +-
 definitions/grib2/section.1.def     | 16 +++++++++++-----
 tests/grib2_chemicals_aerosols.sh   | 28 +++++++++++++++-------------
 tests/grib2_paramid_chemid_split.sh | 11 ++++++++---
 5 files changed, 36 insertions(+), 22 deletions(-)

diff --git a/definitions/grib2/boot.def b/definitions/grib2/boot.def
index 99b47bc56..85ffa02a8 100644
--- a/definitions/grib2/boot.def
+++ b/definitions/grib2/boot.def
@@ -16,6 +16,7 @@ constant tablesVersionLatest = 33 : edition_specific;
 # See ECC-1426
 # GRIB messages with tablesVersion greater than or equal will have paramId and chemId
 constant tablesVersionChemParamSplit = 31 : hidden;
+transient chem_param_split = 0 : hidden;
 
 constant false = 0 : hidden;
 constant true = 1 : hidden;
diff --git a/definitions/grib2/parameters.def b/definitions/grib2/parameters.def
index 49c4c6ed6..e7a20d20a 100644
--- a/definitions/grib2/parameters.def
+++ b/definitions/grib2/parameters.def
@@ -26,7 +26,7 @@ concept cfVarNameLegacyECMF(defaultShortName,"cfVarName.legacy.def",conceptsMast
 concept cfVarNameECMF (cfVarNameLegacyECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy;
 concept cfVarName (cfVarNameECMF,"cfVarName.def",conceptsDir2,conceptsDir1): no_copy,dump;
 
-if (tablesVersion >= tablesVersionChemParamSplit ) {
+if (chem_param_split == 1) {
   # Note: 0 is a valid chemId
   concept chemId (minus_one,"chemId.def",conceptsDir2,conceptsDir1): no_copy,long_type,dump;
   concept chemName (unknown,"chemName.def",conceptsDir2,conceptsDir1): no_copy,dump;
diff --git a/definitions/grib2/section.1.def b/definitions/grib2/section.1.def
index 696de13bc..150002937 100644
--- a/definitions/grib2/section.1.def
+++ b/definitions/grib2/section.1.def
@@ -29,10 +29,16 @@ transient masterDir="grib2/tables/[tablesVersion]";
 if (tablesVersion > tablesVersionLatest) {
   transient masterDir="grib2/tables/[tablesVersionLatest]";
 }
-when (tablesVersion!=255) {
-  set masterDir="grib2/tables/[tablesVersion]";
-} else {
-  set masterDir="grib2/tables/4";
+
+# See ECC-1426: chemId and paramId split feature.
+# The WHEN statement is needed for tablesVersion changed in memory
+when (tablesVersion >= tablesVersionChemParamSplit) {
+  set chem_param_split = 1;
+}
+# The IF statement is needed when reading an existing message
+if (tablesVersion >= tablesVersionChemParamSplit) {
+  remove chem_param_split;
+  transient chem_param_split = 1;
 }
 
 codetable[1] localTablesVersion 'grib2/tables/local/[centreForLocal]/1.1.table' ;
@@ -97,7 +103,7 @@ concept stepType {
     "mode"    = {selectStepTemplateInterval=1; stepTypeInternal="mode";}
 }
 
-if ( tablesVersion < tablesVersionChemParamSplit ) {
+if (chem_param_split == 0) {
   # 0=atmospheric chemical constituents
   # 1=atmospheric chemical constituents based on a distribution function
   # 2=atmospheric chemical constituents with source or sink
diff --git a/tests/grib2_chemicals_aerosols.sh b/tests/grib2_chemicals_aerosols.sh
index 43cca2042..e0cd6c626 100755
--- a/tests/grib2_chemicals_aerosols.sh
+++ b/tests/grib2_chemicals_aerosols.sh
@@ -15,11 +15,13 @@ temp=temp.$label
 temp1=temp.$label.1
 sample2=$ECCODES_SAMPLES_PATH/GRIB2.tmpl
 
-latest=`${tools_dir}/grib_get -p tablesVersionLatest $sample2`
+# Use a tablesVersion BEFORE the split of chemId/paramId
+tv_prev=`${tools_dir}/grib_get -p tablesVersionChemParamSplit $sample2`
+tv_prev=$((tv_prev - 1))
 
 # ECC-1417: Replace key aerosolType with constituentType
 # ------------------------------------------------------
-${tools_dir}/grib_set -s tablesVersion=$latest,paramId=210072 $sample2 $temp
+${tools_dir}/grib_set -s tablesVersion=$tv_prev,paramId=210072 $sample2 $temp
 grib_check_key_equals $temp aerosolType         62026
 grib_check_key_equals $temp aerosolTypeName     'Particulate matter'
 grib_check_key_equals $temp constituentType     62026
@@ -28,10 +30,10 @@ grib_check_key_equals $temp constituentTypeName 'Particulate matter'
 
 # Check latest chemical/aerosol code tables
 # -----------------------------------------
-${tools_dir}/grib_set -s tablesVersion=$latest,paramId=217224  $sample2 $temp
+${tools_dir}/grib_set -s tablesVersion=$tv_prev,paramId=217224  $sample2 $temp
 grib_check_key_equals $temp constituentTypeName "Aceto nitrile CH3CN"
 
-${tools_dir}/grib_set -s tablesVersion=$latest,paramId=210249  $sample2 $temp
+${tools_dir}/grib_set -s tablesVersion=$tv_prev,paramId=210249  $sample2 $temp
 grib_check_key_equals $temp aerosolTypeName "Ammonium dry"
 
 
@@ -39,27 +41,27 @@ grib_check_key_equals $temp aerosolTypeName "Ammonium dry"
 # Deterministic instantaneous
 # =============================
 # Plain chemicals
-${tools_dir}/grib_set -s tablesVersion=$latest,is_chemical=1 $sample2 $temp
+${tools_dir}/grib_set -s is_chemical=1 $sample2 $temp
 grib_check_key_equals $temp productDefinitionTemplateNumber '40'
 grib_check_key_equals $temp constituentType '0'
 
 # Chemicals with source and sink
-${tools_dir}/grib_set -s tablesVersion=$latest,is_chemical_srcsink=1 $sample2 $temp
+${tools_dir}/grib_set -s is_chemical_srcsink=1 $sample2 $temp
 grib_check_key_equals $temp productDefinitionTemplateNumber '76'
 grib_check_key_equals $temp constituentType,sourceSinkChemicalPhysicalProcess '0 255'
 
 # Chemicals with distribution function
-${tools_dir}/grib_set -s tablesVersion=$latest,is_chemical_distfn=1 $sample2 $temp
+${tools_dir}/grib_set -s is_chemical_distfn=1 $sample2 $temp
 grib_check_key_equals $temp productDefinitionTemplateNumber '57'
 grib_check_key_equals $temp constituentType,numberOfModeOfDistribution,modeNumber '0 0 0'
 
 # Plain aerosols
-${tools_dir}/grib_set -s tablesVersion=$latest,is_aerosol=1 $sample2 $temp
+${tools_dir}/grib_set -s is_aerosol=1 $sample2 $temp
 grib_check_key_equals $temp productDefinitionTemplateNumber '48'
 grib_check_key_equals $temp aerosolType,typeOfSizeInterval,typeOfWavelengthInterval '0 0 0'
 
 # Aerosol optical
-${tools_dir}/grib_set -s tablesVersion=$latest,is_aerosol_optical=1 $sample2 $temp
+${tools_dir}/grib_set -s is_aerosol_optical=1 $sample2 $temp
 grib_check_key_equals $temp productDefinitionTemplateNumber '48'
 #${tools_dir}/grib_dump -O $temp
 
@@ -68,22 +70,22 @@ grib_check_key_equals $temp productDefinitionTemplateNumber '48'
 # Deterministic interval-based
 # =============================
 # Plain chemicals
-${tools_dir}/grib_set -s tablesVersion=$latest,stepType=accum,is_chemical=1 $sample2 $temp
+${tools_dir}/grib_set -s stepType=accum,is_chemical=1 $sample2 $temp
 grib_check_key_equals $temp productDefinitionTemplateNumber '42'
 grib_check_key_equals $temp constituentType '0'
 
 # Chemicals with source and sink
-${tools_dir}/grib_set -s tablesVersion=$latest,stepType=accum,is_chemical_srcsink=1 $sample2 $temp
+${tools_dir}/grib_set -s stepType=accum,is_chemical_srcsink=1 $sample2 $temp
 grib_check_key_equals $temp productDefinitionTemplateNumber '78'
 grib_check_key_equals $temp constituentType,sourceSinkChemicalPhysicalProcess '0 255'
 
 # Chemicals with distribution function
-${tools_dir}/grib_set -s tablesVersion=$latest,stepType=accum,is_chemical_distfn=1 $sample2 $temp
+${tools_dir}/grib_set -s stepType=accum,is_chemical_distfn=1 $sample2 $temp
 grib_check_key_equals $temp productDefinitionTemplateNumber '67'
 grib_check_key_equals $temp constituentType,numberOfModeOfDistribution,modeNumber '0 0 0'
 
 # Plain aerosols
-${tools_dir}/grib_set -s tablesVersion=$latest,stepType=accum,is_aerosol=1 $sample2 $temp
+${tools_dir}/grib_set -s stepType=accum,is_aerosol=1 $sample2 $temp
 grib_check_key_equals $temp productDefinitionTemplateNumber '46'
 grib_check_key_equals $temp aerosolType,typeOfSizeInterval '0 0'
 
diff --git a/tests/grib2_paramid_chemid_split.sh b/tests/grib2_paramid_chemid_split.sh
index dc60c5b4b..4e28c5f52 100755
--- a/tests/grib2_paramid_chemid_split.sh
+++ b/tests/grib2_paramid_chemid_split.sh
@@ -35,7 +35,7 @@ result=$( ${tools_dir}/grib_get -f -p chemName,chemId $tempGribA )
 # Now expect paramId 402000 and chemId 17
 ${tools_dir}/grib_set -s tablesVersion=$tablesVersionSplit $tempGribA $tempGribB
 
-grib_check_key_equals $tempGribB paramId,chemId "402000 17"
+grib_check_key_equals $tempGribB chem_param_split,paramId,chemId "1 402000 17"
 grib_check_key_equals $tempGribB chemName "Nitrogen dioxide"
 grib_check_key_equals $tempGribB name "Mass mixing ratio"
 
@@ -43,11 +43,16 @@ grib_check_key_equals $tempGribB name "Mass mixing ratio"
 result=$( ${tools_dir}/grib_get -f -p is_chemical $tempGribB )
 [ "$result" = "not_found" ]
 
+# Test with filter
+echo 'set tablesVersion=32; assert(chemId == 17); write;' | ${tools_dir}/grib_filter -o $tempGribB - $tempGribA
+grib_check_key_equals $tempGribB chem_param_split,paramId,chemName "1 402000 Nitrogen dioxide"
 
-# Test non-chemical e.g. temperature. Now chem keys are present but unknown
+
+# Test a non-chemical e.g., temperature. Now chem keys are present but unknown
 ${tools_dir}/grib_set -s tablesVersion=$tablesVersionSplit,discipline=0,parameterCategory=0,parameterNumber=0 $sample  $tempGribA
-grib_check_key_equals $tempGribA "chemId,chemName,chemShortName" "-1 unknown unknown"
+grib_check_key_equals $tempGribA "chem_param_split,chemId,chemName,chemShortName" "1 -1 unknown unknown"
 
+grib_check_key_equals $sample chem_param_split 0
 
 # Clean up
 rm -f $tempGribA $tempGribB