ECC-806: Local definitions precedence order (Initial try)

definitions/boot.def

@@ -11,7 +11,8 @@ constant definitionFilesVersion="2.0.0.0" : hidden;
 constant internalVersion=30 : hidden;
 meta checkInternalVersion check_internal_version(internalVersion) : hidden;
 
-UseEcmfConventions = getenv("ECCODES_USE_ECMF_CONVENTIONS","1") :hidden ;
+# ECC-806: Local concepts precedence order
+transient preferLocalConcepts = 0 : hidden;
 
 constant defaultTypeOfLevel="unknown" : hidden;
 
@@ -24,7 +25,6 @@ if (gribDataQualityChecks) {
 # If the environment variable is not defined, the key will be 0
 GRIBEX_boustrophedonic = getenv("ECCODES_GRIBEX_BOUSTROPHEDONIC","0")  :hidden;
 
-
 constant zero=0 : hidden;
 constant one=1 : hidden;
 constant hundred=100 : hidden;

definitions/grib1/section.1.def

@@ -8,6 +8,14 @@ constant conceptsLocalDirAll="grib1/localConcepts/[centre:s]" : hidden;
 constant tablesMasterDir="grib1" : hidden;
 constant tablesLocalDir="grib1/local/[centre:s]" : hidden;
 
+if (preferLocalConcepts) {
+    constant conceptsDir1 = conceptsMasterDir : hidden;
+    constant conceptsDir2 = conceptsLocalDirAll : hidden;
+} else {
+    constant conceptsDir1 = conceptsLocalDirAll : hidden;
+    constant conceptsDir2 = conceptsMasterDir : hidden;
+}
+
 transient productionStatusOfProcessedData=0;
 position offsetSection1;
 section_length[3] section1Length ;
@@ -53,7 +61,7 @@ alias levelType=indicatorOfTypeOfLevel;
 transient pressureUnits="hPa";
 
 concept_nofail typeOfLevelECMF      (unknown,         "typeOfLevel.def",conceptsMasterDir,conceptsLocalDirECMF); 
-concept_nofail vertical.typeOfLevel (typeOfLevelECMF, "typeOfLevel.def",conceptsMasterDir,conceptsLocalDirAll);
+concept_nofail vertical.typeOfLevel (typeOfLevelECMF, "typeOfLevel.def",conceptsDir2,conceptsDir1);
 
 when ( typeOfLevel is "isobaricInPa" ) { set pressureUnits="Pa"; }
 else { set pressureUnits="hPa";}
@@ -71,7 +79,7 @@ if ( indicatorOfTypeOfLevel == 101 or
      indicatorOfTypeOfLevel == 120 or
      indicatorOfTypeOfLevel == 121 or
      indicatorOfTypeOfLevel == 128 or
-     indicatorOfTypeOfLevel == 141    )
+     indicatorOfTypeOfLevel == 141 )
 {
     unsigned[1] topLevel : can_be_missing,dump;
     unsigned[1] bottomLevel : can_be_missing,dump;
@@ -148,20 +156,20 @@ if(table2Version >= 128) {
 #}
 
 concept paramIdECMF (defaultParameter,"paramId.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy;
-concept paramId (paramIdECMF,"paramId.def",conceptsMasterDir,conceptsLocalDirAll): long_type,dump;
+concept paramId (paramIdECMF,"paramId.def",conceptsDir2,conceptsDir1): long_type,dump;
 # transient pid = paramId : hidden;
 
 concept cfNameECMF(defaultName,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only;
-concept cfName(cfNameECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirAll) : dump,no_copy,read_only;
+concept cfName(cfNameECMF,"cfName.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only;
 
 concept cfVarNameECMF(defaultName,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only;
-concept cfVarName(cfVarNameECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirAll) : dump,no_copy,read_only;
+concept cfVarName(cfVarNameECMF,"cfVarName.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only;
 
 concept unitsECMF(defaultName,"units.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only;
-concept units(unitsECMF,"units.def",conceptsMasterDir,conceptsLocalDirAll) : dump,no_copy,read_only;
+concept units(unitsECMF,"units.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only;
 
 concept nameECMF(defaultName,"name.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only;
-concept name(nameECMF,"name.def",conceptsMasterDir,conceptsLocalDirAll) : dump,no_copy,read_only;
+concept name(nameECMF,"name.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only;
 
 signed[2] decimalScaleFactor :dump;
 transient setLocalDefinition= 0 : no_copy;
@@ -177,7 +185,7 @@ meta julianDay  julian_day(dataDate,hour,minute,second) : edition_specific;
 
 codetable[1] stepUnits 'stepUnits.table' = 1 : transient,dump,no_copy;
 
-concept_nofail stepType (timeRangeIndicator, "stepType.def", conceptsMasterDir, conceptsLocalDirAll);
+concept_nofail stepType (timeRangeIndicator, "stepType.def", conceptsDir2, conceptsDir1);
 
 #alias stepTypeInternal=stepType;
 #alias lengthOfTimeRange=numberIncludedInAverage;
@@ -287,9 +295,8 @@ section_padding section1Padding : read_only;
 #  padtoeven evenpadding_sec1(offsetSection1,section1Length);
 #}
 
-
 concept shortNameECMF (defaultShortName,"shortName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy;
-concept ls.shortName (shortNameECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirAll) : no_copy,dump;
+concept ls.shortName (shortNameECMF,"shortName.def",conceptsDir2,conceptsDir1) : no_copy,dump;
 meta ifsParam ifs_param(paramId,type);
 
 alias parameter.paramId=paramId;
@@ -309,7 +316,7 @@ alias time.endStep=endStep;
 alias time.stepType=stepType;
 
 # ECC-457: GRIB1 to GRIB2 conversion
-concept_nofail stepTypeForConversion (unknown, "stepTypeForConversion.def", conceptsMasterDir, conceptsLocalDirAll);
+concept_nofail stepTypeForConversion (unknown, "stepTypeForConversion.def", conceptsDir2, conceptsDir1);
 if (stepTypeForConversion is "accum" ) {
     alias productDefinitionTemplateNumber=eight;
 }

definitions/grib2/parameters.def

@@ -5,30 +5,30 @@ transient dummyc=0: hidden;
 
 concept paramIdLegacyECMF(defaultParameter,"paramId.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): long_type,no_copy,hidden;
 concept paramIdECMF (paramIdLegacyECMF,"paramId.def",conceptsMasterDir,conceptsLocalDirECMF): long_type,no_copy;
-concept paramId (paramIdECMF,"paramId.def",conceptsMasterDir,conceptsLocalDirAll): long_type;
+concept paramId (paramIdECMF,"paramId.def",conceptsDir2,conceptsDir1): long_type;
 
 concept shortNameLegacyECMF(defaultShortName,"shortName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden;
 concept shortNameECMF (shortNameLegacyECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump;
-concept ls.shortName (shortNameECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirAll): no_copy,dump;
+concept ls.shortName (shortNameECMF,"shortName.def",conceptsDir2,conceptsDir1): no_copy,dump;
 
 concept unitsLegacyECMF(defaultName,"units.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden;
 concept unitsECMF (unitsLegacyECMF,"units.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy;
-concept units (unitsECMF,"units.def",conceptsMasterDir,conceptsLocalDirAll): no_copy;
+concept units (unitsECMF,"units.def",conceptsDir2,conceptsDir1): no_copy;
 
 concept nameLegacyECMF(defaultName,"name.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden;
 concept nameECMF(nameLegacyECMF,"name.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump;
-concept name(nameECMF,"name.def",conceptsMasterDir,conceptsLocalDirAll): no_copy,dump;
+concept name(nameECMF,"name.def",conceptsDir2,conceptsDir1): no_copy,dump;
 
 concept cfNameLegacyECMF(defaultShortName,"cfName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden;
 concept cfNameECMF(cfNameLegacyECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,dump;
-concept cfName(cfNameECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirAll) : no_copy,dump;
+concept cfName(cfNameECMF,"cfName.def",conceptsDir2,conceptsDir1) : no_copy,dump;
 
 concept cfVarNameLegacyECMF(defaultShortName,"cfVarName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden;
 concept cfVarNameECMF (cfVarNameLegacyECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump;
-concept cfVarName (cfVarNameECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirAll): no_copy,dump;
+concept cfVarName (cfVarNameECMF,"cfVarName.def",conceptsDir2,conceptsDir1): no_copy,dump;
 
 # modelName: Contribution from Daniel Lee @ DWD
-concept modelName (defaultName,"modelName.def",conceptsMasterDir,conceptsLocalDirAll): no_copy,dump,read_only;
+concept modelName (defaultName,"modelName.def",conceptsDir2,conceptsDir1): no_copy,dump,read_only;
 
 template_nofail names "grib2/products_[productionStatusOfProcessedData].def";
 

definitions/grib2/section.1.def

@@ -136,3 +136,11 @@ concept is_uerra(zero) {
 constant conceptsMasterDir="grib2" : hidden;
 constant conceptsLocalDirAll="grib2/localConcepts/[centre:s]" : hidden;
 constant conceptsLocalDirECMF="grib2/localConcepts/ecmf" : hidden;
+
+if (preferLocalConcepts) {
+    constant conceptsDir1 = conceptsMasterDir : hidden;
+    constant conceptsDir2 = conceptsLocalDirAll : hidden;
+} else {
+    constant conceptsDir1 = conceptsLocalDirAll : hidden;
+    constant conceptsDir2 = conceptsMasterDir : hidden;
+}

definitions/grib2/template.4.horizontal.def

@@ -24,7 +24,7 @@ unsigned[4] scaledValueOfSecondFixedSurface = missing() : can_be_missing,dump,no
 
 transient pressureUnits="hPa";
 
-concept_nofail vertical.typeOfLevel (unknown,"typeOfLevelConcept.def",conceptsMasterDir,conceptsLocalDirAll);
+concept_nofail vertical.typeOfLevel (unknown,"typeOfLevelConcept.def",conceptsDir2,conceptsDir1);
 
 alias levelType=typeOfFirstFixedSurface;
 

tests/CMakeLists.txt

@@ -127,6 +127,7 @@ if( HAVE_BUILD_TOOLS )
         bufr_ecc-887
         grib_ecc-490
         grib_ecc-756
+        grib_ecc-806
         grib_ecc-873
         grib_ecc-600
         grib_ecc-923

tests/grib_ecc-806.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+# (C) Copyright 2005- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+set -u
+# --------------------------------------------
+# This is the test for the JIRA issue ECC-806
+# Local definitions precedence order
+# --------------------------------------------
+label="grib_ecc-806-test"
+tempGrb=temp.${label}.grb
+tempOut=temp.${label}.txt
+tempErr=temp.${label}.err
+
+# This NCEP grib2 file has the keys
+#   discipline = 0
+#   parameterCategory = 1
+#   parameterNumber = 11
+# Which is normally matched by 3066 but when overriden by NCEP local definitions
+# should become 260056
+# This assumes our definitions contain the local kwbc concepts
+# See definitions/grib2/localConcepts/kwbc/paramId.def
+
+input=$data_dir/gfs.complex.mvmu.grib2
+res=`${tools_dir}/grib_get -p paramId,shortName,units,name $input`
+[ "$res" = "3066 sde m Snow depth" ]
+
+# Change the precedence rule
+res=`${tools_dir}/grib_get -s preferLocalConcepts=1 -p paramId,shortName,units,name $input`
+[ "$res" = "260056 sdwe kg m**-2 Water equivalent of accumulated snow depth (deprecated)" ]
+
+# Clean up
+rm -f $tempGrb $tempOut $tempErr