From f1b79391d3c415494921255c1d7f6930d5bf957c Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 5 Oct 2020 15:32:01 +0100 Subject: [PATCH] ECC-806: Local definitions precedence order (Initial try) --- definitions/boot.def | 4 +-- definitions/grib1/section.1.def | 29 +++++++++------ definitions/grib2/parameters.def | 14 ++++---- definitions/grib2/section.1.def | 8 +++++ definitions/grib2/template.4.horizontal.def | 2 +- tests/CMakeLists.txt | 1 + tests/grib_ecc-806.sh | 40 +++++++++++++++++++++ 7 files changed, 77 insertions(+), 21 deletions(-) create mode 100755 tests/grib_ecc-806.sh diff --git a/definitions/boot.def b/definitions/boot.def index ca68cf583..22edf96e1 100644 --- a/definitions/boot.def +++ b/definitions/boot.def @@ -11,7 +11,8 @@ constant definitionFilesVersion="2.0.0.0" : hidden; constant internalVersion=30 : hidden; meta checkInternalVersion check_internal_version(internalVersion) : hidden; -UseEcmfConventions = getenv("ECCODES_USE_ECMF_CONVENTIONS","1") :hidden ; +# ECC-806: Local concepts precedence order +transient preferLocalConcepts = 0 : hidden; constant defaultTypeOfLevel="unknown" : hidden; @@ -24,7 +25,6 @@ if (gribDataQualityChecks) { # If the environment variable is not defined, the key will be 0 GRIBEX_boustrophedonic = getenv("ECCODES_GRIBEX_BOUSTROPHEDONIC","0") :hidden; - constant zero=0 : hidden; constant one=1 : hidden; constant hundred=100 : hidden; diff --git a/definitions/grib1/section.1.def b/definitions/grib1/section.1.def index 91f891a41..14c4a8bd8 100644 --- a/definitions/grib1/section.1.def +++ b/definitions/grib1/section.1.def @@ -8,6 +8,14 @@ constant conceptsLocalDirAll="grib1/localConcepts/[centre:s]" : hidden; constant tablesMasterDir="grib1" : hidden; constant tablesLocalDir="grib1/local/[centre:s]" : hidden; +if (preferLocalConcepts) { + constant conceptsDir1 = conceptsMasterDir : hidden; + constant conceptsDir2 = conceptsLocalDirAll : hidden; +} else { + constant conceptsDir1 = conceptsLocalDirAll : hidden; + constant conceptsDir2 = conceptsMasterDir : hidden; +} + transient productionStatusOfProcessedData=0; position offsetSection1; section_length[3] section1Length ; @@ -53,7 +61,7 @@ alias levelType=indicatorOfTypeOfLevel; transient pressureUnits="hPa"; concept_nofail typeOfLevelECMF (unknown, "typeOfLevel.def",conceptsMasterDir,conceptsLocalDirECMF); -concept_nofail vertical.typeOfLevel (typeOfLevelECMF, "typeOfLevel.def",conceptsMasterDir,conceptsLocalDirAll); +concept_nofail vertical.typeOfLevel (typeOfLevelECMF, "typeOfLevel.def",conceptsDir2,conceptsDir1); when ( typeOfLevel is "isobaricInPa" ) { set pressureUnits="Pa"; } else { set pressureUnits="hPa";} @@ -71,7 +79,7 @@ if ( indicatorOfTypeOfLevel == 101 or indicatorOfTypeOfLevel == 120 or indicatorOfTypeOfLevel == 121 or indicatorOfTypeOfLevel == 128 or - indicatorOfTypeOfLevel == 141 ) + indicatorOfTypeOfLevel == 141 ) { unsigned[1] topLevel : can_be_missing,dump; unsigned[1] bottomLevel : can_be_missing,dump; @@ -148,20 +156,20 @@ if(table2Version >= 128) { #} concept paramIdECMF (defaultParameter,"paramId.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy; -concept paramId (paramIdECMF,"paramId.def",conceptsMasterDir,conceptsLocalDirAll): long_type,dump; +concept paramId (paramIdECMF,"paramId.def",conceptsDir2,conceptsDir1): long_type,dump; # transient pid = paramId : hidden; concept cfNameECMF(defaultName,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only; -concept cfName(cfNameECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirAll) : dump,no_copy,read_only; +concept cfName(cfNameECMF,"cfName.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only; concept cfVarNameECMF(defaultName,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only; -concept cfVarName(cfVarNameECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirAll) : dump,no_copy,read_only; +concept cfVarName(cfVarNameECMF,"cfVarName.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only; concept unitsECMF(defaultName,"units.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only; -concept units(unitsECMF,"units.def",conceptsMasterDir,conceptsLocalDirAll) : dump,no_copy,read_only; +concept units(unitsECMF,"units.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only; concept nameECMF(defaultName,"name.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only; -concept name(nameECMF,"name.def",conceptsMasterDir,conceptsLocalDirAll) : dump,no_copy,read_only; +concept name(nameECMF,"name.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only; signed[2] decimalScaleFactor :dump; transient setLocalDefinition= 0 : no_copy; @@ -177,7 +185,7 @@ meta julianDay julian_day(dataDate,hour,minute,second) : edition_specific; codetable[1] stepUnits 'stepUnits.table' = 1 : transient,dump,no_copy; -concept_nofail stepType (timeRangeIndicator, "stepType.def", conceptsMasterDir, conceptsLocalDirAll); +concept_nofail stepType (timeRangeIndicator, "stepType.def", conceptsDir2, conceptsDir1); #alias stepTypeInternal=stepType; #alias lengthOfTimeRange=numberIncludedInAverage; @@ -287,9 +295,8 @@ section_padding section1Padding : read_only; # padtoeven evenpadding_sec1(offsetSection1,section1Length); #} - concept shortNameECMF (defaultShortName,"shortName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy; -concept ls.shortName (shortNameECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirAll) : no_copy,dump; +concept ls.shortName (shortNameECMF,"shortName.def",conceptsDir2,conceptsDir1) : no_copy,dump; meta ifsParam ifs_param(paramId,type); alias parameter.paramId=paramId; @@ -309,7 +316,7 @@ alias time.endStep=endStep; alias time.stepType=stepType; # ECC-457: GRIB1 to GRIB2 conversion -concept_nofail stepTypeForConversion (unknown, "stepTypeForConversion.def", conceptsMasterDir, conceptsLocalDirAll); +concept_nofail stepTypeForConversion (unknown, "stepTypeForConversion.def", conceptsDir2, conceptsDir1); if (stepTypeForConversion is "accum" ) { alias productDefinitionTemplateNumber=eight; } diff --git a/definitions/grib2/parameters.def b/definitions/grib2/parameters.def index 066b49feb..632902561 100644 --- a/definitions/grib2/parameters.def +++ b/definitions/grib2/parameters.def @@ -5,30 +5,30 @@ transient dummyc=0: hidden; concept paramIdLegacyECMF(defaultParameter,"paramId.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): long_type,no_copy,hidden; concept paramIdECMF (paramIdLegacyECMF,"paramId.def",conceptsMasterDir,conceptsLocalDirECMF): long_type,no_copy; -concept paramId (paramIdECMF,"paramId.def",conceptsMasterDir,conceptsLocalDirAll): long_type; +concept paramId (paramIdECMF,"paramId.def",conceptsDir2,conceptsDir1): long_type; concept shortNameLegacyECMF(defaultShortName,"shortName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden; concept shortNameECMF (shortNameLegacyECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump; -concept ls.shortName (shortNameECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirAll): no_copy,dump; +concept ls.shortName (shortNameECMF,"shortName.def",conceptsDir2,conceptsDir1): no_copy,dump; concept unitsLegacyECMF(defaultName,"units.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden; concept unitsECMF (unitsLegacyECMF,"units.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy; -concept units (unitsECMF,"units.def",conceptsMasterDir,conceptsLocalDirAll): no_copy; +concept units (unitsECMF,"units.def",conceptsDir2,conceptsDir1): no_copy; concept nameLegacyECMF(defaultName,"name.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden; concept nameECMF(nameLegacyECMF,"name.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump; -concept name(nameECMF,"name.def",conceptsMasterDir,conceptsLocalDirAll): no_copy,dump; +concept name(nameECMF,"name.def",conceptsDir2,conceptsDir1): no_copy,dump; concept cfNameLegacyECMF(defaultShortName,"cfName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden; concept cfNameECMF(cfNameLegacyECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,dump; -concept cfName(cfNameECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirAll) : no_copy,dump; +concept cfName(cfNameECMF,"cfName.def",conceptsDir2,conceptsDir1) : no_copy,dump; concept cfVarNameLegacyECMF(defaultShortName,"cfVarName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden; concept cfVarNameECMF (cfVarNameLegacyECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump; -concept cfVarName (cfVarNameECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirAll): no_copy,dump; +concept cfVarName (cfVarNameECMF,"cfVarName.def",conceptsDir2,conceptsDir1): no_copy,dump; # modelName: Contribution from Daniel Lee @ DWD -concept modelName (defaultName,"modelName.def",conceptsMasterDir,conceptsLocalDirAll): no_copy,dump,read_only; +concept modelName (defaultName,"modelName.def",conceptsDir2,conceptsDir1): no_copy,dump,read_only; template_nofail names "grib2/products_[productionStatusOfProcessedData].def"; diff --git a/definitions/grib2/section.1.def b/definitions/grib2/section.1.def index 72415c998..cca229b19 100644 --- a/definitions/grib2/section.1.def +++ b/definitions/grib2/section.1.def @@ -136,3 +136,11 @@ concept is_uerra(zero) { constant conceptsMasterDir="grib2" : hidden; constant conceptsLocalDirAll="grib2/localConcepts/[centre:s]" : hidden; constant conceptsLocalDirECMF="grib2/localConcepts/ecmf" : hidden; + +if (preferLocalConcepts) { + constant conceptsDir1 = conceptsMasterDir : hidden; + constant conceptsDir2 = conceptsLocalDirAll : hidden; +} else { + constant conceptsDir1 = conceptsLocalDirAll : hidden; + constant conceptsDir2 = conceptsMasterDir : hidden; +} diff --git a/definitions/grib2/template.4.horizontal.def b/definitions/grib2/template.4.horizontal.def index d3739c0b1..071ae07b0 100755 --- a/definitions/grib2/template.4.horizontal.def +++ b/definitions/grib2/template.4.horizontal.def @@ -24,7 +24,7 @@ unsigned[4] scaledValueOfSecondFixedSurface = missing() : can_be_missing,dump,no transient pressureUnits="hPa"; -concept_nofail vertical.typeOfLevel (unknown,"typeOfLevelConcept.def",conceptsMasterDir,conceptsLocalDirAll); +concept_nofail vertical.typeOfLevel (unknown,"typeOfLevelConcept.def",conceptsDir2,conceptsDir1); alias levelType=typeOfFirstFixedSurface; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9940abddd..54156ef9c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -127,6 +127,7 @@ if( HAVE_BUILD_TOOLS ) bufr_ecc-887 grib_ecc-490 grib_ecc-756 + grib_ecc-806 grib_ecc-873 grib_ecc-600 grib_ecc-923 diff --git a/tests/grib_ecc-806.sh b/tests/grib_ecc-806.sh new file mode 100755 index 000000000..8fd5e640d --- /dev/null +++ b/tests/grib_ecc-806.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# (C) Copyright 2005- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +# + +. ./include.sh +set -u +# -------------------------------------------- +# This is the test for the JIRA issue ECC-806 +# Local definitions precedence order +# -------------------------------------------- +label="grib_ecc-806-test" +tempGrb=temp.${label}.grb +tempOut=temp.${label}.txt +tempErr=temp.${label}.err + +# This NCEP grib2 file has the keys +# discipline = 0 +# parameterCategory = 1 +# parameterNumber = 11 +# Which is normally matched by 3066 but when overriden by NCEP local definitions +# should become 260056 +# This assumes our definitions contain the local kwbc concepts +# See definitions/grib2/localConcepts/kwbc/paramId.def + +input=$data_dir/gfs.complex.mvmu.grib2 +res=`${tools_dir}/grib_get -p paramId,shortName,units,name $input` +[ "$res" = "3066 sde m Snow depth" ] + +# Change the precedence rule +res=`${tools_dir}/grib_get -s preferLocalConcepts=1 -p paramId,shortName,units,name $input` +[ "$res" = "260056 sdwe kg m**-2 Water equivalent of accumulated snow depth (deprecated)" ] + +# Clean up +rm -f $tempGrb $tempOut $tempErr