Merge branch 'develop' into feature/modernisation_nearest

definitions/boot.def

@@ -12,7 +12,8 @@ constant internalVersion=30 : hidden;
 meta checkInternalVersion check_internal_version(internalVersion) : hidden;
 
 # ECC-806: Local concepts precedence order
-transient preferLocalConcepts = 0 : hidden;
+preferLocalConceptsEnvVar = getenv("ECCODES_GRIB_PREFER_LOCAL_CONCEPTS","0") : hidden;
+transient preferLocalConcepts = preferLocalConceptsEnvVar : hidden;
 
 constant defaultTypeOfLevel="unknown" : hidden;
 

definitions/create_def.pl

@@ -38,7 +38,8 @@ sub create_cfName {
     attribute.id=grib.attribute_id and
     centre.id=grib_encoding.centre_id and
     units.id=param.units_id and
-    param.id=cf.grib1_ecmwf order by
+    param.id=cf.grib1_ecmwf and
+    grib_encoding.is_legacy=0 order by
     edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
 EOF
 
@@ -98,6 +99,83 @@ EOF
     close(TAR);
 }
 
+sub create_cfName_legacy {
+    my $p; my %seen; 
+    my ($key) = "cfName";
+    my $field = "cf.name";
+
+    my $query= <<"EOF";
+    select $field,force128,edition,
+    centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName from
+    param,grib_encoding,grib,attribute,centre,units,cf where
+    param.hide_def=0 and
+#    param.retired=0 and
+    grib_encoding.id=grib.encoding_id and
+    param.id=grib_encoding.param_id and
+    attribute.id=grib.attribute_id and
+    centre.id=grib_encoding.centre_id and
+    units.id=param.units_id and
+    param.id=cf.grib1_ecmwf and
+    grib_encoding.is_legacy=1 order by
+    edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
+EOF
+
+    my $qh=$dbh->prepare($query);
+    $qh->execute();
+
+    # file containing the list of grib api parameters files we want to tar and 
+    # distribute to users for them to download and update their list of parameter
+    # to the latest
+    #open(TAR,$tarfilesflag ? ">>" : ">","tarfiles.txt") or die "Count not open file tarfiles.txt: $!";
+    #$tarfilesflag=1;
+
+    while (my ($keyval,$force128,$edition,$centre,$paramId,$attribute,$value,$name,$shortName)=$qh->fetchrow_array )
+    {
+        if ($centre eq "wmo" ) { $conceptDir=""; }
+        else { $conceptDir="/localConcepts/$centre"; }
+
+        if ($filebase ne "$basedir/grib$edition$conceptDir") {
+            if ($filebase) {
+                print $out "}\n";
+                close $out;
+            }
+            $filebase="$basedir/grib$edition$conceptDir";
+            mkpath($filebase);
+
+            print TAR "grib$edition$conceptDir/$key.legacy.def\n";
+            #system("cp -f $filebase/$key.def $filebase/$key.def.orig");
+            open($out,"> $filebase/$key.legacy.def")
+                or die "unable to open $filebase/$key.legacy.def";
+            print $out "# Automatically generated by $0, do not edit\n";
+            $p=();
+        }
+        if ($p ne $paramId || exists($seen{$attribute}) ) {
+           if ($p) { print $out "\t}\n"; }
+           print $out "#$name\n" ;
+           print $out "\'".$keyval."\' = {\n" ;
+           $p=$paramId;
+           %seen=();
+        }
+        $seen{$attribute}=1;
+        print "($key=$keyval) $edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
+        # we need to allow strings in the attribute_value field 
+        # for the moment we apply a patch here
+        if ($attribute =~ /stepType/ ) {
+            $value="\"accum\"";
+        }
+        if ($value eq '') {
+            $value="missing()";
+        }
+        print $out "\t $attribute = $value ;\n" ;
+    }
+    if ($filebase) {
+       print $out "}\n";
+       close $out;
+    }
+
+    close(TAR);
+}
+
 sub create_def {
     my $p; my %seen; 
     my ($key) =@_;
@@ -117,7 +195,8 @@ sub create_def {
         param.id=grib_encoding.param_id and
         attribute.id=grib.attribute_id and
         centre.id=grib_encoding.centre_id and
-        units.id=param.units_id
+        units.id=param.units_id and
+        grib_encoding.is_legacy=0
         order by edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
 EOF
 
@@ -183,29 +262,27 @@ EOF
     close(TAR);
 }
 
-# See ECC-1886
+sub create_def_legacy {
-sub create_cfVarName {
     my $p; my %seen; 
     my ($key) =@_;
     my $field=$key;
 
-    #if ($key =~ /paramId/) { $field="param.id"; } 
+    if ($key =~ /paramId/) { $field="param.id"; } 
-    #if ($key =~ /name/) { $field="param.name"; } 
+    if ($key =~ /name/) { $field="param.name"; } 
-    #if ($key =~ /units/) { $field="units.name"; } 
+    if ($key =~ /units/) { $field="units.name"; } 
-    if ($key =~ /cfVarName/) { $field="cfVarName"; }
 
     my $query= <<"EOF";
         select $field,force128,edition,
         centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName
         from param,grib_encoding,grib,attribute,centre,units where
         param.hide_def=0 and
-        param.retired=0 and
+#        param.retired=0 and
         grib_encoding.id=grib.encoding_id and
         param.id=grib_encoding.param_id and
         attribute.id=grib.attribute_id and
         centre.id=grib_encoding.centre_id and
-        units.id=param.units_id
+        units.id=param.units_id and
-        and cfVarName IS NOT NULL
+        grib_encoding.is_legacy=1
         order by edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
 EOF
 
@@ -237,10 +314,10 @@ EOF
             #copy("$filebase/$key.def","$filebase/$key.def.bkp") 
             #  or die ("unable to copy $filebase/$key.def");
 
-            print TAR "grib$edition$conceptDir/$key.def\n";
+            print TAR "grib$edition$conceptDir/$key.legacy.def\n";
             #system("cp -f $filebase/$key.def $filebase/$key.def.orig");
-            open($out,"> $filebase/$key.def")
+            open($out,"> $filebase/$key.legacy.def")
-                or die "unable to open $filebase/$key.def";
+                or die "unable to open $filebase/$key.legacy.def";
             print $out "# Automatically generated by $0, do not edit\n";
             $p=();
         }
@@ -281,7 +358,8 @@ sub create_paramId_def {
     grib_encoding.id=grib.encoding_id and
     param.id=grib_encoding.param_id and
     attribute.id=grib.attribute_id and
-    centre.id=grib_encoding.centre_id
+    centre.id=grib_encoding.centre_id and
+    grib_encoding.is_legacy=0
     order by edition,centre_id,param.o,param.id,attribute.o";
     my $qh=$dbh->prepare($query);
     $qh->execute();
@@ -355,13 +433,15 @@ sub create_def_old {
 }
 
 create_def("paramId");
+create_def_legacy("paramId");
 create_def("shortName");
+create_def_legacy("shortName");
 create_def("name");
+create_def_legacy("name");
 create_def("units");
-# Note: The cfVarName.def files are smaller than the rest. We only store the
+create_def_legacy("units");
-#       minimum set necessary. See ECC-1886
-create_cfVarName("cfVarName");
 create_cfName("cfName");
+create_cfName_legacy("cfName");
 
 # #create_paramId_def();
 

definitions/grib1/local.98.16.def

@@ -29,7 +29,7 @@ transient hourOfEndOfOverallTimeInterval=23;
 transient minuteOfEndOfOverallTimeInterval=59;
 transient secondOfEndOfOverallTimeInterval=59;
 
-transient indicatorOfUnitForTimeRange=3;
+transient indicatorOfUnitForTimeRange=3; # month
 transient lengthOfTimeRange=1;
 unsigned[1] averagingPeriod : dump ;
 

definitions/grib1/local.98.19.def

@@ -12,6 +12,7 @@ alias perturbationNumber=number;
 
 unsigned[1] ensembleSize  : dump;
 alias totalNumber=ensembleSize;
+alias numberOfForecastsInEnsemble=ensembleSize;
 
 meta quantile sprintf("%s:%s",number,ensembleSize);
 

definitions/grib1/section.1.def

@@ -334,3 +334,11 @@ meta md5Product md5(offsetSection1,section1Length,gridDefinition,section1Flags,d
 
 # ECC-1806
 concept_nofail paramIdForConversion(zero, "paramIdForConversion.def", conceptsDir2, conceptsDir1) : long_type,read_only;
+
+# ECC-1954
+if (productDefinitionTemplateNumber == 0 && defined(perturbationNumber) && defined(numberOfForecastsInEnsemble) && numberOfForecastsInEnsemble > 0) {
+  alias productDefinitionTemplateNumber = one; # ensemble instant
+}
+if (productDefinitionTemplateNumber == 8 && defined(perturbationNumber) && defined(numberOfForecastsInEnsemble) && numberOfForecastsInEnsemble > 0) {
+  alias productDefinitionTemplateNumber = eleven; # ensemble interval
+}

definitions/grib2/local/ecmf/section4_extras.def

@@ -5,7 +5,7 @@ if (centre is "ecmf" or datasetForLocal is "era6") {
  concept modelName(unknown, "modelNameConcept.def", conceptsMasterDir, conceptsLocalDirAll): no_copy, dump, read_only;
 
  if (modelName isnot "unknown") {
-    concept modelVersion(unknown, "modelVersionConcept_[modelName].def", conceptsMasterDir, conceptsLocalDirAll): no_copy, dump, read_only;
+    concept modelVersion(unknown, "modelVersionConcept.[modelName].def", conceptsMasterDir, conceptsLocalDirAll): no_copy, dump, read_only;
     # alias ls.model = modelName;
     # alias mars.model = modelName;
  }

definitions/grib2/localConcepts/ecmf/modelVersionConcept.IFS.def

@@ -0,0 +1,18 @@
+'cy50r3' = { generatingProcessIdentifier = 163; }
+'cy50r2' = { generatingProcessIdentifier = 162; }
+'cy50r1' = { generatingProcessIdentifier = 161; }
+'cy49r3' = { generatingProcessIdentifier = 160; }
+'cy49r2' = { generatingProcessIdentifier = 159; }
+'cy49r1' = { generatingProcessIdentifier = 158; }
+'cy48r3' = { generatingProcessIdentifier = 157; }
+'climatedt' = { generatingProcessIdentifier = 156; }
+'cy48r2' = { generatingProcessIdentifier = 155; }
+'cy48r1' = { generatingProcessIdentifier = 154; }
+'cy47r3' = { generatingProcessIdentifier = 153; }
+'cy47r2' = { generatingProcessIdentifier = 152; }
+'cy47r1' = { generatingProcessIdentifier = 151; }
+'cy46r1' = { generatingProcessIdentifier = 150; }
+'cy45r1' = { generatingProcessIdentifier = 149; }
+'cy43r3' = { generatingProcessIdentifier = 148; }
+'cy43r1' = { generatingProcessIdentifier = 147; }
+'cy41r2' = { generatingProcessIdentifier = 146; }

definitions/grib2/localConcepts/ecmf/modelVersionConcept_IFS.def

@@ -1,10 +0,0 @@
-'cy49r1' = { generatingProcessIdentifier = 155; } 
-'cy48r1' = { generatingProcessIdentifier = 154; }
-'cy47r3' = { generatingProcessIdentifier = 153; }
-'cy47r2' = { generatingProcessIdentifier = 152; }
-'cy47r1' = { generatingProcessIdentifier = 151; }
-'cy46r1' = { generatingProcessIdentifier = 150; }
-'cy45r1' = { generatingProcessIdentifier = 149; }
-'cy43r3' = { generatingProcessIdentifier = 148; }
-'cy43r1' = { generatingProcessIdentifier = 147; }
-'cy41r2' = { generatingProcessIdentifier = 146; }

definitions/grib2/name.legacy.def

@@ -1,3 +1,60 @@
+# Automatically generated by ./create_def.pl, do not edit
+#Instantaneous eastward turbulent surface stress
+'Instantaneous eastward turbulent surface stress' = {
+	 discipline = 0 ;
+	 parameterCategory = 2 ;
+	 parameterNumber = 38 ;
+	 typeOfFirstFixedSurface = 1 ;
+	}
+#Instantaneous northward turbulent surface stress
+'Instantaneous northward turbulent surface stress' = {
+	 discipline = 0 ;
+	 parameterCategory = 2 ;
+	 parameterNumber = 37 ;
+	 typeOfFirstFixedSurface = 1 ;
+	}
+#Minimum dew point depression
+'Minimum dew point depression' = {
+	 discipline = 0 ;
+	 parameterCategory = 0 ;
+	 parameterNumber = 14 ;
+	}
+#Maximum relative humidity
+'Maximum relative humidity' = {
+	 discipline = 0 ;
+	 parameterCategory = 1 ;
+	 parameterNumber = 27 ;
+	}
+#Maximum absolute humidity
+'Maximum absolute humidity' = {
+	 discipline = 0 ;
+	 parameterCategory = 1 ;
+	 parameterNumber = 28 ;
+	}
+#Maximum wind speed
+'Maximum wind speed' = {
+	 discipline = 0 ;
+	 parameterCategory = 2 ;
+	 parameterNumber = 21 ;
+	}
+#Net long wave radiation flux (surface)
+'Net long wave radiation flux (surface)' = {
+	 discipline = 0 ;
+	 parameterCategory = 5 ;
+	 parameterNumber = 0 ;
+	}
+#Net long wave radiation flux (top of atmosphere)
+'Net long wave radiation flux (top of atmosphere)' = {
+	 discipline = 0 ;
+	 parameterCategory = 5 ;
+	 parameterNumber = 1 ;
+	}
+#Maximum snow albedo
+'Maximum snow albedo' = {
+	 discipline = 0 ;
+	 parameterCategory = 19 ;
+	 parameterNumber = 17 ;
+	}
 #Volumetric soil moisture content
 'Volumetric soil moisture content' = {
 	 discipline = 2 ;
@@ -52,20 +109,6 @@
 	 parameterCategory = 3 ;
 	 parameterNumber = 9 ;
 	}
-#Instantaneous eastward turbulent surface stress
-'Instantaneous eastward turbulent surface stress' = {
- discipline = 0 ;
- parameterCategory = 2 ;
- parameterNumber = 38 ;
- typeOfFirstFixedSurface = 1 ;
-}
-#Instantaneous northward turbulent surface stress
-'Instantaneous northward turbulent surface stress' = {
- discipline = 0 ;
- parameterCategory = 2 ;
- parameterNumber = 37 ;
- typeOfFirstFixedSurface = 1 ;
-}
 #Evaporation in the last 6 hours
 'Evaporation in the last 6 hours' = {
 	 discipline = 0 ;
@@ -75,7 +118,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 6 ;
-}
+	}
 #Time-mean evapotranspiration rate in the last 24h
 'Time-mean evapotranspiration rate in the last 24h' = {
 	 discipline = 2 ;
@@ -84,7 +127,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-integrated potential evapotranspiration rate in the last 24h
 'Time-integrated potential evapotranspiration rate in the last 24h' = {
 	 discipline = 2 ;
@@ -93,7 +136,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean potential evapotranspiration rate in the last 24h
 'Time-mean potential evapotranspiration rate in the last 24h' = {
 	 discipline = 2 ;
@@ -102,7 +145,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean volumetric soil moisture
 'Time-mean volumetric soil moisture' = {
 	 discipline = 2 ;
@@ -111,7 +154,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-integrated water runoff and drainage rate in the last 24h
 'Time-integrated water runoff and drainage rate in the last 24h' = {
 	 discipline = 2 ;
@@ -120,7 +163,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean water runoff and drainage rate in the last 24h
 'Time-mean water runoff and drainage rate in the last 24h' = {
 	 discipline = 2 ;
@@ -129,7 +172,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean snow depth water equivalent
 'Time-mean snow depth water equivalent' = {
 	 discipline = 0 ;
@@ -138,7 +181,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean skin temperature
 'Time-mean skin temperature' = {
 	 discipline = 0 ;
@@ -147,7 +190,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-integrated snow melt rate in the last 24h
 'Time-integrated snow melt rate in the last 24h' = {
 	 discipline = 2 ;
@@ -156,7 +199,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Upward sea water velocity
 'Upward sea water velocity' = {
 	 discipline = 10 ;
@@ -164,7 +207,7 @@
 	 parameterNumber = 27 ;
 	 typeOfFirstFixedSurface = 168 ;
 	 typeOfSecondFixedSurface = 168 ;
-}
+	}
 #Time-mean upward sea water velocity
 'Time-mean upward sea water velocity' = {
 	 discipline = 10 ;
@@ -173,82 +216,34 @@
 	 typeOfFirstFixedSurface = 168 ;
 	 typeOfSecondFixedSurface = 168 ;
 	 typeOfStatisticalProcessing = 0 ;
-}
+	}
-#Maximum individual wave height
-'Maximum individual wave height' = {
- discipline = 10 ;
- parameterCategory = 0 ;
- parameterNumber = 24 ;
-}
-#Soil temperature
-'Soil temperature' = {
- discipline = 2 ;
- parameterCategory = 0 ;
- parameterNumber = 2 ;
-}
 #Maximum temperature
 'Maximum temperature' = {
 	 discipline = 0 ;
 	 parameterCategory = 0 ;
 	 parameterNumber = 4 ;
-}
+	}
 #Minimum temperature
 'Minimum temperature' = {
 	 discipline = 0 ;
 	 parameterCategory = 0 ;
 	 parameterNumber = 5 ;
-}
+	}
-#Minimum dew point depression
+#Envelop-maximum individual wave height
-'Minimum dew point depression' = {
+'Envelop-maximum individual wave height' = {
- discipline = 0 ;
+	 discipline = 10 ;
 	 parameterCategory = 0 ;
- parameterNumber = 14 ;
+	 parameterNumber = 24 ;
-}
+	}
-#Maximum relative humidity
+#Soil moisture
-'Maximum relative humidity' = {
+'Soil moisture' = {
- discipline = 0 ;
+	 discipline = 2 ;
- parameterCategory = 1 ;
+	 parameterCategory = 0 ;
- parameterNumber = 27 ;
+	 parameterNumber = 22 ;
-}
+	}
-#Maximum absolute humidity
-'Maximum absolute humidity' = {
- discipline = 0 ;
- parameterCategory = 1 ;
- parameterNumber = 28 ;
-}
-#Maximum wind speed
-'Maximum wind speed' = {
- discipline = 0 ;
- parameterCategory = 2 ;
- parameterNumber = 21 ;
-}
-#Net long wave radiation flux (surface)
-'Net long wave radiation flux (surface)' = {
- discipline = 0 ;
- parameterCategory = 5 ;
- parameterNumber = 0 ;
-}
-#Net long wave radiation flux (top of atmosphere)
-'Net long wave radiation flux (top of atmosphere)' = {
- discipline = 0 ;
- parameterCategory = 5 ;
- parameterNumber = 1 ;
-}
-#Maximum snow albedo
-'Maximum snow albedo' = {
- discipline = 0 ;
- parameterCategory = 19 ;
- parameterNumber = 17 ;
-}
 #Soil temperature
 'Soil temperature' = {
 	 discipline = 2 ;
 	 parameterCategory = 0 ;
 	 parameterNumber = 2 ;
 }
-#Soil moisture
-'Soil moisture' = {
- discipline = 2 ;
- parameterCategory = 0 ;
- parameterNumber = 22 ;
-}

definitions/grib2/paramId.legacy.def

@@ -1,3 +1,60 @@
+# Automatically generated by ./create_def.pl, do not edit
+#Instantaneous eastward turbulent surface stress
+'229' = {
+	 discipline = 0 ;
+	 parameterCategory = 2 ;
+	 parameterNumber = 38 ;
+	 typeOfFirstFixedSurface = 1 ;
+	}
+#Instantaneous northward turbulent surface stress
+'230' = {
+	 discipline = 0 ;
+	 parameterCategory = 2 ;
+	 parameterNumber = 37 ;
+	 typeOfFirstFixedSurface = 1 ;
+	}
+#Minimum dew point depression
+'260006' = {
+	 discipline = 0 ;
+	 parameterCategory = 0 ;
+	 parameterNumber = 14 ;
+	}
+#Maximum relative humidity
+'260023' = {
+	 discipline = 0 ;
+	 parameterCategory = 1 ;
+	 parameterNumber = 27 ;
+	}
+#Maximum absolute humidity
+'260024' = {
+	 discipline = 0 ;
+	 parameterCategory = 1 ;
+	 parameterNumber = 28 ;
+	}
+#Maximum wind speed
+'260064' = {
+	 discipline = 0 ;
+	 parameterCategory = 2 ;
+	 parameterNumber = 21 ;
+	}
+#Net long wave radiation flux (surface)
+'260095' = {
+	 discipline = 0 ;
+	 parameterCategory = 5 ;
+	 parameterNumber = 0 ;
+	}
+#Net long wave radiation flux (top of atmosphere)
+'260096' = {
+	 discipline = 0 ;
+	 parameterCategory = 5 ;
+	 parameterNumber = 1 ;
+	}
+#Maximum snow albedo
+'260161' = {
+	 discipline = 0 ;
+	 parameterCategory = 19 ;
+	 parameterNumber = 17 ;
+	}
 #Volumetric soil moisture content
 '260185' = {
 	 discipline = 2 ;
@@ -52,20 +109,6 @@
 	 parameterCategory = 3 ;
 	 parameterNumber = 9 ;
 	}
-#Instantaneous eastward turbulent surface stress
-'229' = {
- discipline = 0 ;
- parameterCategory = 2 ;
- parameterNumber = 38 ;
- typeOfFirstFixedSurface = 1 ;
-}
-#Instantaneous northward turbulent surface stress
-'230' = {
- discipline = 0 ;
- parameterCategory = 2 ;
- parameterNumber = 37 ;
- typeOfFirstFixedSurface = 1 ;
-}
 #Evaporation in the last 6 hours
 '260265' = {
 	 discipline = 0 ;
@@ -75,7 +118,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 6 ;
-}
+	}
 #Time-mean evapotranspiration rate in the last 24h
 '260435' = {
 	 discipline = 2 ;
@@ -84,7 +127,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-integrated potential evapotranspiration rate in the last 24h
 '260437' = {
 	 discipline = 2 ;
@@ -93,7 +136,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean potential evapotranspiration rate in the last 24h
 '260438' = {
 	 discipline = 2 ;
@@ -102,7 +145,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean volumetric soil moisture
 '260440' = {
 	 discipline = 2 ;
@@ -111,7 +154,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-integrated water runoff and drainage rate in the last 24h
 '260444' = {
 	 discipline = 2 ;
@@ -120,7 +163,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean water runoff and drainage rate in the last 24h
 '260445' = {
 	 discipline = 2 ;
@@ -129,7 +172,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean snow depth water equivalent
 '260472' = {
 	 discipline = 0 ;
@@ -138,7 +181,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean skin temperature
 '260473' = {
 	 discipline = 0 ;
@@ -147,7 +190,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-integrated snow melt rate in the last 24h
 '260476' = {
 	 discipline = 2 ;
@@ -156,7 +199,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Upward sea water velocity
 '262507' = {
 	 discipline = 10 ;
@@ -164,7 +207,7 @@
 	 parameterNumber = 27 ;
 	 typeOfFirstFixedSurface = 168 ;
 	 typeOfSecondFixedSurface = 168 ;
-}
+	}
 #Time-mean upward sea water velocity
 '263507' = {
 	 discipline = 10 ;
@@ -173,82 +216,34 @@
 	 typeOfFirstFixedSurface = 168 ;
 	 typeOfSecondFixedSurface = 168 ;
 	 typeOfStatisticalProcessing = 0 ;
-}
+	}
-#Maximum individual wave height
-'140218' = {
- discipline = 10 ;
- parameterCategory = 0 ;
- parameterNumber = 24 ;
-}
-#Soil temperature
-'228139' = {
- discipline = 2 ;
- parameterCategory = 0 ;
- parameterNumber = 2 ;
-}
 #Maximum temperature
 '3015' = {
 	 discipline = 0 ;
 	 parameterCategory = 0 ;
 	 parameterNumber = 4 ;
-}
+	}
 #Minimum temperature
 '3016' = {
 	 discipline = 0 ;
 	 parameterCategory = 0 ;
 	 parameterNumber = 5 ;
-}
+	}
-#Minimum dew point depression
+#Envelop-maximum individual wave height
-'260006' = {
+'140218' = {
- discipline = 0 ;
+	 discipline = 10 ;
 	 parameterCategory = 0 ;
- parameterNumber = 14 ;
+	 parameterNumber = 24 ;
-}
+	}
-#Maximum relative humidity
+#Soil moisture
-'260023' = {
+'228039' = {
- discipline = 0 ;
+	 discipline = 2 ;
- parameterCategory = 1 ;
+	 parameterCategory = 0 ;
- parameterNumber = 27 ;
+	 parameterNumber = 22 ;
-}
+	}
-#Maximum absolute humidity
-'260024' = {
- discipline = 0 ;
- parameterCategory = 1 ;
- parameterNumber = 28 ;
-}
-#Maximum wind speed
-'260064' = {
- discipline = 0 ;
- parameterCategory = 2 ;
- parameterNumber = 21 ;
-}
-#Net long wave radiation flux (surface)
-'260095' = {
- discipline = 0 ;
- parameterCategory = 5 ;
- parameterNumber = 0 ;
-}
-#Net long wave radiation flux (top of atmosphere)
-'260096' = {
- discipline = 0 ;
- parameterCategory = 5 ;
- parameterNumber = 1 ;
-}
-#Maximum snow albedo
-'260161' = {
- discipline = 0 ;
- parameterCategory = 19 ;
- parameterNumber = 17 ;
-}
 #Soil temperature
 '228139' = {
 	 discipline = 2 ;
 	 parameterCategory = 0 ;
 	 parameterNumber = 2 ;
 }
-#Soil moisture
-'228039' = {
- discipline = 2 ;
- parameterCategory = 0 ;
- parameterNumber = 22 ;
-}

definitions/grib2/shortName.legacy.def

@@ -1,3 +1,60 @@
+# Automatically generated by ./create_def.pl, do not edit
+#Instantaneous eastward turbulent surface stress
+'iews' = {
+	 discipline = 0 ;
+	 parameterCategory = 2 ;
+	 parameterNumber = 38 ;
+	 typeOfFirstFixedSurface = 1 ;
+	}
+#Instantaneous northward turbulent surface stress
+'inss' = {
+	 discipline = 0 ;
+	 parameterCategory = 2 ;
+	 parameterNumber = 37 ;
+	 typeOfFirstFixedSurface = 1 ;
+	}
+#Minimum dew point depression
+'mindpd' = {
+	 discipline = 0 ;
+	 parameterCategory = 0 ;
+	 parameterNumber = 14 ;
+	}
+#Maximum relative humidity
+'maxrh' = {
+	 discipline = 0 ;
+	 parameterCategory = 1 ;
+	 parameterNumber = 27 ;
+	}
+#Maximum absolute humidity
+'maxah' = {
+	 discipline = 0 ;
+	 parameterCategory = 1 ;
+	 parameterNumber = 28 ;
+	}
+#Maximum wind speed
+'maxgust' = {
+	 discipline = 0 ;
+	 parameterCategory = 2 ;
+	 parameterNumber = 21 ;
+	}
+#Net long wave radiation flux (surface)
+'nlwrs' = {
+	 discipline = 0 ;
+	 parameterCategory = 5 ;
+	 parameterNumber = 0 ;
+	}
+#Net long wave radiation flux (top of atmosphere)
+'nlwrt' = {
+	 discipline = 0 ;
+	 parameterCategory = 5 ;
+	 parameterNumber = 1 ;
+	}
+#Maximum snow albedo
+'mxsalb' = {
+	 discipline = 0 ;
+	 parameterCategory = 19 ;
+	 parameterNumber = 17 ;
+	}
 #Volumetric soil moisture content
 'soilw' = {
 	 discipline = 2 ;
@@ -52,20 +109,6 @@
 	 parameterCategory = 3 ;
 	 parameterNumber = 9 ;
 	}
-#Instantaneous eastward turbulent surface stress
-'iews' = {
- discipline = 0 ;
- parameterCategory = 2 ;
- parameterNumber = 38 ;
- typeOfFirstFixedSurface = 1 ;
-}
-#Instantaneous northward turbulent surface stress
-'inss' = {
- discipline = 0 ;
- parameterCategory = 2 ;
- parameterNumber = 37 ;
- typeOfFirstFixedSurface = 1 ;
-}
 #Evaporation in the last 6 hours
 'eva06' = {
 	 discipline = 0 ;
@@ -75,7 +118,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 6 ;
-}
+	}
 #Time-mean evapotranspiration rate in the last 24h
 'avg_et24' = {
 	 discipline = 2 ;
@@ -84,7 +127,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-integrated potential evapotranspiration rate in the last 24h
 'acc_pet24' = {
 	 discipline = 2 ;
@@ -93,7 +136,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean potential evapotranspiration rate in the last 24h
 'avg_pet24' = {
 	 discipline = 2 ;
@@ -102,7 +145,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean volumetric soil moisture
 'avg_swv24' = {
 	 discipline = 2 ;
@@ -111,7 +154,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-integrated water runoff and drainage rate in the last 24h
 'acc_rod24' = {
 	 discipline = 2 ;
@@ -120,7 +163,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean water runoff and drainage rate in the last 24h
 'avg_rod24' = {
 	 discipline = 2 ;
@@ -129,7 +172,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean snow depth water equivalent
 'avg_sd24' = {
 	 discipline = 0 ;
@@ -138,7 +181,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean skin temperature
 'avg_skt24' = {
 	 discipline = 0 ;
@@ -147,7 +190,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-integrated snow melt rate in the last 24h
 'acc_smr24' = {
 	 discipline = 2 ;
@@ -156,7 +199,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Upward sea water velocity
 'wo' = {
 	 discipline = 10 ;
@@ -164,7 +207,7 @@
 	 parameterNumber = 27 ;
 	 typeOfFirstFixedSurface = 168 ;
 	 typeOfSecondFixedSurface = 168 ;
-}
+	}
 #Time-mean upward sea water velocity
 'avg_wo' = {
 	 discipline = 10 ;
@@ -173,82 +216,34 @@
 	 typeOfFirstFixedSurface = 168 ;
 	 typeOfSecondFixedSurface = 168 ;
 	 typeOfStatisticalProcessing = 0 ;
-}
+	}
-#Maximum individual wave height
-'hmax' = {
- discipline = 10 ;
- parameterCategory = 0 ;
- parameterNumber = 24 ;
-}
-#Soil temperature
-'st' = {
- discipline = 2 ;
- parameterCategory = 0 ;
- parameterNumber = 2 ;
-}
 #Maximum temperature
 'tmax' = {
 	 discipline = 0 ;
 	 parameterCategory = 0 ;
 	 parameterNumber = 4 ;
-}
+	}
 #Minimum temperature
 'tmin' = {
 	 discipline = 0 ;
 	 parameterCategory = 0 ;
 	 parameterNumber = 5 ;
-}
+	}
-#Minimum dew point depression
+#Envelop-maximum individual wave height
-'mindpd' = {
+'hmax' = {
- discipline = 0 ;
+	 discipline = 10 ;
 	 parameterCategory = 0 ;
- parameterNumber = 14 ;
+	 parameterNumber = 24 ;
-}
+	}
-#Maximum relative humidity
+#Soil moisture
-'maxrh' = {
+'sm' = {
- discipline = 0 ;
+	 discipline = 2 ;
- parameterCategory = 1 ;
+	 parameterCategory = 0 ;
- parameterNumber = 27 ;
+	 parameterNumber = 22 ;
-}
+	}
-#Maximum absolute humidity
-'maxah' = {
- discipline = 0 ;
- parameterCategory = 1 ;
- parameterNumber = 28 ;
-}
-#Maximum wind speed
-'maxgust' = {
- discipline = 0 ;
- parameterCategory = 2 ;
- parameterNumber = 21 ;
-}
-#Net long wave radiation flux (surface)
-'nlwrs' = {
- discipline = 0 ;
- parameterCategory = 5 ;
- parameterNumber = 0 ;
-}
-#Net long wave radiation flux (top of atmosphere)
-'nlwrt' = {
- discipline = 0 ;
- parameterCategory = 5 ;
- parameterNumber = 1 ;
-}
-#Maximum snow albedo
-'mxsalb' = {
- discipline = 0 ;
- parameterCategory = 19 ;
- parameterNumber = 17 ;
-}
 #Soil temperature
 'st' = {
 	 discipline = 2 ;
 	 parameterCategory = 0 ;
 	 parameterNumber = 2 ;
 }
-#Soil moisture
-'sm' = {
- discipline = 2 ;
- parameterCategory = 0 ;
- parameterNumber = 22 ;
-}

definitions/grib2/units.legacy.def

@@ -1,3 +1,60 @@
+# Automatically generated by ./create_def.pl, do not edit
+#Instantaneous eastward turbulent surface stress
+'N m**-2' = {
+	 discipline = 0 ;
+	 parameterCategory = 2 ;
+	 parameterNumber = 38 ;
+	 typeOfFirstFixedSurface = 1 ;
+	}
+#Instantaneous northward turbulent surface stress
+'N m**-2' = {
+	 discipline = 0 ;
+	 parameterCategory = 2 ;
+	 parameterNumber = 37 ;
+	 typeOfFirstFixedSurface = 1 ;
+	}
+#Minimum dew point depression
+'K' = {
+	 discipline = 0 ;
+	 parameterCategory = 0 ;
+	 parameterNumber = 14 ;
+	}
+#Maximum relative humidity
+'%' = {
+	 discipline = 0 ;
+	 parameterCategory = 1 ;
+	 parameterNumber = 27 ;
+	}
+#Maximum absolute humidity
+'kg m**-3' = {
+	 discipline = 0 ;
+	 parameterCategory = 1 ;
+	 parameterNumber = 28 ;
+	}
+#Maximum wind speed
+'m s**-1' = {
+	 discipline = 0 ;
+	 parameterCategory = 2 ;
+	 parameterNumber = 21 ;
+	}
+#Net long wave radiation flux (surface)
+'W m**-2' = {
+	 discipline = 0 ;
+	 parameterCategory = 5 ;
+	 parameterNumber = 0 ;
+	}
+#Net long wave radiation flux (top of atmosphere)
+'W m**-2' = {
+	 discipline = 0 ;
+	 parameterCategory = 5 ;
+	 parameterNumber = 1 ;
+	}
+#Maximum snow albedo
+'%' = {
+	 discipline = 0 ;
+	 parameterCategory = 19 ;
+	 parameterNumber = 17 ;
+	}
 #Volumetric soil moisture content
 'Proportion' = {
 	 discipline = 2 ;
@@ -52,20 +109,6 @@
 	 parameterCategory = 3 ;
 	 parameterNumber = 9 ;
 	}
-#Instantaneous eastward turbulent surface stress
-'N m**-2' = {
- discipline = 0 ;
- parameterCategory = 2 ;
- parameterNumber = 38 ;
- typeOfFirstFixedSurface = 1 ;
-}
-#Instantaneous northward turbulent surface stress
-'N m**-2' = {
- discipline = 0 ;
- parameterCategory = 2 ;
- parameterNumber = 37 ;
- typeOfFirstFixedSurface = 1 ;
-}
 #Evaporation in the last 6 hours
 'kg m**-2' = {
 	 discipline = 0 ;
@@ -75,7 +118,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 6 ;
-}
+	}
 #Time-mean evapotranspiration rate in the last 24h
 'kg m**-2 s**-1' = {
 	 discipline = 2 ;
@@ -84,7 +127,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-integrated potential evapotranspiration rate in the last 24h
 'kg m**-2' = {
 	 discipline = 2 ;
@@ -93,7 +136,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean potential evapotranspiration rate in the last 24h
 'kg m**-2 s**-1' = {
 	 discipline = 2 ;
@@ -102,7 +145,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean volumetric soil moisture
 'm**3 m**-3' = {
 	 discipline = 2 ;
@@ -111,7 +154,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-integrated water runoff and drainage rate in the last 24h
 'kg m**-2' = {
 	 discipline = 2 ;
@@ -120,7 +163,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean water runoff and drainage rate in the last 24h
 'kg m**-2 s**-1' = {
 	 discipline = 2 ;
@@ -129,7 +172,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean snow depth water equivalent
 'kg m**-2' = {
 	 discipline = 0 ;
@@ -138,7 +181,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-mean skin temperature
 'K' = {
 	 discipline = 0 ;
@@ -147,7 +190,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 0 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Time-integrated snow melt rate in the last 24h
 'kg m**-2' = {
 	 discipline = 2 ;
@@ -156,7 +199,7 @@
 	 indicatorOfUnitForTimeRange = 1 ;
 	 typeOfStatisticalProcessing = 1 ;
 	 lengthOfTimeRange = 24 ;
-}
+	}
 #Upward sea water velocity
 'm s**-1' = {
 	 discipline = 10 ;
@@ -164,7 +207,7 @@
 	 parameterNumber = 27 ;
 	 typeOfFirstFixedSurface = 168 ;
 	 typeOfSecondFixedSurface = 168 ;
-}
+	}
 #Time-mean upward sea water velocity
 'm s**-1' = {
 	 discipline = 10 ;
@@ -173,82 +216,34 @@
 	 typeOfFirstFixedSurface = 168 ;
 	 typeOfSecondFixedSurface = 168 ;
 	 typeOfStatisticalProcessing = 0 ;
-}
+	}
-#Maximum individual wave height
-'m' = {
- discipline = 10 ;
- parameterCategory = 0 ;
- parameterNumber = 24 ;
-}
-#Soil temperature
-'K' = {
- discipline = 2 ;
- parameterCategory = 0 ;
- parameterNumber = 2 ;
-}
 #Maximum temperature
 'K' = {
 	 discipline = 0 ;
 	 parameterCategory = 0 ;
 	 parameterNumber = 4 ;
-}
+	}
 #Minimum temperature
 'K' = {
 	 discipline = 0 ;
 	 parameterCategory = 0 ;
 	 parameterNumber = 5 ;
-}
+	}
-#Minimum dew point depression
+#Envelop-maximum individual wave height
-'K' = {
+'m' = {
- discipline = 0 ;
+	 discipline = 10 ;
 	 parameterCategory = 0 ;
- parameterNumber = 14 ;
+	 parameterNumber = 24 ;
-}
+	}
-#Maximum relative humidity
+#Soil moisture
-'%' = {
- discipline = 0 ;
- parameterCategory = 1 ;
- parameterNumber = 27 ;
-}
-#Maximum absolute humidity
 'kg m**-3' = {
- discipline = 0 ;
+	 discipline = 2 ;
- parameterCategory = 1 ;
+	 parameterCategory = 0 ;
- parameterNumber = 28 ;
+	 parameterNumber = 22 ;
-}
+	}
-#Maximum wind speed
-'m s**-1' = {
- discipline = 0 ;
- parameterCategory = 2 ;
- parameterNumber = 21 ;
-}
-#Net long wave radiation flux (surface)
-'W m**-2' = {
- discipline = 0 ;
- parameterCategory = 5 ;
- parameterNumber = 0 ;
-}
-#Net long wave radiation flux (top of atmosphere)
-'W m**-2' = {
- discipline = 0 ;
- parameterCategory = 5 ;
- parameterNumber = 1 ;
-}
-#Maximum snow albedo
-'%' = {
- discipline = 0 ;
- parameterCategory = 19 ;
- parameterNumber = 17 ;
-}
 #Soil temperature
 'K' = {
 	 discipline = 2 ;
 	 parameterCategory = 0 ;
 	 parameterNumber = 2 ;
 }
-#Soil moisture
-'kg m**-3' = {
- discipline = 2 ;
- parameterCategory = 0 ;
- parameterNumber = 22 ;
-}

definitions/mars/grib.mmsf.an.def

@@ -1,6 +1,3 @@
-#no step in type an
-unalias mars.step;
-
 if (class is "od" || class is "me" || class is "en" ||
     class is "c3" || class is "ci")
 {

src/accessor/grib_accessor_class_ascii.cc

@@ -66,21 +66,41 @@ int grib_accessor_ascii_t::pack_string(const char* val, size_t* len)
 {
     grib_handle* hand = grib_handle_of_accessor(this);
     const size_t alen = length_;
-    if (len[0] > (alen + 1)) {
+    if (*len > (alen + 1)) {
         grib_context_log(context_, GRIB_LOG_ERROR,
-                         "pack_string: Wrong size (%zu) for %s, it contains %ld values",
+                         "%s: Buffer too small for %s. It is %zu bytes long (input string len=%zu)",
-                         len[0], name_, length_ + 1);
+                         class_name_, name_, alen, *len);
-        len[0] = 0;
+        *len = alen;
         return GRIB_BUFFER_TOO_SMALL;
     }
 
     for (size_t i = 0; i < alen; i++) {
-        if (i < len[0])
+        if (i < *len)
             hand->buffer->data[offset_ + i] = val[i];
         else
             hand->buffer->data[offset_ + i] = 0;
     }
 
+    // TODO(masn): Make this an error.
+    // But we have to allow this case unfortunately as returning an error breaks
+    // clients e.g. grib1 local def 40 has marsDomain of 2 bytes but local def 21
+    // has the same key with 1 byte! Legacy stuff that cannot be changed easily.
+    // So at least issue a warning
+    if (*len > alen) {
+        // Decode the string and compare with the incoming value
+        size_t size = 0;
+        if (grib_get_string_length_acc(this, &size) == GRIB_SUCCESS) {
+            char* value = (char*)grib_context_malloc_clear(context_, size);
+            if (value) {
+                if (this->unpack_string(value, &size) == GRIB_SUCCESS && !STR_EQUAL(val, value)) {
+                    fprintf(stderr, "ECCODES WARNING :  String input '%s' truncated to '%s'. Key %s is %zu byte(s)\n",
+                            val, value, name_, alen);
+                }
+                grib_context_free(context_, value);
+            }
+        }
+    }
+
     return GRIB_SUCCESS;
 }
 

src/action_class_alias.cc

@@ -178,6 +178,20 @@ static int create_accessor(grib_section* p, grib_action* act, grib_loader* h)
                                  act->name_space, act->name, y->name_);
                 /* printf("[%s %s]\n",y->all_names_[i], y->all_name_spaces_[i]); */
 
+                /* 
+                 * ECC-1898: Remove accessor from cache
+                 * This workaround was disabled because it was causing problems with the unaliasing mars.step,
+                 * i.e., when unaliasing "mars.step" it also unaliases "step"
+                 */
+
+                // TODO(maee): Implement a new hash function, which uses the name and the name_space as well
+
+                //grib_handle* hand = grib_handle_of_accessor(y);
+                //if (hand->use_trie && y->all_name_spaces_[i] != NULL && strcmp(y->name_, act->name) != 0) {
+                //    int id = grib_hash_keys_get_id(hand->context->keys, act->name);
+                //    hand->accessors[id] = NULL;
+                //}
+
                 while (i < MAX_ACCESSOR_NAMES - 1) {
                     y->all_names_[i]       = y->all_names_[i + 1];
                     y->all_name_spaces_[i] = y->all_name_spaces_[i + 1];

src/eccodes_prototypes.h

@@ -486,6 +486,7 @@ void grib_file_close(const char* filename, int force, int* err);
 void grib_file_close_all(int* err);
 grib_file* grib_get_file(const char* filename, int* err);
 void grib_file_delete(grib_file* file);
+void grib_file_pool_print(const char* title, FILE* out);
 
 /* grib_geography.cc */
 int grib_get_gaussian_latitudes(long trunc, double* lats);

src/grib_context.cc

@@ -1027,7 +1027,7 @@ void* grib_context_buffer_malloc_clear(const grib_context* c, size_t size)
 
 void grib_context_set_memory_proc(grib_context* c, grib_malloc_proc m, grib_free_proc f, grib_realloc_proc r)
 {
-    fprintf(stderr, "Warning: The %s function is deprecated and will be removed later.\n", __func__);
+    fprintf(stderr, "ECCODES WARNING :  The %s function is deprecated and will be removed in a future release.\n", __func__);
     c->free_mem    = f;
     c->alloc_mem   = m;
     c->realloc_mem = r;
@@ -1035,14 +1035,14 @@ void grib_context_set_memory_proc(grib_context* c, grib_malloc_proc m, grib_free
 
 void grib_context_set_persistent_memory_proc(grib_context* c, grib_malloc_proc m, grib_free_proc f)
 {
-    fprintf(stderr, "Warning: The %s function is deprecated and will be removed later.\n", __func__);
+    fprintf(stderr, "ECCODES WARNING :  The %s function is deprecated and will be removed in a future release.\n", __func__);
     c->free_persistent_mem  = f;
     c->alloc_persistent_mem = m;
 }
 
 void grib_context_set_buffer_memory_proc(grib_context* c, grib_malloc_proc m, grib_free_proc f, grib_realloc_proc r)
 {
-    fprintf(stderr, "Warning: The %s function is deprecated and will be removed later.\n", __func__);
+    fprintf(stderr, "ECCODES WARNING :  The %s function is deprecated and will be removed in a future release.\n", __func__);
     c->free_buffer_mem    = f;
     c->alloc_buffer_mem   = m;
     c->realloc_buffer_mem = r;

src/grib_filepool.cc

@@ -9,6 +9,8 @@
  */
 
 #include "grib_api_internal.h"
+#include <cstdio>
+
 #define GRIB_MAX_OPENED_FILES 200
 
 #if GRIB_PTHREADS
@@ -423,15 +425,22 @@ void grib_file_delete(grib_file* file)
     //    }
     //}
 
-    if (file->name)
+    free(file->name); file->name = 0;
-        free(file->name);
+    free(file->mode); file->mode = 0;
-    if (file->mode)
+    free(file->buffer); file->buffer = 0;
-        free(file->mode);
-
-    if (file->buffer) {
-        free(file->buffer);
-    }
     grib_context_free(file->context, file);
     /* file = NULL; */
     GRIB_MUTEX_UNLOCK(&mutex1);
 }
+
+void grib_file_pool_print(const char* title, FILE* out)
+{
+    int i = 0;
+    grib_file* file = file_pool.first;
+    printf("%s: size=%zu, num_opened_files=%d\n", title, file_pool.size, file_pool.number_of_opened_files);
+    while (file) {
+        printf("%s:\tfile_pool entry %d = %s\n", title, i++, file->name);
+        file = file->next;
+    }
+    printf("\n");
+}

tests/CMakeLists.txt

@@ -114,6 +114,7 @@ if( HAVE_BUILD_TOOLS )
         grib2_wave_spectra
         grib_element
         grib_suppressed
+        grib_destine_mars_keys
         grib_2nd_order_numValues
         grib_ecc-136
         grib_ecc-530
@@ -142,7 +143,6 @@ if( HAVE_BUILD_TOOLS )
         grib_ecc-1654
         grib_ecc-1671
         grib_ecc-1708
-        grib_ecc-1691
         grib_ecc-1766
         grib_ecc-1829
         bufr_ecc-1028
@@ -276,6 +276,7 @@ if( HAVE_BUILD_TOOLS )
         grib_ecc-1764
         grib_ecc-1792
         grib_ecc-1806
+        #grib_ecc-1898
         grib_ecc-1907
         grib_ecc-1941
         grib_ecc-1942

tests/grib1to2.sh

@@ -11,6 +11,7 @@
 . ./include.ctest.sh
 
 REDIRECT=/dev/null
+label="grib1to2_test"
 
 files="constant_field\
  reduced_gaussian_pressure_level_constant \
@@ -29,6 +30,8 @@ files="constant_field\
  spherical_pressure_level \
  spherical_model_level "
 
+tempLog=temp.$label.log
+
 for f in `echo $files`
 do
   file=${data_dir}/$f
@@ -53,7 +56,7 @@ done
 echo "ECC-457,ECC-1298 ECMWF total precipitation..."
 # ---------------------------------------------------
 input=${data_dir}/tp_ecmwf.grib
-output=temp.grib1to2.grib
+output=temp.$label.grib
 ${tools_dir}/grib_set -s edition=2 $input $output
 res=`${tools_dir}/grib_get -w count=1 -p edition,paramId,units $output`
 [ "$res" = "2 228 m" ]
@@ -93,8 +96,8 @@ grib_check_key_equals $output    shapeOfTheEarth 0
 
 echo "ECC-1329: Cannot convert runoff (paramId=205)"
 # --------------------------------------------------------
-temp1="temp1.grib1to2.grib1"
+temp1="temp1.$label.grib1"
-temp2="temp2.grib1to2.grib2"
+temp2="temp2.$label.grib2"
 ${tools_dir}/grib_set -s paramId=205,P1=240,marsType=fc $sample_g1 $temp1
 ${tools_dir}/grib_set -s edition=2 $temp1 $temp2
 grib_check_key_equals $temp2 discipline,stepType,shortName,paramId '2 accum ro 205'
@@ -111,10 +114,20 @@ for sn in e lsp pev sro uvb; do
     ${tools_dir}/grib_compare -e -b param $temp1 $temp2
 done
 
+
+echo "ECC-1954: preserve ensemble product template..."
+# --------------------------------------------------------
+${tools_dir}/grib_set -s \
+  indicatorOfParameter=49,P2=1,timeRangeIndicator=2,localDefinitionNumber=30,perturbationNumber=1,numberOfForecastsInEnsemble=51 \
+  $sample_g1 $temp1
+${tools_dir}/grib_set -s edition=2 $temp1 $temp2
+grib_check_key_equals $temp2 productDefinitionTemplateNumber,perturbationNumber '11 1'
+
+
 # Turn on (brief) DEBUGGING messages
 sample_g1=$ECCODES_SAMPLES_PATH/GRIB1.tmpl
-output=temp.grib1to2.grib
+output=temp.$label.grib
-ECCODES_DEBUG=-1 ${tools_dir}/grib_set -s edition=2 $sample_g1 $output
+ECCODES_DEBUG=-1 ${tools_dir}/grib_set -s edition=2 $sample_g1 $output > $tempLog 2>&1
 
 # -G option
 sample_g1=$ECCODES_SAMPLES_PATH/GRIB1.tmpl
@@ -124,3 +137,4 @@ ${tools_dir}/grib_set -G -s edition=2 $sample_g1 $temp2
 # Clean up
 rm -f $output
 rm -f $temp1 $temp2
+rm -f $tempLog

tests/grib_destine_mars_keys.sh

@@ -14,7 +14,7 @@ REDIRECT=/dev/null
 
 # ECC-1691: Destination Earth ClimateDT metadata support
 
-label="grib_ecc-1691_test"
+label="grib_destine_mars_keys_test"
 temp_grib_a=temp.$label.a.grib
 temp_grib_b=temp.$label.b.grib
 destine_sample=temp.$label.destine.grib

tests/grib_ecc-1898.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+# (C) Copyright 2005- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# 
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.ctest.sh
+
+label="grib_ecc-1898_test"
+
+tempGrib=temp.$label.grib
+tempOut=temp.${label}.txt
+
+$tools_dir/grib_dump -Da $samp_dir/reduced_gg_pl_32_grib2.tmpl > $tempOut
+grep -q '288-288 g2level level = 1000 \[vertical.level, mars.levelist\]' $tempOut
+
+# For given MARS stream/types, we unalias the mars.levelist key
+$tools_dir/grib_set -s stream=gfas,type=gsd $samp_dir/reduced_gg_pl_32_grib2.tmpl $tempGrib
+
+# Check that the key is unaliased
+$tools_dir/grib_dump -Da $tempGrib > $tempOut
+grep -q '288-288 g2level level = 1000 \[vertical.level\]' $tempOut
+
+$tools_dir/grib_dump -p levelist -Da $tempGrib 2> $tempOut
+grep -q 'levelist: Key/value not found' $tempOut
+
+result=$($tools_dir/grib_get -fp levelist $tempGrib)
+[ "$result" = "not_found" ]
+
+result=$($tools_dir/grib_get -fp mars.levelist $tempGrib)
+[ "$result" = "not_found" ]
+
+# Clean up
+rm -f $tempGrib $tempOut

tests/grib_ecc-806.sh

@@ -17,7 +17,6 @@
 label="grib_ecc-806_test"
 tempGrb=temp.${label}.grb
 tempOut=temp.${label}.txt
-tempErr=temp.${label}.err
 
 # This NCEP grib2 file has the keys
 #   discipline = 0
@@ -36,5 +35,17 @@ res=`${tools_dir}/grib_get -p paramId,shortName,units,name $input`
 res=`${tools_dir}/grib_get -s preferLocalConcepts=1 -p paramId,shortName,units,name $input`
 [ "$res" = "260056 sdwe kg m**-2 Water equivalent of accumulated snow depth (deprecated)" ]
 
+# Test the environment variable too
+export ECCODES_GRIB_PREFER_LOCAL_CONCEPTS=1
+grib_check_key_equals $input preferLocalConceptsEnvVar,preferLocalConcepts '1 1'
+res=`${tools_dir}/grib_get -p paramId,shortName,units,name $input`
+[ "$res" = "260056 sdwe kg m**-2 Water equivalent of accumulated snow depth (deprecated)" ]
+
+export ECCODES_GRIB_PREFER_LOCAL_CONCEPTS=0
+grib_check_key_equals $input preferLocalConceptsEnvVar,preferLocalConcepts '0 0'
+unset ECCODES_GRIB_PREFER_LOCAL_CONCEPTS
+grib_check_key_equals $input preferLocalConceptsEnvVar,preferLocalConcepts '0 0'
+
+
 # Clean up
-rm -f $tempGrb $tempOut $tempErr
+rm -f $tempGrb $tempOut

tests/grib_keys_iter.cc

@@ -17,12 +17,11 @@
 
 int main(int argc, char* argv[])
 {
-    FILE* f         = NULL;
     grib_handle* h = NULL;
     int err = 0;
 
     Assert(argc == 2);
-    f = fopen(argv[1], "rb");
+    FILE* f = fopen(argv[1], "rb");
     Assert(f);
 
     while ((h = grib_handle_new_from_file(0, f, &err)) != NULL) {
@@ -39,6 +38,8 @@ int main(int argc, char* argv[])
             int type = 0;
             GRIB_CHECK(grib_get_native_type(h, name, &type), 0);
             Assert( type > 0 && type < 7 );
+            int ktype = grib_keys_iterator_get_native_type(kiter);
+            Assert(type == ktype);
             const char* type_name = grib_get_type_name(type);
             Assert( !STR_EQUAL(type_name, "unknown") );
             printf("%s = %s (%d)\n", name, type_name, type);
@@ -48,6 +49,16 @@ int main(int argc, char* argv[])
                 size_t vlen = MAX_VAL_LEN;
                 GRIB_CHECK(grib_get_string(h, name, value, &vlen), name);
                 Assert( strlen(value) > 0 );
+                int e = grib_keys_iterator_get_string(kiter, value, &vlen);
+                Assert(!e);
+                Assert( STR_EQUAL(name, value) );
+            }
+            if (STR_EQUAL(name, "editionNumber")) {
+                long lVal = 0;
+                size_t llen = 1;
+                int e = grib_keys_iterator_get_long(kiter, &lVal, &llen);
+                Assert(!e);
+                Assert(lVal == 1 || lVal == 2);
             }
         }
 

tests/grib_uerra.sh

@@ -98,5 +98,16 @@ $grib2_sample $temp1
 grib_check_key_equals $temp1 'marsExpver,mars.expver' '0078 0078'
 
 
+# Stream 'dame'
+${tools_dir}/grib_set -s productionStatusOfProcessedData=10 $grib2_sample $temp1
+grib_check_key_equals $temp1 'mars.time,mars.step' '1200 0'
+
+${tools_dir}/grib_set -s \
+ productionStatusOfProcessedData=10,productDefinitionTemplateNumber=8,outerLoopTypeOfTimeIncrement=1,outerLoopLengthOfTimeRange=21 \
+ $grib2_sample $temp1
+grib_check_key_equals $temp1 'mars.stream' 'dame'
+result=$(${tools_dir}/grib_get -fp mars.time,mars.step $temp1)
+[ "$result" = "not_found not_found" ]
+
 # Clean up
 rm -f $temp1 $temp2 $tempSample

tests/unit_tests.cc

@@ -855,6 +855,12 @@ static void test_grib_get_binary_scale_fact()
     Assert( result == 0 );
 }
 
+static void test_filepool()
+{
+    printf("Running %s ...\n", __func__);
+    grib_file_pool_print("file_pool contents", stdout);
+}
+
 int main(int argc, char** argv)
 {
     printf("Doing unit tests. ecCodes version = %ld\n", grib_get_api_version());
@@ -924,6 +930,7 @@ int main(int argc, char** argv)
     test_grib2_choose_PDTN();
     test_codes_is_feature_enabled();
     test_codes_get_features();
+    test_filepool();
 
     return 0;
 }