Merge branch 'feature/ECC-1620_sub-hourly' of github.com:ecmwf/eccodes into feature/ECC-1620_sub-hourly

.appveyor.yml

@@ -9,7 +9,7 @@ branches:
     - develop
     - master
 
-image: Visual Studio 2015
+image: Visual Studio 2022
 
 environment:
   CONDA: c:\Miniconda37-x64
@@ -62,8 +62,7 @@ platform:
   - x64
 
 before_build:
-  - cmd: call "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64
+  - cmd: call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsall.bat" x86_amd64
-  - cmd: call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_amd64
 
 build_script:
   - cmd: cd %ECCODES_SRC%

src/grib_accessor_class_data_g1second_order_constant_width_packing.cc

@@ -258,14 +258,22 @@ static int unpack_double(grib_accessor* a, double* values, size_t* len)
         return ret;
 
     secondaryBitmap = (long*)grib_context_malloc_clear(a->context, sizeof(long) * numberOfSecondOrderPackedValues);
+    if (!secondaryBitmap)
+        return GRIB_OUT_OF_MEMORY;
+
     grib_decode_long_array(buf, &pos, 1, numberOfSecondOrderPackedValues, secondaryBitmap);
     pos = 8 * ((pos + 7) / 8);
 
     firstOrderValues = (long*)grib_context_malloc_clear(a->context, sizeof(long) * numberOfGroups);
+    if (!firstOrderValues)
+        return GRIB_OUT_OF_MEMORY;
+
     grib_decode_long_array(buf, &pos, widthOfFirstOrderValues, numberOfGroups, firstOrderValues);
     pos = 8 * ((pos + 7) / 8);
 
     X = (long*)grib_context_malloc_clear(a->context, sizeof(long) * numberOfSecondOrderPackedValues);
+    if (!X)
+        return GRIB_OUT_OF_MEMORY;
 
     if (groupWidth > 0) {
         grib_decode_long_array(buf, &pos, groupWidth, numberOfSecondOrderPackedValues, X);
@@ -273,7 +281,11 @@ static int unpack_double(grib_accessor* a, double* values, size_t* len)
         i = -1;
         while (n < numberOfSecondOrderPackedValues) {
             i += secondaryBitmap[n];
-            X[n] = firstOrderValues[i] + X[n];
+            long fovi = 0;
+            // ECC-1703
+            if ( i >=0 && i < numberOfGroups )
+                fovi = firstOrderValues[i];
+            X[n] = fovi + X[n];
             n++;
         }
     }
@@ -282,7 +294,10 @@ static int unpack_double(grib_accessor* a, double* values, size_t* len)
         i = -1;
         while (n < numberOfSecondOrderPackedValues) {
             i += secondaryBitmap[n];
-            X[n] = firstOrderValues[i];
+            long fovi = 0;
+            if ( i >=0 && i < numberOfGroups )
+                fovi = firstOrderValues[i];
+            X[n] = fovi;
             n++;
         }
     }

src/grib_accessor_class_validity_time.cc

@@ -17,6 +17,7 @@
    CLASS      = accessor
    SUPER      = grib_accessor_class_long
    IMPLEMENTS = unpack_long
+   IMPLEMENTS = unpack_string
    IMPLEMENTS = init;dump
    MEMBERS=const char* date
    MEMBERS=const char* time
@@ -39,6 +40,7 @@ or edit "accessor.class" and rerun ./make_class.pl
 */
 
 static int unpack_long(grib_accessor*, long* val, size_t* len);
+static int unpack_string(grib_accessor*, char*, size_t* len);
 static void dump(grib_accessor*, grib_dumper*);
 static void init(grib_accessor*, const long, grib_arguments*);
 
@@ -84,7 +86,7 @@ static grib_accessor_class _grib_accessor_class_validity_time = {
     0,              /* unpack_double */
     0,               /* unpack_float */
     0,                /* pack_string */
-    0,              /* unpack_string */
+    &unpack_string,              /* unpack_string */
     0,          /* pack_string_array */
     0,        /* unpack_string_array */
     0,                 /* pack_bytes */
@@ -227,3 +229,24 @@ static int unpack_long(grib_accessor* a, long* val, size_t* len)
 
     return GRIB_SUCCESS;
 }
+
+static int unpack_string(grib_accessor* a, char* val, size_t* len)
+{
+    int err = 0;
+    long v  = 0;
+    size_t lsize = 1;
+
+    err = unpack_long(a, &v, &lsize);
+    if (err) return err;
+
+    if (*len < 5) {
+        grib_context_log(a->context, GRIB_LOG_ERROR, "Key %s (unpack_string): Buffer too small", a->name);
+        *len = 5;
+        return GRIB_BUFFER_TOO_SMALL;
+    }
+
+    snprintf(val, 64, "%04ld", v);
+
+    len[0] = 5;
+    return GRIB_SUCCESS;
+}

src/grib_expression.cc

@@ -19,7 +19,7 @@ int grib_expression_native_type(grib_handle* h, grib_expression* g)
         c = c->super ? *(c->super) : NULL;
     }
     if (g->cclass)
-        grib_context_log(h->context, GRIB_LOG_ERROR, "No native_type() in %s\n", g->cclass->name);
+        grib_context_log(h->context, GRIB_LOG_ERROR, "No native_type() in %s", g->cclass->name);
     Assert(1 == 0);
     return 0;
 }
@@ -55,7 +55,7 @@ const char* grib_expression_evaluate_string(grib_handle* h, grib_expression* g,
         c = c->super ? *(c->super) : NULL;
     }
     if (g->cclass)
-        grib_context_log(h->context, GRIB_LOG_ERROR, "No evaluate_string() in %s\n", g->cclass->name);
+        grib_context_log(h->context, GRIB_LOG_ERROR, "No evaluate_string() in %s", g->cclass->name);
     *err = GRIB_INVALID_TYPE;
 
     return 0;

tests/CMakeLists.txt

@@ -426,7 +426,7 @@ if( HAVE_BUILD_TOOLS )
 
     ecbuild_add_test( TARGET eccodes_t_grib_to_netcdf
                       TYPE SCRIPT
-                      CONDITION HAVE_NETCDF AND ENABLE_EXTRA_TESTS
+                      CONDITION HAVE_NETCDF AND ENABLE_EXTRA_TESTS AND NOT ECCODES_ON_WINDOWS
                       COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_to_netcdf.sh
                       TEST_DEPENDS eccodes_download_gribs )
 

tests/grib_filter.sh

@@ -378,6 +378,14 @@ cat >$tempFilt <<EOF
 EOF
 ${tools_dir}/grib_filter $tempFilt $ECCODES_SAMPLES_PATH/GRIB2.tmpl #> $tempOut
 
+# Decode an integer key as string
+cat >$tempFilt <<EOF
+ print "[scaleFactorOfSecondFixedSurface:s]";
+EOF
+${tools_dir}/grib_filter $tempFilt $ECCODES_SAMPLES_PATH/GRIB2.tmpl > $tempOut
+cat $tempOut
+grep "MISSING" $tempOut
+
 
 # Clean up
 rm -f $tempGrib $tempFilt $tempOut $tempRef

tests/grib_step.sh

@@ -17,6 +17,9 @@ tempGrb=${data_dir}/temp.$label.out.grib
 templog=${data_dir}/temp.$label.log
 rm -f $templog $tempGrb
 
+grib1_sample=$ECCODES_SAMPLES_PATH/GRIB1.tmpl
+grib2_sample=$ECCODES_SAMPLES_PATH/GRIB2.tmpl
+
 for i in 0 10
 do
   for s in 0 1200 600 6000
@@ -83,8 +86,6 @@ hourEnd=$1; dayEnd=$2
 
 # ECC-134 case-sensitivity
 # --------------------------
-grib1_sample=$ECCODES_SAMPLES_PATH/GRIB1.tmpl
-grib2_sample=$ECCODES_SAMPLES_PATH/GRIB2.tmpl
 temp=temp.step.$$.grib
 # M is for Month (code 3)
 ${tools_dir}/grib_set -s indicatorOfUnitOfTimeRange=M $grib1_sample $temp
@@ -147,6 +148,14 @@ grib_check_key_equals $input "dataDate,dataTime,step" "20061205 1200 6"
 grib_check_key_equals $input "validityDate,validityTime" "20061205 1800"
 grib_check_key_equals $input "validityDateTime:s" "20061205 001800"
 
+# ECC-1704: Key validityTime as string
+# -----------------------------------------------
+result=$( ${tools_dir}/grib_get -p validityTime:s -s dataTime=0000 $grib2_sample )
+[ "$result" = "0000" ]
+input=$data_dir/simple.grib
+grib_check_key_equals $input "validityTime:s" "0600"
+
+
 # Key julianDay
 # -----------------------------------------------
 input=${data_dir}/sample.grib2
@@ -157,7 +166,6 @@ grib_check_key_equals $temp day  7
 
 # Seconds (ignored)
 # -----------------------------------------------
-grib2_sample=$ECCODES_SAMPLES_PATH/GRIB2.tmpl
 ${tools_dir}/grib_ls -s second=9 -n time $grib2_sample 2>$templog
 # Something should have been written to stderr
 [ -s $templog ]

tests/grib_to_netcdf.sh

@@ -36,6 +36,7 @@ if command -v "ncdump" >/dev/null 2>&1; then
     NC_DUMPER="ncdump"
 fi
 
+
 echo "Test ECC-1041: One parameter with different expvers ..."
 # ------------------------------------------------------------
 # This has 5 messages, all 'tp'. Change the first message to have a different expver
@@ -48,23 +49,22 @@ if test "x$NC_DUMPER" != "x"; then
     grep -q "short tp_0001" $tempText
 fi
 
-if [ $ECCODES_ON_WINDOWS -eq 0 ]; then
+echo "Test HDF5 decoding ..."
-    echo "Test HDF5 decoding ..."
+# ---------------------------
-    # ---------------------------
+# Note: this is only available in NetCDF-4. So need to check if the command worked with -k3
-    # Note: this is only available in NetCDF-4. So need to check if the command worked with -k3
+input=${data_dir}/sample.grib2
-    input=${data_dir}/sample.grib2
+set +e
-    set +e
+${tools_dir}/grib_to_netcdf -k3 -o $tempNetcdf $input 2>/dev/null
-    ${tools_dir}/grib_to_netcdf -k3 -o $tempNetcdf $input 2>/dev/null
+stat=$?
-    stat=$?
+set -e
-    set -e
+if [ $stat -eq 0 ]; then
-    if [ $stat -eq 0 ]; then
+    have_netcdf4=1
-        have_netcdf4=1
+    ${tools_dir}/grib_dump -TA -O $tempNetcdf
-        ${tools_dir}/grib_dump -TA -O $tempNetcdf
+    res=`${tools_dir}/grib_get -TA -p identifier $tempNetcdf`
-        res=`${tools_dir}/grib_get -TA -p identifier $tempNetcdf`
+    [ "$res" = "HDF5" ]
-        [ "$res" = "HDF5" ]
-    fi
 fi
 
+
 grib_files="\
  regular_latlon_surface.grib2 \
  regular_latlon_surface.grib1 \