From 86d85d60c198dd0366ecfad925653431416966a5 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 10 Oct 2023 11:27:00 +0000 Subject: [PATCH 1/9] Testing: Decode integer key as string --- tests/grib_filter.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/grib_filter.sh b/tests/grib_filter.sh index dee06e373..6a7ebb38f 100755 --- a/tests/grib_filter.sh +++ b/tests/grib_filter.sh @@ -378,6 +378,14 @@ cat >$tempFilt < $tempOut +# Decode an integer key as string +cat >$tempFilt < $tempOut +cat $tempOut +grep "MISSING" $tempOut + # Clean up rm -f $tempGrib $tempFilt $tempOut $tempRef From 5a1f65bed0b88d19796cf0ff585499e6bff032ff Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 10 Oct 2023 17:49:20 +0100 Subject: [PATCH 2/9] Windows: Upgrade to Visual Studio 2022 --- .appveyor.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 4b0a3eaaa..096e0e4c8 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -9,7 +9,7 @@ branches: - develop - master -image: Visual Studio 2015 +image: Visual Studio 2022 environment: CONDA: c:\Miniconda37-x64 @@ -62,8 +62,7 @@ platform: - x64 before_build: - - cmd: call "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 - - cmd: call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_amd64 + - cmd: call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 build_script: - cmd: cd %ECCODES_SRC% From 5b702b03fcd9413087de63dc157d14f283cbe72b Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 10 Oct 2023 18:31:23 +0100 Subject: [PATCH 3/9] Testing --- tests/grib_to_netcdf.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/grib_to_netcdf.sh b/tests/grib_to_netcdf.sh index 9e2151b7c..7a2a6f7ad 100755 --- a/tests/grib_to_netcdf.sh +++ b/tests/grib_to_netcdf.sh @@ -36,19 +36,19 @@ if command -v "ncdump" >/dev/null 2>&1; then NC_DUMPER="ncdump" fi -echo "Test ECC-1041: One parameter with different expvers ..." -# ------------------------------------------------------------ -# This has 5 messages, all 'tp'. Change the first message to have a different expver -input=${data_dir}/tp_ecmwf.grib -${tools_dir}/grib_set -w stepRange=12 -s experimentVersionNumber=0005 $input $tempGrib -${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib -if test "x$NC_DUMPER" != "x"; then - $NC_DUMPER -h $tempNetcdf > $tempText - grep -q "short tp_0005" $tempText - grep -q "short tp_0001" $tempText -fi - if [ $ECCODES_ON_WINDOWS -eq 0 ]; then + echo "Test ECC-1041: One parameter with different expvers ..." + # ------------------------------------------------------------ + # This has 5 messages, all 'tp'. Change the first message to have a different expver + input=${data_dir}/tp_ecmwf.grib + ${tools_dir}/grib_set -w stepRange=12 -s experimentVersionNumber=0005 $input $tempGrib + ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib + if test "x$NC_DUMPER" != "x"; then + $NC_DUMPER -h $tempNetcdf > $tempText + grep -q "short tp_0005" $tempText + grep -q "short tp_0001" $tempText + fi + echo "Test HDF5 decoding ..." # --------------------------- # Note: this is only available in NetCDF-4. So need to check if the command worked with -k3 From 71c091f8f95318f45ad29107c2356b32378c9b3a Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 10 Oct 2023 21:46:57 +0100 Subject: [PATCH 4/9] Testing: grib_to_netcdf --- tests/CMakeLists.txt | 2 +- tests/grib_to_netcdf.sh | 52 ++++++++++++++++++++--------------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 595baae99..a2abbfdc3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -425,7 +425,7 @@ if( HAVE_BUILD_TOOLS ) ecbuild_add_test( TARGET eccodes_t_grib_to_netcdf TYPE SCRIPT - CONDITION HAVE_NETCDF AND ENABLE_EXTRA_TESTS + CONDITION HAVE_NETCDF AND ENABLE_EXTRA_TESTS AND NOT ECCODES_ON_WINDOWS COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_to_netcdf.sh TEST_DEPENDS eccodes_download_gribs ) diff --git a/tests/grib_to_netcdf.sh b/tests/grib_to_netcdf.sh index 7a2a6f7ad..e83f9a620 100755 --- a/tests/grib_to_netcdf.sh +++ b/tests/grib_to_netcdf.sh @@ -36,35 +36,35 @@ if command -v "ncdump" >/dev/null 2>&1; then NC_DUMPER="ncdump" fi -if [ $ECCODES_ON_WINDOWS -eq 0 ]; then - echo "Test ECC-1041: One parameter with different expvers ..." - # ------------------------------------------------------------ - # This has 5 messages, all 'tp'. Change the first message to have a different expver - input=${data_dir}/tp_ecmwf.grib - ${tools_dir}/grib_set -w stepRange=12 -s experimentVersionNumber=0005 $input $tempGrib - ${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib - if test "x$NC_DUMPER" != "x"; then - $NC_DUMPER -h $tempNetcdf > $tempText - grep -q "short tp_0005" $tempText - grep -q "short tp_0001" $tempText - fi - echo "Test HDF5 decoding ..." - # --------------------------- - # Note: this is only available in NetCDF-4. So need to check if the command worked with -k3 - input=${data_dir}/sample.grib2 - set +e - ${tools_dir}/grib_to_netcdf -k3 -o $tempNetcdf $input 2>/dev/null - stat=$? - set -e - if [ $stat -eq 0 ]; then - have_netcdf4=1 - ${tools_dir}/grib_dump -TA -O $tempNetcdf - res=`${tools_dir}/grib_get -TA -p identifier $tempNetcdf` - [ "$res" = "HDF5" ] - fi +echo "Test ECC-1041: One parameter with different expvers ..." +# ------------------------------------------------------------ +# This has 5 messages, all 'tp'. Change the first message to have a different expver +input=${data_dir}/tp_ecmwf.grib +${tools_dir}/grib_set -w stepRange=12 -s experimentVersionNumber=0005 $input $tempGrib +${tools_dir}/grib_to_netcdf -o $tempNetcdf $tempGrib +if test "x$NC_DUMPER" != "x"; then + $NC_DUMPER -h $tempNetcdf > $tempText + grep -q "short tp_0005" $tempText + grep -q "short tp_0001" $tempText fi +echo "Test HDF5 decoding ..." +# --------------------------- +# Note: this is only available in NetCDF-4. So need to check if the command worked with -k3 +input=${data_dir}/sample.grib2 +set +e +${tools_dir}/grib_to_netcdf -k3 -o $tempNetcdf $input 2>/dev/null +stat=$? +set -e +if [ $stat -eq 0 ]; then + have_netcdf4=1 + ${tools_dir}/grib_dump -TA -O $tempNetcdf + res=`${tools_dir}/grib_get -TA -p identifier $tempNetcdf` + [ "$res" = "HDF5" ] +fi + + grib_files="\ regular_latlon_surface.grib2 \ regular_latlon_surface.grib1 \ From 43ae680c47758ef8b51c6acd3a6677eed234b2c4 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 12 Oct 2023 16:49:25 +0100 Subject: [PATCH 5/9] Error messages --- src/grib_expression.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/grib_expression.cc b/src/grib_expression.cc index 9a443c67c..c3cd36305 100644 --- a/src/grib_expression.cc +++ b/src/grib_expression.cc @@ -19,7 +19,7 @@ int grib_expression_native_type(grib_handle* h, grib_expression* g) c = c->super ? *(c->super) : NULL; } if (g->cclass) - grib_context_log(h->context, GRIB_LOG_ERROR, "No native_type() in %s\n", g->cclass->name); + grib_context_log(h->context, GRIB_LOG_ERROR, "No native_type() in %s", g->cclass->name); Assert(1 == 0); return 0; } @@ -55,7 +55,7 @@ const char* grib_expression_evaluate_string(grib_handle* h, grib_expression* g, c = c->super ? *(c->super) : NULL; } if (g->cclass) - grib_context_log(h->context, GRIB_LOG_ERROR, "No evaluate_string() in %s\n", g->cclass->name); + grib_context_log(h->context, GRIB_LOG_ERROR, "No evaluate_string() in %s", g->cclass->name); *err = GRIB_INVALID_TYPE; return 0; From 7101651b466ced8f07412e421a0b8c40aa348463 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 12 Oct 2023 17:49:21 +0000 Subject: [PATCH 6/9] ECC-1703 --- ...ta_g1second_order_constant_width_packing.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/grib_accessor_class_data_g1second_order_constant_width_packing.cc b/src/grib_accessor_class_data_g1second_order_constant_width_packing.cc index eba3ecc24..c3f74204d 100644 --- a/src/grib_accessor_class_data_g1second_order_constant_width_packing.cc +++ b/src/grib_accessor_class_data_g1second_order_constant_width_packing.cc @@ -258,14 +258,22 @@ static int unpack_double(grib_accessor* a, double* values, size_t* len) return ret; secondaryBitmap = (long*)grib_context_malloc_clear(a->context, sizeof(long) * numberOfSecondOrderPackedValues); + if (!secondaryBitmap) + return GRIB_OUT_OF_MEMORY; + grib_decode_long_array(buf, &pos, 1, numberOfSecondOrderPackedValues, secondaryBitmap); pos = 8 * ((pos + 7) / 8); firstOrderValues = (long*)grib_context_malloc_clear(a->context, sizeof(long) * numberOfGroups); + if (!firstOrderValues) + return GRIB_OUT_OF_MEMORY; + grib_decode_long_array(buf, &pos, widthOfFirstOrderValues, numberOfGroups, firstOrderValues); pos = 8 * ((pos + 7) / 8); X = (long*)grib_context_malloc_clear(a->context, sizeof(long) * numberOfSecondOrderPackedValues); + if (!X) + return GRIB_OUT_OF_MEMORY; if (groupWidth > 0) { grib_decode_long_array(buf, &pos, groupWidth, numberOfSecondOrderPackedValues, X); @@ -273,7 +281,10 @@ static int unpack_double(grib_accessor* a, double* values, size_t* len) i = -1; while (n < numberOfSecondOrderPackedValues) { i += secondaryBitmap[n]; - X[n] = firstOrderValues[i] + X[n]; + long fovi = 0; + if ( i >=0 && i < numberOfGroups ) + fovi = firstOrderValues[i]; + X[n] = fovi + X[n]; n++; } } @@ -282,7 +293,10 @@ static int unpack_double(grib_accessor* a, double* values, size_t* len) i = -1; while (n < numberOfSecondOrderPackedValues) { i += secondaryBitmap[n]; - X[n] = firstOrderValues[i]; + long fovi = 0; + if ( i >=0 && i < numberOfGroups ) + fovi = firstOrderValues[i]; + X[n] = fovi; n++; } } From fa9f8ea763ec304d444d61df0bb10fb7de3a6731 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 12 Oct 2023 17:50:18 +0000 Subject: [PATCH 7/9] ECC-1703: GRIB: AddressSanitizer: heap-buffer-overflow --- ..._accessor_class_data_g1second_order_constant_width_packing.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/grib_accessor_class_data_g1second_order_constant_width_packing.cc b/src/grib_accessor_class_data_g1second_order_constant_width_packing.cc index c3f74204d..079e91829 100644 --- a/src/grib_accessor_class_data_g1second_order_constant_width_packing.cc +++ b/src/grib_accessor_class_data_g1second_order_constant_width_packing.cc @@ -282,6 +282,7 @@ static int unpack_double(grib_accessor* a, double* values, size_t* len) while (n < numberOfSecondOrderPackedValues) { i += secondaryBitmap[n]; long fovi = 0; + // ECC-1703 if ( i >=0 && i < numberOfGroups ) fovi = firstOrderValues[i]; X[n] = fovi + X[n]; From db19ba940592464b017732e6b7d909e11c5a4ad7 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 13 Oct 2023 12:17:50 +0100 Subject: [PATCH 8/9] ECC-1704: GRIB: Implement unpack_string for validityTime --- src/grib_accessor_class_validity_time.cc | 25 +++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/grib_accessor_class_validity_time.cc b/src/grib_accessor_class_validity_time.cc index 36af6cb98..e07e71e15 100644 --- a/src/grib_accessor_class_validity_time.cc +++ b/src/grib_accessor_class_validity_time.cc @@ -17,6 +17,7 @@ CLASS = accessor SUPER = grib_accessor_class_long IMPLEMENTS = unpack_long + IMPLEMENTS = unpack_string IMPLEMENTS = init;dump MEMBERS=const char* date MEMBERS=const char* time @@ -39,6 +40,7 @@ or edit "accessor.class" and rerun ./make_class.pl */ static int unpack_long(grib_accessor*, long* val, size_t* len); +static int unpack_string(grib_accessor*, char*, size_t* len); static void dump(grib_accessor*, grib_dumper*); static void init(grib_accessor*, const long, grib_arguments*); @@ -84,7 +86,7 @@ static grib_accessor_class _grib_accessor_class_validity_time = { 0, /* unpack_double */ 0, /* unpack_float */ 0, /* pack_string */ - 0, /* unpack_string */ + &unpack_string, /* unpack_string */ 0, /* pack_string_array */ 0, /* unpack_string_array */ 0, /* pack_bytes */ @@ -227,3 +229,24 @@ static int unpack_long(grib_accessor* a, long* val, size_t* len) return GRIB_SUCCESS; } + +static int unpack_string(grib_accessor* a, char* val, size_t* len) +{ + int err = 0; + long v = 0; + size_t lsize = 1; + + err = unpack_long(a, &v, &lsize); + if (err) return err; + + if (*len < 5) { + grib_context_log(a->context, GRIB_LOG_ERROR, "Key %s (unpack_string): Buffer too small", a->name); + *len = 5; + return GRIB_BUFFER_TOO_SMALL; + } + + snprintf(val, 64, "%04ld", v); + + len[0] = 5; + return GRIB_SUCCESS; +} From 993cc42e4c032299ffd627c6973550304567c73f Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 13 Oct 2023 12:28:32 +0100 Subject: [PATCH 9/9] ECC-1704: Test --- tests/grib_step.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/grib_step.sh b/tests/grib_step.sh index 2b1ca37b6..0a5cf7425 100755 --- a/tests/grib_step.sh +++ b/tests/grib_step.sh @@ -17,6 +17,9 @@ tempGrb=${data_dir}/temp.$label.out.grib templog=${data_dir}/temp.$label.log rm -f $templog $tempGrb +grib1_sample=$ECCODES_SAMPLES_PATH/GRIB1.tmpl +grib2_sample=$ECCODES_SAMPLES_PATH/GRIB2.tmpl + for i in 0 10 do for s in 0 1200 600 6000 @@ -83,8 +86,6 @@ hourEnd=$1; dayEnd=$2 # ECC-134 case-sensitivity # -------------------------- -grib1_sample=$ECCODES_SAMPLES_PATH/GRIB1.tmpl -grib2_sample=$ECCODES_SAMPLES_PATH/GRIB2.tmpl temp=temp.step.$$.grib # M is for Month (code 3) ${tools_dir}/grib_set -s indicatorOfUnitOfTimeRange=M $grib1_sample $temp @@ -147,6 +148,14 @@ grib_check_key_equals $input "dataDate,dataTime,step" "20061205 1200 6" grib_check_key_equals $input "validityDate,validityTime" "20061205 1800" grib_check_key_equals $input "validityDateTime:s" "20061205 001800" +# ECC-1704: Key validityTime as string +# ----------------------------------------------- +result=$( ${tools_dir}/grib_get -p validityTime:s -s dataTime=0000 $grib2_sample ) +[ "$result" = "0000" ] +input=$data_dir/simple.grib +grib_check_key_equals $input "validityTime:s" "0600" + + # Key julianDay # ----------------------------------------------- input=${data_dir}/sample.grib2 @@ -157,7 +166,6 @@ grib_check_key_equals $temp day 7 # Seconds (ignored) # ----------------------------------------------- -grib2_sample=$ECCODES_SAMPLES_PATH/GRIB2.tmpl ${tools_dir}/grib_ls -s second=9 -n time $grib2_sample 2>$templog # Something should have been written to stderr [ -s $templog ]