From 0f098d5df200b005c2dcddef982b7f4f2f69129b Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 15 Jul 2022 18:18:50 +0100 Subject: [PATCH 01/13] ECC-1427: GRIB Performance: Improve decoding speed of CCSDS packing --- src/grib_accessor_class_data_ccsds_packing.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/grib_accessor_class_data_ccsds_packing.c b/src/grib_accessor_class_data_ccsds_packing.c index 1b145ca0f..d9aba9064 100644 --- a/src/grib_accessor_class_data_ccsds_packing.c +++ b/src/grib_accessor_class_data_ccsds_packing.c @@ -311,10 +311,13 @@ static int unpack_double(grib_accessor* a, double* val, size_t* len) /* printf("bscale=%g dscale=%g reference_value=%g\n",bscale,dscale,reference_value); */ pos = 0; p = decoded; +#if 0 for (i = 0; i < n_vals; i++) { val[i] = (double)(((grib_decode_unsigned_long(p, &pos, bits8) * bscale) + reference_value) * dscale); } - /*-------------------------------------------*/ +#endif + /* ECC-1427: Performance improvement */ + grib_decode_double_array(decoded, &pos, bits8 , reference_value, bscale, dscale, n_vals, val); *len = n_vals; cleanup: From ed56dd3efec0fcbd7f18bc693d6b4a59c66596ad Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 15 Jul 2022 18:22:52 +0100 Subject: [PATCH 02/13] ECC-1427: clean up --- src/grib_accessor_class_data_ccsds_packing.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/grib_accessor_class_data_ccsds_packing.c b/src/grib_accessor_class_data_ccsds_packing.c index d9aba9064..ff123d834 100644 --- a/src/grib_accessor_class_data_ccsds_packing.c +++ b/src/grib_accessor_class_data_ccsds_packing.c @@ -224,7 +224,6 @@ static int unpack_double(grib_accessor* a, double* val, size_t* len) size_t n_vals = 0; size_t size = 0; unsigned char* decoded = NULL; - unsigned char* p = NULL; long pos = 0; long nn = 0; @@ -310,8 +309,8 @@ static int unpack_double(grib_accessor* a, double* val, size_t* len) /* printf("bscale=%g dscale=%g reference_value=%g\n",bscale,dscale,reference_value); */ pos = 0; - p = decoded; #if 0 + p = decoded; for (i = 0; i < n_vals; i++) { val[i] = (double)(((grib_decode_unsigned_long(p, &pos, bits8) * bscale) + reference_value) * dscale); } From ab287f9e0b6d248e6b914782656b0c4ca2781afb Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 18 Jul 2022 11:17:03 +0100 Subject: [PATCH 03/13] ECC-1427: clean up --- src/grib_accessor_class_data_ccsds_packing.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/grib_accessor_class_data_ccsds_packing.c b/src/grib_accessor_class_data_ccsds_packing.c index ff123d834..e761430e8 100644 --- a/src/grib_accessor_class_data_ccsds_packing.c +++ b/src/grib_accessor_class_data_ccsds_packing.c @@ -224,6 +224,7 @@ static int unpack_double(grib_accessor* a, double* val, size_t* len) size_t n_vals = 0; size_t size = 0; unsigned char* decoded = NULL; + /*unsigned char* p = NULL;*/ long pos = 0; long nn = 0; @@ -310,7 +311,7 @@ static int unpack_double(grib_accessor* a, double* val, size_t* len) /* printf("bscale=%g dscale=%g reference_value=%g\n",bscale,dscale,reference_value); */ pos = 0; #if 0 - p = decoded; + p = decoded; for (i = 0; i < n_vals; i++) { val[i] = (double)(((grib_decode_unsigned_long(p, &pos, bits8) * bscale) + reference_value) * dscale); } From 4af92cba8381bc9f76817cc4c9e5673f65cadba2 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 19 Jul 2022 12:45:42 +0100 Subject: [PATCH 04/13] Performance: Exclude METAR definition files from MEMFS --- memfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memfs.py b/memfs.py index c12a59dff..b63123f66 100755 --- a/memfs.py +++ b/memfs.py @@ -66,7 +66,7 @@ print("MEMFS: Starting") # Exclude experimental features e.g. GRIB3 and TAF # The BUFR codetables is not used in the engine -EXCLUDED = ["grib3", "codetables", "taf", "stations", "grib1_mlgrib2_ieee32"] +EXCLUDED = ["grib3", "codetables", "taf", "metar", "stations", "grib1_mlgrib2_ieee32"] EXCLUDE = { None: [], From 18339324ed03b333fc574d205ccac904c8978e6c Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 19 Jul 2022 16:55:46 +0100 Subject: [PATCH 05/13] ECC-1407: grib_util_set_spec --- src/grib_util.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/grib_util.c b/src/grib_util.c index 45500b599..414892322 100644 --- a/src/grib_util.c +++ b/src/grib_util.c @@ -1477,8 +1477,17 @@ grib_handle* grib_util_set_spec2(grib_handle* h, switch (packing_spec->accuracy) { case GRIB_UTIL_ACCURACY_SAME_BITS_PER_VALUES_AS_INPUT: { long bitsPerValue = 0; - Assert(grib_get_long(h, "bitsPerValue", &bitsPerValue) == 0); - SET_LONG_VALUE("bitsPerValue", bitsPerValue); + if ((packing_spec->packing_type == GRIB_UTIL_PACKING_TYPE_GRID_SIMPLE || + packing_spec->packing_type == GRIB_UTIL_PACKING_TYPE_CCSDS) && + strcmp(input_packing_type, "grid_ieee")==0) + { + SET_LONG_VALUE("bitsPerValue", 32); + } + else + { + Assert(grib_get_long(h, "bitsPerValue", &bitsPerValue) == 0); + SET_LONG_VALUE("bitsPerValue", bitsPerValue); + } } break; case GRIB_UTIL_ACCURACY_USE_PROVIDED_BITS_PER_VALUES: From 8dd44dfd530f4d86201e08a1190d2959cf2ae6a7 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 21 Jul 2022 12:58:09 +0100 Subject: [PATCH 06/13] Scripts: Allow multiple parameter IDs to be passed in --- definitions/create_legacy_def.sh | 59 ++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/definitions/create_legacy_def.sh b/definitions/create_legacy_def.sh index 291862b20..23300813c 100755 --- a/definitions/create_legacy_def.sh +++ b/definitions/create_legacy_def.sh @@ -2,10 +2,10 @@ set -eu # Usage: -# create_legacy_def.sh $paramId +# create_legacy_def.sh $paramId $paramId... # # This script will insert the local ECMWF GRIB2 representation -# for that paramId into the files: +# for each paramId into the files: # definitions/grib2/localConcepts/ecmf/paramId.legacy.def # definitions/grib2/localConcepts/ecmf/shortName.legacy.def # etc @@ -22,7 +22,13 @@ set -eu # The parameterNumber = $paramId - parameterCategory*1000 # -pid=$1 +if [ $# -eq 0 ]; then + echo "Usage: $0 pid1 pid2 ..." 2>&1 + exit 1 +fi + +# Input list of paramId values +pids="$@" sample2=samples/GRIB2.tmpl temp=$TMPDIR/temp.create_legacy_def.grib @@ -49,23 +55,38 @@ output_def() fi } -grib_set -s paramId=$pid $sample2 $temp -name=$(grib_get -p name $temp) -shortName=$(grib_get -p shortName $temp) -units=$(grib_get -p units $temp) -cfName=$(grib_get -p cfName $temp) -cfVarName=$(grib_get -p cfVarName $temp) +count=0 +for pid in $pids; do + echo "Doing $pid..." + set +e + grib_set -s paramId=$pid $sample2 $temp 2>/dev/null + status=$? + set -e + if [ $status -ne 0 ]; then + grib_set -s stepType=accum,paramId=$pid $sample2 $temp + fi + name=$(grib_get -p name $temp) + shortName=$(grib_get -p shortName $temp) + units=$(grib_get -p units $temp) + cfName=$(grib_get -p cfName $temp) + cfVarName=$(grib_get -p cfVarName $temp) + count=$((count+1)) -dis=192 -cat=$((pid/1000)) -num=$((pid - cat*1000)) + dis=192 + cat=$((pid/1000)) + num=$((pid - cat*1000)) -output_def "$name" "$pid" $dis $cat $num >> $defs/paramId.legacy.def -output_def "$name" "$name" $dis $cat $num >> $defs/name.legacy.def -output_def "$name" "$shortName" $dis $cat $num >> $defs/shortName.legacy.def -output_def "$name" "$units" $dis $cat $num >> $defs/units.legacy.def -output_def "$name" "$cfVarName" $dis $cat $num >> $defs/cfVarName.legacy.def -output_def "$name" "$cfName" $dis $cat $num >> $defs/cfName.legacy.def + output_def "$name" "$pid" $dis $cat $num >> $defs/paramId.legacy.def + output_def "$name" "$name" $dis $cat $num >> $defs/name.legacy.def + output_def "$name" "$shortName" $dis $cat $num >> $defs/shortName.legacy.def + output_def "$name" "$units" $dis $cat $num >> $defs/units.legacy.def + output_def "$name" "$cfVarName" $dis $cat $num >> $defs/cfVarName.legacy.def + output_def "$name" "$cfName" $dis $cat $num >> $defs/cfName.legacy.def -echo "Files updated. Check directory $defs" +done + +echo "Number of legacy parameters added: $count" +if [ $count -gt 0 ]; then + echo "Files updated. Check directory $defs" +fi rm -f $temp From b7724edc271b7b9622226444cd50606d717777e7 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 21 Jul 2022 13:38:07 +0100 Subject: [PATCH 07/13] Tools: man page --- tools/grib_options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/grib_options.c b/tools/grib_options.c index a76ff7758..ee546cf3f 100644 --- a/tools/grib_options.c +++ b/tools/grib_options.c @@ -105,7 +105,7 @@ static grib_options_help grib_options_help_list[] = { { "T:", "T | B | M | A", "Message type. T->GTS, B->BUFR, M->METAR (Experimental), A->Any (Experimental).\n\t\t\t\tThe input file is interpreted according to the message type.\n" }, { "V", 0, "Version.\n" }, { "W:", "width", "\n\t\tMinimum width of each column in output. Default is 10.\n" }, - { "X:", "offset", "\n\t\tInput file offset in bytes. Processing of the input file will start from \"offset\".\n" }, + { "X:", "offset", "\n\t\tInput file offset in bytes. Processing of the input file will start from the given offset.\n" }, { "x", 0, "Fast parsing option, only headers are loaded.\n" }, { "k:", "key1,key2,...", "\n\t\tSpecify a list of keys to index on. By default the input files are indexed on the MARS keys." From 78d3d066bd18af59aac6c1f00c8e9fac68bf829b Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 22 Jul 2022 17:57:02 +0100 Subject: [PATCH 08/13] Testing: Rename labels --- tests/bufr_change_edition.sh | 2 +- tests/bufr_indexing.sh | 2 +- tests/codes_split_file.sh | 2 +- tests/grib1to3.sh | 2 +- tests/grib2_templates.sh | 2 +- tests/grib2_version.sh | 2 +- tests/grib2to3.sh | 2 +- tests/grib3_templates.sh | 2 +- tests/grib_change_scanning.sh | 2 +- tests/grib_check_gaussian_grids.sh | 2 +- tests/grib_complex.sh | 2 +- tests/grib_data_quality_checks.sh | 2 +- tests/grib_dump_debug.sh | 2 +- tests/grib_dump_json.sh | 2 +- tests/grib_efas.sh | 2 +- tests/grib_g1fcperiod.sh | 2 +- tests/grib_g1monthlydate.sh | 2 +- tests/grib_grid_unstructured.sh | 2 +- tests/grib_indexing.sh | 2 +- tests/grib_lam_bf.sh | 2 +- tests/grib_lam_gp.sh | 2 +- tests/grib_levtype.sh | 2 +- tests/grib_mars_keys.sh | 2 +- tests/grib_packing_order.sh | 2 +- tests/grib_png.sh | 2 +- tests/grib_tigge_conversions1.sh | 2 +- tests/grib_tigge_conversions2.sh | 2 +- 27 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/bufr_change_edition.sh b/tests/bufr_change_edition.sh index 04feb7d72..98228556a 100755 --- a/tests/bufr_change_edition.sh +++ b/tests/bufr_change_edition.sh @@ -13,7 +13,7 @@ cd ${data_dir}/bufr # Define a common label for all the tmp files -label="bufr_change_edition" +label="bufr_change_edition_test" fBufrTmp=${label}".bufr.tmp" diff --git a/tests/bufr_indexing.sh b/tests/bufr_indexing.sh index da12caf12..a3030d46c 100755 --- a/tests/bufr_indexing.sh +++ b/tests/bufr_indexing.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="bufr_indexing" +label="bufr_indexing_test" tempIndex=temp.$label.$$.idx tempOut=temp.$label.$$.out tempRef=temp.$label.$$.ref diff --git a/tests/codes_split_file.sh b/tests/codes_split_file.sh index e9209cdf3..88021b4c2 100755 --- a/tests/codes_split_file.sh +++ b/tests/codes_split_file.sh @@ -11,7 +11,7 @@ . ./include.ctest.sh pwd # Define a common label for all the tmp files -label="codes_split_file" +label="codes_split_file_test" temp=$label.temp.grib # Do all the work in a temporary directory diff --git a/tests/grib1to3.sh b/tests/grib1to3.sh index de7bab965..9b6dc40d4 100755 --- a/tests/grib1to3.sh +++ b/tests/grib1to3.sh @@ -12,7 +12,7 @@ REDIRECT=/dev/null -label="grib1to3" +label="grib1to3_test" temp=temp.$label.grib3 sample_g1=$ECCODES_SAMPLES_PATH/GRIB1.tmpl ${tools_dir}/grib_set -s editionNumber=3 $sample_g1 $temp diff --git a/tests/grib2_templates.sh b/tests/grib2_templates.sh index 5d4f263d1..c1e062b20 100755 --- a/tests/grib2_templates.sh +++ b/tests/grib2_templates.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh set -u -label="grib2_templates" +label="grib2_templates_test" temp1=temp1.$label.grib2 temp2=temp2.$label.grib2 diff --git a/tests/grib2_version.sh b/tests/grib2_version.sh index 14e219a30..4f1661836 100755 --- a/tests/grib2_version.sh +++ b/tests/grib2_version.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib2_version" +label="grib2_version_test" if [ ! -d "$ECCODES_DEFINITION_PATH" ]; then echo "Test $0 disabled. No definitions directory" exit 0 diff --git a/tests/grib2to3.sh b/tests/grib2to3.sh index ece486fdc..ae4a4d339 100755 --- a/tests/grib2to3.sh +++ b/tests/grib2to3.sh @@ -11,7 +11,7 @@ REDIRECT=/dev/null -label="grib2to3" +label="grib2to3_test" temp=temp.$label.grib3 sample_g2=$ECCODES_SAMPLES_PATH/GRIB2.tmpl ${tools_dir}/grib_set -s editionNumber=3 $sample_g2 $temp diff --git a/tests/grib3_templates.sh b/tests/grib3_templates.sh index dcc90c48f..decba118a 100755 --- a/tests/grib3_templates.sh +++ b/tests/grib3_templates.sh @@ -11,7 +11,7 @@ REDIRECT=/dev/null -label="grib3_templates" +label="grib3_templates_test" temp1=temp1.$label.grib3 temp2=temp2.$label.grib3 diff --git a/tests/grib_change_scanning.sh b/tests/grib_change_scanning.sh index 65b4360a0..9bd8f8561 100755 --- a/tests/grib_change_scanning.sh +++ b/tests/grib_change_scanning.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib_change_scanning" +label="grib_change_scanning_test" editions="1 2" gridTypes="regular_ll rotated_ll" diff --git a/tests/grib_check_gaussian_grids.sh b/tests/grib_check_gaussian_grids.sh index d1bb7a0b8..ec6adb8b7 100755 --- a/tests/grib_check_gaussian_grids.sh +++ b/tests/grib_check_gaussian_grids.sh @@ -9,7 +9,7 @@ . ./include.ctest.sh -label="grib_check_gaussian_grids" +label="grib_check_gaussian_grids_test" temp=temp.$label.grib diff --git a/tests/grib_complex.sh b/tests/grib_complex.sh index 14d7a86a2..05f5663e8 100755 --- a/tests/grib_complex.sh +++ b/tests/grib_complex.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib_complex" +label="grib_complex_test" temp=${label}".grib.tmp" temp1=${label}".1.tmp" temp2=${label}".2.tmp" diff --git a/tests/grib_data_quality_checks.sh b/tests/grib_data_quality_checks.sh index 8cd47b0cc..6a6177cfd 100755 --- a/tests/grib_data_quality_checks.sh +++ b/tests/grib_data_quality_checks.sh @@ -13,7 +13,7 @@ # --------------------------------------------------------- # Tests for data quality checks # --------------------------------------------------------- -label="grib_data_quality" +label="grib_data_quality_test" tempOut=temp.1.${label}.out temp2=temp.2.${label}.out tempErr=temp.${label}.err diff --git a/tests/grib_dump_debug.sh b/tests/grib_dump_debug.sh index cc84ac62e..d5bceb4f7 100755 --- a/tests/grib_dump_debug.sh +++ b/tests/grib_dump_debug.sh @@ -9,7 +9,7 @@ # . ./include.ctest.sh -label="grib_dump_debug" +label="grib_dump_debug_test" temp=temp.$label.txt REDIRECT=/dev/null diff --git a/tests/grib_dump_json.sh b/tests/grib_dump_json.sh index 83eb70e69..81b3d94ef 100755 --- a/tests/grib_dump_json.sh +++ b/tests/grib_dump_json.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib_dump_json" +label="grib_dump_json_test" temp=${label}".temp.json" REDIRECT=/dev/null diff --git a/tests/grib_efas.sh b/tests/grib_efas.sh index 7a9da86f8..2c6541b79 100755 --- a/tests/grib_efas.sh +++ b/tests/grib_efas.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib_efas" +label="grib_efas_test" sample=temp.sample.$label.grib temp1=temp1.$label.grib temp2=temp2.$label.grib diff --git a/tests/grib_g1fcperiod.sh b/tests/grib_g1fcperiod.sh index 97f6bf46b..173f6e898 100755 --- a/tests/grib_g1fcperiod.sh +++ b/tests/grib_g1fcperiod.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib_g1fcperiod" +label="grib_g1fcperiod_test" sample_g1=$ECCODES_SAMPLES_PATH/GRIB1.tmpl temp=temp.$label.grib diff --git a/tests/grib_g1monthlydate.sh b/tests/grib_g1monthlydate.sh index 9825f32c7..06f478ec1 100755 --- a/tests/grib_g1monthlydate.sh +++ b/tests/grib_g1monthlydate.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib_g1monthlydate" +label="grib_g1monthlydate_test" sample_g1=$ECCODES_SAMPLES_PATH/GRIB1.tmpl temp=temp.$label.grib diff --git a/tests/grib_grid_unstructured.sh b/tests/grib_grid_unstructured.sh index 24d0ffa29..8a68dc2b8 100755 --- a/tests/grib_grid_unstructured.sh +++ b/tests/grib_grid_unstructured.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib_grid_unstructured" +label="grib_grid_unstructured_test" # Only relevant for GRIB2 sample=$ECCODES_SAMPLES_PATH/GRIB2.tmpl diff --git a/tests/grib_indexing.sh b/tests/grib_indexing.sh index b3f6a7583..841778f09 100755 --- a/tests/grib_indexing.sh +++ b/tests/grib_indexing.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib_indexing" +label="grib_indexing_test" temp=temp.$label.index.out infile=${data_dir}/index.grib diff --git a/tests/grib_lam_bf.sh b/tests/grib_lam_bf.sh index 84ecc37f6..b1071f7e8 100755 --- a/tests/grib_lam_bf.sh +++ b/tests/grib_lam_bf.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib_lam_bf" +label="grib_lam_bf_test" temp=temp.$label.txt if [ $ECCODES_ON_WINDOWS -eq 1 ]; then diff --git a/tests/grib_lam_gp.sh b/tests/grib_lam_gp.sh index 16e65f498..65724a13a 100755 --- a/tests/grib_lam_gp.sh +++ b/tests/grib_lam_gp.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib_lam_gp" +label="grib_lam_gp_test" temp=temp.$label.txt dump_and_check() diff --git a/tests/grib_levtype.sh b/tests/grib_levtype.sh index a46b1ee19..d6a8cc522 100755 --- a/tests/grib_levtype.sh +++ b/tests/grib_levtype.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh set -u -label="grib_levtype" +label="grib_levtype_test" if [ ! -d "$ECCODES_DEFINITION_PATH" ]; then echo "Test $0 disabled. No definitions directory" diff --git a/tests/grib_mars_keys.sh b/tests/grib_mars_keys.sh index 40cc6a05a..5188d83b1 100755 --- a/tests/grib_mars_keys.sh +++ b/tests/grib_mars_keys.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib_mars_keys" +label="grib_mars_keys_test" tempOut=temp.${label}.out tempGrib=temp.${label}.grib tempRef=temp.${label}.ref diff --git a/tests/grib_packing_order.sh b/tests/grib_packing_order.sh index 684874e54..3cadb3f91 100755 --- a/tests/grib_packing_order.sh +++ b/tests/grib_packing_order.sh @@ -11,7 +11,7 @@ . ./include.ctest.sh REDIRECT=/dev/null -label="grib_packing_order" +label="grib_packing_order_test" temp=${label}".grib.tmp" temp_simple1=${label}".simple1.tmp" temp_simple2=${label}".simple2.tmp" diff --git a/tests/grib_png.sh b/tests/grib_png.sh index 6acf763e9..5d28981a3 100755 --- a/tests/grib_png.sh +++ b/tests/grib_png.sh @@ -10,7 +10,7 @@ . ./include.ctest.sh -label="grib_png" +label="grib_png_test" temp=${label}".grib.tmp" temp1=${label}".1.tmp" temp2=${label}".2.tmp" diff --git a/tests/grib_tigge_conversions1.sh b/tests/grib_tigge_conversions1.sh index b105a9d98..088066951 100755 --- a/tests/grib_tigge_conversions1.sh +++ b/tests/grib_tigge_conversions1.sh @@ -16,7 +16,7 @@ REDIRECT=/dev/null -label="grib_tigge_conv1" +label="grib_tigge_conv1_test" dir="${data_dir}/tigge" temp1="temp.${label}.grib1_" temp2="temp.${label}.grib2_" diff --git a/tests/grib_tigge_conversions2.sh b/tests/grib_tigge_conversions2.sh index 3d1c0f947..4bf8666b4 100755 --- a/tests/grib_tigge_conversions2.sh +++ b/tests/grib_tigge_conversions2.sh @@ -16,7 +16,7 @@ REDIRECT=/dev/null -label="grib_tigge_conv2" +label="grib_tigge_conv2_test" dir="${data_dir}/tigge" temp1="temp.${label}.grib1_" temp2="temp.${label}.grib2_" From 6de4bbc03fd491d8e8a9de3e36e6ef52fc288c14 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sat, 23 Jul 2022 17:17:27 +0100 Subject: [PATCH 09/13] Testing: Rename test lists --- tests/CMakeLists.txt | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a090dc81f..734b9a63f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -62,7 +62,8 @@ endforeach() if( HAVE_BUILD_TOOLS ) # These tests do not require any data downloads - list(APPEND tests_no_data_reqd + # and are generally quick + list(APPEND tests_basic unit_tests julian grib_dump_samples @@ -111,12 +112,9 @@ if( HAVE_BUILD_TOOLS ) bufr_ecc-359 bufr_ecc-517 bufr_rdbSubTypes - bufr_templates grib_efas grib_sh_imag grib_sh_spectral_complex - pseudo_diag - wrap grib_levtype grib_typeOfLevel grib_grid_unstructured @@ -126,14 +124,16 @@ if( HAVE_BUILD_TOOLS ) grib_g1day_of_the_year_date grib_g1fcperiod) - # These tests do require data downloads - list(APPEND tests_data_reqd + # These tests require data downloads + # and/or take much longer + list(APPEND tests_extra grib_data_quality_checks grib_bpv_limit grib_complex grib_double_cmp grib_change_packing grib_grid_space_view + bufr_templates bufr_dump_data bufr_dump_descriptors bufr_dump_subset @@ -200,6 +200,8 @@ if( HAVE_BUILD_TOOLS ) gts_ls gts_count gts_compare + wrap + pseudo_diag metar_ls metar_get metar_dump @@ -264,21 +266,21 @@ if( HAVE_BUILD_TOOLS ) grib_mars_keys) if( HAVE_FORTRAN AND ENABLE_EXTRA_TESTS ) - list(APPEND tests_data_reqd bufr_dump_encode_fortran) - list(APPEND tests_data_reqd bufr_dump_decode_fortran) + list(APPEND tests_extra bufr_dump_encode_fortran) + list(APPEND tests_extra bufr_dump_decode_fortran) endif() if( ENABLE_EXTRA_TESTS ) - list(APPEND tests_data_reqd grib_util_set_spec) - list(APPEND tests_data_reqd grib_padding) - list(APPEND tests_data_reqd grib_tigge_conversions1) - list(APPEND tests_data_reqd grib_tigge_conversions2) - list(APPEND tests_data_reqd bufr_dump_encode_C) - list(APPEND tests_data_reqd bufr_dump_decode_C) - list(APPEND tests_no_data_reqd list_codetable_flagtable_keys) + list(APPEND tests_extra grib_util_set_spec) + list(APPEND tests_extra grib_padding) + list(APPEND tests_extra grib_tigge_conversions1) + list(APPEND tests_extra grib_tigge_conversions2) + list(APPEND tests_extra bufr_dump_encode_C) + list(APPEND tests_extra bufr_dump_decode_C) + list(APPEND tests_extra list_codetable_flagtable_keys) endif() # These tests do not require any data downloads - foreach( test ${tests_no_data_reqd} ) + foreach( test ${tests_basic} ) ecbuild_add_test( TARGET eccodes_t_${test} TYPE SCRIPT LABELS "sanity" @@ -293,7 +295,7 @@ if( HAVE_BUILD_TOOLS ) # Note: making the test dependent on the grib files (with DEPENDS) # means they will be downloaded at "make" time # rather than when you do "ctest". Use TEST_DEPENDS instead - foreach( test ${tests_data_reqd} ) + foreach( test ${tests_extra} ) ecbuild_add_test( TARGET eccodes_t_${test} TYPE SCRIPT CONDITION ENABLE_EXTRA_TESTS From f8d11512a10caae5e49d69547d34413da63059db Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 25 Jul 2022 17:23:00 +0100 Subject: [PATCH 10/13] CI: Enable MEMFS for Bamboo --- bamboo/flags.cmake | 1 + bamboo/macosx1010-flags.cmake | 2 +- bamboo/opensuse131-CLANG-flags.cmake | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bamboo/flags.cmake b/bamboo/flags.cmake index a6e3c1324..2c5a809e2 100644 --- a/bamboo/flags.cmake +++ b/bamboo/flags.cmake @@ -1,2 +1,3 @@ SET(ENABLE_EXTRA_TESTS ON CACHE BOOL "Enable extra tests") SET(ENABLE_ECCODES_THREADS ON CACHE BOOL "Enable POSIX threads") +SET(ENABLE_MEMFS ON CACHE BOOL "Enable MEMFS") diff --git a/bamboo/macosx1010-flags.cmake b/bamboo/macosx1010-flags.cmake index fba3c2980..c92fa490e 100644 --- a/bamboo/macosx1010-flags.cmake +++ b/bamboo/macosx1010-flags.cmake @@ -1,2 +1,2 @@ # All python tests fail due to error loading the SWIG generated extension module -SET( ENABLE_PYTHON OFF CACHE BOOL "Disable Python") +SET( ENABLE_PYTHON2 OFF CACHE BOOL "Disable Python") diff --git a/bamboo/opensuse131-CLANG-flags.cmake b/bamboo/opensuse131-CLANG-flags.cmake index 37054dde9..dbd5afbba 100644 --- a/bamboo/opensuse131-CLANG-flags.cmake +++ b/bamboo/opensuse131-CLANG-flags.cmake @@ -5,3 +5,5 @@ # All python tests fails due to error loading the shared python lib: undefined symbol: __tsan_init #SET( ENABLE_PYTHON OFF CACHE BOOL "Disable Python") + +SET(ENABLE_MEMFS ON CACHE BOOL "Enable MEMFS") From a677da2908d0634aa5ac9d7158a9fc94863162ce Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 26 Jul 2022 13:23:32 +0100 Subject: [PATCH 11/13] ECC-1430: GRIB2: Add extra values for stepType: 'severity' and 'mode' --- definitions/grib2/section.1.def | 2 ++ definitions/grib2/template.4.statistical.def | 2 ++ tests/grib_step.sh | 12 ++++++++++++ 3 files changed, 16 insertions(+) diff --git a/definitions/grib2/section.1.def b/definitions/grib2/section.1.def index a9bca14ce..719a9399c 100644 --- a/definitions/grib2/section.1.def +++ b/definitions/grib2/section.1.def @@ -93,6 +93,8 @@ concept stepType { "ratio" = {selectStepTemplateInterval=1; stepTypeInternal="ratio";} "stdanom" = {selectStepTemplateInterval=1; stepTypeInternal="stdanom";} "sum" = {selectStepTemplateInterval=1; stepTypeInternal="sum";} + "severity" = {selectStepTemplateInterval=1; stepTypeInternal="severity";} + "mode" = {selectStepTemplateInterval=1; stepTypeInternal="mode";} } # 0=atmospheric chemical constituents diff --git a/definitions/grib2/template.4.statistical.def b/definitions/grib2/template.4.statistical.def index 35390d536..c9307b04e 100644 --- a/definitions/grib2/template.4.statistical.def +++ b/definitions/grib2/template.4.statistical.def @@ -68,6 +68,8 @@ if (numberOfTimeRange == 1 || numberOfTimeRange == 2) { "ratio" = {typeOfStatisticalProcessing=9;} "stdanom" = {typeOfStatisticalProcessing=10;} "sum" = {typeOfStatisticalProcessing=11;} + "severity" = {typeOfStatisticalProcessing=100;} + "mode" = {typeOfStatisticalProcessing=101;} } meta startStep step_in_units(forecastTime,indicatorOfUnitOfTimeRange,stepUnits, indicatorOfUnitForTimeRange,lengthOfTimeRange) : no_copy; diff --git a/tests/grib_step.sh b/tests/grib_step.sh index 362bf9d03..1bc7e208b 100755 --- a/tests/grib_step.sh +++ b/tests/grib_step.sh @@ -62,6 +62,7 @@ diff ${data_dir}/step_grib1.log ${templog} rm -f ${templog} # GRIB-180 +# ------------ # Set PDT 4.8 where you can find the EndOfOverallTimeInterval keys grib2File=${data_dir}/reduced_latlon_surface_constant.grib2 ${tools_dir}/grib_set -sproductDefinitionTemplateNumber=8 $grib2File ${grib2File}.p8tmp @@ -81,6 +82,7 @@ hourEnd=$1; dayEnd=$2 # ECC-134 case-sensitivity +# -------------------------- grib1_sample=$ECCODES_SAMPLES_PATH/GRIB1.tmpl grib2_sample=$ECCODES_SAMPLES_PATH/GRIB2.tmpl temp=temp.step.$$.grib @@ -93,6 +95,7 @@ unit=`${tools_dir}/grib_get -p indicatorOfUnitOfTimeRange $temp` [ "$unit" = "3" ] # m is for Minute (code 0) +# ------------------------- ${tools_dir}/grib_set -s indicatorOfUnitOfTimeRange=m $grib1_sample $temp unit=`${tools_dir}/grib_get -p unitOfTimeRange $temp` [ "$unit" = "0" ] @@ -101,9 +104,18 @@ unit=`${tools_dir}/grib_get -p indicatorOfUnitOfTimeRange $temp` [ "$unit" = "0" ] # ECC-457 +# --------- input=${data_dir}/tp_ecmwf.grib stepRange=`${tools_dir}/grib_get -w count=1 -p stepRange,startStep,endStep,stepType $input` [ "$stepRange" = "12 12 12 instant" ] +# ECC-1430: stepType: 'severity' and 'mode' +# ----------------------------------------- +${tools_dir}/grib_set -s stepType=severity,paramId=260318 $grib2_sample $temp +grib_check_key_equals $temp productDefinitionTemplateNumber,typeOfStatisticalProcessing '8 100' +${tools_dir}/grib_set -s stepType=mode,paramId=260320 $grib2_sample $temp +grib_check_key_equals $temp productDefinitionTemplateNumber,typeOfStatisticalProcessing '8 101' + +# Clean up rm -f $temp rm -f $grib2File.p8tmp ${grib2File}.tmp x.grib From b111d0b70bc26b49da8cc0d5d76a6c349e5c41ca Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 26 Jul 2022 19:05:11 +0100 Subject: [PATCH 12/13] Tools: Fix key name --- tools/grib_check_gaussian_grid.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/grib_check_gaussian_grid.c b/tools/grib_check_gaussian_grid.c index 03ec2c0d2..0ff8e656c 100644 --- a/tools/grib_check_gaussian_grid.c +++ b/tools/grib_check_gaussian_grid.c @@ -144,7 +144,7 @@ static int process_file(const char* filename) } if (lon1 != 0) { - error(filename, msg_num, "latitudeOfFirstGridPointInDegrees=%f but should be 0\n", lon1); + error(filename, msg_num, "longitudeOfFirstGridPointInDegrees=%f but should be 0\n", lon1); } expected_lon2 = 360.0 - 360.0 / (4 * N); From 714415054d31ffc2523376aedcce7c59e1dbf825 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 28 Jul 2022 12:34:26 +0100 Subject: [PATCH 13/13] ECC-762 and ECC-1432 --- data/ls.log | 12 ++++----- src/grib_nearest.c | 5 +--- src/grib_nearest_class_reduced.c | 45 +++++++++++++++++++++++++++++++- tests/grib_nearest_test.sh | 16 ++++++++++++ 4 files changed, 67 insertions(+), 11 deletions(-) diff --git a/data/ls.log b/data/ls.log index 4e290e613..20d9557f4 100644 --- a/data/ls.log +++ b/data/ls.log @@ -75,17 +75,17 @@ index=1290 index=1171 index=1170 reduced_gaussian_sub_area.grib1 -#3 -index=19985 -index=19984 +#1 index=19665 +index=19985 index=19664 +index=19984 reduced_gaussian_sub_area.grib2 -#3 -index=19985 -index=19984 +#1 index=19665 +index=19985 index=19664 +index=19984 reduced_gaussian_surface.grib1 #4 index=1291 diff --git a/src/grib_nearest.c b/src/grib_nearest.c index a30245232..5a69622d5 100644 --- a/src/grib_nearest.c +++ b/src/grib_nearest.c @@ -342,10 +342,7 @@ int grib_nearest_find_generic( double lat1 = 0, lat2 = 0; /* inlat will be between these */ const double LAT_DELTA = 10.0; /* in degrees */ - if (grib_is_missing(h, Ni_keyname, &ret)) { - grib_context_log(h->context, GRIB_LOG_DEBUG, "Key '%s' is missing", Ni_keyname); - return ret ? ret : GRIB_GEOCALCULUS_PROBLEM; - } + /* Note: If this is being called for a REDUCED grid, its Ni will be missing */ if (grib_is_missing(h, Nj_keyname, &ret)) { grib_context_log(h->context, GRIB_LOG_DEBUG, "Key '%s' is missing", Nj_keyname); diff --git a/src/grib_nearest_class_reduced.c b/src/grib_nearest_class_reduced.c index 598985892..0b75b1167 100644 --- a/src/grib_nearest_class_reduced.c +++ b/src/grib_nearest_class_reduced.c @@ -130,17 +130,60 @@ static int init(grib_nearest* nearest, grib_handle* h, grib_arguments* args) typedef void (*get_reduced_row_proc)(long pl, double lon_first, double lon_last, long* npoints, long* ilon_first, long* ilon_last); +static int find_global(grib_nearest* nearest, grib_handle* h, + double inlat, double inlon, unsigned long flags, + double* outlats, double* outlons, double* values, + double* distances, int* indexes, size_t* len); + static int is_legacy(grib_handle* h) { long is_legacy = 0; return (grib_get_long(h, "legacyGaussSubarea", &is_legacy) == GRIB_SUCCESS && is_legacy == 1); } -/* Old implementation in src/deprecated/grib_nearest_class_reduced.old */ + static int find(grib_nearest* nearest, grib_handle* h, double inlat, double inlon, unsigned long flags, double* outlats, double* outlons, double* values, double* distances, int* indexes, size_t* len) +{ + int err = 0; + grib_nearest_reduced* self = (grib_nearest_reduced*)nearest; + + if (self->global) { + err = find_global(nearest, h, + inlat, inlon, flags, + outlats, outlons, values, + distances, indexes, len); + } + else + { + /* ECC-762, ECC-1432: Use brute force generic algorithm + * for reduced grid subareas. Review in the future + */ + int lons_count = 0; /*dummy*/ + + err = grib_nearest_find_generic( + nearest, h, inlat, inlon, flags, + self->values_key, + "Ni", + self->Nj, + &(self->lats), + &(self->lats_count), + &(self->lons), + &(lons_count), + &(self->distances), + outlats, outlons, + values, distances, indexes, len); + } + return err; +} + +/* Old implementation in src/deprecated/grib_nearest_class_reduced.old */ +static int find_global(grib_nearest* nearest, grib_handle* h, + double inlat, double inlon, unsigned long flags, + double* outlats, double* outlons, double* values, + double* distances, int* indexes, size_t* len) { grib_nearest_reduced* self = (grib_nearest_reduced*)nearest; int ret = 0, kk = 0, ii = 0, jj = 0; diff --git a/tests/grib_nearest_test.sh b/tests/grib_nearest_test.sh index 4799d06aa..1ca9e379a 100755 --- a/tests/grib_nearest_test.sh +++ b/tests/grib_nearest_test.sh @@ -119,6 +119,22 @@ ${tools_dir}/grib_ls -l 0,0,1 $tempGrib > $temp grep -q "Grid Point chosen #2 index=480 " $temp +# Subarea: Reduced Gaussian grid +# ------------------------------ +input_grb=${data_dir}/reduced_gaussian_sub_area.grib1 +${tools_dir}/grib_get -F%3.6g -l 89.656,220,1 $input_grb > $temp +grep -q "274.038" $temp +${tools_dir}/grib_ls -l 89.656,220,1 $input_grb > $temp +grep -q "Grid Point chosen #1 index=0 " $temp + +${tools_dir}/grib_ls -l 0.225,399.6,1 $input_grb > $temp +grep -q "Grid Point chosen #1 index=53563 " $temp + +${tools_dir}/grib_get -F%3.6g -l 11.91,366.3,1 $input_grb > $temp +grep -q "299.632" $temp +${tools_dir}/grib_ls -l 11.91,366.3,1 $input_grb > $temp +grep -q "Grid Point chosen #1 index=43089 " $temp + # Clean up rm -f $temp $tempRef $tempGrib