From e22bc13bf624d568194137804abef0b10466ea3d Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 2 Jan 2024 15:12:57 +0000 Subject: [PATCH 01/40] Testing: Fortran codes_any_scan_file and codes_any_new_from_scanned_file --- examples/F90/CMakeLists.txt | 1 + examples/F90/codes_scan_file.f90 | 35 ++++++++++++++++++++++++++++++++ examples/F90/codes_scan_file.sh | 20 ++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 examples/F90/codes_scan_file.f90 create mode 100755 examples/F90/codes_scan_file.sh diff --git a/examples/F90/CMakeLists.txt b/examples/F90/CMakeLists.txt index ead88a487..880a47f37 100644 --- a/examples/F90/CMakeLists.txt +++ b/examples/F90/CMakeLists.txt @@ -22,6 +22,7 @@ if( HAVE_BUILD_TOOLS ) list( APPEND tests_extra grib_index codes_dump + codes_scan_file grib_copy_message bufr_copy_message grib_get_keys diff --git a/examples/F90/codes_scan_file.f90 b/examples/F90/codes_scan_file.f90 new file mode 100644 index 000000000..5136f126a --- /dev/null +++ b/examples/F90/codes_scan_file.f90 @@ -0,0 +1,35 @@ +! (C) Copyright 2005- ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! +! In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +! virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +! +! +program codes_scan_file + use eccodes + implicit none + integer, parameter :: max_strsize = 200 + integer :: ifile, cnt, level, step + integer :: i, igrib + character(len=max_strsize) :: infile_name + + call getarg(1, infile_name) + + call codes_open_file(ifile, infile_name, 'r') + + call codes_any_scan_file(ifile,cnt) + + i = 45 + call codes_any_new_from_scanned_file(ifile,i,igrib) + call codes_get(igrib, 'level', level) + call codes_get(igrib, 'stepRange', step) + + print *, 'Num messages=', cnt + print *, 'Msg ',i,' level=',level, ' step=', step + + call codes_release(igrib) + call codes_close_file(ifile) + +end program diff --git a/examples/F90/codes_scan_file.sh b/examples/F90/codes_scan_file.sh new file mode 100755 index 000000000..80ca95bc3 --- /dev/null +++ b/examples/F90/codes_scan_file.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# (C) Copyright 2005- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. + +. ./include.ctest.sh + +temp='temp.eccodes_f_codes_scan_file.txt' + +input=../../data/index.grib +${examples_dir}/eccodes_f_codes_scan_file $input > $temp + +grep -q "Num messages= *384" $temp +grep -q "level= *700 step= *60" $temp + +rm -f $temp From 4047c5764c08e6d2c29fc4911a93c3a2d31d3ab1 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 2 Jan 2024 16:01:37 +0000 Subject: [PATCH 02/40] Testing: codes_any_new_from_scanned_file --- examples/F90/codes_scan_file.f90 | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/examples/F90/codes_scan_file.f90 b/examples/F90/codes_scan_file.f90 index 5136f126a..7c0438275 100644 --- a/examples/F90/codes_scan_file.f90 +++ b/examples/F90/codes_scan_file.f90 @@ -12,7 +12,7 @@ program codes_scan_file implicit none integer, parameter :: max_strsize = 200 integer :: ifile, cnt, level, step - integer :: i, igrib + integer :: i, igrib, iret character(len=max_strsize) :: infile_name call getarg(1, infile_name) @@ -22,7 +22,7 @@ program codes_scan_file call codes_any_scan_file(ifile,cnt) i = 45 - call codes_any_new_from_scanned_file(ifile,i,igrib) + call codes_any_new_from_scanned_file(ifile, i, igrib) call codes_get(igrib, 'level', level) call codes_get(igrib, 'stepRange', step) @@ -30,6 +30,16 @@ program codes_scan_file print *, 'Msg ',i,' level=',level, ' step=', step call codes_release(igrib) + + ! Invalid msg number + i = 450 + call codes_any_new_from_scanned_file(ifile, i, igrib, iret) + if (iret /= GRIB_INVALID_ARGUMENT) then + call codes_check(iret, 'codes_any_new_from_scanned_file', 'exit') + else + print *,'Invalid message index returned error (as expected)' + end if + call codes_close_file(ifile) end program From 4cd68b210fd508c09db5c611f3a63a0a2e0627b9 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 2 Jan 2024 16:04:02 +0000 Subject: [PATCH 03/40] Fortran: any_f_scan_file_ check for errors --- fortran/grib_fortran.c | 51 ++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/fortran/grib_fortran.c b/fortran/grib_fortran.c index e744090dd..a07a3110f 100644 --- a/fortran/grib_fortran.c +++ b/fortran/grib_fortran.c @@ -1441,52 +1441,59 @@ int any_f_scan_file_(int* fid, int* n) { /* this needs a callback to a destructor*/ /* grib_oarray_delete_content(c,binary_messages); */ - grib_oarray_delete(c,info_messages); - info_messages=grib_oarray_new(c,1000,1000); + grib_oarray_delete(c, info_messages); + info_messages=grib_oarray_new(c, 1000, 1000); if (f) { while (err!=GRIB_END_OF_FILE) { - data = wmo_read_any_from_file_malloc ( f, 0,&olen,&offset,&err ); + data = wmo_read_any_from_file_malloc ( f, 0, &olen, &offset, &err ); msg=(l_message_info*)grib_context_malloc_clear(c,sizeof(l_message_info)); - msg->offset=offset; - msg->size=olen; - - if (err==0 && data) grib_oarray_push(c,info_messages,msg); - grib_context_free(c,data); + msg->offset = offset; + msg->size = olen; + + if (err == 0 && data) grib_oarray_push(c, info_messages, msg); + grib_context_free(c, data); } - if (err==GRIB_END_OF_FILE) err=0; + if (err == GRIB_END_OF_FILE) err = 0; } - *n=info_messages->n; + *n = info_messages->n; return err; } /*****************************************************************************/ -int any_f_new_from_scanned_file_(int* fid,int* msgid,int* gid) +int any_f_new_from_scanned_file_(int* fid, int* msgid, int* gid) { grib_handle *h = NULL; - grib_context* c=grib_context_get_default(); - int err=0; + grib_context* c = grib_context_get_default(); + int err = 0; FILE* f = get_file(*fid); - /* fortran convention of 1 based index*/ - const int n=*msgid-1; + if (info_messages == NULL) { + return GRIB_INVALID_ARGUMENT; + } + if (*msgid < 1 || *msgid > info_messages->n) { + return GRIB_INVALID_ARGUMENT; + } - l_message_info* msg=(l_message_info*)grib_oarray_get(info_messages,n); + /* fortran convention of 1-based index */ + const int n = *msgid - 1; + + l_message_info* msg=(l_message_info*)grib_oarray_get(info_messages, n); if (msg && f) { - GRIB_MUTEX_INIT_ONCE(&once,&init); + GRIB_MUTEX_INIT_ONCE(&once, &init); GRIB_MUTEX_LOCK(&read_mutex); - fseeko(f,msg->offset,SEEK_SET); - h=any_new_from_file (c,f,&err); + fseeko(f, msg->offset, SEEK_SET); + h = any_new_from_file (c, f, &err); GRIB_MUTEX_UNLOCK(&read_mutex); } if (err) return err; - if(h){ - push_handle(h,gid); + if (h) { + push_handle(h, gid); return GRIB_SUCCESS; } else { - *gid=-1; + *gid = -1; return GRIB_END_OF_FILE; } } From 1be36946be386f6effff5354e4bf93d43d5bfc56 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 2 Jan 2024 16:23:21 +0000 Subject: [PATCH 04/40] ECC-1740: GRIB2: Addition to concept combinationOfAttributesOfTile --- definitions/grib2/combinationOfAttributesOfTileConcept.def | 1 + 1 file changed, 1 insertion(+) diff --git a/definitions/grib2/combinationOfAttributesOfTileConcept.def b/definitions/grib2/combinationOfAttributesOfTileConcept.def index 465913ca8..93d23dd1e 100644 --- a/definitions/grib2/combinationOfAttributesOfTileConcept.def +++ b/definitions/grib2/combinationOfAttributesOfTileConcept.def @@ -1,5 +1,6 @@ # Concept combinationOfAttributesOfTile +'UNDEF' = {attributeOfTile = [0];} 'UNMOD' = {attributeOfTile = [1];} 'SNOW' = {attributeOfTile = [2];} 'FLOOD' = {attributeOfTile = [3];} From 5636bdfaec14206a7c509afc4b57dddcd497e056 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 2 Jan 2024 16:54:26 +0000 Subject: [PATCH 05/40] Fortran: ISO C90 forbids mixed declarations and code --- fortran/grib_fortran.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fortran/grib_fortran.c b/fortran/grib_fortran.c index a07a3110f..258b9a003 100644 --- a/fortran/grib_fortran.c +++ b/fortran/grib_fortran.c @@ -1467,6 +1467,10 @@ int any_f_new_from_scanned_file_(int* fid, int* msgid, int* gid) grib_context* c = grib_context_get_default(); int err = 0; FILE* f = get_file(*fid); + l_message_info* msg = NULL; + + /* fortran convention of 1-based index */ + const int n = *msgid - 1; if (info_messages == NULL) { return GRIB_INVALID_ARGUMENT; @@ -1475,10 +1479,7 @@ int any_f_new_from_scanned_file_(int* fid, int* msgid, int* gid) return GRIB_INVALID_ARGUMENT; } - /* fortran convention of 1-based index */ - const int n = *msgid - 1; - - l_message_info* msg=(l_message_info*)grib_oarray_get(info_messages, n); + msg = (l_message_info*)grib_oarray_get(info_messages, n); if (msg && f) { GRIB_MUTEX_INIT_ONCE(&once, &init); From 279ff53a4c7f7a11c218a93bc9e0727284181817 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 3 Jan 2024 10:55:32 +0000 Subject: [PATCH 06/40] Definitions: GTS formatting --- definitions/gts/boot.def | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/definitions/gts/boot.def b/definitions/gts/boot.def index f2a2d3d5b..02b3292e9 100644 --- a/definitions/gts/boot.def +++ b/definitions/gts/boot.def @@ -39,14 +39,13 @@ alias ls.CCCC=CCCC; alias ls.YY=YY; alias ls.GG=GG; alias ls.gg=gg; -position endOfHeadersMarker; +position endOfHeadersMarker; message[4] theMessage; -meta lengthOfHeaders evaluate( endOfHeadersMarker-startOfHeaders); -meta md5Headers md5(startOfHeaders,lengthOfHeaders); +meta lengthOfHeaders evaluate(endOfHeadersMarker-startOfHeaders); +meta md5Headers md5(startOfHeaders, lengthOfHeaders); -ascii[4] endMark; +ascii[4] endMark; position totalLength; alias ls.totalLength=totalLength; - From 7424b760966485f48b3f78640777ee5986a16752 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 3 Jan 2024 10:55:55 +0000 Subject: [PATCH 07/40] Testing: GTS compare --- tests/gts_compare.sh | 13 +++++++++++++ tools/gts_compare.cc | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/gts_compare.sh b/tests/gts_compare.sh index 3fd648a1d..6b57e0369 100755 --- a/tests/gts_compare.sh +++ b/tests/gts_compare.sh @@ -64,7 +64,20 @@ set -e # Add correct blocklist ${tools_dir}/gts_compare -b GG $gts_file $fGtsTmp +#---------------------------------------------------- +# Compare using -c +#---------------------------------------------------- +temp1=temp.$label.1.gts +temp2=temp.$label.2.gts +# Pick two messages which do have different contents +${tools_dir}/gts_copy -w count=1 $gts_file $temp1 +${tools_dir}/gts_copy -w count=4 $gts_file $temp2 +${tools_dir}/gts_compare -c theMessage $temp1 $temp2 +rm -f $temp1 $temp2 + +#---------------------------------------------------- # Test with file of the same name in a dir +#---------------------------------------------------- tempDir=temp.$label.dir rm -fr $tempDir mkdir $tempDir diff --git a/tools/gts_compare.cc b/tools/gts_compare.cc index fee4f8e00..c9bb66af7 100644 --- a/tools/gts_compare.cc +++ b/tools/gts_compare.cc @@ -611,8 +611,10 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h case GRIB_TYPE_BYTES: if (verbose) printf(" as bytes\n"); - if (options->mode == MODE_GTS) + if (options->mode == MODE_GTS) { + // We do not want to compare the message itself return 0; + } if (len1 < 2) len1 = 512; if (len2 < 2) From e6aafdb7dc6d14078a24f656b3556ecc1c8e2e1f Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 3 Jan 2024 11:52:23 +0000 Subject: [PATCH 08/40] Testing: METAR compare --- tests/metar_compare.sh | 12 ++++++++++ tools/grib_compare.cc | 28 +++++++++++------------ tools/metar_compare.cc | 52 +++++++++++++++++++++++++----------------- 3 files changed, 56 insertions(+), 36 deletions(-) diff --git a/tests/metar_compare.sh b/tests/metar_compare.sh index 6311e8597..2d43f289c 100755 --- a/tests/metar_compare.sh +++ b/tests/metar_compare.sh @@ -65,6 +65,18 @@ grep -q "DIFFERENCE == string.*theMessage" $fLog # The -d option should have created these files rm -f error1_1.metar error2_1.metar error1_2.metar error2_2.metar +#---------------------------------------------------- +# Compare a key of type double +#---------------------------------------------------- +temp1=temp.$label.metar.1 +temp2=temp.$label.metar.2 +${tools_dir}/metar_copy -w count=1 $metar_file $temp1 +${tools_dir}/metar_copy -w count=2 $metar_file $temp2 +# absolute diff. = 16.53, relative diff. = 0.381315 +${tools_dir}/metar_compare -c latitude -R latitude=0.4 $temp1 $temp2 +${tools_dir}/metar_compare -c latitude -A 17 $temp1 $temp2 +rm -f $temp1 $temp2 + #---------------------------------------------------- # Test: comparing with and without the -b switch #---------------------------------------------------- diff --git a/tools/grib_compare.cc b/tools/grib_compare.cc index 9d99498fb..9b9cef404 100644 --- a/tools/grib_compare.cc +++ b/tools/grib_compare.cc @@ -383,21 +383,19 @@ int grib_tool_init(grib_runtime_options* options) } } - { - /* Check for 2nd file being a directory. If so, we assume user is comparing to a file */ - /* with the same name as first file in that directory */ - grib_tools_file* infile = options->infile; /* the 2nd file in comparison */ - if (infile) { - if (path_is_directory(infile->name)) { - /* Take the filename of the 1st file and append to dir */ - char bufr[2048] = {0,}; - /* options->infile_extra->name is the 1st file */ - snprintf(bufr, sizeof(bufr), "%s%c%s", - infile->name, - get_dir_separator_char(), - extract_filename(options->infile_extra->name)); - infile->name = strdup(bufr); - } + // Check for 2nd file being a directory. If so, we assume user is comparing to a file + // with the same name as first file in that directory + grib_tools_file* infile = options->infile; // the 2nd file in comparison + if (infile) { + if (path_is_directory(infile->name)) { + // Take the filename of the 1st file and append to dir + char bufr[2048] = {0,}; + // options->infile_extra->name is the 1st file + snprintf(bufr, sizeof(bufr), "%s%c%s", + infile->name, + get_dir_separator_char(), + extract_filename(options->infile_extra->name)); + infile->name = strdup(bufr); } } diff --git a/tools/metar_compare.cc b/tools/metar_compare.cc index a1b3af529..c275273bb 100644 --- a/tools/metar_compare.cc +++ b/tools/metar_compare.cc @@ -279,21 +279,29 @@ int grib_tool_init(grib_runtime_options* options) if (grib_options_on("t:")) tolerance_factor = atof(grib_options_get_option("t:")); - { - /* Check for 2nd file being a directory. If so, we assume user is comparing to a file */ - /* with the same name as first file in that directory */ - grib_tools_file* infile = options->infile; /* the 2nd file in comparison */ - if (infile) { - if (path_is_directory(infile->name)) { - /* Take the filename of the 1st file and append to dir */ - char bufr[2048] = {0,}; - /* options->infile_extra->name is the 1st file */ - snprintf(bufr, 2048, "%s%c%s", - infile->name, - get_dir_separator_char(), - extract_filename(options->infile_extra->name)); - infile->name = strdup(bufr); - } + if (grib_options_on("R:")) { + char* sarg = grib_options_get_option("R:"); + options->tolerance_count = MAX_KEYS; + int err = parse_keyval_string(tool_name, sarg, 1, GRIB_TYPE_DOUBLE, options->tolerance, &(options->tolerance_count)); + if (err == GRIB_INVALID_ARGUMENT) { + usage(); + exit(1); + } + } + + // Check for 2nd file being a directory. If so, we assume user is comparing to a file + // with the same name as first file in that directory + grib_tools_file* infile = options->infile; // the 2nd file in comparison + if (infile) { + if (path_is_directory(infile->name)) { + // Take the filename of the 1st file and append to dir + char bufr[2048] = {0,}; + // options->infile_extra->name is the 1st file + snprintf(bufr, 2048, "%s%c%s", + infile->name, + get_dir_separator_char(), + extract_filename(options->infile_extra->name)); + infile->name = strdup(bufr); } } @@ -693,8 +701,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h pv1 = dval1; pv2 = dval2; value_tolerance *= tolerance_factor; - if (verbose) - printf(" (%d values) tolerance=%g\n", (int)len1, value_tolerance); + if (verbose) { + printf(" (%d values) tolerance=%g \t", (int)len1, value_tolerance); + if (compare_double == &compare_double_absolute) + printf("using compare_double_absolute"); + if (compare_double == &compare_double_relative) + printf("using compare_double_relative"); + printf("\n"); + } for (i = 0; i < len1; i++) { if ((diff = compare_double(pv1++, pv2++, &value_tolerance)) != 0) { countdiff++; @@ -828,7 +842,6 @@ static int compare_all_dump_keys(grib_handle* h1, grib_handle* h2, grib_runtime_ while (grib_keys_iterator_next(iter)) { grib_accessor* xa = grib_keys_iterator_get_accessor(iter); name = grib_keys_iterator_get_name(iter); - /* printf("----- comparing %s\n",name); */ if (blocklisted(name)) continue; @@ -864,7 +877,6 @@ static int compare_handles(grib_handle* h1, grib_handle* h2, grib_runtime_option } while (grib_keys_iterator_next(iter)) { name = grib_keys_iterator_get_name(iter); - /*printf("----- comparing %s\n",name);*/ if (blocklisted(name)) continue; @@ -886,7 +898,6 @@ static int compare_handles(grib_handle* h1, grib_handle* h2, grib_runtime_option const void *msg1 = NULL, *msg2 = NULL; size_t size1 = 0, size2 = 0; int memcmp_ret = 0; - /* int ii=0; */ GRIB_CHECK_NOLINE(grib_get_message(h1, &msg1, &size1), 0); GRIB_CHECK_NOLINE(grib_get_message(h2, &msg2, &size2), 0); if (size1 == size2 && !(memcmp_ret = memcmp(msg1, msg2, size1))) { @@ -905,7 +916,6 @@ static int compare_handles(grib_handle* h1, grib_handle* h2, grib_runtime_option } while (grib_keys_iterator_next(iter)) { name = grib_keys_iterator_get_name(iter); - /*printf("----- comparing %s\n",name);*/ if (blocklisted(name)) continue; From af7778cbdde1a429a1e36fa7d526d2fced8e3903 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 3 Jan 2024 13:17:23 +0000 Subject: [PATCH 09/40] Tools: Error messages --- tools/bufr_compare.cc | 30 ++++++++++++++---------------- tools/grib_compare.cc | 24 ++++++++++++------------ tools/gts_compare.cc | 20 ++++++++++---------- tools/metar_compare.cc | 24 ++++++++++++------------ 4 files changed, 48 insertions(+), 50 deletions(-) diff --git a/tools/bufr_compare.cc b/tools/bufr_compare.cc index 294a1da26..9a1905944 100644 --- a/tools/bufr_compare.cc +++ b/tools/bufr_compare.cc @@ -714,7 +714,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* handle1, g if (type1 == GRIB_TYPE_UNDEFINED && (err = grib_get_native_type(handle1, name, &type1)) != GRIB_SUCCESS) { printInfo(handle1); - printf("Oops... cannot get type of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err)); + printf("Error: cannot get type of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err)); save_error(c, name); return err; } @@ -727,7 +727,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* handle1, g return err; } printInfo(handle1); - printf("Oops... cannot get type of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err)); + printf("Error: cannot get type of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err)); save_error(c, name); return err; } @@ -747,7 +747,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* handle1, g if ((err = grib_get_size(handle1, name, &len1)) != GRIB_SUCCESS) { printInfo(handle1); - printf("Oops... cannot get size of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err)); + printf("Error: cannot get size of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err)); save_error(c, name); return err; } @@ -761,7 +761,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* handle1, g } printInfo(handle1); - printf("Oops... cannot get size of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err)); + printf("Error: cannot get size of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err)); save_error(c, name); return err; } @@ -826,13 +826,13 @@ static int compare_values(grib_runtime_options* options, grib_handle* handle1, g sval2 = (char*)grib_context_malloc(handle2->context, slen2 * sizeof(char)); if ((err1 = grib_get_string(handle1, name, sval1, &slen1)) != GRIB_SUCCESS) { printInfo(handle1); - printf("Oops... cannot get string value of [%s] in %s field: %s\n", + printf("Error: cannot get string value of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err1)); save_error(c, name); } if ((err2 = grib_get_string(handle2, name, sval2, &slen2)) != GRIB_SUCCESS) { printInfo(handle1); - printf("Oops... cannot get string value of [%s] in %s field: %s\n", + printf("Error: cannot get string value of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err2)); save_error(c, name); } @@ -855,13 +855,13 @@ static int compare_values(grib_runtime_options* options, grib_handle* handle1, g if ((err1 = grib_get_string_array(handle1, name, svals1, &len1)) != GRIB_SUCCESS) { printInfo(handle1); - printf("Oops... cannot get string value of [%s] in %s field: %s\n", + printf("Error: cannot get string value of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err1)); save_error(c, name); } if ((err2 = grib_get_string_array(handle2, name, svals2, &len2)) != GRIB_SUCCESS) { printInfo(handle1); - printf("Oops... cannot get string value of [%s] in %s field: %s\n", + printf("Error: cannot get string value of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err2)); save_error(c, name); } @@ -910,14 +910,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* handle1, g if ((err1 = grib_get_long_array(handle1, name, lval1, &len1)) != GRIB_SUCCESS) { printInfo(handle1); - printf("Oops... cannot get long value of [%s] in %s field: %s\n", + printf("Error: cannot get long value of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err1)); save_error(c, name); } if ((err2 = grib_get_long_array(handle2, name, lval2, &len2)) != GRIB_SUCCESS) { printInfo(handle1); - printf("Oops... cannot get long value of [%s] in %s field: %s\n", + printf("Error: cannot get long value of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err2)); save_error(c, name); } @@ -995,14 +995,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* handle1, g if ((err1 = grib_get_double_array(handle1, name, dval1, &len1)) != GRIB_SUCCESS) { printInfo(handle1); - printf("Oops... cannot get double value of [%s] in %s field: %s\n", + printf("Error: cannot get double value of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err1)); save_error(c, name); } if ((err2 = grib_get_double_array(handle2, name, dval2, &len2)) != GRIB_SUCCESS) { printInfo(handle1); - printf("Oops... cannot get double value of [%s] in %s field: %s\n", + printf("Error: cannot get double value of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err2)); save_error(c, name); } @@ -1104,14 +1104,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* handle1, g if ((err1 = grib_get_bytes(handle1, name, uval1, &len1)) != GRIB_SUCCESS) { printInfo(handle1); save_error(c, name); - printf("Oops... cannot get bytes value of [%s] in %s field: %s\n", + printf("Error: cannot get bytes value of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err1)); } if ((err2 = grib_get_bytes(handle2, name, uval2, &len2)) != GRIB_SUCCESS) { printInfo(handle1); save_error(c, name); - printf("Oops... cannot get bytes value of [%s] in %s field: %s\n", + printf("Error: cannot get bytes value of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err2)); } @@ -1316,7 +1316,6 @@ static int compare_handles(grib_handle* handle1, grib_handle* handle2, grib_runt } while (grib_keys_iterator_next(iter)) { name = grib_keys_iterator_get_name(iter); - /*printf("----- comparing %s\n",name);*/ if (blocklisted(name)) continue; @@ -1336,7 +1335,6 @@ static int compare_handles(grib_handle* handle1, grib_handle* handle2, grib_runt const void *msg1 = NULL, *msg2 = NULL; size_t size1 = 0, size2 = 0; int memcmp_ret = 0; - /* int ii=0; */ GRIB_CHECK_NOLINE(grib_get_message(handle1, &msg1, &size1), 0); GRIB_CHECK_NOLINE(grib_get_message(handle2, &msg2, &size2), 0); if (size1 == size2 && !(memcmp_ret = memcmp(msg1, msg2, size1))) { diff --git a/tools/grib_compare.cc b/tools/grib_compare.cc index 9b9cef404..cace206fe 100644 --- a/tools/grib_compare.cc +++ b/tools/grib_compare.cc @@ -711,7 +711,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if (type1 == GRIB_TYPE_UNDEFINED && (err = grib_get_native_type(h1, name, &type1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get type of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err)); + printf("Error: cannot get type of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err)); save_error(c, name); return err; } @@ -724,7 +724,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h return err; } printInfo(h1); - printf("Oops... cannot get type of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err)); + printf("Error: cannot get type of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err)); save_error(c, name); return err; } @@ -746,7 +746,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err = grib_get_size(h1, name, &len1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get size of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err)); + printf("Error: cannot get size of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err)); save_error(c, name); return err; } @@ -760,7 +760,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h } printInfo(h1); - printf("Oops... cannot get size of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err)); + printf("Error: cannot get size of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err)); save_error(c, name); return err; } @@ -818,14 +818,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err1 = grib_get_string(h1, name, sval1, &len1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get string value of [%s] in %s field: %s\n", + printf("Error: cannot get string value of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err1)); save_error(c, name); } if ((err2 = grib_get_string(h2, name, sval2, &len2)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get string value of [%s] in %s field: %s\n", + printf("Error: cannot get string value of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err2)); save_error(c, name); } @@ -875,14 +875,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err1 = grib_get_long_array(h1, name, lval1, &len1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get long value of [%s] in %s field: %s\n", + printf("Error: cannot get long value of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err1)); save_error(c, name); } if ((err2 = grib_get_long_array(h2, name, lval2, &len2)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get long value of [%s] in %s field: %s\n", + printf("Error: cannot get long value of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err2)); save_error(c, name); } @@ -1000,14 +1000,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err1 = grib_get_double_array(h1, name, dval1, &len1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get double value of [%s] in %s field: %s\n", + printf("Error: cannot get double value of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err1)); save_error(c, name); } if ((err2 = grib_get_double_array(h2, name, dval2, &len2)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get double value of [%s] in %s field: %s\n", + printf("Error: cannot get double value of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err2)); save_error(c, name); } @@ -1121,14 +1121,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err1 = grib_get_bytes(h1, name, uval1, &len1)) != GRIB_SUCCESS) { printInfo(h1); save_error(c, name); - printf("Oops... cannot get bytes value of [%s] in %s field: %s\n", + printf("Error: cannot get bytes value of [%s] in %s field: %s\n", name, first_str, grib_get_error_message(err1)); } if ((err2 = grib_get_bytes(h2, name, uval2, &len2)) != GRIB_SUCCESS) { printInfo(h1); save_error(c, name); - printf("Oops... cannot get bytes value of [%s] in %s field: %s\n", + printf("Error: cannot get bytes value of [%s] in %s field: %s\n", name, second_str, grib_get_error_message(err2)); } diff --git a/tools/gts_compare.cc b/tools/gts_compare.cc index c9bb66af7..849f84554 100644 --- a/tools/gts_compare.cc +++ b/tools/gts_compare.cc @@ -406,7 +406,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if (type1 == GRIB_TYPE_UNDEFINED && (err = grib_get_native_type(h1, name, &type1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get type of [%s] in 1st field: %s\n", name, grib_get_error_message(err)); + printf("Error: cannot get type of [%s] in 1st field: %s\n", name, grib_get_error_message(err)); save_error(c, name); return err; } @@ -419,7 +419,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h return err; } printInfo(h1); - printf("Oops... cannot get type of [%s] in 2nd field: %s\n", name, grib_get_error_message(err)); + printf("Error: cannot get type of [%s] in 2nd field: %s\n", name, grib_get_error_message(err)); save_error(c, name); return err; } @@ -443,7 +443,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err = grib_get_size(h1, name, &len1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get size of [%s] in 1st field: %s\n", name, grib_get_error_message(err)); + printf("Error: cannot get size of [%s] in 1st field: %s\n", name, grib_get_error_message(err)); save_error(c, name); return err; } @@ -457,7 +457,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h } printInfo(h1); - printf("Oops... cannot get size of [%s] in 2nd field: %s\n", name, grib_get_error_message(err)); + printf("Error: cannot get size of [%s] in 2nd field: %s\n", name, grib_get_error_message(err)); save_error(c, name); return err; } @@ -515,14 +515,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err1 = grib_get_string(h1, name, sval1, &len1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get string value of [%s] in 1st field: %s\n", + printf("Error: cannot get string value of [%s] in 1st field: %s\n", name, grib_get_error_message(err1)); save_error(c, name); } if ((err2 = grib_get_string(h2, name, sval2, &len2)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get string value of [%s] in 2nd field: %s\n", + printf("Error: cannot get string value of [%s] in 2nd field: %s\n", name, grib_get_error_message(err2)); save_error(c, name); } @@ -556,14 +556,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err1 = grib_get_long_array(h1, name, lval1, &len1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get long value of [%s] in 1st field: %s\n", + printf("Error: cannot get long value of [%s] in 1st field: %s\n", name, grib_get_error_message(err1)); save_error(c, name); } if ((err2 = grib_get_long_array(h2, name, lval2, &len2)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get long value of [%s] in 2nd field: %s\n", + printf("Error: cannot get long value of [%s] in 2nd field: %s\n", name, grib_get_error_message(err2)); save_error(c, name); } @@ -625,14 +625,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err1 = grib_get_bytes(h1, name, uval1, &len1)) != GRIB_SUCCESS) { printInfo(h1); save_error(c, name); - printf("Oops... cannot get bytes value of [%s] in 1st field: %s\n", + printf("Error: cannot get bytes value of [%s] in 1st field: %s\n", name, grib_get_error_message(err1)); } if ((err2 = grib_get_bytes(h2, name, uval2, &len2)) != GRIB_SUCCESS) { printInfo(h1); save_error(c, name); - printf("Oops... cannot get bytes value of [%s] in 2nd field: %s\n", + printf("Error: cannot get bytes value of [%s] in 2nd field: %s\n", name, grib_get_error_message(err2)); } diff --git a/tools/metar_compare.cc b/tools/metar_compare.cc index c275273bb..c8e824387 100644 --- a/tools/metar_compare.cc +++ b/tools/metar_compare.cc @@ -478,7 +478,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if (type1 == GRIB_TYPE_UNDEFINED && (err = grib_get_native_type(h1, name, &type1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get type of [%s] in 1st field: %s\n", name, grib_get_error_message(err)); + printf("Error: cannot get type of [%s] in 1st field: %s\n", name, grib_get_error_message(err)); save_error(c, name); return err; } @@ -491,7 +491,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h return err; } printInfo(h1); - printf("Oops... cannot get type of [%s] in 2nd field: %s\n", name, grib_get_error_message(err)); + printf("Error: cannot get type of [%s] in 2nd field: %s\n", name, grib_get_error_message(err)); save_error(c, name); return err; } @@ -505,7 +505,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err = grib_get_size(h1, name, &len1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get size of [%s] in 1st field: %s\n", name, grib_get_error_message(err)); + printf("Error: cannot get size of [%s] in 1st field: %s\n", name, grib_get_error_message(err)); save_error(c, name); return err; } @@ -519,7 +519,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h } printInfo(h1); - printf("Oops... cannot get size of [%s] in 2nd field: %s\n", name, grib_get_error_message(err)); + printf("Error: cannot get size of [%s] in 2nd field: %s\n", name, grib_get_error_message(err)); save_error(c, name); return err; } @@ -567,14 +567,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err1 = grib_get_string(h1, name, sval1, &len1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get string value of [%s] in 1st field: %s\n", + printf("Error: cannot get string value of [%s] in 1st field: %s\n", name, grib_get_error_message(err1)); save_error(c, name); } if ((err2 = grib_get_string(h2, name, sval2, &len2)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get string value of [%s] in 2nd field: %s\n", + printf("Error: cannot get string value of [%s] in 2nd field: %s\n", name, grib_get_error_message(err2)); save_error(c, name); } @@ -608,14 +608,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err1 = grib_get_long_array(h1, name, lval1, &len1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get long value of [%s] in 1st field: %s\n", + printf("Error: cannot get long value of [%s] in 1st field: %s\n", name, grib_get_error_message(err1)); save_error(c, name); } if ((err2 = grib_get_long_array(h2, name, lval2, &len2)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get long value of [%s] in 2nd field: %s\n", + printf("Error: cannot get long value of [%s] in 2nd field: %s\n", name, grib_get_error_message(err2)); save_error(c, name); } @@ -674,14 +674,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err1 = grib_get_double_array(h1, name, dval1, &len1)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get double value of [%s] in 1st field: %s\n", + printf("Error: cannot get double value of [%s] in 1st field: %s\n", name, grib_get_error_message(err1)); save_error(c, name); } if ((err2 = grib_get_double_array(h2, name, dval2, &len2)) != GRIB_SUCCESS) { printInfo(h1); - printf("Oops... cannot get double value of [%s] in 2nd field: %s\n", + printf("Error: cannot get double value of [%s] in 2nd field: %s\n", name, grib_get_error_message(err2)); save_error(c, name); } @@ -770,14 +770,14 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h if ((err1 = grib_get_bytes(h1, name, uval1, &len1)) != GRIB_SUCCESS) { printInfo(h1); save_error(c, name); - printf("Oops... cannot get bytes value of [%s] in 1st field: %s\n", + printf("Error: cannot get bytes value of [%s] in 1st field: %s\n", name, grib_get_error_message(err1)); } if ((err2 = grib_get_bytes(h2, name, uval2, &len2)) != GRIB_SUCCESS) { printInfo(h1); save_error(c, name); - printf("Oops... cannot get bytes value of [%s] in 2nd field: %s\n", + printf("Error: cannot get bytes value of [%s] in 2nd field: %s\n", name, grib_get_error_message(err2)); } From f80aa163f1e2345551b6e2d8248df490aae5dffb Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 3 Jan 2024 13:17:53 +0000 Subject: [PATCH 10/40] Testing: compare with -f --- tests/bufr_compare.sh | 11 +++++++++++ tests/grib_compare.sh | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/tests/bufr_compare.sh b/tests/bufr_compare.sh index fb0b11ea9..56fc335f8 100755 --- a/tests/bufr_compare.sh +++ b/tests/bufr_compare.sh @@ -328,6 +328,17 @@ set -e [ $status -ne 0 ] grep -q "Failed to unpack 2nd message" $fLog +# ---------------------------------------- +# Summary mode (-f) +# ---------------------------------------- +set +e +${tools_dir}/bufr_compare -f aaen_55.bufr aben_55.bufr > $fLog 2>&1 +status=$? +set -e +[ $status -eq 1 ] +grep -q "Summary of different key values" $fLog + + # Clean up # ------------- diff --git a/tests/grib_compare.sh b/tests/grib_compare.sh index 7b813dc74..65cbd2bb7 100755 --- a/tests/grib_compare.sh +++ b/tests/grib_compare.sh @@ -175,6 +175,16 @@ ${tools_dir}/grib_compare -b $BLACKLIST -R all=2 $temp1 $temp2 cp ${data_dir}/tigge_cf_ecmwf.grib2 $temp1 ${tools_dir}/grib_compare -w typeOfLevel=surface ${data_dir}/tigge_cf_ecmwf.grib2 $temp1 +# ---------------------------------------- +# Summary mode (-f) +# ---------------------------------------- +set +e +${tools_dir}/grib_compare -f ${data_dir}/tigge_cf_ecmwf.grib2 ${data_dir}/tigge_pf_ecmwf.grib2 > $outfile 2>&1 +status=$? +set -e +[ $status -eq 1 ] +grep -q "indicatorOfUnitForTimeIncrement . 7 different" $outfile + # ---------------------------------------- # ECC-651: Two-way (symmetric) comparison From 7e9a6b4078c76c08ac807320e6d968f490320852 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 3 Jan 2024 16:30:36 +0000 Subject: [PATCH 11/40] Tools: Error messages --- tools/grib_get.cc | 1 + tools/grib_ls.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/grib_get.cc b/tools/grib_get.cc index 6251c436f..19ae34a3e 100644 --- a/tools/grib_get.cc +++ b/tools/grib_get.cc @@ -113,6 +113,7 @@ int grib_tool_init(grib_runtime_options* options) grib_handle* hh; FILE* f = fopen(options->latlon_mask, "r"); if (!f) { + fprintf(stderr, "%s: unable to open mask file %s\n", tool_name, options->latlon_mask); perror(options->latlon_mask); exit(1); } diff --git a/tools/grib_ls.cc b/tools/grib_ls.cc index 2be9333e4..ab53e200f 100644 --- a/tools/grib_ls.cc +++ b/tools/grib_ls.cc @@ -137,6 +137,7 @@ int grib_tool_init(grib_runtime_options* options) int idx_overall = -1; FILE* f = fopen(options->latlon_mask, "r"); if (!f) { + fprintf(stderr, "%s: unable to open mask file %s\n", tool_name, options->latlon_mask); perror(options->latlon_mask); exit(1); } From 4ac57746267aa27aaa9efd52c8274c7eb61f7f6d Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 3 Jan 2024 16:31:07 +0000 Subject: [PATCH 12/40] Testing: Error conditions --- tests/grib_get_fail.sh | 30 ++++++++++++++++++++++++++++++ tests/grib_ls.sh | 10 ++++++++++ 2 files changed, 40 insertions(+) diff --git a/tests/grib_get_fail.sh b/tests/grib_get_fail.sh index f4a6102e3..d7d0d5b1b 100755 --- a/tests/grib_get_fail.sh +++ b/tests/grib_get_fail.sh @@ -50,6 +50,36 @@ set -e grep -q "Key Nj cannot be 0" $tempText +set +e +${tools_dir}/grib_get -l 0,0,5 $ECCODES_SAMPLES_PATH/reduced_ll_sfc_grib1.tmpl > $tempText 2>&1 +status=$? +set -e +[ $status -ne 0 ] +grep -q "Wrong mode given" $tempText + + +set +e +${tools_dir}/grib_get -l 0,0,1,nonexistingmask $ECCODES_SAMPLES_PATH/reduced_ll_sfc_grib1.tmpl > $tempText 2>&1 +status=$? +set -e +[ $status -ne 0 ] +cat $tempText +grep -q "unable to open mask file" $tempText + + +# ------------------------ +# Unreadable message +# ------------------------ +outfile=temp.$label.out +echo GRIB > $outfile +set +e +${tools_dir}/grib_get -p edition $outfile /dev/null > $tempText 2>&1 +status=$? +set -e +[ $status -ne 0 ] +grep -q "unreadable message" $tempText +rm -f $outfile + # Clean up rm -f $tempText diff --git a/tests/grib_ls.sh b/tests/grib_ls.sh index 2a6f6c5ab..1101d4d52 100755 --- a/tests/grib_ls.sh +++ b/tests/grib_ls.sh @@ -225,6 +225,8 @@ grib_check_key_equals $file 'expver:s' '0001' ${tools_dir}/grib_ls -j -l0,0 -p referenceValue:d $data_dir/sample.grib2 ${tools_dir}/grib_ls -j -l0,0 -p referenceValue:i $data_dir/sample.grib2 +${tools_dir}/grib_get -l0,0,4 $data_dir/sample.grib2 + set +e ${tools_dir}/grib_ls -l0,0,666 $data_dir/sample.grib2 > $tempText 2>&1 status=$? @@ -233,5 +235,13 @@ set -e grep -q "Wrong mode given" $tempText +set +e +${tools_dir}/grib_ls -l0,0,1,nonexistingmask $data_dir/sample.grib2 > $tempText 2>&1 +status=$? +set -e +[ $status -ne 0 ] +grep -q "unable to open mask file" $tempText + + # Clean up rm -f $temp1 $temp2 $tempText $tempLog From 0090d8f0cd9868af06609bd45a6020de8ad12a01 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 3 Jan 2024 16:59:58 +0000 Subject: [PATCH 13/40] Testing: gts_dump --- tests/CMakeLists.txt | 1 + tests/gts_dump.sh | 27 +++++++++++++++++++++++++++ tools/gts_dump.cc | 10 +--------- 3 files changed, 29 insertions(+), 9 deletions(-) create mode 100755 tests/gts_dump.sh diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4766e0f73..4e790b091 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -254,6 +254,7 @@ if( HAVE_BUILD_TOOLS ) gts_ls gts_count gts_compare + gts_dump wrap taf pseudo_diag diff --git a/tests/gts_dump.sh b/tests/gts_dump.sh new file mode 100755 index 000000000..b8c8db2e8 --- /dev/null +++ b/tests/gts_dump.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# (C) Copyright 2005- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +# + +. ./include.ctest.sh + +label="gts_dump_test" + +fLog=${label}.log +fTmp=${label}.tmp.txt + +# Enter data dir +cd ${data_dir}/gts + +gts_file=EGRR20150317121020_00493212.DAT +${tools_dir}/gts_dump -w count=1 $gts_file +${tools_dir}/gts_dump -Dat $gts_file +${tools_dir}/gts_dump -OH $gts_file + +# Clean up +rm -f $fLog $fTmp diff --git a/tools/gts_dump.cc b/tools/gts_dump.cc index 0f297393b..eff44c548 100644 --- a/tools/gts_dump.cc +++ b/tools/gts_dump.cc @@ -8,12 +8,6 @@ * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. */ -/* - * Implementation: gts_dump - * - * - */ - #include "grib_tools.h" grib_option grib_options[] = { @@ -41,7 +35,6 @@ const char* tool_usage = "[options] file file ..."; int grib_options_count = sizeof(grib_options) / sizeof(grib_option); /** -* gts_dump * Dump the content of a GTS file * */ @@ -57,11 +50,10 @@ int grib_tool_before_getopt(grib_runtime_options* options) int grib_tool_init(grib_runtime_options* options) { - int opt = grib_options_on("C") + grib_options_on("O") + grib_options_on("D"); + int opt = grib_options_on("O") + grib_options_on("D"); options->dump_mode = (char*)"default"; - if (opt > 1) { printf("%s: simultaneous O/D options not allowed\n", tool_name); exit(1); From c857a1d45faa11e07d880ef877a727c24d397aad Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 3 Jan 2024 17:13:38 +0000 Subject: [PATCH 14/40] Testing: metar_dump -j --- tests/metar_dump.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/metar_dump.sh b/tests/metar_dump.sh index 3be590228..1dfb10874 100755 --- a/tests/metar_dump.sh +++ b/tests/metar_dump.sh @@ -46,4 +46,9 @@ ${tools_dir}/metar_dump $f 2> $REDIRECT > $res_dump diff $ref_dump $res_dump >$REDIRECT 2> $REDIRECT +# JSON +${tools_dir}/metar_dump -j $f + + +# Clean up rm -f $fLog $res_dump From 58a3afbba818446b0632a13dbf85a3f134659529 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 3 Jan 2024 17:16:48 +0000 Subject: [PATCH 15/40] Testing: metar_dump -w --- tests/metar_dump.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/metar_dump.sh b/tests/metar_dump.sh index 1dfb10874..cd9074c22 100755 --- a/tests/metar_dump.sh +++ b/tests/metar_dump.sh @@ -49,6 +49,8 @@ diff $ref_dump $res_dump >$REDIRECT 2> $REDIRECT # JSON ${tools_dir}/metar_dump -j $f +# Skip +${tools_dir}/metar_dump -w count=11 $f # Clean up rm -f $fLog $res_dump From f05870d89a873ed62106f08e06c0e08ee996c8c8 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 3 Jan 2024 17:17:58 +0000 Subject: [PATCH 16/40] Testing: metar_dump -d --- tests/metar_dump.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/metar_dump.sh b/tests/metar_dump.sh index cd9074c22..86389a0e0 100755 --- a/tests/metar_dump.sh +++ b/tests/metar_dump.sh @@ -46,6 +46,9 @@ ${tools_dir}/metar_dump $f 2> $REDIRECT > $res_dump diff $ref_dump $res_dump >$REDIRECT 2> $REDIRECT +# Data +${tools_dir}/metar_dump -d $f + # JSON ${tools_dir}/metar_dump -j $f From 05f6952ab44f4d614c349351c26fe168ea306890 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 3 Jan 2024 17:35:09 +0000 Subject: [PATCH 17/40] Testing: grib_get_data --- tests/grib_iterator.sh | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/tests/grib_iterator.sh b/tests/grib_iterator.sh index 4b8eec5f7..393b3e680 100755 --- a/tests/grib_iterator.sh +++ b/tests/grib_iterator.sh @@ -51,7 +51,7 @@ ${tools_dir}/grib_set -s Ni=33 $samp_dir/GRIB2.tmpl $tempGrib set +e ${tools_dir}/grib_get_data $tempGrib > $tempText 2>&1 status=$? -set +e +set -e [ $status -ne 0 ] grep -q "Grid description is wrong or inconsistent" $tempText @@ -60,16 +60,44 @@ ${tools_dir}/grib_set -s Ni=MISSING $samp_dir/GRIB2.tmpl $tempGrib set +e ${tools_dir}/grib_get_data $tempGrib > $tempText 2>&1 status=$? -set +e +set -e [ $status -ne 0 ] grep -q "Grid description is wrong or inconsistent" $tempText +set +e ${tools_dir}/grib_ls -s Ni=missing -j -p latLonValues $data_dir/sample.grib2 > $tempText 2>&1 -cat $tempText +status=$? +set -e +[ $status -ne 0 ] grep -q "Key Ni cannot be 'missing' for a regular grid" $tempText grep -q "latlonvalues: Unable to create iterator" $tempText +# -w option +${tools_dir}/grib_get_data -w count=11 $data_dir/tigge_cf_ecmwf.grib2 > $tempText + + +# ------------------------ +# Bad key +# ------------------------ +${tools_dir}/grib_get_data -f -p nonexistingkey $data_dir/sample.grib2 > $tempText +grep -q "not found" $tempText + + +# ------------------------ +# Unreadable message +# ------------------------ +echo GRIB > $tempGrib +set +e +${tools_dir}/grib_get_data $tempGrib > $tempText 2>&1 +status=$? +set -e +[ $status -ne 0 ] +cat $tempText +grep -q "unreadable message" $tempText + + + # Clean up rm -f $tempText $tempGrib From 8709a6d32d0f7bc4d5e6a280647a39baa30311f3 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 5 Jan 2024 16:05:59 +0000 Subject: [PATCH 18/40] Testing: Gaussian sub-area legacy --- data/grib_data_files.txt | 2 ++ tests/grib_iterator.sh | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/data/grib_data_files.txt b/data/grib_data_files.txt index 0647dc142..f6d3a9124 100644 --- a/data/grib_data_files.txt +++ b/data/grib_data_files.txt @@ -90,3 +90,5 @@ gfs.complex.mvmu.grib2 mercator.grib2 run_length_packing.grib2 boustrophedonic.grib1 +reduced_gaussian_sub_area.legacy.grib1 + diff --git a/tests/grib_iterator.sh b/tests/grib_iterator.sh index 393b3e680..45d6543ca 100755 --- a/tests/grib_iterator.sh +++ b/tests/grib_iterator.sh @@ -26,11 +26,11 @@ files="reduced_latlon_surface.grib1 \ regular_latlon_surface.grib2" for f in $files; do - file=${data_dir}/$f - # Must exclude the first line of grib_get_data which is "Latitude Longitude Value" - iterator_count=`${tools_dir}/grib_get_data -m 9999:missing -f -p centre -F "%g" -w count=1 $file | grep -v Lat |wc -l ` - numberOfPoints=`${tools_dir}/grib_get -w count=1 -p numberOfPoints $file` - [ $numberOfPoints = ${iterator_count} ] + file=${data_dir}/$f + # Must exclude the first line of grib_get_data which is "Latitude Longitude Value" + iterator_count=`${tools_dir}/grib_get_data -m 9999:missing -f -p centre -F "%g" -w count=1 $file | grep -v Lat |wc -l ` + numberOfPoints=`${tools_dir}/grib_get -w count=1 -p numberOfPoints $file` + [ $numberOfPoints = ${iterator_count} ] done @@ -97,6 +97,16 @@ set -e cat $tempText grep -q "unreadable message" $tempText +# Legacy Gaussian sub-area (produced by old ProdGen) +# See ECC-906: +# grib_get_data not working correctly with old-style sub-areas of reduced grids +# ------------------------------------------------- +input=$data_dir/reduced_gaussian_sub_area.legacy.grib1 +${tools_dir}/grib_get_data $input > $tempText +grib_check_key_equals $input legacyGaussSubarea 1 + +ECCODES_DEBUG=-1 ${tools_dir}/grib_ls -p numberOfDataPoints $input > $tempText 2>&1 +grep -q "LEGACY MODE activated. Count.=253982. changed to num values.=254139" $tempText # Clean up From aca931a8012bca96c76575e493793b4e54e7317b Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 5 Jan 2024 16:53:52 +0000 Subject: [PATCH 19/40] Dead code removal --- src/eccodes_prototypes.h | 2 -- src/grib_value.cc | 39 +++++++++++++++++++-------------------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/eccodes_prototypes.h b/src/eccodes_prototypes.h index c6e6e9e17..0771c2656 100644 --- a/src/eccodes_prototypes.h +++ b/src/eccodes_prototypes.h @@ -1162,8 +1162,6 @@ int grib_set_float_array(grib_handle* h, const char* name, const float* val, siz int grib_set_long_array_internal(grib_handle* h, const char* name, const long* val, size_t length); int grib_set_long_array(grib_handle* h, const char* name, const long* val, size_t length); int grib_get_long_internal(grib_handle* h, const char* name, long* val); -int grib_is_in_dump(const grib_handle* h, const char* name); -int grib_attributes_count(const grib_accessor* a, size_t* size); int grib_get_long(const grib_handle* h, const char* name, long* val); int grib_get_double_internal(grib_handle* h, const char* name, double* val); int grib_get_double(const grib_handle* h, const char* name, double* val); diff --git a/src/grib_value.cc b/src/grib_value.cc index cac46c01a..50cdca970 100644 --- a/src/grib_value.cc +++ b/src/grib_value.cc @@ -951,27 +951,26 @@ int grib_get_long_internal(grib_handle* h, const char* name, long* val) return ret; } -int grib_is_in_dump(const grib_handle* h, const char* name) -{ - const grib_accessor* a = grib_find_accessor(h, name); - if (a != NULL && (a->flags & GRIB_ACCESSOR_FLAG_DUMP)) - return 1; - else - return 0; -} +// int grib_is_in_dump(const grib_handle* h, const char* name) +// { +// const grib_accessor* a = grib_find_accessor(h, name); +// if (a != NULL && (a->flags & GRIB_ACCESSOR_FLAG_DUMP)) +// return 1; +// else +// return 0; +// } -int grib_attributes_count(const grib_accessor* a, size_t* size) -{ - if (a) { - *size = 0; - while (a->attributes[*size] != NULL) { - (*size)++; - } - return GRIB_SUCCESS; - } - - return GRIB_NOT_FOUND; -} +// int grib_attributes_count(const grib_accessor* a, size_t* size) +// { +// if (a) { +// *size = 0; +// while (a->attributes[*size] != NULL) { +// (*size)++; +// } +// return GRIB_SUCCESS; +// } +// return GRIB_NOT_FOUND; +// } int grib_get_long(const grib_handle* h, const char* name, long* val) { From 731c329e41a111dc35c9ada4c1fa49710dc69c8e Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 5 Jan 2024 16:54:42 +0000 Subject: [PATCH 20/40] Testing: Iterator with DEBUG enabled --- tests/grib_iterator.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/grib_iterator.sh b/tests/grib_iterator.sh index 45d6543ca..8c9a4fbf8 100755 --- a/tests/grib_iterator.sh +++ b/tests/grib_iterator.sh @@ -102,11 +102,18 @@ grep -q "unreadable message" $tempText # grib_get_data not working correctly with old-style sub-areas of reduced grids # ------------------------------------------------- input=$data_dir/reduced_gaussian_sub_area.legacy.grib1 -${tools_dir}/grib_get_data $input > $tempText -grib_check_key_equals $input legacyGaussSubarea 1 +if [ -f "$input" ]; then + ${tools_dir}/grib_get_data $input > $tempText + grib_check_key_equals $input legacyGaussSubarea 1 -ECCODES_DEBUG=-1 ${tools_dir}/grib_ls -p numberOfDataPoints $input > $tempText 2>&1 -grep -q "LEGACY MODE activated. Count.=253982. changed to num values.=254139" $tempText + ECCODES_DEBUG=-1 ${tools_dir}/grib_ls -p numberOfDataPoints $input > $tempText 2>&1 + grep -q "LEGACY MODE activated. Count.=253982. changed to num values.=254139" $tempText +fi + +# Iterate with DEBUG on +input=$ECCODES_SAMPLES_PATH/reduced_gg_pl_32_grib2.tmpl +ECCODES_DEBUG=1 ${tools_dir}/grib_get_data $input > $tempText 2>&1 +grep "global num points=6114" $tempText # Clean up From 90ed7d1e670baf5e13ef15300a2d0343c8136474 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 5 Jan 2024 17:02:12 +0000 Subject: [PATCH 21/40] Dead code removal --- src/eccodes_prototypes.h | 1 - src/grib_value.cc | 152 +++++++++++++++++++-------------------- 2 files changed, 74 insertions(+), 79 deletions(-) diff --git a/src/eccodes_prototypes.h b/src/eccodes_prototypes.h index 0771c2656..d2d29d60e 100644 --- a/src/eccodes_prototypes.h +++ b/src/eccodes_prototypes.h @@ -1201,7 +1201,6 @@ int grib_get_long_array_internal(grib_handle* h, const char* name, long* val, si int grib_get_long_array(const grib_handle* h, const char* name, long* val, size_t* length); grib_key_value_list* grib_key_value_list_clone(grib_context* c, grib_key_value_list* list); void grib_key_value_list_delete(grib_context* c, grib_key_value_list* kvl); -int grib_get_key_value_list(grib_handle* h, grib_key_value_list* list); int grib_get_values(grib_handle* h, grib_values* args, size_t count); int grib_set_values(grib_handle* h, grib_values* args, size_t count); int grib_get_nearest_smaller_value(grib_handle* h, const char* name, double val, double* nearest); diff --git a/src/grib_value.cc b/src/grib_value.cc index 50cdca970..513adc2ac 100644 --- a/src/grib_value.cc +++ b/src/grib_value.cc @@ -1598,74 +1598,70 @@ static void grib_clean_key_value(grib_context* c, grib_key_value_list* kv) kv->size = 0; } -static int grib_get_key_value(grib_handle* h, grib_key_value_list* kv) -{ - int err = 0; - size_t size = 0; - grib_keys_iterator* iter = NULL; - grib_key_value_list* list = NULL; - - if (kv->has_value) - grib_clean_key_value(h->context, kv); - - err = grib_get_size(h, kv->name, &size); - if (err) { - kv->error = err; - return err; - } - if (size == 0) - size = 512; - - switch (kv->type) { - case GRIB_TYPE_LONG: - kv->long_value = (long*)grib_context_malloc_clear(h->context, size * sizeof(long)); - err = grib_get_long_array(h, kv->name, kv->long_value, &size); - kv->error = err; - break; - case GRIB_TYPE_DOUBLE: - kv->double_value = (double*)grib_context_malloc_clear(h->context, size * sizeof(double)); - err = grib_get_double_array(h, kv->name, kv->double_value, &size); - kv->error = err; - break; - case GRIB_TYPE_STRING: - grib_get_string_length(h, kv->name, &size); - kv->string_value = (char*)grib_context_malloc_clear(h->context, size * sizeof(char)); - err = grib_get_string(h, kv->name, kv->string_value, &size); - kv->error = err; - break; - case GRIB_TYPE_BYTES: - kv->string_value = (char*)grib_context_malloc_clear(h->context, size * sizeof(char)); - err = grib_get_bytes(h, kv->name, (unsigned char*)kv->string_value, &size); - kv->error = err; - break; - case CODES_NAMESPACE: - iter = grib_keys_iterator_new(h, 0, kv->name); - list = (grib_key_value_list*)grib_context_malloc_clear(h->context, sizeof(grib_key_value_list)); - kv->namespace_value = list; - while (grib_keys_iterator_next(iter)) { - list->name = grib_keys_iterator_get_name(iter); - err = grib_get_native_type(h, list->name, &(list->type)); - if (err) - return err; - err = grib_get_key_value(h, list); - if (err) - return err; - list->next = (grib_key_value_list*)grib_context_malloc_clear(h->context, sizeof(grib_key_value_list)); - list = list->next; - } - grib_keys_iterator_delete(iter); - break; - - default: - err = grib_get_native_type(h, kv->name, &(kv->type)); - if (err) - return err; - err = grib_get_key_value(h, kv); - break; - } - kv->has_value = 1; - return err; -} +// static int grib_get_key_value(grib_handle* h, grib_key_value_list* kv) +// { +// int err = 0; +// size_t size = 0; +// grib_keys_iterator* iter = NULL; +// grib_key_value_list* list = NULL; +// if (kv->has_value) +// grib_clean_key_value(h->context, kv); +// err = grib_get_size(h, kv->name, &size); +// if (err) { +// kv->error = err; +// return err; +// } +// if (size == 0) +// size = 512; +// switch (kv->type) { +// case GRIB_TYPE_LONG: +// kv->long_value = (long*)grib_context_malloc_clear(h->context, size * sizeof(long)); +// err = grib_get_long_array(h, kv->name, kv->long_value, &size); +// kv->error = err; +// break; +// case GRIB_TYPE_DOUBLE: +// kv->double_value = (double*)grib_context_malloc_clear(h->context, size * sizeof(double)); +// err = grib_get_double_array(h, kv->name, kv->double_value, &size); +// kv->error = err; +// break; +// case GRIB_TYPE_STRING: +// grib_get_string_length(h, kv->name, &size); +// kv->string_value = (char*)grib_context_malloc_clear(h->context, size * sizeof(char)); +// err = grib_get_string(h, kv->name, kv->string_value, &size); +// kv->error = err; +// break; +// case GRIB_TYPE_BYTES: +// kv->string_value = (char*)grib_context_malloc_clear(h->context, size * sizeof(char)); +// err = grib_get_bytes(h, kv->name, (unsigned char*)kv->string_value, &size); +// kv->error = err; +// break; +// case CODES_NAMESPACE: +// iter = grib_keys_iterator_new(h, 0, kv->name); +// list = (grib_key_value_list*)grib_context_malloc_clear(h->context, sizeof(grib_key_value_list)); +// kv->namespace_value = list; +// while (grib_keys_iterator_next(iter)) { +// list->name = grib_keys_iterator_get_name(iter); +// err = grib_get_native_type(h, list->name, &(list->type)); +// if (err) +// return err; +// err = grib_get_key_value(h, list); +// if (err) +// return err; +// list->next = (grib_key_value_list*)grib_context_malloc_clear(h->context, sizeof(grib_key_value_list)); +// list = list->next; +// } +// grib_keys_iterator_delete(iter); +// break; +// default: +// err = grib_get_native_type(h, kv->name, &(kv->type)); +// if (err) +// return err; +// err = grib_get_key_value(h, kv); +// break; +// } +// kv->has_value = 1; +// return err; +// } grib_key_value_list* grib_key_value_list_clone(grib_context* c, grib_key_value_list* list) { @@ -1696,16 +1692,16 @@ void grib_key_value_list_delete(grib_context* c, grib_key_value_list* kvl) } } -int grib_get_key_value_list(grib_handle* h, grib_key_value_list* list) -{ - int ret = 0; - grib_key_value_list* kvl = list; - while (kvl) { - ret = grib_get_key_value(h, kvl); - kvl = kvl->next; - } - return ret; -} +// int grib_get_key_value_list(grib_handle* h, grib_key_value_list* list) +// { +// int ret = 0; +// grib_key_value_list* kvl = list; +// while (kvl) { +// ret = grib_get_key_value(h, kvl); +// kvl = kvl->next; +// } +// return ret; +// } int grib_get_values(grib_handle* h, grib_values* args, size_t count) { From 1016c2e0f1aadde302ffc7cb3f7eced72a73bf62 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 5 Jan 2024 17:22:03 +0000 Subject: [PATCH 22/40] Dead code removal --- src/eccodes_prototypes.h | 2 - src/grib_value.cc | 88 ++++++++++++++++++++-------------------- 2 files changed, 43 insertions(+), 47 deletions(-) diff --git a/src/eccodes_prototypes.h b/src/eccodes_prototypes.h index d2d29d60e..ba44e941a 100644 --- a/src/eccodes_prototypes.h +++ b/src/eccodes_prototypes.h @@ -1199,8 +1199,6 @@ int grib_get_string_array(const grib_handle* h, const char* name, char** val, si int ecc__grib_get_long_array_internal(const grib_handle* h, grib_accessor* a, long* val, size_t buffer_len, size_t* decoded_length); int grib_get_long_array_internal(grib_handle* h, const char* name, long* val, size_t* length); int grib_get_long_array(const grib_handle* h, const char* name, long* val, size_t* length); -grib_key_value_list* grib_key_value_list_clone(grib_context* c, grib_key_value_list* list); -void grib_key_value_list_delete(grib_context* c, grib_key_value_list* kvl); int grib_get_values(grib_handle* h, grib_values* args, size_t count); int grib_set_values(grib_handle* h, grib_values* args, size_t count); int grib_get_nearest_smaller_value(grib_handle* h, const char* name, double val, double* nearest); diff --git a/src/grib_value.cc b/src/grib_value.cc index 513adc2ac..538e73c2a 100644 --- a/src/grib_value.cc +++ b/src/grib_value.cc @@ -1579,24 +1579,24 @@ int grib_get_long_array(const grib_handle* h, const char* name, long* val, size_ return ret; } -static void grib_clean_key_value(grib_context* c, grib_key_value_list* kv) -{ - if (kv->long_value) - grib_context_free(c, kv->long_value); - kv->long_value = NULL; - if (kv->double_value) - grib_context_free(c, kv->double_value); - kv->double_value = NULL; - if (kv->string_value) - grib_context_free(c, kv->string_value); - kv->string_value = NULL; - if (kv->namespace_value) - grib_key_value_list_delete(c, kv->namespace_value); - kv->namespace_value = NULL; - kv->error = 0; - kv->has_value = 0; - kv->size = 0; -} +// static void grib_clean_key_value(grib_context* c, grib_key_value_list* kv) +// { +// if (kv->long_value) +// grib_context_free(c, kv->long_value); +// kv->long_value = NULL; +// if (kv->double_value) +// grib_context_free(c, kv->double_value); +// kv->double_value = NULL; +// if (kv->string_value) +// grib_context_free(c, kv->string_value); +// kv->string_value = NULL; +// if (kv->namespace_value) +// grib_key_value_list_delete(c, kv->namespace_value); +// kv->namespace_value = NULL; +// kv->error = 0; +// kv->has_value = 0; +// kv->size = 0; +// } // static int grib_get_key_value(grib_handle* h, grib_key_value_list* kv) // { @@ -1663,34 +1663,32 @@ static void grib_clean_key_value(grib_context* c, grib_key_value_list* kv) // return err; // } -grib_key_value_list* grib_key_value_list_clone(grib_context* c, grib_key_value_list* list) -{ - grib_key_value_list* next = list; - grib_key_value_list* the_clone = (grib_key_value_list*)grib_context_malloc_clear(c, sizeof(grib_key_value_list)); - grib_key_value_list* p = the_clone; +// grib_key_value_list* grib_key_value_list_clone(grib_context* c, grib_key_value_list* list) +// { +// grib_key_value_list* next = list; +// grib_key_value_list* the_clone = (grib_key_value_list*)grib_context_malloc_clear(c, sizeof(grib_key_value_list)); +// grib_key_value_list* p = the_clone; +// while (next && next->name) { +// p->name = grib_context_strdup(c, next->name); +// p->type = next->type; +// next = next->next; +// } +// return the_clone; +// } - while (next && next->name) { - p->name = grib_context_strdup(c, next->name); - p->type = next->type; - next = next->next; - } - return the_clone; -} - -void grib_key_value_list_delete(grib_context* c, grib_key_value_list* kvl) -{ - grib_key_value_list* next = kvl; - grib_key_value_list* p = NULL; - while (next) { - p = next->next; - if (next->type == CODES_NAMESPACE) - grib_key_value_list_delete(c, next->namespace_value); - - grib_clean_key_value(c, next); - grib_context_free(c, next); - next = p; - } -} +// void grib_key_value_list_delete(grib_context* c, grib_key_value_list* kvl) +// { +// grib_key_value_list* next = kvl; +// grib_key_value_list* p = NULL; +// while (next) { +// p = next->next; +// if (next->type == CODES_NAMESPACE) +// grib_key_value_list_delete(c, next->namespace_value); +// grib_clean_key_value(c, next); +// grib_context_free(c, next); +// next = p; +// } +// } // int grib_get_key_value_list(grib_handle* h, grib_key_value_list* list) // { From 23854304362a69c605a61a14d5932085d161c412 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 5 Jan 2024 18:57:22 +0000 Subject: [PATCH 23/40] Testing: Fortran nearest --- examples/F90/CMakeLists.txt | 2 ++ examples/F90/grib_nearest_four_single.f90 | 31 +++++++++++++++++++++++ examples/F90/grib_nearest_four_single.sh | 12 +++++++++ examples/F90/grib_set_data_force.f90 | 15 +++++++---- 4 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 examples/F90/grib_nearest_four_single.f90 create mode 100755 examples/F90/grib_nearest_four_single.sh diff --git a/examples/F90/CMakeLists.txt b/examples/F90/CMakeLists.txt index 880a47f37..738ccec6e 100644 --- a/examples/F90/CMakeLists.txt +++ b/examples/F90/CMakeLists.txt @@ -37,6 +37,7 @@ if( HAVE_BUILD_TOOLS ) grib_elements grib_nearest grib_nearest_single + grib_nearest_four_single grib_precision grib_print_data grib_set_keys @@ -90,6 +91,7 @@ else() grib_nearest grib_elements grib_nearest_single + grib_nearest_four_single grib_precision grib_print_data grib_set_missing diff --git a/examples/F90/grib_nearest_four_single.f90 b/examples/F90/grib_nearest_four_single.f90 new file mode 100644 index 000000000..674745bfd --- /dev/null +++ b/examples/F90/grib_nearest_four_single.f90 @@ -0,0 +1,31 @@ +! (C) Copyright 2005- ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! +! In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +! virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +! +program find_nearest_4single + use eccodes + implicit none + integer :: infile, i + integer :: igrib + real(8) :: inlat = 5, inlon = 10 + real(8) :: outlats(4), outlons(4) + real(8) :: values(4), distances(4) + integer(kind=kindOfInt) :: indexes(4) + + call codes_open_file(infile, '../../data/reduced_gaussian_lsm.grib1', 'r') + call codes_grib_new_from_file(infile, igrib) + + call codes_grib_find_nearest_four_single(igrib, .true., inlat, inlon, outlats, outlons, values, distances, indexes) + call codes_release(igrib) + + call codes_close_file(infile) + + print *, ' outlats outlons values distances indexes' + do i = 1, 4 + write (*, '(F10.3, F10.3, F10.5, F10.3, I8)') outlats(i), outlons(i), values(i), distances(i), indexes(i) + end do +end program diff --git a/examples/F90/grib_nearest_four_single.sh b/examples/F90/grib_nearest_four_single.sh new file mode 100755 index 000000000..22b07624f --- /dev/null +++ b/examples/F90/grib_nearest_four_single.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# (C) Copyright 2005- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. + +. ./include.ctest.sh + +${examples_dir}/eccodes_f_grib_nearest_four_single diff --git a/examples/F90/grib_set_data_force.f90 b/examples/F90/grib_set_data_force.f90 index a5a3db2fc..d8762e649 100644 --- a/examples/F90/grib_set_data_force.f90 +++ b/examples/F90/grib_set_data_force.f90 @@ -13,7 +13,8 @@ program set_data_force integer :: outfile integer :: i, igrib, iret, numberOfValues, cnt real :: d, e - real, dimension(:), allocatable :: values + real(4), dimension(:), allocatable :: values_real4 + real(8), dimension(:), allocatable :: values_real8 integer, parameter :: max_strsize = 200 character(len=max_strsize) :: outfile_name @@ -24,7 +25,8 @@ program set_data_force call codes_get_size(igrib, 'values', numberOfValues) - allocate (values(numberOfValues), stat=iret) + allocate (values_real4(numberOfValues), stat=iret) + allocate (values_real8(numberOfValues), stat=iret) d = 10e-8 e = d cnt = 1 @@ -33,7 +35,8 @@ program set_data_force e = e*10 cnt = 1 end if - values(i) = d + values_real4(i) = d + values_real8(i) = d d = d + e cnt = cnt + 1 end do @@ -42,9 +45,11 @@ program set_data_force call codes_set(igrib, 'bitmapPresent', 1) ! set data values - call codes_set_force(igrib, 'codedValues', values) + call codes_set_force(igrib, 'codedValues', values_real4) + call codes_set_force(igrib, 'codedValues', values_real8) call codes_write(igrib, outfile) call codes_release(igrib) - deallocate (values) + deallocate (values_real4) + deallocate (values_real8) end program set_data_force From 634cff7c6c2bb21b70ed8f3c589fd4623ba09187 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 5 Jan 2024 19:49:41 +0000 Subject: [PATCH 24/40] Testing: Fortran index read --- examples/F90/grib_index.f90 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/F90/grib_index.f90 b/examples/F90/grib_index.f90 index d0d248978..1d928e2ea 100644 --- a/examples/F90/grib_index.f90 +++ b/examples/F90/grib_index.f90 @@ -23,7 +23,7 @@ program index character(len=20) :: oshortName integer :: shortNameSize, numberSize, levelSize, stepSize integer :: i, j, k, l - integer :: idx, igrib, count1 + integer :: idx, idx1, igrib, count1 character(len=10) :: index_file = 'index.idx' ! uncomment following line to load index from file @@ -107,7 +107,10 @@ program index ! save the index to a file for later reuse call codes_index_write(idx, index_file) + call codes_index_read(idx1, index_file) + call codes_index_release(idx) + call codes_index_release(idx1) deallocate (level) deallocate (shortName) From 0f3d2bf9d2a66a8451627d7d10ed81ae7e876682 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sat, 6 Jan 2024 13:54:53 +0000 Subject: [PATCH 25/40] Testing: codes_any_new_from_loaded --- examples/F90/CMakeLists.txt | 1 + examples/F90/codes_load_file.f90 | 43 ++++++++++++++++++++++++++++++++ examples/F90/codes_load_file.sh | 12 +++++++++ 3 files changed, 56 insertions(+) create mode 100644 examples/F90/codes_load_file.f90 create mode 100755 examples/F90/codes_load_file.sh diff --git a/examples/F90/CMakeLists.txt b/examples/F90/CMakeLists.txt index 738ccec6e..c8c291514 100644 --- a/examples/F90/CMakeLists.txt +++ b/examples/F90/CMakeLists.txt @@ -23,6 +23,7 @@ if( HAVE_BUILD_TOOLS ) grib_index codes_dump codes_scan_file + codes_load_file grib_copy_message bufr_copy_message grib_get_keys diff --git a/examples/F90/codes_load_file.f90 b/examples/F90/codes_load_file.f90 new file mode 100644 index 000000000..0c4d809dd --- /dev/null +++ b/examples/F90/codes_load_file.f90 @@ -0,0 +1,43 @@ +! (C) Copyright 2005- ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! +! In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +! virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +! +! +program codes_load_file + use eccodes + implicit none + + integer :: ifile, cnt, level, step + integer :: i, igrib, iret + character(len=32) :: infile_name = '../../data/index.grib' + + call codes_open_file(ifile, infile_name, 'r') + + call codes_any_load_all_from_file(ifile, cnt) + + i = 45 + call codes_any_new_from_loaded(i, igrib) + call codes_get(igrib, 'level', level) + call codes_get(igrib, 'stepRange', step) + + print *, 'Num messages=', cnt + print *, 'Msg ',i,' level=',level, ' step=', step + + call codes_release(igrib) + + ! Invalid msg number + i = 450 + call codes_any_new_from_loaded(i, igrib, iret) + if (iret /= GRIB_INVALID_ARGUMENT) then + call codes_check(iret, 'Error', 'codes_any_new_from_loaded should have failed') + else + print *,'Invalid message index returned error (as expected)' + end if + + call codes_close_file(ifile) + +end program diff --git a/examples/F90/codes_load_file.sh b/examples/F90/codes_load_file.sh new file mode 100755 index 000000000..77d5ac169 --- /dev/null +++ b/examples/F90/codes_load_file.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# (C) Copyright 2005- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. + +. ./include.ctest.sh + +${examples_dir}/eccodes_f_codes_load_file From 7bd5d6228baf499e2c4e131e1c189ca6ab45069e Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sat, 6 Jan 2024 13:55:05 +0000 Subject: [PATCH 26/40] Cleanup --- fortran/grib_fortran.c | 80 ++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/fortran/grib_fortran.c b/fortran/grib_fortran.c index 258b9a003..2c0cbb0d4 100644 --- a/fortran/grib_fortran.c +++ b/fortran/grib_fortran.c @@ -1439,7 +1439,7 @@ int any_f_scan_file_(int* fid, int* n) { grib_context* c=grib_context_get_default(); /* this needs a callback to a destructor*/ - /* grib_oarray_delete_content(c,binary_messages); */ + /* grib_oarray_delete_content(c, info_messages); */ grib_oarray_delete(c, info_messages); info_messages=grib_oarray_new(c, 1000, 1000); @@ -1500,64 +1500,65 @@ int any_f_new_from_scanned_file_(int* fid, int* msgid, int* gid) } /*****************************************************************************/ -int any_f_load_all_from_file_(int* fid,int* n) { +int any_f_load_all_from_file_(int* fid, int* n) { int err = 0; off_t offset=0; - void *data = NULL; + void* data = NULL; size_t olen = 0; l_binary_message* msg=0; FILE* f = get_file(*fid); - grib_context* c=grib_context_get_default(); + grib_context* c = grib_context_get_default(); /* this needs a callback to a destructor*/ - /* grib_oarray_delete_content(c,binary_messages); */ + /* grib_oarray_delete_content(c, binary_messages); */ - grib_oarray_delete(c,binary_messages); - binary_messages=grib_oarray_new(c,1000,1000); + grib_oarray_delete(c, binary_messages); + binary_messages = grib_oarray_new(c, 1000, 1000); if (f) { - while (err!=GRIB_END_OF_FILE) { - data = wmo_read_any_from_file_malloc ( f, 0,&olen,&offset,&err ); - msg=(l_binary_message*)grib_context_malloc_clear(c,sizeof(l_binary_message)); - msg->data=data; - msg->size=olen; + while (err != GRIB_END_OF_FILE) { + data = wmo_read_any_from_file_malloc (f, 0,&olen, &offset, &err); + msg = (l_binary_message*)grib_context_malloc_clear(c,sizeof(l_binary_message)); + msg->data = data; + msg->size = olen; - if (err==0 && data) grib_oarray_push(c,binary_messages,msg); + if (err == 0 && data) grib_oarray_push(c, binary_messages, msg); } - if (err==GRIB_END_OF_FILE) err=0; + if (err == GRIB_END_OF_FILE) err = 0; } - *n=binary_messages->n; + *n = binary_messages->n; return err; } /*****************************************************************************/ -int any_f_new_from_loaded_(int* msgid,int* gid) +int any_f_new_from_loaded_(int* msgid, int* gid) { - grib_handle *h = NULL; - grib_context* c=grib_context_get_default(); + grib_handle* h = NULL; + grib_context* c = grib_context_get_default(); /* fortran convention of 1 based index*/ - const int n=*msgid-1; + const int n = *msgid - 1; - l_binary_message* msg=(l_binary_message*)grib_oarray_get(binary_messages,n); + l_binary_message* msg = (l_binary_message*)grib_oarray_get(binary_messages, n); if (msg && msg->data) - h=grib_handle_new_from_message_copy (c,msg->data,msg->size); + h = grib_handle_new_from_message_copy(c, msg->data, msg->size); - if(h){ - push_handle(h,gid); + if (h) { + push_handle(h, gid); return GRIB_SUCCESS; - } else { - *gid=-1; + } + else { + *gid = -1; return GRIB_END_OF_FILE; } } /*****************************************************************************/ int codes_f_clear_loaded_from_file_(void) { - grib_context* c=grib_context_get_default(); + grib_context* c = grib_context_get_default(); /* grib_oarray_delete_content(c,binary_messages); */ - grib_oarray_delete(c,binary_messages); + grib_oarray_delete(c, binary_messages); return GRIB_SUCCESS; } @@ -1565,27 +1566,28 @@ int codes_f_clear_loaded_from_file_(void) { int grib_f_count_in_file_(int* fid,int* n) { int err = 0; FILE* f = get_file(*fid); - if (f) err=grib_count_in_file(0, f,n); + if (f) err = grib_count_in_file(0, f, n); return err; } /*****************************************************************************/ -int any_f_new_from_file_(int* fid, int* gid){ - int err = 0; - FILE* f = get_file(*fid); - grib_handle *h = NULL; +int any_f_new_from_file_(int* fid, int* gid) { + int err = 0; + FILE* f = get_file(*fid); + grib_handle* h = NULL; - if(f){ - h = codes_handle_new_from_file(0,f,PRODUCT_ANY,&err); - if(h){ - push_handle(h,gid); + if (f) { + h = codes_handle_new_from_file(0, f, PRODUCT_ANY, &err); + if (h) { + push_handle(h, gid); return GRIB_SUCCESS; - } else { - *gid=-1; + } + else { + *gid = -1; return GRIB_END_OF_FILE; } } - *gid=-1; + *gid = -1; return GRIB_INVALID_FILE; } From 3f266ab17f9551f7efcc49bb4f6a37550f7bccfc Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sat, 6 Jan 2024 13:55:19 +0000 Subject: [PATCH 27/40] Testing: Padding --- tests/grib_padding.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/grib_padding.sh b/tests/grib_padding.sh index daa64fe8f..6aeb757ec 100755 --- a/tests/grib_padding.sh +++ b/tests/grib_padding.sh @@ -15,6 +15,11 @@ REDIRECT=/dev/null tempGrib=temp.local.$label.grib1 tempFilt=temp.local.$label.filt +if [ $ECCODES_ON_WINDOWS -eq 1 ]; then + echo "$0: This test is currently disabled on Windows" + exit 0 +fi + ${tools_dir}/grib_set -s setLocalDefinition=1 ${data_dir}/regular_latlon_surface.grib1 $tempGrib cat > $tempFilt < Date: Sat, 6 Jan 2024 20:49:08 +0000 Subject: [PATCH 28/40] Testing: Fortran scan/load and GTS compare --- examples/F90/codes_load_file.f90 | 2 +- examples/F90/codes_load_file.sh | 11 +++- examples/F90/codes_scan_file.f90 | 10 ++-- examples/F90/codes_scan_file.sh | 7 ++- tests/gts_compare.sh | 14 +++++ tools/gts_compare.cc | 95 +++----------------------------- 6 files changed, 40 insertions(+), 99 deletions(-) diff --git a/examples/F90/codes_load_file.f90 b/examples/F90/codes_load_file.f90 index 0c4d809dd..3572b4b4d 100644 --- a/examples/F90/codes_load_file.f90 +++ b/examples/F90/codes_load_file.f90 @@ -13,7 +13,7 @@ program codes_load_file integer :: ifile, cnt, level, step integer :: i, igrib, iret - character(len=32) :: infile_name = '../../data/index.grib' + character(len=32) :: infile_name = '../../data/index.grib' call codes_open_file(ifile, infile_name, 'r') diff --git a/examples/F90/codes_load_file.sh b/examples/F90/codes_load_file.sh index 77d5ac169..6a4fa78d8 100755 --- a/examples/F90/codes_load_file.sh +++ b/examples/F90/codes_load_file.sh @@ -9,4 +9,13 @@ . ./include.ctest.sh -${examples_dir}/eccodes_f_codes_load_file +label='eccodes_f_codes_load_file' +temp=temp.$label.txt + +# The input file is hard coded => data/index.grib +${examples_dir}/eccodes_f_codes_load_file > $temp + +grep -q "Num messages= *384" $temp +grep -q "level= *700 step= *60" $temp + +rm -f $temp diff --git a/examples/F90/codes_scan_file.f90 b/examples/F90/codes_scan_file.f90 index 7c0438275..9c21ef77b 100644 --- a/examples/F90/codes_scan_file.f90 +++ b/examples/F90/codes_scan_file.f90 @@ -10,12 +10,10 @@ program codes_scan_file use eccodes implicit none - integer, parameter :: max_strsize = 200 - integer :: ifile, cnt, level, step - integer :: i, igrib, iret - character(len=max_strsize) :: infile_name - call getarg(1, infile_name) + integer :: ifile, cnt, level, step + integer :: i, igrib, iret + character(len=32) :: infile_name = '../../data/index.grib' call codes_open_file(ifile, infile_name, 'r') @@ -35,7 +33,7 @@ program codes_scan_file i = 450 call codes_any_new_from_scanned_file(ifile, i, igrib, iret) if (iret /= GRIB_INVALID_ARGUMENT) then - call codes_check(iret, 'codes_any_new_from_scanned_file', 'exit') + call codes_check(iret, 'Error', 'codes_any_new_from_scanned_file should have failed') else print *,'Invalid message index returned error (as expected)' end if diff --git a/examples/F90/codes_scan_file.sh b/examples/F90/codes_scan_file.sh index 80ca95bc3..4df8398b5 100755 --- a/examples/F90/codes_scan_file.sh +++ b/examples/F90/codes_scan_file.sh @@ -9,10 +9,11 @@ . ./include.ctest.sh -temp='temp.eccodes_f_codes_scan_file.txt' +label='eccodes_f_codes_scan_file' +temp=temp.$label.txt -input=../../data/index.grib -${examples_dir}/eccodes_f_codes_scan_file $input > $temp +# The input file is hard coded => data/index.grib +${examples_dir}/eccodes_f_codes_scan_file > $temp grep -q "Num messages= *384" $temp grep -q "level= *700 step= *60" $temp diff --git a/tests/gts_compare.sh b/tests/gts_compare.sh index 6b57e0369..0860530c3 100755 --- a/tests/gts_compare.sh +++ b/tests/gts_compare.sh @@ -73,6 +73,20 @@ temp2=temp.$label.2.gts ${tools_dir}/gts_copy -w count=1 $gts_file $temp1 ${tools_dir}/gts_copy -w count=4 $gts_file $temp2 ${tools_dir}/gts_compare -c theMessage $temp1 $temp2 +${tools_dir}/gts_compare -c theMessage -a $temp1 $temp2 + +set +e +${tools_dir}/gts_compare -c ls:n $temp1 $temp2 +status=$? +set -e +[ $status -eq 1 ] + +set +e +${tools_dir}/gts_compare -c ls:n -a $temp1 $temp2 +status=$? +set -e +[ $status -eq 1 ] + rm -f $temp1 $temp2 #---------------------------------------------------- diff --git a/tools/gts_compare.cc b/tools/gts_compare.cc index 849f84554..2d3392526 100644 --- a/tools/gts_compare.cc +++ b/tools/gts_compare.cc @@ -395,7 +395,6 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h int isMissing1 = 0, isMissing2 = 0; char *sval1 = NULL, *sval2 = NULL; - unsigned char *uval1 = NULL, *uval2 = NULL; long *lval1 = NULL, *lval2 = NULL; grib_context* c = h1->context; @@ -424,23 +423,12 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h return err; } - /* - if(type1 != type2) - { - printInfo(h1); - printf("Warning, [%s] has different types: 1st field: [%s], 2nd field: [%s]\n", - name,grib_get_type_name(type1),grib_get_type_name(type2)); - return GRIB_TYPE_MISMATCH; - } - */ - if (type1 == GRIB_TYPE_LABEL) return err; if (type1 == GRIB_TYPE_SECTION) return err; - if ((err = grib_get_size(h1, name, &len1)) != GRIB_SUCCESS) { printInfo(h1); printf("Error: cannot get size of [%s] in 1st field: %s\n", name, grib_get_error_message(err)); @@ -462,16 +450,6 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h return err; } - /* - if(len1 != len2 && type1 != GRIB_TYPE_STRING) - { - printInfo(h1); - printf("[%s] has different size: 1st field: %ld, 2nd field: %ld\n",name,(long)len1,(long)len2); - save_error(c,name); - return GRIB_COUNT_MISMATCH; - } - */ - if (options->mode != MODE_GTS) { /* TODO: Ignore missing values for keys in GTS. Not yet implemented */ isMissing1 = ((grib_is_missing(h1, name, &err1) == 1) && (err1 == 0)) ? 1 : 0; @@ -485,8 +463,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h } if (isMissing1 == 1) { - if (verbose) - printf(" is set to missing in 1st field\n"); + if (verbose) printf(" is set to missing in 1st field\n"); printInfo(h1); printf("%s is set to missing in 1st field but is not missing in 2nd field\n", name); err1 = GRIB_VALUE_MISMATCH; @@ -495,8 +472,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h } if (isMissing2 == 1) { - if (verbose) - printf(" is set to missing in 1st field\n"); + if (verbose) printf(" is set to missing in 1st field\n"); printInfo(h1); printf("%s is set to missing in 2nd field but is not missing in 1st field\n", name); err1 = GRIB_VALUE_MISMATCH; @@ -506,8 +482,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h switch (type1) { case GRIB_TYPE_STRING: - if (verbose) - printf(" as string\n"); + if (verbose) printf(" as string\n"); grib_get_string_length(h1, name, &len1); grib_get_string_length(h2, name, &len2); sval1 = (char*)grib_context_malloc(h1->context, len1 * sizeof(char)); @@ -548,8 +523,7 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h break; case GRIB_TYPE_LONG: - if (verbose) - printf(" as long\n"); + if (verbose) printf(" as long\n"); lval1 = (long*)grib_context_malloc(h1->context, len1 * sizeof(long)); lval2 = (long*)grib_context_malloc(h2->context, len2 * sizeof(long)); @@ -609,71 +583,19 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h break; case GRIB_TYPE_BYTES: - if (verbose) - printf(" as bytes\n"); + if (verbose) printf(" as bytes\n"); if (options->mode == MODE_GTS) { // We do not want to compare the message itself return 0; } - if (len1 < 2) - len1 = 512; - if (len2 < 2) - len2 = 512; - uval1 = (unsigned char*)grib_context_malloc(h1->context, len1 * sizeof(unsigned char)); - uval2 = (unsigned char*)grib_context_malloc(h2->context, len2 * sizeof(unsigned char)); - - if ((err1 = grib_get_bytes(h1, name, uval1, &len1)) != GRIB_SUCCESS) { - printInfo(h1); - save_error(c, name); - printf("Error: cannot get bytes value of [%s] in 1st field: %s\n", - name, grib_get_error_message(err1)); - } - - if ((err2 = grib_get_bytes(h2, name, uval2, &len2)) != GRIB_SUCCESS) { - printInfo(h1); - save_error(c, name); - printf("Error: cannot get bytes value of [%s] in 2nd field: %s\n", - name, grib_get_error_message(err2)); - } - - if (err1 == GRIB_SUCCESS && err2 == GRIB_SUCCESS) { - if (memcmp(uval1, uval2, len1) != 0) { - size_t i; - for (i = 0; i < len1; i++) - if (uval1[i] != uval2[i]) { - printInfo(h1); - save_error(c, name); - if (len1 == 1) - printf("[%s] byte values are different: [%02x] and [%02x]\n", - name, uval1[i], uval2[i]); - else - printf("[%s] byte value %zu of %ld are different: [%02x] and [%02x]\n", - name, i, (long)len1, uval1[i], uval2[i]); - - err1 = GRIB_VALUE_MISMATCH; - break; - } - err1 = GRIB_VALUE_MISMATCH; - } - } - - grib_context_free(h1->context, uval1); - grib_context_free(h2->context, uval2); - - if (err1) - return err1; - if (err2) - return err2; break; case GRIB_TYPE_LABEL: - if (verbose) - printf(" as label\n"); + if (verbose) printf(" as label\n"); break; default: - if (verbose) - printf("\n"); + if (verbose) printf("\n"); printInfo(h1); save_error(c, name); printf("Cannot compare [%s], unsupported type %d\n", name, type1); @@ -699,7 +621,6 @@ static int compare_all_dump_keys(grib_handle* h1, grib_handle* h2, grib_runtime_ while (grib_keys_iterator_next(iter)) { grib_accessor* xa = grib_keys_iterator_get_accessor(iter); name = grib_keys_iterator_get_name(iter); - /* printf("----- comparing %s\n",name); */ if (blocklisted(name)) continue; @@ -735,7 +656,6 @@ static int compare_handles(grib_handle* h1, grib_handle* h2, grib_runtime_option } while (grib_keys_iterator_next(iter)) { name = grib_keys_iterator_get_name(iter); - /*printf("----- comparing %s\n",name);*/ if (blocklisted(name)) continue; @@ -796,7 +716,6 @@ static int compare_handles(grib_handle* h1, grib_handle* h2, grib_runtime_option } while (grib_keys_iterator_next(iter)) { name = grib_keys_iterator_get_name(iter); - /*printf("----- comparing %s\n",name);*/ if (blocklisted(name)) continue; From 1458690dcf85f2bdabfcfcdd6cf4ae7e1e7dd68a Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sat, 6 Jan 2024 21:35:38 +0000 Subject: [PATCH 29/40] Tools: Dead code removal --- tools/gts_compare.cc | 2 +- tools/metar_compare.cc | 51 +----------------------------------------- 2 files changed, 2 insertions(+), 51 deletions(-) diff --git a/tools/gts_compare.cc b/tools/gts_compare.cc index 2d3392526..b7692f65a 100644 --- a/tools/gts_compare.cc +++ b/tools/gts_compare.cc @@ -383,7 +383,7 @@ static void save_error(grib_context* c, const char* key) } } -static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_handle* h2, const char* name, int type) +static int compare_values(const grib_runtime_options* options, grib_handle* h1, grib_handle* h2, const char* name, int type) { size_t len1 = 0; size_t len2 = 0; diff --git a/tools/metar_compare.cc b/tools/metar_compare.cc index c8e824387..48db23534 100644 --- a/tools/metar_compare.cc +++ b/tools/metar_compare.cc @@ -452,7 +452,7 @@ static void save_error(grib_context* c, const char* key) } } -static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_handle* h2, const char* name, int type) +static int compare_values(const grib_runtime_options* options, grib_handle* h1, grib_handle* h2, const char* name, int type) { size_t len1 = 0; size_t len2 = 0; @@ -464,7 +464,6 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h int isMissing1 = 0, isMissing2 = 0; char *sval1 = NULL, *sval2 = NULL; - unsigned char *uval1 = NULL, *uval2 = NULL; double *dval1 = NULL, *dval2 = NULL; long *lval1 = NULL, *lval2 = NULL; double maxdiff = 0; @@ -760,54 +759,6 @@ static int compare_values(grib_runtime_options* options, grib_handle* h1, grib_h printf(" as bytes\n"); if (options->mode == MODE_METAR) return 0; - if (len1 < 2) - len1 = 512; - if (len2 < 2) - len2 = 512; - uval1 = (unsigned char*)grib_context_malloc(h1->context, len1 * sizeof(unsigned char)); - uval2 = (unsigned char*)grib_context_malloc(h2->context, len2 * sizeof(unsigned char)); - - if ((err1 = grib_get_bytes(h1, name, uval1, &len1)) != GRIB_SUCCESS) { - printInfo(h1); - save_error(c, name); - printf("Error: cannot get bytes value of [%s] in 1st field: %s\n", - name, grib_get_error_message(err1)); - } - - if ((err2 = grib_get_bytes(h2, name, uval2, &len2)) != GRIB_SUCCESS) { - printInfo(h1); - save_error(c, name); - printf("Error: cannot get bytes value of [%s] in 2nd field: %s\n", - name, grib_get_error_message(err2)); - } - - if (err1 == GRIB_SUCCESS && err2 == GRIB_SUCCESS) { - if (memcmp(uval1, uval2, len1) != 0) { - for (i = 0; i < len1; i++) - if (uval1[i] != uval2[i]) { - printInfo(h1); - save_error(c, name); - if (len1 == 1) - printf("[%s] byte values are different: [%02x] and [%02x]\n", - name, uval1[i], uval2[i]); - else - printf("[%s] byte value %d of %ld are different: [%02x] and [%02x]\n", - name, i, (long)len1, uval1[i], uval2[i]); - - err1 = GRIB_VALUE_MISMATCH; - break; - } - err1 = GRIB_VALUE_MISMATCH; - } - } - - grib_context_free(h1->context, uval1); - grib_context_free(h2->context, uval2); - - if (err1) - return err1; - if (err2) - return err2; break; case GRIB_TYPE_LABEL: From 25ca99dc0baee368b4c04873f673f72e4a7b90c7 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sun, 7 Jan 2024 13:22:05 +0000 Subject: [PATCH 30/40] Testing: HDF5 --- samples/hdf5.tmpl | Bin 0 -> 342 bytes tests/CMakeLists.txt | 1 + tests/hdf5.sh | 29 +++++++++++++++++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 samples/hdf5.tmpl create mode 100755 tests/hdf5.sh diff --git a/samples/hdf5.tmpl b/samples/hdf5.tmpl new file mode 100644 index 0000000000000000000000000000000000000000..a517bc7dd5eb27f84006276c1b3f631dcd906ef7 GIT binary patch literal 342 zcmeD5aB<`1lHy|G;9!7(|4VzTjB5F62zpyp}WW%8CNg~fz-e>!e}lg q2A~maybM5!$=&cU)EowP7=vQkHi+XJ8BSt|px9=d2F%mquK@tT?mdkF literal 0 HcmV?d00001 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4e790b091..a02568dbf 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -256,6 +256,7 @@ if( HAVE_BUILD_TOOLS ) gts_compare gts_dump wrap + hdf5 taf pseudo_diag metar_ls diff --git a/tests/hdf5.sh b/tests/hdf5.sh new file mode 100755 index 000000000..1f8d90a72 --- /dev/null +++ b/tests/hdf5.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# (C) Copyright 2005- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +# + +. ./include.ctest.sh + +label="hdf5_test" + +tempOut=temp.${label}.out +tempTxt=temp.${label}.txt +tempRef=temp.${label}.ref + +input=$ECCODES_SAMPLES_PATH/hdf5.tmpl + +${tools_dir}/grib_dump -TA -O $input > $tempTxt +grep -q "versionNumberOfSuperblock = 2" $tempTxt +grep -q "endOfFileAddress = 342" $tempTxt + +id=`${tools_dir}/grib_get -TA -p identifier $input` +[ "$id" = "HDF5" ] + +# Clean up +rm -f $tempOut $tempRef $tempTxt From b5470b229742f564a579680d386e1807fb72051e Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sun, 7 Jan 2024 14:07:23 +0000 Subject: [PATCH 31/40] Dead code removal: TAF and METAR --- src/eccodes_prototypes.h | 2 - src/grib_io.cc | 91 +++++++++++++++++++--------------------- 2 files changed, 42 insertions(+), 51 deletions(-) diff --git a/src/eccodes_prototypes.h b/src/eccodes_prototypes.h index ba44e941a..ba88a265f 100644 --- a/src/eccodes_prototypes.h +++ b/src/eccodes_prototypes.h @@ -1020,8 +1020,6 @@ int wmo_read_any_from_file(FILE* f, void* buffer, size_t* len); int wmo_read_grib_from_file(FILE* f, void* buffer, size_t* len); int wmo_read_bufr_from_file(FILE* f, void* buffer, size_t* len); int wmo_read_gts_from_file(FILE* f, void* buffer, size_t* len); -int wmo_read_taf_from_file(FILE* f, void* buffer, size_t* len); -int wmo_read_metar_from_file(FILE* f, void* buffer, size_t* len); int wmo_read_any_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset); int wmo_read_grib_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset); int wmo_read_bufr_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset); diff --git a/src/grib_io.cc b/src/grib_io.cc index ce8d2aa65..978885c09 100644 --- a/src/grib_io.cc +++ b/src/grib_io.cc @@ -1267,57 +1267,50 @@ int wmo_read_gts_from_file(FILE* f, void* buffer, size_t* len) return err; } -int wmo_read_taf_from_file(FILE* f, void* buffer, size_t* len) -{ - int err; - user_buffer_t u; - reader r; +// int wmo_read_taf_from_file(FILE* f, void* buffer, size_t* len) +// { +// int err; +// user_buffer_t u; +// reader r; +// u.user_buffer = buffer; +// u.buffer_size = *len; - u.user_buffer = buffer; - u.buffer_size = *len; +// r.read_data = f; +// r.read = &stdio_read; +// r.alloc_data = &u; +// r.alloc = &user_provider_buffer; +// r.headers_only = 0; +// r.seek = &stdio_seek; +// r.seek_from_start = &stdio_seek_from_start; +// r.tell = &stdio_tell; +// r.offset = 0; +// r.message_size = 0; +// err = read_any_taf(&r); +// *len = r.message_size; +// return err; +// } - r.read_data = f; - r.read = &stdio_read; - r.alloc_data = &u; - r.alloc = &user_provider_buffer; - r.headers_only = 0; - r.seek = &stdio_seek; - r.seek_from_start = &stdio_seek_from_start; - r.tell = &stdio_tell; - r.offset = 0; - r.message_size = 0; - - err = read_any_taf(&r); - *len = r.message_size; - - return err; -} - -int wmo_read_metar_from_file(FILE* f, void* buffer, size_t* len) -{ - int err; - user_buffer_t u; - reader r; - - u.user_buffer = buffer; - u.buffer_size = *len; - - r.read_data = f; - r.read = &stdio_read; - r.alloc_data = &u; - r.alloc = &user_provider_buffer; - r.headers_only = 0; - r.seek = &stdio_seek; - r.seek_from_start = &stdio_seek_from_start; - r.tell = &stdio_tell; - r.offset = 0; - r.message_size = 0; - - err = read_any_metar(&r); - *len = r.message_size; - - return err; -} +// int wmo_read_metar_from_file(FILE* f, void* buffer, size_t* len) +// { +// int err; +// user_buffer_t u; +// reader r; +// u.user_buffer = buffer; +// u.buffer_size = *len; +// r.read_data = f; +// r.read = &stdio_read; +// r.alloc_data = &u; +// r.alloc = &user_provider_buffer; +// r.headers_only = 0; +// r.seek = &stdio_seek; +// r.seek_from_start = &stdio_seek_from_start; +// r.tell = &stdio_tell; +// r.offset = 0; +// r.message_size = 0; +// err = read_any_metar(&r); +// *len = r.message_size; +// return err; +// } /*================== */ From 5eb9379428b50c9ae354bb3a104b4b2c8999ac23 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sun, 7 Jan 2024 14:10:33 +0000 Subject: [PATCH 32/40] Testing: Fix for Windows --- tests/hdf5.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/hdf5.sh b/tests/hdf5.sh index 1f8d90a72..c1c41597e 100755 --- a/tests/hdf5.sh +++ b/tests/hdf5.sh @@ -12,6 +12,11 @@ label="hdf5_test" +if [ $ECCODES_ON_WINDOWS -eq 1 ]; then + echo "$0: This test is currently disabled on Windows" + exit 0 +fi + tempOut=temp.${label}.out tempTxt=temp.${label}.txt tempRef=temp.${label}.ref From 28b7d13eea65ad8b35df5f838a8f2945eff567cd Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sun, 7 Jan 2024 15:26:58 +0000 Subject: [PATCH 33/40] Testing: grib_repair/grib_read_any_from_memory --- src/grib_handle.cc | 2 ++ src/grib_io.cc | 2 -- tests/CMakeLists.txt | 1 + tests/grib_repair.sh | 25 +++++++++++++++++++++++++ 4 files changed, 28 insertions(+), 2 deletions(-) create mode 100755 tests/grib_repair.sh diff --git a/src/grib_handle.cc b/src/grib_handle.cc index d4c94e4e3..f476ab9f2 100644 --- a/src/grib_handle.cc +++ b/src/grib_handle.cc @@ -428,6 +428,8 @@ grib_handle* codes_handle_new_from_file(grib_context* c, FILE* f, ProductKind pr return metar_new_from_file(c, f, error); if (product == PRODUCT_GTS) return gts_new_from_file(c, f, error); + //if (product == PRODUCT_TAF) + // return taf_new_from_file(c, f, error); if (product == PRODUCT_ANY) return any_new_from_file(c, f, error); diff --git a/src/grib_io.cc b/src/grib_io.cc index 978885c09..d4d876cca 100644 --- a/src/grib_io.cc +++ b/src/grib_io.cc @@ -1274,7 +1274,6 @@ int wmo_read_gts_from_file(FILE* f, void* buffer, size_t* len) // reader r; // u.user_buffer = buffer; // u.buffer_size = *len; - // r.read_data = f; // r.read = &stdio_read; // r.alloc_data = &u; @@ -1724,7 +1723,6 @@ int grib_read_any_from_memory(grib_context* ctx, unsigned char** data, size_t* d err = read_any(&r, /*no_alloc=*/0, 1, ECCODES_READS_BUFR, ECCODES_READS_HDF5, ECCODES_READS_WRAP); *len = r.message_size; - *data_length = m.data_len; *data = m.data; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a02568dbf..7f36206d0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -160,6 +160,7 @@ if( HAVE_BUILD_TOOLS ) # and/or take much longer list(APPEND tests_extra grib_data_quality_checks + grib_repair grib_g1monthlydate grib_g1fcperiod grib_bpv_limit diff --git a/tests/grib_repair.sh b/tests/grib_repair.sh new file mode 100755 index 000000000..3d7178408 --- /dev/null +++ b/tests/grib_repair.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# (C) Copyright 2005- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +# + +. ./include.ctest.sh + +label="grib_repair_test" +tempText=temp.$label.txt +tempGrib=temp.$label.grib + +export ECCODES_GRIB_REPAIR_MAX_NUM_MESSAGES=3 + +${tools_dir}/grib_repair $data_dir/bad.grib $tempGrib > $tempText 2>&1 +${tools_dir}/grib_ls $tempGrib + +grep -q "Wrong message length" $tempText + +# Clean up +rm -f $tempText $tempGrib From 22104b872b03689ff45f12583e5bbdee1b9f7b59 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sun, 7 Jan 2024 15:48:25 +0000 Subject: [PATCH 34/40] Remove deprecated files --- examples/F90/include.sh | 59 -------------------------------------- examples/python/include.sh | 32 --------------------- 2 files changed, 91 deletions(-) delete mode 100755 examples/F90/include.sh delete mode 100644 examples/python/include.sh diff --git a/examples/F90/include.sh b/examples/F90/include.sh deleted file mode 100755 index 74403ee37..000000000 --- a/examples/F90/include.sh +++ /dev/null @@ -1,59 +0,0 @@ -# (C) Copyright 2005- ECMWF. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# -# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by -# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. - -CMAKE_INCLUDE_FILE=include.ctest.sh -if [ -f "$CMAKE_INCLUDE_FILE" ]; then - # This is the config file for Cmake tests - . ./$CMAKE_INCLUDE_FILE - -else - set -eax - - echo - echo "TEST: $0" - - if [ -z "${data_dir}" ] - then - cd ../../ - cpath=`pwd` - ECCODES_DEFINITION_PATH=$cpath/definitions - export ECCODES_DEFINITION_PATH - ECCODES_SAMPLES_PATH=$cpath/samples - export ECCODES_SAMPLES_PATH - tools_dir=$cpath/tools - examples_dir=$cpath/examples/F90 - data_dir=$cpath/data - samples_dir=$cpath/samples - - if test "x$ECCODES_TEST_WITH_VALGRIND" != "x"; then - tools_dir="valgrind --error-exitcode=1 -q $cpath/tools" - examples_dir="valgrind --error-exitcode=1 -q $cpath/examples/F90" - fi - - else - echo "Skipping test $0" - exit - fi - - cd "$cpath/examples/F90" - - if [ -z "${GRIB_API_INCLUDE}" ] - then - GRIB_API_INCLUDE=`pwd`/src - fi - - if [ -z "${GRIB_API_LIB}" ] - then - GRIB_API_LIB=`pwd`/src - fi - - # Download the data needed for tests - ${data_dir}/download.sh "${data_dir}" - - set -u -fi diff --git a/examples/python/include.sh b/examples/python/include.sh deleted file mode 100644 index e91fdd3d6..000000000 --- a/examples/python/include.sh +++ /dev/null @@ -1,32 +0,0 @@ -CMAKE_INCLUDE_FILE=include.ctest.sh -if [ -f "$CMAKE_INCLUDE_FILE" ]; then - # This is the config file for Cmake tests - . ./$CMAKE_INCLUDE_FILE - -else - set -eax - echo - echo "TEST: $0" - - cpath=$TOPBUILDDIR - ECCODES_DEFINITION_PATH=$cpath/definitions - export ECCODES_DEFINITION_PATH - ECCODES_SAMPLES_PATH=$cpath/samples - export ECCODES_SAMPLES_PATH - tools_dir=$cpath/tools - examples_dir=$cpath/examples/python - data_dir=$cpath/data - examples_src=$examples_dir - - PYTHONPATH=$cpath/python:$cpath/python/.libs:$PYTHONPATH - export PYTHONPATH - - HAVE_MEMFS=0 - ECCODES_ON_WINDOWS=0 - - # Download the data needed for tests - ${data_dir}/download.sh "${data_dir}" - - set -u - -fi From 393efa3b702d4e3ed5ccf1369f0ef301831705cb Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sun, 7 Jan 2024 16:19:43 +0000 Subject: [PATCH 35/40] Testing: Cleanup --- tests/grib_ecc-1212.sh | 3 --- tests/grib_indexing.sh | 3 --- 2 files changed, 6 deletions(-) diff --git a/tests/grib_ecc-1212.sh b/tests/grib_ecc-1212.sh index 7ae426756..e1f32e596 100755 --- a/tests/grib_ecc-1212.sh +++ b/tests/grib_ecc-1212.sh @@ -66,7 +66,6 @@ cat > $tempRef < $tempOut # edition centre date time dataType gridType typeOfLevel level shortName packingType echo "2 ecmf 20200805 1200 an regular_ll surface 0 t grid_simple" > $tempRef -cat $tempOut diff -w $tempRef $tempOut # Check "time" namespace @@ -84,7 +82,6 @@ ${tools_dir}/grib_get -n time $tempGrib > $tempOut echo "h 20200804 0000 36" > $tempRef diff -w $tempRef $tempOut - # numberOfForecastsUsedInLocalTime > 1 # ------------------------------------ cat > $tempFilt </dev/null # Must remove first two lines (filename specifics) ${tools_dir}/grib_dump ${tempIndex} | sed '1,2d' > $tempOut -#cat $tempOut cat > $tempRef <$tempOut status=$? set -e [ $status -ne 0 ] -cat $tempOut grep -q "Indexes contained in the input files have different keys" $tempOut rm -f $tempIndex1 $tempIndex2 $tempOut @@ -130,7 +128,6 @@ ${tools_dir}/grib_compare -v $tempIndex2 $tempIndex1 2>$tempOut status=$? set -e [ $status -ne 0 ] -cat $tempOut grep -q "Indexes contained in the input files have different keys" $tempOut rm -f $tempIndex1 $tempIndex2 $tempOut From 557da6b2de43435e782dea995316ec73d6ae688c Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sun, 7 Jan 2024 16:20:03 +0000 Subject: [PATCH 36/40] Testing: extract offsets --- tests/extract_offsets.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/extract_offsets.sh b/tests/extract_offsets.sh index 828a4dd3f..b36005cb9 100755 --- a/tests/extract_offsets.sh +++ b/tests/extract_offsets.sh @@ -48,5 +48,13 @@ set -e [ $status -ne 0 ] grep -q "Wrong message length" $tempLog +set +e +$EXEC ${test_dir}/extract_offsets nonexistentfile > $tempLog 2>&1 +status=$? +set -e +[ $status -ne 0 ] +grep -q "Unable to read file" $tempLog + + # Clean up rm -f $temp1 $temp2 $tempLog From 290e1ba3211ff70b8c74917ed8e43f5c7f867e9d Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sun, 7 Jan 2024 16:34:22 +0000 Subject: [PATCH 37/40] Testing: Failing test --- tests/grib_repair.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/grib_repair.sh b/tests/grib_repair.sh index 3d7178408..794b75307 100755 --- a/tests/grib_repair.sh +++ b/tests/grib_repair.sh @@ -16,7 +16,9 @@ tempGrib=temp.$label.grib export ECCODES_GRIB_REPAIR_MAX_NUM_MESSAGES=3 +ls -l $data_dir/bad.grib ${tools_dir}/grib_repair $data_dir/bad.grib $tempGrib > $tempText 2>&1 +cat $tempText ${tools_dir}/grib_ls $tempGrib grep -q "Wrong message length" $tempText From 1c0c10cedb4925a9576c8739d2c98791af283a40 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sun, 7 Jan 2024 16:52:57 +0000 Subject: [PATCH 38/40] Testing: Fix grib_repair test --- tests/CMakeLists.txt | 5 ++++- tests/grib_repair.sh | 14 +++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7f36206d0..94bbc2882 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -160,7 +160,6 @@ if( HAVE_BUILD_TOOLS ) # and/or take much longer list(APPEND tests_extra grib_data_quality_checks - grib_repair grib_g1monthlydate grib_g1fcperiod grib_bpv_limit @@ -363,6 +362,10 @@ if( HAVE_BUILD_TOOLS ) TYPE SCRIPT CONDITION ECCODES_INSTALL_EXTRA_TOOLS COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_check_gaussian_grids.sh ) + ecbuild_add_test( TARGET eccodes_t_grib_repair + TYPE SCRIPT + CONDITION ECCODES_INSTALL_EXTRA_TOOLS AND ENABLE_EXTRA_TESTS + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_repair.sh ) # Note: making the test dependent on the grib files (with DEPENDS) diff --git a/tests/grib_repair.sh b/tests/grib_repair.sh index 794b75307..430fa89cb 100755 --- a/tests/grib_repair.sh +++ b/tests/grib_repair.sh @@ -14,14 +14,14 @@ label="grib_repair_test" tempText=temp.$label.txt tempGrib=temp.$label.grib -export ECCODES_GRIB_REPAIR_MAX_NUM_MESSAGES=3 +if [ -e "${tools_dir}/grib_repair" ]; then + export ECCODES_GRIB_REPAIR_MAX_NUM_MESSAGES=3 + ${tools_dir}/grib_repair $data_dir/bad.grib $tempGrib > $tempText 2>&1 + cat $tempText + ${tools_dir}/grib_ls $tempGrib -ls -l $data_dir/bad.grib -${tools_dir}/grib_repair $data_dir/bad.grib $tempGrib > $tempText 2>&1 -cat $tempText -${tools_dir}/grib_ls $tempGrib - -grep -q "Wrong message length" $tempText + grep -q "Wrong message length" $tempText +fi # Clean up rm -f $tempText $tempGrib From 574bdc080ad82236325b3432392008702a09c3b2 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sun, 7 Jan 2024 17:28:36 +0000 Subject: [PATCH 39/40] Testing: bufr_filter unreadable message --- tests/bufr_filter_fail.sh | 16 ++++++++++++++++ tests/grib_repair.sh | 21 ++++++++++++++------- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/tests/bufr_filter_fail.sh b/tests/bufr_filter_fail.sh index ed38ba64c..2ab8b3ef0 100755 --- a/tests/bufr_filter_fail.sh +++ b/tests/bufr_filter_fail.sh @@ -105,6 +105,22 @@ cat $tempErr grep -q "Input output problem" $tempErr +# ------------------------ +# Unreadable message +# ------------------------ +cat > $fRules < $outfile +set +e +${tools_dir}/bufr_filter $fRules $outfile > $tempErr 2>&1 +status=$? +set -e +[ $status -ne 0 ] +grep -q "unreadable message" $tempErr +rm -f $outfile + # Clean up rm -f $fLog $fRules $tempErr diff --git a/tests/grib_repair.sh b/tests/grib_repair.sh index 430fa89cb..876ebdff0 100755 --- a/tests/grib_repair.sh +++ b/tests/grib_repair.sh @@ -3,7 +3,7 @@ # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -# +# # In applying this licence, ECMWF does not waive the privileges and immunities granted to it by # virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. # @@ -12,16 +12,23 @@ label="grib_repair_test" tempText=temp.$label.txt -tempGrib=temp.$label.grib +tempGoodGribs=temp.$label.good.grib +tempBadGribs=temp.$label.bad.grib if [ -e "${tools_dir}/grib_repair" ]; then export ECCODES_GRIB_REPAIR_MAX_NUM_MESSAGES=3 - ${tools_dir}/grib_repair $data_dir/bad.grib $tempGrib > $tempText 2>&1 - cat $tempText - ${tools_dir}/grib_ls $tempGrib - + ${tools_dir}/grib_repair $data_dir/bad.grib $tempGoodGribs $tempBadGribs > $tempText 2>&1 grep -q "Wrong message length" $tempText + + count=$( ${tools_dir}/grib_count $tempGoodGribs ) + [ $count -eq 1 ] + + count=$( ${tools_dir}/grib_count $tempBadGribs ) + [ $count -eq 3 ] + + ${tools_dir}/grib_ls $tempGoodGribs + ${tools_dir}/grib_ls $tempBadGribs fi # Clean up -rm -f $tempText $tempGrib +rm -f $tempText $tempGoodGribs $tempBadGribs From 808aa46225e20697eb9f7eee63a208e21091fd20 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sun, 7 Jan 2024 17:52:23 +0000 Subject: [PATCH 40/40] Testing: Error conditions (unreadable message) --- tests/bufr_compare.sh | 11 +++++++++++ tests/bufr_get.sh | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/tests/bufr_compare.sh b/tests/bufr_compare.sh index 56fc335f8..819900b15 100755 --- a/tests/bufr_compare.sh +++ b/tests/bufr_compare.sh @@ -339,6 +339,17 @@ set -e grep -q "Summary of different key values" $fLog +# ---------------------------------------- +# Unreadable message +# ---------------------------------------- +echo BUFR > $fBufrTmp +set +e +${tools_dir}/bufr_compare $fBufrTmp $fBufrTmp > $fLog 2>&1 +status=$? +set -e +[ $status -ne 0 ] +grep -q "unreadable message" $fLog + # Clean up # ------------- diff --git a/tests/bufr_get.sh b/tests/bufr_get.sh index 57dade77e..0449cfc92 100755 --- a/tests/bufr_get.sh +++ b/tests/bufr_get.sh @@ -120,6 +120,18 @@ result=`${tools_dir}/bufr_get -p unpack:s,heightOfStation aaen_55.bufr` [ "$result" = "0 858000" ] +# ---------------------------------------- +# Unreadable message +# ---------------------------------------- +echo BUFR > $fTmp +set +e +${tools_dir}/bufr_get -p edition $fTmp > $fLog 2>&1 +status=$? +set -e +[ $status -ne 0 ] +grep -q "unreadable message" $fLog + + # Clean up rm -f $fLog $fTmp $res_get $tempRef