From a516cdcc5fc99cf104ba9f64b5006ecad7b18a78 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 8 Jul 2021 14:36:35 +0100 Subject: [PATCH 1/5] ECC-1261: GRIB local definition 12: MARS 'time' key should be HHMM not single digit --- definitions/grib1/local.98.12.def | 6 +++++- definitions/grib2/local.98.12.def | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/definitions/grib1/local.98.12.def b/definitions/grib1/local.98.12.def index 3a65f98b6..d0b76d61d 100644 --- a/definitions/grib1/local.98.12.def +++ b/definitions/grib1/local.98.12.def @@ -57,8 +57,12 @@ alias totalNumber=numberOfForecastsInEnsemble; unsigned[4] indexingDate: dump; # MARS archiving date (YYYYMMDD) unsigned[2] indexingTime: dump; # MARS archiving time (HHMM) +transient indexingTimeHH = indexingTime/100 : hidden; +transient indexingTimeMM = indexingTime - 100 * indexingTimeHH : hidden; +meta indexingTimeHHMM sprintf("%.2d%.2d",indexingTimeHH,indexingTimeMM); + alias mars.date = indexingDate; -alias mars.time = indexingTime; +alias mars.time = indexingTimeHHMM; # spareSetToZero pad padding_loc12_1(50); diff --git a/definitions/grib2/local.98.12.def b/definitions/grib2/local.98.12.def index 8632fd6a6..355362b5e 100644 --- a/definitions/grib2/local.98.12.def +++ b/definitions/grib2/local.98.12.def @@ -8,7 +8,11 @@ alias local.methodNumber=methodNumber; unsigned[4] indexingDate: dump; # MARS archiving date (YYYYMMDD) unsigned[2] indexingTime: dump; # MARS archiving time (HHMM) +transient indexingTimeHH = indexingTime/100 : hidden; +transient indexingTimeMM = indexingTime - 100 * indexingTimeHH : hidden; +meta indexingTimeHHMM sprintf("%.2d%.2d",indexingTimeHH,indexingTimeMM); + alias mars.date = indexingDate; -alias mars.time = indexingTime; +alias mars.time = indexingTimeHHMM; pad padding_loc12_1(50); From 14be017ab6e3f967fe85b07f125104f841aebb20 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 9 Jul 2021 17:14:32 +0100 Subject: [PATCH 2/5] ECC-1261: Added test --- tests/CMakeLists.txt | 1 + tests/grib_ecc-1261.sh | 48 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100755 tests/grib_ecc-1261.sh diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 751dbc7a4..579922867 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -87,6 +87,7 @@ if( HAVE_BUILD_TOOLS ) grib_ecc-1255 grib_ecc-1258 grib_ecc-1260 + grib_ecc-1261 bufr_ecc-1028 bufr_ecc-1195 bufr_ecc-1259 diff --git a/tests/grib_ecc-1261.sh b/tests/grib_ecc-1261.sh new file mode 100755 index 000000000..3c4d4899e --- /dev/null +++ b/tests/grib_ecc-1261.sh @@ -0,0 +1,48 @@ +#!/bin/sh +# (C) Copyright 2005- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +# + +. ./include.sh +set -u + +label="grib_ecc-1261-test" + +tempFilt="temp.${label}.filt" +tempGrib="temp.${label}.grib" + +sample_grib1=$ECCODES_SAMPLES_PATH/GRIB1.tmpl +sample_grib2=$ECCODES_SAMPLES_PATH/GRIB2.tmpl + +test_ld12() +{ + samplef=$1 + itime=$2 # input indexingTime + otime=$3 # output (expected) + + cat > $tempFilt << EOF + set setLocalDefinition=1; + set localDefinitionNumber=12; + set indexingTime = $itime; + write; +EOF + ${tools_dir}/grib_filter -o $tempGrib $tempFilt $samplef + grib_check_key_equals $tempGrib "mars.time" "$otime" + rm -f $tempFilt $tempGrib +} + +test_ld12 $sample_grib1 "0" "0000" +test_ld12 $sample_grib1 "12" "0012" +test_ld12 $sample_grib1 "2400" "2400" +test_ld12 $sample_grib1 "1423" "1423" + +test_ld12 $sample_grib2 "0" "0000" +test_ld12 $sample_grib2 "12" "0012" +test_ld12 $sample_grib2 "2400" "2400" +test_ld12 $sample_grib2 "1423" "1423" + From 84652311435c2dbb5a387db8a4c0c44405b03ba3 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 16 Sep 2021 16:47:25 +0100 Subject: [PATCH 3/5] ECC-1284: bufr_dump: Display of missing strings in arrays --- examples/F90/CMakeLists.txt | 1 + examples/F90/bufr_ecc-1284.f90 | 74 +++++++++++++++++++++++++++++ examples/F90/bufr_ecc-1284.sh | 69 +++++++++++++++++++++++++++ src/grib_dumper_class_bufr_simple.c | 9 +++- src/grib_dumper_class_json.c | 9 +++- 5 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 examples/F90/bufr_ecc-1284.f90 create mode 100755 examples/F90/bufr_ecc-1284.sh diff --git a/examples/F90/CMakeLists.txt b/examples/F90/CMakeLists.txt index 93b6435a1..ecd3a3d1a 100644 --- a/examples/F90/CMakeLists.txt +++ b/examples/F90/CMakeLists.txt @@ -11,6 +11,7 @@ if( HAVE_BUILD_TOOLS ) list( APPEND tests_sanity grib_set_pv grib_set_data + bufr_ecc-1284 grib_ecc-671 ) list( APPEND tests_extra grib_index diff --git a/examples/F90/bufr_ecc-1284.f90 b/examples/F90/bufr_ecc-1284.f90 new file mode 100644 index 000000000..ae3fd5fea --- /dev/null +++ b/examples/F90/bufr_ecc-1284.f90 @@ -0,0 +1,74 @@ +! (C) Copyright 2005- ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! +! In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +! virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +! +! +program bufr_encode + use eccodes + implicit none + integer :: iret + integer :: outfile + integer :: ibufr + integer, parameter :: max_strsize = 100 + character(len=max_strsize) , dimension(:),allocatable :: svalues + character(9) :: missing_string + character(len=max_strsize) :: outfile_name + + call getarg(1, outfile_name) + + call codes_bufr_new_from_samples(ibufr,'BUFR4',iret) + if (iret/=CODES_SUCCESS) then + print *,'ERROR creating BUFR from BUFR4' + stop 1 + endif + call codes_set(ibufr,'masterTableNumber',0) + call codes_set(ibufr,'bufrHeaderSubCentre',0) + call codes_set(ibufr,'bufrHeaderCentre',98) + call codes_set(ibufr,'updateSequenceNumber',0) + call codes_set(ibufr,'dataCategory',1) + call codes_set(ibufr,'dataSubCategory',110) + call codes_set(ibufr,'masterTablesVersionNumber',13) + call codes_set(ibufr,'typicalYearOfCentury',12) + call codes_set(ibufr,'typicalMonth',10) + call codes_set(ibufr,'typicalDay',31) + call codes_set(ibufr,'typicalHour',0) + call codes_set(ibufr,'typicalMinute',2) + call codes_set(ibufr,'numberOfSubsets',10) + call codes_set(ibufr,'observedData',1) + call codes_set(ibufr,'compressedData',1) + + ! Create the structure of the data section + call codes_set(ibufr,'unexpandedDescriptors',307022) + + missing_string = char(255)//char(255)//char(255)//char(255)//char(255)//char(255)//char(255)//char(255)//char(255) + + allocate(svalues(10)) + svalues=(/"ARD2-LPTR", & + "ETH2-LPTR", & + "ETH2-LPTR", & + "KALT-LPTR", & + "KALT-LPTR", & + "FHBB-LPTR", & + "FHBB-LPTR", & + "LIND-LPTR", & + "LIND-LPTR", & + "EPFL-LPTR" /) + svalues(3) = missing_string + svalues(10) = missing_string + call codes_set_string_array(ibufr,'stationOrSiteName',svalues) + + ! Encode the keys back in the data section + call codes_set(ibufr,'pack',1) + + call codes_open_file(outfile,outfile_name,'w') + print *,'Wrote ',outfile_name + call codes_write(ibufr,outfile) + call codes_close_file(outfile) + call codes_release(ibufr) + + if(allocated(svalues)) deallocate(svalues) +end program bufr_encode diff --git a/examples/F90/bufr_ecc-1284.sh b/examples/F90/bufr_ecc-1284.sh new file mode 100755 index 000000000..f7a1e04ee --- /dev/null +++ b/examples/F90/bufr_ecc-1284.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# (C) Copyright 2005- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +# + +. ./include.sh +set -u + +label="bufr_ecc-1284_test_f" +TEMP_DUMP=temp.$label.dump +TEMP_TEXT=temp.$label.txt +TEMP_REF=temp.$label.ref +TEMP_BUFR=temp.$label.bufr + +# Create a BUFR which has some missing entries in the string array stationOrSiteName +${examples_dir}/eccodes_f_bufr_ecc-1284 $TEMP_BUFR + +# -------------- +# Plain dump +# -------------- +${tools_dir}/bufr_dump -p $TEMP_BUFR > $TEMP_DUMP +awk '/stationOrSiteName/{x=NR+10}(NR<=x){print}' $TEMP_DUMP > $TEMP_TEXT + +cat > $TEMP_REF << EOF +stationOrSiteName={ "ARD2-LPTR", + "ETH2-LPTR", + MISSING, + "KALT-LPTR", + "KALT-LPTR", + "FHBB-LPTR", + "FHBB-LPTR", + "LIND-LPTR", + "LIND-LPTR", + MISSING +} +EOF + +diff $TEMP_REF $TEMP_TEXT + +# -------------- +# JSON dump +# -------------- +${tools_dir}/bufr_dump $TEMP_BUFR > $TEMP_DUMP +awk '/stationOrSiteName/{x=NR+12}(NR<=x){print}' $TEMP_DUMP > $TEMP_TEXT + +cat > $TEMP_REF << EOF + "key" : "stationOrSiteName", + "value" : + [ "ARD2-LPTR", + "ETH2-LPTR", + null, + "KALT-LPTR", + "KALT-LPTR", + "FHBB-LPTR", + "FHBB-LPTR", + "LIND-LPTR", + "LIND-LPTR", + null + ], +EOF +diff $TEMP_REF $TEMP_TEXT + +# Clean up +rm -f $TEMP_BUFR $TEMP_DUMP $TEMP_TEXT $TEMP_REF diff --git a/src/grib_dumper_class_bufr_simple.c b/src/grib_dumper_class_bufr_simple.c index 6ae765c69..785b41c2e 100644 --- a/src/grib_dumper_class_bufr_simple.c +++ b/src/grib_dumper_class_bufr_simple.c @@ -568,6 +568,7 @@ static void dump_string_array(grib_dumper* d, grib_accessor* a, const char* comm size_t size = 0, i = 0; grib_context* c = a->context; int err = 0; + int is_missing = 0; long count = 0; int r = 0; grib_handle* h = grib_handle_of_accessor(a); @@ -601,9 +602,13 @@ static void dump_string_array(grib_dumper* d, grib_accessor* a, const char* comm fprintf(self->dumper.out, "{"); for (i = 0; i < size - 1; i++) { - fprintf(self->dumper.out, " \"%s\",\n", values[i]); + is_missing = grib_is_missing_string(a, (unsigned char*)values[i], strlen(values[i])); + if (is_missing) fprintf(self->dumper.out, " %s,\n", "MISSING"); + else fprintf(self->dumper.out, " \"%s\",\n", values[i]); } - fprintf(self->dumper.out, " \"%s\"\n", values[i]); + is_missing = grib_is_missing_string(a, (unsigned char*)values[i], strlen(values[i])); + if (is_missing) fprintf(self->dumper.out, " %s\n", "MISSING"); + else fprintf(self->dumper.out, " \"%s\"\n", values[i]); fprintf(self->dumper.out, "}\n"); diff --git a/src/grib_dumper_class_json.c b/src/grib_dumper_class_json.c index 8a526005c..ccc1f0874 100644 --- a/src/grib_dumper_class_json.c +++ b/src/grib_dumper_class_json.c @@ -381,6 +381,7 @@ static void dump_string_array(grib_dumper* d, grib_accessor* a, const char* comm size_t size = 0, i = 0; grib_context* c = NULL; int err = 0; + int is_missing = 0; long count = 0; c = a->context; @@ -423,9 +424,13 @@ static void dump_string_array(grib_dumper* d, grib_accessor* a, const char* comm fprintf(self->dumper.out, "\n%-*s[", depth, " "); depth += 2; for (i = 0; i < size - 1; i++) { - fprintf(self->dumper.out, "%-*s\"%s\",\n", depth, " ", values[i]); + is_missing = grib_is_missing_string(a, (unsigned char*)values[i], strlen(values[i])); + if (is_missing) fprintf(self->dumper.out, "%-*s%s,\n", depth, " ", "null"); + else fprintf(self->dumper.out, "%-*s\"%s\",\n", depth, " ", values[i]); } - fprintf(self->dumper.out, "%-*s\"%s\"\n", depth, " ", values[i]); + is_missing = grib_is_missing_string(a, (unsigned char*)values[i], strlen(values[i])); + if (is_missing) fprintf(self->dumper.out, "%-*s%s", depth, " ", "null"); + else fprintf(self->dumper.out, "%-*s\"%s\"", depth, " ", values[i]); depth -= 2; fprintf(self->dumper.out, "\n%-*s]", depth, " "); From b594c9cc65cbd4699c668a6efbf9e2d0311775be Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 16 Sep 2021 17:07:38 +0100 Subject: [PATCH 4/5] ECC-1284: clean up --- examples/F90/bufr_ecc-1284.f90 | 2 ++ examples/F90/bufr_ecc-1284.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/examples/F90/bufr_ecc-1284.f90 b/examples/F90/bufr_ecc-1284.f90 index ae3fd5fea..132086659 100644 --- a/examples/F90/bufr_ecc-1284.f90 +++ b/examples/F90/bufr_ecc-1284.f90 @@ -44,6 +44,7 @@ program bufr_encode ! Create the structure of the data section call codes_set(ibufr,'unexpandedDescriptors',307022) + ! Create a string with characters all set to 255 i.e., all bits=1 missing_string = char(255)//char(255)//char(255)//char(255)//char(255)//char(255)//char(255)//char(255)//char(255) allocate(svalues(10)) @@ -57,6 +58,7 @@ program bufr_encode "LIND-LPTR", & "LIND-LPTR", & "EPFL-LPTR" /) + ! Change some entries to be missing (undefined) svalues(3) = missing_string svalues(10) = missing_string call codes_set_string_array(ibufr,'stationOrSiteName',svalues) diff --git a/examples/F90/bufr_ecc-1284.sh b/examples/F90/bufr_ecc-1284.sh index f7a1e04ee..56b635717 100755 --- a/examples/F90/bufr_ecc-1284.sh +++ b/examples/F90/bufr_ecc-1284.sh @@ -24,6 +24,7 @@ ${examples_dir}/eccodes_f_bufr_ecc-1284 $TEMP_BUFR # Plain dump # -------------- ${tools_dir}/bufr_dump -p $TEMP_BUFR > $TEMP_DUMP +# From the dump output, grab 10 lines after stationOrSiteName awk '/stationOrSiteName/{x=NR+10}(NR<=x){print}' $TEMP_DUMP > $TEMP_TEXT cat > $TEMP_REF << EOF From e337247173f6894ae97f6b15d9a9eb10d0bd6e14 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 17 Sep 2021 18:13:17 +0100 Subject: [PATCH 5/5] ECC-1284: use the Fortran REPEAT statement --- examples/F90/bufr_ecc-1284.f90 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/F90/bufr_ecc-1284.f90 b/examples/F90/bufr_ecc-1284.f90 index 132086659..69600a1c7 100644 --- a/examples/F90/bufr_ecc-1284.f90 +++ b/examples/F90/bufr_ecc-1284.f90 @@ -10,13 +10,13 @@ program bufr_encode use eccodes implicit none - integer :: iret - integer :: outfile - integer :: ibufr - integer, parameter :: max_strsize = 100 + integer :: iret, outfile + integer :: ibufr + integer, parameter :: max_strsize = 100 + character (len=1), parameter :: missing_char = char(255) + character(9) :: missing_string + character(len=max_strsize) :: outfile_name character(len=max_strsize) , dimension(:),allocatable :: svalues - character(9) :: missing_string - character(len=max_strsize) :: outfile_name call getarg(1, outfile_name) @@ -44,8 +44,8 @@ program bufr_encode ! Create the structure of the data section call codes_set(ibufr,'unexpandedDescriptors',307022) - ! Create a string with characters all set to 255 i.e., all bits=1 - missing_string = char(255)//char(255)//char(255)//char(255)//char(255)//char(255)//char(255)//char(255)//char(255) + ! Create a string with characters all set to 255 (all bits=1) + missing_string = repeat(missing_char, 9) allocate(svalues(10)) svalues=(/"ARD2-LPTR", &