From e4e641d28fa2dd95c2f652512427ce39ca09bfda Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 26 Apr 2022 11:27:46 +0100 Subject: [PATCH 1/4] Testing: use of 'accuracy' to encode (IEEE) --- tests/grib_ieee.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/grib_ieee.sh b/tests/grib_ieee.sh index 6409aa6fc..c9db74506 100755 --- a/tests/grib_ieee.sh +++ b/tests/grib_ieee.sh @@ -175,8 +175,12 @@ stats1=`${tools_dir}/grib_get -M -F%.3f -p skew,kurt $infile` stats2=`${tools_dir}/grib_get -M -F%.3f -p skew,kurt $temp` [ "$stats1" = "$stats2" ] +# The same thing but use 'accuracy' instead of 'precision' +temp2=temp2.grib_ieee.grib +${tools_dir}/grib_set -r -s accuracy=64 $infile $temp2 +cmp $temp $temp2 - +rm -f $temp2 rm -f $temp ################################## From 4270ffaaa33319daa69a53e297909ac4483fcc32 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 26 Apr 2022 11:39:34 +0100 Subject: [PATCH 2/4] Cleanup: src/grib_templates.h deprecated --- src/CMakeLists.txt | 1 - src/grib_templates.h | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ff3bcaea8..76c008161 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -373,7 +373,6 @@ list( APPEND eccodes_src_files grib_iterator_factory.h grib_nearest_class.h grib_nearest_factory.h - grib_templates.h grib_yacc.h md5.h md5.c diff --git a/src/grib_templates.h b/src/grib_templates.h index 1b1ae7333..cc8175e0c 100644 --- a/src/grib_templates.h +++ b/src/grib_templates.h @@ -61,7 +61,7 @@ unsigned char _grib_template_GRIB2[] = { }; static grib_templates templates[] = { - {"GRIB1", _grib_template_GRIB1, 107,}, - {"GRIB2", _grib_template_GRIB2, 184,}, + {"GRIB1", _grib_template_GRIB1, 107,}, + {"GRIB2", _grib_template_GRIB2, 184,}, }; #endif From 5b521ced3a11192a187fd04b4bf31a9bc37544ee Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 26 Apr 2022 12:03:24 +0100 Subject: [PATCH 3/4] Testing: use of 'accuracy' to encode (IEEE) --- tests/grib_change_packing.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/grib_change_packing.sh b/tests/grib_change_packing.sh index e645c30fe..5b5e27a54 100755 --- a/tests/grib_change_packing.sh +++ b/tests/grib_change_packing.sh @@ -86,6 +86,16 @@ stats2=`${tools_dir}/grib_get -F%.2f -p skew,kurt $temp` [ "$stats1" = "$stats2" ] +# Test 'accuracy' key +# ----------------------- +input=${data_dir}/reduced_gaussian_model_level.grib1 +${tools_dir}/grib_set -r -s packingType=grid_ieee,accuracy=64 $input $temp +grib_check_key_equals $temp precision '2' +stats1=`${tools_dir}/grib_get -F%.2f -p skew,kurt $input` +stats2=`${tools_dir}/grib_get -F%.2f -p skew,kurt $temp` +[ "$stats1" = "$stats2" ] + + # Change grib packing test # ------------------------- test_packing $grib1 $packing1 From 4f3c47fc1296a9aeec1a6616a1a73eb855889c33 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Tue, 26 Apr 2022 12:46:01 +0100 Subject: [PATCH 4/4] Cleanup --- src/grib_accessor_class_bufr_data_array.c | 91 ++++++++--------------- 1 file changed, 32 insertions(+), 59 deletions(-) diff --git a/src/grib_accessor_class_bufr_data_array.c b/src/grib_accessor_class_bufr_data_array.c index d267ad589..9778cb528 100644 --- a/src/grib_accessor_class_bufr_data_array.c +++ b/src/grib_accessor_class_bufr_data_array.c @@ -2905,8 +2905,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st data = buffer->data; err = get_descriptors(a); - if (err) - return err; + if (err) return err; descriptors = self->expanded->v; if (!descriptors) { @@ -2926,7 +2925,6 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st if (flag != PROCESS_ENCODE) { self->numericValues = grib_vdarray_new(c, 1000, 1000); self->stringValues = grib_vsarray_new(c, 10, 10); - /*printf("dbg process_elements: Create a new one %p\n", (void*)(self->stringValues));*/ if (self->elementsDescriptorsIndex) grib_viarray_delete(c, self->elementsDescriptorsIndex); @@ -2935,15 +2933,13 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st if (flag != PROCESS_DECODE) { /* Operator 203YYY: key OVERRIDDEN_REFERENCE_VALUES_KEY */ err = grib_get_size(h, OVERRIDDEN_REFERENCE_VALUES_KEY, &self->refValListSize); - if (err) - return err; + if (err) return err; if (self->refValList) grib_context_free(c, self->refValList); if (self->refValListSize > 0) { self->refValList = (long*)grib_context_malloc_clear(c, self->refValListSize * sizeof(long)); err = grib_get_long_array(grib_handle_of_accessor(a), OVERRIDDEN_REFERENCE_VALUES_KEY, self->refValList, &self->refValListSize); - if (err) - return err; + if (err) return err; } } @@ -2960,6 +2956,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st end = self->compressedData == 1 ? 1 : grib_iarray_used_size(self->iss_list); } + /* Go through all subsets */ for (iiss = 0; iiss < end; iiss++) { icount = 1; if (self->compressedData == 0 && self->iss_list) { @@ -2976,7 +2973,6 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st elementsDescriptorsIndex = grib_iarray_new(c, DYN_ARRAY_SIZE_INIT, DYN_ARRAY_SIZE_INCR); if (!self->compressedData) { dval = grib_darray_new(c, DYN_ARRAY_SIZE_INIT, DYN_ARRAY_SIZE_INCR); - /* sval=grib_sarray_new(c,10,10); */ } } else { @@ -2991,8 +2987,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st numberOfNestedRepetitions = 0; for (i = 0; i < numberOfDescriptors; i++) { - if (c->debug) grib_context_log(c, GRIB_LOG_DEBUG, "BUFR data processing: elementNumber=%ld code=%6.6ld", - icount++, descriptors[i]->code); + if (c->debug) grib_context_log(c, GRIB_LOG_DEBUG, "BUFR data processing: elementNumber=%ld code=%6.6ld", icount++, descriptors[i]->code); switch (descriptors[i]->F) { case 0: /* Table B element */ @@ -3004,8 +2999,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st } err = codec_element(c, self, iss, buffer, data, &pos, i, 0, elementIndex, dval, sval); - if (err) - return err; + if (err) return err; elementIndex++; break; case 1: @@ -3017,11 +3011,9 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st n[inr] = numberOfElementsToRepeat[inr]; i++; - /* ECC-517 */ - data = buffer->data; + data = buffer->data; /* ECC-517 */ err = codec_replication(c, self, iss, buffer, data, &pos, i, elementIndex, dval, &(numberOfRepetitions[inr])); - if (err) - return err; + if (err) return err; startRepetition[inr] = i; nn[inr] = numberOfRepetitions[inr]; @@ -3053,8 +3045,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st case 2: /* Operator */ switch (descriptors[i]->X) { - case 3: - /* Change reference values */ + case 3: /* Change reference values */ if (self->compressedData == 1 && flag != PROCESS_DECODE) { grib_context_log(c, GRIB_LOG_ERROR, "process_elements: operator %d not supported for encoding compressed data", descriptors[i]->X); return GRIB_INTERNAL_ERROR; @@ -3066,8 +3057,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st if (iss == 0 && flag == PROCESS_DECODE) { /*Write out the contents of the TableB overridden reference values to the transient array key*/ err = tableB_override_set_key(h, self); - if (err) - return err; + if (err) return err; } if (flag != PROCESS_DECODE) { /* Encoding operator 203YYY */ @@ -3092,8 +3082,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st tableB_override_clear(c, self); if (flag != PROCESS_DECODE) { err = check_overridden_reference_values(c, self->refValList, self->refValListSize, numBits); - if (err) - return err; + if (err) return err; } } /*grib_iarray_push(elementsDescriptorsIndex,i);*/ @@ -3105,8 +3094,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st descriptors[i]->width = descriptors[i]->Y * 8; descriptors[i]->type = BUFR_DESCRIPTOR_TYPE_STRING; err = codec_element(c, self, iss, buffer, data, &pos, i, 0, elementIndex, dval, sval); - if (err) - return err; + if (err) return err; if (flag != PROCESS_ENCODE) grib_iarray_push(elementsDescriptorsIndex, i); elementIndex++; @@ -3147,10 +3135,8 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st push_zero_element(self, dval); elementIndex++; break; - case 24: - /*first-order statistical values marker operator*/ - case 32: - /*replaced/retained values marker operator*/ + case 24: /* First-order statistical values marker operator */ + case 32: /* Replaced/retained values marker operator */ if (descriptors[i]->Y == 255) { index = get_next_bitmap_descriptor_index(self, elementsDescriptorsIndex, dval); if (index < 0) { /* Return value is an error code not an index */ @@ -3158,8 +3144,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st return err; } err = codec_element(c, self, iss, buffer, data, &pos, index, 0, elementIndex, dval, sval); - if (err) - return err; + if (err) return err; /* self->expanded->v[index] */ if (flag != PROCESS_ENCODE) grib_iarray_push(elementsDescriptorsIndex, i); @@ -3174,7 +3159,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st elementIndex++; } break; - case 23: + case 23: /* Substituted values operator */ if (descriptors[i]->Y == 255) { index = get_next_bitmap_descriptor_index(self, elementsDescriptorsIndex, dval); if (index < 0) { /* Return value is an error code not an index */ @@ -3182,8 +3167,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st return err; } err = codec_element(c, self, iss, buffer, data, &pos, index, 0, elementIndex, dval, sval); - if (err) - return err; + if (err) return err; /* self->expanded->v[index] */ if (flag != PROCESS_ENCODE) grib_iarray_push(elementsDescriptorsIndex, i); @@ -3195,8 +3179,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st push_zero_element(self, dval); if (descriptors[i + 1] && descriptors[i + 1]->code != 236000 && descriptors[i + 1]->code != 237000) { err = build_bitmap(self, data, &pos, elementIndex, elementsDescriptorsIndex, i); - if (err) - return err; + if (err) return err; } } else if (flag == PROCESS_ENCODE) { @@ -3207,15 +3190,13 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st grib_iarray_push(elementsDescriptorsIndex, i); if (descriptors[i + 1] && descriptors[i + 1]->code != 236000 && descriptors[i + 1]->code != 237000) { err = build_bitmap_new_data(self, data, &pos, elementIndex, elementsDescriptorsIndex, i); - if (err) - return err; + if (err) return err; } } elementIndex++; } break; - case 25: - /*difference statistical values marker operator*/ + case 25: /* Difference statistical values marker operator */ if (descriptors[i]->Y == 255) { index = get_next_bitmap_descriptor_index(self, elementsDescriptorsIndex, dval); if (index < 0) { /* Return value is an error code not an index */ @@ -3228,8 +3209,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st err = codec_element(c, self, iss, buffer, data, &pos, index, bd, elementIndex, dval, sval); grib_bufr_descriptor_delete(bd); - if (err) - return err; + if (err) return err; /* self->expanded->v[index] */ if (flag != PROCESS_ENCODE) grib_iarray_push(elementsDescriptorsIndex, i); @@ -3243,8 +3223,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st elementIndex++; } break; - case 35: - /* cancel bitmap */ + case 35: /* Cancel backward data reference (cancel bitmap) */ if (flag != PROCESS_ENCODE) { grib_iarray_push(elementsDescriptorsIndex, i); if (decoding) @@ -3254,15 +3233,13 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st } elementIndex++; break; - case 36: - /* bitmap */ + case 36: /* Define data present bit-map */ if (flag == PROCESS_DECODE) { grib_iarray_push(elementsDescriptorsIndex, i); if (decoding) push_zero_element(self, dval); err = build_bitmap(self, data, &pos, elementIndex, elementsDescriptorsIndex, i); - if (err) - return err; + if (err) return err; } else if (flag == PROCESS_ENCODE) { restart_bitmap(self); @@ -3270,13 +3247,11 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st else if (flag == PROCESS_NEW_DATA) { grib_iarray_push(elementsDescriptorsIndex, i); err = build_bitmap_new_data(self, data, &pos, elementIndex, elementsDescriptorsIndex, i); - if (err) - return err; + if (err) return err; } elementIndex++; break; - case 37: - /* reuse defined bitmap */ + case 37: /* Use defined data present bit-map = reuse defined bitmap */ if (flag != PROCESS_ENCODE) { grib_iarray_push(elementsDescriptorsIndex, i); if (decoding) @@ -3295,11 +3270,10 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st } /* F == 2 */ break; case 9: - /* associated field */ + /* Associated field */ if (descriptors[i]->X == 99 && descriptors[i]->Y == 999) { err = codec_element(c, self, iss, buffer, data, &pos, i, 0, elementIndex, dval, sval); - if (err) - return err; + if (err) return err; if (flag != PROCESS_ENCODE) grib_iarray_push(elementsDescriptorsIndex, i); elementIndex++; @@ -3313,7 +3287,7 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st return err; } /* switch F */ - /* delayed repetition check */ + /* Delayed repetition check */ innr = numberOfNestedRepetitions - 1; for (ir = innr; ir >= 0; ir--) { if (nn[ir]) { @@ -3347,16 +3321,16 @@ static int process_elements(grib_accessor* a, int flag, long onlySubset, long st } } } - } + } /* for all descriptors */ + if (flag != PROCESS_ENCODE) grib_viarray_push(c, self->elementsDescriptorsIndex, elementsDescriptorsIndex); if (decoding && !self->compressedData) { grib_vdarray_push(c, self->numericValues, dval); } - } + } /* for all subsets */ /*if (c->debug) grib_vdarray_print("process_elements: self->numericValues", self->numericValues);*/ - if (decoding) { err = create_keys(a, 0, 0, 0); self->bitsToEndData = totalSize; @@ -3379,7 +3353,6 @@ static void dump(grib_accessor* a, grib_dumper* dumper) /* int err=process_elements(a,PROCESS_DECODE); */ /* grib_dump_section(dumper,a,self->dataKeys->block); */ - return; }