core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECC-428: Inconsistent number of values when decoding compressed BUFR data

This commit is contained in:
Shahram Najm 2019-07-31 18:52:53 +01:00
parent 3fc3ec85bb
commit 6a5749ecb7
5 changed files with 161 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
src/grib_accessor_class_bufr_data_array.c
View File

@ -559,33 +559,33 @@ static int decode_string_array(grib_context* c, unsigned char* data, long* pos,
sval=(char*)grib_context_malloc_clear(c,modifiedWidth/8+1); sval=(char*)grib_context_malloc_clear(c,modifiedWidth/8+1);
CHECK_END_DATA_RETURN(c, self, modifiedWidth, *err); CHECK_END_DATA_RETURN(c, self, modifiedWidth, *err);
if (*err) { if (*err) {
grib_sarray_push(c,sa,sval); grib_sarray_push(c,sa,sval);
grib_vsarray_push(c,self->stringValues,sa); grib_vsarray_push(c,self->stringValues,sa);
return ret; return ret;
} }
grib_decode_string(data,pos,modifiedWidth/8,sval); grib_decode_string(data,pos,modifiedWidth/8,sval);
CHECK_END_DATA_RETURN(c, self, 6, *err); CHECK_END_DATA_RETURN(c, self, 6, *err);
if (*err) { if (*err) {
grib_sarray_push(c,sa,sval); grib_sarray_push(c,sa,sval);
grib_vsarray_push(c,self->stringValues,sa); grib_vsarray_push(c,self->stringValues,sa);
return ret; return ret;
} }
width=grib_decode_unsigned_long(data,pos,6); width=grib_decode_unsigned_long(data,pos,6);
if (width) { if (width) {
CHECK_END_DATA_RETURN(c, self, width*8*self->numberOfSubsets, *err); CHECK_END_DATA_RETURN(c, self, width*8*self->numberOfSubsets, *err);
if (*err) { if (*err) {
grib_sarray_push(c,sa,sval); grib_sarray_push(c,sa,sval);
grib_vsarray_push(c,self->stringValues,sa); grib_vsarray_push(c,self->stringValues,sa);
return ret; return ret;
} }
grib_context_free(c,sval); grib_context_free(c,sval);
for (j=0;j<self->numberOfSubsets;j++) { for (j=0;j<self->numberOfSubsets;j++) {
sval=(char*)grib_context_malloc_clear(c,width+1); sval=(char*)grib_context_malloc_clear(c,width+1);
grib_decode_string(data,pos,width,sval); grib_decode_string(data,pos,width,sval);
grib_sarray_push(c,sa,sval); grib_sarray_push(c,sa,sval);
} }
} else { } else {
grib_sarray_push(c,sa,sval); grib_sarray_push(c,sa,sval);
} }
grib_vsarray_push(c,self->stringValues,sa); grib_vsarray_push(c,self->stringValues,sa);
return ret; return ret;
@ -609,13 +609,13 @@ static grib_darray* decode_double_array(grib_context* c,unsigned char* data,long
CHECK_END_DATA_RETURN(c, self, modifiedWidth+6, NULL); CHECK_END_DATA_RETURN(c, self, modifiedWidth+6, NULL);
if (*err) { if (*err) {
dval=GRIB_MISSING_DOUBLE; dval=GRIB_MISSING_DOUBLE;
lval=0; lval=0;
grib_context_log(c, GRIB_LOG_DEBUG," modifiedWidth=%ld lval=%ld dval=%g", modifiedWidth,lval,dval); grib_context_log(c, GRIB_LOG_DEBUG," modifiedWidth=%ld lval=%ld dval=%g", modifiedWidth,lval,dval);
ret=grib_darray_new(c,DYN_ARRAY_SIZE_INIT,DYN_ARRAY_SIZE_INCR); ret=grib_darray_new(c,DYN_ARRAY_SIZE_INIT,DYN_ARRAY_SIZE_INCR);
grib_darray_push(c,ret,dval); grib_darray_push(c,ret,dval);
*err=0; *err=0;
return ret; return ret;
} }
lval=grib_decode_unsigned_long(data,pos,modifiedWidth); lval=grib_decode_unsigned_long(data,pos,modifiedWidth);
localReference=(long)lval+modifiedReference; localReference=(long)lval+modifiedReference;
@ -625,13 +625,13 @@ static grib_darray* decode_double_array(grib_context* c,unsigned char* data,long
if (localWidth) { if (localWidth) {
CHECK_END_DATA_RETURN(c, self, localWidth*self->numberOfSubsets, NULL); CHECK_END_DATA_RETURN(c, self, localWidth*self->numberOfSubsets, NULL);
if (*err) { if (*err) {
dval=GRIB_MISSING_DOUBLE; dval=GRIB_MISSING_DOUBLE;
lval=0; lval=0;
grib_context_log(c, GRIB_LOG_DEBUG," modifiedWidth=%ld lval=%ld dval=%g", modifiedWidth,lval,dval); grib_context_log(c, GRIB_LOG_DEBUG," modifiedWidth=%ld lval=%ld dval=%g", modifiedWidth,lval,dval);
ret=grib_darray_new(c,DYN_ARRAY_SIZE_INIT,DYN_ARRAY_SIZE_INCR); ret=grib_darray_new(c,DYN_ARRAY_SIZE_INIT,DYN_ARRAY_SIZE_INCR);
grib_darray_push(c,ret,dval); grib_darray_push(c,ret,dval);
*err=0; *err=0;
return ret; return ret;
} }
for (j=0;j<self->numberOfSubsets;j++) { for (j=0;j<self->numberOfSubsets;j++) {
lval=grib_decode_unsigned_long(data,pos,localWidth); lval=grib_decode_unsigned_long(data,pos,localWidth);
@ -643,13 +643,21 @@ static grib_darray* decode_double_array(grib_context* c,unsigned char* data,long
grib_darray_push(c,ret,dval); grib_darray_push(c,ret,dval);
} }
} else { } else {
// ECC-428
if (grib_is_all_bits_one(lval,modifiedWidth) && canBeMissing) { if (grib_is_all_bits_one(lval,modifiedWidth) && canBeMissing) {
dval=GRIB_MISSING_DOUBLE; dval=GRIB_MISSING_DOUBLE;
} else { } else {
dval=localReference*modifiedFactor; dval=localReference*modifiedFactor;
} }
grib_context_log(c, GRIB_LOG_DEBUG," modifiedWidth=%ld lval=%ld dval=%g", modifiedWidth,lval,dval); if(c->bufr_multi_element_constant_arrays) {
grib_darray_push(c,ret,dval); grib_context_log(c, GRIB_LOG_DEBUG," modifiedWidth=%ld lval=%ld dval=%g (const array multi values)", modifiedWidth,lval,dval,bd->code);
for (j=0;j<self->numberOfSubsets;j++) {
grib_darray_push(c,ret,dval);
}
} else {
grib_context_log(c, GRIB_LOG_DEBUG," modifiedWidth=%ld lval=%ld dval=%g (const array single value)", modifiedWidth,lval,dval,bd->code);
grib_darray_push(c,ret,dval);
}
} }
return ret; return ret;
@ -671,10 +679,10 @@ static int encode_string_array(grib_context* c,grib_buffer* buff,long* pos, bufr
if (n<=0) return GRIB_NO_VALUES; if (n<=0) return GRIB_NO_VALUES;
if (grib_sarray_used_size(stringValues)==1) { if (grib_sarray_used_size(stringValues)==1) {
n=1; n=1;
ival=0; ival=0;
} else { } else {
ival=self->iss_list->v[0]; ival=self->iss_list->v[0];
} }
if (n>grib_sarray_used_size(stringValues)) if (n>grib_sarray_used_size(stringValues))
@ -708,7 +716,7 @@ static void set_missing_long_to_double(grib_darray* dvalues)
/* ECC-750: The 'factor' argument is 10^-scale */ /* ECC-750: The 'factor' argument is 10^-scale */
static int descriptor_get_min_max(bufr_descriptor* bd, long width, long reference, double factor, static int descriptor_get_min_max(bufr_descriptor* bd, long width, long reference, double factor,
double* minAllowed, double* maxAllowed) double* minAllowed, double* maxAllowed)
{ {
/* Maximum value is allowed to be the largest number (all bits 1) which means it's MISSING */ /* Maximum value is allowed to be the largest number (all bits 1) which means it's MISSING */
unsigned long max1 = (1UL << width) - 1; /* Highest value for number with 'width' bits */ unsigned long max1 = (1UL << width) - 1; /* Highest value for number with 'width' bits */
@ -768,11 +776,11 @@ static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos, bufr
if (*v > maxAllowed || *v < minAllowed) { if (*v > maxAllowed || *v < minAllowed) {
if (dont_fail_if_out_of_range) { if (dont_fail_if_out_of_range) {
grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Value (%g) out of range (minAllowed=%g, maxAllowed=%g)." grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Value (%g) out of range (minAllowed=%g, maxAllowed=%g)."
" Setting it to missing value\n", bd->shortName, *v, minAllowed, maxAllowed); " Setting it to missing value\n", bd->shortName, *v, minAllowed, maxAllowed);
grib_set_bits_on(buff->data,pos,modifiedWidth); grib_set_bits_on(buff->data,pos,modifiedWidth);
} else { } else {
grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Value (%g) out of range (minAllowed=%g, maxAllowed=%g).", grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Value (%g) out of range (minAllowed=%g, maxAllowed=%g).",
bd->shortName, *v, minAllowed, maxAllowed); bd->shortName, *v, minAllowed, maxAllowed);
return GRIB_OUT_OF_RANGE; /* ECC-611 */ return GRIB_OUT_OF_RANGE; /* ECC-611 */
} }
} else { } else {
@ -790,8 +798,8 @@ static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos, bufr
val0=dvalues->v[self->iss_list->v[0]]; val0=dvalues->v[self->iss_list->v[0]];
is_constant=1; is_constant=1;
for (i=0;i<nvals;i++) { for (i=0;i<nvals;i++) {
values[i]=dvalues->v[self->iss_list->v[i]]; values[i]=dvalues->v[self->iss_list->v[i]];
if (val0 != values[i]) is_constant=0; if (val0 != values[i]) is_constant=0;
} }
v=values; v=values;
@ -822,8 +830,8 @@ static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos, bufr
/* Turn out-of-range values into 'missing' */ /* Turn out-of-range values into 'missing' */
if (*v!=GRIB_MISSING_DOUBLE && (*v < minAllowed || *v > maxAllowed)) { if (*v!=GRIB_MISSING_DOUBLE && (*v < minAllowed || *v > maxAllowed)) {
grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Value at index %ld (%g) out of range (minAllowed=%g, maxAllowed=%g)." grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Value at index %ld (%g) out of range (minAllowed=%g, maxAllowed=%g)."
" Setting it to missing value\n", " Setting it to missing value\n",
bd->shortName, (long)ii, *v, minAllowed, maxAllowed); bd->shortName, (long)ii, *v, minAllowed, maxAllowed);
*v = GRIB_MISSING_DOUBLE; *v = GRIB_MISSING_DOUBLE;
} }
ii++; ii++;
@ -850,12 +858,12 @@ static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos, bufr
} }
if (max>maxAllowed && max!=GRIB_MISSING_DOUBLE) { if (max>maxAllowed && max!=GRIB_MISSING_DOUBLE) {
grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Maximum value (value[%lu]=%g) out of range (maxAllowed=%g).", grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Maximum value (value[%lu]=%g) out of range (maxAllowed=%g).",
bd->shortName, index_of_max, max, maxAllowed, index_of_max); bd->shortName, index_of_max, max, maxAllowed, index_of_max);
return GRIB_OUT_OF_RANGE; return GRIB_OUT_OF_RANGE;
} }
if (min<minAllowed && min!=GRIB_MISSING_DOUBLE) { if (min<minAllowed && min!=GRIB_MISSING_DOUBLE) {
grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Minimum value (value[%lu]=%g) out of range (minAllowed=%g).", grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Minimum value (value[%lu]=%g) out of range (minAllowed=%g).",
bd->shortName, index_of_min, min, minAllowed); bd->shortName, index_of_min, min, minAllowed);
return GRIB_OUT_OF_RANGE; return GRIB_OUT_OF_RANGE;
} }
@ -931,13 +939,13 @@ static int encode_double_value(grib_context* c,grib_buffer* buff,long* pos,bufr_
else if (value>maxAllowed || value<minAllowed) { else if (value>maxAllowed || value<minAllowed) {
if (dont_fail_if_out_of_range) { if (dont_fail_if_out_of_range) {
grib_context_log(c, GRIB_LOG_ERROR, "encode_double_value: %s. Value (%g) out of range (minAllowed=%g, maxAllowed=%g)." grib_context_log(c, GRIB_LOG_ERROR, "encode_double_value: %s. Value (%g) out of range (minAllowed=%g, maxAllowed=%g)."
" Setting it to missing value\n", " Setting it to missing value\n",
bd->shortName, value, minAllowed, maxAllowed); bd->shortName, value, minAllowed, maxAllowed);
value = GRIB_MISSING_DOUBLE; /* Ignore the bad value and instead use 'missing' */ value = GRIB_MISSING_DOUBLE; /* Ignore the bad value and instead use 'missing' */
grib_set_bits_on(buff->data,pos,modifiedWidth); grib_set_bits_on(buff->data,pos,modifiedWidth);
} else { } else {
grib_context_log(c, GRIB_LOG_ERROR, "encode_double_value: %s. Value (%g) out of range (minAllowed=%g, maxAllowed=%g).", grib_context_log(c, GRIB_LOG_ERROR, "encode_double_value: %s. Value (%g) out of range (minAllowed=%g, maxAllowed=%g).",
bd->shortName, value, minAllowed, maxAllowed); bd->shortName, value, minAllowed, maxAllowed);
return GRIB_OUT_OF_RANGE; return GRIB_OUT_OF_RANGE;
} }
} }
@ -1109,28 +1117,28 @@ static int decode_replication(grib_context* c,grib_accessor_bufr_data_array* sel
grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data decoding: \tdelayed replication localReference width=%ld", descriptors[i]->width); grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data decoding: \tdelayed replication localReference width=%ld", descriptors[i]->width);
CHECK_END_DATA_RETURN(c, self, descriptors[i]->width+6, *err); CHECK_END_DATA_RETURN(c, self, descriptors[i]->width+6, *err);
if (*err) { if (*err) {
*numberOfRepetitions=0; *numberOfRepetitions=0;
} else { } else {
localReference=grib_decode_unsigned_long(data,pos,descriptors[i]->width)+descriptors[i]->reference; localReference=grib_decode_unsigned_long(data,pos,descriptors[i]->width)+descriptors[i]->reference;
grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data decoding: \tdelayed replication localWidth width=6"); grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data decoding: \tdelayed replication localWidth width=6");
width=grib_decode_unsigned_long(data,pos,6); width=grib_decode_unsigned_long(data,pos,6);
if (width) { if (width) {
grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data decoding: \tdelayed replication is NOT constant for compressed data!"); grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data decoding: \tdelayed replication is NOT constant for compressed data!");
/* delayed replication number is not constant. NOT IMPLEMENTED */ /* delayed replication number is not constant. NOT IMPLEMENTED */
return GRIB_NOT_IMPLEMENTED; return GRIB_NOT_IMPLEMENTED;
} else { } else {
*numberOfRepetitions=localReference*descriptors[i]->factor; *numberOfRepetitions=localReference*descriptors[i]->factor;
grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data decoding: \tdelayed replication value=%ld",*numberOfRepetitions); grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data decoding: \tdelayed replication value=%ld",*numberOfRepetitions);
} }
} }
} else { } else {
CHECK_END_DATA_RETURN(c, self, descriptors[i]->width, *err); CHECK_END_DATA_RETURN(c, self, descriptors[i]->width, *err);
if (*err) { if (*err) {
*numberOfRepetitions=0; *numberOfRepetitions=0;
} else { } else {
*numberOfRepetitions=grib_decode_unsigned_long(data,pos,descriptors[i]->width)+ *numberOfRepetitions=grib_decode_unsigned_long(data,pos,descriptors[i]->width)+
descriptors[i]->reference*descriptors[i]->factor; descriptors[i]->reference*descriptors[i]->factor;
grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data decoding: \tdelayed replication value=%ld",*numberOfRepetitions); grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data decoding: \tdelayed replication value=%ld",*numberOfRepetitions);
} }
} }
if (self->compressedData) { if (self->compressedData) {
@ -1364,7 +1372,7 @@ static int encode_element(grib_context* c,grib_accessor_bufr_data_array* self,in
err=encode_double_value(c,buff,pos,bd,self,self->numericValues->v[subsetIndex]->v[elementIndex]); err=encode_double_value(c,buff,pos,bd,self,self->numericValues->v[subsetIndex]->v[elementIndex]);
if (err) { if (err) {
grib_context_log(c,GRIB_LOG_ERROR,"Cannot encode %s=%g (subset=%d)", /*subsetIndex starts from 0*/ grib_context_log(c,GRIB_LOG_ERROR,"Cannot encode %s=%g (subset=%d)", /*subsetIndex starts from 0*/
bd->shortName, self->numericValues->v[subsetIndex]->v[elementIndex], subsetIndex+1); bd->shortName, self->numericValues->v[subsetIndex]->v[elementIndex], subsetIndex+1);
} }
} }
} }
@ -1957,9 +1965,9 @@ static GRIB_INLINE int significanceQualifierIndex(int X,int Y)
} }
static GRIB_INLINE void reset_deeper_qualifiers( static GRIB_INLINE void reset_deeper_qualifiers(
grib_accessor* significanceQualifierGroup[], grib_accessor* significanceQualifierGroup[],
const int* const significanceQualifierDepth, const int* const significanceQualifierDepth,
int numElements, int depth) int numElements, int depth)
{ {
int i; int i;
for (i=0;i<numElements;i++) { for (i=0;i<numElements;i++) {
@ -2003,7 +2011,7 @@ static GRIB_INLINE void reset_qualifiers(grib_accessor* significanceQualifierGro
for (i=0;i<number_of_qualifiers;i++) for (i=0;i<number_of_qualifiers;i++)
significanceQualifierGroup[i]=0; significanceQualifierGroup[i]=0;
} }
*/ */
static void grib_convert_to_attribute(grib_accessor* a) static void grib_convert_to_attribute(grib_accessor* a)
{ {
@ -2090,17 +2098,17 @@ static int is_bitmap_start_descriptor(grib_accessors_list* al, int* err)
case 224000: case 224000:
case 225000: case 225000:
case 232000: case 232000:
/*case 236000:*/ /*case 236000:*/
case 237000: case 237000:
/*case 243000:*/ /*case 243000:*/
{ {
#if 0 #if 0
long index[1]; long index[1];
grib_accessor* anindex=grib_accessor_get_attribute(al->accessor,"index"); grib_accessor* anindex=grib_accessor_get_attribute(al->accessor,"index");
grib_unpack_long(anindex,index,&l); grib_unpack_long(anindex,index,&l);
#endif #endif
return 1; return 1;
} }
} }
return 0; return 0;
} }
@ -2132,7 +2140,7 @@ static void print_bitmap_debug_info(grib_context* c, bitmap_s* bitmap, grib_acce
} }
static int bitmap_init(grib_context* c, bitmap_s* bitmap, static int bitmap_init(grib_context* c, bitmap_s* bitmap,
grib_accessors_list* bitmapStart, int bitmapSize, grib_accessors_list* lastAccessorInList) grib_accessors_list* bitmapStart, int bitmapSize, grib_accessors_list* lastAccessorInList)
{ {
int ret=0,i; int ret=0,i;
bitmap->cursor=bitmapStart->next; bitmap->cursor=bitmapStart->next;
@ -2305,7 +2313,7 @@ static int create_keys(grib_accessor* a,long onlySubset,long startSubset,long en
} }
elementFromBitmap=NULL; elementFromBitmap=NULL;
if (descriptor->F==0 && IS_QUALIFIER(descriptor->X) && if (descriptor->F==0 && IS_QUALIFIER(descriptor->X) &&
self->unpackMode==CODES_BUFR_UNPACK_STRUCTURE) { self->unpackMode==CODES_BUFR_UNPACK_STRUCTURE) {
int sidx=significanceQualifierIndex(descriptor->X,descriptor->Y); int sidx=significanceQualifierIndex(descriptor->X,descriptor->Y);
groupNumber++; groupNumber++;
@ -2315,7 +2323,7 @@ static int create_keys(grib_accessor* a,long onlySubset,long startSubset,long en
if (depth < max_depth) { if (depth < max_depth) {
/* If depth >= max_depth, then no entry will be deeper so no need for call */ /* If depth >= max_depth, then no entry will be deeper so no need for call */
reset_deeper_qualifiers(significanceQualifierGroup,significanceQualifierDepth, reset_deeper_qualifiers(significanceQualifierGroup,significanceQualifierDepth,
number_of_qualifiers,depth); number_of_qualifiers,depth);
} }
} else { } else {
/* if (forceGroupClosure) { */ /* if (forceGroupClosure) { */
@ -2362,7 +2370,7 @@ static int create_keys(grib_accessor* a,long onlySubset,long startSubset,long en
groupSection=bitmapGroup[bitmapIndex]->parent; groupSection=bitmapGroup[bitmapIndex]->parent;
depth=bitmapDepth[bitmapIndex]; depth=bitmapDepth[bitmapIndex];
reset_deeper_qualifiers(significanceQualifierGroup,significanceQualifierDepth, reset_deeper_qualifiers(significanceQualifierGroup,significanceQualifierDepth,
number_of_qualifiers,depth); number_of_qualifiers,depth);
/* TODO: This branch is not reached in our tests! */ /* TODO: This branch is not reached in our tests! */
reset_deeper_qualifiers(bitmapGroup,bitmapDepth,MAX_NUMBER_OF_BITMAPS,depth); reset_deeper_qualifiers(bitmapGroup,bitmapDepth,MAX_NUMBER_OF_BITMAPS,depth);
} else { } else {
@ -2470,7 +2478,7 @@ static int create_keys(grib_accessor* a,long onlySubset,long startSubset,long en
case 31021: case 31021:
associatedFieldSignificanceAccessor=elementAccessor; associatedFieldSignificanceAccessor=elementAccessor;
break; break;
/*case 33007:*/ /*case 33007:*/
/* ECC-690: See later */ /* ECC-690: See later */
/* break; */ /* break; */
default: default:
@ -2553,7 +2561,7 @@ static int set_to_missing_if_out_of_range(grib_handle* h)
/* First check if the transient key is set */ /* First check if the transient key is set */
long setToMissingIfOutOfRange=0; long setToMissingIfOutOfRange=0;
if (grib_get_long(h, "setToMissingIfOutOfRange", &setToMissingIfOutOfRange)==GRIB_SUCCESS && if (grib_get_long(h, "setToMissingIfOutOfRange", &setToMissingIfOutOfRange)==GRIB_SUCCESS &&
setToMissingIfOutOfRange != 0) setToMissingIfOutOfRange != 0)
{ {
return 1; return 1;
} }
@ -2767,10 +2775,10 @@ static int process_elements(grib_accessor* a,int flag,long onlySubset,long start
while (ip>=0 && n[ip]==0) { while (ip>=0 && n[ip]==0) {
nn[ip]--; nn[ip]--;
if (nn[ip]<=0) { if (nn[ip]<=0) {
numberOfNestedRepetitions--; numberOfNestedRepetitions--;
} else { } else {
n[ip]=numberOfElementsToRepeat[ip]; n[ip]=numberOfElementsToRepeat[ip];
i=startRepetition[ip]; i=startRepetition[ip];
} }
ip--; ip--;
} }

1
src/grib_api_internal.h
View File

@ -1053,6 +1053,7 @@ struct grib_context
int ieee_packing; int ieee_packing;
int bufrdc_mode; int bufrdc_mode;
int bufr_set_to_missing_if_out_of_range; int bufr_set_to_missing_if_out_of_range;
int bufr_multi_element_constant_arrays;
FILE* log_stream; FILE* log_stream;
grib_trie* classes; grib_trie* classes;
grib_trie* lists; grib_trie* lists;

11
src/grib_context.c
View File

@ -335,6 +335,7 @@ static grib_context default_grib_context = {
0, /* ieee_packing */ 0, /* ieee_packing */
0, /* bufrdc_mode */ 0, /* bufrdc_mode */
0, /* bufr_set_to_missing_if_out_of_range */ 0, /* bufr_set_to_missing_if_out_of_range */
0, /* bufr_multi_element_constant_arrays */
0, /* log_stream */ 0, /* log_stream */
0, /* classes */ 0, /* classes */
0, /* lists */ 0, /* lists */
@ -368,6 +369,7 @@ grib_context* grib_context_get_default()
const char* keep_matrix = NULL; const char* keep_matrix = NULL;
const char* bufrdc_mode = NULL; const char* bufrdc_mode = NULL;
const char* bufr_set_to_missing_if_out_of_range = NULL; const char* bufr_set_to_missing_if_out_of_range = NULL;
const char* bufr_multi_element_constant_arrays = NULL;
const char* file_pool_max_opened_files = NULL; const char* file_pool_max_opened_files = NULL;
#ifdef ENABLE_FLOATING_POINT_EXCEPTIONS #ifdef ENABLE_FLOATING_POINT_EXCEPTIONS
@ -375,8 +377,9 @@ grib_context* grib_context_get_default()
#endif #endif
write_on_fail = codes_getenv("ECCODES_GRIB_WRITE_ON_FAIL"); write_on_fail = codes_getenv("ECCODES_GRIB_WRITE_ON_FAIL");
bufrdc_mode = codes_getenv("ECCODES_BUFRDC_MODE_ON"); bufrdc_mode = getenv("ECCODES_BUFRDC_MODE_ON");
bufr_set_to_missing_if_out_of_range = codes_getenv("ECCODES_BUFR_SET_TO_MISSING_IF_OUT_OF_RANGE"); bufr_set_to_missing_if_out_of_range = getenv("ECCODES_BUFR_SET_TO_MISSING_IF_OUT_OF_RANGE");
bufr_multi_element_constant_arrays = getenv("ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS");
large_constant_fields = codes_getenv("ECCODES_GRIB_LARGE_CONSTANT_FIELDS"); large_constant_fields = codes_getenv("ECCODES_GRIB_LARGE_CONSTANT_FIELDS");
no_abort = codes_getenv("ECCODES_NO_ABORT"); no_abort = codes_getenv("ECCODES_NO_ABORT");
debug = codes_getenv("ECCODES_DEBUG"); debug = codes_getenv("ECCODES_DEBUG");
@ -387,7 +390,7 @@ grib_context* grib_context_get_default()
no_big_group_split = codes_getenv("ECCODES_GRIB_NO_BIG_GROUP_SPLIT"); no_big_group_split = codes_getenv("ECCODES_GRIB_NO_BIG_GROUP_SPLIT");
no_spd = codes_getenv("ECCODES_GRIB_NO_SPD"); no_spd = codes_getenv("ECCODES_GRIB_NO_SPD");
keep_matrix = codes_getenv("ECCODES_GRIB_KEEP_MATRIX"); keep_matrix = codes_getenv("ECCODES_GRIB_KEEP_MATRIX");
file_pool_max_opened_files = codes_getenv("ECCODES_FILE_POOL_MAX_OPENED_FILES"); file_pool_max_opened_files = getenv("ECCODES_FILE_POOL_MAX_OPENED_FILES");
/* On UNIX, when we read from a file we get exactly what is in the file on disk. /* On UNIX, when we read from a file we get exactly what is in the file on disk.
* But on Windows a file can be opened in binary or text mode. In binary mode the system behaves exactly as in UNIX. * But on Windows a file can be opened in binary or text mode. In binary mode the system behaves exactly as in UNIX.
@ -473,6 +476,8 @@ grib_context* grib_context_get_default()
default_grib_context.bufrdc_mode = bufrdc_mode ? atoi(bufrdc_mode) : 0; default_grib_context.bufrdc_mode = bufrdc_mode ? atoi(bufrdc_mode) : 0;
default_grib_context.bufr_set_to_missing_if_out_of_range = bufr_set_to_missing_if_out_of_range ? default_grib_context.bufr_set_to_missing_if_out_of_range = bufr_set_to_missing_if_out_of_range ?
atoi(bufr_set_to_missing_if_out_of_range) : 0; atoi(bufr_set_to_missing_if_out_of_range) : 0;
default_grib_context.bufr_multi_element_constant_arrays = bufr_multi_element_constant_arrays ?
atoi(bufr_multi_element_constant_arrays) : 0;
default_grib_context.file_pool_max_opened_files = file_pool_max_opened_files ? default_grib_context.file_pool_max_opened_files = file_pool_max_opened_files ?
atoi(file_pool_max_opened_files) : DEFAULT_FILE_POOL_MAX_OPENED_FILES; atoi(file_pool_max_opened_files) : DEFAULT_FILE_POOL_MAX_OPENED_FILES;
} }

1
tests/CMakeLists.txt
View File

@ -87,6 +87,7 @@ list( APPEND tests_data_reqd
bufr_keys_iter bufr_keys_iter
bufr_get_element bufr_get_element
bufr_wmo_tables bufr_wmo_tables
bufr_ecc-428
bufr_ecc-197 bufr_ecc-197
bufr_ecc-286 bufr_ecc-286
bufr_ecc-288 bufr_ecc-288

56
tests/bufr_ecc-428.sh Executable file
View File

@ -0,0 +1,56 @@
#!/bin/sh
# Copyright 2005-2019 ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
#
. ./include.sh
# ---------------------------------------------------------
# This is the test for the JIRA issue ECC-428
# Decoding compressed BUFR data:
# Option to have constant arrays with multiple repeated values
# rather than a single value
# ---------------------------------------------------------
cd ${data_dir}/bufr
label="bufr_ecc_428_test"
tempRules=temp.${label}.filter
tempText=temp.${label}.text
tempRef1=temp.${label}.ref1
tempRef2=temp.${label}.ref2
# --------------------------------------------------------
# Test 1
# --------------------------------------------------------
bufrFile=airs_57.bufr # this has 15 subsets
cat > $tempRules <<EOF
set unpack=1;
print "[satelliteIdentifier!0]";
EOF
${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText
echo "784" > $tempRef1
diff $tempRef1 $tempText
export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=1
${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText
echo "784 784 784 784 784 784 784 784 784 784 784 784 784 784 784" > $tempRef2
diff $tempRef2 $tempText
unset ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS
${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText
diff $tempRef1 $tempText
export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=0
${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText
diff $tempRef1 $tempText
# ------------------------
rm -rf $tempRules $tempText $tempRef1 $tempRef2