From fec6c51eac2f39eb7979484cc74de075e30886bc Mon Sep 17 00:00:00 2001 From: Enrico Fucile Date: Tue, 5 Jul 2016 11:31:56 +0100 Subject: [PATCH] several fixes to re-create a message with bufr_dump -Efilter ECC-289 ECC-284 ECC-290 ECC-291 ECC-285 --- definitions/bufr/section.3.def | 4 +-- src/grib_accessor_class_bits.c | 4 +-- src/grib_accessor_class_bufr_data_array.c | 14 ++++++-- src/grib_accessor_class_bufr_data_element.c | 8 +++-- src/grib_dumper_class_filter.c | 32 +++++------------- src/grib_value.c | 1 + tests/bufr_filter.sh | 36 +++++++++++++++------ 7 files changed, 56 insertions(+), 43 deletions(-) diff --git a/definitions/bufr/section.3.def b/definitions/bufr/section.3.def index e808cf19a..6eeba634a 100644 --- a/definitions/bufr/section.3.def +++ b/definitions/bufr/section.3.def @@ -54,10 +54,10 @@ if (section2Present && bufrHeaderCentre==98 && section2Length==52) { if (numberOfSubsets>255 || ( rdbSubtype>=121 && rdbSubtype <=130 ) || rdbSubtype==31) { - meta ls.numberOfObservations bits(keySat,0,16) : dump,long_type,no_copy; + meta ls.localNumberOfObservations bits(keySat,0,16) : dump,long_type,no_copy; meta ls.satelliteID bits(keySat,16,16) : dump,long_type,no_copy; } else { - meta ls.numberOfObservations bits(keySat,0,8) : dump,long_type,no_copy; + meta ls.localNumberOfObservations bits(keySat,0,8) : dump,long_type,no_copy; meta ls.satelliteID bits(keySat,8,16) : dump,long_type,no_copy; } } else { diff --git a/src/grib_accessor_class_bits.c b/src/grib_accessor_class_bits.c index 70aa4421d..5b8cd92fc 100644 --- a/src/grib_accessor_class_bits.c +++ b/src/grib_accessor_class_bits.c @@ -221,7 +221,7 @@ static int unpack_double(grib_accessor* a, double* val, size_t *len) p = h->buffer->data + grib_byte_offset(x); *val=grib_decode_unsigned_long(p,&start,length); - *val=(*val+self->referenceValue)/self->scale; + *val=((long)*val+self->referenceValue)/self->scale; *len=1; @@ -246,7 +246,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len) p=h->buffer->data + grib_byte_offset(x); - lval= *val *self->scale - self->referenceValue; + lval= round(*val * self->scale) - self->referenceValue; return grib_encode_unsigned_longb(p,lval,&start,length); } diff --git a/src/grib_accessor_class_bufr_data_array.c b/src/grib_accessor_class_bufr_data_array.c index 407e46e22..aac9a0531 100644 --- a/src/grib_accessor_class_bufr_data_array.c +++ b/src/grib_accessor_class_bufr_data_array.c @@ -530,6 +530,13 @@ static int encode_string_array(grib_context* c,grib_buffer* buff,long* pos, bufr return err; } +static void set_missing_long_to_double(grib_darray* dvalues) { + size_t i,n=grib_darray_used_size(dvalues); + for (i=0;iv[i]==GRIB_MISSING_LONG) dvalues->v[i]=GRIB_MISSING_DOUBLE; + } +} + static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos, bufr_descriptor* bd, grib_accessor_bufr_data_array* self,grib_darray* dvalues) { @@ -557,6 +564,8 @@ static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos, bufr nvals=self->end - self->start; if (nvals<=0) return GRIB_NO_VALUES; + set_missing_long_to_double(dvalues); + v=dvalues->v; /* is constant */ @@ -1338,7 +1347,8 @@ static void set_creator_name(grib_action* creator,int code) creator->name = (char*)"associatedField"; break; default : - creator->name = (char*)"operator"; + if (code > 204999 && code < 206000) creator->name = (char*)"text"; + else creator->name = (char*)"operator"; break; } } @@ -1429,7 +1439,7 @@ static grib_accessor* create_accessor_from_descriptor(grib_accessor* a,grib_acce break; case 2: set_creator_name(&creator,self->expanded->v[idx]->code); - if (bufr_descriptor_is_marker(self->expanded->v[idx])) { + if (bufr_descriptor_is_marker(self->expanded->v[idx]) ) { elementAccessor = grib_accessor_factory(section, &creator, 0, NULL); if (self->canBeMissing[idx]) elementAccessor->flags |= GRIB_ACCESSOR_FLAG_CAN_BE_MISSING; accessor_bufr_data_element_set_index(elementAccessor,ide); diff --git a/src/grib_accessor_class_bufr_data_element.c b/src/grib_accessor_class_bufr_data_element.c index fabf3532c..cf2479c82 100644 --- a/src/grib_accessor_class_bufr_data_element.c +++ b/src/grib_accessor_class_bufr_data_element.c @@ -388,12 +388,14 @@ static int pack_string(grib_accessor* a, const char* val, size_t *len) grib_accessor_bufr_data_element* self = (grib_accessor_bufr_data_element*)a; int ret=0,idx; - /*long count=0;*/ char* s=NULL; grib_context* c=a->context; - /*count=self->numberOfSubsets;*/ - idx=((int)self->numericValues->v[self->index]->v[0]/1000-1)/self->numberOfSubsets; + if (self->compressedData) { + idx=((int)self->numericValues->v[self->index]->v[0]/1000-1)/self->numberOfSubsets; + } else { + idx=(int)self->numericValues->v[self->subsetNumber]->v[self->index]/1000-1; + } grib_sarray_delete(c,self->stringValues->v[idx]); self->stringValues->v[idx]=grib_sarray_new(c,1,1); s=grib_context_strdup(c,val); diff --git a/src/grib_dumper_class_filter.c b/src/grib_dumper_class_filter.c index 3bb8e32b0..762359133 100644 --- a/src/grib_dumper_class_filter.c +++ b/src/grib_dumper_class_filter.c @@ -181,7 +181,6 @@ static void dump_values(grib_dumper* d,grib_accessor* a) int i,r; int cols=9; long count=0; - double missing_value = GRIB_MISSING_DOUBLE; grib_context* c=a->context; grib_handle* h=grib_handle_of_accessor(a); @@ -201,7 +200,6 @@ static void dump_values(grib_dumper* d,grib_accessor* a) self->begin=0; self->empty=0; - err = grib_set_double(h, "missingValue", missing_value); if (size>1) { int icount=0; @@ -214,17 +212,11 @@ static void dump_values(grib_dumper* d,grib_accessor* a) for (i=0; icols || i==0) {fprintf(self->dumper.out,"\n ");icount=0;} - if (values[i] == missing_value) - fprintf(self->dumper.out,"missing, \n"); - else - fprintf(self->dumper.out,"%g, ", values[i]); + fprintf(self->dumper.out,"%.18e, ", values[i]); icount++; } if (icount>cols || i==0) {fprintf(self->dumper.out,"\n ");icount=0;} - if (grib_is_missing_double(a,values[i])) - fprintf(self->dumper.out, "%s","missing"); - else - fprintf(self->dumper.out, "%g",values[i]); + fprintf(self->dumper.out, "%.18e",values[i]); depth-=2; fprintf(self->dumper.out,"};\n"); @@ -238,7 +230,7 @@ static void dump_values(grib_dumper* d,grib_accessor* a) else fprintf(self->dumper.out,"set %s=",a->name); - fprintf(self->dumper.out,"%g;\n",value); + fprintf(self->dumper.out,"%.18e;\n",value); } } @@ -256,7 +248,7 @@ static void dump_long(grib_dumper* d,grib_accessor* a,const char* comment) long value; size_t size = 1; long *values=NULL; int err = 0; - int i,r; + int i,r,icount; int cols=9; long count=0; grib_handle* h=grib_handle_of_accessor(a); @@ -278,7 +270,7 @@ static void dump_long(grib_dumper* d,grib_accessor* a,const char* comment) self->empty=0; if (size>1) { - int icount=0; + icount=0; if ((r=get_key_rank(h,self->keys,a->name))!=0) fprintf(self->dumper.out,"set #%d#%s=",r,a->name); else @@ -288,19 +280,11 @@ static void dump_long(grib_dumper* d,grib_accessor* a,const char* comment) for (i=0;icols || i==0) {fprintf(self->dumper.out,"\n ");icount=0;} - if (grib_is_missing_long(a,values[i])) { - fprintf(self->dumper.out,"missing, "); - } else { - fprintf(self->dumper.out,"%ld, ",values[i]); - } + fprintf(self->dumper.out,"%ld, ",values[i]); icount++; } if (icount>cols || i==0) {fprintf(self->dumper.out,"\n ");icount=0;} - if (grib_is_missing_long(a,values[i])) { - fprintf(self->dumper.out,"missing "); - } else { - fprintf(self->dumper.out,"%ld ",values[i]); - } + fprintf(self->dumper.out,"%ld ",values[i]); depth-=2; fprintf(self->dumper.out,"};\n"); @@ -349,7 +333,7 @@ static void dump_double(grib_dumper* d,grib_accessor* a,const char* comment) else fprintf(self->dumper.out,"set %s=",a->name); - fprintf(self->dumper.out,"%g;\n",value); + fprintf(self->dumper.out,"%.18e;\n",value); } if (self->isLeaf==0) { diff --git a/src/grib_value.c b/src/grib_value.c index 6f772eebb..8f2c8fa13 100644 --- a/src/grib_value.c +++ b/src/grib_value.c @@ -546,6 +546,7 @@ int grib_is_missing_string(grib_accessor* a,unsigned char* x,size_t len) int ret; size_t i=0; + if (len==0) return 0; ret=1; for (i=0;i ${fOut}.log.ref <> $fLog -echo "file: $f" >> $fLog +for f in $files +do + echo "Test: bufr_dump -Efilter " >> $fLog + echo "file: $f" >> $fLog -${tools_dir}bufr_dump -Efilter $g > $fRules -${tools_dir}bufr_filter -o $o $fRules $f -${tools_dir}bufr_compare $o $g + ${tools_dir}bufr_dump -Efilter $f > $fRules + ${tools_dir}bufr_filter -o ${f}.out $fRules $f + ${tools_dir}bufr_compare ${f}.out $f -rm -f $o $fRules + rm -f ${f}.out $fRules + +done