From ee94336bda61bc24f2a5793404590639f182791b Mon Sep 17 00:00:00 2001 From: Enrico Fucile Date: Fri, 20 Feb 2015 17:44:19 +0000 Subject: [PATCH] bufr_dump -jL is providing a flat list of the content and is still a valid json. Test added in bufr_json.sh. More work done on the attributes. Not final yet. --- src/grib_accessor.c | 10 +- src/grib_accessor_class_bufr_data_array.c | 130 +++++++++++++----- src/grib_accessor_class_bufr_elements_table.c | 13 ++ ...grib_accessor_class_expanded_descriptors.c | 1 - src/grib_api_internal.h | 1 + src/grib_api_prototypes.h | 4 + src/grib_bufr_descriptor.c | 1 + src/grib_dumper_class.c | 22 +++ src/grib_dumper_class_json.c | 2 + tests/bufr_filter.sh | 40 ++++++ tests/bufr_json.sh | 12 +- 11 files changed, 194 insertions(+), 42 deletions(-) diff --git a/src/grib_accessor.c b/src/grib_accessor.c index a1bf10d68..86ff2a3a0 100644 --- a/src/grib_accessor.c +++ b/src/grib_accessor.c @@ -653,9 +653,13 @@ void grib_accessors_list_push(grib_accessors_list* al,grib_accessor* a) { grib_context* c=a->parent->h->context; last=grib_accessors_list_last(al); - last->next=(grib_accessors_list*)grib_context_malloc_clear(c,sizeof(grib_accessors_list)); - last->next->accessor=a; - last->next->prev=last; + if (last->accessor) { + last->next=(grib_accessors_list*)grib_context_malloc_clear(c,sizeof(grib_accessors_list)); + last->next->accessor=a; + last->next->prev=last; + } else { + last->accessor=a; + } } grib_accessors_list* grib_accessors_list_last(grib_accessors_list* al) { diff --git a/src/grib_accessor_class_bufr_data_array.c b/src/grib_accessor_class_bufr_data_array.c index 439f41a9d..e47ac05bc 100644 --- a/src/grib_accessor_class_bufr_data_array.c +++ b/src/grib_accessor_class_bufr_data_array.c @@ -302,6 +302,11 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len) return GRIB_NOT_IMPLEMENTED; } +grib_accessors_list* accessor_bufr_data_array_get_dataAccessors(grib_accessor* a) { + grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a; + return self->dataAccessors; +} + static int get_descriptors(grib_accessor* a) { grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a; int ret=0,i,numberOfDescriptors; @@ -598,7 +603,27 @@ static grib_accessor* create_attribute(char* name,grib_section* section,int type return a; } -static grib_accessor* create_accessor_from_descriptor(grib_accessor* a,grib_section* section,long ide,long subset) { +static void set_creator_name(grib_action* creator,int code) { + switch (code) { + case 223255: + creator->name="substitutedValue"; + break; + case 224255: + creator->name="firstOrderStatisticalValue"; + break; + case 225255: + creator->name="differenceStatisticalValue"; + break; + case 232255: + creator->name="replacedRetainedValue"; + break; + default : + creator->name="unknown"; + break; + } +} + +static grib_accessor* create_accessor_from_descriptor(grib_accessor* a,grib_section* section,long ide,long subset,int dump) { grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a; char code[7]={0,}; int idx=0; @@ -609,7 +634,6 @@ static grib_accessor* create_accessor_from_descriptor(grib_accessor* a,grib_sect grib_action creator = {0, }; creator.op = "bufr_data_element"; creator.name_space = ""; - creator.flags = GRIB_ACCESSOR_FLAG_DUMP; creator.set = 0; operatorCreator.op = "variable"; @@ -618,43 +642,69 @@ static grib_accessor* create_accessor_from_descriptor(grib_accessor* a,grib_sect operatorCreator.set = 0; operatorCreator.name="operator"; + if (dump) { + creator.flags = GRIB_ACCESSOR_FLAG_DUMP; + operatorCreator.flags = GRIB_ACCESSOR_FLAG_DUMP; + } + idx = self->compressedData ? self->elementsDescriptorsIndex->v[0]->v[ide] : self->elementsDescriptorsIndex->v[subset]->v[ide] ; - creator.name=self->expanded->v[idx]->shortName; - /* only table B element are created as they only have a shortName*/ - if (creator.name) { - elementAccessor = grib_accessor_factory(section, &creator, 0, NULL); - if (self->canBeMissing[idx]) elementAccessor->flags |= GRIB_ACCESSOR_FLAG_CAN_BE_MISSING; - accessor_bufr_data_element_set_index(elementAccessor,ide); - accessor_bufr_data_element_set_descriptors(elementAccessor,self->expanded); - accessor_bufr_data_element_set_numericValues(elementAccessor,self->numericValues); - accessor_bufr_data_element_set_stringValues(elementAccessor,self->stringValues); - accessor_bufr_data_element_set_compressedData(elementAccessor,self->compressedData); - accessor_bufr_data_element_set_type(elementAccessor,self->expanded->v[idx]->type); - accessor_bufr_data_element_set_numberOfSubsets(elementAccessor,self->numberOfSubsets); - accessor_bufr_data_element_set_subsetNumber(elementAccessor,subset); + switch (self->expanded->v[idx]->F) { + case 0: + creator.name=self->expanded->v[idx]->shortName; + elementAccessor = grib_accessor_factory(section, &creator, 0, NULL); + if (self->canBeMissing[idx]) elementAccessor->flags |= GRIB_ACCESSOR_FLAG_CAN_BE_MISSING; + accessor_bufr_data_element_set_index(elementAccessor,ide); + accessor_bufr_data_element_set_descriptors(elementAccessor,self->expanded); + accessor_bufr_data_element_set_numericValues(elementAccessor,self->numericValues); + accessor_bufr_data_element_set_stringValues(elementAccessor,self->stringValues); + accessor_bufr_data_element_set_compressedData(elementAccessor,self->compressedData); + accessor_bufr_data_element_set_type(elementAccessor,self->expanded->v[idx]->type); + accessor_bufr_data_element_set_numberOfSubsets(elementAccessor,self->numberOfSubsets); + accessor_bufr_data_element_set_subsetNumber(elementAccessor,subset); - self->expanded->v[idx]->a=elementAccessor; + self->expanded->v[idx]->a=elementAccessor; - sprintf(code,"%06ld",self->expanded->v[idx]->code); - attribute=create_attribute("code",section,GRIB_TYPE_STRING,code,0,0); - grib_accessor_add_attribute(elementAccessor,attribute); + sprintf(code,"%06ld",self->expanded->v[idx]->code); + attribute=create_attribute("code",section,GRIB_TYPE_STRING,code,0,0); + grib_accessor_add_attribute(elementAccessor,attribute); - attribute=create_attribute("units",section,GRIB_TYPE_STRING,self->expanded->v[idx]->units,0,0); - grib_accessor_add_attribute(elementAccessor,attribute); + attribute=create_attribute("units",section,GRIB_TYPE_STRING,self->expanded->v[idx]->units,0,0); + grib_accessor_add_attribute(elementAccessor,attribute); - attribute=create_attribute("scale",section,GRIB_TYPE_LONG,0,0,self->expanded->v[idx]->scale); - grib_accessor_add_attribute(elementAccessor,attribute); + attribute=create_attribute("scale",section,GRIB_TYPE_LONG,0,0,self->expanded->v[idx]->scale); + grib_accessor_add_attribute(elementAccessor,attribute); - attribute=create_attribute("reference",section,GRIB_TYPE_DOUBLE,0,self->expanded->v[idx]->reference,0); - grib_accessor_add_attribute(elementAccessor,attribute); + attribute=create_attribute("reference",section,GRIB_TYPE_DOUBLE,0,self->expanded->v[idx]->reference,0); + grib_accessor_add_attribute(elementAccessor,attribute); - attribute=create_attribute("width",section,GRIB_TYPE_LONG,0,0,self->expanded->v[idx]->width); - grib_accessor_add_attribute(elementAccessor,attribute); - } else { - elementAccessor = grib_accessor_factory(section, &operatorCreator, 0, NULL); - accessor_variable_set_type(elementAccessor,GRIB_TYPE_LONG); + attribute=create_attribute("width",section,GRIB_TYPE_LONG,0,0,self->expanded->v[idx]->width); + grib_accessor_add_attribute(elementAccessor,attribute); + break; + case 2: + if (self->expanded->v[idx]->isMarker) { + set_creator_name(&creator,self->expanded->v[idx]->code); + elementAccessor = grib_accessor_factory(section, &creator, 0, NULL); + if (self->canBeMissing[idx]) elementAccessor->flags |= GRIB_ACCESSOR_FLAG_CAN_BE_MISSING; + accessor_bufr_data_element_set_index(elementAccessor,ide); + accessor_bufr_data_element_set_descriptors(elementAccessor,self->expanded); + accessor_bufr_data_element_set_numericValues(elementAccessor,self->numericValues); + accessor_bufr_data_element_set_stringValues(elementAccessor,self->stringValues); + accessor_bufr_data_element_set_compressedData(elementAccessor,self->compressedData); + accessor_bufr_data_element_set_type(elementAccessor,self->expanded->v[idx]->type); + accessor_bufr_data_element_set_numberOfSubsets(elementAccessor,self->numberOfSubsets); + accessor_bufr_data_element_set_subsetNumber(elementAccessor,subset); + + } else { + elementAccessor = grib_accessor_factory(section, &operatorCreator, 0, NULL); + accessor_variable_set_type(elementAccessor,GRIB_TYPE_LONG); + sprintf(code,"%06ld",self->expanded->v[idx]->code); + attribute=create_attribute("code",section,GRIB_TYPE_STRING,code,0,0); + grib_accessor_add_attribute(elementAccessor,attribute); + } + self->expanded->v[idx]->a=elementAccessor; + break; } return elementAccessor; @@ -736,7 +786,7 @@ static int create_keys(grib_accessor* a) { int incrementBitmapIndex=1; grib_accessor* elementFromBitmap=NULL; int reuseBitmap=0; - int i; + int i,dump=1;; creatorGroup.op = "bufr_group"; creatorGroup.name="groupNumber"; @@ -802,6 +852,7 @@ static int create_keys(grib_accessor* a) { significanceQualifierGroup[sidx]=gaGroup; significanceQualifierDepth[sidx]=depth; incrementBitmapIndex=1; + dump=1; } else if (descriptor->code == 31031 && incrementBitmapIndex!=0) { /* bitmap */ bitmapIndex++; @@ -830,20 +881,25 @@ static int create_keys(grib_accessor* a) { sectionUp=gaGroup->parent; bitmapGroup[bitmapIndex]=gaGroup; bitmapDepth[bitmapIndex]=depth; + dump=1; } else if (descriptor->code == 31031) { + dump=1; bitmapSize[bitmapIndex]++; bitmap.cursor=0; - } else if (descriptor->code == 222000 ) { + } else if (descriptor->code == 222000 || descriptor->code == 224000) { bitmap.referredElement=NULL; qualityPresent=1; + dump=1; } else if (descriptor->code == 236000 || descriptor->code == 237000 ) { bitmap.referredElement=NULL; bitmap.cursor=0; reuseBitmap=1; + dump=0; } else if (descriptor->code == 237255 ) { reuseBitmap=0; bitmap.cursor=0; - } else if (descriptor->X==33 && qualityPresent) { + dump=1; + } else if ( ( descriptor->X==33 || descriptor->isMarker ) && qualityPresent) { if (!bitmap.referredElement) { bitmap.cursor=bitmapStart[bitmapIndex]->next; bitmap.referredElement=bitmap.cursor->prev; @@ -852,12 +908,13 @@ static int create_keys(grib_accessor* a) { } } elementFromBitmap=get_element_from_bitmap(a,&bitmap); + dump=1; } - elementAccessor=create_accessor_from_descriptor(a,section,ide,iss); + elementAccessor=create_accessor_from_descriptor(a,section,ide,iss,dump); if (elementFromBitmap) grib_accessor_add_attribute(elementFromBitmap,elementAccessor); else if (elementAccessor) { - if (descriptor->F == 0) grib_push_accessor(elementAccessor,section->block); + grib_push_accessor(elementAccessor,section->block); grib_accessors_list_push(self->dataAccessors,elementAccessor); } } @@ -1006,7 +1063,8 @@ static int decode_elements(grib_accessor* a) { if (descriptors[i]->Y==255) { index=get_next_bitmap_descriptor_index(self,elementsDescriptorsIndex,dval); decode_element(c,self,iss,data,&pos,index,dval,sval); - grib_iarray_push(elementsDescriptorsIndex,index); + /* self->expanded->v[index] */ + grib_iarray_push(elementsDescriptorsIndex,i); } else { grib_iarray_push(elementsDescriptorsIndex,i); push_zero_element(self,dval); diff --git a/src/grib_accessor_class_bufr_elements_table.c b/src/grib_accessor_class_bufr_elements_table.c index 2a04171d7..55d866f04 100644 --- a/src/grib_accessor_class_bufr_elements_table.c +++ b/src/grib_accessor_class_bufr_elements_table.c @@ -340,6 +340,18 @@ static int bufr_get_from_table(grib_accessor* a,bufr_descriptor* v) { return ret; } +int bufr_is_marker(int code) { + switch (code) { + case 223255: + case 224255: + case 225255: + case 232255: + return 1; + default : + return 0; + } +} + bufr_descriptor* accessor_bufr_elements_table_get_descriptor(grib_accessor* a,int code,int *err) { grib_context* c; bufr_descriptor* v=NULL; @@ -360,6 +372,7 @@ bufr_descriptor* accessor_bufr_elements_table_get_descriptor(grib_accessor* a,in v->F=code/100000; v->X=(code-v->F*100000)/1000; v->Y=(code-v->F*100000)%1000; + v->isMarker=bufr_is_marker(code); switch (v->F) { case 0: diff --git a/src/grib_accessor_class_expanded_descriptors.c b/src/grib_accessor_class_expanded_descriptors.c index 62a8c542a..2865260c7 100644 --- a/src/grib_accessor_class_expanded_descriptors.c +++ b/src/grib_accessor_class_expanded_descriptors.c @@ -165,7 +165,6 @@ typedef struct change_coding_params { double referenceFactor; } change_coding_params ; - static void init(grib_accessor* a, const long len , grib_arguments* args ) { grib_accessor_expanded_descriptors* self = (grib_accessor_expanded_descriptors*)a; diff --git a/src/grib_api_internal.h b/src/grib_api_internal.h index a45d57d39..8dc7be22d 100644 --- a/src/grib_api_internal.h +++ b/src/grib_api_internal.h @@ -765,6 +765,7 @@ struct bufr_descriptor { double factor; double reference; long width; + int isMarker; grib_accessor* a; } ; diff --git a/src/grib_api_prototypes.h b/src/grib_api_prototypes.h index 8e221da5f..d970bf0c3 100644 --- a/src/grib_api_prototypes.h +++ b/src/grib_api_prototypes.h @@ -262,6 +262,7 @@ size_t grib_viarray_used_size(grib_viarray *v); /* grib_accessor_class_bufr_data.c */ /* grib_accessor_class_bufr_data_array.c */ +grib_accessors_list *accessor_bufr_data_array_get_dataAccessors(grib_accessor *a); /* grib_accessor_class_bufr_data_element.c */ void accessor_bufr_data_element_set_index(grib_accessor *a, long index); @@ -276,6 +277,7 @@ void accessor_bufr_data_element_set_elementsDescriptorsIndex(grib_accessor *a, g /* grib_accessor_class_bufr_elements_table.c */ char **str_split(char *a_str, const char a_delim); +int bufr_is_marker(int code); bufr_descriptor *accessor_bufr_elements_table_get_descriptor(grib_accessor *a, int code, int *err); /* grib_accessor_class_bufr_group.c */ @@ -872,8 +874,10 @@ void grib_dump_footer(grib_dumper *d, grib_handle *h); /* grib_dumper_class.c */ grib_dumper *grib_dumper_factory(const char *op, grib_handle *h, FILE *out, unsigned long option_flags, void *arg); void grib_dump_accessors_block(grib_dumper *dumper, grib_block_of_accessors *block); +void grib_dump_accessors_list(grib_dumper *dumper, grib_accessors_list *al); int grib_print(grib_handle *h, const char *name, grib_dumper *d); void grib_dump_content(grib_handle *h, FILE *f, const char *mode, unsigned long option_flags, void *data); +void grib_dump_bufr_flat(grib_accessors_list *al, grib_handle *h, FILE *f, const char *mode, unsigned long option_flags, void *data); /* grib_context.c */ size_t grib_context_read(const grib_context *c, void *ptr, size_t size, void *stream); diff --git a/src/grib_bufr_descriptor.c b/src/grib_bufr_descriptor.c index f5107f4ca..16f2efc76 100644 --- a/src/grib_bufr_descriptor.c +++ b/src/grib_bufr_descriptor.c @@ -63,6 +63,7 @@ int grib_bufr_descriptor_set_code(grib_accessor* tables_accessor,int code,bufr_d v->F=code/100000; v->X=(code-v->F*100000)/1000; v->Y=(code-v->F*100000)%1000; + v->isMarker=bufr_is_marker(code); } else { if (tables_accessor==NULL) return GRIB_NULL_POINTER; d=accessor_bufr_elements_table_get_descriptor(tables_accessor,code,&err); diff --git a/src/grib_dumper_class.c b/src/grib_dumper_class.c index ea98114a0..0ef40e05e 100644 --- a/src/grib_dumper_class.c +++ b/src/grib_dumper_class.c @@ -61,6 +61,19 @@ void grib_dump_accessors_block(grib_dumper* dumper,grib_block_of_accessors* bloc } } +void grib_dump_accessors_list(grib_dumper* dumper,grib_accessors_list* al) +{ + grib_accessors_list* cur=al; + grib_accessors_list* next=al->next; + + while(next) { + grib_accessor_dump(cur->accessor,dumper); + cur=next; + next=cur->next; + } + +} + int grib_print (grib_handle* h, const char* name, grib_dumper *d ){ grib_accessor* act = grib_find_accessor(h, name); @@ -83,3 +96,12 @@ void grib_dump_content(grib_handle* h, FILE* f,const char* mode,unsigned long op grib_dumper_delete(dumper); } +void grib_dump_bufr_flat(grib_accessors_list* al,grib_handle* h, FILE* f,const char* mode,unsigned long option_flags,void *data) +{ + grib_dumper *dumper; + dumper = grib_dumper_factory(mode?mode:"serialize",h,f,option_flags,data); + grib_dump_header(dumper,h); + grib_dump_accessors_list(dumper,al); + grib_dump_footer(dumper,h); + grib_dumper_delete(dumper); +} diff --git a/src/grib_dumper_class_json.c b/src/grib_dumper_class_json.c index 0cd988f70..6d8b19d1f 100644 --- a/src/grib_dumper_class_json.c +++ b/src/grib_dumper_class_json.c @@ -440,6 +440,8 @@ static void dump_section(grib_dumper* d,grib_accessor* a,grib_block_of_accessors fprintf(self->dumper.out,"\n]\n"); } else if (!grib_inline_strcmp(a->name,"groupNumber")) { depth+=2; + if ( (a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0) + return; if (!self->empty) fprintf(self->dumper.out,",\n"); fprintf(self->dumper.out,"%-*s",depth," "); diff --git a/tests/bufr_filter.sh b/tests/bufr_filter.sh index 69ae1c041..36031c3f8 100755 --- a/tests/bufr_filter.sh +++ b/tests/bufr_filter.sh @@ -119,6 +119,46 @@ for statid in 1 3 7 ; do [ -s ${dSplit}/split_98_13_1_1_${statid}.bufr ] done +#----------------------------------------------------------- +# Test: print attributes +#----------------------------------------------------------- +cat > $fRules <units]"; +print "pressure->code=[pressure->code!06d]"; +print "pressure->scale=[pressure->scale]"; +print "pressure->reference=[pressure->reference]"; +print "pressure->width=[pressure->width]"; +print "pressure->perCentConfidence=[pressure->perCentConfidence] [pressure->perCentConfidence->units]"; +print "pressure->perCentConfidence->code=[pressure->perCentConfidence->code!06d]"; +print "pressure->perCentConfidence->scale=[pressure->perCentConfidence->scale]"; +print "pressure->perCentConfidence->reference=[pressure->perCentConfidence->reference]"; +print "pressure->perCentConfidence->width=[pressure->perCentConfidence->width]"; +EOF + +f="syno_1.bufr" +echo "Test: attributes" >> $fLog +echo "file: $f" >> $fLog +${tools_dir}/bufr_filter $fRules $f 2>> $fLog 1>> $fLog + +${tools_dir}/bufr_filter $fRules $f 2>> ${f}.log 1>> ${f}.log +cat > ${f}.ref <code=010004 +pressure->scale=-1 +pressure->reference=0 +pressure->width=14 +pressure->perCentConfidence=74 % +pressure->perCentConfidence->code=033007 +pressure->perCentConfidence->scale=0 +pressure->perCentConfidence->reference=0 +pressure->perCentConfidence->width=7 +EOF + +diff ${f}.ref ${f}.log + +rm -f ${f}.ref ${f}.log + #----------------------------------------------------------- # Test: with nonexistent keys. #----------------------------------------------------------- diff --git a/tests/bufr_json.sh b/tests/bufr_json.sh index bea977bae..cdc3160de 100755 --- a/tests/bufr_json.sh +++ b/tests/bufr_json.sh @@ -8,7 +8,7 @@ # virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. # -set -e +set -ex . ./include.sh @@ -28,7 +28,15 @@ do ${tools_dir}bufr_dump -j $file 2> $REDIRECT > ${file}.json if test "x$JSON_CHECK" != "x"; then - $JSON_CHECK < ${file}.json >$REDIRECT 2> $REDIRECT + json_xs < ${file}.json >$REDIRECT 2> $REDIRECT + fi + + rm -f ${file}.json + + ${tools_dir}bufr_dump -jL $file 2> $REDIRECT > ${file}.json + + if test "x$JSON_CHECK" != "x"; then + json_xs < ${file}.json >$REDIRECT 2> $REDIRECT fi rm -f ${file}.json