core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bufr_dump -jL is providing a flat list of the content and is still a valid json. Test added in bufr_json.sh. More work done on the attributes. Not final yet.

This commit is contained in:
Enrico Fucile 2015-02-20 17:44:19 +00:00
parent 5cf67b838a
commit ee94336bda
11 changed files with 194 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/grib_accessor.c
View File

@ -653,9 +653,13 @@ void grib_accessors_list_push(grib_accessors_list* al,grib_accessor* a) {
grib_context* c=a->parent->h->context;
last=grib_accessors_list_last(al);
last->next=(grib_accessors_list*)grib_context_malloc_clear(c,sizeof(grib_accessors_list));
last->next->accessor=a;
last->next->prev=last;
if (last->accessor) {
last->next=(grib_accessors_list*)grib_context_malloc_clear(c,sizeof(grib_accessors_list));
last->next->accessor=a;
last->next->prev=last;
} else {
last->accessor=a;
}
}
grib_accessors_list* grib_accessors_list_last(grib_accessors_list* al) {

130
src/grib_accessor_class_bufr_data_array.c
View File

@ -302,6 +302,11 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
return GRIB_NOT_IMPLEMENTED;
}
grib_accessors_list* accessor_bufr_data_array_get_dataAccessors(grib_accessor* a) {
grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a;
return self->dataAccessors;
}
static int get_descriptors(grib_accessor* a) {
grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a;
int ret=0,i,numberOfDescriptors;
@ -598,7 +603,27 @@ static grib_accessor* create_attribute(char* name,grib_section* section,int type
return a;
}
static grib_accessor* create_accessor_from_descriptor(grib_accessor* a,grib_section* section,long ide,long subset) {
static void set_creator_name(grib_action* creator,int code) {
switch (code) {
case 223255:
creator->name="substitutedValue";
break;
case 224255:
creator->name="firstOrderStatisticalValue";
break;
case 225255:
creator->name="differenceStatisticalValue";
break;
case 232255:
creator->name="replacedRetainedValue";
break;
default :
creator->name="unknown";
break;
}
}
static grib_accessor* create_accessor_from_descriptor(grib_accessor* a,grib_section* section,long ide,long subset,int dump) {
grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a;
char code[7]={0,};
int idx=0;
@ -609,7 +634,6 @@ static grib_accessor* create_accessor_from_descriptor(grib_accessor* a,grib_sect
grib_action creator = {0, };
creator.op = "bufr_data_element";
creator.name_space = "";
creator.flags = GRIB_ACCESSOR_FLAG_DUMP;
creator.set = 0;
operatorCreator.op = "variable";
@ -618,43 +642,69 @@ static grib_accessor* create_accessor_from_descriptor(grib_accessor* a,grib_sect
operatorCreator.set = 0;
operatorCreator.name="operator";
if (dump) {
creator.flags = GRIB_ACCESSOR_FLAG_DUMP;
operatorCreator.flags = GRIB_ACCESSOR_FLAG_DUMP;
}
idx = self->compressedData ? self->elementsDescriptorsIndex->v[0]->v[ide] :
self->elementsDescriptorsIndex->v[subset]->v[ide] ;
creator.name=self->expanded->v[idx]->shortName;
/* only table B element are created as they only have a shortName*/
if (creator.name) {
elementAccessor = grib_accessor_factory(section, &creator, 0, NULL);
if (self->canBeMissing[idx]) elementAccessor->flags |= GRIB_ACCESSOR_FLAG_CAN_BE_MISSING;
accessor_bufr_data_element_set_index(elementAccessor,ide);
accessor_bufr_data_element_set_descriptors(elementAccessor,self->expanded);
accessor_bufr_data_element_set_numericValues(elementAccessor,self->numericValues);
accessor_bufr_data_element_set_stringValues(elementAccessor,self->stringValues);
accessor_bufr_data_element_set_compressedData(elementAccessor,self->compressedData);
accessor_bufr_data_element_set_type(elementAccessor,self->expanded->v[idx]->type);
accessor_bufr_data_element_set_numberOfSubsets(elementAccessor,self->numberOfSubsets);
accessor_bufr_data_element_set_subsetNumber(elementAccessor,subset);
switch (self->expanded->v[idx]->F) {
case 0:
creator.name=self->expanded->v[idx]->shortName;
elementAccessor = grib_accessor_factory(section, &creator, 0, NULL);
if (self->canBeMissing[idx]) elementAccessor->flags |= GRIB_ACCESSOR_FLAG_CAN_BE_MISSING;
accessor_bufr_data_element_set_index(elementAccessor,ide);
accessor_bufr_data_element_set_descriptors(elementAccessor,self->expanded);
accessor_bufr_data_element_set_numericValues(elementAccessor,self->numericValues);
accessor_bufr_data_element_set_stringValues(elementAccessor,self->stringValues);
accessor_bufr_data_element_set_compressedData(elementAccessor,self->compressedData);
accessor_bufr_data_element_set_type(elementAccessor,self->expanded->v[idx]->type);
accessor_bufr_data_element_set_numberOfSubsets(elementAccessor,self->numberOfSubsets);
accessor_bufr_data_element_set_subsetNumber(elementAccessor,subset);
self->expanded->v[idx]->a=elementAccessor;
self->expanded->v[idx]->a=elementAccessor;
sprintf(code,"%06ld",self->expanded->v[idx]->code);
attribute=create_attribute("code",section,GRIB_TYPE_STRING,code,0,0);
grib_accessor_add_attribute(elementAccessor,attribute);
sprintf(code,"%06ld",self->expanded->v[idx]->code);
attribute=create_attribute("code",section,GRIB_TYPE_STRING,code,0,0);
grib_accessor_add_attribute(elementAccessor,attribute);
attribute=create_attribute("units",section,GRIB_TYPE_STRING,self->expanded->v[idx]->units,0,0);
grib_accessor_add_attribute(elementAccessor,attribute);
attribute=create_attribute("units",section,GRIB_TYPE_STRING,self->expanded->v[idx]->units,0,0);
grib_accessor_add_attribute(elementAccessor,attribute);
attribute=create_attribute("scale",section,GRIB_TYPE_LONG,0,0,self->expanded->v[idx]->scale);
grib_accessor_add_attribute(elementAccessor,attribute);
attribute=create_attribute("scale",section,GRIB_TYPE_LONG,0,0,self->expanded->v[idx]->scale);
grib_accessor_add_attribute(elementAccessor,attribute);
attribute=create_attribute("reference",section,GRIB_TYPE_DOUBLE,0,self->expanded->v[idx]->reference,0);
grib_accessor_add_attribute(elementAccessor,attribute);
attribute=create_attribute("reference",section,GRIB_TYPE_DOUBLE,0,self->expanded->v[idx]->reference,0);
grib_accessor_add_attribute(elementAccessor,attribute);
attribute=create_attribute("width",section,GRIB_TYPE_LONG,0,0,self->expanded->v[idx]->width);
grib_accessor_add_attribute(elementAccessor,attribute);
} else {
elementAccessor = grib_accessor_factory(section, &operatorCreator, 0, NULL);
accessor_variable_set_type(elementAccessor,GRIB_TYPE_LONG);
attribute=create_attribute("width",section,GRIB_TYPE_LONG,0,0,self->expanded->v[idx]->width);
grib_accessor_add_attribute(elementAccessor,attribute);
break;
case 2:
if (self->expanded->v[idx]->isMarker) {
set_creator_name(&creator,self->expanded->v[idx]->code);
elementAccessor = grib_accessor_factory(section, &creator, 0, NULL);
if (self->canBeMissing[idx]) elementAccessor->flags |= GRIB_ACCESSOR_FLAG_CAN_BE_MISSING;
accessor_bufr_data_element_set_index(elementAccessor,ide);
accessor_bufr_data_element_set_descriptors(elementAccessor,self->expanded);
accessor_bufr_data_element_set_numericValues(elementAccessor,self->numericValues);
accessor_bufr_data_element_set_stringValues(elementAccessor,self->stringValues);
accessor_bufr_data_element_set_compressedData(elementAccessor,self->compressedData);
accessor_bufr_data_element_set_type(elementAccessor,self->expanded->v[idx]->type);
accessor_bufr_data_element_set_numberOfSubsets(elementAccessor,self->numberOfSubsets);
accessor_bufr_data_element_set_subsetNumber(elementAccessor,subset);
} else {
elementAccessor = grib_accessor_factory(section, &operatorCreator, 0, NULL);
accessor_variable_set_type(elementAccessor,GRIB_TYPE_LONG);
sprintf(code,"%06ld",self->expanded->v[idx]->code);
attribute=create_attribute("code",section,GRIB_TYPE_STRING,code,0,0);
grib_accessor_add_attribute(elementAccessor,attribute);
}
self->expanded->v[idx]->a=elementAccessor;
break;
}
return elementAccessor;
@ -736,7 +786,7 @@ static int create_keys(grib_accessor* a) {
int incrementBitmapIndex=1;
grib_accessor* elementFromBitmap=NULL;
int reuseBitmap=0;
int i;
int i,dump=1;;
creatorGroup.op = "bufr_group";
creatorGroup.name="groupNumber";
@ -802,6 +852,7 @@ static int create_keys(grib_accessor* a) {
significanceQualifierGroup[sidx]=gaGroup;
significanceQualifierDepth[sidx]=depth;
incrementBitmapIndex=1;
dump=1;
} else if (descriptor->code == 31031 && incrementBitmapIndex!=0) {
/* bitmap */
bitmapIndex++;
@ -830,20 +881,25 @@ static int create_keys(grib_accessor* a) {
sectionUp=gaGroup->parent;
bitmapGroup[bitmapIndex]=gaGroup;
bitmapDepth[bitmapIndex]=depth;
dump=1;
} else if (descriptor->code == 31031) {
dump=1;
bitmapSize[bitmapIndex]++;
bitmap.cursor=0;
} else if (descriptor->code == 222000 ) {
} else if (descriptor->code == 222000 || descriptor->code == 224000) {
bitmap.referredElement=NULL;
qualityPresent=1;
dump=1;
} else if (descriptor->code == 236000 || descriptor->code == 237000 ) {
bitmap.referredElement=NULL;
bitmap.cursor=0;
reuseBitmap=1;
dump=0;
} else if (descriptor->code == 237255 ) {
reuseBitmap=0;
bitmap.cursor=0;
} else if (descriptor->X==33 && qualityPresent) {
dump=1;
} else if ( ( descriptor->X==33 || descriptor->isMarker ) && qualityPresent) {
if (!bitmap.referredElement) {
bitmap.cursor=bitmapStart[bitmapIndex]->next;
bitmap.referredElement=bitmap.cursor->prev;
@ -852,12 +908,13 @@ static int create_keys(grib_accessor* a) {
}
}
elementFromBitmap=get_element_from_bitmap(a,&bitmap);
dump=1;
}
elementAccessor=create_accessor_from_descriptor(a,section,ide,iss);
elementAccessor=create_accessor_from_descriptor(a,section,ide,iss,dump);
if (elementFromBitmap) grib_accessor_add_attribute(elementFromBitmap,elementAccessor);
else if (elementAccessor) {
if (descriptor->F == 0) grib_push_accessor(elementAccessor,section->block);
grib_push_accessor(elementAccessor,section->block);
grib_accessors_list_push(self->dataAccessors,elementAccessor);
}
}
@ -1006,7 +1063,8 @@ static int decode_elements(grib_accessor* a) {
if (descriptors[i]->Y==255) {
index=get_next_bitmap_descriptor_index(self,elementsDescriptorsIndex,dval);
decode_element(c,self,iss,data,&pos,index,dval,sval);
grib_iarray_push(elementsDescriptorsIndex,index);
/* self->expanded->v[index] */
grib_iarray_push(elementsDescriptorsIndex,i);
} else {
grib_iarray_push(elementsDescriptorsIndex,i);
push_zero_element(self,dval);

13
src/grib_accessor_class_bufr_elements_table.c
View File

@ -340,6 +340,18 @@ static int bufr_get_from_table(grib_accessor* a,bufr_descriptor* v) {
return ret;
}
int bufr_is_marker(int code) {
switch (code) {
case 223255:
case 224255:
case 225255:
case 232255:
return 1;
default :
return 0;
}
}
bufr_descriptor* accessor_bufr_elements_table_get_descriptor(grib_accessor* a,int code,int *err) {
grib_context* c;
bufr_descriptor* v=NULL;
@ -360,6 +372,7 @@ bufr_descriptor* accessor_bufr_elements_table_get_descriptor(grib_accessor* a,in
v->F=code/100000;
v->X=(code-v->F*100000)/1000;
v->Y=(code-v->F*100000)%1000;
v->isMarker=bufr_is_marker(code);
switch (v->F) {
case 0:

1
src/grib_accessor_class_expanded_descriptors.c
View File

@ -165,7 +165,6 @@ typedef struct change_coding_params {
double referenceFactor;
} change_coding_params ;
static void init(grib_accessor* a, const long len , grib_arguments* args )
{
grib_accessor_expanded_descriptors* self = (grib_accessor_expanded_descriptors*)a;

1
src/grib_api_internal.h
View File

@ -765,6 +765,7 @@ struct bufr_descriptor {
double factor;
double reference;
long width;
int isMarker;
grib_accessor* a;
} ;

4
src/grib_api_prototypes.h
View File

@ -262,6 +262,7 @@ size_t grib_viarray_used_size(grib_viarray *v);
/* grib_accessor_class_bufr_data.c */
/* grib_accessor_class_bufr_data_array.c */
grib_accessors_list *accessor_bufr_data_array_get_dataAccessors(grib_accessor *a);
/* grib_accessor_class_bufr_data_element.c */
void accessor_bufr_data_element_set_index(grib_accessor *a, long index);
@ -276,6 +277,7 @@ void accessor_bufr_data_element_set_elementsDescriptorsIndex(grib_accessor *a, g
/* grib_accessor_class_bufr_elements_table.c */
char **str_split(char *a_str, const char a_delim);
int bufr_is_marker(int code);
bufr_descriptor *accessor_bufr_elements_table_get_descriptor(grib_accessor *a, int code, int *err);
/* grib_accessor_class_bufr_group.c */
@ -872,8 +874,10 @@ void grib_dump_footer(grib_dumper *d, grib_handle *h);
/* grib_dumper_class.c */
grib_dumper *grib_dumper_factory(const char *op, grib_handle *h, FILE *out, unsigned long option_flags, void *arg);
void grib_dump_accessors_block(grib_dumper *dumper, grib_block_of_accessors *block);
void grib_dump_accessors_list(grib_dumper *dumper, grib_accessors_list *al);
int grib_print(grib_handle *h, const char *name, grib_dumper *d);
void grib_dump_content(grib_handle *h, FILE *f, const char *mode, unsigned long option_flags, void *data);
void grib_dump_bufr_flat(grib_accessors_list *al, grib_handle *h, FILE *f, const char *mode, unsigned long option_flags, void *data);
/* grib_context.c */
size_t grib_context_read(const grib_context *c, void *ptr, size_t size, void *stream);

1
src/grib_bufr_descriptor.c
View File

@ -63,6 +63,7 @@ int grib_bufr_descriptor_set_code(grib_accessor* tables_accessor,int code,bufr_d
v->F=code/100000;
v->X=(code-v->F*100000)/1000;
v->Y=(code-v->F*100000)%1000;
v->isMarker=bufr_is_marker(code);
} else {
if (tables_accessor==NULL) return GRIB_NULL_POINTER;
d=accessor_bufr_elements_table_get_descriptor(tables_accessor,code,&err);

22
src/grib_dumper_class.c
View File

@ -61,6 +61,19 @@ void grib_dump_accessors_block(grib_dumper* dumper,grib_block_of_accessors* bloc
}
}
void grib_dump_accessors_list(grib_dumper* dumper,grib_accessors_list* al)
{
grib_accessors_list* cur=al;
grib_accessors_list* next=al->next;
while(next) {
grib_accessor_dump(cur->accessor,dumper);
cur=next;
next=cur->next;
}
}
int grib_print (grib_handle* h, const char* name, grib_dumper *d ){
grib_accessor* act = grib_find_accessor(h, name);
@ -83,3 +96,12 @@ void grib_dump_content(grib_handle* h, FILE* f,const char* mode,unsigned long op
grib_dumper_delete(dumper);
}
void grib_dump_bufr_flat(grib_accessors_list* al,grib_handle* h, FILE* f,const char* mode,unsigned long option_flags,void *data)
{
grib_dumper *dumper;
dumper = grib_dumper_factory(mode?mode:"serialize",h,f,option_flags,data);
grib_dump_header(dumper,h);
grib_dump_accessors_list(dumper,al);
grib_dump_footer(dumper,h);
grib_dumper_delete(dumper);
}

2
src/grib_dumper_class_json.c
View File

@ -440,6 +440,8 @@ static void dump_section(grib_dumper* d,grib_accessor* a,grib_block_of_accessors
fprintf(self->dumper.out,"\n]\n");
} else if (!grib_inline_strcmp(a->name,"groupNumber")) {
depth+=2;
if ( (a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0)
return;
if (!self->empty) fprintf(self->dumper.out,",\n");
fprintf(self->dumper.out,"%-*s",depth," ");

40
tests/bufr_filter.sh
View File

@ -119,6 +119,46 @@ for statid in 1 3 7 ; do
[ -s ${dSplit}/split_98_13_1_1_${statid}.bufr ]
done
#-----------------------------------------------------------
# Test: print attributes
#-----------------------------------------------------------
cat > $fRules <<EOF
set unpack=1;
print "pressure=[pressure] [pressure->units]";
print "pressure->code=[pressure->code!06d]";
print "pressure->scale=[pressure->scale]";
print "pressure->reference=[pressure->reference]";
print "pressure->width=[pressure->width]";
print "pressure->perCentConfidence=[pressure->perCentConfidence] [pressure->perCentConfidence->units]";
print "pressure->perCentConfidence->code=[pressure->perCentConfidence->code!06d]";
print "pressure->perCentConfidence->scale=[pressure->perCentConfidence->scale]";
print "pressure->perCentConfidence->reference=[pressure->perCentConfidence->reference]";
print "pressure->perCentConfidence->width=[pressure->perCentConfidence->width]";
EOF
f="syno_1.bufr"
echo "Test: attributes" >> $fLog
echo "file: $f" >> $fLog
${tools_dir}/bufr_filter $fRules $f 2>> $fLog 1>> $fLog
${tools_dir}/bufr_filter $fRules $f 2>> ${f}.log 1>> ${f}.log
cat > ${f}.ref <<EOF
pressure=100910 Pa
pressure->code=010004
pressure->scale=-1
pressure->reference=0
pressure->width=14
pressure->perCentConfidence=74 %
pressure->perCentConfidence->code=033007
pressure->perCentConfidence->scale=0
pressure->perCentConfidence->reference=0
pressure->perCentConfidence->width=7
EOF
diff ${f}.ref ${f}.log
rm -f ${f}.ref ${f}.log
#-----------------------------------------------------------
# Test: with nonexistent keys.
#-----------------------------------------------------------

12
tests/bufr_json.sh
View File

@ -8,7 +8,7 @@
# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
#
set -e
set -ex
. ./include.sh
@ -28,7 +28,15 @@ do
${tools_dir}bufr_dump -j $file 2> $REDIRECT > ${file}.json
if test "x$JSON_CHECK" != "x"; then
$JSON_CHECK < ${file}.json >$REDIRECT 2> $REDIRECT
json_xs < ${file}.json >$REDIRECT 2> $REDIRECT
fi
rm -f ${file}.json
${tools_dir}bufr_dump -jL $file 2> $REDIRECT > ${file}.json
if test "x$JSON_CHECK" != "x"; then
json_xs < ${file}.json >$REDIRECT 2> $REDIRECT
fi
rm -f ${file}.json