diff --git a/data/bufr/make_bufrdc_ref.ksh b/data/bufr/make_bufrdc_ref.ksh index d3a73d808..c73a506f5 100755 --- a/data/bufr/make_bufrdc_ref.ksh +++ b/data/bufr/make_bufrdc_ref.ksh @@ -1,9 +1,12 @@ #!/bin/ksh set -ex -decode=/var/tmp/mac/p4/bufrdc/releases/000402/examples/bufr_decode_data +base=/var/tmp/mac/p4/bufrdc/releases/000402/examples +decode=$base/bufr_decode_data +print_descriptors=$base/bufr_print_descriptors for f in *.bufr do - $decode -i $f | sed 's/MISSING/-1.00000000000000e+100/g' | sed -e '/ECMWF/,+8d' > $f.ref + $decode -i $f | sed 's/MISSING/-1.00000000000000e+100/g' | sed -e '/ECMWF/,+8d' > $f.num.ref + #$print_descriptors -i $f | sed -e '/ECMWF/,+8d'| sed 's/ //g' > $f.desc.ref done diff --git a/definitions/bufr/section.3.def b/definitions/bufr/section.3.def index 2a656bc22..9766eb0bf 100644 --- a/definitions/bufr/section.3.def +++ b/definitions/bufr/section.3.def @@ -67,7 +67,21 @@ position offsetDescriptors; meta numberOfUnexpandedDescriptors evaluate( (section3Length - 7) / 2 ) ; meta unexpandedDescriptors unexpanded_descriptors(numberOfUnexpandedDescriptors) :dump; -meta expandedDescriptors expanded_descriptors(unexpandedDescriptors,sequences) ; +transient elementCode="000000"; +elementAbbreviation=dict_search("element.table",elementCode,1,tablesMasterDir,tablesLocalDir) : string_type,hidden ; +elementType=dict_search("element.table",elementCode,2,tablesMasterDir,tablesLocalDir) : string_type,hidden ; +elementName=dict_search("element.table",elementCode,3,tablesMasterDir,tablesLocalDir) : string_type,hidden ; +elementUnit=dict_search("element.table",elementCode,4,tablesMasterDir,tablesLocalDir) : string_type,hidden ; +elementScale=dict_search("element.table",elementCode,5,tablesMasterDir,tablesLocalDir) : string_type,hidden ; +elementReference=dict_search("element.table",elementCode,6,tablesMasterDir,tablesLocalDir) : string_type,hidden ; +elementWidth=dict_search("element.table",elementCode,7,tablesMasterDir,tablesLocalDir) : string_type,hidden ; + +meta expandedDescriptors expanded_descriptors(expandedDescriptors,0,unexpandedDescriptors,sequences, + elementCode,elementUnit,elementScale,elementReference,elementWidth); +meta expandedFactors expanded_descriptors(expandedDescriptors,1); +meta expandedReferences expanded_descriptors(expandedDescriptors,2); +meta expandedWidths expanded_descriptors(expandedDescriptors,3); + meta bufrdcExpandedDescriptors bufrdc_expanded_descriptors(expandedDescriptors); #smart_table NAME (VALUES,FILE_NAME,MASTER_DIRECTORY,LOCAL_DIRECTORY,WIDTH_OF_CODE_IN_BITS,EXTRA_DIRECTORY,EXTRA_FILE_NAME); @@ -83,8 +97,6 @@ meta crex_unit smart_table_column(expanded,7,0) : long_type; meta crex_scale smart_table_column(expanded,8,0) : long_type; meta crex_width smart_table_column(expanded,9,0) : long_type; -meta hasDelayedReplication bufr_has_delayed_replication(expandedDescriptorsAfterOperators); - position endDescriptors; section_padding section3Padding; meta lengthDescriptors evaluate(endDescriptors-offsetDescriptors); diff --git a/definitions/bufr/section.4.def b/definitions/bufr/section.4.def index da38bbc53..34b124538 100644 --- a/definitions/bufr/section.4.def +++ b/definitions/bufr/section.4.def @@ -19,12 +19,12 @@ transient subsetNumber=0; if (compressedData) { meta numericValues bufr_data_array(offsetSection4,offsetBeforeData,offsetEndSection4,section4Length, numberOfDataSubsets,subsetNumber, - expandedDescriptors,type,unit,reference,scale,width, + expanded,type,unit,reference,scale,width, stringValues,elementsDescriptorsIndex,compressedData) ; } else { meta numericValues bufr_data_array(offsetSection4,offsetBeforeData,offsetEndSection4,section4Length, numberOfDataSubsets,subsetNumber, - expandedDescriptors,type,unit,reference,scale,width, + expanded,types,units,references,scales,widths, stringValues,elementsDescriptorsIndex,compressedData) ; } diff --git a/definitions/bufr/tables/0/local/1/98/0/sequence.def b/definitions/bufr/tables/0/local/1/98/0/sequence.def index c948a1307..6d8d0d362 100644 --- a/definitions/bufr/tables/0/local/1/98/0/sequence.def +++ b/definitions/bufr/tables/0/local/1/98/0/sequence.def @@ -44,6 +44,11 @@ "309195" = [ 301195,302004,101000,031001,303014 ] "309196" = [ 301196,302004,101000,031001,303014 ] "309198" = [ 301198,302004,101000,031001,303014 ] +"310061" = [ 001007, 001033, 001034, 002019, 002020, 301011, 301012, 207003, 004006, + 207000, 005040, 005041, 005043, 033079, 033080, 033078, 301021, 201129, + 007002, 201000, 007024, 005021, 007025, 005022, 025075, 111000, 031002, + 005042, 202131, 002153, 002154, 202000, 002104, 012066, 012163, 012158, + 012159, 033081 ] "310193" = [ 301250,303250,302250,303249,303251 ] "310194" = [ 301250,303250,302250,303249,303252 ] "310195" = [ 301250,303250,304250 ] diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1c54fc587..81b7c606e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -38,6 +38,8 @@ list( APPEND grib_api_srcs grib_accessor.c grib_concept.c grib_hash_array.c + grib_bufr_descriptor.c + grib_bufr_descriptors_array.c grib_darray.c grib_sarray.c grib_vdarray.c diff --git a/src/Makefile.am b/src/Makefile.am index f13c2d149..cf562312d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -47,6 +47,8 @@ libgrib_api_la_prototypes= \ grib_accessor.c \ grib_concept.c \ grib_hash_array.c \ + grib_bufr_descriptor.c \ + grib_bufr_descriptors_array.c \ grib_darray.c \ grib_sarray.c \ grib_iarray.c \ diff --git a/src/grib_accessor_class_bufr_data_array.c b/src/grib_accessor_class_bufr_data_array.c index 904ce9870..eaed7a4ab 100644 --- a/src/grib_accessor_class_bufr_data_array.c +++ b/src/grib_accessor_class_bufr_data_array.c @@ -308,6 +308,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len) return GRIB_NOT_IMPLEMENTED; } +#if 0 static void apply_early_operators(grib_accessor *a) { long* descriptors; int i,j,F,X,Y,elementsToRepeat,jReplication; @@ -392,6 +393,7 @@ static void apply_early_operators(grib_accessor *a) { self->expandedDescriptors=descriptors; } +#endif static int get_descriptors(grib_accessor* a) { int err=0; @@ -487,7 +489,7 @@ static int get_descriptors(grib_accessor* a) { err=grib_get_long(h,self->compressedDataName,&(self->compressedData)); err=grib_get_long(h,self->subsetNumberName,&(self->subsetNumber)); - apply_early_operators(a); + /* apply_early_operators(a); */ return err; } diff --git a/src/grib_accessor_class_bufrdc_expanded_descriptors.c b/src/grib_accessor_class_bufrdc_expanded_descriptors.c index 927ac0d1c..e67676262 100644 --- a/src/grib_accessor_class_bufrdc_expanded_descriptors.c +++ b/src/grib_accessor_class_bufrdc_expanded_descriptors.c @@ -183,7 +183,7 @@ static int unpack_long (grib_accessor* a, long* val, size_t *len) rlen=0; for (i=0;i 221999 ) + if ((v[i] < 100000 || v[i] > 221999 ) ) val[rlen++]=v[i]; } *len=rlen; diff --git a/src/grib_accessor_class_dictionary.c b/src/grib_accessor_class_dictionary.c index c9e97115a..03347cf76 100644 --- a/src/grib_accessor_class_dictionary.c +++ b/src/grib_accessor_class_dictionary.c @@ -26,6 +26,8 @@ MEMBERS = const char* dictionary MEMBERS = const char* key MEMBERS = long column + MEMBERS = const char* masterDir + MEMBERS = const char* localDir END_CLASS_DEF */ @@ -56,6 +58,8 @@ typedef struct grib_accessor_dictionary { const char* dictionary; const char* key; long column; + const char* masterDir; + const char* localDir; } grib_accessor_dictionary; extern grib_accessor_class* grib_accessor_class_gen; @@ -144,6 +148,8 @@ static void init(grib_accessor* a, const long len, grib_arguments* params) { self->dictionary = grib_arguments_get_string(a->parent->h,params,n++); self->key = grib_arguments_get_name(a->parent->h,params,n++); self->column = grib_arguments_get_long(a->parent->h,params,n++); + self->masterDir = grib_arguments_get_name(a->parent->h,params,n++); + self->localDir = grib_arguments_get_name(a->parent->h,params,n++); a->length = 0; a->flags |= GRIB_ACCESSOR_FLAG_READ_ONLY; @@ -157,14 +163,45 @@ static grib_trie* load_dictionary(grib_context* c,grib_accessor* a, int* err) { char* filename=NULL; char line[1024]={0,}; char key[1024]={0,}; + char masterDir[1024]={0,}; + char localDir[1024]={0,}; + char name[1024]={0,}; + char localName[1024]={0,}; + char recomposed[1024]={0,}; + char localRecomposed[1024]={0,}; + char dictName[1024]={0,}; + char *localFilename=0; char* list=0; + size_t len=1024; grib_trie* dictionary=NULL; FILE* f=NULL; int i=0; + grib_handle* h=a->parent->h; *err=GRIB_SUCCESS; - filename=grib_context_full_defs_path(c,self->dictionary); + len=1024; + if (self->masterDir != NULL) grib_get_string(h,self->masterDir,masterDir,&len); + len=1024; + if (self->localDir != NULL) grib_get_string(h,self->localDir,localDir,&len); + + if (*masterDir!=0) { + sprintf(name,"%s/%s",masterDir,self->dictionary); + grib_recompose_name(h, NULL,name, recomposed,0); + filename=grib_context_full_defs_path(c,recomposed); + } else { + filename=grib_context_full_defs_path(c,self->dictionary); + } + + if (*localDir!=0) { + sprintf(localName,"%s/%s",localDir,self->dictionary); + grib_recompose_name(h, NULL,localName, localRecomposed,0); + localFilename=grib_context_full_defs_path(c,localRecomposed); + sprintf(dictName,"%s:%s",localFilename,filename); + } else { + sprintf(dictName,"%s",filename); + } + if (!filename) { grib_context_log(c,GRIB_LOG_ERROR,"unable to find def file %s",self->dictionary); *err=GRIB_FILE_NOT_FOUND; @@ -172,7 +209,7 @@ static grib_trie* load_dictionary(grib_context* c,grib_accessor* a, int* err) { } else { grib_context_log(c,GRIB_LOG_DEBUG,"found def file %s",filename); } - dictionary=grib_trie_get(c->lists,filename); + dictionary=grib_trie_get(c->lists,dictName); if (dictionary) { grib_context_log(c,GRIB_LOG_DEBUG,"using dictionary %s from cache",self->dictionary); return dictionary; @@ -197,10 +234,28 @@ static grib_trie* load_dictionary(grib_context* c,grib_accessor* a, int* err) { grib_trie_insert(dictionary,key,list); } - grib_trie_insert(c->lists,filename,dictionary); - fclose(f); + if (localFilename!=0) { + f=fopen(localFilename,"r"); + if (!f) {*err=GRIB_IO_PROBLEM; return NULL;} + + while(fgets(line,sizeof(line)-1,f)) { + i=0; + while (line[i] != '|' && line[i] != 0) { + key[i]=line[i]; + i++; + } + key[i]=0; + list=grib_context_malloc_clear(c,strlen(line)+1); + memcpy(list,line,strlen(line)); + grib_trie_insert(dictionary,key,list); + } + + + fclose(f); + } + grib_trie_insert(c->lists,filename,dictionary); return dictionary; } diff --git a/src/grib_accessor_class_expanded_descriptors.c b/src/grib_accessor_class_expanded_descriptors.c index 36d8e8efc..3b8b48ffb 100644 --- a/src/grib_accessor_class_expanded_descriptors.c +++ b/src/grib_accessor_class_expanded_descriptors.c @@ -27,8 +27,15 @@ IMPLEMENTS = init;dump;destroy IMPLEMENTS = value_count MEMBERS = const char* unexpandedDescriptors MEMBERS = const char* sequence -MEMBERS = long* expanded -MEMBERS = size_t expandedSize +MEMBERS = const char* expandedName +MEMBERS = const char* elementCode +MEMBERS = const char* elementUnit +MEMBERS = const char* elementScale +MEMBERS = const char* elementReference +MEMBERS = const char* elementWidth +MEMBERS = bufr_descriptors_array* expanded +MEMBERS = int rank +MEMBERS = grib_accessor* expandedAccessor MEMBERS = int dirty END_CLASS_DEF @@ -60,8 +67,15 @@ typedef struct grib_accessor_expanded_descriptors { /* Members defined in expanded_descriptors */ const char* unexpandedDescriptors; const char* sequence; - long* expanded; - size_t expandedSize; + const char* expandedName; + const char* elementCode; + const char* elementUnit; + const char* elementScale; + const char* elementReference; + const char* elementWidth; + bufr_descriptors_array* expanded; + int rank; + grib_accessor* expandedAccessor; int dirty; } grib_accessor_expanded_descriptors; @@ -146,12 +160,35 @@ static void init_class(grib_accessor_class* c) /* END_CLASS_IMP */ +#define MYDEBUG 0 + +typedef struct change_coding_params { + int associatedFieldWidth; + int localDescriptorWidth; + int extraWidth; + int extraScale; + double referenceFactor; +} change_coding_params ; + + static void init(grib_accessor* a, const long len , grib_arguments* args ) { grib_accessor_expanded_descriptors* self = (grib_accessor_expanded_descriptors*)a; int n=0; + self->expandedName=grib_arguments_get_name(a->parent->h,args,n++); + self->rank=grib_arguments_get_long(a->parent->h,args,n++); + if (self->rank!=0) { + self->expandedAccessor=grib_find_accessor(a->parent->h,self->expandedName); + } else { + self->expandedAccessor=0; + } self->unexpandedDescriptors=grib_arguments_get_name(a->parent->h,args,n++); self->sequence=grib_arguments_get_name(a->parent->h,args,n++); + self->elementCode=grib_arguments_get_name(a->parent->h,args,n++); + self->elementUnit=grib_arguments_get_name(a->parent->h,args,n++); + self->elementScale=grib_arguments_get_name(a->parent->h,args,n++); + self->elementReference=grib_arguments_get_name(a->parent->h,args,n++); + self->elementWidth=grib_arguments_get_name(a->parent->h,args,n++); self->dirty=1; self->expanded=0; a->length = 0; @@ -162,148 +199,375 @@ static void dump(grib_accessor* a, grib_dumper* dumper) grib_dump_long(dumper,a,NULL); } -size_t __expand(grib_accessor* a,grib_iarray* unexpanded,grib_iarray* expanded, int* err) { - int FF,F,X,Y,k,j; +static bufr_descriptors_array* _expand(grib_accessor* a,bufr_descriptors_array* unexpanded,change_coding_params* ccp,int *err); + +static int depth=-1; + +static size_t __expand(grib_accessor* a,bufr_descriptors_array* unexpanded,bufr_descriptors_array* expanded, + change_coding_params* ccp,int* err) { + int k,j,i; grib_accessor_expanded_descriptors* self = (grib_accessor_expanded_descriptors*)a; + long scale,reference,width; size_t size; + char code[]="000000"; + size_t len_code=6; + char unit[100]={0,}; + size_t len_unit=100; long* v; - long u; - long* ur; + bufr_descriptor* u; + bufr_descriptor* vv; + bufr_descriptor** ur; int idx; - long u0; + bufr_descriptor* u0; grib_context* c=a->parent->h->context; - long us; + bufr_descriptor* us; + bufr_descriptors_array* inner_expanded; + bufr_descriptors_array* inner_unexpanded; +#if MYDEBUG + int idepth; +#endif - if (grib_iarray_used_size(unexpanded)==0) return 0; - us=grib_iarray_get(unexpanded,0); + if (grib_bufr_descriptors_array_used_size(unexpanded)==0) return 0; + + us=grib_bufr_descriptors_array_get(unexpanded,0); *err=0; - F=us/100000; - X=(us-F*100000)/1000; - Y=(us-F*100000)%1000; - /* printf("\n__expand ==> %d-%.2d-%.3d\n",F,X,Y); */ - switch (F) { +#if MYDEBUG + for (idepth=0;idepth %d-%02d-%03d\n",us->F,us->X,us->Y); +#endif + switch (us->F) { case 3: /* sequence */ - u=grib_iarray_pop_front(unexpanded); - /* printf("+++ pop %ld\n",u); */ + u=grib_bufr_descriptors_array_pop_front(unexpanded); +#if MYDEBUG + for (idepth=0;idepthcode); +#endif /*this is to get the sequence elements of the sequence unexpanded[i] */ - *err=grib_set_long(a->parent->h,self->sequence,u); + *err=grib_set_long(a->parent->h,self->sequence,u->code); *err=grib_get_size(a->parent->h,self->sequence,&size); if (*err) return 0; v=grib_context_malloc_clear(c,sizeof(long)*size); *err=grib_get_long_array(a->parent->h,self->sequence,v,&size); if (*err) return 0; - expanded=grib_iarray_push_array(expanded,v,size); - /* printf("=== push array \n"); */ - /* for (i=0;in;i++) printf("=== %ld\n",grib_iarray_get(expanded,i)); */ + inner_unexpanded=grib_bufr_descriptors_array_new(c,100,100); + inner_expanded=grib_bufr_descriptors_array_new(c,100,100); + for (i=0;in;i++) { + for (idepth=0;idepthv[i]->code); + } +#endif + expanded=grib_bufr_descriptors_array_append(expanded,inner_expanded); + size=grib_bufr_descriptors_array_used_size(inner_expanded); break; case 1: - if (Y==0) { - u=grib_iarray_pop_front(unexpanded); - /* printf("+++ pop %ld\n",u); */ - /* printf("=== push %ld\n",u); */ - grib_iarray_push(expanded,u); + if (us->Y==0) { + /* delayed replication */ + bufr_descriptor* uidx=0; + u=grib_bufr_descriptors_array_pop_front(unexpanded); +#if MYDEBUG + for (idepth=0;idepthcode); + for (idepth=0;idepthcode); +#endif + grib_bufr_descriptors_array_push(expanded,u); idx=expanded->n-1; - u=grib_iarray_pop_front(unexpanded); - /* printf("+++ pop %ld\n",u); */ - /* printf("=== push %ld\n",u); */ - grib_iarray_push(expanded,u); + u=grib_bufr_descriptors_array_pop_front(unexpanded); +#if MYDEBUG + for (idepth=0;idepthcode); + for (idepth=0;idepthcode); +#endif + grib_bufr_descriptors_array_push(expanded,u); size=0; - for (j=0;jX;j++) { + u0=grib_bufr_descriptors_array_pop_front(unexpanded); + grib_bufr_descriptors_array_push(inner_unexpanded,u0); +#if MYDEBUG + for (idepth=0;idepthcode); +#endif } - grib_iarray_set(expanded,idx,size*1000+100000); + inner_expanded=_expand(a,inner_unexpanded,ccp,err); + size=grib_bufr_descriptors_array_used_size(inner_expanded); +#if MYDEBUG + for (i=0;in;i++) { + for (idepth=0;idepthv[i]->code); + } +#endif + expanded=grib_bufr_descriptors_array_append(expanded,inner_expanded); + uidx=grib_bufr_descriptors_array_get(expanded,idx); + grib_bufr_descriptor_set_code(uidx,size*1000+100000); size++; } else { - u=grib_iarray_pop_front(unexpanded); - /* printf("+++ pop %ld\n",u); */ - size=X*Y; - ur=grib_context_malloc_clear(c,X*sizeof(long)); - for (j=0;jcode); +#endif + size=us->X*us->Y; + ur=(bufr_descriptor**)grib_context_malloc_clear(c,us->X*sizeof(bufr_descriptor)); + for (j=0;jX;j++) { + ur[j]=grib_bufr_descriptors_array_pop_front(unexpanded); +#if MYDEBUG + for (idepth=0;idepthcode); +#endif } - for (k=0;kY;k++) { + for (j=0;jX;j++) { + grib_bufr_descriptors_array_push(inner_unexpanded,ur[j]); } } + inner_expanded=_expand(a,inner_unexpanded,ccp,err); +#if MYDEBUG + for (i=0;in;i++) { + for (idepth=0;idepthv[i]->code); + } +#endif + expanded=grib_bufr_descriptors_array_append(expanded,inner_expanded); + size=grib_bufr_descriptors_array_used_size(inner_expanded); grib_context_free(c,ur); } break; + case 0: + u=grib_bufr_descriptors_array_pop_front(unexpanded); +#if MYDEBUG + for (idepth=0;idepthcode); +#endif + if ((u->flags & BUFR_DESCRIPTOR_FLAG_HAS_VALUES) == 0) { + sprintf(code,"%06ld",u->code); + len_code=strlen(code); + grib_set_string(a->parent->h,self->elementCode,code,&len_code); + *err=grib_get_long(a->parent->h,self->elementScale,&scale); + if (*err==GRIB_SUCCESS) { + *err=grib_get_long(a->parent->h,self->elementReference,&reference); + *err=grib_get_long(a->parent->h,self->elementWidth,&width); + grib_bufr_descriptor_set_values(u,scale,reference,width); + *err=grib_get_string(a->parent->h,self->elementUnit,unit,&len_unit); + if (!strcmp(unit,"CCITTIA5")) u->flags |= BUFR_DESCRIPTOR_FLAG_IS_STRING; + else if (strstr(unit,"TABLE") ) { + if (strstr(unit,"FLAG")) { + u->flags |= BUFR_DESCRIPTOR_FLAG_IS_FLAG; + } else { + u->flags |= BUFR_DESCRIPTOR_FLAG_IS_TABLE; + } + } + u->flags |= BUFR_DESCRIPTOR_FLAG_HAS_VALUES; + } else { + grib_context_log(c,GRIB_LOG_ERROR,"element %s not in table B",code); + return 0; + } + } +#if MYDEBUG + for (idepth=0;idepthcode, + (u->flags & BUFR_DESCRIPTOR_FLAG_HAS_VALUES)==BUFR_DESCRIPTOR_FLAG_HAS_VALUES, + (u->flags & BUFR_DESCRIPTOR_FLAG_IS_FLAG)==BUFR_DESCRIPTOR_FLAG_IS_FLAG, + (u->flags & BUFR_DESCRIPTOR_FLAG_IS_TABLE)==BUFR_DESCRIPTOR_FLAG_IS_TABLE, + (u->flags & BUFR_DESCRIPTOR_FLAG_IS_STRING)==BUFR_DESCRIPTOR_FLAG_IS_STRING, + u->scale,u->reference,u->width); +#endif + if ((u->flags & ( BUFR_DESCRIPTOR_FLAG_IS_FLAG | + BUFR_DESCRIPTOR_FLAG_IS_TABLE | + BUFR_DESCRIPTOR_FLAG_IS_STRING)) == 0) { + u->width= ccp->localDescriptorWidth>0 ? ccp->localDescriptorWidth : u->width+ccp->extraWidth; + u->reference=u->reference* ccp->referenceFactor; + u->scale=u->scale+ccp->extraScale; + } +#if MYDEBUG + printf("->(%ld %g %ld)\n",u->scale,u->reference,u->width); +#endif + grib_bufr_descriptors_array_push(expanded,u); + size=1; + if (ccp->associatedFieldWidth) { + bufr_descriptor* au=grib_bufr_descriptor_new(c,999999); + au->width=ccp->associatedFieldWidth; +#if MYDEBUG + for (idepth=0;idepthcode,au->scale,au->reference,au->width); +#endif + grib_bufr_descriptors_array_push(expanded,au); + size++; + /* bufrdc bug!!*/ + ccp->associatedFieldWidth=0; + } + break; + + case 2: + u=grib_bufr_descriptors_array_pop_front(unexpanded); +#if MYDEBUG + for (idepth=0;idepthcode); +#endif + switch(us->X) { + case 1: + ccp->extraWidth = us->Y ? us->Y-128 : 0; + size=0; + break; + case 2: + ccp->extraScale = us->Y ? us->Y-128 : 0; + size=0; + break; + case 4: + /* associated field*/ + ccp->associatedFieldWidth=us->Y; + break; + case 6: + /*signify data width*/ + ccp->localDescriptorWidth=us->Y; + size=0; + break; + case 7: + if (us->Y) { + ccp->extraScale = us->Y; + ccp->referenceFactor=grib_power(us->Y,10); + ccp->extraWidth=((10*us->Y)+2)/3; + } else { + ccp->extraWidth=0; + ccp->extraScale=0; + ccp->referenceFactor=1; + } + size=0; + break; + default: +#if MYDEBUG + for (idepth=0;idepthcode); +#endif + grib_bufr_descriptors_array_push(expanded,u); + size=1; + } + break; + default: - u=grib_iarray_pop_front(unexpanded); - /* printf("+++ pop front %ld\n",u); */ - /* printf("=== push %ld\n",u); */ - grib_iarray_push(expanded,u); + u=grib_bufr_descriptors_array_pop_front(unexpanded); +#if MYDEBUG + for (idepth=0;idepthcode); + for (idepth=0;idepthcode); +#endif + grib_bufr_descriptors_array_push(expanded,u); size=1; } - /* printf("__expand <== %d-%.2d-%.3d (%ld)\n",F,X,Y,size); */ +#if MYDEBUG + for (idepth=0;idepthF,us->X,us->Y,size); +#endif return size; } -static long* _expand(grib_accessor* a,long* u,size_t unexpandedSize,size_t *size,int *err) +static bufr_descriptors_array* _expand(grib_accessor* a,bufr_descriptors_array* unexpanded,change_coding_params* ccp,int *err) { - grib_iarray* expanded=NULL; - long *expanded_array=0; - long *ar=0; - size_t i=0; - grib_iarray* unexpanded; + bufr_descriptors_array* expanded=NULL; grib_context* c=a->parent->h->context; - expanded=grib_iarray_new(c,unexpandedSize,100); - unexpanded=grib_iarray_new_from_array(c,u,unexpandedSize); +#if MYDEBUG + int idepth; +#endif - *size=0; - while (unexpanded->n) - __expand(a,unexpanded,expanded,err); + depth++; - ar=grib_iarray_get_array(expanded); - expanded_array=grib_context_malloc_clear(c,sizeof(long)*expanded->n); - for (i=0;in;i++) expanded_array[i]=ar[i]; - *size=expanded->n; - grib_iarray_delete(expanded); - grib_context_free(c,ar); - grib_iarray_delete(unexpanded); - return expanded_array; -} + expanded=grib_bufr_descriptors_array_new(c,100,100); -static long* expand(grib_accessor* a,size_t *expandedSize,int *err) -{ - grib_accessor_expanded_descriptors* self = (grib_accessor_expanded_descriptors*)a; - long *expanded=0; - size_t unexpandedSize=0; - long* unexpanded=NULL; - grib_context* c=a->parent->h->context; - - *err=grib_get_size(a->parent->h,self->unexpandedDescriptors,&unexpandedSize); - if (*err) return NULL; - unexpanded=grib_context_malloc_clear(c,sizeof(long)*unexpandedSize); - if (!unexpanded) {*err=GRIB_OUT_OF_MEMORY; return NULL;} - *err=grib_get_long_array(a->parent->h,self->unexpandedDescriptors,unexpanded,&unexpandedSize); - if (*err) return NULL; - - *expandedSize=unexpandedSize; - while (1) { - expanded=_expand(a,unexpanded,unexpandedSize,expandedSize,err); - /* printf("unexpandedSize=%ld expandedSize=%ld\n",unexpandedSize,*expandedSize); */ - grib_context_free(c,unexpanded); - if (unexpandedSize==*expandedSize) break; - unexpanded=expanded; - expanded=0; - unexpandedSize=*expandedSize; +#if MYDEBUG + { + int i; + for (idepth=0;idepth \n"); + for (i=0;in;i++) { + bufr_descriptor* xx=grib_bufr_descriptors_array_get(unexpanded,i); + for (idepth=0;idepthcode); + } + for (idepth=0;idepthn) { + __expand(a,unexpanded,expanded,ccp,err); } +#if MYDEBUG + { + int i; + for (idepth=0;idepth \n"); + for (i=0;in;i++) { + bufr_descriptor* xx=grib_bufr_descriptors_array_get(expanded,i); + for (idepth=0;idepthcode); + if (xx->flags & BUFR_DESCRIPTOR_FLAG_HAS_VALUES) { + printf("%ld %g %ld",xx->scale,xx->reference,xx->width); + } + printf("\n"); + } + for (idepth=0;idepthparent->h->context; + + if (!self->dirty) return err; + self->dirty=0; + + + grib_bufr_descriptors_array_delete(self->expanded); + err=grib_get_size(a->parent->h,self->unexpandedDescriptors,&unexpandedSize); + if (err) return err; + u=grib_context_malloc_clear(c,sizeof(long)*unexpandedSize); + if (!u) {err=GRIB_OUT_OF_MEMORY; return err;} + err=grib_get_long_array(a->parent->h,self->unexpandedDescriptors,u,&unexpandedSize); + if (err) return err; + + unexpanded=grib_bufr_descriptors_array_new(c,unexpandedSize,100); + for (i=0;iexpanded=_expand(a,unexpanded,&ccp,&err); + grib_bufr_descriptors_array_delete(unexpanded); + + return err; } @@ -314,15 +578,9 @@ static int unpack_long (grib_accessor* a, long* val, size_t *len) size_t rlen=0; size_t i; - if (self->dirty) { - if (self->expanded) - grib_context_free(a->parent->h->context,self->expanded); - self->expanded=expand(a,&rlen,&ret); - self->expandedSize=rlen; - self->dirty=0; - if (ret) return ret; - } - + ret=expand(a); + if (ret) return ret; + rlen=grib_bufr_descriptors_array_used_size(self->expanded); if(*len < rlen) { @@ -332,8 +590,8 @@ static int unpack_long (grib_accessor* a, long* val, size_t *len) return GRIB_ARRAY_TOO_SMALL; } - *len = self->expandedSize; - for (i=0;i<*len;i++) val[i]=self->expanded[i]; + *len = rlen; + for (i=0;i<*len;i++) val[i]=self->expanded->v[i]->code; return GRIB_SUCCESS; } @@ -343,7 +601,6 @@ static int pack_long (grib_accessor* a, const long* val, size_t *len) grib_accessor_expanded_descriptors* self = (grib_accessor_expanded_descriptors*)a; self->dirty=1; return GRIB_NOT_IMPLEMENTED; - } @@ -354,16 +611,13 @@ static int value_count(grib_accessor* a,long* rlen) grib_accessor_expanded_descriptors* self = (grib_accessor_expanded_descriptors*)a; *rlen=0; - if (self->expanded) grib_context_free(c,self->expanded); - self->expanded=expand(a,&(self->expandedSize),&ret); - self->dirty=0; + ret=expand(a); if (ret) { - grib_context_log(a->parent->h->context,GRIB_LOG_ERROR, - "%s unable to compute size",a->name); - return ret; + grib_context_log(c,GRIB_LOG_ERROR,"%s unable to compute size",a->name); + return ret; } - *rlen=self->expandedSize; + *rlen = grib_bufr_descriptors_array_used_size(self->expanded); return ret; } diff --git a/src/grib_api.h b/src/grib_api.h index ad5ace19f..c2d0362a5 100644 --- a/src/grib_api.h +++ b/src/grib_api.h @@ -205,6 +205,8 @@ typedef struct grib_iarray grib_iarray; typedef struct grib_vdarray grib_vdarray; typedef struct grib_vsarray grib_vsarray; typedef struct grib_viarray grib_viarray; +typedef struct bufr_descriptor bufr_descriptor; +typedef struct bufr_descriptors_array bufr_descriptors_array; grib_fieldset *grib_fieldset_new_from_files(grib_context *c, char *filenames[], int nfiles, char **keys, int nkeys, char *where_string, char *order_by_string, int *err); diff --git a/src/grib_api_internal.h b/src/grib_api_internal.h index 4464e7760..4f32c7bc6 100644 --- a/src/grib_api_internal.h +++ b/src/grib_api_internal.h @@ -514,6 +514,12 @@ struct grib_accessor }; +#define BUFR_DESCRIPTOR_FLAG_HAS_VALUES (1<<1) +#define BUFR_DESCRIPTOR_FLAG_DEFAULT_VALUES (1<<2) +#define BUFR_DESCRIPTOR_FLAG_MODIFIED_VALUES (1<<3) +#define BUFR_DESCRIPTOR_FLAG_IS_TABLE (1<<4) +#define BUFR_DESCRIPTOR_FLAG_IS_FLAG (1<<5) +#define BUFR_DESCRIPTOR_FLAG_IS_STRING (1<<6) #define GRIB_ACCESSOR_FLAG_READ_ONLY (1<<1) #define GRIB_ACCESSOR_FLAG_DUMP (1<<2) @@ -744,6 +750,26 @@ struct grib_viarray { grib_context* context; } ; +struct bufr_descriptor { + long code; + int F; + int X; + int Y; + int flags; + long scale; + double reference; + long width; +} ; + +struct bufr_descriptors_array { + bufr_descriptor** v; + size_t size; + size_t n; + size_t incsize; + size_t number_of_pop_front; + grib_context* context; +} ; + #define MAX_SET_VALUES 10 #define MAX_ACCESSOR_CACHE 100 diff --git a/src/grib_api_prototypes.h b/src/grib_api_prototypes.h index 64df80a07..f1c6cfb4f 100644 --- a/src/grib_api_prototypes.h +++ b/src/grib_api_prototypes.h @@ -145,6 +145,29 @@ grib_hash_array_value *grib_integer_hash_array_value_new(grib_context *c, const grib_hash_array_value *grib_double_hash_array_value_new(grib_context *c, const char *name, grib_darray *array); void grib_hash_array_value_delete(grib_context *c, grib_hash_array_value *v); +/* grib_bufr_descriptor.c */ +bufr_descriptor *grib_bufr_descriptor_new(grib_context *c, int code); +bufr_descriptor *grib_bufr_descriptor_clone(grib_context *c, bufr_descriptor *d); +void grib_bufr_descriptor_set_code(bufr_descriptor *v, int code); +void grib_bufr_descriptor_set_values(bufr_descriptor *v, int scale, int reference, int width); +void grib_bufr_descriptor_delete(grib_context *c, bufr_descriptor *v); + +/* grib_bufr_descriptors_array.c */ +bufr_descriptors_array *grib_bufr_descriptors_array_new(grib_context *c, size_t size, size_t incsize); +bufr_descriptor *grib_bufr_descriptors_array_pop(bufr_descriptors_array *a); +bufr_descriptor *grib_bufr_descriptors_array_pop_front(bufr_descriptors_array *a); +bufr_descriptors_array *grib_bufr_descriptors_array_resize_to(bufr_descriptors_array *v, size_t newsize); +bufr_descriptors_array *grib_bufr_descriptors_array_resize(bufr_descriptors_array *v); +bufr_descriptors_array *grib_bufr_descriptors_array_push(bufr_descriptors_array *v, bufr_descriptor *val); +bufr_descriptors_array *grib_bufr_descriptors_array_append(bufr_descriptors_array *v, bufr_descriptors_array *ar); +bufr_descriptors_array *grib_bufr_descriptors_array_push_front(bufr_descriptors_array *v, bufr_descriptor *val); +bufr_descriptor *grib_bufr_descriptors_array_get(bufr_descriptors_array *a, size_t i); +void grib_bufr_descriptors_array_set(bufr_descriptors_array *a, size_t i, bufr_descriptor *v); +void grib_bufr_descriptors_array_delete(bufr_descriptors_array *v); +void grib_bufr_descriptors_array_delete_array(bufr_descriptors_array *v); +bufr_descriptor **grib_bufr_descriptors_array_get_array(bufr_descriptors_array *v); +size_t grib_bufr_descriptors_array_used_size(bufr_descriptors_array *v); + /* grib_darray.c */ grib_darray *grib_darray_new(grib_context *c, size_t size, size_t incsize); grib_darray *grib_darray_resize(grib_context *c, grib_darray *v); @@ -525,7 +548,6 @@ second_order_packed *grib_get_second_order_groups(grib_context *c, const unsigne /* grib_accessor_class_unexpanded_descriptors.c */ /* grib_accessor_class_expanded_descriptors.c */ -size_t __expand(grib_accessor *a, grib_iarray *unexpanded, grib_iarray *expanded, int *err); /* grib_accessor_class_bufrdc_expanded_descriptors.c */ @@ -699,6 +721,14 @@ void grib_timer_print(grib_timer *t); void grib_timer_partial_rate(grib_timer *t, double start, long total); void grib_print_all_timers(void); void grib_reset_all_timers(void); +grib_timer *grib_get_timer(grib_context *c, const char *name, const char *statname, int elapsed); +int grib_timer_start(grib_timer *t); +int grib_timer_stop(grib_timer *t, long total); +double grib_timer_value(grib_timer *t); +void grib_timer_print(grib_timer *t); +void grib_timer_partial_rate(grib_timer *t, double start, long total); +void grib_print_all_timers(void); +void grib_reset_all_timers(void); /* grib_ibmfloat.c */ unsigned long grib_ibm_to_long(double x); @@ -717,6 +747,10 @@ double grib_ieeefloat_error(double x); double grib_long_to_ieee(unsigned long x); unsigned long grib_ieee_nearest_smaller_to_long(double x); int grib_nearest_smaller_ieee_float(double a, double *ret); +double grib_ieeefloat_error(double x); +double grib_long_to_ieee(unsigned long x); +int grib_nearest_smaller_ieee_float(double a, double *x); +unsigned long grib_ieee_to_long(double x); unsigned long grib_ieee64_to_long(double x); double grib_long_to_ieee64(unsigned long x); int grib_ieee_decode_array(grib_context *c, unsigned char *buf, size_t nvals, int bytes, double *val); diff --git a/src/grib_bufr_descriptor.c b/src/grib_bufr_descriptor.c new file mode 100644 index 000000000..67b27e594 --- /dev/null +++ b/src/grib_bufr_descriptor.c @@ -0,0 +1,61 @@ +/* + * Copyright 2005-2014 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by + * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. + */ + + +#include "grib_api_internal.h" + +bufr_descriptor* grib_bufr_descriptor_new(grib_context* c,int code) { + bufr_descriptor* v=NULL; + + if (!c) c=grib_context_get_default(); + + v=(bufr_descriptor*)grib_context_malloc_clear(c,sizeof(bufr_descriptor)); + if (!v) { + grib_context_log(c,GRIB_LOG_ERROR, + "grib_bufr_descriptor_new unable to allocate %d bytes\n",sizeof(bufr_descriptor)); + return NULL; + } + + grib_bufr_descriptor_set_code(v,code); + + return v; +} + +bufr_descriptor* grib_bufr_descriptor_clone(grib_context* c,bufr_descriptor* d) { + + if (!d) return NULL; + + return grib_bufr_descriptor_new(c,d->code); +} + +void grib_bufr_descriptor_set_code(bufr_descriptor* v,int code) { + if (!v) return; + + v->code=code; + v->F=code/100000; + v->X=(code-v->F*100000)/1000; + v->Y=(code-v->F*100000)%1000; +} + +void grib_bufr_descriptor_set_values(bufr_descriptor* v,int scale,int reference,int width) { + if (!v) return; + v->scale=scale; + v->reference=reference; + v->width=width; + v->flags=BUFR_DESCRIPTOR_FLAG_HAS_VALUES; +} + +void grib_bufr_descriptor_delete(grib_context* c,bufr_descriptor* v) { + + if (!v) return; + + grib_context_free(c,v); +} + diff --git a/src/grib_bufr_descriptors_array.c b/src/grib_bufr_descriptors_array.c new file mode 100644 index 000000000..9e0f40a7f --- /dev/null +++ b/src/grib_bufr_descriptors_array.c @@ -0,0 +1,184 @@ +/* + * Copyright 2005-2014 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by + * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. + */ + +#include "grib_api_internal.h" + +bufr_descriptors_array* grib_bufr_descriptors_array_new(grib_context* c,size_t size,size_t incsize) { + bufr_descriptors_array* v=NULL; + + if (!c) c=grib_context_get_default(); + + v=(bufr_descriptors_array*)grib_context_malloc(c,sizeof(bufr_descriptors_array)); + if (!v) { + grib_context_log(c,GRIB_LOG_ERROR, + "bufr_descriptors_array_new unable to allocate %d bytes\n",sizeof(bufr_descriptors_array)); + return NULL; + } + v->context=c; + v->size=size; + v->n=0; + v->incsize=incsize; + v->v=(bufr_descriptor**)grib_context_malloc(c,sizeof(bufr_descriptor*)*size); + v->number_of_pop_front=0; + if (!v->v) { + grib_context_log(c,GRIB_LOG_ERROR, + "grib_bufr_descriptors_array_new unable to allocate %d bytes\n",sizeof(bufr_descriptor)*size); + return NULL; + } + return v; +} + +bufr_descriptor* grib_bufr_descriptors_array_pop(bufr_descriptors_array* a) { + a->n-=1; + return a->v[a->n]; +} + +bufr_descriptor* grib_bufr_descriptors_array_pop_front(bufr_descriptors_array* a) { + bufr_descriptor* v=a->v[0]; + /* size_t i=0; */ + if (a->n==0) Assert(0); + a->n--; + a->v++; + a->number_of_pop_front++; + /* for (i=0;in;i++) a->v[i]=a->v[i+1]; */ + + return v; +} + +bufr_descriptors_array* grib_bufr_descriptors_array_resize_to(bufr_descriptors_array* v,size_t newsize) { + bufr_descriptor** newv; + size_t i; + grib_context* c=v->context; + + if (newsizesize) return v; + + if (!c) c=grib_context_get_default(); + + newv=grib_context_malloc_clear(c,newsize*sizeof(bufr_descriptor*)); + if (!newv) { + grib_context_log(c,GRIB_LOG_ERROR, + "grib_bufr_descriptors_array_resize unable to allocate %d bytes\n",sizeof(bufr_descriptor*)*newsize); + return NULL; + } + + for (i=0;in;i++) newv[i]=v->v[i]; + + v->v-=v->number_of_pop_front; + grib_context_free(c,v->v); + + v->v=newv; + v->size=newsize; + v->number_of_pop_front=0; + + return v; +} + +bufr_descriptors_array* grib_bufr_descriptors_array_resize(bufr_descriptors_array* v) { + int newsize=v->incsize+v->size; + + return grib_bufr_descriptors_array_resize_to(v,newsize); +} + +bufr_descriptors_array* grib_bufr_descriptors_array_push(bufr_descriptors_array* v,bufr_descriptor* val) { + size_t start_size=100; + size_t start_incsize=100; + + if (!v) v=grib_bufr_descriptors_array_new(0,start_size,start_incsize); + + if (v->n >= v->size-v->number_of_pop_front) + v=grib_bufr_descriptors_array_resize(v); + + v->v[v->n]=val; + v->n++; + return v; +} + +bufr_descriptors_array* grib_bufr_descriptors_array_append(bufr_descriptors_array* v,bufr_descriptors_array* ar) { + size_t start_size=100; + size_t start_incsize=100; + size_t i; + + if (!v) v=grib_bufr_descriptors_array_new(0,start_size,start_incsize); + + for (i=0;in;i++) + grib_bufr_descriptors_array_push(v,ar->v[i]); + + grib_context_free(v->context,ar); + ar=0; + + return v; +} + +bufr_descriptors_array* grib_bufr_descriptors_array_push_front(bufr_descriptors_array* v,bufr_descriptor* val) { + size_t start_size=100; + size_t start_incsize=100; + int i; + if (!v) v=grib_bufr_descriptors_array_new(0,start_size,start_incsize); + + if (v->number_of_pop_front) { + v->v--; + v->number_of_pop_front--; + } else { + if (v->n >= v->size) v=grib_bufr_descriptors_array_resize(v); + for (i=v->n;i>0;i--) v[i]=v[i-1]; + } + v->v[0]=val; + v->n++; + + return v; +} + +bufr_descriptor* grib_bufr_descriptors_array_get(bufr_descriptors_array* a,size_t i) { + return a->v[i]; +} + +void grib_bufr_descriptors_array_set(bufr_descriptors_array* a,size_t i,bufr_descriptor* v) { + a->v[i]=v; +} + +void grib_bufr_descriptors_array_delete(bufr_descriptors_array* v) { + grib_context* c; + + if (!v) return; + c=v->context; + + grib_bufr_descriptors_array_delete_array(v); + + grib_context_free(c,v); +} + +void grib_bufr_descriptors_array_delete_array(bufr_descriptors_array* v) { + grib_context* c; + int i; + + if (!v) return; + c=v->context; + + if (v->v) { + bufr_descriptor** vv = v->v - v->number_of_pop_front; + for (i=0;in;i++) grib_bufr_descriptor_delete(c,vv[i]); + + grib_context_free(c,vv); + } +} + +bufr_descriptor** grib_bufr_descriptors_array_get_array(bufr_descriptors_array* v) { + bufr_descriptor** vv; + size_t i; + grib_context* c=grib_context_get_default(); + + vv=grib_context_malloc_clear(c,sizeof(bufr_descriptor*)*v->n); + for (i=0;in;i++) vv[i]=grib_bufr_descriptor_clone(c,v->v[i]); + + return vv; +} + +size_t grib_bufr_descriptors_array_used_size(bufr_descriptors_array* v) {return v->n;} + diff --git a/tests/bufrdc_desc_ref.sh b/tests/bufrdc_desc_ref.sh new file mode 100755 index 000000000..7714fa43d --- /dev/null +++ b/tests/bufrdc_desc_ref.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# Copyright 2005-2014 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +# + +set -e +. ./include.sh + +REDIRECT=/dev/null + +cat > bufrdc_desc_ref.filter< $REDIRECT > $res_desc + + diff $ref_desc $res_desc > $diff_desc 2> $diff_desc + + rm -f $res_desc $diff_desc +done + +rm -f bufrdc_desc_ref.filter diff --git a/tests/bufrdc_ref.sh b/tests/bufrdc_ref.sh index 0ffd71567..c1fd45945 100755 --- a/tests/bufrdc_ref.sh +++ b/tests/bufrdc_ref.sh @@ -8,32 +8,41 @@ # virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. # +set -ex . ./include.sh REDIRECT=/dev/null -cat > bufrdc_ref.filter< bufrdc_num_ref.filter< $REDIRECT > $res_num + #set +e + ${tools_dir}bufr_filter bufrdc_num_ref.filter $file 2> $REDIRECT > $res_num + if [ $? != 0 ] + then + mv $file $file.no + fi # Cannot use plain diff. We need to compare FLOAT NUMBERS with a tolerance - #numdiff $ref_num $res_num >$REDIRECT - perl number_compare.pl $ref_num $res_num >$REDIRECT 2> $REDIRECT + #perl number_compare.pl $ref_num $res_num >$REDIRECT 2> $REDIRECT + numdiff $ref_num $res_num > $diff_num 2> $diff_num + if [ $? != 0 ] + then + mv $file $file.no + fi - rm -f $res_num + #rm -f $res_num $diff_num done