diff --git a/definitions/bufr/section.3.def b/definitions/bufr/section.3.def index bad6f73fd..2a656bc22 100644 --- a/definitions/bufr/section.3.def +++ b/definitions/bufr/section.3.def @@ -68,6 +68,7 @@ meta numberOfUnexpandedDescriptors evaluate( (section3Length - 7) / 2 ) ; meta unexpandedDescriptors unexpanded_descriptors(numberOfUnexpandedDescriptors) :dump; meta expandedDescriptors expanded_descriptors(unexpandedDescriptors,sequences) ; +meta bufrdcExpandedDescriptors bufrdc_expanded_descriptors(expandedDescriptors); #smart_table NAME (VALUES,FILE_NAME,MASTER_DIRECTORY,LOCAL_DIRECTORY,WIDTH_OF_CODE_IN_BITS,EXTRA_DIRECTORY,EXTRA_FILE_NAME); smart_table expanded (expandedDescriptors,"element.table",tablesMasterDir,tablesLocalDir,18,rootTablesDir,"operators.table") ; diff --git a/definitions/bufr/section.4.def b/definitions/bufr/section.4.def index b7bc0afb8..da38bbc53 100644 --- a/definitions/bufr/section.4.def +++ b/definitions/bufr/section.4.def @@ -20,12 +20,12 @@ if (compressedData) { meta numericValues bufr_data_array(offsetSection4,offsetBeforeData,offsetEndSection4,section4Length, numberOfDataSubsets,subsetNumber, expandedDescriptors,type,unit,reference,scale,width, - stringValues,elementsFXY,compressedData) ; + stringValues,elementsDescriptorsIndex,compressedData) ; } else { meta numericValues bufr_data_array(offsetSection4,offsetBeforeData,offsetEndSection4,section4Length, numberOfDataSubsets,subsetNumber, expandedDescriptors,type,unit,reference,scale,width, - stringValues,elementsFXY,compressedData) ; + stringValues,elementsDescriptorsIndex,compressedData) ; } meta unpack unpack_bufr_values(numericValues); diff --git a/definitions/bufr/tables/0/local/1/98/0/element.table b/definitions/bufr/tables/0/local/1/98/0/element.table index 172cb1bc9..72ce1f5da 100644 --- a/definitions/bufr/tables/0/local/1/98/0/element.table +++ b/definitions/bufr/tables/0/local/1/98/0/element.table @@ -113,6 +113,13 @@ 013231|largeScalePrecipitation|double|LARGE SCALE PRECIPITATION|KG/M**2|1|-1|14|0|0| 013240|airDensity|double|AIR DENSITY|KG/M**3|3|0|10|0|0| 013241|convectiveAvailablePotentialEnergy|double|CONVECTIVE AVAILABLE POTENTIAL ENERGY|J/KG|1|0|17|0|0| +014001|longWaveRadiationIntegratedOver24Hours|long|LONG-WAVE RADIATION, INTEGRATED OVER 24 HOURS|JM-2|-3|-2048|12|J M-2|-3|4 +014002|longWaveRadiationIntegratedOverPeriodSpecified|long|LONG-WAVE RADIATION, INTEGRATED OVER PERIOD SPECIFIED|JM-2|-3|-2048|12|J M-2|-3|4 +014003|shortWaveRadiationIntegratedOver24Hours|long|SHORT-WAVE RADIATION, INTEGRATED OVER 24 HOURS|JM-2|-3|-2048|12|J M-2|-3|4 +014004|shortWaveRadiationIntegratedOverPeriodSpecified|long|SHORT-WAVE RADIATION, INTEGRATED OVER PERIOD SPECIFIED|JM-2|-3|-2048|12|J M-2|-3|4 +014028|globalSolarRadiationIntegratedOverPeriodSpecified|long|GLOBAL SOLAR RADIATION (HIGH ACCURACY), INTEGRATED OVER PERIOD SPECIFIED|JM-2|-2|0|16|J M-2|-2|5 +014029|diffuseSolarRadiationIntegratedOverPeriodSpecified|long|DIFFUSE SOLAR RADIATION (HIGH ACCURACY), INTEGRATED OVER PERIOD SPECIFIED|JM-2|-2|0|16|J M-2|-2|5 +014030|directSolarRadiationIntegratedOverPeriodSpecified|long|DIRECT SOLAR RADIATION (HIGH ACCURACY), INTEGRATED OVER PERIOD SPECIFIED|JM-2|-2|0|16|J M-2|-2|5 015202|integratedElectronDensity|double|INTEGRATED ELECTRON DENSITY|1/M**2|3|13000|12|0|0| 015231|atmosphericPathDelayInSatelliteSignal|double|ATMOSPHERIC PATH DELAY IN SATELLITE SIGNAL|M|4|0|15|0|0| 020192|specialPhenomena|long|SPECIAL PHENOMENA|CODE TABLE 20192|0|0|14|0|0| diff --git a/definitions/bufr/tables/0/local/1/98/0/sequence.def b/definitions/bufr/tables/0/local/1/98/0/sequence.def index 3bed26af3..c948a1307 100644 --- a/definitions/bufr/tables/0/local/1/98/0/sequence.def +++ b/definitions/bufr/tables/0/local/1/98/0/sequence.def @@ -16,6 +16,7 @@ "301241" = [ 201141,202130,007001,201000,202000 ] "301242" = [ 021207,021208,021209,021210,021211,021212 ] "301250" = [ 301193,301011,301013,301021 ] +"302175" = [ 008021, 004025, 013055, 013058, 008021 ] "302205" = [ 201131,202129,022021,201000,202000 ] "302206" = [ 302205,201130,202129,022011,201000,202000,022001,011001,011002 ] "302207" = [ 022193,022194,022195,022196,022197 ] @@ -29,6 +30,16 @@ "304250" = [ 002231,002232,008012,007024,002057,008021,004001,004002,004003,004004, 008021,004024,110004,008021,004004,004005,004006,008021,004004,004005, 004006,011001,011002,103010,002231,007004,012001 ] +"308008" = [ 001003, 001020, 001005, 002001, 002036, 002149, 301011, 301012, 008021, + 301011, 301012, 008021, 301021, 027004, 028004, 007030, 001051, 002148, + 001012, 001014, 002040, 033022, 033023, 033027, 022063, 302021, 302022, + 302023, 008081, 025026, 008081, 025026, 008081, 025026, 008081, 002034, + 022060, 007070, 002190, 025086, 002035, 002168, 020031, 002038, 306004, + 002030, 306005, 007031, 008081, 012064, 302001, 008081, 007032, 007033, + 012101, 012103, 013003, 007032, 007033, 008082, 007033, 002169, 002002, + 008021, 004025, 011001, 011002, 008021, 004025, 011043, 011041, 008082, + 007033, 007032, 004024, 013011, 007032, 008021, 004024, 014021, 008021, + 025028, 025028, 025028 ] "309194" = [ 301194,302004,101000,031001,303014 ] "309195" = [ 301195,302004,101000,031001,303014 ] "309196" = [ 301196,302004,101000,031001,303014 ] @@ -43,6 +54,9 @@ 107000,031002,301021,005021,103000,031001,002121,007040,015037 ] "311193" = [ 301197,301011,301012,301023,008004,007004,008021,011001,011002,011031, 011034,011035,012001,012003,013003,020041 ] +"312060" = [ 025060, 025062, 040001, 040002, 021062, 021151, 021152, 021153, 021154, 021062, + 021062, 040003, 040004, 040005, 040006, 040007, 020065, 040008, 040009, 040010 + ] "312200" = [ 301237,301238,029002,021206,104012,006232,102012,005232,021205,021196 ] "312201" = [ 301237,101003,301239,011012,011011,021197 ] diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 38486bda7..1c54fc587 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -194,6 +194,7 @@ list( APPEND grib_api_srcs grib_accessor_class_g2grid.c grib_accessor_class_unexpanded_descriptors.c grib_accessor_class_expanded_descriptors.c + grib_accessor_class_bufrdc_expanded_descriptors.c grib_accessor_class_data_apply_bitmap.c grib_accessor_class_data_apply_boustrophedonic.c grib_accessor_class_data_apply_boustrophedonic_bitmap.c diff --git a/src/Makefile.am b/src/Makefile.am index 25cd3d84c..f13c2d149 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -203,6 +203,7 @@ libgrib_api_la_prototypes= \ grib_accessor_class_g2grid.c \ grib_accessor_class_unexpanded_descriptors.c \ grib_accessor_class_expanded_descriptors.c \ + grib_accessor_class_bufrdc_expanded_descriptors.c \ grib_accessor_class_data_apply_bitmap.c \ grib_accessor_class_data_apply_boustrophedonic.c \ grib_accessor_class_data_apply_boustrophedonic_bitmap.c \ diff --git a/src/grib_accessor_class.h b/src/grib_accessor_class.h index 1eb29e8ee..c7020c0ac 100644 --- a/src/grib_accessor_class.h +++ b/src/grib_accessor_class.h @@ -20,6 +20,7 @@ extern grib_accessor_class* grib_accessor_class_bufr_group_number; extern grib_accessor_class* grib_accessor_class_bufr_has_delayed_replication; extern grib_accessor_class* grib_accessor_class_bufr_subset_number; extern grib_accessor_class* grib_accessor_class_bufr_uncompressed_data; +extern grib_accessor_class* grib_accessor_class_bufrdc_expanded_descriptors; extern grib_accessor_class* grib_accessor_class_bytes; extern grib_accessor_class* grib_accessor_class_change_scanning_direction; extern grib_accessor_class* grib_accessor_class_codeflag; diff --git a/src/grib_accessor_class_bufr_data_array.c b/src/grib_accessor_class_bufr_data_array.c index a4c220092..904ce9870 100644 --- a/src/grib_accessor_class_bufr_data_array.c +++ b/src/grib_accessor_class_bufr_data_array.c @@ -38,10 +38,11 @@ MEMBERS= const char* scaleName MEMBERS= const char* widthName MEMBERS= const char* stringValuesName - MEMBERS= const char* elementsFXYName + MEMBERS= const char* elementsDescriptorsIndexName MEMBERS= const char* compressedDataName MEMBERS= long* expandedDescriptors MEMBERS= int* type + MEMBERS= int* canBeMissing MEMBERS= long* reference MEMBERS= double* factor MEMBERS= long* width @@ -51,8 +52,11 @@ MEMBERS= size_t numberOfDescriptors MEMBERS= grib_vdarray* numericValues MEMBERS= grib_vsarray* stringValues - MEMBERS= grib_viarray* elementsFXY + MEMBERS= grib_viarray* elementsDescriptorsIndex MEMBERS= int do_decode + MEMBERS= int bitmapStartReferredDescriptorsIndex + MEMBERS= int bitmapEndReferredDescriptorsIndex + MEMBERS= int bitmapCurrentRank END_CLASS_DEF @@ -100,10 +104,11 @@ typedef struct grib_accessor_bufr_data_array { const char* scaleName; const char* widthName; const char* stringValuesName; - const char* elementsFXYName; + const char* elementsDescriptorsIndexName; const char* compressedDataName; long* expandedDescriptors; int* type; + int* canBeMissing; long* reference; double* factor; long* width; @@ -113,8 +118,11 @@ typedef struct grib_accessor_bufr_data_array { size_t numberOfDescriptors; grib_vdarray* numericValues; grib_vsarray* stringValues; - grib_viarray* elementsFXY; + grib_viarray* elementsDescriptorsIndex; int do_decode; + int bitmapStartReferredDescriptorsIndex; + int bitmapEndReferredDescriptorsIndex; + int bitmapCurrentRank; } grib_accessor_bufr_data_array; extern grib_accessor_class* grib_accessor_class_gen; @@ -191,6 +199,12 @@ static void init_class(grib_accessor_class* c) /* END_CLASS_IMP */ +static int can_be_missing(int descriptor) { + int ret=1; + if (descriptor==31031) ret=0; + return ret; +} + static long init_length(grib_accessor* a) { grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a; @@ -223,12 +237,13 @@ static void init(grib_accessor* a,const long v, grib_arguments* params) self->scaleName = grib_arguments_get_name(a->parent->h,params,n++); self->widthName = grib_arguments_get_name(a->parent->h,params,n++); self->stringValuesName = grib_arguments_get_name(a->parent->h,params,n++); - self->elementsFXYName = grib_arguments_get_name(a->parent->h,params,n++); + self->elementsDescriptorsIndexName = grib_arguments_get_name(a->parent->h,params,n++); self->compressedDataName = grib_arguments_get_name(a->parent->h,params,n++); self->do_decode=1; - self->elementsFXY=0; + self->elementsDescriptorsIndex=0; self->numericValues=0; self->stringValues=0; + self->bitmapCurrentRank=-1; a->parent->h->unpacked=0; a->length = init_length(a); @@ -239,10 +254,15 @@ static void init(grib_accessor* a,const long v, grib_arguments* params) static void self_clear(grib_context* c,grib_accessor_bufr_data_array* self) { grib_context_free(c,self->expandedDescriptors); grib_context_free(c,self->type); + grib_context_free(c,self->canBeMissing); grib_context_free(c,self->reference); grib_context_free(c,self->width); grib_context_free(c,self->factor); + grib_vdarray_delete(c,self->numericValues); + grib_vsarray_delete(c,self->stringValues); + grib_viarray_delete(c,self->elementsDescriptorsIndex); } + static int get_native_type(grib_accessor* a){ return GRIB_TYPE_DOUBLE; } @@ -288,6 +308,91 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len) return GRIB_NOT_IMPLEMENTED; } +static void apply_early_operators(grib_accessor *a) { + long* descriptors; + int i,j,F,X,Y,elementsToRepeat,jReplication; + int extraWidth,extraScale,localDescriptorWidth; + double referenceFactor; + grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a; + grib_context* c=a->parent->h->context; + + descriptors=grib_context_malloc_clear(c,self->numberOfDescriptors*sizeof(long)); + + extraWidth=0; + extraScale=0; + referenceFactor=1; + localDescriptorWidth=-1; + j=0; + elementsToRepeat=0; + for (i=0;inumberOfDescriptors;i++) { + F=self->expandedDescriptors[i]/100000; + X=(self->expandedDescriptors[i]-F*100000)/1000; + Y=(self->expandedDescriptors[i]-F*100000)%1000; + if (F==2) { + switch(X) { + case 1: + extraWidth = Y ? Y-128 : 0; + if (elementsToRepeat>0) descriptors[jReplication]-=1000; + break; + case 2: + extraScale = Y ? Y-128 : 0; + if (elementsToRepeat>0) descriptors[jReplication]-=1000; + break; + /* case 4: */ + /* associated field*/ + /* associatedFieldWidth=Y; */ + /* break; */ + case 6: + /*signify data width*/ + localDescriptorWidth=Y; + if (elementsToRepeat>0) descriptors[jReplication]-=1000; + break; + case 7: + if (Y) { + extraScale = Y; + referenceFactor=grib_power(Y,10); + extraWidth=((10*Y)+2)/3; + } else { + extraWidth=0; + extraScale=0; + referenceFactor=1; + } + if (elementsToRepeat>0) descriptors[jReplication]-=1000; + break; + default: + descriptors[j]=self->expandedDescriptors[i]; + self->type[j]=self->type[i]; + self->width[j]=self->width[i]; + self->reference[j]=self->reference[i]; + self->factor[j]=self->factor[i]; + self->canBeMissing[j]=self->canBeMissing[i]; + elementsToRepeat--; + j++; + } + } else { + if (F==1) { elementsToRepeat=X; jReplication=j; } + if (self->type[i]==BUFR_TYPE_CODETABLE || self->type[i]==BUFR_TYPE_FLAGTABLE) { + self->width[j]=self->width[i]; + self->reference[j]=self->reference[i]; + self->factor[j]=self->factor[i]; + } else { + self->width[j]= localDescriptorWidth>0 ? localDescriptorWidth : self->width[i]+extraWidth; + self->reference[j]=self->reference[i]*referenceFactor; + self->factor[j]=extraScale ? self->factor[i]*grib_power(-extraScale,10) : self->factor[i]; + } + self->canBeMissing[j]=self->canBeMissing[i]; + descriptors[j]=self->expandedDescriptors[i]; + self->type[j]=self->type[i]; + elementsToRepeat--; + j++; + } + } + self->numberOfDescriptors=j; + grib_context_free(c,self->expandedDescriptors); + self->expandedDescriptors=descriptors; + +} + static int get_descriptors(grib_accessor* a) { int err=0; size_t size=0; @@ -325,13 +430,19 @@ static int get_descriptors(grib_accessor* a) { return err; self->type=grib_context_malloc_clear(c,self->numberOfDescriptors*sizeof(int)); + size=self->numberOfDescriptors; ctype=(char**)grib_context_malloc_clear(c,size*sizeof(char*)); err=grib_get_string_array(h,self->typeName,ctype,&size); + size=self->numberOfDescriptors; + self->canBeMissing=grib_context_malloc_clear(c,self->numberOfDescriptors*sizeof(int)); cunits=(char**)grib_context_malloc_clear(c,size*sizeof(char*)); + err=grib_get_string_array(h,self->unitsName,cunits,&size); + for (i=0;inumberOfDescriptors;i++) { + self->canBeMissing[i]=can_be_missing(self->expandedDescriptors[i]); if (*(ctype[i]) =='s') { self->type[i]=BUFR_TYPE_STRING; } else { @@ -376,19 +487,20 @@ static int get_descriptors(grib_accessor* a) { err=grib_get_long(h,self->compressedDataName,&(self->compressedData)); err=grib_get_long(h,self->subsetNumberName,&(self->subsetNumber)); + apply_early_operators(a); + return err; } static grib_sarray* decode_string_array(grib_context* c,unsigned char* data,long* pos, int i, - grib_accessor_bufr_data_array* self, - int extraWidth,int extraScale,double referenceFactor,int associatedFieldWidth,int localDescriptorWidth) { + grib_accessor_bufr_data_array* self) { grib_sarray* ret=NULL; char* sval=0; int j,modifiedWidth,modifiedReference,width; double modifiedFactor; - modifiedWidth= extraWidth ? self->width[i]+extraWidth : self->width[i]; - modifiedReference= referenceFactor ? self->reference[i]*referenceFactor : self->reference[i]; - modifiedFactor= extraScale ? self->factor[i]*grib_power(-extraScale,10) : self->factor[i]; + modifiedWidth= self->width[i]; + modifiedReference= self->reference[i]; + modifiedFactor= self->factor[i]; ret=grib_sarray_new(c,10,10); sval=grib_context_malloc_clear(c,modifiedWidth/8+1); @@ -410,38 +522,25 @@ static grib_sarray* decode_string_array(grib_context* c,unsigned char* data,long } static grib_darray* decode_double_array(grib_context* c,unsigned char* data,long* pos,int i, - grib_accessor_bufr_data_array* self,int canBeMissing, - int extraWidth,int extraScale,double referenceFactor,int associatedFieldWidth,int localDescriptorWidth) { + grib_accessor_bufr_data_array* self) { grib_darray* ret=NULL; int j; unsigned long lval; - long localReference; - int width,modifiedWidth,modifiedReference; + int localReference,localWidth,modifiedWidth,modifiedReference; double modifiedFactor,dval; - if (self->type[i]==BUFR_TYPE_CODETABLE || self->type[i]==BUFR_TYPE_FLAGTABLE) { - modifiedReference= self->reference[i]; - modifiedFactor= self->factor[i]; - modifiedWidth= self->width[i]; - } else { - modifiedReference= self->reference[i]*referenceFactor; - modifiedFactor= extraScale ? self->factor[i]*grib_power(-extraScale,10) : self->factor[i]; - - if (localDescriptorWidth) { - modifiedWidth=localDescriptorWidth; - } else { - modifiedWidth= self->width[i]+extraWidth; - } - } + modifiedReference= self->reference[i]; + modifiedFactor= self->factor[i]; + modifiedWidth= self->width[i]; lval=grib_decode_unsigned_long(data,pos,modifiedWidth); localReference=(long)lval+modifiedReference; - width=grib_decode_unsigned_long(data,pos,6); + localWidth=grib_decode_unsigned_long(data,pos,6); ret=grib_darray_new(c,100,100); - if (width) { + if (localWidth) { for (j=0;jnumberOfDataSubsets;j++) { - lval=grib_decode_unsigned_long(data,pos,width); - if (grib_is_all_bits_one(lval,width) && canBeMissing) { + lval=grib_decode_unsigned_long(data,pos,localWidth); + if (grib_is_all_bits_one(lval,localWidth) && self->canBeMissing[i]) { dval=GRIB_MISSING_DOUBLE; } else { dval=((long)lval+localReference)*modifiedFactor; @@ -449,7 +548,7 @@ static grib_darray* decode_double_array(grib_context* c,unsigned char* data,long grib_darray_push(c,ret,dval); } } else { - if (grib_is_all_bits_one(lval,modifiedWidth) && canBeMissing) { + if (grib_is_all_bits_one(lval,modifiedWidth) && self->canBeMissing[i]) { dval=GRIB_MISSING_DOUBLE; } else { dval=localReference*modifiedFactor; @@ -461,14 +560,13 @@ static grib_darray* decode_double_array(grib_context* c,unsigned char* data,long } static char* decode_string_value(grib_context* c,unsigned char* data,long* pos, int i, - grib_accessor_bufr_data_array* self, - int extraWidth,int extraScale,double referenceFactor,int associatedFieldWidth,int localDescriptorWidth) { + grib_accessor_bufr_data_array* self) { char* sval=0; int modifiedWidth,modifiedReference; double modifiedFactor; - modifiedWidth= extraWidth ? self->width[i]+extraWidth : self->width[i]; - modifiedReference= referenceFactor ? self->reference[i]*referenceFactor : self->reference[i]; - modifiedFactor= extraScale ? self->factor[i]*grib_power(-extraScale,10) : self->factor[i]; + modifiedWidth= self->width[i]; + modifiedReference= self->reference[i]; + modifiedFactor= self->factor[i]; sval=grib_context_malloc_clear(c,modifiedWidth/8+1); grib_decode_string(data,pos,modifiedWidth/8,sval); @@ -477,29 +575,17 @@ static char* decode_string_value(grib_context* c,unsigned char* data,long* pos, } static double decode_double_value(grib_context* c,unsigned char* data,long* pos,int i, - grib_accessor_bufr_data_array* self,int canBeMissing, - int extraWidth,int extraScale,double referenceFactor,int associatedFieldWidth,int localDescriptorWidth) { + grib_accessor_bufr_data_array* self) { unsigned long lval; int modifiedWidth,modifiedReference; double modifiedFactor,dval; - if (self->type[i]==BUFR_TYPE_CODETABLE || self->type[i]==BUFR_TYPE_FLAGTABLE) { - modifiedReference= self->reference[i]; - modifiedFactor= self->factor[i]; - modifiedWidth= self->width[i]; - } else { - modifiedReference= self->reference[i]*referenceFactor; - modifiedFactor= extraScale ? self->factor[i]*grib_power(-extraScale,10) : self->factor[i]; - - if (localDescriptorWidth) { - modifiedWidth=localDescriptorWidth; - } else { - modifiedWidth= self->width[i]+extraWidth; - } - } + modifiedReference= self->reference[i]; + modifiedFactor= self->factor[i]; + modifiedWidth= self->width[i]; lval=grib_decode_unsigned_long(data,pos,modifiedWidth); - if (grib_is_all_bits_one(lval,modifiedWidth) && canBeMissing) { + if (grib_is_all_bits_one(lval,modifiedWidth) && self->canBeMissing[i]) { dval=GRIB_MISSING_DOUBLE; } else { dval=((long)lval+modifiedReference)*modifiedFactor; @@ -507,34 +593,129 @@ static double decode_double_value(grib_context* c,unsigned char* data,long* pos, return dval; } -static int can_be_missing(int F,int X,int Y) { - int ret=1; - if (F==0 && X==31 && Y==31) ret=0; +static void decode_element(grib_context* c,grib_accessor_bufr_data_array* self, + unsigned char* data,long *pos,int i,grib_darray* dval,grib_sarray* sval) { + grib_darray* dar=0; + grib_sarray* sar=0; + int index=0,sar_size,ii; + char* csval=0; + double cdval=0; + if (self->type[i]==BUFR_TYPE_STRING) { + /* string */ + if (self->compressedData) { + sar=decode_string_array(c,data,pos,i,self); + grib_vsarray_push(c,self->stringValues,sar); + index=grib_vsarray_used_size(self->stringValues); + sar_size=grib_sarray_used_size(sar); + dar=grib_darray_new(c,sar_size,10); + for (ii=0;iiv[ii]); + grib_darray_push(c,dar,x); + } + grib_vdarray_push(c,self->numericValues,dar); + } else { + csval=decode_string_value(c,data,pos,i,self); + grib_sarray_push(c,sval,csval); + index=grib_sarray_used_size(sval); + cdval=index*1000+strlen(csval); + grib_darray_push(c,dval,cdval); + } + } else { + /* numeric or codetable or flagtable */ + if (self->compressedData) { + dar=decode_double_array(c,data,pos,i,self); + grib_vdarray_push(c,self->numericValues,dar); + } else { + cdval=decode_double_value(c,data,pos,i,self); + grib_darray_push(c,dval,cdval); + } + } +} + +static int build_bitmap(grib_accessor_bufr_data_array *self,unsigned char* data,long* pos,int iBitmapOperator) { + int bitmapSize,iDelayedReplication=0; + int i,localReference,width; + long ppos; + grib_accessor* a=(grib_accessor*)self; + grib_context* c=a->parent->h->context; + + switch (self->expandedDescriptors[iBitmapOperator]) { + case 236000: + i=iBitmapOperator; + while (self->expandedDescriptors[i]>200000) i--; + self->bitmapEndReferredDescriptorsIndex=i; + i=iBitmapOperator+1; + if (self->expandedDescriptors[i]==101000) { + iDelayedReplication=iBitmapOperator+2; + Assert( self->expandedDescriptors[iDelayedReplication]==31001 || + self->expandedDescriptors[iDelayedReplication]==31002 ); + i=iDelayedReplication; + if (self->compressedData) { + ppos=*pos; + localReference=grib_decode_unsigned_long(data,pos,self->width[i])+self->reference[i]; + width=grib_decode_unsigned_long(data,pos,6); + *pos=ppos; + if (width) { + /* delayed replication number is not constant. NOT IMPLEMENTED */ + Assert(0); + } else { + bitmapSize=localReference*self->factor[i]; + } + } else { + ppos=*pos; + bitmapSize=grib_decode_unsigned_long(data,pos,self->width[i])+self->reference[i]*self->factor[i]; + *pos=ppos; + } + } else if (self->expandedDescriptors[i]==31031){ + bitmapSize=1; + while (self->expandedDescriptors[i]==31031) {bitmapSize++;i++;} + } + self->bitmapStartReferredDescriptorsIndex=self->bitmapEndReferredDescriptorsIndex-bitmapSize; + self->bitmapCurrentRank=0; + break; + default : + grib_context_log(c,GRIB_LOG_ERROR,"unsupported operator %d\n", + self->expandedDescriptors[iBitmapOperator]); + return GRIB_INTERNAL_ERROR; + } + return GRIB_SUCCESS; +} + +static int get_next_bitmap_descriptor_index(grib_accessor_bufr_data_array *self) { + int ret=self->bitmapStartReferredDescriptorsIndex+self->bitmapCurrentRank; + self->bitmapCurrentRank++; return ret; } +static void push_zero_element(grib_accessor_bufr_data_array* self,grib_darray* dval) { + grib_darray* d=0; + grib_accessor* a=(grib_accessor*)self; + grib_context* c=a->parent->h->context; + if (self->compressedData) { + d=grib_darray_new(c,1,100); + grib_darray_push(c,d,0); + grib_vdarray_push(c,self->numericValues,d); + } else { + grib_darray_push(c,dval,0); + } +} + static int decode_elements(grib_accessor* a) { int err=0; int *F=0,*X=0,*Y=0; - int extraWidth,extraScale=0,associatedFieldWidth=0,localDescriptorWidth=0; - double referenceFactor=1; + int associatedFieldWidth=0,localDescriptorWidth=0; int nn; int numberOfElementsToRepeat=0,numberOfRepetitions=0; int startRepetition=0; unsigned char* data=0; - int i,canBeMissing=1; - grib_sarray* sval=0; - grib_darray* dval; - grib_iarray* elementsFXY=0; + int i; + grib_iarray* elementsDescriptorsIndex=0; long localReference=0,width=0; long pos=0; - int index,sval_size,ii; - int iss,end,elementIndex; - double cdval; - char* csval; + int iss,end,elementIndex,index; - grib_vdarray* dvalues = NULL; - grib_vsarray* svalues = NULL; + grib_darray* dval = NULL; + grib_sarray* sval = NULL; grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a; grib_handle* h=a->parent->h; @@ -559,61 +740,33 @@ static int decode_elements(grib_accessor* a) { Y[i]=(self->expandedDescriptors[i]-F[i]*100000)%1000; } - dvalues=grib_vdarray_new(c,100,100); - svalues=grib_vsarray_new(c,10,10); - extraWidth=0; - extraScale=0; - referenceFactor=1; + if (self->numericValues) { + grib_vdarray_delete_content(c,self->numericValues); + grib_vdarray_delete(c,self->numericValues); + grib_vsarray_delete_content(c,self->stringValues); + grib_vsarray_delete(c,self->stringValues); + } + self->numericValues=grib_vdarray_new(c,100,100); + self->stringValues=grib_vsarray_new(c,10,10); - if (self->elementsFXY) grib_viarray_delete(c,self->elementsFXY); - self->elementsFXY=grib_viarray_new(c,100,100); + if (self->elementsDescriptorsIndex) grib_viarray_delete(c,self->elementsDescriptorsIndex); + self->elementsDescriptorsIndex=grib_viarray_new(c,100,100); end= self->compressedData ? 1 : self->numberOfDataSubsets; for (iss=0;isscompressedData) { dval=grib_darray_new(c,100,100); sval=grib_sarray_new(c,10,10); } for (i=0;inumberOfDescriptors;i++) { - canBeMissing=can_be_missing(F[i],X[i],Y[i]); - elementIndex=grib_iarray_used_size(elementsFXY); + elementIndex=grib_iarray_used_size(elementsDescriptorsIndex); switch(F[i]) { case 0: /* Table B element */ - grib_iarray_push(elementsFXY,self->expandedDescriptors[i]); - if (self->type[i]==BUFR_TYPE_STRING) { - /* string */ - if (self->compressedData) { - sval=decode_string_array(c,data,&pos,i,self,extraWidth,extraScale,referenceFactor,associatedFieldWidth,localDescriptorWidth); - grib_vsarray_push(c,svalues,sval); - index=grib_vsarray_used_size(svalues)-1; - sval_size=grib_sarray_used_size(sval); - dval=grib_darray_new(c,sval_size,10); - for (ii=0;iiv[ii]); - grib_darray_push(c,dval,x); - } - grib_vdarray_push(c,dvalues,dval); - } else { - csval=decode_string_value(c,data,&pos,i,self,extraWidth,extraScale,referenceFactor,associatedFieldWidth,localDescriptorWidth); - grib_sarray_push(c,sval,csval); - index=grib_sarray_used_size(sval); - sval_size=grib_sarray_used_size(sval); - cdval=index*1000+strlen(csval); - grib_darray_push(c,dval,cdval); - } - } else { - /* numeric or codetable or flagtable */ - if (self->compressedData) { - dval=decode_double_array(c,data,&pos,i,self,canBeMissing,extraWidth,extraScale,referenceFactor,associatedFieldWidth,localDescriptorWidth); - grib_vdarray_push(c,dvalues,dval); - } else { - cdval=decode_double_value(c,data,&pos,i,self,canBeMissing,extraWidth,extraScale,referenceFactor,associatedFieldWidth,localDescriptorWidth); - grib_darray_push(c,dval,cdval); - } - } + grib_iarray_push(elementsDescriptorsIndex,i); + decode_element(c,self,data,&pos,i,dval,sval); break; case 1: /* Delayed replication */ @@ -634,11 +787,11 @@ static int decode_elements(grib_accessor* a) { numberOfRepetitions=grib_decode_unsigned_long(data,&pos,self->width[i])+self->reference[i]*self->factor[i]; startRepetition=i; } - grib_iarray_push(elementsFXY,self->expandedDescriptors[i]); + grib_iarray_push(elementsDescriptorsIndex,i); if (self->compressedData) { dval=grib_darray_new(c,1,100); grib_darray_push(c,dval,(double)numberOfRepetitions); - grib_vdarray_push(c,dvalues,dval); + grib_vdarray_push(c,self->numericValues,dval); } else { grib_darray_push(c,dval,(double)numberOfRepetitions); } @@ -648,60 +801,60 @@ static int decode_elements(grib_accessor* a) { associatedFieldWidth=0; localDescriptorWidth=0; switch(X[i]) { - case 1: - extraWidth = Y[i] ? Y[i]-128 : 0; - break; - case 2: - extraScale = Y[i] ? Y[i]-128 : 0; - break; - /* case 4: */ - /* associated field*/ - /* associatedFieldWidth=Y[i]; */ - /* break; */ - case 6: - /*signify data width*/ - localDescriptorWidth=Y[i]; - break; - case 7: - if (Y) { - extraScale = Y[i]; - referenceFactor=grib_power(Y[i],10); - extraWidth=((10*Y[i])+2)/3; - } else { - extraWidth=0; - extraScale=0; - referenceFactor=1; - } - break; case 22: - case 23: - case 24: - case 25: case 26: case 27: case 29: case 30: case 31: - case 32: case 33: case 34: - case 35: - case 36: - case 37: case 38: case 39: case 40: case 41: case 42: - grib_iarray_push(elementsFXY,self->expandedDescriptors[i]); - if (self->compressedData) { - dval=grib_darray_new(c,1,100); - grib_darray_push(c,dval,0); - grib_vdarray_push(c,dvalues,dval); + grib_iarray_push(elementsDescriptorsIndex,i); + push_zero_element(self,dval); + break; + case 23: + /* substituted values marker operator */ + case 24: + /*first-order statistical values marker operator*/ + case 32: + /*replaced/retained values marker operator*/ + if (Y[i]==255) { + index=get_next_bitmap_descriptor_index(self); + decode_element(c,self,data,&pos,index,dval,sval); + grib_iarray_push(elementsDescriptorsIndex,index); } else { - grib_darray_push(c,dval,0); + grib_iarray_push(elementsDescriptorsIndex,i); + push_zero_element(self,dval); } break; + case 25: + /*difference statistical values marker operator*/ + break; + case 35: + /* cancel bitmap */ + grib_iarray_push(elementsDescriptorsIndex,i); + push_zero_element(self,dval); + if (Y[i]==0) self->bitmapCurrentRank=-1; + break; + case 36: + /* bitmap */ + grib_iarray_push(elementsDescriptorsIndex,i); + push_zero_element(self,dval); + build_bitmap(self,data,&pos,i); + break; + case 37: + /* reuse defined bitmap */ + grib_iarray_push(elementsDescriptorsIndex,i); + push_zero_element(self,dval); + if (Y[i]==0) self->bitmapCurrentRank=0; + /* cancel reuse */ + else self->bitmapCurrentRank=-1; + break; default : grib_context_log(c,GRIB_LOG_ERROR,"unsupported operator %d\n",X[i]); return GRIB_INTERNAL_ERROR; @@ -724,21 +877,13 @@ static int decode_elements(grib_accessor* a) { } } - grib_viarray_push(c,self->elementsFXY,elementsFXY); + grib_viarray_push(c,self->elementsDescriptorsIndex,elementsDescriptorsIndex); if (!self->compressedData) { - grib_vdarray_push(c,dvalues,dval); - grib_vsarray_push(c,svalues,sval); + grib_vdarray_push(c,self->numericValues,dval); + grib_vsarray_push(c,self->stringValues,sval); } } - if (self->numericValues) { - grib_vdarray_delete_content(c,self->numericValues); - grib_vdarray_delete(c,self->numericValues); - grib_vsarray_delete_content(c,self->stringValues); - grib_vsarray_delete(c,self->stringValues); - } - self->numericValues=dvalues; - self->stringValues=svalues; return err; } @@ -778,11 +923,11 @@ static int value_count(grib_accessor* a,long* count) } } else { if (subsetNumber>0) { - *count=grib_iarray_used_size(self->elementsFXY->v[subsetNumber-1]); + *count=grib_iarray_used_size(self->elementsDescriptorsIndex->v[subsetNumber-1]); } else { *count=0; for (i=0;inumberOfDataSubsets;i++) - *count+=grib_iarray_used_size(self->elementsFXY->v[i]); + *count+=grib_iarray_used_size(self->elementsDescriptorsIndex->v[i]); } } @@ -827,14 +972,14 @@ static int unpack_double(grib_accessor* a, double* val, size_t *len) { } } else { if (n>0) { - elementsInSubset=grib_iarray_used_size(self->elementsFXY->v[n]); + elementsInSubset=grib_iarray_used_size(self->elementsDescriptorsIndex->v[n]); for (i=0;inumericValues->v[n-1]->v[i]; } else { ii=0; for (k=0;kelementsFXY->v[k]); + elementsInSubset=grib_iarray_used_size(self->elementsDescriptorsIndex->v[k]); for (i=0;inumericValues->v[k]->v[i]; + val[ii++]=self->numericValues->v[k]->v[i]; } } } diff --git a/src/grib_accessor_class_bufrdc_expanded_descriptors.c b/src/grib_accessor_class_bufrdc_expanded_descriptors.c new file mode 100644 index 000000000..927ac0d1c --- /dev/null +++ b/src/grib_accessor_class_bufrdc_expanded_descriptors.c @@ -0,0 +1,231 @@ +/* +* Copyright 2005-2014 ECMWF. +* +* This software is licensed under the terms of the Apache Licence Version 2.0 +* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +* +* In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +* virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +*/ + +/* Sequences and replication (not delayed) are resolved in this class. +Number of elements to which a delayed replication applies are recomputed to +take account of the expansion. +expanded descriptors cannot contain sequences and only delayed replication +can appear +*/ + +#include "grib_api_internal.h" +/* +This is used by make_class.pl + +START_CLASS_DEF +CLASS = accessor +SUPER = grib_accessor_class_long +IMPLEMENTS = unpack_long +IMPLEMENTS = unpack_string_array +IMPLEMENTS = init;dump;destroy +IMPLEMENTS = value_count +MEMBERS = const char* expandedDescriptors +MEMBERS = grib_accessor* expandedDescriptorsAccessor + +END_CLASS_DEF + +*/ + +/* START_CLASS_IMP */ + +/* + +Don't edit anything between START_CLASS_IMP and END_CLASS_IMP +Instead edit values between START_CLASS_DEF and END_CLASS_DEF +or edit "accessor.class" and rerun ./make_class.pl + +*/ + +static int unpack_long(grib_accessor*, long* val,size_t *len); +static int unpack_string_array (grib_accessor*, char**, size_t *len); +static int value_count(grib_accessor*,long*); +static void destroy(grib_context*,grib_accessor*); +static void dump(grib_accessor*, grib_dumper*); +static void init(grib_accessor*,const long, grib_arguments* ); +static void init_class(grib_accessor_class*); + +typedef struct grib_accessor_bufrdc_expanded_descriptors { + grib_accessor att; +/* Members defined in gen */ +/* Members defined in long */ +/* Members defined in bufrdc_expanded_descriptors */ + const char* expandedDescriptors; + grib_accessor* expandedDescriptorsAccessor; +} grib_accessor_bufrdc_expanded_descriptors; + +extern grib_accessor_class* grib_accessor_class_long; + +static grib_accessor_class _grib_accessor_class_bufrdc_expanded_descriptors = { + &grib_accessor_class_long, /* super */ + "bufrdc_expanded_descriptors", /* name */ + sizeof(grib_accessor_bufrdc_expanded_descriptors), /* size */ + 0, /* inited */ + &init_class, /* init_class */ + &init, /* init */ + 0, /* post_init */ + &destroy, /* free mem */ + &dump, /* describes himself */ + 0, /* get length of section */ + 0, /* get length of string */ + &value_count, /* get number of values */ + 0, /* get number of bytes */ + 0, /* get offset to bytes */ + 0, /* get native type */ + 0, /* get sub_section */ + 0, /* grib_pack procedures long */ + 0, /* grib_pack procedures long */ + 0, /* grib_pack procedures long */ + &unpack_long, /* grib_unpack procedures long */ + 0, /* grib_pack procedures double */ + 0, /* grib_unpack procedures double */ + 0, /* grib_pack procedures string */ + 0, /* grib_unpack procedures string */ + 0, /* grib_pack array procedures string */ + &unpack_string_array, /* grib_unpack array procedures string */ + 0, /* grib_pack procedures bytes */ + 0, /* grib_unpack procedures bytes */ + 0, /* pack_expression */ + 0, /* notify_change */ + 0, /* update_size */ + 0, /* preferred_size */ + 0, /* resize */ + 0, /* nearest_smaller_value */ + 0, /* next accessor */ + 0, /* compare vs. another accessor */ + 0, /* unpack only ith value */ + 0, /* unpack a subarray */ + 0, /* clear */ +}; + + +grib_accessor_class* grib_accessor_class_bufrdc_expanded_descriptors = &_grib_accessor_class_bufrdc_expanded_descriptors; + + +static void init_class(grib_accessor_class* c) +{ + c->next_offset = (*(c->super))->next_offset; + c->string_length = (*(c->super))->string_length; + c->byte_count = (*(c->super))->byte_count; + c->byte_offset = (*(c->super))->byte_offset; + c->get_native_type = (*(c->super))->get_native_type; + c->sub_section = (*(c->super))->sub_section; + c->pack_missing = (*(c->super))->pack_missing; + c->is_missing = (*(c->super))->is_missing; + c->pack_long = (*(c->super))->pack_long; + c->pack_double = (*(c->super))->pack_double; + c->unpack_double = (*(c->super))->unpack_double; + c->pack_string = (*(c->super))->pack_string; + c->unpack_string = (*(c->super))->unpack_string; + c->pack_string_array = (*(c->super))->pack_string_array; + c->pack_bytes = (*(c->super))->pack_bytes; + c->unpack_bytes = (*(c->super))->unpack_bytes; + c->pack_expression = (*(c->super))->pack_expression; + c->notify_change = (*(c->super))->notify_change; + c->update_size = (*(c->super))->update_size; + c->preferred_size = (*(c->super))->preferred_size; + c->resize = (*(c->super))->resize; + c->nearest_smaller_value = (*(c->super))->nearest_smaller_value; + c->next = (*(c->super))->next; + c->compare = (*(c->super))->compare; + c->unpack_double_element = (*(c->super))->unpack_double_element; + c->unpack_double_subarray = (*(c->super))->unpack_double_subarray; + c->clear = (*(c->super))->clear; +} + +/* END_CLASS_IMP */ + +static void init(grib_accessor* a, const long len , grib_arguments* args ) +{ + grib_accessor_bufrdc_expanded_descriptors* self = (grib_accessor_bufrdc_expanded_descriptors*)a; + int n=0; + self->expandedDescriptors=grib_arguments_get_name(a->parent->h,args,n++); + self->expandedDescriptorsAccessor=0; + a->length = 0; + a->flags |= GRIB_ACCESSOR_FLAG_READ_ONLY; +} + +static void dump(grib_accessor* a, grib_dumper* dumper) +{ + grib_dump_string(dumper,a,NULL); +} + +static grib_accessor* get_accessor(grib_accessor* a) { + grib_accessor_bufrdc_expanded_descriptors* self = (grib_accessor_bufrdc_expanded_descriptors*)a; + if (!self->expandedDescriptorsAccessor) { + self->expandedDescriptorsAccessor=grib_find_accessor(a->parent->h,self->expandedDescriptors); + } + return self->expandedDescriptorsAccessor; +} + +static int unpack_long (grib_accessor* a, long* val, size_t *len) +{ + grib_accessor* descriptors=0; + size_t rlen=0,l; + long lenall=0; + size_t i; + long* v=0; + grib_context* c=a->parent->h->context; + + descriptors=get_accessor(a); + if (!descriptors) return GRIB_NOT_FOUND; + + grib_value_count(a,&lenall); + v=grib_context_malloc_clear(c,sizeof(long)*lenall); + l=lenall; + grib_unpack_long(descriptors,v,&l); + + rlen=0; + for (i=0;i 221999 ) + val[rlen++]=v[i]; + } + *len=rlen; + + return GRIB_SUCCESS; +} + +static int unpack_string_array (grib_accessor* a, char** buffer, size_t *len) +{ + grib_accessor* descriptors=0; + grib_context* c=a->parent->h->context; + long l=0; + size_t size,i; + char buf[25]={0,}; + long* v=0; + + descriptors=get_accessor(a); + if (!descriptors) return GRIB_NOT_FOUND; + + value_count(descriptors,&l); + if (l>*len) return GRIB_ARRAY_TOO_SMALL; + + v=grib_context_malloc_clear(c,sizeof(long)*l); + size=l; + unpack_long(a,v,&size); + + for (i=0;i