core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECC-128 part 1

This commit is contained in:
Enrico Fucile 2015-06-03 14:52:29 +01:00
parent 8ba4d03232
commit 5b1a787f63
12 changed files with 270 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
definitions/bufr/section.4.def
View File

@ -27,6 +27,11 @@ if (compressedData) {
meta unpack unpack_bufr_values(numericValues) ;
when (changed(unexpandedDescriptors)) {
# resize data section and unpack
set unpack=3;
}
section_padding section4Padding;
position offsetEndSection4;
meta md5Data md5(offsetSection4,section4Length);

1
fortran/grib_api_constants.h
View File

@ -1,3 +1,4 @@
integer, parameter,public :: GRIB_OUT_OF_RANGE = -65
integer, parameter,public :: GRIB_UNSUPPORTED_EDITION = -64
integer, parameter,public :: GRIB_ATTRIBUTE_NOT_FOUND = -63
integer, parameter,public :: GRIB_TOO_MANY_ATTRIBUTES = -62

2
python/grib_errors.h
View File

@ -135,5 +135,7 @@ Error codes returned by the grib_api functions.
#define GRIB_ATTRIBUTE_NOT_FOUND -63
/** Edition not supported. */
#define GRIB_UNSUPPORTED_EDITION -64
/** Value out of coding range */
#define GRIB_OUT_OF_RANGE -65
/*! @}*/
#endif

227
src/grib_accessor_class_bufr_data_array.c
View File

@ -195,15 +195,17 @@ static void init_class(grib_accessor_class* c)
/* END_CLASS_IMP */
typedef int (*codec_element_proc) (grib_context* c,grib_accessor_bufr_data_array* self,int subsetIndex, grib_buffer* b,unsigned char* data,long *pos,int i,grib_darray* dval,grib_sarray* sval);
static int can_be_missing(int descriptor) {
int ret=1;
if (descriptor==31031) ret=0;
return ret;
}
static void restart_bitmap(grib_accessor_bufr_data_array *self) {
static void restart_bitmap(grib_accessor_bufr_data_array *self) {
self->bitmapCurrent=-1;
self->bitmapCurrentElementsDescriptorsIndex=self->bitmapStartElementsDescriptorsIndex-1;
self->bitmapCurrentElementsDescriptorsIndex=self->bitmapStartElementsDescriptorsIndex-1;
}
static void cancel_bitmap(grib_accessor_bufr_data_array *self) { self->bitmapCurrent=-1;self->bitmapStart=-1; }
@ -443,6 +445,139 @@ static grib_darray* decode_double_array(grib_context* c,unsigned char* data,long
return ret;
}
static int encode_string_array(grib_context* c,grib_buffer* buff,long* pos, int i,
grib_accessor_bufr_data_array* self,grib_sarray* stringValues) {
char* sval=0;
int err=0;
int j,modifiedWidth,modifiedReference,width;
double modifiedFactor;
modifiedWidth= self->expanded->v[i]->width;
modifiedReference= self->expanded->v[i]->reference;
modifiedFactor= self->expanded->v[i]->factor;
grib_buffer_set_ulength(c,buff,modifiedWidth);
grib_encode_string(buff->data,pos,modifiedWidth/8,stringValues->v[0]);
width= stringValues->n > 1 ? modifiedWidth : 0;
grib_buffer_set_ulength(c,buff,6);
grib_encode_unsigned_long(buff->data,width/8,pos,6);
if (width) {
grib_buffer_set_ulength(c,buff,6);
grib_buffer_set_ulength(c,buff,width*stringValues->n);
for (j=0;j<stringValues->n;j++) {
grib_encode_string(buff->data,pos,width,stringValues->v[j]);
}
}
return err;
}
static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos,int i,
grib_accessor_bufr_data_array* self,grib_darray* dvalues) {
int err=0;
int j;
unsigned long lval;
int localReference,localWidth,modifiedWidth,modifiedReference,reference;
int localRange;
double modifiedFactor,dval;
size_t nvals,ii;
double min,max,maxAllowed;
double* v=NULL;
modifiedReference= self->expanded->v[i]->reference;
modifiedFactor= self->expanded->v[i]->factor;
modifiedWidth= self->expanded->v[i]->width;
maxAllowed=(grib_power(modifiedWidth,2)+modifiedReference)*modifiedFactor;
nvals=grib_darray_used_size(dvalues);
v=dvalues->v;
ii=0;
do {
min=*v;
max=*v;
v++;
ii++;
} while (*v==GRIB_MISSING_DOUBLE && ii<nvals);
while (ii<nvals) {
if (*v<min && *v!=GRIB_MISSING_DOUBLE) min=*v;
if (*v>max && *v!=GRIB_MISSING_DOUBLE) max=*v;
ii++;
v++;
}
if (max>maxAllowed) return GRIB_OUT_OF_RANGE;
if (min<0) return GRIB_OUT_OF_RANGE;
reference=min/modifiedFactor;
localReference=reference-modifiedReference;
if (max!=min) {
localRange = (max/modifiedFactor-localReference-modifiedReference);
localWidth=(long)ceil(log((double)localRange)/log(2.0));
} else {
localWidth=0;
}
grib_buffer_set_ulength(c,buff,modifiedWidth);
grib_encode_unsigned_long(buff->data,localReference,pos,modifiedWidth);
grib_buffer_set_ulength(c,buff,6);
grib_encode_unsigned_long(buff->data,localWidth,pos,6);
if (localWidth) {
grib_buffer_set_ulength(c,buff,self->numberOfSubsets*localWidth);
for (j=0;j<self->numberOfSubsets;j++) {
if (dvalues->v[j]==GRIB_MISSING_DOUBLE) {
grib_set_bits_on(buff->data,pos,localWidth);
} else {
lval=(long)((dvalues->v[j]-min)/modifiedFactor);
grib_encode_unsigned_long(buff->data,lval,pos,localWidth);
}
}
}
return err;
}
static int encode_double_value(grib_context* c,grib_buffer* buff,long* pos,int i,
grib_accessor_bufr_data_array* self,double value) {
unsigned long lval;
double maxAllowed;
int err=0;
int modifiedWidth,modifiedReference;
double modifiedFactor;
double dval=0;
modifiedReference= self->expanded->v[i]->reference;
modifiedFactor= self->expanded->v[i]->factor;
modifiedWidth= self->expanded->v[i]->width;
grib_buffer_set_ulength(c,buff,modifiedWidth);
if (value==GRIB_MISSING_DOUBLE) {
grib_set_bits_on(buff->data,pos,modifiedWidth);
} else {
maxAllowed=(grib_power(modifiedWidth,2)+modifiedReference)*modifiedFactor;
if (value>maxAllowed) return GRIB_OUT_OF_RANGE;
lval=(long)(value/modifiedFactor-modifiedReference);
grib_encode_unsigned_long(buff->data,lval,pos,modifiedWidth);
}
return err;
}
static int encode_string_value(grib_context* c,grib_buffer* buff,long* pos, int i,
grib_accessor_bufr_data_array* self,char* sval) {
int err=0;
int len;
len= self->expanded->v[i]->width / 8;
grib_buffer_set_ulength(c,buff,self->expanded->v[i]->width);
grib_encode_string(buff->data,pos,len,sval);
return err;
}
static char* decode_string_value(grib_context* c,unsigned char* data,long* pos, int i,
grib_accessor_bufr_data_array* self,int *err) {
char* sval=0;
@ -487,7 +622,7 @@ static double decode_double_value(grib_context* c,unsigned char* data,long* pos,
}
static int decode_element(grib_context* c,grib_accessor_bufr_data_array* self,int subsetIndex,
unsigned char* data,long *pos,int i,grib_darray* dval,grib_sarray* sval) {
grib_buffer* b,unsigned char* data,long *pos,int i,grib_darray* dval,grib_sarray* sval) {
grib_darray* dar=0;
grib_sarray* sar=0;
int index=0,ii;
@ -529,6 +664,45 @@ static int decode_element(grib_context* c,grib_accessor_bufr_data_array* self,in
return err;
}
static int encode_new_element(grib_context* c,grib_accessor_bufr_data_array* self,int subsetIndex,
grib_buffer* buff,unsigned char* data,long *pos,int i,grib_darray* dval,grib_sarray* sval) {
int index=0,ii;
char* csval=0;
unsigned char missingChar=0xFF;
double cdval=GRIB_MISSING_DOUBLE,x;
int err=0;
size_t slen;
grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data encoding: \tcode=%6.6ld width=%ld",
self->expanded->v[i]->code,self->expanded->v[i]->width);
if (self->expanded->v[i]->type==BUFR_DESCRIPTOR_TYPE_STRING) {
/* string */
slen=self->expanded->v[i]->width/8;
csval=grib_context_malloc_clear(c,slen);
for (ii=0;ii<slen;ii++) csval[ii]=missingChar;
if (self->compressedData) {
grib_sarray* stringValues=grib_sarray_new(c,1,1);
grib_sarray_push(c,stringValues,csval);
err=encode_string_array(c,buff,pos,i,self,stringValues);
grib_sarray_delete_content(c,stringValues);
grib_sarray_delete(c,stringValues);
} else {
err=encode_string_value(c,buff,pos,i,self,csval);
}
} else {
/* numeric or codetable or flagtable */
if (self->compressedData) {
grib_darray* doubleValues=grib_darray_new(c,1,1);
grib_darray_push(c,doubleValues,cdval);
err=encode_double_array(c,buff,pos,i,self,doubleValues);
grib_darray_delete(c,doubleValues);
} else {
err=encode_double_value(c,buff,pos,i,self,cdval);
}
}
return err;
}
static int build_bitmap(grib_accessor_bufr_data_array *self,unsigned char* data,long* pos,grib_iarray* elementsDescriptorsIndex,int iBitmapOperator) {
int bitmapSize=0,iDelayedReplication=0;
int i,localReference,width,bitmapEndElementsDescriptorsIndex;
@ -1072,7 +1246,11 @@ static int create_keys(grib_accessor* a) {
#define MAX_NESTED_REPLICATIONS 8
static int decode_elements(grib_accessor* a) {
#define PROCESS_DECODE 0
#define PROCESS_NEW_DATA 1
#define PROCESS_ENCODE 2
static int process_elements(grib_accessor* a,int flag) {
int err=0;
int associatedFieldWidth=0,localDescriptorWidth=0;
long inr,innr,ir;
@ -1092,6 +1270,8 @@ static int decode_elements(grib_accessor* a) {
long totalSize;
bufr_descriptor** descriptors=0;
long icount;
grib_buffer* buffer=NULL;
codec_element_proc codec_element;
grib_darray* dval = NULL;
grib_sarray* sval = NULL;
@ -1106,8 +1286,26 @@ static int decode_elements(grib_accessor* a) {
self->do_decode=0;
a->parent->h->unpacked=1;
data = (unsigned char*)h->buffer->data;
pos=a->offset*8;
if (flag==PROCESS_ENCODE) return GRIB_NOT_IMPLEMENTED;
switch (flag) {
case PROCESS_DECODE:
buffer=h->buffer;
pos=a->offset*8;
codec_element=&decode_element;
break;
case PROCESS_NEW_DATA:
buffer=grib_create_growable_buffer(c);
pos=0;
codec_element=&encode_new_element;
break;
case PROCESS_ENCODE:
return GRIB_NOT_IMPLEMENTED;
break;
default :
return GRIB_NOT_IMPLEMENTED;
}
data=(unsigned char*)buffer->data;
err=get_descriptors(a);
if (err) return err;
@ -1147,7 +1345,7 @@ static int decode_elements(grib_accessor* a) {
grib_iarray_push(elementsDescriptorsIndex,i);
if (descriptors[i]->code==31031 && !is_bitmap_start_defined(self))
self->bitmapStart=grib_iarray_used_size(elementsDescriptorsIndex)-1;
err=decode_element(c,self,iss,data,&pos,i,dval,sval);
err=codec_element(c,self,iss,buffer,data,&pos,i,dval,sval);
if (err) return err;
break;
case 1:
@ -1204,7 +1402,7 @@ static int decode_elements(grib_accessor* a) {
descriptors[i]->width=descriptors[i]->Y*8;
descriptors[i]->type=BUFR_DESCRIPTOR_TYPE_STRING;
grib_iarray_push(elementsDescriptorsIndex,i);
err=decode_element(c,self,iss,data,&pos,i,dval,sval);
err=codec_element(c,self,iss,buffer,data,&pos,i,dval,sval);
if (err) return err;
break;
case 22:
@ -1231,7 +1429,7 @@ static int decode_elements(grib_accessor* a) {
/*replaced/retained values marker operator*/
if (descriptors[i]->Y==255) {
index=get_next_bitmap_descriptor_index(self,elementsDescriptorsIndex,dval);
err=decode_element(c,self,iss,data,&pos,index,dval,sval);
err=codec_element(c,self,iss,buffer,data,&pos,index,dval,sval);
if (err) return err;
/* self->expanded->v[index] */
grib_iarray_push(elementsDescriptorsIndex,i);
@ -1272,7 +1470,7 @@ static int decode_elements(grib_accessor* a) {
/* associated field */
if (descriptors[i]->X==99 && descriptors[i]->Y==999) {
grib_iarray_push(elementsDescriptorsIndex,i);
err=decode_element(c,self,iss,data,&pos,i,dval,sval);
err=codec_element(c,self,iss,buffer,data,&pos,i,dval,sval);
if (err) return err;
} else {
return GRIB_INTERNAL_ERROR;
@ -1328,7 +1526,7 @@ static int decode_elements(grib_accessor* a) {
static void dump(grib_accessor* a, grib_dumper* dumper)
{
int err=decode_elements(a);
int err=process_elements(a,PROCESS_DECODE);
grib_dump_section(dumper,a,a->sub_section->block);
@ -1342,7 +1540,7 @@ static int value_count(grib_accessor* a,long* count)
grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a;
grib_context* c=a->parent->h->context;
err=decode_elements(a);
err=process_elements(a,PROCESS_DECODE);
if (err) return err;
err=grib_get_long(a->parent->h,self->subsetNumberName,&subsetNumber);
@ -1376,12 +1574,15 @@ static int value_count(grib_accessor* a,long* count)
static int unpack_double(grib_accessor* a, double* val, size_t *len) {
int err=0,i,k,ii;
long n=0;
int proc_flag=PROCESS_DECODE;
size_t l=0,elementsInSubset;
long numberOfSubsets=0;
grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a;
grib_context* c=a->parent->h->context;
err=decode_elements(a);
if (self->unpackMode == CODES_BUFR_NEW_DATA) proc_flag=PROCESS_NEW_DATA;
err=process_elements(a,proc_flag);
if (err) return err;
if (!val) return err;

1
src/grib_accessor_class_unpack_bufr_values.c
View File

@ -205,6 +205,7 @@ static int pack_long (grib_accessor* a, const long* val, size_t *len)
grib_accessor* data=(grib_accessor*)self->data_accessor;
if (*val==2) unpackMode=CODES_BUFR_UNPACK_FLAT;
if (*val==3) unpackMode=CODES_BUFR_NEW_DATA;
accessor_bufr_data_array_set_unpackMode(data,unpackMode);

2
src/grib_api.h
View File

@ -1583,5 +1583,7 @@ Error codes returned by the grib_api functions.
#define GRIB_ATTRIBUTE_NOT_FOUND -63
/** Edition not supported. */
#define GRIB_UNSUPPORTED_EDITION -64
/** Value out of coding range */
#define GRIB_OUT_OF_RANGE -65
/*! @}*/
#endif

3
src/grib_api_internal.h
View File

@ -196,7 +196,8 @@ extern "C" {
#define CODES_GTS 4
#define CODES_BUFR_UNPACK_STRUCTURE 0
#define CODES_BUFR_UNPACK_FLAT 1
#define CODES_BUFR_UNPACK_FLAT 1
#define CODES_BUFR_NEW_DATA 2
#define MAX_SMART_TABLE_COLUMNS 20
#define MAX_CODETABLE_ENTRIES 65536

2
src/grib_api_prototypes.h
View File

@ -769,6 +769,7 @@ int grib_init_accessor_from_handle(grib_loader *loader, grib_accessor *ga, grib_
unsigned long grib_decode_unsigned_byte_long(const unsigned char *p, long o, int l);
long grib_decode_signed_long(const unsigned char *p, long o, int l);
int grib_encode_signed_long(unsigned char *p, long val, long o, int l);
void grib_set_bits_on(unsigned char *p, long *bitp, long nbits);
int grib_get_bit(const unsigned char *p, long bitp);
void grib_set_bit(unsigned char *p, long bitp, int val);
long grib_decode_signed_longb(const unsigned char *p, long *bitp, long nbits);
@ -1393,6 +1394,7 @@ const char *grib_get_git_sha1(void);
/* grib_bits_any_endian.c */
int grib_is_all_bits_one(long val, long nbits);
int grib_encode_string(const unsigned char *bitStream, long *bitOffset, size_t numberOfCharacters, char *string);
char *grib_decode_string(const unsigned char *bitStream, long *bitOffset, size_t numberOfCharacters, char *string);
unsigned long grib_decode_unsigned_long(const unsigned char *p, long *bitp, long nbits);
int grib_encode_unsigned_long(unsigned char *p, unsigned long val, long *bitp, long nbits);

7
src/grib_bits.c
View File

@ -108,6 +108,13 @@ static void grib_set_bit_on( unsigned char* p, long* bitp){
(*bitp)++;
}
void grib_set_bits_on( unsigned char* p, long* bitp,long nbits){
int i;
for (i=0;i<nbits;i++) {
grib_set_bit_on(p,bitp);
}
}
static void grib_set_bit_off( unsigned char* p, long* bitp){
p += *bitp/8;

32
src/grib_bits_any_endian.c
View File

@ -40,6 +40,38 @@ int grib_is_all_bits_one(long val, long nbits) {
}
int grib_encode_string(const unsigned char* bitStream, long *bitOffset, size_t numberOfCharacters,char* string)
{
size_t i;
int err=0;
long byteOffset = *bitOffset / 8;
int remainder = *bitOffset % 8;
unsigned char c;
unsigned char* p;
char* s=string;
unsigned char mask[] ={ 0, 0x80, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE };
int remainderComplement=8-remainder;
if (numberOfCharacters==0) return err;
p=(unsigned char*)bitStream+byteOffset;
if ( remainder == 0 ) {
memcpy(p,string,numberOfCharacters);
*bitOffset+=numberOfCharacters*8;
return err;
}
for (i=0;i<numberOfCharacters;i++) {
c=(*s)>>remainderComplement;
*p |= c;
p++;
*p = (*s) & mask[remainder];
s++;
}
*bitOffset+=numberOfCharacters*8;
return err;
}
char* grib_decode_string(const unsigned char* bitStream, long *bitOffset, size_t numberOfCharacters,char* string)
{

1
src/grib_errors.c
View File

@ -68,6 +68,7 @@ static const char *errors[] = {
"Too many attributes. Increase MAX_ACCESSOR_ATTRIBUTES", /* -62 GRIB_TOO_MANY_ATTRIBUTES */
"Attribute not found.", /* -63 GRIB_ATTRIBUTE_NOT_FOUND */
"Edition not supported.", /* -64 GRIB_UNSUPPORTED_EDITION */
"Value out of coding range", /* -65 GRIB_OUT_OF_RANGE */
"Value mismatch", /* 1 GRIB_VALUE_MISMATCH */
"double values are different", /* 2 GRIB_DOUBLE_VALUE_MISMATCH */
"long values are different", /* 3 GRIB_LONG_VALUE_MISMATCH */

1
src/grib_errors.txt
View File

@ -70,3 +70,4 @@ GRIB_ATTRIBUTE_CLASH Attribute is already present, cannot add
GRIB_TOO_MANY_ATTRIBUTES Too many attributes. Increase MAX_ACCESSOR_ATTRIBUTES
GRIB_ATTRIBUTE_NOT_FOUND Attribute not found.
GRIB_UNSUPPORTED_EDITION Edition not supported.
GRIB_OUT_OF_RANGE Value out of coding range