core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GRIB-262: Conversion from GRIB2 to GRIB1 works without error for L137 data

This commit is contained in:
Shahram Najm 2013-06-12 10:25:48 +01:00
parent 5ff5f1b28b
commit a76ddaf47f
6 changed files with 239 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/grib_accessor_class_g1_message_length.c
View File

@ -166,8 +166,6 @@ int grib_get_g1_message_size(grib_handle* h,grib_accessor* tl,grib_accessor* s4,
slen = tlen - s4->offset - 4; /* 4 is for 7777 */;
/*printf("DECODING large grib total=%ld section4=%ld\n",tlen,slen);*/
}
@ -177,11 +175,10 @@ int grib_get_g1_message_size(grib_handle* h,grib_accessor* tl,grib_accessor* s4,
return GRIB_SUCCESS;
}
static int pack_long(grib_accessor* a, const long* val,size_t *len)
{
grib_accessor_g1_message_length *self = (grib_accessor_g1_message_length*)a;
grib_accessor_class* super = *(a->cclass->super);
/*grib_accessor_class* super = *(a->cclass->super);*/
/* Here we assume that the totalLength will be coded AFTER the section4 length, and
the section4 length will be overwritten by the totalLength accessor for large GRIBs */
@ -191,36 +188,39 @@ static int pack_long(grib_accessor* a, const long* val,size_t *len)
long t120;
int ret;
tlen = *val;
if((tlen < 0x800000 || !a->parent->h->context->gribex_mode_on) && tlen < 0xFFFFFF )
{
/* printf("ENCODING small grib total = %ld\n",tlen); */
return super->pack_long(a,val,len);
/*return super->pack_long(a,val,len);*/
/* Do not directly call pack_long on base class */
/* because in this special case we want to skip the checks. */
/* So we call the helper function which has an extra argument */
return pack_long_unsigned_helper(a,val,len, /*check=*/0);
}
if(!s4) return GRIB_NOT_FOUND;
/* special case for large GRIBs */
tlen -= 4;
t120 = (tlen+119)/120;
slen = t120*120 - tlen;
tlen = 0x800000 | t120;
/* printf("ENCODING large grib total = %ld tlen=%ld slen=%ld \n",*val,tlen,slen); */
*len = 1;
if((ret = grib_pack_long(s4,&slen,len)) != GRIB_SUCCESS)
return ret;
*len = 1;
/* Do not do the length checks in this special case */
if((ret = pack_long_unsigned_helper(a,&tlen,len,/*check=*/0)) != GRIB_SUCCESS)
return ret;
/*
if((ret = super->pack_long(a,&tlen,len)) != GRIB_SUCCESS)
return ret;
*/
{
long total_length = -1, sec4_length = -1;
@ -231,7 +231,6 @@ static int pack_long(grib_accessor* a, const long* val,size_t *len)
&sec4_length);
Assert(total_length == *val);
}
return GRIB_SUCCESS;

26
src/grib_accessor_class_g1_section4_length.c
View File

@ -136,6 +136,7 @@ static void init(grib_accessor* a, const long len , grib_arguments* args )
static int pack_long(grib_accessor* a, const long* val,size_t *len)
{
#if 0
grib_accessor_class* super = *(a->cclass->super);
/* Here we assume that the totalLength will be coded AFTER the section4 length, and
@ -144,24 +145,29 @@ static int pack_long(grib_accessor* a, const long* val,size_t *len)
/*printf("UPDATING sec4len %ld\n",*val);*/
return super->pack_long(a,val,len);
#endif
/* Do not directly call pack_long on base class */
/* because in this special case we want to skip the checks. */
/* So we call the helper function which has an extra argument */
return pack_long_unsigned_helper(a,val,len, /*check=*/0);
}
static int unpack_long(grib_accessor* a, long* val,size_t *len)
{
grib_accessor_g1_section4_length *self = (grib_accessor_g1_section4_length*)a;
int ret;
long total_length, sec4_length;
if((ret = grib_get_g1_message_size(a->parent->h,
grib_find_accessor(a->parent->h,self->total_length),
a,
&total_length,
&sec4_length)) != GRIB_SUCCESS)
return ret;
grib_find_accessor(a->parent->h,self->total_length),
a,
&total_length,
&sec4_length)) != GRIB_SUCCESS)
return ret;
*val = sec4_length;
return GRIB_SUCCESS;
}

221
src/grib_accessor_class_unsigned.c
View File

@ -132,8 +132,6 @@ static void init_class(grib_accessor_class* c)
/* END_CLASS_IMP */
static void init(grib_accessor* a, const long len , grib_arguments* arg )
{
grib_accessor_unsigned* self = (grib_accessor_unsigned*)a;
@ -143,7 +141,7 @@ static void init(grib_accessor* a, const long len , grib_arguments* arg )
if (a->flags & GRIB_ACCESSOR_FLAG_TRANSIENT) {
a->length=0;
if (!a->vvalue)
if (!a->vvalue)
a->vvalue=grib_context_malloc_clear(a->parent->h->context,sizeof(grib_virtual_value));
a->vvalue->type=GRIB_TYPE_LONG;
a->vvalue->length=len;
@ -171,7 +169,90 @@ static unsigned long ones[] = {
0xffffffff,
};
static int unpack_long (grib_accessor* a, long* val, size_t *len)
int pack_long_unsigned_helper(grib_accessor* a, const long* val, size_t *len, int check)
{
grib_accessor_unsigned* self = (grib_accessor_unsigned*)a;
int ret = 0;
long off = 0;
unsigned long rlen = grib_value_count(a);
size_t buflen = 0;
unsigned char *buf = NULL;
unsigned long i = 0;
unsigned long missing = 0;
if(a->flags & GRIB_ACCESSOR_FLAG_CAN_BE_MISSING)
{
Assert(self->nbytes <= 4);
missing = ones[self->nbytes];
}
if (a->flags & GRIB_ACCESSOR_FLAG_TRANSIENT) {
a->vvalue->lval=val[0];
if(missing && val[0] == GRIB_MISSING_LONG)
a->vvalue->missing=1;
else
a->vvalue->missing=0;
return GRIB_SUCCESS;
}
if(*len < 1)
{
grib_context_log(a->parent->h->context, GRIB_LOG_ERROR, "Wrong size for %s it contains %d values ", a->name , 1 );
len[0] = 0;
return GRIB_ARRAY_TOO_SMALL;
}
if (rlen == 1) {
long v = val[0];
if (missing)
if(v == GRIB_MISSING_LONG)
v = missing;
/* Check if value fits into number of bits */
if (check) {
const long nbits = self->nbytes*8;
if ( nbits < 32 && v != GRIB_MISSING_LONG ) {
unsigned long maxval = (1 << nbits)-1;
if (v > maxval) {
grib_context_log(a->parent->h->context, GRIB_LOG_ERROR,
"Key \"%s\": Trying to encode value of %ld but the maximum allowable value is %ld (number of bits=%ld)\n",
a->name, v, maxval, nbits);
return GRIB_ENCODING_ERROR;
}
}
}
off = a->offset*8;
ret = grib_encode_unsigned_long(a->parent->h->buffer->data, v, &off, self->nbytes*8);
if (ret == GRIB_SUCCESS) len[0] = 1;
if (*len > 1) grib_context_log(a->parent->h->context, GRIB_LOG_WARNING, "grib_accessor_unsigned : Trying to pack %d values in a scalar %s, packing first value", *len, a->name );
len[0] = 1;
return ret;
}
/* TODO: We assume that there are no missing values if there are more that 1 value */
buflen = *len*self->nbytes;
buf = grib_context_malloc(a->parent->h->context,buflen);
for(i=0; i < *len;i++)
grib_encode_unsigned_long(buf, val[i] , &off, self->nbytes*8);
ret = grib_set_long_internal(a->parent->h,grib_arguments_get_name(a->parent->h,self->arg,0),*len);
if(ret == GRIB_SUCCESS)
grib_buffer_replace(a, buf, buflen,1,1);
else
*len = 0;
grib_context_free(a->parent->h->context,buf);
return ret;
}
static int unpack_long(grib_accessor* a, long* val, size_t *len)
{
grib_accessor_unsigned* self = (grib_accessor_unsigned*)a;
unsigned long rlen = grib_value_count(a);
@ -209,132 +290,62 @@ static int unpack_long (grib_accessor* a, long* val, size_t *len)
return GRIB_SUCCESS;
}
static int pack_long (grib_accessor* a, const long* val, size_t *len)
static int pack_long(grib_accessor* a, const long* val, size_t *len)
{
grib_accessor_unsigned* self = (grib_accessor_unsigned*)a;
int ret = 0;
long off = 0;
unsigned long rlen = grib_value_count(a);
size_t buflen = 0;
unsigned char *buf = NULL;
unsigned long i = 0;
unsigned long missing = 0;
if(a->flags & GRIB_ACCESSOR_FLAG_CAN_BE_MISSING)
{
Assert(self->nbytes <= 4);
missing = ones[self->nbytes];
}
if (a->flags & GRIB_ACCESSOR_FLAG_TRANSIENT) {
a->vvalue->lval=val[0];
if(missing && val[0] == GRIB_MISSING_LONG)
a->vvalue->missing=1;
else
a->vvalue->missing=0;
return GRIB_SUCCESS;
}
if(*len < 1)
{
grib_context_log(a->parent->h->context, GRIB_LOG_ERROR, "Wrong size for %s it contains %d values ", a->name , 1 );
len[0] = 0;
return GRIB_ARRAY_TOO_SMALL;
}
if (rlen == 1){
long v = val[0];
#if 1
if(missing)
if(v == GRIB_MISSING_LONG)
v = missing;
#endif
off = a->offset*8;
ret = grib_encode_unsigned_long(a->parent->h->buffer->data, v , &off, self->nbytes*8);
if (ret == GRIB_SUCCESS) len[0] = 1;
if (*len > 1) grib_context_log(a->parent->h->context, GRIB_LOG_WARNING, "grib_accessor_unsigned : Trying to pack %d values in a scalar %s, packing first value", *len, a->name );
len[0] = 1;
return ret;
}
/* TODO: We assume that there are no missing values if there are more that 1 value */
buflen = *len*self->nbytes;
buf = grib_context_malloc(a->parent->h->context,buflen);
for(i=0; i < *len;i++)
grib_encode_unsigned_long(buf, val[i] , &off, self->nbytes*8);
ret = grib_set_long_internal(a->parent->h,grib_arguments_get_name(a->parent->h,self->arg,0),*len);
if(ret == GRIB_SUCCESS)
grib_buffer_replace(a, buf, buflen,1,1);
else
*len = 0;
grib_context_free(a->parent->h->context,buf);
return ret;
/* See GRIB-262 as example of why we do the checks */
return pack_long_unsigned_helper(a,val,len, /*check=*/1);
}
static long byte_count(grib_accessor* a){
return a->length;
return a->length;
}
static long value_count(grib_accessor* a)
{
grib_accessor_unsigned* self = (grib_accessor_unsigned*)a;
long len = 0;
int ret =0;
if(!self->arg) return 1;
ret = grib_get_long_internal(a->parent->h,grib_arguments_get_name(a->parent->h,self->arg,0),&len);
if(ret == GRIB_SUCCESS) return len;
return 1;
grib_accessor_unsigned* self = (grib_accessor_unsigned*)a;
long len = 0;
int ret =0;
if(!self->arg) return 1;
ret = grib_get_long_internal(a->parent->h,grib_arguments_get_name(a->parent->h,self->arg,0),&len);
if(ret == GRIB_SUCCESS) return len;
return 1;
}
static long byte_offset(grib_accessor* a){
return a->offset;
return a->offset;
}
static void update_size(grib_accessor* a,size_t s)
{
a->length = s;
a->length = s;
}
static long next_offset(grib_accessor* a){
return grib_byte_offset(a)+grib_byte_count(a);
return grib_byte_offset(a)+grib_byte_count(a);
}
static int is_missing(grib_accessor* a){
int i=0;
unsigned char ff=0xff;
unsigned long offset=a->offset;
int i=0;
unsigned char ff=0xff;
unsigned long offset=a->offset;
if (a->length==0) {
Assert(a->vvalue!=NULL);
return a->vvalue->missing;
}
if (a->length==0) {
Assert(a->vvalue!=NULL);
return a->vvalue->missing;
}
for (i=0;i<a->length;i++) {
if (a->parent->h->buffer->data[offset] != ff) {
return 0;
}
offset++;
}
return 1;
for (i=0;i<a->length;i++) {
if (a->parent->h->buffer->data[offset] != ff) {
return 0;
}
offset++;
}
return 1;
}
static void destroy(grib_context* context,grib_accessor* a)
{
if (a->vvalue != NULL)
grib_context_free(context, a->vvalue);
if (a->vvalue != NULL)
grib_context_free(context, a->vvalue);
a->vvalue=NULL;
a->vvalue=NULL;
}

1
src/grib_api_prototypes.h
View File

@ -535,6 +535,7 @@ int grib_index_search(grib_index *index, grib_index_key *keys);
/* grib_accessor_class_statistics_spectral.c */
/* grib_accessor_class_unsigned.c */
int pack_long_unsigned_helper(grib_accessor* a, const long* val, size_t *len, int check);
/* grib_accessor_class_unsigned_bits.c */

185
src/grib_dependency.c
View File

@ -17,146 +17,139 @@
static grib_handle* handle_of(grib_accessor* observed)
{
grib_handle *h = observed->parent->h;
while(h->main) h = h->main;
return h;
grib_handle *h = observed->parent->h;
while(h->main) h = h->main;
return h;
}
void grib_dependency_add(grib_accessor* observer,grib_accessor* observed)
{
grib_handle *h = handle_of(observed);
grib_dependency *d = h->dependencies;
grib_dependency *last = 0;
grib_handle *h = handle_of(observed);
grib_dependency *d = h->dependencies;
grib_dependency *last = 0;
/*
printf("observe %p %p %s %s\n",(void*)observed,(void*)observer, observed?observed->name:"NULL",
observer?observer->name:"NULL");
*/
/*printf("observe %p %p %s %s\n",(void*)observed,(void*)observer, observed?observed->name:"NULL",
observer?observer->name:"NULL");*/
if(!observer || !observed)
{
return;
}
if(!observer || !observed)
{
return;
}
/* Assert(h == handle_of(observer)); */
/* Assert(h == handle_of(observer)); */
/* Check if already in list */
while(d)
{
if(d->observer == observer && d->observed == observed)
return;
last = d;
d = d->next;
}
/* Check if already in list */
while(d)
{
if(d->observer == observer && d->observed == observed)
return;
last = d;
d = d->next;
}
#if 0
d = h->dependencies;
while(d)
{
last = d;
d = d->next;
}
d = h->dependencies;
while(d)
{
last = d;
d = d->next;
}
#endif
d = (grib_dependency*)grib_context_malloc_clear(h->context,sizeof(grib_dependency));
Assert(d);
d = (grib_dependency*)grib_context_malloc_clear(h->context,sizeof(grib_dependency));
Assert(d);
d->observed = observed;
d->observer = observer;
d->next = 0;
d->observed = observed;
d->observer = observer;
d->next = 0;
/*
printf("observe %p %p %s %s\n",(void*)observed,(void*)observer, observed->name,observer->name);
*/
/*printf("observe %p %p %s %s\n",(void*)observed,(void*)observer, observed->name,observer->name);*/
#if 0
d->next = h->dependencies;
h->dependencies = d;
d->next = h->dependencies;
h->dependencies = d;
#endif
if(last)
last->next = d;
else
h->dependencies = d;
if(last)
last->next = d;
else
h->dependencies = d;
}
void grib_dependency_remove_observed(grib_accessor* observed)
{
grib_handle *h = handle_of(observed);
grib_dependency *d = h->dependencies;
/* printf("%s\n",observed->name); */
grib_handle *h = handle_of(observed);
grib_dependency *d = h->dependencies;
/* printf("%s\n",observed->name); */
while(d)
{
if(d->observed == observed)
{
/* TODO: Notify observer...*/
d->observed = 0;/*
printf("grib_dependency_remove_observed %s\n",observed->name); */
}
d = d->next;
}
while(d)
{
if(d->observed == observed)
{
/* TODO: Notify observer...*/
d->observed = 0;/*printf("grib_dependency_remove_observed %s\n",observed->name); */
}
d = d->next;
}
}
/* TODO: Notification must go from outer blocks to inner block */
int grib_dependency_notify_change(grib_accessor* observed)
{
grib_handle *h = handle_of(observed);
grib_dependency *d = h->dependencies;
int ret = GRIB_SUCCESS;
grib_handle *h = handle_of(observed);
grib_dependency *d = h->dependencies;
int ret = GRIB_SUCCESS;
/* Do a two pass mark&sweep, in case some depedancies
are added while we notify */
/*Do a two pass mark&sweep, in case some dependencies are added while we notify*/
while(d)
{
d->run = (d->observed == observed && d->observer != 0);
d = d->next;
}
while(d)
{
d->run = (d->observed == observed && d->observer != 0);
d = d->next;
}
d = h->dependencies;
while(d)
{
if(d->run)
{
/*printf("grib_dependency_notify_change %s %s %p\n",observed->name,d->observer ? d->observer->name : "?", (void*)d->observer);*/
if( d->observer && (ret = grib_accessor_notify_change(d->observer,observed))
!= GRIB_SUCCESS) return ret;
}
d = d->next;
}
return ret;
d = h->dependencies;
while(d)
{
if(d->run)
{
/*printf("grib_dependency_notify_change %s %s %p\n",observed->name,d->observer ? d->observer->name : "?", (void*)d->observer);*/
if( d->observer && (ret = grib_accessor_notify_change(d->observer,observed))
!= GRIB_SUCCESS) return ret;
}
d = d->next;
}
return ret;
}
void grib_dependency_remove_observer(grib_accessor* observer)
{
grib_handle *h = handle_of(observer);
grib_dependency *d = h->dependencies;
grib_handle *h = handle_of(observer);
grib_dependency *d = h->dependencies;
if (!observer) return;
while(d)
{
if(d->observer == observer)
{
d->observer = 0;
if (!observer) return;
while(d)
{
if(d->observer == observer)
{
d->observer = 0;
}
d = d->next;
}
d = d->next;
}
}
void grib_dependency_observe_expression(grib_accessor* observer,grib_expression* e)
{
grib_expression_add_dependency(e,observer);
grib_expression_add_dependency(e,observer);
}
void grib_dependency_observe_arguments(grib_accessor* observer,grib_arguments* a)
{
while(a)
{
grib_dependency_observe_expression(observer,a->expression);
a = a->next;
}
while(a)
{
grib_dependency_observe_expression(observer,a->expression);
a = a->next;
}
}

9
src/grib_value.c
View File

@ -62,7 +62,7 @@ int grib_set_long_internal(grib_handle* h, const char* name, long val) {
a = grib_find_accessor(h, name);
if (h->context->debug==-1)
printf("GRIB_API DEBUG grib_set_long %s=%ld\n",name,(long)val);
printf("GRIB_API DEBUG grib_set_long_internal %s=%ld\n",name,(long)val);
if(a){
ret = grib_pack_long(a, &val, &l);
@ -110,7 +110,7 @@ int grib_set_double_internal(grib_handle* h, const char* name, double val) {
a = grib_find_accessor(h, name);
if (h->context->debug==-1)
printf("GRIB_API DEBUG grib_set_double %s=%g\n",name,val);
printf("GRIB_API DEBUG grib_set_double_internal %s=%g\n",name,val);
if(a){
ret = grib_pack_double(a, &val, &l);
@ -325,7 +325,7 @@ int grib_set_string_internal(grib_handle* h, const char* name,
a = grib_find_accessor(h, name);
if (h->context->debug==-1)
printf("GRIB_API DEBUG grib_set_string %s=%s\n",name,val);
printf("GRIB_API DEBUG grib_set_string_internal %s=%s\n",name,val);
if(a){
ret = grib_pack_string(a, val, length);
@ -576,7 +576,7 @@ int grib_set_double_array_internal(grib_handle* h, const char* name, const doubl
int ret=0;
if (h->context->debug==-1)
printf("GRIB_API DEBUG grib_set_double_array %ld values\n",(long)length);
printf("GRIB_API DEBUG grib_set_double_array_internal %ld values\n",(long)length);
if (length==0) {
grib_accessor* a = grib_find_accessor(h, name);
@ -1369,3 +1369,4 @@ int grib_values_check(grib_handle* h, grib_values* values, int count) {
return 0;
}