core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECC-403: BUFR: codes_is_missing reports 1 on keys that are not missing

This commit is contained in:
Shahram Najm 2020-12-14 13:42:37 +00:00
parent 3a420404a0 52f75237f6
commit 9a6aa9bd7c
5 changed files with 176 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
examples/C/bufr_missing.c
View File

@ -15,7 +15,7 @@
*/
#include "eccodes.h"
#include <assert.h>
int main(int argc, char* argv[])
{
@ -26,6 +26,7 @@ int main(int argc, char* argv[])
double doubleVal;
int err = 0;
int is_missing = 0;
int cnt = 0;
const char* infile = "../../data/bufr/syno_1.bufr";
@ -51,7 +52,6 @@ int main(int argc, char* argv[])
/* the value of this key is missing in the message*/
CODES_CHECK(codes_get_double(h, "relativeHumidity", &doubleVal), 0);
printf(" relativeHumidity: %.2f\n", doubleVal);
/* we will print "value missing" */
if (doubleVal == CODES_MISSING_DOUBLE) {
@ -61,6 +61,10 @@ int main(int argc, char* argv[])
printf(" --> value present\n");
}
is_missing = codes_is_missing(h, "relativeHumidity", &err);
assert(!err);
assert(is_missing == 1);
/* delete handle */
codes_handle_delete(h);

5
examples/python/bufr_get_keys.py
View File

@ -60,6 +60,7 @@ def example():
print(' %s: %s' % (key, codes_get(bufr, key)))
except CodesInternalError as err:
print('Error with key="%s" : %s' % (key, err.msg))
assert codes_is_missing(bufr, key) == 0
# Native type float
key = 'airTemperatureAt2M'
@ -67,6 +68,7 @@ def example():
print(' %s: %s' % (key, codes_get(bufr, key)))
except CodesInternalError as err:
print('Error with key="%s" : %s' % (key, err.msg))
assert codes_is_missing(bufr, key) == 0
# Native type string
key = 'typicalDate'
@ -75,6 +77,9 @@ def example():
except CodesInternalError as err:
print('Error with key="%s" : %s' % (key, err.msg))
assert codes_is_missing(bufr, 'relativeHumidity') == 1
assert codes_is_missing(bufr, '#5#cloudAmount') == 1
# --------------------------------
# get values for an array
# --------------------------------

212
src/grib_accessor_class_bufr_data_element.c
View File

@ -22,7 +22,7 @@
SUPER = grib_accessor_class_gen
IMPLEMENTS = init;dump
IMPLEMENTS = unpack_string;unpack_string_array;unpack_long; unpack_double
IMPLEMENTS = unpack_double_element
IMPLEMENTS = unpack_double_element ; is_missing
IMPLEMENTS = pack_long; pack_double ; pack_string_array; pack_string
IMPLEMENTS = value_count; get_native_type; make_clone; destroy
MEMBERS = long index
@ -51,6 +51,7 @@ or edit "accessor.class" and rerun ./make_class.pl
*/
static int get_native_type(grib_accessor*);
static int is_missing(grib_accessor*);
static int pack_double(grib_accessor*, const double* val, size_t* len);
static int pack_long(grib_accessor*, const long* val, size_t* len);
static int pack_string(grib_accessor*, const char*, size_t* len);
@ -87,46 +88,46 @@ typedef struct grib_accessor_bufr_data_element
extern grib_accessor_class* grib_accessor_class_gen;
static grib_accessor_class _grib_accessor_class_bufr_data_element = {
&grib_accessor_class_gen, /* super */
"bufr_data_element", /* name */
sizeof(grib_accessor_bufr_data_element), /* size */
0, /* inited */
&init_class, /* init_class */
&init, /* init */
0, /* post_init */
&destroy, /* free mem */
&dump, /* describes himself */
0, /* get length of section */
0, /* get length of string */
&value_count, /* get number of values */
0, /* get number of bytes */
0, /* get offset to bytes */
&get_native_type, /* get native type */
0, /* get sub_section */
0, /* grib_pack procedures long */
0, /* grib_pack procedures long */
&pack_long, /* grib_pack procedures long */
&unpack_long, /* grib_unpack procedures long */
&pack_double, /* grib_pack procedures double */
&unpack_double, /* grib_unpack procedures double */
&pack_string, /* grib_pack procedures string */
&unpack_string, /* grib_unpack procedures string */
&pack_string_array, /* grib_pack array procedures string */
&unpack_string_array, /* grib_unpack array procedures string */
0, /* grib_pack procedures bytes */
0, /* grib_unpack procedures bytes */
0, /* pack_expression */
0, /* notify_change */
0, /* update_size */
0, /* preferred_size */
0, /* resize */
0, /* nearest_smaller_value */
0, /* next accessor */
0, /* compare vs. another accessor */
&unpack_double_element, /* unpack only ith value */
0, /* unpack a subarray */
0, /* clear */
&make_clone, /* clone accessor */
&grib_accessor_class_gen, /* super */
"bufr_data_element", /* name */
sizeof(grib_accessor_bufr_data_element), /* size */
0, /* inited */
&init_class, /* init_class */
&init, /* init */
0, /* post_init */
&destroy, /* free mem */
&dump, /* describes himself */
0, /* get length of section */
0, /* get length of string */
&value_count, /* get number of values */
0, /* get number of bytes */
0, /* get offset to bytes */
&get_native_type, /* get native type */
0, /* get sub_section */
0, /* grib_pack procedures long */
&is_missing, /* grib_pack procedures long */
&pack_long, /* grib_pack procedures long */
&unpack_long, /* grib_unpack procedures long */
&pack_double, /* grib_pack procedures double */
&unpack_double, /* grib_unpack procedures double */
&pack_string, /* grib_pack procedures string */
&unpack_string, /* grib_unpack procedures string */
&pack_string_array, /* grib_pack array procedures string */
&unpack_string_array, /* grib_unpack array procedures string */
0, /* grib_pack procedures bytes */
0, /* grib_unpack procedures bytes */
0, /* pack_expression */
0, /* notify_change */
0, /* update_size */
0, /* preferred_size */
0, /* resize */
0, /* nearest_smaller_value */
0, /* next accessor */
0, /* compare vs. another accessor */
&unpack_double_element, /* unpack only ith value */
0, /* unpack a subarray */
0, /* clear */
&make_clone, /* clone accessor */
};
@ -135,25 +136,24 @@ grib_accessor_class* grib_accessor_class_bufr_data_element = &_grib_accessor_cla
static void init_class(grib_accessor_class* c)
{
c->next_offset = (*(c->super))->next_offset;
c->string_length = (*(c->super))->string_length;
c->byte_count = (*(c->super))->byte_count;
c->byte_offset = (*(c->super))->byte_offset;
c->sub_section = (*(c->super))->sub_section;
c->pack_missing = (*(c->super))->pack_missing;
c->is_missing = (*(c->super))->is_missing;
c->pack_bytes = (*(c->super))->pack_bytes;
c->unpack_bytes = (*(c->super))->unpack_bytes;
c->pack_expression = (*(c->super))->pack_expression;
c->notify_change = (*(c->super))->notify_change;
c->update_size = (*(c->super))->update_size;
c->preferred_size = (*(c->super))->preferred_size;
c->resize = (*(c->super))->resize;
c->nearest_smaller_value = (*(c->super))->nearest_smaller_value;
c->next = (*(c->super))->next;
c->compare = (*(c->super))->compare;
c->unpack_double_subarray = (*(c->super))->unpack_double_subarray;
c->clear = (*(c->super))->clear;
c->next_offset = (*(c->super))->next_offset;
c->string_length = (*(c->super))->string_length;
c->byte_count = (*(c->super))->byte_count;
c->byte_offset = (*(c->super))->byte_offset;
c->sub_section = (*(c->super))->sub_section;
c->pack_missing = (*(c->super))->pack_missing;
c->pack_bytes = (*(c->super))->pack_bytes;
c->unpack_bytes = (*(c->super))->unpack_bytes;
c->pack_expression = (*(c->super))->pack_expression;
c->notify_change = (*(c->super))->notify_change;
c->update_size = (*(c->super))->update_size;
c->preferred_size = (*(c->super))->preferred_size;
c->resize = (*(c->super))->resize;
c->nearest_smaller_value = (*(c->super))->nearest_smaller_value;
c->next = (*(c->super))->next;
c->compare = (*(c->super))->compare;
c->unpack_double_subarray = (*(c->super))->unpack_double_subarray;
c->clear = (*(c->super))->clear;
}
/* END_CLASS_IMP */
@ -639,3 +639,95 @@ static void destroy(grib_context* ct, grib_accessor* a)
i++;
}
}
#define MAX_STRING_SIZE 4096
/* Return 1 if BUFR element(s) is/are missing, 0 otherwise. In case of decoding errors, also return 0 */
static int is_missing(grib_accessor* a)
{
const int ktype = get_native_type(a);
int err = 0, result = 1; // default: assume all are missing
long count = 0;
size_t i = 0, size = 1, size2 = 0;
grib_context* c = a->context;
if (ktype == GRIB_TYPE_LONG) {
long* values = NULL;
long value = 0;
value_count(a, &count);
size = size2 = count;
if (size > 1) {
values = (long*)grib_context_malloc_clear(c, sizeof(long) * size);
err = grib_unpack_long(a, values, &size2);
}
else {
err = grib_unpack_long(a, &value, &size2);
}
if (err) return 0; // TODO: no way of propagating the error up
Assert(size2 == size);
if (size > 1) {
for (i = 0; i < size; i++) {
if (!grib_is_missing_long(a, values[i])) {
result = 0; // at least one not missing
break;
}
}
grib_context_free(c, values);
} else {
result = grib_is_missing_long(a, value);
}
}
else if (ktype == GRIB_TYPE_DOUBLE) {
double value = 0;
double* values = NULL;
value_count(a, &count);
size = size2 = count;
if (size > 1) {
values = (double*)grib_context_malloc_clear(c, sizeof(double) * size);
err = grib_unpack_double(a, values, &size2);
}
else {
err = grib_unpack_double(a, &value, &size2);
}
if (err) return 0; // TODO: no way of propagating the error up
Assert(size2 == size);
if (size > 1) {
for (i = 0; i < size; ++i) {
if (!grib_is_missing_double(a, values[i])) {
result = 0;
break;
}
}
grib_context_free(c, values);
}
else {
result = grib_is_missing_double(a, value);
}
}
else if (ktype == GRIB_TYPE_STRING) {
char** values = NULL;
value_count(a, &count);
size = count;
if (size > 1) {
values = (char**)grib_context_malloc_clear(a->context, size * sizeof(char*));
err = grib_unpack_string_array(a, values, &size);
if (err) return 0; // TODO: no way of propagating the error up
for (i = 0; i < size; i++) {
if (!grib_is_missing_string(a, (unsigned char*)values[i], size)) {
result = 0;
break;
}
}
for (i = 0; i < size; i++) grib_context_free(c, values[i]);
grib_context_free(c, values);
} else {
char value[MAX_STRING_SIZE] = {0,}; /* See ECC-710 */
size = MAX_STRING_SIZE;
err = grib_unpack_string(a, value, &size);
if (err) return 0; // TODO: no way of propagating the error up
result = grib_is_missing_string(a, (unsigned char*)value, size);
}
}
return result;
}

15
src/grib_expression_class_functor.c
View File

@ -106,9 +106,14 @@ static int evaluate_long(grib_expression* g, grib_handle* h, long* lres)
if (p) {
long val = 0;
int err = 0;
err = grib_get_long_internal(h, p, &val);
if (err)
return err;
if (h->product_kind == PRODUCT_BUFR) {
int ismiss = grib_is_missing(h, p, &err);
if (err) return err;
*lres = ismiss;
return GRIB_SUCCESS;
}
err = grib_get_long_internal(h, p, &val);
if (err) return err;
/* Note: This does not cope with keys like typeOfSecondFixedSurface
* which are codetable entries with values like 255: this value is
* not classed as 'missing'!
@ -117,8 +122,10 @@ static int evaluate_long(grib_expression* g, grib_handle* h, long* lres)
*lres = (val == GRIB_MISSING_LONG);
return GRIB_SUCCESS;
}
else
else {
/* No arguments means return the actual integer missing value */
*lres = GRIB_MISSING_LONG;
}
return GRIB_SUCCESS;
}

2
tests/bufr_filter.sh
View File

@ -485,8 +485,10 @@ cat > $fRules <<EOF
set unpack=1;
transient m1 = missing(heightOfBarometerAboveMeanSeaLevel);
transient m2 = missing(blockNumber);
transient m3 = missing(stationOrSiteName);
assert ( m1 == 1 );
assert ( m2 == 1 );
assert ( m3 == 1 );
EOF
f="$ECCODES_SAMPLES_PATH/BUFR4.tmpl"
${tools_dir}/codes_bufr_filter $fRules $f