From 2f90f4a3e68826ff10a8334a3be271deaf57ff71 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 11 Dec 2020 18:17:30 +0000 Subject: [PATCH 1/3] ECC-403: codes_is_missing reports 1 on keys that are not missing --- src/grib_accessor_class_bufr_data_element.c | 212 ++++++++++++++------ src/grib_expression_class_functor.c | 15 +- 2 files changed, 163 insertions(+), 64 deletions(-) diff --git a/src/grib_accessor_class_bufr_data_element.c b/src/grib_accessor_class_bufr_data_element.c index c332b081f..c74e22bfb 100644 --- a/src/grib_accessor_class_bufr_data_element.c +++ b/src/grib_accessor_class_bufr_data_element.c @@ -22,7 +22,7 @@ SUPER = grib_accessor_class_gen IMPLEMENTS = init;dump IMPLEMENTS = unpack_string;unpack_string_array;unpack_long; unpack_double - IMPLEMENTS = unpack_double_element + IMPLEMENTS = unpack_double_element ; is_missing IMPLEMENTS = pack_long; pack_double ; pack_string_array; pack_string IMPLEMENTS = value_count; get_native_type; make_clone; destroy MEMBERS = long index @@ -51,6 +51,7 @@ or edit "accessor.class" and rerun ./make_class.pl */ static int get_native_type(grib_accessor*); +static int is_missing(grib_accessor*); static int pack_double(grib_accessor*, const double* val, size_t* len); static int pack_long(grib_accessor*, const long* val, size_t* len); static int pack_string(grib_accessor*, const char*, size_t* len); @@ -87,46 +88,46 @@ typedef struct grib_accessor_bufr_data_element extern grib_accessor_class* grib_accessor_class_gen; static grib_accessor_class _grib_accessor_class_bufr_data_element = { - &grib_accessor_class_gen, /* super */ - "bufr_data_element", /* name */ - sizeof(grib_accessor_bufr_data_element), /* size */ - 0, /* inited */ - &init_class, /* init_class */ - &init, /* init */ - 0, /* post_init */ - &destroy, /* free mem */ - &dump, /* describes himself */ - 0, /* get length of section */ - 0, /* get length of string */ - &value_count, /* get number of values */ - 0, /* get number of bytes */ - 0, /* get offset to bytes */ - &get_native_type, /* get native type */ - 0, /* get sub_section */ - 0, /* grib_pack procedures long */ - 0, /* grib_pack procedures long */ - &pack_long, /* grib_pack procedures long */ - &unpack_long, /* grib_unpack procedures long */ - &pack_double, /* grib_pack procedures double */ - &unpack_double, /* grib_unpack procedures double */ - &pack_string, /* grib_pack procedures string */ - &unpack_string, /* grib_unpack procedures string */ - &pack_string_array, /* grib_pack array procedures string */ - &unpack_string_array, /* grib_unpack array procedures string */ - 0, /* grib_pack procedures bytes */ - 0, /* grib_unpack procedures bytes */ - 0, /* pack_expression */ - 0, /* notify_change */ - 0, /* update_size */ - 0, /* preferred_size */ - 0, /* resize */ - 0, /* nearest_smaller_value */ - 0, /* next accessor */ - 0, /* compare vs. another accessor */ - &unpack_double_element, /* unpack only ith value */ - 0, /* unpack a subarray */ - 0, /* clear */ - &make_clone, /* clone accessor */ + &grib_accessor_class_gen, /* super */ + "bufr_data_element", /* name */ + sizeof(grib_accessor_bufr_data_element), /* size */ + 0, /* inited */ + &init_class, /* init_class */ + &init, /* init */ + 0, /* post_init */ + &destroy, /* free mem */ + &dump, /* describes himself */ + 0, /* get length of section */ + 0, /* get length of string */ + &value_count, /* get number of values */ + 0, /* get number of bytes */ + 0, /* get offset to bytes */ + &get_native_type, /* get native type */ + 0, /* get sub_section */ + 0, /* grib_pack procedures long */ + &is_missing, /* grib_pack procedures long */ + &pack_long, /* grib_pack procedures long */ + &unpack_long, /* grib_unpack procedures long */ + &pack_double, /* grib_pack procedures double */ + &unpack_double, /* grib_unpack procedures double */ + &pack_string, /* grib_pack procedures string */ + &unpack_string, /* grib_unpack procedures string */ + &pack_string_array, /* grib_pack array procedures string */ + &unpack_string_array, /* grib_unpack array procedures string */ + 0, /* grib_pack procedures bytes */ + 0, /* grib_unpack procedures bytes */ + 0, /* pack_expression */ + 0, /* notify_change */ + 0, /* update_size */ + 0, /* preferred_size */ + 0, /* resize */ + 0, /* nearest_smaller_value */ + 0, /* next accessor */ + 0, /* compare vs. another accessor */ + &unpack_double_element, /* unpack only ith value */ + 0, /* unpack a subarray */ + 0, /* clear */ + &make_clone, /* clone accessor */ }; @@ -135,25 +136,24 @@ grib_accessor_class* grib_accessor_class_bufr_data_element = &_grib_accessor_cla static void init_class(grib_accessor_class* c) { - c->next_offset = (*(c->super))->next_offset; - c->string_length = (*(c->super))->string_length; - c->byte_count = (*(c->super))->byte_count; - c->byte_offset = (*(c->super))->byte_offset; - c->sub_section = (*(c->super))->sub_section; - c->pack_missing = (*(c->super))->pack_missing; - c->is_missing = (*(c->super))->is_missing; - c->pack_bytes = (*(c->super))->pack_bytes; - c->unpack_bytes = (*(c->super))->unpack_bytes; - c->pack_expression = (*(c->super))->pack_expression; - c->notify_change = (*(c->super))->notify_change; - c->update_size = (*(c->super))->update_size; - c->preferred_size = (*(c->super))->preferred_size; - c->resize = (*(c->super))->resize; - c->nearest_smaller_value = (*(c->super))->nearest_smaller_value; - c->next = (*(c->super))->next; - c->compare = (*(c->super))->compare; - c->unpack_double_subarray = (*(c->super))->unpack_double_subarray; - c->clear = (*(c->super))->clear; + c->next_offset = (*(c->super))->next_offset; + c->string_length = (*(c->super))->string_length; + c->byte_count = (*(c->super))->byte_count; + c->byte_offset = (*(c->super))->byte_offset; + c->sub_section = (*(c->super))->sub_section; + c->pack_missing = (*(c->super))->pack_missing; + c->pack_bytes = (*(c->super))->pack_bytes; + c->unpack_bytes = (*(c->super))->unpack_bytes; + c->pack_expression = (*(c->super))->pack_expression; + c->notify_change = (*(c->super))->notify_change; + c->update_size = (*(c->super))->update_size; + c->preferred_size = (*(c->super))->preferred_size; + c->resize = (*(c->super))->resize; + c->nearest_smaller_value = (*(c->super))->nearest_smaller_value; + c->next = (*(c->super))->next; + c->compare = (*(c->super))->compare; + c->unpack_double_subarray = (*(c->super))->unpack_double_subarray; + c->clear = (*(c->super))->clear; } /* END_CLASS_IMP */ @@ -639,3 +639,95 @@ static void destroy(grib_context* ct, grib_accessor* a) i++; } } + +#define MAX_STRING_SIZE 4096 +/* Return 1 if BUFR element(s) is/are missing, 0 otherwise. In case of decoding errors, also return 0 */ +static int is_missing(grib_accessor* a) +{ + const int ktype = get_native_type(a); + int err = 0, result = 1; // default: assume all are missing + long count = 0; + size_t i = 0, size = 1, size2 = 0; + grib_context* c = a->context; + + if (ktype == GRIB_TYPE_LONG) { + long* values = NULL; + long value = 0; + + value_count(a, &count); + size = size2 = count; + if (size > 1) { + values = (long*)grib_context_malloc_clear(c, sizeof(long) * size); + err = grib_unpack_long(a, values, &size2); + } + else { + err = grib_unpack_long(a, &value, &size2); + } + if (err) return 0; // TODO: no way of propagating the error up + Assert(size2 == size); + if (size > 1) { + for (i = 0; i < size; i++) { + if (!grib_is_missing_long(a, values[i])) { + result = 0; // at least one not missing + break; + } + } + grib_context_free(c, values); + } else { + result = grib_is_missing_long(a, value); + } + } + else if (ktype == GRIB_TYPE_DOUBLE) { + double value = 0; + double* values = NULL; + + value_count(a, &count); + size = size2 = count; + if (size > 1) { + values = (double*)grib_context_malloc_clear(c, sizeof(double) * size); + err = grib_unpack_double(a, values, &size2); + } + else { + err = grib_unpack_double(a, &value, &size2); + } + if (err) return 0; // TODO: no way of propagating the error up + Assert(size2 == size); + if (size > 1) { + for (i = 0; i < size; ++i) { + if (!grib_is_missing_double(a, values[i])) { + result = 0; + break; + } + } + grib_context_free(c, values); + } + else { + result = grib_is_missing_double(a, value); + } + } + else if (ktype == GRIB_TYPE_STRING) { + char** values = NULL; + value_count(a, &count); + size = count; + if (size > 1) { + values = (char**)grib_context_malloc_clear(a->context, size * sizeof(char*)); + err = grib_unpack_string_array(a, values, &size); + if (err) return 0; // TODO: no way of propagating the error up + for (i = 0; i < size; i++) { + if (!grib_is_missing_string(a, (unsigned char*)values[i], size)) { + result = 0; + break; + } + } + for (i = 0; i < size; i++) grib_context_free(c, values[i]); + grib_context_free(c, values); + } else { + char value[MAX_STRING_SIZE] = {0,}; /* See ECC-710 */ + size = MAX_STRING_SIZE; + err = grib_unpack_string(a, value, &size); + if (err) return 0; // TODO: no way of propagating the error up + result = grib_is_missing_string(a, (unsigned char*)value, size); + } + } + return result; +} diff --git a/src/grib_expression_class_functor.c b/src/grib_expression_class_functor.c index 4919ba5df..34cf3d5a6 100644 --- a/src/grib_expression_class_functor.c +++ b/src/grib_expression_class_functor.c @@ -106,9 +106,14 @@ static int evaluate_long(grib_expression* g, grib_handle* h, long* lres) if (p) { long val = 0; int err = 0; - err = grib_get_long_internal(h, p, &val); - if (err) - return err; + if (h->product_kind == PRODUCT_BUFR) { + int ismiss = grib_is_missing(h, p, &err); + if (err) return err; + *lres = ismiss; + return GRIB_SUCCESS; + } + err = grib_get_long_internal(h, p, &val); + if (err) return err; /* Note: This does not cope with keys like typeOfSecondFixedSurface * which are codetable entries with values like 255: this value is * not classed as 'missing'! @@ -117,8 +122,10 @@ static int evaluate_long(grib_expression* g, grib_handle* h, long* lres) *lres = (val == GRIB_MISSING_LONG); return GRIB_SUCCESS; } - else + else { + /* No arguments means return the actual integer missing value */ *lres = GRIB_MISSING_LONG; + } return GRIB_SUCCESS; } From 15652829ff679198b704de25b653c1413846790a Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Fri, 11 Dec 2020 18:28:41 +0000 Subject: [PATCH 2/3] ECC-403: test is_missing for string key --- tests/bufr_filter.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/bufr_filter.sh b/tests/bufr_filter.sh index 6afe9d46a..511c2df24 100755 --- a/tests/bufr_filter.sh +++ b/tests/bufr_filter.sh @@ -485,8 +485,10 @@ cat > $fRules < Date: Fri, 11 Dec 2020 19:27:33 +0000 Subject: [PATCH 3/3] ECC-403: C and Python API tests --- examples/C/bufr_missing.c | 8 ++++++-- examples/python/bufr_get_keys.py | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/examples/C/bufr_missing.c b/examples/C/bufr_missing.c index 2ff58225f..bbb7adef4 100644 --- a/examples/C/bufr_missing.c +++ b/examples/C/bufr_missing.c @@ -15,7 +15,7 @@ */ #include "eccodes.h" - +#include int main(int argc, char* argv[]) { @@ -26,6 +26,7 @@ int main(int argc, char* argv[]) double doubleVal; int err = 0; + int is_missing = 0; int cnt = 0; const char* infile = "../../data/bufr/syno_1.bufr"; @@ -51,7 +52,6 @@ int main(int argc, char* argv[]) /* the value of this key is missing in the message*/ CODES_CHECK(codes_get_double(h, "relativeHumidity", &doubleVal), 0); - printf(" relativeHumidity: %.2f\n", doubleVal); /* we will print "value missing" */ if (doubleVal == CODES_MISSING_DOUBLE) { @@ -61,6 +61,10 @@ int main(int argc, char* argv[]) printf(" --> value present\n"); } + is_missing = codes_is_missing(h, "relativeHumidity", &err); + assert(!err); + assert(is_missing == 1); + /* delete handle */ codes_handle_delete(h); diff --git a/examples/python/bufr_get_keys.py b/examples/python/bufr_get_keys.py index a204c69a2..35a1c6a0f 100644 --- a/examples/python/bufr_get_keys.py +++ b/examples/python/bufr_get_keys.py @@ -60,6 +60,7 @@ def example(): print(' %s: %s' % (key, codes_get(bufr, key))) except CodesInternalError as err: print('Error with key="%s" : %s' % (key, err.msg)) + assert codes_is_missing(bufr, key) == 0 # Native type float key = 'airTemperatureAt2M' @@ -67,6 +68,7 @@ def example(): print(' %s: %s' % (key, codes_get(bufr, key))) except CodesInternalError as err: print('Error with key="%s" : %s' % (key, err.msg)) + assert codes_is_missing(bufr, key) == 0 # Native type string key = 'typicalDate' @@ -75,6 +77,9 @@ def example(): except CodesInternalError as err: print('Error with key="%s" : %s' % (key, err.msg)) + assert codes_is_missing(bufr, 'relativeHumidity') == 1 + assert codes_is_missing(bufr, '#5#cloudAmount') == 1 + # -------------------------------- # get values for an array # --------------------------------