core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECC-830: BUFR encoding: set to missing if value out of range (key)

This commit is contained in:
Shahram Najm 2018-12-05 17:56:25 +00:00
parent b7d656e942
commit c6395fedcc
3 changed files with 69 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
definitions/bufr/boot.def
View File

@ -9,6 +9,7 @@
lookup[1] ed (7,editionNumber);
transient missingValue=999999 : hidden;
transient setToMissingIfOutOfRange=0 : hidden;
# This gets updated twice a year by WMO.
# See http://www.wmo.int/pages/prog/www/WMOCodes/WMO306_vI2/LatestVERSION/LatestVERSION.html

31
src/grib_accessor_class_bufr_data_array.c
View File

@ -69,6 +69,7 @@
MEMBERS = long* refValList
MEMBERS = long refValIndex
MEMBERS = bufr_tableb_override* tableb_override
MEMBERS = int set_to_missing_if_out_of_range
END_CLASS_DEF
@ -147,6 +148,7 @@ typedef struct grib_accessor_bufr_data_array {
long* refValList;
long refValIndex;
bufr_tableb_override* tableb_override;
int set_to_missing_if_out_of_range;
} grib_accessor_bufr_data_array;
extern grib_accessor_class* grib_accessor_class_gen;
@ -401,6 +403,7 @@ static void init(grib_accessor* a,const long v, grib_arguments* params)
self->refValList=NULL; /* Operator 203YYY: overridden reference values array */
self->refValIndex=0; /* Operator 203YYY: index into overridden reference values array */
self->tableb_override = NULL; /* Operator 203YYY: Table B lookup linked list */
self->set_to_missing_if_out_of_range = 0; /* By default fail if out of range */
a->length=0;
self->bitsToEndData=get_length(a)*8;
@ -449,6 +452,7 @@ static void self_clear(grib_context* c,grib_accessor_bufr_data_array* self)
if (self->refValList) grib_context_free(c, self->refValList);
self->refValIndex=0;
tableB_override_clear(c, self);
self->set_to_missing_if_out_of_range = 0;
}
static int get_native_type(grib_accessor* a)
@ -732,7 +736,8 @@ static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos, bufr
int thereIsAMissing=0;
int is_constant;
double val0;
const int dont_fail_if_out_of_range = c->bufr_set_to_missing_if_out_of_range;/* ECC-379 */
/* ECC-379, ECC-830 */
const int dont_fail_if_out_of_range = self->set_to_missing_if_out_of_range;
if (self->iss_list==NULL) {
grib_context_log(c, GRIB_LOG_ERROR,"encode_double_array: self->iss_list==NULL");
@ -763,7 +768,7 @@ static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos, bufr
if (*v > maxAllowed || *v < minAllowed) {
if (dont_fail_if_out_of_range) {
grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Value (%g) out of range (minAllowed=%g, maxAllowed=%g)."
"Setting it to missing value\n", bd->shortName, *v, minAllowed, maxAllowed);
" Setting it to missing value\n", bd->shortName, *v, minAllowed, maxAllowed);
grib_set_bits_on(buff->data,pos,modifiedWidth);
} else {
grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Value (%g) out of range (minAllowed=%g, maxAllowed=%g).",
@ -817,7 +822,7 @@ static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos, bufr
/* Turn out-of-range values into 'missing' */
if (*v!=GRIB_MISSING_DOUBLE && (*v < minAllowed || *v > maxAllowed)) {
grib_context_log(c, GRIB_LOG_ERROR, "encode_double_array: %s. Value at index %ld (%g) out of range (minAllowed=%g, maxAllowed=%g)."
"Setting it to missing value\n",
" Setting it to missing value\n",
bd->shortName, (long)ii, *v, minAllowed, maxAllowed);
*v = GRIB_MISSING_DOUBLE;
}
@ -910,7 +915,8 @@ static int encode_double_value(grib_context* c,grib_buffer* buff,long* pos,bufr_
int err=0;
int modifiedWidth,modifiedReference;
double modifiedFactor;
const int dont_fail_if_out_of_range = c->bufr_set_to_missing_if_out_of_range; /* ECC-379 */
/* ECC-379, ECC-830 */
const int dont_fail_if_out_of_range = self->set_to_missing_if_out_of_range;
modifiedReference= bd->reference;
modifiedFactor= bd->factor;
@ -925,7 +931,7 @@ static int encode_double_value(grib_context* c,grib_buffer* buff,long* pos,bufr_
else if (value>maxAllowed || value<minAllowed) {
if (dont_fail_if_out_of_range) {
grib_context_log(c, GRIB_LOG_ERROR, "encode_double_value: %s. Value (%g) out of range (minAllowed=%g, maxAllowed=%g)."
"Setting it to missing value\n",
" Setting it to missing value\n",
bd->shortName, value, minAllowed, maxAllowed);
value = GRIB_MISSING_DOUBLE; /* Ignore the bad value and instead use 'missing' */
grib_set_bits_on(buff->data,pos,modifiedWidth);
@ -2478,6 +2484,19 @@ static void set_input_bitmap(grib_handle* h,grib_accessor_bufr_data_array *self)
}
}
static int set_to_missing_if_out_of_range(grib_handle* h)
{
/* First check if the transient key is set */
long setToMissingIfOutOfRange=0;
if (grib_get_long(h, "setToMissingIfOutOfRange", &setToMissingIfOutOfRange)==GRIB_SUCCESS &&
setToMissingIfOutOfRange != 0)
{
return 1;
}
/* Then check the environment variable via the context */
return h->context->bufr_set_to_missing_if_out_of_range;
}
static int process_elements(grib_accessor* a,int flag,long onlySubset,long startSubset,long endSubset)
{
int err=0;
@ -2535,6 +2554,7 @@ static int process_elements(grib_accessor* a,int flag,long onlySubset,long start
decoding=0;
do_clean=1;
self->do_decode=1;
self->set_to_missing_if_out_of_range = set_to_missing_if_out_of_range(h);
pos=0;
codec_element=&encode_new_element;
codec_replication=&encode_new_replication;
@ -2548,6 +2568,7 @@ static int process_elements(grib_accessor* a,int flag,long onlySubset,long start
decoding=0;
do_clean=0;
self->do_decode=0;
self->set_to_missing_if_out_of_range = set_to_missing_if_out_of_range(h);
pos=0;
codec_element=&encode_element;
grib_get_long(grib_handle_of_accessor(a),"extractSubset",&onlySubset);

43
tests/bufr_ecc-379.sh
View File

@ -11,7 +11,7 @@
. ./include.sh
# ---------------------------------------------------------
# This is the test for the JIRA issue ECC-379
# This is the test for the JIRA issue ECC-379 (Also ECC-830)
# BUFR encoding failing when value out of range
# ---------------------------------------------------------
cd ${data_dir}/bufr
@ -96,5 +96,46 @@ export ECCODES_BUFR_SET_TO_MISSING_IF_OUT_OF_RANGE=1
${tools_dir}/codes_bufr_filter -o $tempOut $tempRules $BufrFile
unset ECCODES_BUFR_SET_TO_MISSING_IF_OUT_OF_RANGE
# --------------------------------------------------------
# Test 4: use the key setToMissingIfOutOfRange
# --------------------------------------------------------
BufrFile=airc_144.bufr
cat > $tempRules <<EOF
set unpack=1;
set setToMissingIfOutOfRange=1;
set latitude=9999;
set pack=1;
write;
EOF
# Will pass now
${tools_dir}/codes_bufr_filter -o $tempOut $tempRules $BufrFile
echo 'set unpack=1; print "lat is now=[latitude]";' | ${tools_dir}/codes_bufr_filter - $tempOut
# --------------------------------------------------------
# Test 5: set setToMissingIfOutOfRange for one message only
# --------------------------------------------------------
BufrFile=syno_multi.bufr
cat > $tempRules <<EOF
set unpack=1;
if (count==2) {set setToMissingIfOutOfRange=1;}
set latitude=5000;
set pack=1;
write;
EOF
# syno_multi has 3 messages but only one will get written out
# and its latitude will be missing
rm -f $tempOut
${tools_dir}/codes_bufr_filter -f -o $tempOut $tempRules $BufrFile
count=`${tools_dir}/bufr_count $BufrFile`
[ $count -eq 3 ]
count=`${tools_dir}/bufr_count $tempOut`
[ $count -eq 1 ]
lat=`${tools_dir}/bufr_get -s unpack=1 -p latitude $tempOut`
[ "$lat" = "MISSING" ]
# ------------------------
rm -rf $tempOut $tempRules $tempText