diff --git a/definitions/boot.def b/definitions/boot.def index 83ea9f2ee..069df6900 100644 --- a/definitions/boot.def +++ b/definitions/boot.def @@ -15,6 +15,11 @@ UseEcmfConventions = getenv("ECCODES_USE_ECMF_CONVENTIONS","1") :hidden ; constant defaultTypeOfLevel="unknown" : hidden; +gribDataQualityChecks = getenv("ECCODES_GRIB_DATA_QUALITY_CHECKS","0") : hidden; +if (gribDataQualityChecks) { + template LIMITS "param_limits.def"; +} + # GRIBEX special boustrophedonic mode. See GRIB-472 # If the environment variable is not defined, the key will be 0 GRIBEX_boustrophedonic = getenv("ECCODES_GRIBEX_BOUSTROPHEDONIC","0") :hidden; diff --git a/definitions/param_limits.def b/definitions/param_limits.def new file mode 100644 index 000000000..24616f996 --- /dev/null +++ b/definitions/param_limits.def @@ -0,0 +1,9 @@ +constant default_min_val = -1e9 : long_type, hidden; +constant default_max_val = +1e9 : long_type, hidden; + +concept param_value_min(default_min_val) { + 0 = { paramId=167; } +} : long_type, hidden; +concept param_value_max(default_max_val) { + 373 = { paramId=167; } +} : long_type, hidden; diff --git a/src/action_class_concept.c b/src/action_class_concept.c index 19589c458..71e3ab2fc 100644 --- a/src/action_class_concept.c +++ b/src/action_class_concept.c @@ -305,3 +305,97 @@ static grib_concept_value* get_concept(grib_handle* h, grib_action_concept* self GRIB_MUTEX_UNLOCK(&mutex); return result; } + +static int concept_condition_expression_true(grib_handle* h, grib_concept_condition* c, char* exprVal) +{ + long lval; + long lres=0; + int ok = 0; + int err=0; + const int type = grib_expression_native_type(h,c->expression); + + switch(type) + { + case GRIB_TYPE_LONG: + grib_expression_evaluate_long(h,c->expression,&lres); + ok = (grib_get_long(h,c->name,&lval) == GRIB_SUCCESS) && + (lval == lres); + if (ok) sprintf(exprVal, "%ld", lres); + break; + + case GRIB_TYPE_DOUBLE: { + double dval; + double dres=0.0; + grib_expression_evaluate_double(h,c->expression,&dres); + ok = (grib_get_double(h,c->name,&dval) == GRIB_SUCCESS) && + (dval == dres); + if (ok) sprintf(exprVal, "%g", dres); + break; + } + + case GRIB_TYPE_STRING: { + const char *cval; + char buf[80]; + char tmp[80]; + size_t len = sizeof(buf); + size_t size=sizeof(tmp); + + ok = (grib_get_string(h,c->name,buf,&len) == GRIB_SUCCESS) && + ((cval = grib_expression_evaluate_string(h,c->expression,tmp,&size,&err)) != NULL) && + (err==0) && (strcmp(buf,cval) == 0); + if (ok) sprintf(exprVal, "%s", cval); + break; + } + + default: + /* TODO: */ + break; + } + return ok; +} + +/* Caller has to allocate space for the result. + * INPUTS: h, key and value (can be NULL) + * OUTPUT: result + * Example: key='typeOfLevel' whose value is 'mixedLayerDepth', + * result='typeOfFirstFixedSurface=169,typeOfSecondFixedSurface=255' + */ +int get_concept_condition_string(grib_handle* h, const char* key, const char* value, char* result) +{ + int err = 0; + int length = 0; + char strVal[64]={0,}; + char exprVal[256]={0,}; + const char* pValue = value; + size_t len = sizeof(strVal); + grib_concept_value* concept_value = NULL; + grib_accessor* acc = grib_find_accessor(h, key); + if (!acc) return GRIB_NOT_FOUND; + + if (!value) { + err = grib_get_string(h, key, strVal,&len); + if (err) return GRIB_INTERNAL_ERROR; + pValue = strVal; + } + + concept_value = action_concept_get_concept(acc); + while (concept_value) { + grib_concept_condition* concept_condition = concept_value->conditions; + + if (strcmp(pValue, concept_value->name)==0) { + while (concept_condition) { + grib_expression* expression = concept_condition->expression; + Assert(expression); + if (concept_condition_expression_true(h, concept_condition, exprVal)) { + length += sprintf(result+length, "%s%s=%s", + (length==0?"":","),concept_condition->name, exprVal); + } + concept_condition = concept_condition->next; + } + } + + concept_value = concept_value->next; + } + if (length == 0) return GRIB_CONCEPT_NO_MATCH; + return GRIB_SUCCESS; +} diff --git a/src/grib_accessor_class_concept.c b/src/grib_accessor_class_concept.c index 40d2f2c1a..fcfb44c90 100644 --- a/src/grib_accessor_class_concept.c +++ b/src/grib_accessor_class_concept.c @@ -615,6 +615,14 @@ static int unpack_string (grib_accessor* a, char* val, size_t *len) } strcpy(val,p); *len = slen; +#if 0 + if (a->context->debug==1) { + int err = 0; + char result[1024] = {0,}; + err = get_concept_condition_string(grib_handle_of_accessor(a), a->name, val, result); + if (!err) fprintf(stderr, "ECCODES DEBUG concept name=%s, value=%s, conditions=%s\n", a->name, val, result); + } +#endif return GRIB_SUCCESS; } diff --git a/src/grib_accessor_class_data_simple_packing.c b/src/grib_accessor_class_data_simple_packing.c index f08c6c188..50e7a7695 100644 --- a/src/grib_accessor_class_data_simple_packing.c +++ b/src/grib_accessor_class_data_simple_packing.c @@ -505,12 +505,26 @@ static int producing_large_constant_fields(const grib_context* c, grib_handle* h } #endif -static int check_range(const double val) +static int check_range(grib_handle* h, const double min_val, const double max_val) { - if (val < DBL_MAX && val > -DBL_MAX) - return GRIB_SUCCESS; - else + int result = GRIB_SUCCESS; + grib_context* ctx = h->context; + + if ( !(min_val < DBL_MAX && min_val > -DBL_MAX) ) { + grib_context_log(ctx, GRIB_LOG_ERROR, "Minimum value out of range: %g", min_val); return GRIB_ENCODING_ERROR; + } + if ( !(max_val < DBL_MAX && max_val > -DBL_MAX) ) { + grib_context_log(ctx, GRIB_LOG_ERROR, "Maximum value out of range: %g", max_val); + return GRIB_ENCODING_ERROR; + } + + /* Data Quality checks */ + if (ctx->grib_data_quality_checks) { + result = grib_util_grib_data_quality_check(h, min_val, max_val); + } + + return result; } static int pack_double(grib_accessor* a, const double* val, size_t *len) @@ -577,12 +591,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len) else if (val[i] < min ) min = val[i]; } #endif - if ((err = check_range(max)) != GRIB_SUCCESS) { - grib_context_log(a->context,GRIB_LOG_ERROR,"Maximum value out of range: %g", max); - return err; - } - if ((err = check_range(min)) != GRIB_SUCCESS) { - grib_context_log(a->context,GRIB_LOG_ERROR,"Minimum value out of range: %g", min); + if ((err = check_range(gh, min, max)) != GRIB_SUCCESS) { return err; } diff --git a/src/grib_api_internal.h b/src/grib_api_internal.h index 43fa2722e..5028d7ff3 100644 --- a/src/grib_api_internal.h +++ b/src/grib_api_internal.h @@ -1056,6 +1056,7 @@ struct grib_context int bufrdc_mode; int bufr_set_to_missing_if_out_of_range; int bufr_multi_element_constant_arrays; + int grib_data_quality_checks; FILE* log_stream; grib_trie* classes; grib_trie* lists; diff --git a/src/grib_api_prototypes.h b/src/grib_api_prototypes.h index 5aba681d0..2dc932a06 100644 --- a/src/grib_api_prototypes.h +++ b/src/grib_api_prototypes.h @@ -67,6 +67,7 @@ grib_action *grib_action_create_when(grib_context *context, grib_expression *exp grib_concept_value *action_concept_get_concept(grib_accessor *a); int action_concept_get_nofail(grib_accessor *a); grib_action *grib_action_create_concept(grib_context *context, const char *name, grib_concept_value *concept, const char *basename, const char *name_space, const char *defaultkey, const char *masterDir, const char *localDir, const char *ecmfDir, int flags, int nofail); +int get_concept_condition_string(grib_handle* h, const char* key, const char* value, char* result); /* action_class_hash_array.c */ grib_action *grib_action_create_hash_array(grib_context *context, const char *name, grib_hash_array_value *hash_array, const char *basename, const char *name_space, const char *defaultkey, const char *masterDir, const char *localDir, const char *ecmfDir, int flags, int nofail); @@ -1483,6 +1484,7 @@ int grib2_is_PDTN_AerosolOptical(long productDefinitionTemplateNumber); int grib2_select_PDTN(int is_eps, int is_instant, int is_chemical, int is_chemical_distfn, int is_aerosol, int is_aerosol_optical); int is_grib_index_file(const char *filename); size_t sum_of_pl_array(const long* pl, size_t plsize); +int grib_util_grib_data_quality_check(grib_handle* h, double min_val, double max_val); /* bufr_util.c */ int compute_bufr_key_rank(grib_handle *h, grib_string_list *keys, const char *key); diff --git a/src/grib_context.c b/src/grib_context.c index da8b61d2e..e2616c08d 100644 --- a/src/grib_context.c +++ b/src/grib_context.c @@ -308,13 +308,13 @@ static grib_context default_grib_context = { &default_seek, /* lfile seek procedure */ &default_feof, /* file feof procedure */ - &default_log, /* logging_procedure */ - &default_print, /* print procedure */ - 0, /* grib_codetable* */ - 0, /* grib_smart_table* */ - 0, /* char* outfilename */ - 0, /* int multi_support_on */ - 0, /* grib_multi_support* multi_support*/ + &default_log, /* output_log */ + &default_print, /* print */ + 0, /* codetable */ + 0, /* smart_table */ + 0, /* outfilename */ + 0, /* multi_support_on */ + 0, /* multi_support */ 0, /* grib_definition_files_dir */ 0, /* handle_file_count */ 0, /* handle_total_count */ @@ -323,9 +323,9 @@ static grib_context default_grib_context = { 0, /* gts_header_on */ 0, /* gribex_mode_on */ 0, /* large_constant_fields */ - 0, /* grib_itrie* keys */ + 0, /* keys */ 0, /* keys_count */ - 0, /* grib_itrie* concepts_index */ + 0, /* concepts_index */ 0, /* concepts_count */ {0,}, /* concepts */ 0, /* hash_array_index */ @@ -337,6 +337,7 @@ static grib_context default_grib_context = { 0, /* bufrdc_mode */ 0, /* bufr_set_to_missing_if_out_of_range */ 0, /* bufr_multi_element_constant_arrays */ + 0, /* grib_data_quality_checks */ 0, /* log_stream */ 0, /* classes */ 0, /* lists */ @@ -371,6 +372,7 @@ grib_context* grib_context_get_default() const char* bufrdc_mode = NULL; const char* bufr_set_to_missing_if_out_of_range = NULL; const char* bufr_multi_element_constant_arrays = NULL; + const char* grib_data_quality_checks = NULL; const char* file_pool_max_opened_files = NULL; #ifdef ENABLE_FLOATING_POINT_EXCEPTIONS @@ -381,6 +383,7 @@ grib_context* grib_context_get_default() bufrdc_mode = getenv("ECCODES_BUFRDC_MODE_ON"); bufr_set_to_missing_if_out_of_range = getenv("ECCODES_BUFR_SET_TO_MISSING_IF_OUT_OF_RANGE"); bufr_multi_element_constant_arrays = getenv("ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS"); + grib_data_quality_checks = getenv("ECCODES_GRIB_DATA_QUALITY_CHECKS"); large_constant_fields = codes_getenv("ECCODES_GRIB_LARGE_CONSTANT_FIELDS"); no_abort = codes_getenv("ECCODES_NO_ABORT"); debug = codes_getenv("ECCODES_DEBUG"); @@ -458,6 +461,16 @@ grib_context* grib_context_get_default() } } + /* Definitions path extra: Added at the head of (i.e. before) existing path */ + { + const char* defs_extra = getenv("ECCODES_EXTRA_DEFINITION_PATH"); + if (defs_extra) { + char buffer[DEF_PATH_MAXLEN]; + ecc_snprintf(buffer, DEF_PATH_MAXLEN, "%s:%s", defs_extra, default_grib_context.grib_definition_files_path); + default_grib_context.grib_definition_files_path = strdup(buffer); + } + } + grib_context_log(&default_grib_context, GRIB_LOG_DEBUG, "Definitions path: %s", default_grib_context.grib_definition_files_path); grib_context_log(&default_grib_context, GRIB_LOG_DEBUG, "Samples path: %s", @@ -479,6 +492,8 @@ grib_context* grib_context_get_default() atoi(bufr_set_to_missing_if_out_of_range) : 0; default_grib_context.bufr_multi_element_constant_arrays = bufr_multi_element_constant_arrays ? atoi(bufr_multi_element_constant_arrays) : 0; + default_grib_context.grib_data_quality_checks = grib_data_quality_checks ? + atoi(grib_data_quality_checks) : 0; default_grib_context.file_pool_max_opened_files = file_pool_max_opened_files ? atoi(file_pool_max_opened_files) : DEFAULT_FILE_POOL_MAX_OPENED_FILES; } diff --git a/src/grib_util.c b/src/grib_util.c index a5b584c12..7e5d7e94b 100644 --- a/src/grib_util.c +++ b/src/grib_util.c @@ -2048,3 +2048,67 @@ size_t sum_of_pl_array(const long* pl, size_t plsize) } return count; } + +int grib_util_grib_data_quality_check(grib_handle* h, double min_val, double max_val) +{ + int err = 0; + long min_field_value_allowed=0, max_field_value_allowed=0; + long paramId = 0; + double dmin_allowed=0, dmax_allowed=0; + grib_context* ctx = h->context; + int is_error = 1; + /* + * If grib_data_quality_checks == 1, limits failure results in an error + * If grib_data_quality_checks == 2, limits failure results in a warning + */ + Assert( ctx->grib_data_quality_checks == 1 || ctx->grib_data_quality_checks == 2 ); + is_error = (ctx->grib_data_quality_checks == 1); + + /* The limit keys must exist if we are here */ + err = grib_get_long(h, "param_value_min", &min_field_value_allowed); + if (err) { + grib_context_log(ctx, GRIB_LOG_ERROR,"grib_data_quality_check: Could not get param_value_min"); + return err; + } + err = grib_get_long(h, "param_value_max", &max_field_value_allowed); + if (err) { + grib_context_log(ctx, GRIB_LOG_ERROR,"grib_data_quality_check: Could not get param_value_max"); + return err; + } + + dmin_allowed = (double)min_field_value_allowed; + dmax_allowed = (double)max_field_value_allowed; + + if (min_val < dmin_allowed) { + char description[1024] = {0,}; + if (get_concept_condition_string(h, "param_value_min", NULL, description)==GRIB_SUCCESS) { + fprintf(stderr, "ECCODES %s : (%s): minimum (%g) is less than the allowable limit (%g)\n", + (is_error? "ERROR":"WARNING"), description, min_val, dmin_allowed); + } else { + if (grib_get_long(h, "paramId", ¶mId) == GRIB_SUCCESS) { + fprintf(stderr, "ECCODES %s : (paramId=%ld): minimum (%g) is less than the default allowable limit (%g)\n", + (is_error? "ERROR":"WARNING"), paramId, min_val, dmin_allowed); + } + } + if (is_error) { + return GRIB_OUT_OF_RANGE; /* Failure */ + } + } + if (max_val > dmax_allowed) { + char description[1024] = {0,}; + if (get_concept_condition_string(h, "param_value_max", NULL, description)==GRIB_SUCCESS) { + fprintf(stderr, "ECCODES %s : (%s): maximum (%g) is more than the allowable limit (%g)\n", + (is_error? "ERROR":"WARNING"), description, max_val, dmax_allowed); + } else { + if (grib_get_long(h, "paramId", ¶mId) == GRIB_SUCCESS) { + fprintf(stderr, "ECCODES %s : (paramId=%ld): maximum (%g) is more than the default allowable limit (%g)\n", + (is_error? "ERROR":"WARNING"), paramId, max_val, dmax_allowed); + } + } + if (is_error) { + return GRIB_OUT_OF_RANGE; /* Failure */ + } + } + + return GRIB_SUCCESS; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 61b6d3fe8..bb4efaf3b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -70,6 +70,7 @@ list( APPEND tests_no_data_reqd ) # These tests do require data downloads list( APPEND tests_data_reqd + grib_data_quality_checks bpv_limit grib_complex grib_double_cmp diff --git a/tests/grib_data_quality_checks.sh b/tests/grib_data_quality_checks.sh new file mode 100755 index 000000000..65a08ce79 --- /dev/null +++ b/tests/grib_data_quality_checks.sh @@ -0,0 +1,96 @@ +#!/bin/sh +# Copyright 2005-2019 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +# + +. ./include.sh +set -u +# --------------------------------------------------------- +# Tests for data quality checks +# --------------------------------------------------------- +label="grib_data_quality" +tempOut=temp.1.${label}.out +temp2=temp.2.${label}.out +tempErr=temp.${label}.err + +# Start with clean environment +unset ECCODES_GRIB_DATA_QUALITY_CHECKS +unset ECCODES_EXTRA_DEFINITION_PATH + + +input1=${data_dir}/reduced_gaussian_surface.grib1 +input2=${data_dir}/reduced_gaussian_surface.grib2 +grib_check_key_equals $input1 paramId 167 +grib_check_key_equals $input2 paramId 167 + +# Data quality checks disabled. Create huge values for temperature +${tools_dir}/grib_set -s scaleValuesBy=100 $input1 $tempOut +${tools_dir}/grib_set -s scaleValuesBy=100 $input2 $tempOut + +# Data quality checks enabled. Repacking should fail +export ECCODES_GRIB_DATA_QUALITY_CHECKS=1 +set +e +${tools_dir}/grib_copy -r $tempOut /dev/null 2>$tempErr +status=$? +set -e +[ $status -ne 0 ] +grep -q 'more than the allowable limit' $tempErr + + +# Data quality checks enabled but only as a warning. Repacking should pass +export ECCODES_GRIB_DATA_QUALITY_CHECKS=2 +${tools_dir}/grib_copy -r $tempOut /dev/null 2>$tempErr +grep -q 'more than the allowable limit' $tempErr + + +# Data quality checks enabled. Scaling should fail +export ECCODES_GRIB_DATA_QUALITY_CHECKS=1 +set +e +${tools_dir}/grib_set -s scaleValuesBy=100 $input1 $tempOut 2>$tempErr +status=$? +set -e +[ $status -ne 0 ] +grep -q 'GRIB1 simple packing: unable to set values' $tempErr +grep -q 'allowable limit' $tempErr + +set +e +${tools_dir}/grib_set -s scaleValuesBy=100 $input2 $tempOut 2>$tempErr +status=$? +set -e +[ $status -ne 0 ] +grep -q 'GRIB2 simple packing: unable to set values' $tempErr +grep -q 'allowable limit' $tempErr + + +# Override the defaults +# ---------------------- +tempDir=tempdir.$label +rm -rf $tempDir +mkdir -p $tempDir +# Set a large limit for temperature +cat > $tempDir/param_limits.def <grib_definition_files_path); printf("Definition files path can be changed by setting ECCODES_DEFINITION_PATH environment variable\n"); } printf("\n"); @@ -118,7 +118,7 @@ int main( int argc,char* argv[]) if ((path=codes_getenv("ECCODES_DEFINITION_PATH")) != NULL) { printf("%s",path); } else { - printf("%s",ECCODES_DEFINITION_PATH); + printf("%s",context->grib_definition_files_path); } }