core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECC-992: GRIB encoding: Data quality checks (limits)

This commit is contained in:
Shahram Najm 2020-01-10 12:06:43 +00:00
parent aefd5f362a 397ec353a6
commit 5153078859
13 changed files with 353 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
definitions/boot.def
View File

@ -15,6 +15,11 @@ UseEcmfConventions = getenv("ECCODES_USE_ECMF_CONVENTIONS","1") :hidden ;
constant defaultTypeOfLevel="unknown" : hidden;
gribDataQualityChecks = getenv("ECCODES_GRIB_DATA_QUALITY_CHECKS","0") : hidden;
if (gribDataQualityChecks) {
template LIMITS "param_limits.def";
}
# GRIBEX special boustrophedonic mode. See GRIB-472
# If the environment variable is not defined, the key will be 0
GRIBEX_boustrophedonic = getenv("ECCODES_GRIBEX_BOUSTROPHEDONIC","0") :hidden;

9
definitions/param_limits.def Normal file
View File

@ -0,0 +1,9 @@
constant default_min_val = -1e9 : long_type, hidden;
constant default_max_val = +1e9 : long_type, hidden;
concept param_value_min(default_min_val) {
0 = { paramId=167; }
} : long_type, hidden;
concept param_value_max(default_max_val) {
373 = { paramId=167; }
} : long_type, hidden;

94
src/action_class_concept.c
View File

@ -305,3 +305,97 @@ static grib_concept_value* get_concept(grib_handle* h, grib_action_concept* self
GRIB_MUTEX_UNLOCK(&mutex);
return result;
}
static int concept_condition_expression_true(grib_handle* h, grib_concept_condition* c, char* exprVal)
{
long lval;
long lres=0;
int ok = 0;
int err=0;
const int type = grib_expression_native_type(h,c->expression);
switch(type)
{
case GRIB_TYPE_LONG:
grib_expression_evaluate_long(h,c->expression,&lres);
ok = (grib_get_long(h,c->name,&lval) == GRIB_SUCCESS) &&
(lval == lres);
if (ok) sprintf(exprVal, "%ld", lres);
break;
case GRIB_TYPE_DOUBLE: {
double dval;
double dres=0.0;
grib_expression_evaluate_double(h,c->expression,&dres);
ok = (grib_get_double(h,c->name,&dval) == GRIB_SUCCESS) &&
(dval == dres);
if (ok) sprintf(exprVal, "%g", dres);
break;
}
case GRIB_TYPE_STRING: {
const char *cval;
char buf[80];
char tmp[80];
size_t len = sizeof(buf);
size_t size=sizeof(tmp);
ok = (grib_get_string(h,c->name,buf,&len) == GRIB_SUCCESS) &&
((cval = grib_expression_evaluate_string(h,c->expression,tmp,&size,&err)) != NULL) &&
(err==0) && (strcmp(buf,cval) == 0);
if (ok) sprintf(exprVal, "%s", cval);
break;
}
default:
/* TODO: */
break;
}
return ok;
}
/* Caller has to allocate space for the result.
* INPUTS: h, key and value (can be NULL)
* OUTPUT: result
* Example: key='typeOfLevel' whose value is 'mixedLayerDepth',
* result='typeOfFirstFixedSurface=169,typeOfSecondFixedSurface=255'
*/
int get_concept_condition_string(grib_handle* h, const char* key, const char* value, char* result)
{
int err = 0;
int length = 0;
char strVal[64]={0,};
char exprVal[256]={0,};
const char* pValue = value;
size_t len = sizeof(strVal);
grib_concept_value* concept_value = NULL;
grib_accessor* acc = grib_find_accessor(h, key);
if (!acc) return GRIB_NOT_FOUND;
if (!value) {
err = grib_get_string(h, key, strVal,&len);
if (err) return GRIB_INTERNAL_ERROR;
pValue = strVal;
}
concept_value = action_concept_get_concept(acc);
while (concept_value) {
grib_concept_condition* concept_condition = concept_value->conditions;
if (strcmp(pValue, concept_value->name)==0) {
while (concept_condition) {
grib_expression* expression = concept_condition->expression;
Assert(expression);
if (concept_condition_expression_true(h, concept_condition, exprVal)) {
length += sprintf(result+length, "%s%s=%s",
(length==0?"":","),concept_condition->name, exprVal);
}
concept_condition = concept_condition->next;
}
}
concept_value = concept_value->next;
}
if (length == 0) return GRIB_CONCEPT_NO_MATCH;
return GRIB_SUCCESS;
}

8
src/grib_accessor_class_concept.c
View File

@ -615,6 +615,14 @@ static int unpack_string (grib_accessor* a, char* val, size_t *len)
}
strcpy(val,p);
*len = slen;
#if 0
if (a->context->debug==1) {
int err = 0;
char result[1024] = {0,};
err = get_concept_condition_string(grib_handle_of_accessor(a), a->name, val, result);
if (!err) fprintf(stderr, "ECCODES DEBUG concept name=%s, value=%s, conditions=%s\n", a->name, val, result);
}
#endif
return GRIB_SUCCESS;
}

29
src/grib_accessor_class_data_simple_packing.c
View File

@ -505,12 +505,26 @@ static int producing_large_constant_fields(const grib_context* c, grib_handle* h
}
#endif
static int check_range(const double val)
static int check_range(grib_handle* h, const double min_val, const double max_val)
{
if (val < DBL_MAX && val > -DBL_MAX)
return GRIB_SUCCESS;
else
int result = GRIB_SUCCESS;
grib_context* ctx = h->context;
if ( !(min_val < DBL_MAX && min_val > -DBL_MAX) ) {
grib_context_log(ctx, GRIB_LOG_ERROR, "Minimum value out of range: %g", min_val);
return GRIB_ENCODING_ERROR;
}
if ( !(max_val < DBL_MAX && max_val > -DBL_MAX) ) {
grib_context_log(ctx, GRIB_LOG_ERROR, "Maximum value out of range: %g", max_val);
return GRIB_ENCODING_ERROR;
}
/* Data Quality checks */
if (ctx->grib_data_quality_checks) {
result = grib_util_grib_data_quality_check(h, min_val, max_val);
}
return result;
}
static int pack_double(grib_accessor* a, const double* val, size_t *len)
@ -577,12 +591,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
else if (val[i] < min ) min = val[i];
}
#endif
if ((err = check_range(max)) != GRIB_SUCCESS) {
grib_context_log(a->context,GRIB_LOG_ERROR,"Maximum value out of range: %g", max);
return err;
}
if ((err = check_range(min)) != GRIB_SUCCESS) {
grib_context_log(a->context,GRIB_LOG_ERROR,"Minimum value out of range: %g", min);
if ((err = check_range(gh, min, max)) != GRIB_SUCCESS) {
return err;
}

1
src/grib_api_internal.h
View File

@ -1056,6 +1056,7 @@ struct grib_context
int bufrdc_mode;
int bufr_set_to_missing_if_out_of_range;
int bufr_multi_element_constant_arrays;
int grib_data_quality_checks;
FILE* log_stream;
grib_trie* classes;
grib_trie* lists;

2
src/grib_api_prototypes.h
View File

@ -67,6 +67,7 @@ grib_action *grib_action_create_when(grib_context *context, grib_expression *exp
grib_concept_value *action_concept_get_concept(grib_accessor *a);
int action_concept_get_nofail(grib_accessor *a);
grib_action *grib_action_create_concept(grib_context *context, const char *name, grib_concept_value *concept, const char *basename, const char *name_space, const char *defaultkey, const char *masterDir, const char *localDir, const char *ecmfDir, int flags, int nofail);
int get_concept_condition_string(grib_handle* h, const char* key, const char* value, char* result);
/* action_class_hash_array.c */
grib_action *grib_action_create_hash_array(grib_context *context, const char *name, grib_hash_array_value *hash_array, const char *basename, const char *name_space, const char *defaultkey, const char *masterDir, const char *localDir, const char *ecmfDir, int flags, int nofail);
@ -1483,6 +1484,7 @@ int grib2_is_PDTN_AerosolOptical(long productDefinitionTemplateNumber);
int grib2_select_PDTN(int is_eps, int is_instant, int is_chemical, int is_chemical_distfn, int is_aerosol, int is_aerosol_optical);
int is_grib_index_file(const char *filename);
size_t sum_of_pl_array(const long* pl, size_t plsize);
int grib_util_grib_data_quality_check(grib_handle* h, double min_val, double max_val);
/* bufr_util.c */
int compute_bufr_key_rank(grib_handle *h, grib_string_list *keys, const char *key);

33
src/grib_context.c
View File

@ -308,13 +308,13 @@ static grib_context default_grib_context = {
&default_seek, /* lfile seek procedure */
&default_feof, /* file feof procedure */
&default_log, /* logging_procedure */
&default_print, /* print procedure */
0, /* grib_codetable* */
0, /* grib_smart_table* */
0, /* char* outfilename */
0, /* int multi_support_on */
0, /* grib_multi_support* multi_support*/
&default_log, /* output_log */
&default_print, /* print */
0, /* codetable */
0, /* smart_table */
0, /* outfilename */
0, /* multi_support_on */
0, /* multi_support */
0, /* grib_definition_files_dir */
0, /* handle_file_count */
0, /* handle_total_count */
@ -323,9 +323,9 @@ static grib_context default_grib_context = {
0, /* gts_header_on */
0, /* gribex_mode_on */
0, /* large_constant_fields */
0, /* grib_itrie* keys */
0, /* keys */
0, /* keys_count */
0, /* grib_itrie* concepts_index */
0, /* concepts_index */
0, /* concepts_count */
{0,}, /* concepts */
0, /* hash_array_index */
@ -337,6 +337,7 @@ static grib_context default_grib_context = {
0, /* bufrdc_mode */
0, /* bufr_set_to_missing_if_out_of_range */
0, /* bufr_multi_element_constant_arrays */
0, /* grib_data_quality_checks */
0, /* log_stream */
0, /* classes */
0, /* lists */
@ -371,6 +372,7 @@ grib_context* grib_context_get_default()
const char* bufrdc_mode = NULL;
const char* bufr_set_to_missing_if_out_of_range = NULL;
const char* bufr_multi_element_constant_arrays = NULL;
const char* grib_data_quality_checks = NULL;
const char* file_pool_max_opened_files = NULL;
#ifdef ENABLE_FLOATING_POINT_EXCEPTIONS
@ -381,6 +383,7 @@ grib_context* grib_context_get_default()
bufrdc_mode = getenv("ECCODES_BUFRDC_MODE_ON");
bufr_set_to_missing_if_out_of_range = getenv("ECCODES_BUFR_SET_TO_MISSING_IF_OUT_OF_RANGE");
bufr_multi_element_constant_arrays = getenv("ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS");
grib_data_quality_checks = getenv("ECCODES_GRIB_DATA_QUALITY_CHECKS");
large_constant_fields = codes_getenv("ECCODES_GRIB_LARGE_CONSTANT_FIELDS");
no_abort = codes_getenv("ECCODES_NO_ABORT");
debug = codes_getenv("ECCODES_DEBUG");
@ -458,6 +461,16 @@ grib_context* grib_context_get_default()
}
}
/* Definitions path extra: Added at the head of (i.e. before) existing path */
{
const char* defs_extra = getenv("ECCODES_EXTRA_DEFINITION_PATH");
if (defs_extra) {
char buffer[DEF_PATH_MAXLEN];
ecc_snprintf(buffer, DEF_PATH_MAXLEN, "%s:%s", defs_extra, default_grib_context.grib_definition_files_path);
default_grib_context.grib_definition_files_path = strdup(buffer);
}
}
grib_context_log(&default_grib_context, GRIB_LOG_DEBUG, "Definitions path: %s",
default_grib_context.grib_definition_files_path);
grib_context_log(&default_grib_context, GRIB_LOG_DEBUG, "Samples path: %s",
@ -479,6 +492,8 @@ grib_context* grib_context_get_default()
atoi(bufr_set_to_missing_if_out_of_range) : 0;
default_grib_context.bufr_multi_element_constant_arrays = bufr_multi_element_constant_arrays ?
atoi(bufr_multi_element_constant_arrays) : 0;
default_grib_context.grib_data_quality_checks = grib_data_quality_checks ?
atoi(grib_data_quality_checks) : 0;
default_grib_context.file_pool_max_opened_files = file_pool_max_opened_files ?
atoi(file_pool_max_opened_files) : DEFAULT_FILE_POOL_MAX_OPENED_FILES;
}

64
src/grib_util.c
View File

@ -2048,3 +2048,67 @@ size_t sum_of_pl_array(const long* pl, size_t plsize)
}
return count;
}
int grib_util_grib_data_quality_check(grib_handle* h, double min_val, double max_val)
{
int err = 0;
long min_field_value_allowed=0, max_field_value_allowed=0;
long paramId = 0;
double dmin_allowed=0, dmax_allowed=0;
grib_context* ctx = h->context;
int is_error = 1;
/*
* If grib_data_quality_checks == 1, limits failure results in an error
* If grib_data_quality_checks == 2, limits failure results in a warning
*/
Assert( ctx->grib_data_quality_checks == 1 || ctx->grib_data_quality_checks == 2 );
is_error = (ctx->grib_data_quality_checks == 1);
/* The limit keys must exist if we are here */
err = grib_get_long(h, "param_value_min", &min_field_value_allowed);
if (err) {
grib_context_log(ctx, GRIB_LOG_ERROR,"grib_data_quality_check: Could not get param_value_min");
return err;
}
err = grib_get_long(h, "param_value_max", &max_field_value_allowed);
if (err) {
grib_context_log(ctx, GRIB_LOG_ERROR,"grib_data_quality_check: Could not get param_value_max");
return err;
}
dmin_allowed = (double)min_field_value_allowed;
dmax_allowed = (double)max_field_value_allowed;
if (min_val < dmin_allowed) {
char description[1024] = {0,};
if (get_concept_condition_string(h, "param_value_min", NULL, description)==GRIB_SUCCESS) {
fprintf(stderr, "ECCODES %s : (%s): minimum (%g) is less than the allowable limit (%g)\n",
(is_error? "ERROR":"WARNING"), description, min_val, dmin_allowed);
} else {
if (grib_get_long(h, "paramId", &paramId) == GRIB_SUCCESS) {
fprintf(stderr, "ECCODES %s : (paramId=%ld): minimum (%g) is less than the default allowable limit (%g)\n",
(is_error? "ERROR":"WARNING"), paramId, min_val, dmin_allowed);
}
}
if (is_error) {
return GRIB_OUT_OF_RANGE; /* Failure */
}
}
if (max_val > dmax_allowed) {
char description[1024] = {0,};
if (get_concept_condition_string(h, "param_value_max", NULL, description)==GRIB_SUCCESS) {
fprintf(stderr, "ECCODES %s : (%s): maximum (%g) is more than the allowable limit (%g)\n",
(is_error? "ERROR":"WARNING"), description, max_val, dmax_allowed);
} else {
if (grib_get_long(h, "paramId", &paramId) == GRIB_SUCCESS) {
fprintf(stderr, "ECCODES %s : (paramId=%ld): maximum (%g) is more than the default allowable limit (%g)\n",
(is_error? "ERROR":"WARNING"), paramId, max_val, dmax_allowed);
}
}
if (is_error) {
return GRIB_OUT_OF_RANGE; /* Failure */
}
}
return GRIB_SUCCESS;
}

1
tests/CMakeLists.txt
View File

@ -70,6 +70,7 @@ list( APPEND tests_no_data_reqd
)
# These tests do require data downloads
list( APPEND tests_data_reqd
grib_data_quality_checks
bpv_limit
grib_complex
grib_double_cmp

96
tests/grib_data_quality_checks.sh Executable file
View File

@ -0,0 +1,96 @@
#!/bin/sh
# Copyright 2005-2019 ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
#
. ./include.sh
set -u
# ---------------------------------------------------------
# Tests for data quality checks
# ---------------------------------------------------------
label="grib_data_quality"
tempOut=temp.1.${label}.out
temp2=temp.2.${label}.out
tempErr=temp.${label}.err
# Start with clean environment
unset ECCODES_GRIB_DATA_QUALITY_CHECKS
unset ECCODES_EXTRA_DEFINITION_PATH
input1=${data_dir}/reduced_gaussian_surface.grib1
input2=${data_dir}/reduced_gaussian_surface.grib2
grib_check_key_equals $input1 paramId 167
grib_check_key_equals $input2 paramId 167
# Data quality checks disabled. Create huge values for temperature
${tools_dir}/grib_set -s scaleValuesBy=100 $input1 $tempOut
${tools_dir}/grib_set -s scaleValuesBy=100 $input2 $tempOut
# Data quality checks enabled. Repacking should fail
export ECCODES_GRIB_DATA_QUALITY_CHECKS=1
set +e
${tools_dir}/grib_copy -r $tempOut /dev/null 2>$tempErr
status=$?
set -e
[ $status -ne 0 ]
grep -q 'more than the allowable limit' $tempErr
# Data quality checks enabled but only as a warning. Repacking should pass
export ECCODES_GRIB_DATA_QUALITY_CHECKS=2
${tools_dir}/grib_copy -r $tempOut /dev/null 2>$tempErr
grep -q 'more than the allowable limit' $tempErr
# Data quality checks enabled. Scaling should fail
export ECCODES_GRIB_DATA_QUALITY_CHECKS=1
set +e
${tools_dir}/grib_set -s scaleValuesBy=100 $input1 $tempOut 2>$tempErr
status=$?
set -e
[ $status -ne 0 ]
grep -q 'GRIB1 simple packing: unable to set values' $tempErr
grep -q 'allowable limit' $tempErr
set +e
${tools_dir}/grib_set -s scaleValuesBy=100 $input2 $tempOut 2>$tempErr
status=$?
set -e
[ $status -ne 0 ]
grep -q 'GRIB2 simple packing: unable to set values' $tempErr
grep -q 'allowable limit' $tempErr
# Override the defaults
# ----------------------
tempDir=tempdir.$label
rm -rf $tempDir
mkdir -p $tempDir
# Set a large limit for temperature
cat > $tempDir/param_limits.def <<EOF
constant default_min_val = -1e9 : long_type, hidden;
constant default_max_val = +1e9 : long_type, hidden;
concept param_value_min(default_min_val) {
0 = { paramId=167; }
} : long_type, hidden;
concept param_value_max(default_max_val) {
40000 = { paramId=167; }
} : long_type, hidden;
EOF
# Command should succeed
export ECCODES_GRIB_DATA_QUALITY_CHECKS=1
export ECCODES_EXTRA_DEFINITION_PATH=$test_dir/$tempDir
${tools_dir}/grib_set -s scaleValuesBy=100 $input1 $tempOut
# Clean up
rm -rf $tempDir
rm -f $tempOut $tempErr

27
tests/unit_tests.c
View File

@ -1425,10 +1425,37 @@ static void test_assertion_catching()
assertion_caught = 0;
}
static void test_concept_condition_strings()
{
int err = 0;
char result[1024] = {0,};
grib_handle* h = grib_handle_new_from_samples(0, "GRIB2");
err = get_concept_condition_string(h, "typeOfLevel", NULL, result);
assert ( !err );
assert( strcmp(result, "typeOfFirstFixedSurface=1,typeOfSecondFixedSurface=255")==0 );
err = get_concept_condition_string(h, "paramId", NULL, result);
assert ( !err );
assert( strcmp(result, "discipline=0,parameterCategory=0,parameterNumber=0")==0 );
err = get_concept_condition_string(h, "gridType", NULL, result);
assert ( !err );
/*printf("%s\n", result);*/
assert( strcmp(result, "gridDefinitionTemplateNumber=0,PLPresent=0")==0 );
err = get_concept_condition_string(h, "stepType", NULL, result);
assert ( !err );
assert( strcmp(result, "selectStepTemplateInstant=1,stepTypeInternal=instant")==0 );
}
int main(int argc, char** argv)
{
/*printf("Doing unit tests. ecCodes version = %ld\n", grib_get_api_version());*/
test_concept_condition_strings();
test_assertion_catching();
test_gaussian_latitude_640();

6
tools/codes_info.c
View File

@ -35,6 +35,7 @@ int main( int argc,char* argv[])
int major=ECCODES_MAJOR_VERSION;
int minor=ECCODES_MINOR_VERSION;
int revision=ECCODES_REVISION_VERSION;
grib_context* context = grib_context_get_default();
while (1) {
int c = getopt (argc, argv, "vds");
@ -61,7 +62,6 @@ int main( int argc,char* argv[])
if (nfiles != 0) usage_and_exit(argv[0]);
if (print_flags == INFO_PRINT_ALL) {
grib_context* context = grib_context_get_default();
printf("\n");
printf("%s Version %d.%d.%d",
grib_get_package_name(), major,minor,revision);
@ -90,7 +90,7 @@ int main( int argc,char* argv[])
printf("(This is for backward compatibility. "
"It is recommended you use ECCODES_DEFINITION_PATH instead!)\n");
} else {
printf("Default definition files path is used: %s\n",ECCODES_DEFINITION_PATH);
printf("Default definition files path is used: %s\n",context->grib_definition_files_path);
printf("Definition files path can be changed by setting ECCODES_DEFINITION_PATH environment variable\n");
}
printf("\n");
@ -118,7 +118,7 @@ int main( int argc,char* argv[])
if ((path=codes_getenv("ECCODES_DEFINITION_PATH")) != NULL) {
printf("%s",path);
} else {
printf("%s",ECCODES_DEFINITION_PATH);
printf("%s",context->grib_definition_files_path);
}
}