ECC-992: GRIB encoding: Data quality checks (limits)

definitions/boot.def

@@ -15,6 +15,11 @@ UseEcmfConventions = getenv("ECCODES_USE_ECMF_CONVENTIONS","1") :hidden ;
 
 constant defaultTypeOfLevel="unknown" : hidden;
 
+gribDataQualityChecks = getenv("ECCODES_GRIB_DATA_QUALITY_CHECKS","0") : hidden;
+if (gribDataQualityChecks) {
+    template LIMITS "param_limits.def";
+}
+
 # GRIBEX special boustrophedonic mode. See GRIB-472
 # If the environment variable is not defined, the key will be 0
 GRIBEX_boustrophedonic = getenv("ECCODES_GRIBEX_BOUSTROPHEDONIC","0")  :hidden;

definitions/param_limits.def

@@ -0,0 +1,9 @@
+constant default_min_val = -1e9 : long_type, hidden;
+constant default_max_val = +1e9 : long_type, hidden;
+
+concept param_value_min(default_min_val) {
+    0  = { paramId=167; }
+} : long_type, hidden;
+concept param_value_max(default_max_val) {
+    373 = { paramId=167; }
+} : long_type, hidden;

src/action_class_concept.c

@@ -305,3 +305,97 @@ static grib_concept_value* get_concept(grib_handle* h, grib_action_concept* self
     GRIB_MUTEX_UNLOCK(&mutex);
     return result;
 }
+
+static int concept_condition_expression_true(grib_handle* h, grib_concept_condition* c, char* exprVal)
+{
+    long lval;
+    long lres=0;
+    int ok = 0;
+    int err=0;
+    const int type = grib_expression_native_type(h,c->expression);
+
+    switch(type)
+    {
+    case GRIB_TYPE_LONG:
+        grib_expression_evaluate_long(h,c->expression,&lres);
+        ok =  (grib_get_long(h,c->name,&lval) == GRIB_SUCCESS) &&
+                (lval == lres);
+        if (ok) sprintf(exprVal, "%ld", lres);
+        break;
+
+    case GRIB_TYPE_DOUBLE: {
+        double dval;
+        double dres=0.0;
+        grib_expression_evaluate_double(h,c->expression,&dres);
+        ok = (grib_get_double(h,c->name,&dval) == GRIB_SUCCESS) &&
+                (dval == dres);
+        if (ok) sprintf(exprVal, "%g", dres);
+        break;
+    }
+
+    case GRIB_TYPE_STRING: {
+        const char *cval;
+        char buf[80];
+        char tmp[80];
+        size_t len = sizeof(buf);
+        size_t size=sizeof(tmp);
+
+        ok = (grib_get_string(h,c->name,buf,&len) == GRIB_SUCCESS) &&
+        ((cval = grib_expression_evaluate_string(h,c->expression,tmp,&size,&err)) != NULL) &&
+        (err==0) && (strcmp(buf,cval) == 0);
+        if (ok) sprintf(exprVal, "%s", cval);
+        break;
+    }
+
+    default:
+        /* TODO: */
+        break;
+    }
+    return ok;
+}
+
+/* Caller has to allocate space for the result.
+ * INPUTS: h, key and value (can be NULL)
+ * OUTPUT: result
+ * Example: key='typeOfLevel' whose value is 'mixedLayerDepth',
+ * result='typeOfFirstFixedSurface=169,typeOfSecondFixedSurface=255'
+ */
+int get_concept_condition_string(grib_handle* h, const char* key, const char* value, char* result)
+{
+    int err = 0;
+    int length = 0;
+    char strVal[64]={0,};
+    char exprVal[256]={0,};
+    const char* pValue = value;
+    size_t len = sizeof(strVal);
+    grib_concept_value* concept_value = NULL;
+    grib_accessor* acc = grib_find_accessor(h, key);
+    if (!acc) return GRIB_NOT_FOUND;
+
+    if (!value) {
+        err = grib_get_string(h, key, strVal,&len);
+        if (err) return GRIB_INTERNAL_ERROR;
+        pValue = strVal;
+    }
+
+    concept_value = action_concept_get_concept(acc);
+    while (concept_value) {
+        grib_concept_condition* concept_condition = concept_value->conditions;
+
+        if (strcmp(pValue, concept_value->name)==0) {
+            while (concept_condition) {
+                grib_expression* expression = concept_condition->expression;
+                Assert(expression);
+                if (concept_condition_expression_true(h, concept_condition, exprVal)) {
+                    length += sprintf(result+length, "%s%s=%s",
+                                    (length==0?"":","),concept_condition->name, exprVal);
+                }
+                concept_condition = concept_condition->next;
+            }
+        }
+
+        concept_value = concept_value->next;
+    }
+    if (length == 0) return GRIB_CONCEPT_NO_MATCH;
+    return GRIB_SUCCESS;
+}

src/grib_accessor_class_concept.c

@@ -615,6 +615,14 @@ static int unpack_string (grib_accessor* a, char* val, size_t *len)
     }
     strcpy(val,p);
     *len = slen;
+#if 0
+    if (a->context->debug==1) {
+        int err = 0;
+        char result[1024] = {0,};
+        err = get_concept_condition_string(grib_handle_of_accessor(a), a->name, val, result);
+        if (!err) fprintf(stderr, "ECCODES DEBUG concept name=%s, value=%s, conditions=%s\n", a->name, val, result);
+    }
+#endif
     return GRIB_SUCCESS;
 }
 

src/grib_accessor_class_data_simple_packing.c

@@ -505,12 +505,26 @@ static int producing_large_constant_fields(const grib_context* c, grib_handle* h
 }
 #endif
 
-static int check_range(const double val)
+static int check_range(grib_handle* h, const double min_val, const double max_val)
 {
-    if (val < DBL_MAX && val > -DBL_MAX)
+    int result = GRIB_SUCCESS;
-        return GRIB_SUCCESS;
+    grib_context* ctx = h->context;
-    else
+    
+    if ( !(min_val < DBL_MAX && min_val > -DBL_MAX) ) {
+        grib_context_log(ctx, GRIB_LOG_ERROR, "Minimum value out of range: %g", min_val);
         return GRIB_ENCODING_ERROR;
+    }
+    if ( !(max_val < DBL_MAX && max_val > -DBL_MAX) ) {
+        grib_context_log(ctx, GRIB_LOG_ERROR, "Maximum value out of range: %g", max_val);
+        return GRIB_ENCODING_ERROR;
+    }
+
+    /* Data Quality checks */
+    if (ctx->grib_data_quality_checks) {
+        result = grib_util_grib_data_quality_check(h, min_val, max_val);
+    }
+
+    return result;
 }
 
 static int pack_double(grib_accessor* a, const double* val, size_t *len)
@@ -577,12 +591,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
         else if (val[i] < min ) min = val[i];
     }
 #endif
-    if ((err = check_range(max)) != GRIB_SUCCESS) {
+    if ((err = check_range(gh, min, max)) != GRIB_SUCCESS) {
-        grib_context_log(a->context,GRIB_LOG_ERROR,"Maximum value out of range: %g", max);
-        return err;
-    }
-    if ((err = check_range(min)) != GRIB_SUCCESS) {
-        grib_context_log(a->context,GRIB_LOG_ERROR,"Minimum value out of range: %g", min);
         return err;
     }
 

src/grib_api_internal.h

@@ -1056,6 +1056,7 @@ struct grib_context
     int                             bufrdc_mode;
     int                             bufr_set_to_missing_if_out_of_range;
     int                             bufr_multi_element_constant_arrays;
+    int                             grib_data_quality_checks;
     FILE*                           log_stream;
     grib_trie*                      classes;
     grib_trie*                      lists;

src/grib_api_prototypes.h

@@ -67,6 +67,7 @@ grib_action *grib_action_create_when(grib_context *context, grib_expression *exp
 grib_concept_value *action_concept_get_concept(grib_accessor *a);
 int action_concept_get_nofail(grib_accessor *a);
 grib_action *grib_action_create_concept(grib_context *context, const char *name, grib_concept_value *concept, const char *basename, const char *name_space, const char *defaultkey, const char *masterDir, const char *localDir, const char *ecmfDir, int flags, int nofail);
+int get_concept_condition_string(grib_handle* h, const char* key, const char* value, char* result);
 
 /* action_class_hash_array.c */
 grib_action *grib_action_create_hash_array(grib_context *context, const char *name, grib_hash_array_value *hash_array, const char *basename, const char *name_space, const char *defaultkey, const char *masterDir, const char *localDir, const char *ecmfDir, int flags, int nofail);
@@ -1483,6 +1484,7 @@ int grib2_is_PDTN_AerosolOptical(long productDefinitionTemplateNumber);
 int grib2_select_PDTN(int is_eps, int is_instant, int is_chemical, int is_chemical_distfn, int is_aerosol, int is_aerosol_optical);
 int is_grib_index_file(const char *filename);
 size_t sum_of_pl_array(const long* pl, size_t plsize);
+int grib_util_grib_data_quality_check(grib_handle* h, double min_val, double max_val);
 
 /* bufr_util.c */
 int compute_bufr_key_rank(grib_handle *h, grib_string_list *keys, const char *key);

src/grib_context.c

@@ -308,13 +308,13 @@ static grib_context default_grib_context = {
         &default_seek,                /* lfile seek procedure       */
         &default_feof,                /* file feof procedure        */
 
-        &default_log,                 /* logging_procedure          */
+        &default_log,                 /* output_log                 */
-        &default_print,               /* print procedure            */
+        &default_print,               /* print                      */
-        0,                            /* grib_codetable*            */
+        0,                            /* codetable                  */
-        0,                            /* grib_smart_table*          */
+        0,                            /* smart_table                */
-        0,                            /* char* outfilename          */
+        0,                            /* outfilename                */
-        0,                            /* int multi_support_on       */
+        0,                            /* multi_support_on           */
-        0,                            /* grib_multi_support* multi_support*/
+        0,                            /* multi_support              */
         0,                            /* grib_definition_files_dir  */
         0,                            /* handle_file_count          */
         0,                            /* handle_total_count         */
@@ -323,9 +323,9 @@ static grib_context default_grib_context = {
         0,                            /* gts_header_on              */
         0,                            /* gribex_mode_on             */
         0,                            /* large_constant_fields      */
-        0,                            /* grib_itrie* keys           */
+        0,                            /* keys                       */
         0,                            /* keys_count                 */
-        0,                            /* grib_itrie* concepts_index */
+        0,                            /* concepts_index             */
         0,                            /* concepts_count             */
         {0,},                         /* concepts                   */
         0,                            /* hash_array_index           */
@@ -337,6 +337,7 @@ static grib_context default_grib_context = {
         0,                            /* bufrdc_mode                */
         0,                            /* bufr_set_to_missing_if_out_of_range */
         0,                            /* bufr_multi_element_constant_arrays */
+        0,                            /* grib_data_quality_checks   */
         0,                            /* log_stream                 */
         0,                            /* classes                    */
         0,                            /* lists                      */
@@ -371,6 +372,7 @@ grib_context* grib_context_get_default()
         const char* bufrdc_mode = NULL;
         const char* bufr_set_to_missing_if_out_of_range = NULL;
         const char* bufr_multi_element_constant_arrays = NULL;
+        const char* grib_data_quality_checks = NULL;
         const char* file_pool_max_opened_files = NULL;
 
 #ifdef ENABLE_FLOATING_POINT_EXCEPTIONS
@@ -381,6 +383,7 @@ grib_context* grib_context_get_default()
         bufrdc_mode = getenv("ECCODES_BUFRDC_MODE_ON");
         bufr_set_to_missing_if_out_of_range = getenv("ECCODES_BUFR_SET_TO_MISSING_IF_OUT_OF_RANGE");
         bufr_multi_element_constant_arrays = getenv("ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS");
+        grib_data_quality_checks = getenv("ECCODES_GRIB_DATA_QUALITY_CHECKS");
         large_constant_fields = codes_getenv("ECCODES_GRIB_LARGE_CONSTANT_FIELDS");
         no_abort = codes_getenv("ECCODES_NO_ABORT");
         debug = codes_getenv("ECCODES_DEBUG");
@@ -458,6 +461,16 @@ grib_context* grib_context_get_default()
             }
         }
 
+        /* Definitions path extra: Added at the head of (i.e. before) existing path */
+        {
+            const char* defs_extra = getenv("ECCODES_EXTRA_DEFINITION_PATH");
+            if (defs_extra) {
+                char buffer[DEF_PATH_MAXLEN];
+                ecc_snprintf(buffer, DEF_PATH_MAXLEN, "%s:%s", defs_extra, default_grib_context.grib_definition_files_path);
+                default_grib_context.grib_definition_files_path = strdup(buffer);
+            }
+        }
+
         grib_context_log(&default_grib_context, GRIB_LOG_DEBUG, "Definitions path: %s",
                 default_grib_context.grib_definition_files_path);
         grib_context_log(&default_grib_context, GRIB_LOG_DEBUG, "Samples path:     %s",
@@ -479,6 +492,8 @@ grib_context* grib_context_get_default()
                 atoi(bufr_set_to_missing_if_out_of_range) : 0;
         default_grib_context.bufr_multi_element_constant_arrays = bufr_multi_element_constant_arrays ?
                 atoi(bufr_multi_element_constant_arrays) : 0;
+        default_grib_context.grib_data_quality_checks = grib_data_quality_checks ?
+                atoi(grib_data_quality_checks) : 0;
         default_grib_context.file_pool_max_opened_files = file_pool_max_opened_files ?
                 atoi(file_pool_max_opened_files) : DEFAULT_FILE_POOL_MAX_OPENED_FILES;
     }

src/grib_util.c

@@ -2048,3 +2048,67 @@ size_t sum_of_pl_array(const long* pl, size_t plsize)
     }
     return count;
 }
+
+int grib_util_grib_data_quality_check(grib_handle* h, double min_val, double max_val)
+{
+    int err = 0;
+    long min_field_value_allowed=0, max_field_value_allowed=0;
+    long paramId = 0;
+    double dmin_allowed=0, dmax_allowed=0;
+    grib_context* ctx = h->context;
+    int is_error = 1;
+    /*
+     * If grib_data_quality_checks == 1, limits failure results in an error
+     * If grib_data_quality_checks == 2, limits failure results in a warning
+     */
+    Assert( ctx->grib_data_quality_checks == 1 || ctx->grib_data_quality_checks == 2 );
+    is_error = (ctx->grib_data_quality_checks == 1);
+
+    /* The limit keys must exist if we are here */
+    err = grib_get_long(h, "param_value_min", &min_field_value_allowed);
+    if (err) {
+        grib_context_log(ctx, GRIB_LOG_ERROR,"grib_data_quality_check: Could not get param_value_min");
+        return err;
+    }
+    err = grib_get_long(h, "param_value_max", &max_field_value_allowed);
+    if (err) {
+        grib_context_log(ctx, GRIB_LOG_ERROR,"grib_data_quality_check: Could not get param_value_max");
+        return err;
+    }
+
+    dmin_allowed = (double)min_field_value_allowed;
+    dmax_allowed = (double)max_field_value_allowed;
+
+    if (min_val < dmin_allowed) {
+        char description[1024] = {0,};
+        if (get_concept_condition_string(h, "param_value_min", NULL, description)==GRIB_SUCCESS) {
+            fprintf(stderr, "ECCODES %s   :  (%s): minimum (%g) is less than the allowable limit (%g)\n",
+                             (is_error? "ERROR":"WARNING"), description, min_val, dmin_allowed);
+        } else {
+            if (grib_get_long(h, "paramId", &paramId) == GRIB_SUCCESS) {
+                fprintf(stderr, "ECCODES %s   :  (paramId=%ld): minimum (%g) is less than the default allowable limit (%g)\n",
+                             (is_error? "ERROR":"WARNING"), paramId, min_val, dmin_allowed);
+            }
+        }
+        if (is_error) {
+            return GRIB_OUT_OF_RANGE; /* Failure */
+        }
+    }
+    if (max_val > dmax_allowed) {
+        char description[1024] = {0,};
+        if (get_concept_condition_string(h, "param_value_max", NULL, description)==GRIB_SUCCESS) {
+            fprintf(stderr, "ECCODES %s   :  (%s): maximum (%g) is more than the allowable limit (%g)\n",
+                             (is_error? "ERROR":"WARNING"), description, max_val, dmax_allowed);
+        } else {
+            if (grib_get_long(h, "paramId", &paramId) == GRIB_SUCCESS) {
+                fprintf(stderr, "ECCODES %s   :  (paramId=%ld): maximum (%g) is more than the default allowable limit (%g)\n",
+                             (is_error? "ERROR":"WARNING"), paramId, max_val, dmax_allowed);
+            }
+        }
+        if (is_error) {
+            return GRIB_OUT_OF_RANGE; /* Failure */
+        }
+    }
+
+    return GRIB_SUCCESS;
+}

tests/CMakeLists.txt

@@ -70,6 +70,7 @@ list( APPEND tests_no_data_reqd
 )
 # These tests do require data downloads
 list( APPEND tests_data_reqd
+    grib_data_quality_checks
     bpv_limit
     grib_complex
     grib_double_cmp

tests/grib_data_quality_checks.sh

@@ -0,0 +1,96 @@
+#!/bin/sh
+# Copyright 2005-2019 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+set -u
+# ---------------------------------------------------------
+# Tests for data quality checks
+# ---------------------------------------------------------
+label="grib_data_quality"
+tempOut=temp.1.${label}.out
+temp2=temp.2.${label}.out
+tempErr=temp.${label}.err
+
+# Start with clean environment
+unset ECCODES_GRIB_DATA_QUALITY_CHECKS
+unset ECCODES_EXTRA_DEFINITION_PATH
+
+
+input1=${data_dir}/reduced_gaussian_surface.grib1
+input2=${data_dir}/reduced_gaussian_surface.grib2
+grib_check_key_equals $input1 paramId 167
+grib_check_key_equals $input2 paramId 167
+
+# Data quality checks disabled. Create huge values for temperature
+${tools_dir}/grib_set -s scaleValuesBy=100 $input1 $tempOut
+${tools_dir}/grib_set -s scaleValuesBy=100 $input2 $tempOut
+
+# Data quality checks enabled. Repacking should fail
+export ECCODES_GRIB_DATA_QUALITY_CHECKS=1
+set +e
+${tools_dir}/grib_copy -r $tempOut /dev/null  2>$tempErr
+status=$?
+set -e
+[ $status -ne 0 ]
+grep -q 'more than the allowable limit' $tempErr
+
+
+# Data quality checks enabled but only as a warning. Repacking should pass
+export ECCODES_GRIB_DATA_QUALITY_CHECKS=2
+${tools_dir}/grib_copy -r $tempOut /dev/null  2>$tempErr
+grep -q 'more than the allowable limit' $tempErr
+
+
+# Data quality checks enabled. Scaling should fail
+export ECCODES_GRIB_DATA_QUALITY_CHECKS=1
+set +e
+${tools_dir}/grib_set -s scaleValuesBy=100 $input1 $tempOut 2>$tempErr
+status=$?
+set -e
+[ $status -ne 0 ]
+grep -q 'GRIB1 simple packing: unable to set values' $tempErr
+grep -q 'allowable limit' $tempErr
+
+set +e
+${tools_dir}/grib_set -s scaleValuesBy=100 $input2 $tempOut 2>$tempErr
+status=$?
+set -e
+[ $status -ne 0 ]
+grep -q 'GRIB2 simple packing: unable to set values' $tempErr
+grep -q 'allowable limit' $tempErr
+
+
+# Override the defaults
+# ----------------------
+tempDir=tempdir.$label
+rm -rf $tempDir
+mkdir -p $tempDir
+# Set a large limit for temperature
+cat > $tempDir/param_limits.def <<EOF
+ constant default_min_val = -1e9 : long_type, hidden;
+ constant default_max_val = +1e9 : long_type, hidden;
+ concept param_value_min(default_min_val) {
+    0  = { paramId=167; }
+ } : long_type, hidden;
+ concept param_value_max(default_max_val) {
+    40000 = { paramId=167; }
+ } : long_type, hidden;
+EOF
+
+# Command should succeed
+export ECCODES_GRIB_DATA_QUALITY_CHECKS=1
+export ECCODES_EXTRA_DEFINITION_PATH=$test_dir/$tempDir
+${tools_dir}/grib_set -s scaleValuesBy=100 $input1 $tempOut
+
+
+
+# Clean up
+rm -rf $tempDir
+rm -f $tempOut $tempErr

tests/unit_tests.c

@@ -1425,10 +1425,37 @@ static void test_assertion_catching()
     assertion_caught = 0;
 }
 
+static void test_concept_condition_strings()
+{
+    int err = 0;
+    char result[1024] = {0,};
+    grib_handle* h = grib_handle_new_from_samples(0, "GRIB2");
+
+    err = get_concept_condition_string(h, "typeOfLevel", NULL, result);
+    assert ( !err );
+    assert( strcmp(result, "typeOfFirstFixedSurface=1,typeOfSecondFixedSurface=255")==0 );
+
+    err = get_concept_condition_string(h, "paramId", NULL, result);
+    assert ( !err );
+    assert( strcmp(result, "discipline=0,parameterCategory=0,parameterNumber=0")==0 );
+
+    err = get_concept_condition_string(h, "gridType", NULL, result);
+    assert ( !err );
+    /*printf("%s\n", result);*/
+    assert( strcmp(result, "gridDefinitionTemplateNumber=0,PLPresent=0")==0 );
+
+    err = get_concept_condition_string(h, "stepType", NULL, result);
+    assert ( !err );
+    assert( strcmp(result, "selectStepTemplateInstant=1,stepTypeInternal=instant")==0 );
+
+}
+
 int main(int argc, char** argv)
 {
     /*printf("Doing unit tests. ecCodes version = %ld\n", grib_get_api_version());*/
 
+    test_concept_condition_strings();
+
     test_assertion_catching();
 
     test_gaussian_latitude_640();

tools/codes_info.c

@@ -35,6 +35,7 @@ int main( int argc,char* argv[])
     int major=ECCODES_MAJOR_VERSION;
     int minor=ECCODES_MINOR_VERSION;
     int revision=ECCODES_REVISION_VERSION;
+    grib_context* context = grib_context_get_default();
 
     while (1) {
         int c = getopt (argc, argv, "vds");
@@ -61,7 +62,6 @@ int main( int argc,char* argv[])
     if (nfiles != 0) usage_and_exit(argv[0]);
 
     if (print_flags ==  INFO_PRINT_ALL) {
-        grib_context* context = grib_context_get_default();
         printf("\n");
         printf("%s Version %d.%d.%d",
                 grib_get_package_name(), major,minor,revision);
@@ -90,7 +90,7 @@ int main( int argc,char* argv[])
             printf("(This is for backward compatibility. "
                    "It is recommended you use ECCODES_DEFINITION_PATH instead!)\n");
         } else {
-            printf("Default definition files path is used: %s\n",ECCODES_DEFINITION_PATH);
+            printf("Default definition files path is used: %s\n",context->grib_definition_files_path);
             printf("Definition files path can be changed by setting ECCODES_DEFINITION_PATH environment variable\n");
         }
         printf("\n");
@@ -118,7 +118,7 @@ int main( int argc,char* argv[])
         if ((path=codes_getenv("ECCODES_DEFINITION_PATH")) != NULL) {
             printf("%s",path);
         } else {
-            printf("%s",ECCODES_DEFINITION_PATH);
+            printf("%s",context->grib_definition_files_path);
         }
     }