core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECC-992: GRIB encoding: Data Quality Checks (initial attempt)

This commit is contained in:
Shahram Najm 2019-10-10 18:21:05 +01:00
parent be02024939
commit a918e9fa6e
9 changed files with 59 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
examples/C/large_grib1.c
View File

@ -29,8 +29,8 @@ int main()
}
for (i=0; i<ni*nj; i++) {
double r = rand();
values[i] = 10*r;
double r = rand() * 1.0 / RAND_MAX;
values[i] = r;
}
h = codes_grib_handle_new_from_samples(0, "GRIB1");

2
src/grib_accessor_class_data_g1simple_packing.c
View File

@ -283,7 +283,7 @@ static int pack_double(grib_accessor* a, const double* cval, size_t *len)
case GRIB_SUCCESS:
break;
default:
grib_context_log(a->context,GRIB_LOG_ERROR,"unable to compute packing parameters\n");
grib_context_log(a->context,GRIB_LOG_ERROR,"GRIB1 simple packing: unable to set values");
return ret;
}

2
src/grib_accessor_class_data_g2simple_packing.c
View File

@ -218,7 +218,7 @@ static int pack_double(grib_accessor* a, const double* cval, size_t *len)
case GRIB_SUCCESS:
break;
default:
grib_context_log(a->context,GRIB_LOG_ERROR,"unable to compute packing parameters\n");
grib_context_log(a->context,GRIB_LOG_ERROR,"GRIB2 simple packing: unable to set values");
return ret;
}

20
src/grib_accessor_class_data_simple_packing.c
View File

@ -505,12 +505,22 @@ static int producing_large_constant_fields(const grib_context* c, grib_handle* h
}
#endif
static int check_range(const double val)
static int check_range(grib_handle* h, const double val)
{
int result = GRIB_SUCCESS;
grib_context* ctx = h->context;
if (val < DBL_MAX && val > -DBL_MAX)
return GRIB_SUCCESS;
result = GRIB_SUCCESS;
else
return GRIB_ENCODING_ERROR;
result = GRIB_ENCODING_ERROR;
/* Data Quality checks */
if (ctx->grib_data_quality_checks && result == GRIB_SUCCESS) {
/*TODO: get limits for the current parameter*/
result = grib_util_grib_data_quality_check(h, val);
}
return result;
}
static int pack_double(grib_accessor* a, const double* val, size_t *len)
@ -577,11 +587,11 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
else if (val[i] < min ) min = val[i];
}
#endif
if ((err = check_range(max)) != GRIB_SUCCESS) {
if ((err = check_range(gh, max)) != GRIB_SUCCESS) {
grib_context_log(a->context,GRIB_LOG_ERROR,"Maximum value out of range: %g", max);
return err;
}
if ((err = check_range(min)) != GRIB_SUCCESS) {
if ((err = check_range(gh, min)) != GRIB_SUCCESS) {
grib_context_log(a->context,GRIB_LOG_ERROR,"Minimum value out of range: %g", min);
return err;
}

1
src/grib_api_internal.h
View File

@ -1053,6 +1053,7 @@ struct grib_context
int bufrdc_mode;
int bufr_set_to_missing_if_out_of_range;
int bufr_multi_element_constant_arrays;
int grib_data_quality_checks;
FILE* log_stream;
grib_trie* classes;
grib_trie* lists;

1
src/grib_api_prototypes.h
View File

@ -1479,6 +1479,7 @@ int is_index_file(const char *filename);
char get_dir_separator_char(void);
char *codes_getenv(const char *name);
size_t sum_of_pl_array(const long* pl, size_t plsize);
int grib_util_grib_data_quality_check(grib_handle* h, double val);
/* bufr_util.c */
int compute_bufr_key_rank(grib_handle *h, grib_string_list *keys, const char *key);

23
src/grib_context.c
View File

@ -308,13 +308,13 @@ static grib_context default_grib_context = {
&default_seek, /* lfile seek procedure */
&default_feof, /* file feof procedure */
&default_log, /* logging_procedure */
&default_print, /* print procedure */
0, /* grib_codetable* */
0, /* grib_smart_table* */
0, /* char* outfilename */
0, /* int multi_support_on */
0, /* grib_multi_support* multi_support*/
&default_log, /* output_log */
&default_print, /* print */
0, /* codetable */
0, /* smart_table */
0, /* outfilename */
0, /* multi_support_on */
0, /* multi_support */
0, /* grib_definition_files_dir */
0, /* handle_file_count */
0, /* handle_total_count */
@ -323,9 +323,9 @@ static grib_context default_grib_context = {
0, /* gts_header_on */
0, /* gribex_mode_on */
0, /* large_constant_fields */
0, /* grib_itrie* keys */
0, /* keys */
0, /* keys_count */
0, /* grib_itrie* concepts_index */
0, /* concepts_index */
0, /* concepts_count */
{0,}, /* concepts */
0, /* hash_array_index */
@ -337,6 +337,7 @@ static grib_context default_grib_context = {
0, /* bufrdc_mode */
0, /* bufr_set_to_missing_if_out_of_range */
0, /* bufr_multi_element_constant_arrays */
0, /* grib_data_quality_checks */
0, /* log_stream */
0, /* classes */
0, /* lists */
@ -371,6 +372,7 @@ grib_context* grib_context_get_default()
const char* bufrdc_mode = NULL;
const char* bufr_set_to_missing_if_out_of_range = NULL;
const char* bufr_multi_element_constant_arrays = NULL;
const char* grib_data_quality_checks = NULL;
const char* file_pool_max_opened_files = NULL;
#ifdef ENABLE_FLOATING_POINT_EXCEPTIONS
@ -381,6 +383,7 @@ grib_context* grib_context_get_default()
bufrdc_mode = getenv("ECCODES_BUFRDC_MODE_ON");
bufr_set_to_missing_if_out_of_range = getenv("ECCODES_BUFR_SET_TO_MISSING_IF_OUT_OF_RANGE");
bufr_multi_element_constant_arrays = getenv("ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS");
grib_data_quality_checks = getenv("ECCODES_GRIB_DATA_QUALITY_CHECKS");
large_constant_fields = codes_getenv("ECCODES_GRIB_LARGE_CONSTANT_FIELDS");
no_abort = codes_getenv("ECCODES_NO_ABORT");
debug = codes_getenv("ECCODES_DEBUG");
@ -479,6 +482,8 @@ grib_context* grib_context_get_default()
atoi(bufr_set_to_missing_if_out_of_range) : 0;
default_grib_context.bufr_multi_element_constant_arrays = bufr_multi_element_constant_arrays ?
atoi(bufr_multi_element_constant_arrays) : 0;
default_grib_context.grib_data_quality_checks = grib_data_quality_checks ?
atoi(grib_data_quality_checks) : 0;
default_grib_context.file_pool_max_opened_files = file_pool_max_opened_files ?
atoi(file_pool_max_opened_files) : DEFAULT_FILE_POOL_MAX_OPENED_FILES;
}

22
src/grib_util.c
View File

@ -2039,3 +2039,25 @@ size_t sum_of_pl_array(const long* pl, size_t plsize)
}
return count;
}
int grib_util_grib_data_quality_check(grib_handle* h, double val)
{
/* TODO: From the paramId of the handle, get its limits. For now hardcoded */
long paramId = 0;
char shortName[256]={0,};
size_t len = sizeof(shortName);
/*const double MIN_FIELD_VALUE_ALLOWED = -2e7;*/
const double MAX_FIELD_VALUE_ALLOWED = +2e7;
if (val > MAX_FIELD_VALUE_ALLOWED) {
if (grib_get_long(h, "paramId", &paramId) == GRIB_SUCCESS &&
grib_get_string(h,"shortName",shortName,&len) == GRIB_SUCCESS)
{
grib_context_log(h->context, GRIB_LOG_ERROR, "Parameter %d (%s): value %g exceeds the maximum limit of %g",
paramId, shortName, val, MAX_FIELD_VALUE_ALLOWED);
}
return GRIB_OUT_OF_RANGE;
}
return GRIB_SUCCESS;
}

4
tests/grib_encode_pthreads.c
View File

@ -76,11 +76,11 @@ static int encode_file(char *input_file, char *output_file)
GRIB_CHECK(grib_get_size(clone_handle, "values", &values_len),0);
values = (double*)malloc(values_len*sizeof(double));
d=10e-8;
d=10e-10;
e=d;
count=1;
for (i=0;i<values_len;i++) {
if (count>100) {e*=10; count=1;}
if (count>1000) {e*=2; count=1;}
values[i]=d;
d+=e;
count++;