core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECC-1066 and ECC-1067: GRIB encoding: Data quality checks

This commit is contained in:
Shahram Najm 2020-01-27 13:20:31 +00:00
parent 0bf6d79c0d
commit ca594113a6
6 changed files with 278 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
definitions/grib1/section.1.def
View File

@ -149,6 +149,7 @@ if(table2Version >= 128) {
concept paramIdECMF (defaultParameter,"paramId.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy;
concept paramId (paramIdECMF,"paramId.def",conceptsMasterDir,conceptsLocalDirAll): long_type,dump;
transient pid = paramId : hidden;
concept cfNameECMF(defaultName,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only;
concept cfName(cfNameECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirAll) : dump,no_copy,read_only;

1
definitions/grib2/parameters.def
View File

@ -7,6 +7,7 @@ constant conceptsLocalDirECMF="grib2/localConcepts/ecmf" : hidden;
concept paramIdECMF (defaultParameter,"paramId.def",conceptsMasterDir,conceptsLocalDirECMF): long_type,no_copy;
concept paramId (paramIdECMF,"paramId.def",conceptsMasterDir,conceptsLocalDirAll): long_type;
transient pid = paramId : hidden;
concept shortNameECMF (defaultShortName,"shortName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,dump;
concept ls.shortName (shortNameECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirAll) : no_copy,dump;

201
definitions/param_limits.def
View File

@ -1,9 +1,198 @@
constant default_min_val = -1e9 : long_type, hidden;
constant default_max_val = +1e9 : long_type, hidden;
constant default_min_val = -1e9 : double_type, hidden;
constant default_max_val = +1e9 : double_type, hidden;
concept param_value_min(default_min_val) {
0 = { paramId=167; }
} : long_type, hidden;
-150 = { pid=165; }
-100 = { pid=166; }
0 = { pid=260260; }
0 = { pid=228028; }
0 = { pid=49; }
0 = { pid=207; }
25 = { pid=168; }
0 = { pid=260242; }
160 = { pid=167; }
-0.1 = { pid=260428; }
-0.1 = { pid=260423; }
-1e+09 = { pid=260427; }
0 = { pid=260509; }
5 = { pid=151175; }
0 = { pid=260257; }
0 = { pid=59; }
-60000 = { pid=228001; }
-0.05 = { pid=228143; }
0 = { pid=151163; }
-3.5 = { pid=151131; }
-100000 = { pid=180; }
-10 = { pid=260259; }
-13000 = { pid=129; }
-1300 = { pid=156; }
0 = { pid=3075; }
0 = { pid=172; }
-0.05 = { pid=3062; }
0 = { pid=260210; }
0 = { pid=3073; }
160 = { pid=121; }
85000 = { pid=151; }
270 = { pid=151126; }
-1 = { pid=140230; }
0 = { pid=140221; }
0 = { pid=3074; }
150 = { pid=122; }
0 = { pid=140214; }
-3.5 = { pid=151132; }
-100000 = { pid=181; }
0 = { pid=151225; }
-1300 = { pid=228002; }
0 = { pid=140231; }
0 = { pid=260430; }
170 = { pid=3; }
-1 = { pid=60; }
100 = { pid=54; }
0 = { pid=157; }
-4 = { pid=151145; }
0 = { pid=151219; }
160 = { pid=34; }
0 = { pid=31; }
0 = { pid=174098; }
0 = { pid=140229; }
120 = { pid=235; }
20 = { pid=228032; }
10 = { pid=33; }
0 = { pid=3066; }
-1e-10 = { pid=228141; }
-1 = { pid=228144; }
0.005 = { pid=260367; }
-1000 = { pid=260364; }
0 = { pid=228039; }
0 = { pid=228087; }
-20 = { pid=228086; }
170 = { pid=260360; }
170 = { pid=228139; }
170 = { pid=228096; }
170 = { pid=228095; }
0 = { pid=43; }
0 = { pid=247; }
0 = { pid=246; }
-0.1 = { pid=133; }
0 = { pid=189; }
-1e+08 = { pid=147; }
-1e+07 = { pid=176; }
-1e+08 = { pid=177; }
43000 = { pid=134; }
0 = { pid=173; }
-0.001 = { pid=174008; }
-1e+08 = { pid=146; }
-10 = { pid=169; }
140 = { pid=130; }
-0.1 = { pid=260264; }
-1e+08 = { pid=179; }
-1e-06 = { pid=228164; }
-3 = { pid=260057; }
-50 = { pid=136; }
-0.05 = { pid=228228; }
-250 = { pid=131; }
-250 = { pid=132; }
-30 = { pid=135; }
0 = { pid=260199; }
-0.001 = { pid=228205; }
0 = { pid=3031; }
0 = { pid=10; }
} : double_type, hidden;
concept param_value_max(default_max_val) {
373 = { paramId=167; }
} : long_type, hidden;
150 = { pid=165; }
100 = { pid=166; }
360.1 = { pid=260260; }
140 = { pid=228028; }
100 = { pid=49; }
300 = { pid=207; }
350 = { pid=168; }
160 = { pid=260242; }
370 = { pid=167; }
1e+09 = { pid=260428; }
1e+09 = { pid=260423; }
0.1 = { pid=260427; }
100 = { pid=260509; }
50 = { pid=151175; }
100 = { pid=260257; }
40000 = { pid=59; }
5 = { pid=228001; }
130 = { pid=228143; }
1500 = { pid=151163; }
3.5 = { pid=151131; }
100000 = { pid=180; }
5 = { pid=260259; }
350000 = { pid=129; }
35000 = { pid=156; }
100 = { pid=3075; }
1 = { pid=172; }
130 = { pid=3062; }
1 = { pid=260210; }
100 = { pid=3073; }
380 = { pid=121; }
125000 = { pid=151; }
308 = { pid=151126; }
360.5 = { pid=140230; }
35 = { pid=140221; }
100 = { pid=3074; }
330 = { pid=122; }
35 = { pid=140214; }
3.5 = { pid=151132; }
100000 = { pid=181; }
4000 = { pid=151225; }
8888 = { pid=228002; }
50 = { pid=140231; }
30 = { pid=260430; }
1200 = { pid=3; }
1 = { pid=60; }
108000 = { pid=54; }
180 = { pid=157; }
4 = { pid=151145; }
50 = { pid=151219; }
320 = { pid=34; }
1.001 = { pid=31; }
15 = { pid=174098; }
35 = { pid=140229; }
380 = { pid=235; }
100 = { pid=228032; }
1000 = { pid=33; }
5 = { pid=3066; }
15000 = { pid=228141; }
50 = { pid=228144; }
100 = { pid=260367; }
1000 = { pid=260364; }
2000 = { pid=228039; }
2000 = { pid=228087; }
2000 = { pid=228086; }
350 = { pid=260360; }
350 = { pid=228139; }
350 = { pid=228096; }
350 = { pid=228095; }
10 = { pid=43; }
0.01 = { pid=247; }
1e+06 = { pid=246; }
0.1 = { pid=133; }
3600 = { pid=189; }
1e+08 = { pid=147; }
1e+07 = { pid=176; }
1e+06 = { pid=177; }
115000 = { pid=134; }
10 = { pid=173; }
100 = { pid=174008; }
1e+08 = { pid=146; }
1e+09 = { pid=169; }
400 = { pid=130; }
1e+09 = { pid=260264; }
-1000 = { pid=179; }
101 = { pid=228164; }
150 = { pid=260057; }
220 = { pid=136; }
130 = { pid=228228; }
250 = { pid=131; }
250 = { pid=132; }
30 = { pid=135; }
1 = { pid=260199; }
30 = { pid=228205; }
360.1 = { pid=3031; }
300 = { pid=10; }
} : double_type, hidden;

23
src/grib_accessor_class_concept.c
View File

@ -434,10 +434,25 @@ static int unpack_double(grib_accessor* a, double* val, size_t* len)
* if (referenceValue > 0 && paramId == 129)
*/
/*return GRIB_NOT_IMPLEMENTED*/
long lval = 0;
int ret = unpack_long(a, &lval, len);
if (ret == GRIB_SUCCESS) {
*val = lval;
int ret = 0;
if (a->flags & GRIB_ACCESSOR_FLAG_LONG_TYPE) {
long lval = 0;
int ret = unpack_long(a, &lval, len);
if (ret == GRIB_SUCCESS) {
*val = lval;
}
} else if (a->flags & GRIB_ACCESSOR_FLAG_DOUBLE_TYPE) {
const char* p = concept_evaluate(a);
if (!p) {
grib_handle* h = grib_handle_of_accessor(a);
if (a->creator->defaultkey)
return grib_get_double_internal(h, a->creator->defaultkey, val);
return GRIB_NOT_FOUND;
}
*val = atof(p);
*len = 1;
}
return ret;
}

10
src/grib_util.c
View File

@ -2180,7 +2180,7 @@ int grib_is_earth_oblate(grib_handle* h)
int grib_util_grib_data_quality_check(grib_handle* h, double min_val, double max_val)
{
int err = 0;
long min_field_value_allowed = 0, max_field_value_allowed = 0;
double min_field_value_allowed = 0, max_field_value_allowed = 0;
long paramId = 0;
double dmin_allowed = 0, dmax_allowed = 0;
grib_context* ctx = h->context;
@ -2193,19 +2193,19 @@ int grib_util_grib_data_quality_check(grib_handle* h, double min_val, double max
is_error = (ctx->grib_data_quality_checks == 1);
/* The limit keys must exist if we are here */
err = grib_get_long(h, "param_value_min", &min_field_value_allowed);
err = grib_get_double(h, "param_value_min", &min_field_value_allowed);
if (err) {
grib_context_log(ctx, GRIB_LOG_ERROR, "grib_data_quality_check: Could not get param_value_min");
return err;
}
err = grib_get_long(h, "param_value_max", &max_field_value_allowed);
err = grib_get_double(h, "param_value_max", &max_field_value_allowed);
if (err) {
grib_context_log(ctx, GRIB_LOG_ERROR, "grib_data_quality_check: Could not get param_value_max");
return err;
}
dmin_allowed = (double)min_field_value_allowed;
dmax_allowed = (double)max_field_value_allowed;
dmin_allowed = min_field_value_allowed;
dmax_allowed = max_field_value_allowed;
if (min_val < dmin_allowed) {
char description[1024] = {0,};

64
tests/grib_data_quality_checks.sh
View File

@ -17,6 +17,11 @@ label="grib_data_quality"
tempOut=temp.1.${label}.out
temp2=temp.2.${label}.out
tempErr=temp.${label}.err
tempGrib1=temp.${label}.grib1
tempGrib2=temp.${label}.grib2
sample_g1=$ECCODES_SAMPLES_PATH/GRIB1.tmpl
sample_g2=$ECCODES_SAMPLES_PATH/GRIB2.tmpl
# Start with clean environment
unset ECCODES_GRIB_DATA_QUALITY_CHECKS
@ -28,11 +33,13 @@ input2=${data_dir}/reduced_gaussian_surface.grib2
grib_check_key_equals $input1 paramId 167
grib_check_key_equals $input2 paramId 167
# Data quality checks disabled. Create huge values for temperature
echo "Data quality checks disabled. Create huge values for temperature..."
# --------------------------------------------------------------------------
${tools_dir}/grib_set -s scaleValuesBy=100 $input1 $tempOut
${tools_dir}/grib_set -s scaleValuesBy=100 $input2 $tempOut
# Data quality checks enabled. Repacking should fail
echo "Data quality checks enabled. Repacking should fail..."
# -----------------------------------------------------------
export ECCODES_GRIB_DATA_QUALITY_CHECKS=1
set +e
${tools_dir}/grib_copy -r $tempOut /dev/null 2>$tempErr
@ -41,14 +48,15 @@ set -e
[ $status -ne 0 ]
grep -q 'more than the allowable limit' $tempErr
# Data quality checks enabled but only as a warning. Repacking should pass
echo "Data quality checks enabled but only as a warning. Repacking should pass..."
# --------------------------------------------------------------------------------
export ECCODES_GRIB_DATA_QUALITY_CHECKS=2
${tools_dir}/grib_copy -r $tempOut /dev/null 2>$tempErr
grep -q 'more than the allowable limit' $tempErr
# Data quality checks enabled. Scaling should fail
echo "Data quality checks enabled. Scaling should fail..."
# --------------------------------------------------------
export ECCODES_GRIB_DATA_QUALITY_CHECKS=1
set +e
${tools_dir}/grib_set -s scaleValuesBy=100 $input1 $tempOut 2>$tempErr
@ -67,8 +75,50 @@ grep -q 'GRIB2 simple packing: unable to set values' $tempErr
grep -q 'allowable limit' $tempErr
# Override the defaults
# ----------------------
echo "Test limits which are doubles..."
# -------------------------------------
pid=151131 # has limits -3.5 and +3.5
${tools_dir}/grib_set -s paramId=$pid $input1 $tempGrib1
${tools_dir}/grib_set -s paramId=$pid $input2 $tempGrib2
minval1=`${tools_dir}/grib_get -p param_value_min $tempGrib1`
maxval1=`${tools_dir}/grib_get -p param_value_max $tempGrib1`
minval2=`${tools_dir}/grib_get -p param_value_min $tempGrib2`
maxval2=`${tools_dir}/grib_get -p param_value_max $tempGrib2`
[ "$minval1" = "-3.5" ]
[ "$maxval1" = "3.5" ]
[ "$minval2" = "-3.5" ]
[ "$maxval2" = "3.5" ]
set +e
${tools_dir}/grib_set -s scaleValuesBy=1.1 $tempGrib1 $tempOut 2>$tempErr
stat1=$?
${tools_dir}/grib_set -s scaleValuesBy=1.1 $tempGrib2 $tempOut 2>$tempErr
stat2=$?
set -e
[ $stat1 -ne 0 ]
[ $stat2 -ne 0 ]
echo "Test close to the limit..."
# ---------------------------------
${tools_dir}/grib_set -s paramId=$pid $sample_g2 $tempGrib2
${tools_dir}/grib_set -s scaleValuesBy=3 $tempGrib2 $tempOut # OK
set +e
${tools_dir}/grib_set -s scaleValuesBy=3.6 $tempGrib2 $tempOut
set -e
[ $status -ne 0 ]
${tools_dir}/grib_set -s edition=1 $tempGrib2 $tempGrib1
${tools_dir}/grib_set -s scaleValuesBy=-3 $tempGrib1 $tempOut # OK
set +e
${tools_dir}/grib_set -s scaleValuesBy=-3.55 $tempGrib1 $tempOut
set -e
[ $status -ne 0 ]
echo "Override the defaults..."
# ------------------------------
tempDir=tempdir.$label
rm -rf $tempDir
mkdir -p $tempDir