core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECC-651: grib_compare/bufr_compare: Enable a 'two-way' switch for symmetric comparison

This commit is contained in:
Shahram Najm 2018-03-13 15:48:00 +00:00
parent eb697ebd2d
commit dddd872c2e
3 changed files with 137 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
tests/grib_compare.sh
View File

@ -20,6 +20,7 @@ rm -f $outfile
${tools_dir}/grib_set -s shortName=2d $infile $outfile
${tools_dir}/grib_compare -b indicatorOfParameter,paramId,shortName $infile $outfile > $REDIRECT
# ----------------------------------------
# Test the -r switch
# ----------------------------------------
infile=${data_dir}/v.grib2
@ -34,6 +35,7 @@ ${tools_dir}/grib_compare -r temp_comp.123 temp_comp.321
rm -f temp_comp.1 temp_comp.2 temp_comp.3 temp_comp.123 temp_comp.321
# ----------------------------------------
# GRIB-797: test last argument being a directory
# ----------------------------------------
temp_dir=tempdir.grib_compare
@ -42,6 +44,7 @@ cp $infile $temp_dir
${tools_dir}/grib_compare $infile $temp_dir
rm -rf $temp_dir
# ----------------------------------------
# ECC-245: blacklist and 2nd order packing
# ----------------------------------------
temp1=grib_compare_temp1.grib
@ -67,6 +70,7 @@ set -e
[ $status -eq 1 ]
# ----------------------------------------
# ECC-355: -R with "all" option
# ----------------------------------------
${tools_dir}/grib_copy -w count=1 ${data_dir}/tigge_cf_ecmwf.grib2 $temp1
@ -77,6 +81,56 @@ ${tools_dir}/grib_compare -b $BLACKLIST -R referenceValue=0.03,codedValues=2 $te
# Now try the "all" option with the highest relative diff value
${tools_dir}/grib_compare -b $BLACKLIST -R all=2 $temp1 $temp2
# ----------------------------------------
# Two way (symmetric) comparison
# ----------------------------------------
sample_g1=$ECCODES_SAMPLES_PATH/GRIB1.tmpl
temp_nold=$temp1
${tools_dir}/grib_set -s deleteLocalDefinition=1 $sample_g1 $temp_nold
# Now the sample has a local definition but $temp_nold does not
set +e
${tools_dir}/grib_compare $temp_nold $sample_g1 > $outfile
status=$?
set -e
[ $status -eq 1 ]
reffile=grib_compare_temp1.ref
cat > $reffile <<EOF
-- GRIB #1 -- shortName=z paramId=129 stepRange=0 levelType=pl level=500 packingType=grid_simple gridType=regular_ll --
long [totalLength]: [84] != [107]
long [section1Length]: [28] != [52]
long [section4Length]: [12] != [11]
EOF
diff $reffile $outfile
# Two-way mode enabled
set +e
${tools_dir}/grib_compare -2 $temp_nold $sample_g1 > $outfile
status=$?
set -e
[ $status -eq 1 ]
cat > $reffile <<EOF
-- GRIB #1 -- shortName=z paramId=129 stepRange=0 levelType=pl level=500 packingType=grid_simple gridType=regular_ll --
long [totalLength]: [84] != [107]
long [section1Length]: [28] != [52]
long [section4Length]: [12] != [11]
[reservedNeedNotBePresent] not found in 1st field
[localDefinitionNumber] not found in 1st field
[marsClass] not found in 1st field
[marsType] not found in 1st field
[marsStream] not found in 1st field
[experimentVersionNumber] not found in 1st field
[perturbationNumber] not found in 1st field
[numberOfForecastsInEnsemble] not found in 1st field
[padding_local1_1] not found in 1st field
EOF
diff $reffile $outfile
rm -f $reffile
# Clean up
# --------------
rm -f $temp1 $temp2
rm -f $outfile

10
tools/bufr_compare.c
View File

@ -486,7 +486,10 @@ int grib_tool_new_handle_action(grib_runtime_options* options, grib_handle* h)
if(compare_handles(global_handle,h,options)) {
error++;
if (!force) exit(1);
if (!two_way) {
/* If two_way mode: Don't exit yet. Show further differences */
if (!force) exit(1);
}
}
if (two_way) {
/* ECC-431 */
@ -494,6 +497,11 @@ int grib_tool_new_handle_action(grib_runtime_options* options, grib_handle* h)
if(compare_handles(h, global_handle, options)) {
error++;
if (!force) exit(1);
} else {
if (error) {
/* Error from first pass */
if (!force) exit(1);
}
}
}

117
tools/grib_compare.c
View File

@ -52,6 +52,14 @@ int error=0;
int count=0;
int lastPrint=0;
int force=0;
int two_way=0; /* set to 1 when '-2' option used */
/* Boolean variable handles_swapped relevant in 'two_way' mode:
* 0 means: h1 is first file, h2 is second file
* 1 means: h1 is second file, h2 is first file
*/
int handles_swapped=0;
double maxAbsoluteError = 1e-19;
int onlyListed=1;
int headerMode=0;
@ -119,6 +127,7 @@ grib_option grib_options[]={
{"r",0,"Compare files in which the messages are not in the same order. This option is time expensive.\n",0,1,0},
{"b:",0,0,0,1,0},
{"e",0,"Edition independent compare. It is used to compare grib edition 1 and 2.\n",0,1,0},
{"2",0,"Enable two-way comparison.\n",0,1,0},
{"c:",0,0,0,1,0},
{"S:","start","First field to be processed.\n",0,1,0},
{"E:","end","Last field to be processed.\n",0,1,0},
@ -183,6 +192,9 @@ int grib_tool_init(grib_runtime_options* options)
if (grib_options_on("f")) force=1;
else force=0;
if (grib_options_on("2")) two_way=1;
else two_way=0;
verbose = grib_options_on("v");
listFromCommandLine=0;
@ -431,7 +443,23 @@ int grib_tool_new_handle_action(grib_runtime_options* options, grib_handle* h)
if(compare_handles(global_handle,h,options)) {
error++;
if (!force) exit(1);
if (!two_way) {
/* If two_way mode: Don't exit yet. Show further differences */
if (!force) exit(1);
}
}
if (two_way) {
/* ECC-431 */
handles_swapped = 1;
if(compare_handles(h, global_handle, options)) {
error++;
if (!force) exit(1);
} else {
if (error) {
/* Error from first pass */
if (!force) exit(1);
}
}
}
grib_handle_delete(global_handle);
@ -562,6 +590,8 @@ static int compare_values(grib_runtime_options* options,grib_handle* h1,grib_han
double packingError1=0,packingError2=0;
double value_tolerance=0;
grib_context* c=h1->context;
char* first_str = (handles_swapped==0? "1st" : "2nd");
char* second_str = (handles_swapped==0? "2nd" : "1st");
type1=type;
type2=type;
@ -573,7 +603,7 @@ static int compare_values(grib_runtime_options* options,grib_handle* h1,grib_han
if( type1==GRIB_TYPE_UNDEFINED && (err = grib_get_native_type(h1,name,&type1)) != GRIB_SUCCESS)
{
printInfo(h1);
printf("Oops... cannot get type of [%s] in 1st field: %s\n",name,grib_get_error_message(err));
printf("Oops... cannot get type of [%s] in %s field: %s\n",name,first_str,grib_get_error_message(err));
save_error(c,name);
return err;
}
@ -583,24 +613,22 @@ static int compare_values(grib_runtime_options* options,grib_handle* h1,grib_han
if(err == GRIB_NOT_FOUND)
{
printInfo(h1);
printf("[%s] not found in 2nd field\n",name);
printf("[%s] not found in %s field\n",name,second_str);
save_error(c,name);
return err;
}
printInfo(h1);
printf("Oops... cannot get type of [%s] in 2nd field: %s\n",name,grib_get_error_message(err));
printf("Oops... cannot get type of [%s] in %s field: %s\n",name,second_str,grib_get_error_message(err));
save_error(c,name);
return err;
}
/*
if(type1 != type2)
{
printInfo(h1);
printf("Warning, [%s] has different types: 1st field: [%s], 2nd field: [%s]\n",
name,grib_get_type_name(type1),grib_get_type_name(type2));
return GRIB_TYPE_MISMATCH;
}
/* if(type1 != type2) {
printInfo(h1);
printf("Warning, [%s] has different types: 1st field: [%s], 2nd field: [%s]\n",
name,grib_get_type_name(type1),grib_get_type_name(type2));
return GRIB_TYPE_MISMATCH;
}
*/
if(type1 == GRIB_TYPE_LABEL)
@ -613,7 +641,7 @@ static int compare_values(grib_runtime_options* options,grib_handle* h1,grib_han
if((err = grib_get_size(h1,name,&len1)) != GRIB_SUCCESS)
{
printInfo(h1);
printf("Oops... cannot get size of [%s] in 1st field: %s\n",name,grib_get_error_message(err));
printf("Oops... cannot get size of [%s] in %s field: %s\n",name,first_str,grib_get_error_message(err));
save_error(c,name);
return err;
}
@ -623,25 +651,28 @@ static int compare_values(grib_runtime_options* options,grib_handle* h1,grib_han
if(err == GRIB_NOT_FOUND)
{
printInfo(h1);
printf("[%s] not found in 2nd field\n",name);
printf("[%s] not found in %s field\n",name,second_str);
save_error(c,name);
return err;
}
printInfo(h1);
printf("Oops... cannot get size of [%s] in 2nd field: %s\n",name,grib_get_error_message(err));
printf("Oops... cannot get size of [%s] in %s field: %s\n",name,second_str,grib_get_error_message(err));
save_error(c,name);
return err;
}
/*
if(len1 != len2 && type1 != GRIB_TYPE_STRING)
{
printInfo(h1);
printf("[%s] has different size: 1st field: %ld, 2nd field: %ld\n",name,(long)len1,(long)len2);
save_error(c,name);
return GRIB_COUNT_MISMATCH;
}
if (handles_swapped) {
/* Comparing a second time with handles swapped. Do not compare keys common to both handles */
return GRIB_SUCCESS;
}
/* if(len1 != len2 && type1 != GRIB_TYPE_STRING) {
printInfo(h1);
printf("[%s] has different size: 1st field: %ld, 2nd field: %ld\n",name,(long)len1,(long)len2);
save_error(c,name);
return GRIB_COUNT_MISMATCH;
}
*/
isMissing1= ( (grib_is_missing(h1,name,&err1)==1) && (err1 == 0) ) ? 1 : 0;
@ -653,18 +684,18 @@ static int compare_values(grib_runtime_options* options,grib_handle* h1,grib_han
}
if (isMissing1==1) {
if (verbose) printf(" is set to missing in 1st field\n");
if (verbose) printf(" is set to missing in %s field\n",first_str);
printInfo(h1);
printf("%s is set to missing in 1st field is not missing in 2nd field\n",name);
printf("%s is set to missing in %s field is not missing in %s field\n",name,first_str,second_str);
err1 = GRIB_VALUE_MISMATCH;
save_error(c,name);
return GRIB_VALUE_MISMATCH;
}
if (isMissing2==1) {
if (verbose) printf(" is set to missing in 1st field\n");
if (verbose) printf(" is set to missing in %s field\n",first_str);
printInfo(h1);
printf("%s is set to missing in 2nd field is not missing in 1st field\n",name);
printf("%s is set to missing in %s field is not missing in %s field\n",name,second_str,first_str);
err1 = GRIB_VALUE_MISMATCH;
save_error(c,name);
return GRIB_VALUE_MISMATCH;
@ -682,16 +713,16 @@ static int compare_values(grib_runtime_options* options,grib_handle* h1,grib_han
if((err1 = grib_get_string(h1,name,sval1,&len1)) != GRIB_SUCCESS)
{
printInfo(h1);
printf("Oops... cannot get string value of [%s] in 1st field: %s\n",
name,grib_get_error_message(err1));
printf("Oops... cannot get string value of [%s] in %s field: %s\n",
name,first_str,grib_get_error_message(err1));
save_error(c,name);
}
if((err2 = grib_get_string(h2,name,sval2,&len2)) != GRIB_SUCCESS)
{
printInfo(h1);
printf("Oops... cannot get string value of [%s] in 2nd field: %s\n",
name,grib_get_error_message(err2));
printf("Oops... cannot get string value of [%s] in %s field: %s\n",
name,second_str,grib_get_error_message(err2));
save_error(c,name);
}
@ -724,16 +755,16 @@ static int compare_values(grib_runtime_options* options,grib_handle* h1,grib_han
if((err1 = grib_get_long_array(h1,name,lval1,&len1)) != GRIB_SUCCESS)
{
printInfo(h1);
printf("Oops... cannot get long value of [%s] in 1st field: %s\n",
name,grib_get_error_message(err1));
printf("Oops... cannot get long value of [%s] in %s field: %s\n",
name,first_str,grib_get_error_message(err1));
save_error(c,name);
}
if((err2 = grib_get_long_array(h2,name,lval2,&len2)) != GRIB_SUCCESS)
{
printInfo(h1);
printf("Oops... cannot get long value of [%s] in 2nd field: %s\n",
name,grib_get_error_message(err2));
printf("Oops... cannot get long value of [%s] in %s field: %s\n",
name,second_str,grib_get_error_message(err2));
save_error(c,name);
}
@ -837,16 +868,16 @@ static int compare_values(grib_runtime_options* options,grib_handle* h1,grib_han
if((err1 = grib_get_double_array(h1,name,dval1,&len1)) != GRIB_SUCCESS)
{
printInfo(h1);
printf("Oops... cannot get double value of [%s] in 1st field: %s\n",
name,grib_get_error_message(err1));
printf("Oops... cannot get double value of [%s] in %s field: %s\n",
name,first_str,grib_get_error_message(err1));
save_error(c,name);
}
if((err2 = grib_get_double_array(h2,name,dval2,&len2)) != GRIB_SUCCESS)
{
printInfo(h1);
printf("Oops... cannot get double value of [%s] in 2nd field: %s\n",
name,grib_get_error_message(err2));
printf("Oops... cannot get double value of [%s] in %s field: %s\n",
name,second_str,grib_get_error_message(err2));
save_error(c,name);
}
@ -945,16 +976,16 @@ static int compare_values(grib_runtime_options* options,grib_handle* h1,grib_han
{
printInfo(h1);
save_error(c,name);
printf("Oops... cannot get bytes value of [%s] in 1st field: %s\n",
name,grib_get_error_message(err1));
printf("Oops... cannot get bytes value of [%s] in %s field: %s\n",
name,first_str,grib_get_error_message(err1));
}
if((err2 = grib_get_bytes(h2,name,uval2,&len2)) != GRIB_SUCCESS)
{
printInfo(h1);
save_error(c,name);
printf("Oops... cannot get bytes value of [%s] in 2nd field: %s\n",
name,grib_get_error_message(err2));
printf("Oops... cannot get bytes value of [%s] in %s field: %s\n",
name,second_str,grib_get_error_message(err2));
}
if(err1 == GRIB_SUCCESS && err2 == GRIB_SUCCESS)