core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECC-745: bufr_compare performance: speed up by skipping extra key attributes

This commit is contained in:
Shahram Najm 2018-09-06 15:13:33 +01:00
parent a70086e846
commit 3eadeaa7be
4 changed files with 8 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
examples/C/bufr_copy_data.sh
View File

@ -22,35 +22,20 @@ rm -f ${TEMP_BUFR} ${TEMP_TEXT} ${REF} ${MYLOG}
cat > ${REF} <<EOF cat > ${REF} <<EOF
== 1 == DIFFERENCE == Different size for "unexpandedDescriptors" [43] [28] == 1 == DIFFERENCE == Different size for "unexpandedDescriptors" [43] [28]
== 1 == DIFFERENCE == [#2#icaoLocationIndicator] not found in 2nd field == 1 == DIFFERENCE == [#2#icaoLocationIndicator] not found in 2nd field
== 1 == DIFFERENCE == [#2#icaoLocationIndicator->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#stationType] not found in 2nd field == 1 == DIFFERENCE == [#2#stationType] not found in 2nd field
== 1 == DIFFERENCE == [#2#stationType->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#year] not found in 2nd field == 1 == DIFFERENCE == [#2#year] not found in 2nd field
== 1 == DIFFERENCE == [#2#year->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#month] not found in 2nd field == 1 == DIFFERENCE == [#2#month] not found in 2nd field
== 1 == DIFFERENCE == [#2#month->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#day] not found in 2nd field == 1 == DIFFERENCE == [#2#day] not found in 2nd field
== 1 == DIFFERENCE == [#2#day->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#hour] not found in 2nd field == 1 == DIFFERENCE == [#2#hour] not found in 2nd field
== 1 == DIFFERENCE == [#2#hour->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#minute] not found in 2nd field == 1 == DIFFERENCE == [#2#minute] not found in 2nd field
== 1 == DIFFERENCE == [#2#minute->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#latitude] not found in 2nd field == 1 == DIFFERENCE == [#2#latitude] not found in 2nd field
== 1 == DIFFERENCE == [#2#latitude->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#longitude] not found in 2nd field == 1 == DIFFERENCE == [#2#longitude] not found in 2nd field
== 1 == DIFFERENCE == [#2#longitude->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#heightOfStation] not found in 2nd field == 1 == DIFFERENCE == [#2#heightOfStation] not found in 2nd field
== 1 == DIFFERENCE == [#2#heightOfStation->units] not found in 2nd field
== 1 == DIFFERENCE == [#4#heightAboveStation] not found in 2nd field == 1 == DIFFERENCE == [#4#heightAboveStation] not found in 2nd field
== 1 == DIFFERENCE == [#4#heightAboveStation->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#windDirection] not found in 2nd field == 1 == DIFFERENCE == [#2#windDirection] not found in 2nd field
== 1 == DIFFERENCE == [#2#windDirection->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind] not found in 2nd field == 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind] not found in 2nd field
== 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind] not found in 2nd field == 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind] not found in 2nd field
== 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#windSpeed] not found in 2nd field == 1 == DIFFERENCE == [#2#windSpeed] not found in 2nd field
== 1 == DIFFERENCE == [#2#windSpeed->units] not found in 2nd field
EOF EOF
INPUT=${data_dir}/bufr/metar_with_2_bias.bufr INPUT=${data_dir}/bufr/metar_with_2_bias.bufr

15
examples/F90/bufr_copy_data.sh
View File

@ -22,35 +22,20 @@ rm -f ${TEMP} ${REF} ${MYLOG}
cat > ${REF} <<EOF cat > ${REF} <<EOF
== 1 == DIFFERENCE == Different size for "unexpandedDescriptors" [43] [28] == 1 == DIFFERENCE == Different size for "unexpandedDescriptors" [43] [28]
== 1 == DIFFERENCE == [#2#icaoLocationIndicator] not found in 2nd field == 1 == DIFFERENCE == [#2#icaoLocationIndicator] not found in 2nd field
== 1 == DIFFERENCE == [#2#icaoLocationIndicator->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#stationType] not found in 2nd field == 1 == DIFFERENCE == [#2#stationType] not found in 2nd field
== 1 == DIFFERENCE == [#2#stationType->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#year] not found in 2nd field == 1 == DIFFERENCE == [#2#year] not found in 2nd field
== 1 == DIFFERENCE == [#2#year->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#month] not found in 2nd field == 1 == DIFFERENCE == [#2#month] not found in 2nd field
== 1 == DIFFERENCE == [#2#month->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#day] not found in 2nd field == 1 == DIFFERENCE == [#2#day] not found in 2nd field
== 1 == DIFFERENCE == [#2#day->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#hour] not found in 2nd field == 1 == DIFFERENCE == [#2#hour] not found in 2nd field
== 1 == DIFFERENCE == [#2#hour->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#minute] not found in 2nd field == 1 == DIFFERENCE == [#2#minute] not found in 2nd field
== 1 == DIFFERENCE == [#2#minute->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#latitude] not found in 2nd field == 1 == DIFFERENCE == [#2#latitude] not found in 2nd field
== 1 == DIFFERENCE == [#2#latitude->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#longitude] not found in 2nd field == 1 == DIFFERENCE == [#2#longitude] not found in 2nd field
== 1 == DIFFERENCE == [#2#longitude->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#heightOfStation] not found in 2nd field == 1 == DIFFERENCE == [#2#heightOfStation] not found in 2nd field
== 1 == DIFFERENCE == [#2#heightOfStation->units] not found in 2nd field
== 1 == DIFFERENCE == [#4#heightAboveStation] not found in 2nd field == 1 == DIFFERENCE == [#4#heightAboveStation] not found in 2nd field
== 1 == DIFFERENCE == [#4#heightAboveStation->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#windDirection] not found in 2nd field == 1 == DIFFERENCE == [#2#windDirection] not found in 2nd field
== 1 == DIFFERENCE == [#2#windDirection->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind] not found in 2nd field == 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind] not found in 2nd field
== 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind] not found in 2nd field == 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind] not found in 2nd field
== 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#windSpeed] not found in 2nd field == 1 == DIFFERENCE == [#2#windSpeed] not found in 2nd field
== 1 == DIFFERENCE == [#2#windSpeed->units] not found in 2nd field
EOF EOF
INPUT=metar_with_2_bias.bufr INPUT=metar_with_2_bias.bufr

15
examples/python/bufr_copy_data.sh
View File

@ -22,35 +22,20 @@ rm -f ${TEMP} ${REF} ${MYLOG}
cat > ${REF} <<EOF cat > ${REF} <<EOF
== 1 == DIFFERENCE == Different size for "unexpandedDescriptors" [43] [28] == 1 == DIFFERENCE == Different size for "unexpandedDescriptors" [43] [28]
== 1 == DIFFERENCE == [#2#icaoLocationIndicator] not found in 2nd field == 1 == DIFFERENCE == [#2#icaoLocationIndicator] not found in 2nd field
== 1 == DIFFERENCE == [#2#icaoLocationIndicator->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#stationType] not found in 2nd field == 1 == DIFFERENCE == [#2#stationType] not found in 2nd field
== 1 == DIFFERENCE == [#2#stationType->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#year] not found in 2nd field == 1 == DIFFERENCE == [#2#year] not found in 2nd field
== 1 == DIFFERENCE == [#2#year->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#month] not found in 2nd field == 1 == DIFFERENCE == [#2#month] not found in 2nd field
== 1 == DIFFERENCE == [#2#month->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#day] not found in 2nd field == 1 == DIFFERENCE == [#2#day] not found in 2nd field
== 1 == DIFFERENCE == [#2#day->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#hour] not found in 2nd field == 1 == DIFFERENCE == [#2#hour] not found in 2nd field
== 1 == DIFFERENCE == [#2#hour->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#minute] not found in 2nd field == 1 == DIFFERENCE == [#2#minute] not found in 2nd field
== 1 == DIFFERENCE == [#2#minute->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#latitude] not found in 2nd field == 1 == DIFFERENCE == [#2#latitude] not found in 2nd field
== 1 == DIFFERENCE == [#2#latitude->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#longitude] not found in 2nd field == 1 == DIFFERENCE == [#2#longitude] not found in 2nd field
== 1 == DIFFERENCE == [#2#longitude->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#heightOfStation] not found in 2nd field == 1 == DIFFERENCE == [#2#heightOfStation] not found in 2nd field
== 1 == DIFFERENCE == [#2#heightOfStation->units] not found in 2nd field
== 1 == DIFFERENCE == [#4#heightAboveStation] not found in 2nd field == 1 == DIFFERENCE == [#4#heightAboveStation] not found in 2nd field
== 1 == DIFFERENCE == [#4#heightAboveStation->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#windDirection] not found in 2nd field == 1 == DIFFERENCE == [#2#windDirection] not found in 2nd field
== 1 == DIFFERENCE == [#2#windDirection->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind] not found in 2nd field == 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind] not found in 2nd field
== 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind] not found in 2nd field == 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind] not found in 2nd field
== 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind->units] not found in 2nd field
== 1 == DIFFERENCE == [#2#windSpeed] not found in 2nd field == 1 == DIFFERENCE == [#2#windSpeed] not found in 2nd field
== 1 == DIFFERENCE == [#2#windSpeed->units] not found in 2nd field
EOF EOF
INPUT=metar_with_2_bias.bufr INPUT=metar_with_2_bias.bufr

9
tools/bufr_compare.c
View File

@ -1134,9 +1134,14 @@ static int compare_attributes(grib_handle* handle1, grib_handle* handle2, grib_r
/*long native_type = 0;*/ /*long native_type = 0;*/
grib_accessor* aa = NULL; grib_accessor* aa = NULL;
if ( (a->attributes[i]->flags & GRIB_ACCESSOR_FLAG_DUMP)== 0 ) { if ( (a->attributes[i]->flags & GRIB_ACCESSOR_FLAG_DUMP)== 0 ) {
++i; /* next attribute */ ++i; /* next attribute if accessor is not for dumping */
continue; continue;
} }
if ( (a->attributes[i]->flags & GRIB_ACCESSOR_FLAG_READ_ONLY)!= 0 ) {
++i; /* next attribute if accessor is read-only */
continue;
}
aa = a->attributes[i]; aa = a->attributes[i];
/*native_type = grib_accessor_get_native_type(aa); TODO: read only check? */ /*native_type = grib_accessor_get_native_type(aa); TODO: read only check? */
@ -1186,11 +1191,13 @@ static int compare_all_dump_keys(grib_handle* handle1, grib_handle* handle2, gri
if (!headerMode) { if (!headerMode) {
/* See ECC-333: By setting unpack we get ALL the bufr keys. */ /* See ECC-333: By setting unpack we get ALL the bufr keys. */
/* In headerMode we want just the header ones */ /* In headerMode we want just the header ones */
grib_set_long(handle1,"skipExtraKeyAttributes",1); /* See ECC-745 */
ret = grib_set_long(handle1,"unpack",1); ret = grib_set_long(handle1,"unpack",1);
if (ret != GRIB_SUCCESS) { if (ret != GRIB_SUCCESS) {
grib_context_log(context, GRIB_LOG_ERROR, "Failed to unpack 1st message: %s", grib_get_error_message(ret)); grib_context_log(context, GRIB_LOG_ERROR, "Failed to unpack 1st message: %s", grib_get_error_message(ret));
exit(1); exit(1);
} }
grib_set_long(handle2,"skipExtraKeyAttributes",1); /* See ECC-745 */
ret = grib_set_long(handle2,"unpack",1); ret = grib_set_long(handle2,"unpack",1);
if (ret != GRIB_SUCCESS) { if (ret != GRIB_SUCCESS) {
grib_context_log(context, GRIB_LOG_ERROR, "Failed to unpack 2nd message: %s", grib_get_error_message(ret)); grib_context_log(context, GRIB_LOG_ERROR, "Failed to unpack 2nd message: %s", grib_get_error_message(ret));