From 3eadeaa7beb4f7674e112fe46fbd6883ad563561 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 6 Sep 2018 15:13:33 +0100 Subject: [PATCH] ECC-745: bufr_compare performance: speed up by skipping extra key attributes --- examples/C/bufr_copy_data.sh | 15 --------------- examples/F90/bufr_copy_data.sh | 15 --------------- examples/python/bufr_copy_data.sh | 15 --------------- tools/bufr_compare.c | 9 ++++++++- 4 files changed, 8 insertions(+), 46 deletions(-) diff --git a/examples/C/bufr_copy_data.sh b/examples/C/bufr_copy_data.sh index 3e2a3e263..772f60b71 100755 --- a/examples/C/bufr_copy_data.sh +++ b/examples/C/bufr_copy_data.sh @@ -22,35 +22,20 @@ rm -f ${TEMP_BUFR} ${TEMP_TEXT} ${REF} ${MYLOG} cat > ${REF} <units] not found in 2nd field == 1 == DIFFERENCE == [#2#stationType] not found in 2nd field -== 1 == DIFFERENCE == [#2#stationType->units] not found in 2nd field == 1 == DIFFERENCE == [#2#year] not found in 2nd field -== 1 == DIFFERENCE == [#2#year->units] not found in 2nd field == 1 == DIFFERENCE == [#2#month] not found in 2nd field -== 1 == DIFFERENCE == [#2#month->units] not found in 2nd field == 1 == DIFFERENCE == [#2#day] not found in 2nd field -== 1 == DIFFERENCE == [#2#day->units] not found in 2nd field == 1 == DIFFERENCE == [#2#hour] not found in 2nd field -== 1 == DIFFERENCE == [#2#hour->units] not found in 2nd field == 1 == DIFFERENCE == [#2#minute] not found in 2nd field -== 1 == DIFFERENCE == [#2#minute->units] not found in 2nd field == 1 == DIFFERENCE == [#2#latitude] not found in 2nd field -== 1 == DIFFERENCE == [#2#latitude->units] not found in 2nd field == 1 == DIFFERENCE == [#2#longitude] not found in 2nd field -== 1 == DIFFERENCE == [#2#longitude->units] not found in 2nd field == 1 == DIFFERENCE == [#2#heightOfStation] not found in 2nd field -== 1 == DIFFERENCE == [#2#heightOfStation->units] not found in 2nd field == 1 == DIFFERENCE == [#4#heightAboveStation] not found in 2nd field -== 1 == DIFFERENCE == [#4#heightAboveStation->units] not found in 2nd field == 1 == DIFFERENCE == [#2#windDirection] not found in 2nd field -== 1 == DIFFERENCE == [#2#windDirection->units] not found in 2nd field == 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind] not found in 2nd field -== 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind->units] not found in 2nd field == 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind] not found in 2nd field -== 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind->units] not found in 2nd field == 1 == DIFFERENCE == [#2#windSpeed] not found in 2nd field -== 1 == DIFFERENCE == [#2#windSpeed->units] not found in 2nd field EOF INPUT=${data_dir}/bufr/metar_with_2_bias.bufr diff --git a/examples/F90/bufr_copy_data.sh b/examples/F90/bufr_copy_data.sh index 07e5e66ed..8d99f9e38 100755 --- a/examples/F90/bufr_copy_data.sh +++ b/examples/F90/bufr_copy_data.sh @@ -22,35 +22,20 @@ rm -f ${TEMP} ${REF} ${MYLOG} cat > ${REF} <units] not found in 2nd field == 1 == DIFFERENCE == [#2#stationType] not found in 2nd field -== 1 == DIFFERENCE == [#2#stationType->units] not found in 2nd field == 1 == DIFFERENCE == [#2#year] not found in 2nd field -== 1 == DIFFERENCE == [#2#year->units] not found in 2nd field == 1 == DIFFERENCE == [#2#month] not found in 2nd field -== 1 == DIFFERENCE == [#2#month->units] not found in 2nd field == 1 == DIFFERENCE == [#2#day] not found in 2nd field -== 1 == DIFFERENCE == [#2#day->units] not found in 2nd field == 1 == DIFFERENCE == [#2#hour] not found in 2nd field -== 1 == DIFFERENCE == [#2#hour->units] not found in 2nd field == 1 == DIFFERENCE == [#2#minute] not found in 2nd field -== 1 == DIFFERENCE == [#2#minute->units] not found in 2nd field == 1 == DIFFERENCE == [#2#latitude] not found in 2nd field -== 1 == DIFFERENCE == [#2#latitude->units] not found in 2nd field == 1 == DIFFERENCE == [#2#longitude] not found in 2nd field -== 1 == DIFFERENCE == [#2#longitude->units] not found in 2nd field == 1 == DIFFERENCE == [#2#heightOfStation] not found in 2nd field -== 1 == DIFFERENCE == [#2#heightOfStation->units] not found in 2nd field == 1 == DIFFERENCE == [#4#heightAboveStation] not found in 2nd field -== 1 == DIFFERENCE == [#4#heightAboveStation->units] not found in 2nd field == 1 == DIFFERENCE == [#2#windDirection] not found in 2nd field -== 1 == DIFFERENCE == [#2#windDirection->units] not found in 2nd field == 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind] not found in 2nd field -== 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind->units] not found in 2nd field == 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind] not found in 2nd field -== 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind->units] not found in 2nd field == 1 == DIFFERENCE == [#2#windSpeed] not found in 2nd field -== 1 == DIFFERENCE == [#2#windSpeed->units] not found in 2nd field EOF INPUT=metar_with_2_bias.bufr diff --git a/examples/python/bufr_copy_data.sh b/examples/python/bufr_copy_data.sh index 0c65810f0..b93b4743a 100755 --- a/examples/python/bufr_copy_data.sh +++ b/examples/python/bufr_copy_data.sh @@ -22,35 +22,20 @@ rm -f ${TEMP} ${REF} ${MYLOG} cat > ${REF} <units] not found in 2nd field == 1 == DIFFERENCE == [#2#stationType] not found in 2nd field -== 1 == DIFFERENCE == [#2#stationType->units] not found in 2nd field == 1 == DIFFERENCE == [#2#year] not found in 2nd field -== 1 == DIFFERENCE == [#2#year->units] not found in 2nd field == 1 == DIFFERENCE == [#2#month] not found in 2nd field -== 1 == DIFFERENCE == [#2#month->units] not found in 2nd field == 1 == DIFFERENCE == [#2#day] not found in 2nd field -== 1 == DIFFERENCE == [#2#day->units] not found in 2nd field == 1 == DIFFERENCE == [#2#hour] not found in 2nd field -== 1 == DIFFERENCE == [#2#hour->units] not found in 2nd field == 1 == DIFFERENCE == [#2#minute] not found in 2nd field -== 1 == DIFFERENCE == [#2#minute->units] not found in 2nd field == 1 == DIFFERENCE == [#2#latitude] not found in 2nd field -== 1 == DIFFERENCE == [#2#latitude->units] not found in 2nd field == 1 == DIFFERENCE == [#2#longitude] not found in 2nd field -== 1 == DIFFERENCE == [#2#longitude->units] not found in 2nd field == 1 == DIFFERENCE == [#2#heightOfStation] not found in 2nd field -== 1 == DIFFERENCE == [#2#heightOfStation->units] not found in 2nd field == 1 == DIFFERENCE == [#4#heightAboveStation] not found in 2nd field -== 1 == DIFFERENCE == [#4#heightAboveStation->units] not found in 2nd field == 1 == DIFFERENCE == [#2#windDirection] not found in 2nd field -== 1 == DIFFERENCE == [#2#windDirection->units] not found in 2nd field == 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind] not found in 2nd field -== 1 == DIFFERENCE == [#2#extremeCounterclockwiseWindDirectionOfAVariableWind->units] not found in 2nd field == 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind] not found in 2nd field -== 1 == DIFFERENCE == [#2#extremeClockwiseWindDirectionOfAVariableWind->units] not found in 2nd field == 1 == DIFFERENCE == [#2#windSpeed] not found in 2nd field -== 1 == DIFFERENCE == [#2#windSpeed->units] not found in 2nd field EOF INPUT=metar_with_2_bias.bufr diff --git a/tools/bufr_compare.c b/tools/bufr_compare.c index 79c897e65..2dba432f9 100644 --- a/tools/bufr_compare.c +++ b/tools/bufr_compare.c @@ -1134,9 +1134,14 @@ static int compare_attributes(grib_handle* handle1, grib_handle* handle2, grib_r /*long native_type = 0;*/ grib_accessor* aa = NULL; if ( (a->attributes[i]->flags & GRIB_ACCESSOR_FLAG_DUMP)== 0 ) { - ++i; /* next attribute */ + ++i; /* next attribute if accessor is not for dumping */ continue; } + if ( (a->attributes[i]->flags & GRIB_ACCESSOR_FLAG_READ_ONLY)!= 0 ) { + ++i; /* next attribute if accessor is read-only */ + continue; + } + aa = a->attributes[i]; /*native_type = grib_accessor_get_native_type(aa); TODO: read only check? */ @@ -1186,11 +1191,13 @@ static int compare_all_dump_keys(grib_handle* handle1, grib_handle* handle2, gri if (!headerMode) { /* See ECC-333: By setting unpack we get ALL the bufr keys. */ /* In headerMode we want just the header ones */ + grib_set_long(handle1,"skipExtraKeyAttributes",1); /* See ECC-745 */ ret = grib_set_long(handle1,"unpack",1); if (ret != GRIB_SUCCESS) { grib_context_log(context, GRIB_LOG_ERROR, "Failed to unpack 1st message: %s", grib_get_error_message(ret)); exit(1); } + grib_set_long(handle2,"skipExtraKeyAttributes",1); /* See ECC-745 */ ret = grib_set_long(handle2,"unpack",1); if (ret != GRIB_SUCCESS) { grib_context_log(context, GRIB_LOG_ERROR, "Failed to unpack 2nd message: %s", grib_get_error_message(ret));