From 50a811a24b0d0d7a2a91379752f54a811b06b504 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Thu, 25 Aug 2016 14:41:46 +0100 Subject: [PATCH] ECC-285: bufr_dump -Efilter (now working) --- src/grib_dumper_class_filter.c | 20 +++++++++- tests/bufr_dump_filter.sh | 71 +++++++++------------------------- 2 files changed, 38 insertions(+), 53 deletions(-) diff --git a/src/grib_dumper_class_filter.c b/src/grib_dumper_class_filter.c index d06f3ee9d..f74de3bba 100644 --- a/src/grib_dumper_class_filter.c +++ b/src/grib_dumper_class_filter.c @@ -297,8 +297,26 @@ static void dump_long(grib_dumper* d, grib_accessor* a, const char* comment) grib_value_count(a,&count); size=count; - if ( (a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 || (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0) + if ( (a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 ) return; + + if ( (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0) { + if (self->isLeaf==0) { + char* prefix; + int dofree=0; + + r=compute_key_rank(h,self->keys,a->name); + if (r!=0) { + prefix=grib_context_malloc_clear(c,sizeof(char)*(strlen(a->name)+10)); + dofree=1; + sprintf(prefix,"#%d#%s",r,a->name); + } else prefix=(char*)a->name; + + dump_attributes(d,a,prefix); + if (dofree) grib_context_free(c,prefix); + depth-=2; + } return; + } if (size>1) { values=(long*)grib_context_malloc_clear(a->context,sizeof(long)*size); diff --git a/tests/bufr_dump_filter.sh b/tests/bufr_dump_filter.sh index 8758054e1..71ae3d045 100755 --- a/tests/bufr_dump_filter.sh +++ b/tests/bufr_dump_filter.sh @@ -28,73 +28,40 @@ fBufrTmp=${label}".bufr.tmp" fRules=${label}.filter #----------------------------------------------------------- -# TODO: not all of our BUFR files pass this test!! +# NOTE: not all of our BUFR files pass this test. bufr_filter is limited +# in what it can do compared to Python or Fortran! # The following do not work: -# bufr_filter fails: -# ias1_240.bufr -- too large, memory exhausted! -# tropical_cyclone.bufr -- Segmentation fault -# bufr_compare fails: -# ocea_133.bufr -- delayedDescriptorReplicationFactor->percentConfidence -# pilo_91.bufr -- ditto -# syno_multi.bufr -- lots of diffs -# temp-land-with-substituted-values.bufr -- delayedDescriptorReplicationFactor->percentConfidence -# temp_101.bufr -- #1#delayedDescriptorReplicationFactor->percentConfidence -# temp_102.bufr -- percentConfidence missing -# temp_106.bufr -- percentConfidence missing +# ias1_240.bufr -- too large, parser out of memory +# tropical_cyclone.bufr -- multi message +# syno_multi.bufr -- multi message #----------------------------------------------------------- -files=" -207003.bufr ashs_139.bufr cmwn_87.bufr hirs_55.bufr nos1_208.bufr ship_9.bufr -PraticaTemp.bufr atap_55.bufr cnow_28.bufr iasi_241.bufr nos2_208.bufr smin_49.bufr -aaen_55.bufr ateu_155.bufr cori_156.bufr ifco_208.bufr nos3_208.bufr smis_49.bufr -aben_55.bufr atms_201.bufr crit_202.bufr ikco_217.bufr nos4_208.bufr smiu_49.bufr -ahws_139.bufr atov_55.bufr csrh_189.bufr itrg_208.bufr nos5_208.bufr smos_203.bufr -airc_142.bufr avhm_87.bufr emsg_189.bufr itwt_233.bufr nos6_208.bufr sn4k_165.bufr -airc_144.bufr avhn_87.bufr emsg_87.bufr j2eo_216.bufr nos7_208.bufr soil_7.bufr -airs_57.bufr avhr_58.bufr euwv_87.bufr j2nb_216.bufr nos8_208.bufr ssbt_127.bufr -alws_139.bufr b002_95.bufr fy3a_154.bufr jaso_214.bufr ocea_131.bufr stuk_7.bufr -amda_144.bufr b002_96.bufr fy3b_154.bufr kond_209.bufr ocea_132.bufr syno_1.bufr -amsa_55.bufr b003_56.bufr g2nd_208.bufr maer_207.bufr ocea_21.bufr syno_2.bufr -amsb_55.bufr b004_145.bufr g2to_206.bufr meta_140.bufr pgps_110.bufr syno_3.bufr -amse_55.bufr b005_87.bufr go15_87.bufr metar_with_2_bias.bufr profiler_european.bufr syno_4.bufr -amsu_55.bufr b005_89.bufr goee_87.bufr mhen_55.bufr rada_250.bufr synop_multi_subset.bufr -amv2_87.bufr b006_96.bufr goes_87.bufr mhsa_55.bufr rado_250.bufr tmr7_129.bufr -amv3_87.bufr b007_31.bufr goga_89.bufr mhsb_55.bufr s4kn_165.bufr tros_31.bufr -asbh_139.bufr bssh_170.bufr gosat.bufr mhse_55.bufr sb19_206.bufr uegabe.bufr -asbl_139.bufr bssh_176.bufr grst_26.bufr mloz_206.bufr sbu8_206.bufr wavb_134.bufr -asca_139.bufr bssh_178.bufr gsd1_208.bufr modi_87.bufr ship_11.bufr -asch_139.bufr bssh_180.bufr gsd2_208.bufr modw_87.bufr ship_12.bufr -ascs_139.bufr btem_109.bufr gsd3_208.bufr monw_87.bufr ship_13.bufr -aseh_139.bufr buoy_27.bufr gst4_26.bufr new.bufr ship_14.bufr -asel_139.bufr cmwi_87.bufr hirb_55.bufr nomi_206.bufr ship_19.bufr -" +files=`cat ${data_dir}/bufr/bufr_data_files.txt` -exclude="" +exclude="ias1_240.bufr syno_multi.bufr tropical_cyclone.bufr" for f in $files do - echo "Test: bufr_dump -Efilter " >> $fLog - echo "file: $f" >> $fLog + process_bufr=1 + for ex in $exclude; do + if [ "$f" = "$ex" ]; then process_bufr=0; break; fi + done + if [ $process_bufr = 1 ]; then + echo "Test: bufr_dump -Efilter " >> $fLog + echo "file: $f" >> $fLog - ${tools_dir}bufr_dump -Efilter $f > $fRules - ${tools_dir}bufr_filter -o $fBufrTmp $fRules $f - ${tools_dir}bufr_compare $fBufrTmp $f + ${tools_dir}bufr_dump -Efilter $f > $fRules - # TODO: Some need to be excluded from the JSON compare - compare_dumps=1 - #for ex in $exclude; do - # if [ "$f" = "$ex" ]; then compare_dumps=0; fi - #done + ${tools_dir}bufr_filter -o $fBufrTmp $fRules $f + ${tools_dir}bufr_compare $fBufrTmp $f - if [ $compare_dumps -eq 1 ]; then TEMP_JSON1=${label}.$f.json TEMP_JSON2=${label}.$fBufrTmp.json ${tools_dir}bufr_dump $f > $TEMP_JSON1 ${tools_dir}bufr_dump $fBufrTmp > $TEMP_JSON2 diff $TEMP_JSON1 $TEMP_JSON2 rm -f $TEMP_JSON1 $TEMP_JSON2 + + rm -f $fBufrTmp $fRules fi - - rm -f $fBufrTmp $fRules done - rm -f $fLog $fBufrTmp $fRules