ECC-763: Split tests which work on data from samples

examples/python/CMakeLists.txt

@@ -29,7 +29,6 @@ endforeach()
 #################################################
 list( APPEND tests_basic
    grib_set_pv
-   bufr_read_header
 )
 list( APPEND tests_extra
    grib_clone
@@ -52,6 +51,7 @@ list( APPEND tests_extra
    bufr_expanded
    bufr_get_keys
    bufr_keys_iterator
+   bufr_read_header
    bufr_read_scatterometer
    bufr_read_tropical_cyclone
    bufr_read_synop 

examples/python/bufr_read_header.sh

@@ -12,13 +12,11 @@ fRef=$f".header.ref"
 fRes=$f".header.test.p"
 REDIRECT=/dev/null
 
-if [ -f "$f" ]; then
+# Write the values into a file and compare with reference
-    # Write the values into a file and compare with reference
+$PYTHON $examples_src/bufr_read_header.py $f 2> $REDIRECT > $fRes
-    $PYTHON $examples_src/bufr_read_header.py $f 2> $REDIRECT > $fRes
 
-    #We compare output to the reference by ignoring the whitespaces
+#We compare output to the reference by ignoring the whitespaces
-    diff -w $fRef $fRes
+diff -w $fRef $fRes
-fi
 
 # Now test reading the header of sample files
 $PYTHON $examples_src/bufr_read_header.py ${ECCODES_SAMPLES_PATH}/BUFR3_local.tmpl

tests/CMakeLists.txt

@@ -51,8 +51,8 @@ list( APPEND tests_no_data_reqd
     uerra
     grib_2nd_order_numValues
     julian
-    bufr_dump
+    bufr_dump_samples
-    bufr_json
+    bufr_json_samples
     bufr_ecc-359
     bufr_ecc-517
     bufr_rdbSubTypes
@@ -62,6 +62,7 @@ list( APPEND tests_no_data_reqd
 # These tests do require data downloads
 list( APPEND tests_data_reqd
     grib_double_cmp
+    bufr_dump_data
     bufr_dump_decode_filter
     bufr_dump_encode_filter
     bufrdc_desc_ref
@@ -73,6 +74,7 @@ list( APPEND tests_data_reqd
     bufr_filter
     bufr_filter_extract_datetime
     bufr_filter_extract_area
+    bufr_json_data
     bufr_ls
     bufr_change_edition
     bufr_keys_iter

tests/Makefile.am

@@ -2,9 +2,9 @@ TESTS = definitions.sh \
         grib_calendar.sh \
         bufrdc_desc_ref.sh \
         bufrdc_ref.sh \
-        bufr_dump.sh \
+        bufr_dump_data.sh \
         bufr_filter.sh \
-        bufr_json.sh \
+        bufr_json_data.sh \
         bufr_compare.sh \
         bufr_copy.sh \
         bufr_count.sh \

tests/bufr_dump_data.sh

@@ -11,7 +11,7 @@
 . ./include.sh
 
 #Define a common label for all the tmp files
-label="bufr_dump_test"
+label="bufr_dump_data_test"
 
 #Create log file
 fLog=${label}".log"
@@ -21,11 +21,6 @@ touch $fLog
 #Define tmp bufr files
 fJsonTmp=${label}".json.tmp"
 
-# Test sample BUFR files
-for file in $ECCODES_SAMPLES_PATH/BUFR*.tmpl; do
-  ${tools_dir}/bufr_dump -O $file >/dev/null
-done
-
 if [ $HAVE_MEMFS -eq 1 ]; then
     unset ECCODES_DEFINITION_PATH
     unset ECCODES_SAMPLES_PATH
@@ -39,9 +34,7 @@ REDIRECT=/dev/null
 
 for file in ${bufr_files}
 do
-  if [ -f ${data_dir}/bufr/$file ]; then
+  ${tools_dir}/bufr_dump -O ${data_dir}/bufr/$file >/dev/null
-    ${tools_dir}/bufr_dump -O ${data_dir}/bufr/$file >/dev/null
-  fi
 done
 
 
@@ -49,19 +42,17 @@ done
 # Testing output when ECCODES_DEBUG is enabled
 #==============================================
 file="aaen_55.bufr"
-if [ -f "$file" ]; then
+export ECCODES_DEBUG=1
-  export ECCODES_DEBUG=1
 
-  # By default debug output goes to stderr
+# By default debug output goes to stderr
-  ${tools_dir}/bufr_dump -O ${data_dir}/bufr/$file 2>&1 | grep -q "BUFR data .*ing"
+${tools_dir}/bufr_dump -O ${data_dir}/bufr/$file 2>&1 | grep -q "BUFR data .*ing"
 
-  # Redirect it to stdout
+# Redirect it to stdout
-  export ECCODES_LOG_STREAM=stdout
+export ECCODES_LOG_STREAM=stdout
-  ${tools_dir}/bufr_dump -O ${data_dir}/bufr/$file | grep -q "BUFR data .*ing"
+${tools_dir}/bufr_dump -O ${data_dir}/bufr/$file | grep -q "BUFR data .*ing"
 
-  unset ECCODES_DEBUG
+unset ECCODES_DEBUG
-  unset ECCODES_LOG_STREAM
+unset ECCODES_LOG_STREAM
-fi
 
 #==============================================
 # Testing a malformed bufr file (see ECC-110)

tests/bufr_dump_samples.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+# Copyright 2005-2018 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+
+#Define a common label for all the tmp files
+label="bufr_dump_samples_test"
+
+#Create log file
+fLog=${label}".log"
+rm -f $fLog
+touch $fLog
+
+#Define tmp bufr files
+fJsonTmp=${label}".json.tmp"
+
+# Test sample BUFR files
+for file in $ECCODES_SAMPLES_PATH/BUFR*.tmpl; do
+  ${tools_dir}/bufr_dump -O $file >/dev/null
+done
+
+rm -f $fLog

tests/bufr_json_data.sh

@@ -12,13 +12,6 @@
 
 REDIRECT=/dev/null
 
-# Test sample BUFR files
-# -------------------------
-for file in $ECCODES_SAMPLES_PATH/BUFR*.tmpl; do
-  ${tools_dir}/bufr_dump -js $file >/dev/null
-  ${tools_dir}/bufr_dump -ja $file >/dev/null
-done
-
 cd ${data_dir}/bufr
 
 if [ $HAVE_MEMFS -eq 1 ]; then
@@ -37,33 +30,28 @@ fi
 # -------------------------
 bufr_files=`cat bufr_data_files.txt`
 for file in ${bufr_files}; do
-  if [ -f "$file" ]; then
+  # Test the various JSON dump modes: 'structure', 'all' and 'flat'
-    # Test the various JSON dump modes: 'structure', 'all' and 'flat'
+  for mode in s a f; do
-    for mode in s a f; do
-      rm -f ${file}.json
-      ${tools_dir}/bufr_dump -j$mode $file 2> $REDIRECT > ${file}.json
-      if test "x$JSON_CHECK" != "x"; then
-        json_xs < ${file}.json >$REDIRECT 2> $REDIRECT
-      fi
-    done
-
     rm -f ${file}.json
-  fi
+    ${tools_dir}/bufr_dump -j$mode $file 2> $REDIRECT > ${file}.json
-done
-
-# ECC-233: Test JSON dump when selecting messages with '-w' switch
-file=tropical_cyclone.bufr
-if [ -f "$file" ]; then
-  for c in 1 3 1/3; do
-    ${tools_dir}/bufr_dump -w count=$c $file 2> $REDIRECT > ${file}.json
     if test "x$JSON_CHECK" != "x"; then
       json_xs < ${file}.json >$REDIRECT 2> $REDIRECT
     fi
   done
-fi
+
+  rm -f ${file}.json
+done
+
+# ECC-233: Test JSON dump when selecting messages with '-w' switch
+file=tropical_cyclone.bufr
+for c in 1 3 1/3; do
+  ${tools_dir}/bufr_dump -w count=$c $file 2> $REDIRECT > ${file}.json
+  if test "x$JSON_CHECK" != "x"; then
+    json_xs < ${file}.json >$REDIRECT 2> $REDIRECT
+  fi
+done
+
 
 # ECC-272
 file=aaen_55.bufr
-if [ -f "$file" ]; then
+${tools_dir}/bufr_dump -jf $file | grep -q -w channelRadiance
-  ${tools_dir}/bufr_dump -jf $file | grep -q -w channelRadiance
-fi

tests/bufr_json_samples.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+# Copyright 2005-2018 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+
+
+# Test sample BUFR files
+# -------------------------
+for file in $ECCODES_SAMPLES_PATH/BUFR*.tmpl; do
+  ${tools_dir}/bufr_dump     $file >/dev/null
+  ${tools_dir}/bufr_dump -ja $file >/dev/null
+  ${tools_dir}/bufr_dump -jf $file >/dev/null
+done