From 06bf3ceba1d038f562f478de25cb1b32bb75ea29 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 24 Oct 2018 15:20:20 +0100
Subject: [PATCH] ECC-763: Split tests which work on data from samples

---
 examples/python/CMakeLists.txt            |  2 +-
 examples/python/bufr_read_header.sh       | 10 ++---
 tests/CMakeLists.txt                      |  6 ++-
 tests/Makefile.am                         |  4 +-
 tests/{bufr_dump.sh => bufr_dump_data.sh} | 29 +++++---------
 tests/bufr_dump_samples.sh                | 29 ++++++++++++++
 tests/{bufr_json.sh => bufr_json_data.sh} | 46 +++++++++--------------
 tests/bufr_json_samples.sh                | 20 ++++++++++
 8 files changed, 87 insertions(+), 59 deletions(-)
 rename tests/{bufr_dump.sh => bufr_dump_data.sh} (76%)
 create mode 100755 tests/bufr_dump_samples.sh
 rename tests/{bufr_json.sh => bufr_json_data.sh} (60%)
 create mode 100755 tests/bufr_json_samples.sh

diff --git a/examples/python/CMakeLists.txt b/examples/python/CMakeLists.txt
index f12fb5de8..9585d5169 100644
--- a/examples/python/CMakeLists.txt
+++ b/examples/python/CMakeLists.txt
@@ -29,7 +29,6 @@ endforeach()
 #################################################
 list( APPEND tests_basic
    grib_set_pv
-   bufr_read_header
 )
 list( APPEND tests_extra
    grib_clone
@@ -52,6 +51,7 @@ list( APPEND tests_extra
    bufr_expanded
    bufr_get_keys
    bufr_keys_iterator
+   bufr_read_header
    bufr_read_scatterometer
    bufr_read_tropical_cyclone
    bufr_read_synop 
diff --git a/examples/python/bufr_read_header.sh b/examples/python/bufr_read_header.sh
index 5cf3c16b5..681fa1ce7 100755
--- a/examples/python/bufr_read_header.sh
+++ b/examples/python/bufr_read_header.sh
@@ -12,13 +12,11 @@ fRef=$f".header.ref"
 fRes=$f".header.test.p"
 REDIRECT=/dev/null
 
-if [ -f "$f" ]; then
-    # Write the values into a file and compare with reference
-    $PYTHON $examples_src/bufr_read_header.py $f 2> $REDIRECT > $fRes
+# Write the values into a file and compare with reference
+$PYTHON $examples_src/bufr_read_header.py $f 2> $REDIRECT > $fRes
 
-    #We compare output to the reference by ignoring the whitespaces
-    diff -w $fRef $fRes
-fi
+#We compare output to the reference by ignoring the whitespaces
+diff -w $fRef $fRes
 
 # Now test reading the header of sample files
 $PYTHON $examples_src/bufr_read_header.py ${ECCODES_SAMPLES_PATH}/BUFR3_local.tmpl
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 0e3b21f84..6d9874d23 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -51,8 +51,8 @@ list( APPEND tests_no_data_reqd
     uerra
     grib_2nd_order_numValues
     julian
-    bufr_dump
-    bufr_json
+    bufr_dump_samples
+    bufr_json_samples
     bufr_ecc-359
     bufr_ecc-517
     bufr_rdbSubTypes
@@ -62,6 +62,7 @@ list( APPEND tests_no_data_reqd
 # These tests do require data downloads
 list( APPEND tests_data_reqd
     grib_double_cmp
+    bufr_dump_data
     bufr_dump_decode_filter
     bufr_dump_encode_filter
     bufrdc_desc_ref
@@ -73,6 +74,7 @@ list( APPEND tests_data_reqd
     bufr_filter
     bufr_filter_extract_datetime
     bufr_filter_extract_area
+    bufr_json_data
     bufr_ls
     bufr_change_edition
     bufr_keys_iter
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 7dc791909..ed9dc6c25 100755
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -2,9 +2,9 @@ TESTS = definitions.sh \
         grib_calendar.sh \
         bufrdc_desc_ref.sh \
         bufrdc_ref.sh \
-        bufr_dump.sh \
+        bufr_dump_data.sh \
         bufr_filter.sh \
-        bufr_json.sh \
+        bufr_json_data.sh \
         bufr_compare.sh \
         bufr_copy.sh \
         bufr_count.sh \
diff --git a/tests/bufr_dump.sh b/tests/bufr_dump_data.sh
similarity index 76%
rename from tests/bufr_dump.sh
rename to tests/bufr_dump_data.sh
index 5bcb1ddef..568bfd300 100755
--- a/tests/bufr_dump.sh
+++ b/tests/bufr_dump_data.sh
@@ -11,7 +11,7 @@
 . ./include.sh
 
 #Define a common label for all the tmp files
-label="bufr_dump_test"
+label="bufr_dump_data_test"
 
 #Create log file
 fLog=${label}".log"
@@ -21,11 +21,6 @@ touch $fLog
 #Define tmp bufr files
 fJsonTmp=${label}".json.tmp"
 
-# Test sample BUFR files
-for file in $ECCODES_SAMPLES_PATH/BUFR*.tmpl; do
-  ${tools_dir}/bufr_dump -O $file >/dev/null
-done
-
 if [ $HAVE_MEMFS -eq 1 ]; then
     unset ECCODES_DEFINITION_PATH
     unset ECCODES_SAMPLES_PATH
@@ -39,9 +34,7 @@ REDIRECT=/dev/null
 
 for file in ${bufr_files}
 do
-  if [ -f ${data_dir}/bufr/$file ]; then
-    ${tools_dir}/bufr_dump -O ${data_dir}/bufr/$file >/dev/null
-  fi
+  ${tools_dir}/bufr_dump -O ${data_dir}/bufr/$file >/dev/null
 done
 
 
@@ -49,19 +42,17 @@ done
 # Testing output when ECCODES_DEBUG is enabled
 #==============================================
 file="aaen_55.bufr"
-if [ -f "$file" ]; then
-  export ECCODES_DEBUG=1
+export ECCODES_DEBUG=1
 
-  # By default debug output goes to stderr
-  ${tools_dir}/bufr_dump -O ${data_dir}/bufr/$file 2>&1 | grep -q "BUFR data .*ing"
+# By default debug output goes to stderr
+${tools_dir}/bufr_dump -O ${data_dir}/bufr/$file 2>&1 | grep -q "BUFR data .*ing"
 
-  # Redirect it to stdout
-  export ECCODES_LOG_STREAM=stdout
-  ${tools_dir}/bufr_dump -O ${data_dir}/bufr/$file | grep -q "BUFR data .*ing"
+# Redirect it to stdout
+export ECCODES_LOG_STREAM=stdout
+${tools_dir}/bufr_dump -O ${data_dir}/bufr/$file | grep -q "BUFR data .*ing"
 
-  unset ECCODES_DEBUG
-  unset ECCODES_LOG_STREAM
-fi
+unset ECCODES_DEBUG
+unset ECCODES_LOG_STREAM
 
 #==============================================
 # Testing a malformed bufr file (see ECC-110)
diff --git a/tests/bufr_dump_samples.sh b/tests/bufr_dump_samples.sh
new file mode 100755
index 000000000..ceb5966e1
--- /dev/null
+++ b/tests/bufr_dump_samples.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+# Copyright 2005-2018 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+
+#Define a common label for all the tmp files
+label="bufr_dump_samples_test"
+
+#Create log file
+fLog=${label}".log"
+rm -f $fLog
+touch $fLog
+
+#Define tmp bufr files
+fJsonTmp=${label}".json.tmp"
+
+# Test sample BUFR files
+for file in $ECCODES_SAMPLES_PATH/BUFR*.tmpl; do
+  ${tools_dir}/bufr_dump -O $file >/dev/null
+done
+
+rm -f $fLog
diff --git a/tests/bufr_json.sh b/tests/bufr_json_data.sh
similarity index 60%
rename from tests/bufr_json.sh
rename to tests/bufr_json_data.sh
index 64ba3cc26..3885dc1c8 100755
--- a/tests/bufr_json.sh
+++ b/tests/bufr_json_data.sh
@@ -12,13 +12,6 @@
 
 REDIRECT=/dev/null
 
-# Test sample BUFR files
-# -------------------------
-for file in $ECCODES_SAMPLES_PATH/BUFR*.tmpl; do
-  ${tools_dir}/bufr_dump -js $file >/dev/null
-  ${tools_dir}/bufr_dump -ja $file >/dev/null
-done
-
 cd ${data_dir}/bufr
 
 if [ $HAVE_MEMFS -eq 1 ]; then
@@ -37,33 +30,28 @@ fi
 # -------------------------
 bufr_files=`cat bufr_data_files.txt`
 for file in ${bufr_files}; do
-  if [ -f "$file" ]; then
-    # Test the various JSON dump modes: 'structure', 'all' and 'flat'
-    for mode in s a f; do
-      rm -f ${file}.json
-      ${tools_dir}/bufr_dump -j$mode $file 2> $REDIRECT > ${file}.json
-      if test "x$JSON_CHECK" != "x"; then
-        json_xs < ${file}.json >$REDIRECT 2> $REDIRECT
-      fi
-    done
-
+  # Test the various JSON dump modes: 'structure', 'all' and 'flat'
+  for mode in s a f; do
     rm -f ${file}.json
-  fi
-done
-
-# ECC-233: Test JSON dump when selecting messages with '-w' switch
-file=tropical_cyclone.bufr
-if [ -f "$file" ]; then
-  for c in 1 3 1/3; do
-    ${tools_dir}/bufr_dump -w count=$c $file 2> $REDIRECT > ${file}.json
+    ${tools_dir}/bufr_dump -j$mode $file 2> $REDIRECT > ${file}.json
     if test "x$JSON_CHECK" != "x"; then
       json_xs < ${file}.json >$REDIRECT 2> $REDIRECT
     fi
   done
-fi
+
+  rm -f ${file}.json
+done
+
+# ECC-233: Test JSON dump when selecting messages with '-w' switch
+file=tropical_cyclone.bufr
+for c in 1 3 1/3; do
+  ${tools_dir}/bufr_dump -w count=$c $file 2> $REDIRECT > ${file}.json
+  if test "x$JSON_CHECK" != "x"; then
+    json_xs < ${file}.json >$REDIRECT 2> $REDIRECT
+  fi
+done
+
 
 # ECC-272
 file=aaen_55.bufr
-if [ -f "$file" ]; then
-  ${tools_dir}/bufr_dump -jf $file | grep -q -w channelRadiance
-fi
+${tools_dir}/bufr_dump -jf $file | grep -q -w channelRadiance
diff --git a/tests/bufr_json_samples.sh b/tests/bufr_json_samples.sh
new file mode 100755
index 000000000..28e872d33
--- /dev/null
+++ b/tests/bufr_json_samples.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+# Copyright 2005-2018 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+
+
+# Test sample BUFR files
+# -------------------------
+for file in $ECCODES_SAMPLES_PATH/BUFR*.tmpl; do
+  ${tools_dir}/bufr_dump     $file >/dev/null
+  ${tools_dir}/bufr_dump -ja $file >/dev/null
+  ${tools_dir}/bufr_dump -jf $file >/dev/null
+done