From 82bf7266767bdbd75973ff17b13de5b6393ddceb Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 25 Apr 2018 13:58:19 +0100
Subject: [PATCH 01/66] ECC-604: Added test for timing and analysis

---
 tests/CMakeLists.txt  |   9 +++
 tests/grib_ecc-604.c  | 163 ++++++++++++++++++++++++++++++++++++++++++
 tests/grib_ecc-604.sh |  30 ++++++++
 3 files changed, 202 insertions(+)
 create mode 100644 tests/grib_ecc-604.c
 create mode 100755 tests/grib_ecc-604.sh

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 6214de6ed..0e758efcc 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -217,6 +217,15 @@ if( ENABLE_EXTRA_TESTS AND HAVE_ECCODES_THREADS )
                       TYPE SCRIPT
                       COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_encode_pthreads.sh
     )
+    ecbuild_add_executable( TARGET    grib_ecc-604
+                            NOINSTALL
+                            SOURCES   grib_ecc-604.c
+                            LIBS      eccodes
+    )
+    ecbuild_add_test( TARGET eccodes_t_grib_ecc-604
+                      TYPE SCRIPT
+                      COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_ecc-604.sh
+    )
 endif()
 
 ecbuild_add_test( TARGET eccodes_t_grib_to_netcdf
diff --git a/tests/grib_ecc-604.c b/tests/grib_ecc-604.c
new file mode 100644
index 000000000..5281dc123
--- /dev/null
+++ b/tests/grib_ecc-604.c
@@ -0,0 +1,163 @@
+#include <stdio.h>
+#include <time.h>
+#include <pthread.h>
+#include <assert.h>
+#include <string.h>
+
+#include "grib_api.h"
+
+#define NUM_THREADS 8
+#define FILES_PER_ITERATION 500
+#define TEMPLATE "../../share/eccodes/samples/gg_sfc_grib2.tmpl"
+
+
+static int encode_file(char *template_file, char *output_file)
+{
+    FILE *in = NULL;
+    FILE *out = NULL;
+    grib_handle *source_handle = NULL;
+    const void *buffer = NULL;
+    size_t size = 0;
+    int err = 0;
+ 
+    in = fopen(template_file,"r"); assert(in);
+    out = fopen(output_file,"w");  assert(out);
+ 
+    /* loop over the messages in the source grib and clone them */
+    while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL)
+    {
+        grib_handle *clone_handle = grib_handle_clone(source_handle);
+ 
+        if (clone_handle == NULL) {
+            perror("ERROR: could not clone field");
+            return 1;
+        }
+ 
+        //GRIB_CHECK(grib_set_long(clone_handle, "centre", 250),0);
+
+        size_t values_len= 0;
+        GRIB_CHECK(grib_get_size(clone_handle, "values", &values_len),0);
+
+        int i;
+        double *values = NULL; 
+           values = (double*)malloc(values_len*sizeof(double));
+        long count;
+        double d,e;
+
+
+        d=10e-8;
+        e=d;
+        count=1;
+        for (i=0;i<values_len;i++) {
+            if (count>100) {e*=10; count=1;}
+            values[i]=d;
+            /*printf("%g \n",values[i]);*/
+            d+=e;
+            count++;
+        }
+ 
+        GRIB_CHECK(grib_set_long(clone_handle,"bitsPerValue",16),0);
+ 
+        /* set data values */
+        GRIB_CHECK(grib_set_double_array(clone_handle,"values",values,values_len),0);
+ 
+ 
+        /* get the coded message in a buffer */
+        GRIB_CHECK(grib_get_message(clone_handle,&buffer,&size),0);
+        /* write the buffer to a file */
+        if(fwrite(buffer,1,size,out) != size) {
+            perror(output_file);
+            return 1;
+        }
+        grib_handle_delete(clone_handle);
+        grib_handle_delete(source_handle);
+    }
+ 
+    fclose(out);
+    fclose(in);
+ 
+    return 0;
+}
+
+void* do_stuff(void *data);
+
+/* Structure for passing data to threads */
+struct v
+{
+    int number; 
+    char *data; /* column */
+};
+
+void *runner(void *ptr); /* the thread */
+
+int main(int argc, char **argv)
+{
+    int i;
+    int thread_counter = 0;
+    int parallel = !(argc>1 && strcmp(argv[1],"seq")==0);
+
+    if (parallel) {
+        printf("Running parallel in %d threads.\n", NUM_THREADS);
+    } else {
+        printf("Running sequentially in %d runs.\n", NUM_THREADS);
+    }
+    
+    pthread_t workers[NUM_THREADS];
+    
+    /* We have to create M * N worker threads */
+    for (i = 0; i < NUM_THREADS; i++) {
+
+        struct v *data = (struct v *) malloc(sizeof(struct v));
+        data->number = i;
+        data->data = NULL;
+
+        if (parallel) {
+            /* Now we will create the thread passing it data as a paramater*/
+            pthread_create(&workers[thread_counter], NULL, runner, data);
+            //pthread_join(workers[thread_counter], NULL);
+            thread_counter++;
+        } else {
+            do_stuff(data);
+        }
+    }
+    
+    /* Waiting for threads to complete */
+    if (parallel) {
+        for (i = 0; i < NUM_THREADS; i++)
+        {
+            pthread_join(workers[i], NULL);
+        }
+    }
+    
+    return 0;
+}
+
+void *runner(void *ptr)
+{    
+    do_stuff(ptr);    
+    pthread_exit(0);
+}
+
+void *do_stuff(void *ptr)
+{
+    /* Casting paramater to struct v pointer */
+    struct v *data = ptr;
+
+    char output_file[50];
+
+    for (int i=0; i<FILES_PER_ITERATION;i++) {
+        sprintf(output_file,"output/output_file_%d-%d.grib",data->number,i);
+        encode_file(TEMPLATE,output_file);
+    }
+
+    time_t ltime;
+    struct tm result;
+    char stime[32];
+
+    ltime = time(NULL);
+    localtime_r(&ltime, &result);
+    strftime(stime, 32, "%H:%M:%S", &result); // Try to get milliseconds here too
+//    asctime_r(&result, stime);
+
+    printf("%s: Worker %d finished.\n", stime,data->number);
+}
diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
new file mode 100755
index 000000000..2cef48748
--- /dev/null
+++ b/tests/grib_ecc-604.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+# Copyright 2005-2018 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+
+label="grib_ecc-604"
+
+temp_dir=tempdir.${label}
+rm -fr $temp_dir
+mkdir -p $temp_dir
+cd $temp_dir
+
+OUTPUT=output
+mkdir -p $OUTPUT
+time ${test_dir}/grib_ecc-604
+
+#for ofile in $OUTPUT/output*grib; do
+#  ${tools_dir}/grib_compare -H $ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl $ofile
+#done
+
+# Clean up
+cd $test_dir
+#rm -fr $temp_dir

From 606bcf9e6189f4bca6dd5f0e2c9a985461ad73a4 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 25 Apr 2018 16:14:16 +0100
Subject: [PATCH 02/66] ECC-604: Test

---
 tests/grib_ecc-604.c  |  2 +-
 tests/grib_ecc-604.sh | 12 +++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tests/grib_ecc-604.c b/tests/grib_ecc-604.c
index 5281dc123..a1603cb03 100644
--- a/tests/grib_ecc-604.c
+++ b/tests/grib_ecc-604.c
@@ -7,7 +7,7 @@
 #include "grib_api.h"
 
 #define NUM_THREADS 8
-#define FILES_PER_ITERATION 500
+#define FILES_PER_ITERATION 400
 #define TEMPLATE "../../share/eccodes/samples/gg_sfc_grib2.tmpl"
 
 
diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index 2cef48748..6134a5eae 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -21,9 +21,15 @@ OUTPUT=output
 mkdir -p $OUTPUT
 time ${test_dir}/grib_ecc-604
 
-#for ofile in $OUTPUT/output*grib; do
-#  ${tools_dir}/grib_compare -H $ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl $ofile
-#done
+num=0
+for ofile in $OUTPUT/output*grib; do
+  ${tools_dir}/grib_compare -H $ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl $ofile
+  num=`expr $num + 1`
+  # If there are too many output files, comparing each one will take a long time!
+  if [ $num -gt 1000 ]; then
+    break
+  fi
+done
 
 # Clean up
 cd $test_dir

From 962160a8ece151c72116ba409b492dd9098cae3c Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 25 Apr 2018 16:15:07 +0100
Subject: [PATCH 03/66] ECC-604: Test: clean up

---
 tests/grib_ecc-604.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index 6134a5eae..537549b1f 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -33,4 +33,4 @@ done
 
 # Clean up
 cd $test_dir
-#rm -fr $temp_dir
+rm -fr $temp_dir

From f3ecdf0ee52a3d7cec559d698fa40205ced05ca6 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 25 Apr 2018 18:45:27 +0100
Subject: [PATCH 04/66] ECC-604: Fix error in test

---
 tests/grib_ecc-604.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/grib_ecc-604.c b/tests/grib_ecc-604.c
index a1603cb03..5cc775c47 100644
--- a/tests/grib_ecc-604.c
+++ b/tests/grib_ecc-604.c
@@ -7,7 +7,7 @@
 #include "grib_api.h"
 
 #define NUM_THREADS 8
-#define FILES_PER_ITERATION 400
+#define FILES_PER_ITERATION 150
 #define TEMPLATE "../../share/eccodes/samples/gg_sfc_grib2.tmpl"
 
 
@@ -79,7 +79,7 @@ static int encode_file(char *template_file, char *output_file)
     return 0;
 }
 
-void* do_stuff(void *data);
+void do_stuff(void *data);
 
 /* Structure for passing data to threads */
 struct v
@@ -138,7 +138,7 @@ void *runner(void *ptr)
     pthread_exit(0);
 }
 
-void *do_stuff(void *ptr)
+void do_stuff(void *ptr)
 {
     /* Casting paramater to struct v pointer */
     struct v *data = ptr;

From 91efbc51b9b60a9a7b1e48dad642287e7497590f Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 27 Apr 2018 12:54:55 +0100
Subject: [PATCH 05/66] ECC-604: Test

---
 tests/grib_ecc-604.c  | 49 +++++++++++++++++++------------------------
 tests/grib_ecc-604.sh | 18 ++++++++--------
 2 files changed, 31 insertions(+), 36 deletions(-)

diff --git a/tests/grib_ecc-604.c b/tests/grib_ecc-604.c
index 5cc775c47..3ed17e8c9 100644
--- a/tests/grib_ecc-604.c
+++ b/tests/grib_ecc-604.c
@@ -7,7 +7,7 @@
 #include "grib_api.h"
 
 #define NUM_THREADS 8
-#define FILES_PER_ITERATION 150
+#define FILES_PER_ITERATION 300
 #define TEMPLATE "../../share/eccodes/samples/gg_sfc_grib2.tmpl"
 
 
@@ -19,20 +19,20 @@ static int encode_file(char *template_file, char *output_file)
     const void *buffer = NULL;
     size_t size = 0;
     int err = 0;
- 
+
     in = fopen(template_file,"r"); assert(in);
     out = fopen(output_file,"w");  assert(out);
- 
-    /* loop over the messages in the source grib and clone them */
+
+    /* loop over the messages in the source GRIB and clone them */
     while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL)
     {
         grib_handle *clone_handle = grib_handle_clone(source_handle);
- 
+
         if (clone_handle == NULL) {
             perror("ERROR: could not clone field");
             return 1;
         }
- 
+
         //GRIB_CHECK(grib_set_long(clone_handle, "centre", 250),0);
 
         size_t values_len= 0;
@@ -40,11 +40,10 @@ static int encode_file(char *template_file, char *output_file)
 
         int i;
         double *values = NULL; 
-           values = (double*)malloc(values_len*sizeof(double));
+        values = (double*)malloc(values_len*sizeof(double));
         long count;
         double d,e;
 
-
         d=10e-8;
         e=d;
         count=1;
@@ -55,16 +54,13 @@ static int encode_file(char *template_file, char *output_file)
             d+=e;
             count++;
         }
- 
+
         GRIB_CHECK(grib_set_long(clone_handle,"bitsPerValue",16),0);
- 
+
         /* set data values */
         GRIB_CHECK(grib_set_double_array(clone_handle,"values",values,values_len),0);
- 
- 
-        /* get the coded message in a buffer */
+
         GRIB_CHECK(grib_get_message(clone_handle,&buffer,&size),0);
-        /* write the buffer to a file */
         if(fwrite(buffer,1,size,out) != size) {
             perror(output_file);
             return 1;
@@ -72,20 +68,19 @@ static int encode_file(char *template_file, char *output_file)
         grib_handle_delete(clone_handle);
         grib_handle_delete(source_handle);
     }
- 
+
     fclose(out);
     fclose(in);
- 
+
     return 0;
 }
 
 void do_stuff(void *data);
 
 /* Structure for passing data to threads */
-struct v
-{
+struct v {
     int number; 
-    char *data; /* column */
+    char *data;
 };
 
 void *runner(void *ptr); /* the thread */
@@ -101,9 +96,9 @@ int main(int argc, char **argv)
     } else {
         printf("Running sequentially in %d runs.\n", NUM_THREADS);
     }
-    
+
     pthread_t workers[NUM_THREADS];
-    
+
     /* We have to create M * N worker threads */
     for (i = 0; i < NUM_THREADS; i++) {
 
@@ -112,15 +107,15 @@ int main(int argc, char **argv)
         data->data = NULL;
 
         if (parallel) {
-            /* Now we will create the thread passing it data as a paramater*/
+            /* Now we will create the thread passing it data as an argument */
             pthread_create(&workers[thread_counter], NULL, runner, data);
-            //pthread_join(workers[thread_counter], NULL);
+            /*pthread_join(workers[thread_counter], NULL);*/
             thread_counter++;
         } else {
             do_stuff(data);
         }
     }
-    
+
     /* Waiting for threads to complete */
     if (parallel) {
         for (i = 0; i < NUM_THREADS; i++)
@@ -128,7 +123,7 @@ int main(int argc, char **argv)
             pthread_join(workers[i], NULL);
         }
     }
-    
+
     return 0;
 }
 
@@ -140,7 +135,7 @@ void *runner(void *ptr)
 
 void do_stuff(void *ptr)
 {
-    /* Casting paramater to struct v pointer */
+    /* Cast argument to struct v pointer */
     struct v *data = ptr;
 
     char output_file[50];
@@ -157,7 +152,7 @@ void do_stuff(void *ptr)
     ltime = time(NULL);
     localtime_r(&ltime, &result);
     strftime(stime, 32, "%H:%M:%S", &result); // Try to get milliseconds here too
-//    asctime_r(&result, stime);
+    //    asctime_r(&result, stime);
 
     printf("%s: Worker %d finished.\n", stime,data->number);
 }
diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index 537549b1f..e8bae65c0 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -21,15 +21,15 @@ OUTPUT=output
 mkdir -p $OUTPUT
 time ${test_dir}/grib_ecc-604
 
-num=0
-for ofile in $OUTPUT/output*grib; do
-  ${tools_dir}/grib_compare -H $ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl $ofile
-  num=`expr $num + 1`
-  # If there are too many output files, comparing each one will take a long time!
-  if [ $num -gt 1000 ]; then
-    break
-  fi
-done
+#num=0
+#for ofile in $OUTPUT/output_file_*grib; do
+#  ${tools_dir}/grib_compare -H $ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl $ofile
+#  num=`expr $num + 1`
+#  # If there are too many output files, comparing each one will take a long time!
+#  if [ $num -gt 500 ]; then
+#    break
+#  fi
+#done
 
 # Clean up
 cd $test_dir

From 238924e4276f240a1a38c6e847c6bb16bec0000b Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 1 May 2018 14:24:42 +0100
Subject: [PATCH 06/66] ECC-604: Pass file to test. Fix memory leaks

---
 tests/grib_ecc-604.c  | 62 ++++++++++++++++++++-----------------------
 tests/grib_ecc-604.sh | 11 +++++++-
 2 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/tests/grib_ecc-604.c b/tests/grib_ecc-604.c
index 3ed17e8c9..433108834 100644
--- a/tests/grib_ecc-604.c
+++ b/tests/grib_ecc-604.c
@@ -7,43 +7,35 @@
 #include "grib_api.h"
 
 #define NUM_THREADS 8
-#define FILES_PER_ITERATION 300
-#define TEMPLATE "../../share/eccodes/samples/gg_sfc_grib2.tmpl"
-
+#define FILES_PER_ITERATION 200
+static char* INPUT_FILE = NULL;
 
 static int encode_file(char *template_file, char *output_file)
 {
-    FILE *in = NULL;
-    FILE *out = NULL;
+    FILE *in, *out;
     grib_handle *source_handle = NULL;
     const void *buffer = NULL;
     size_t size = 0;
     int err = 0;
+    double *values;
 
     in = fopen(template_file,"r"); assert(in);
     out = fopen(output_file,"w");  assert(out);
 
     /* loop over the messages in the source GRIB and clone them */
-    while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL)
-    {
-        grib_handle *clone_handle = grib_handle_clone(source_handle);
-
-        if (clone_handle == NULL) {
-            perror("ERROR: could not clone field");
-            return 1;
-        }
-
-        //GRIB_CHECK(grib_set_long(clone_handle, "centre", 250),0);
-
-        size_t values_len= 0;
-        GRIB_CHECK(grib_get_size(clone_handle, "values", &values_len),0);
-
+    while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
         int i;
-        double *values = NULL; 
-        values = (double*)malloc(values_len*sizeof(double));
         long count;
         double d,e;
+        size_t values_len= 0;
 
+        grib_handle *clone_handle = grib_handle_clone(source_handle);
+        assert(clone_handle);
+
+        /*GRIB_CHECK(grib_set_long(clone_handle, "centre", 250),0);*/
+        GRIB_CHECK(grib_get_size(clone_handle, "values", &values_len),0);
+
+        values = (double*)malloc(values_len*sizeof(double));
         d=10e-8;
         e=d;
         count=1;
@@ -67,6 +59,7 @@ static int encode_file(char *template_file, char *output_file)
         }
         grib_handle_delete(clone_handle);
         grib_handle_delete(source_handle);
+        free(values);
     }
 
     fclose(out);
@@ -79,7 +72,7 @@ void do_stuff(void *data);
 
 /* Structure for passing data to threads */
 struct v {
-    int number; 
+    int number;
     char *data;
 };
 
@@ -89,8 +82,14 @@ int main(int argc, char **argv)
 {
     int i;
     int thread_counter = 0;
-    int parallel = !(argc>1 && strcmp(argv[1],"seq")==0);
-
+    int parallel = 1;
+    if (argc<2) {
+        return 1;
+    }
+    INPUT_FILE= argv[1];
+    if (argc>2 && strcmp(argv[2],"seq")==0) {
+        parallel = 0;
+    }
     if (parallel) {
         printf("Running parallel in %d threads.\n", NUM_THREADS);
     } else {
@@ -101,7 +100,6 @@ int main(int argc, char **argv)
 
     /* We have to create M * N worker threads */
     for (i = 0; i < NUM_THREADS; i++) {
-
         struct v *data = (struct v *) malloc(sizeof(struct v));
         data->number = i;
         data->data = NULL;
@@ -116,10 +114,8 @@ int main(int argc, char **argv)
         }
     }
 
-    /* Waiting for threads to complete */
     if (parallel) {
-        for (i = 0; i < NUM_THREADS; i++)
-        {
+        for (i = 0; i < NUM_THREADS; i++)  {
             pthread_join(workers[i], NULL);
         }
     }
@@ -128,8 +124,8 @@ int main(int argc, char **argv)
 }
 
 void *runner(void *ptr)
-{    
-    do_stuff(ptr);    
+{
+    do_stuff(ptr);
     pthread_exit(0);
 }
 
@@ -142,7 +138,7 @@ void do_stuff(void *ptr)
 
     for (int i=0; i<FILES_PER_ITERATION;i++) {
         sprintf(output_file,"output/output_file_%d-%d.grib",data->number,i);
-        encode_file(TEMPLATE,output_file);
+        encode_file(INPUT_FILE,output_file);
     }
 
     time_t ltime;
@@ -151,8 +147,8 @@ void do_stuff(void *ptr)
 
     ltime = time(NULL);
     localtime_r(&ltime, &result);
-    strftime(stime, 32, "%H:%M:%S", &result); // Try to get milliseconds here too
-    //    asctime_r(&result, stime);
+    strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
+    /* asctime_r(&result, stime); */
 
     printf("%s: Worker %d finished.\n", stime,data->number);
 }
diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index e8bae65c0..3fe6e14b9 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -19,8 +19,17 @@ cd $temp_dir
 
 OUTPUT=output
 mkdir -p $OUTPUT
-time ${test_dir}/grib_ecc-604
+input=$ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl
+time ${test_dir}/grib_ecc-604 $input
 
+# Run with forge
+# -----------------
+# module swap forge/18.0.1
+# map ${test_dir}/grib_ecc-604 $input &
+#
+
+# Validate results
+# -----------------
 #num=0
 #for ofile in $OUTPUT/output_file_*grib; do
 #  ${tools_dir}/grib_compare -H $ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl $ofile

From 5e3bbf960cec8eb6bd4613793a9613dc979ff31b Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 1 May 2018 15:44:34 +0100
Subject: [PATCH 07/66] ECC-604: Make test configurable for different number of
 threads/iterations

---
 tests/grib_ecc-604.c  | 40 +++++++++++++++++++++-------------------
 tests/grib_ecc-604.sh |  6 ++++--
 2 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/tests/grib_ecc-604.c b/tests/grib_ecc-604.c
index 433108834..f1e3a677b 100644
--- a/tests/grib_ecc-604.c
+++ b/tests/grib_ecc-604.c
@@ -1,14 +1,12 @@
-#include <stdio.h>
 #include <time.h>
 #include <pthread.h>
 #include <assert.h>
-#include <string.h>
 
 #include "grib_api.h"
 
-#define NUM_THREADS 8
-#define FILES_PER_ITERATION 200
-static char* INPUT_FILE = NULL;
+static size_t NUM_THREADS         = 8;
+static size_t FILES_PER_ITERATION = 300;
+static char* INPUT_FILE           = NULL;
 
 static int encode_file(char *template_file, char *output_file)
 {
@@ -42,7 +40,6 @@ static int encode_file(char *template_file, char *output_file)
         for (i=0;i<values_len;i++) {
             if (count>100) {e*=10; count=1;}
             values[i]=d;
-            /*printf("%g \n",values[i]);*/
             d+=e;
             count++;
         }
@@ -72,7 +69,7 @@ void do_stuff(void *data);
 
 /* Structure for passing data to threads */
 struct v {
-    int number;
+    size_t number;
     char *data;
 };
 
@@ -80,25 +77,30 @@ void *runner(void *ptr); /* the thread */
 
 int main(int argc, char **argv)
 {
-    int i;
+    size_t i;
     int thread_counter = 0;
     int parallel = 1;
-    if (argc<2) {
+    const char* prog = argv[0];
+    char* mode;
+    if (argc!=5) {
+        fprintf(stderr, "Usage:\n\t%s seq file numRuns numIter\nOr\n\t%s par file numThreads numIter\n", prog, prog);
         return 1;
     }
-    INPUT_FILE= argv[1];
-    if (argc>2 && strcmp(argv[2],"seq")==0) {
+    mode = argv[1];
+    INPUT_FILE= argv[2];
+    NUM_THREADS = atol(argv[3]);
+    FILES_PER_ITERATION = atol(argv[4]);
+
+    if (strcmp(mode,"seq")==0) {
         parallel = 0;
     }
     if (parallel) {
-        printf("Running parallel in %d threads.\n", NUM_THREADS);
+        printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
     } else {
-        printf("Running sequentially in %d runs.\n", NUM_THREADS);
+        printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
     }
 
     pthread_t workers[NUM_THREADS];
-
-    /* We have to create M * N worker threads */
     for (i = 0; i < NUM_THREADS; i++) {
         struct v *data = (struct v *) malloc(sizeof(struct v));
         data->number = i;
@@ -133,11 +135,11 @@ void do_stuff(void *ptr)
 {
     /* Cast argument to struct v pointer */
     struct v *data = ptr;
-
+    size_t i;
     char output_file[50];
 
-    for (int i=0; i<FILES_PER_ITERATION;i++) {
-        sprintf(output_file,"output/output_file_%d-%d.grib",data->number,i);
+    for (i=0; i<FILES_PER_ITERATION;i++) {
+        sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
         encode_file(INPUT_FILE,output_file);
     }
 
@@ -150,5 +152,5 @@ void do_stuff(void *ptr)
     strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
     /* asctime_r(&result, stime); */
 
-    printf("%s: Worker %d finished.\n", stime,data->number);
+    printf("%s: Worker %ld finished.\n", stime, data->number);
 }
diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index 3fe6e14b9..964983658 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -17,15 +17,17 @@ rm -fr $temp_dir
 mkdir -p $temp_dir
 cd $temp_dir
 
+NUM_THREADS=8
+NUM_ITER=300
 OUTPUT=output
 mkdir -p $OUTPUT
 input=$ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl
-time ${test_dir}/grib_ecc-604 $input
+time ${test_dir}/grib_ecc-604 par $input $NUM_THREADS $NUM_ITER
 
 # Run with forge
 # -----------------
 # module swap forge/18.0.1
-# map ${test_dir}/grib_ecc-604 $input &
+# map ${test_dir}/grib_ecc-604 par $input $NUM_THREADS $NUM_ITER &
 #
 
 # Validate results

From 4b97f0de71acd01aa4d7dee52a41ae529c544c83 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 2 May 2018 16:01:30 +0100
Subject: [PATCH 08/66] ECC-604: Verify output from test

---
 examples/C/grib_pthreads.c | 3 ++-
 tests/grib_ecc-604.sh      | 5 +++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/examples/C/grib_pthreads.c b/examples/C/grib_pthreads.c
index 443315d4a..15cdcb669 100644
--- a/examples/C/grib_pthreads.c
+++ b/examples/C/grib_pthreads.c
@@ -35,7 +35,7 @@ static void* process_grib(void* threadID)
     CODES_CHECK(codes_set_string(h,"indicatorOfUnitOfTimeRange", "s", &str_len),0);
     CODES_CHECK(codes_set_string(h,"stepUnits", "s", &str_len),0);
     CODES_CHECK(codes_set_long(h, "endStep", 86400), 0);
-    CODES_CHECK(codes_set_long(h,"centre", 80),0);
+    //CODES_CHECK(codes_set_long(h,"centre", 80),0);
 
     CODES_CHECK(codes_get_long(h,"endStep", &step),0);
     CODES_CHECK(codes_get_string(h, "indicatorOfUnitOfTimeRange", mystring, &str_len),0);
@@ -64,6 +64,7 @@ int main(int argc, char** argv)
         printf("Creating thread %ld\n", i);
         error = pthread_create(&threads[i], NULL, process_grib, (void *)i);
         if (error) {
+            assert(0);
             return 1;
         }
     }
diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index 964983658..52c7be28d 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -41,6 +41,11 @@ time ${test_dir}/grib_ecc-604 par $input $NUM_THREADS $NUM_ITER
 #    break
 #  fi
 #done
+echo "Checking every output file is identical..."
+set +x
+res=`cksum $OUTPUT/output_file_* | awk '{print $1}' | sort -u`
+set -x
+[ "$res" = "2572910830" ]
 
 # Clean up
 cd $test_dir

From 9bf77b446dec0912da7513f569d2e51ccbb215d1 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Thu, 3 May 2018 17:52:37 +0100
Subject: [PATCH 09/66] ECC-604: First improvement of threaded performance.
 Needs more testing

---
 src/action.c       | 7 ++++---
 src/grib_context.c | 2 ++
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/action.c b/src/action.c
index 065760f5e..1da14d41e 100644
--- a/src/action.c
+++ b/src/action.c
@@ -115,10 +115,11 @@ int grib_create_accessor(grib_section* p, grib_action* a,  grib_loader* h)
     {
         if(c->create_accessor) {
             int ret;
-            GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
-            GRIB_MUTEX_LOCK(&mutex1);
+            /* ECC-604: Do not lock excessively */
+            /*GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);*/
+            /*GRIB_MUTEX_LOCK(&mutex1);*/
             ret=c->create_accessor(p, a, h);
-            GRIB_MUTEX_UNLOCK(&mutex1);
+            /*GRIB_MUTEX_UNLOCK(&mutex1);*/
             return ret;
         }
         c = c->super ? *(c->super) : NULL;
diff --git a/src/grib_context.c b/src/grib_context.c
index 507fd1548..d73ac1b34 100644
--- a/src/grib_context.c
+++ b/src/grib_context.c
@@ -618,7 +618,9 @@ char *grib_context_full_defs_path(grib_context* c,const char* basename)
     if(*basename == '/' || *basename ==  '.') {
         return (char*)basename;
     } else {
+        GRIB_MUTEX_LOCK(&mutex_c);  /* See ECC-604 */
         fullpath=(grib_string_list*)grib_trie_get(c->def_files,basename);
+        GRIB_MUTEX_UNLOCK(&mutex_c);
         if (fullpath!=NULL) {
             return fullpath->value;
         }

From 3d59d181c5bc49ebe4ff87d3d9deb36f1a19680a Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 8 May 2018 14:11:01 +0100
Subject: [PATCH 10/66] ECC-604: Add extra tests

---
 tests/CMakeLists.txt                       |  10 +-
 tests/{grib_ecc-604.c => grib_ecc-604-1.c} |   3 +
 tests/grib_ecc-604-2.c                     | 156 +++++++++++++++++++++
 tests/grib_ecc-604.sh                      |  50 +++----
 4 files changed, 190 insertions(+), 29 deletions(-)
 rename tests/{grib_ecc-604.c => grib_ecc-604-1.c} (99%)
 create mode 100644 tests/grib_ecc-604-2.c

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 0e758efcc..f7c1c7e93 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -217,9 +217,15 @@ if( ENABLE_EXTRA_TESTS AND HAVE_ECCODES_THREADS )
                       TYPE SCRIPT
                       COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_encode_pthreads.sh
     )
-    ecbuild_add_executable( TARGET    grib_ecc-604
+
+    ecbuild_add_executable( TARGET    grib_ecc-604-1
                             NOINSTALL
-                            SOURCES   grib_ecc-604.c
+                            SOURCES   grib_ecc-604-1.c
+                            LIBS      eccodes
+    )
+    ecbuild_add_executable( TARGET    grib_ecc-604-2
+                            NOINSTALL
+                            SOURCES   grib_ecc-604-2.c
                             LIBS      eccodes
     )
     ecbuild_add_test( TARGET eccodes_t_grib_ecc-604
diff --git a/tests/grib_ecc-604.c b/tests/grib_ecc-604-1.c
similarity index 99%
rename from tests/grib_ecc-604.c
rename to tests/grib_ecc-604-1.c
index f1e3a677b..fb4f72982 100644
--- a/tests/grib_ecc-604.c
+++ b/tests/grib_ecc-604-1.c
@@ -1,3 +1,6 @@
+/*
+ * Test for ECC-604:
+ */
 #include <time.h>
 #include <pthread.h>
 #include <assert.h>
diff --git a/tests/grib_ecc-604-2.c b/tests/grib_ecc-604-2.c
new file mode 100644
index 000000000..331edbf5d
--- /dev/null
+++ b/tests/grib_ecc-604-2.c
@@ -0,0 +1,156 @@
+/*
+ * Test for ECC-604: This version does not clone the GRIB handle
+ */
+
+#include <time.h>
+#include <pthread.h>
+#include <assert.h>
+
+#include "grib_api.h"
+
+static size_t NUM_THREADS         = 8;
+static size_t FILES_PER_ITERATION = 300;
+static char* INPUT_FILE           = NULL;
+
+static int encode_file(char *template_file, char *output_file)
+{
+    FILE *in, *out;
+    grib_handle *source_handle = NULL;
+    const void *buffer = NULL;
+    size_t size = 0;
+    int err = 0;
+    double *values;
+
+    in = fopen(template_file,"r"); assert(in);
+    out = fopen(output_file,"w");  assert(out);
+
+    /* loop over the messages in the source GRIB */
+    while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
+        int i;
+        long count;
+        double d,e;
+        size_t values_len= 0;
+
+        /*GRIB_CHECK(grib_set_long(source_handle, "centre", 250),0);*/
+        GRIB_CHECK(grib_get_size(source_handle, "values", &values_len),0);
+
+        values = (double*)malloc(values_len*sizeof(double));
+        d=10e-8;
+        e=d;
+        count=1;
+        for (i=0;i<values_len;i++) {
+            if (count>100) {e*=10; count=1;}
+            values[i]=d;
+            d+=e;
+            count++;
+        }
+
+        GRIB_CHECK(grib_set_long(source_handle,"bitsPerValue",16),0);
+
+        /* set data values */
+        GRIB_CHECK(grib_set_double_array(source_handle,"values",values,values_len),0);
+
+        GRIB_CHECK(grib_get_message(source_handle,&buffer,&size),0);
+        if(fwrite(buffer,1,size,out) != size) {
+            perror(output_file);
+            return 1;
+        }
+        grib_handle_delete(source_handle);
+        free(values);
+    }
+
+    fclose(out);
+    fclose(in);
+
+    return 0;
+}
+
+void do_stuff(void *data);
+
+/* Structure for passing data to threads */
+struct v {
+    size_t number;
+    char *data;
+};
+
+void *runner(void *ptr); /* the thread */
+
+int main(int argc, char **argv)
+{
+    size_t i;
+    int thread_counter = 0;
+    int parallel = 1;
+    const char* prog = argv[0];
+    char* mode;
+    if (argc!=5) {
+        fprintf(stderr, "Usage:\n\t%s seq file numRuns numIter\nOr\n\t%s par file numThreads numIter\n", prog, prog);
+        return 1;
+    }
+    mode = argv[1];
+    INPUT_FILE= argv[2];
+    NUM_THREADS = atol(argv[3]);
+    FILES_PER_ITERATION = atol(argv[4]);
+
+    if (strcmp(mode,"seq")==0) {
+        parallel = 0;
+    }
+    if (parallel) {
+        printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+    } else {
+        printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+    }
+
+    pthread_t workers[NUM_THREADS];
+    for (i = 0; i < NUM_THREADS; i++) {
+        struct v *data = (struct v *) malloc(sizeof(struct v));
+        data->number = i;
+        data->data = NULL;
+
+        if (parallel) {
+            /* Now we will create the thread passing it data as an argument */
+            pthread_create(&workers[thread_counter], NULL, runner, data);
+            /*pthread_join(workers[thread_counter], NULL);*/
+            thread_counter++;
+        } else {
+            do_stuff(data);
+        }
+    }
+
+    if (parallel) {
+        for (i = 0; i < NUM_THREADS; i++)  {
+            pthread_join(workers[i], NULL);
+        }
+    }
+
+    return 0;
+}
+
+void *runner(void *ptr)
+{
+    do_stuff(ptr);
+    pthread_exit(0);
+}
+
+void do_stuff(void *ptr)
+{
+    /* Cast argument to struct v pointer */
+    struct v *data = ptr;
+    size_t i;
+    char output_file[50];
+
+    for (i=0; i<FILES_PER_ITERATION;i++) {
+        sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
+        encode_file(INPUT_FILE,output_file);
+    }
+
+    time_t ltime;
+    struct tm result;
+    char stime[32];
+
+    ltime = time(NULL);
+    localtime_r(&ltime, &result);
+    strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
+    /* asctime_r(&result, stime); */
+
+    printf("%s: Worker %ld finished.\n", stime, data->number);
+}
diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index 52c7be28d..3f0c3bf3f 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -11,41 +11,37 @@
 . ./include.sh
 
 label="grib_ecc-604"
-
 temp_dir=tempdir.${label}
-rm -fr $temp_dir
-mkdir -p $temp_dir
-cd $temp_dir
+
+validate()
+{
+    echo "Checking every output file is identical..."
+    set +x
+    res=`cksum $OUTPUT/output_file_* | awk '{print $1}' | sort -u`
+    set -x
+    [ "$res" = "2572910830" ]
+}
 
 NUM_THREADS=8
 NUM_ITER=300
 OUTPUT=output
-mkdir -p $OUTPUT
 input=$ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl
-time ${test_dir}/grib_ecc-604 par $input $NUM_THREADS $NUM_ITER
 
-# Run with forge
-# -----------------
-# module swap forge/18.0.1
-# map ${test_dir}/grib_ecc-604 par $input $NUM_THREADS $NUM_ITER &
-#
+rm -fr $temp_dir
+mkdir -p $temp_dir
+cd $temp_dir
+
+# Test 01
+mkdir -p $OUTPUT
+time ${test_dir}/grib_ecc-604-1 par $input $NUM_THREADS $NUM_ITER
+validate
+
+# Test 02
+rm -fr $OUTPUT
+mkdir -p $OUTPUT
+time ${test_dir}/grib_ecc-604-2 par $input $NUM_THREADS $NUM_ITER
+validate
 
-# Validate results
-# -----------------
-#num=0
-#for ofile in $OUTPUT/output_file_*grib; do
-#  ${tools_dir}/grib_compare -H $ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl $ofile
-#  num=`expr $num + 1`
-#  # If there are too many output files, comparing each one will take a long time!
-#  if [ $num -gt 500 ]; then
-#    break
-#  fi
-#done
-echo "Checking every output file is identical..."
-set +x
-res=`cksum $OUTPUT/output_file_* | awk '{print $1}' | sort -u`
-set -x
-[ "$res" = "2572910830" ]
 
 # Clean up
 cd $test_dir

From 25acd14b9149f063bdd5ff059d284c3af67be4ae Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 8 May 2018 17:11:45 +0100
Subject: [PATCH 11/66] ECC-604: Add extra tests

---
 tests/CMakeLists.txt   |  17 ++---
 tests/grib_ecc-604-3.c | 152 +++++++++++++++++++++++++++++++++++++++++
 tests/grib_ecc-604.sh  |  11 ++-
 3 files changed, 168 insertions(+), 12 deletions(-)
 create mode 100644 tests/grib_ecc-604-3.c

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index f7c1c7e93..e84844a9c 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -218,16 +218,13 @@ if( ENABLE_EXTRA_TESTS AND HAVE_ECCODES_THREADS )
                       COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_encode_pthreads.sh
     )
 
-    ecbuild_add_executable( TARGET    grib_ecc-604-1
-                            NOINSTALL
-                            SOURCES   grib_ecc-604-1.c
-                            LIBS      eccodes
-    )
-    ecbuild_add_executable( TARGET    grib_ecc-604-2
-                            NOINSTALL
-                            SOURCES   grib_ecc-604-2.c
-                            LIBS      eccodes
-    )
+    foreach( test grib_ecc-604-1 grib_ecc-604-2 grib_ecc-604-3 )
+        ecbuild_add_executable( TARGET    ${test}
+                                NOINSTALL
+                                SOURCES   ${test}.c
+                                LIBS      eccodes
+        )
+    endforeach()
     ecbuild_add_test( TARGET eccodes_t_grib_ecc-604
                       TYPE SCRIPT
                       COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_ecc-604.sh
diff --git a/tests/grib_ecc-604-3.c b/tests/grib_ecc-604-3.c
new file mode 100644
index 000000000..a84d4ad4e
--- /dev/null
+++ b/tests/grib_ecc-604-3.c
@@ -0,0 +1,152 @@
+/*
+ * Test for ECC-604: No output writing
+ */
+#include <time.h>
+#include <pthread.h>
+#include <assert.h>
+
+#include "grib_api.h"
+
+static size_t NUM_THREADS         = 8;
+static size_t FILES_PER_ITERATION = 300;
+static char* INPUT_FILE           = NULL;
+
+static int decode_file(char *template_file)
+{
+    FILE *in;
+    grib_handle *source_handle = NULL;
+    const void *buffer = NULL;
+    size_t size = 0;
+    int err = 0;
+    double *values;
+
+    in = fopen(template_file,"r"); assert(in);
+
+    /* loop over the messages in the source GRIB and clone them */
+    while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
+        int i;
+        long count;
+        double d,e;
+        size_t values_len= 0;
+
+        grib_handle *clone_handle = grib_handle_clone(source_handle);
+        assert(clone_handle);
+
+        /*GRIB_CHECK(grib_set_long(clone_handle, "centre", 250),0);*/
+        GRIB_CHECK(grib_get_size(clone_handle, "values", &values_len),0);
+
+        values = (double*)malloc(values_len*sizeof(double));
+        d=10e-8;
+        e=d;
+        count=1;
+        for (i=0;i<values_len;i++) {
+            if (count>100) {e*=10; count=1;}
+            values[i]=d;
+            d+=e;
+            count++;
+        }
+
+        GRIB_CHECK(grib_set_long(clone_handle,"bitsPerValue",16),0);
+
+        /* set data values */
+        GRIB_CHECK(grib_set_double_array(clone_handle,"values",values,values_len),0);
+
+        GRIB_CHECK(grib_get_message(clone_handle,&buffer,&size),0);
+
+        grib_handle_delete(clone_handle);
+        grib_handle_delete(source_handle);
+        free(values);
+    }
+
+    fclose(in);
+
+    return 0;
+}
+
+void do_stuff(void *data);
+
+/* Structure for passing data to threads */
+struct v {
+    size_t number;
+    char *data;
+};
+
+void *runner(void *ptr); /* the thread */
+
+int main(int argc, char **argv)
+{
+    size_t i;
+    int thread_counter = 0;
+    int parallel = 1;
+    const char* prog = argv[0];
+    char* mode;
+    if (argc!=5) {
+        fprintf(stderr, "Usage:\n\t%s seq file numRuns numIter\nOr\n\t%s par file numThreads numIter\n", prog, prog);
+        return 1;
+    }
+    mode = argv[1];
+    INPUT_FILE= argv[2];
+    NUM_THREADS = atol(argv[3]);
+    FILES_PER_ITERATION = atol(argv[4]);
+
+    if (strcmp(mode,"seq")==0) {
+        parallel = 0;
+    }
+    if (parallel) {
+        printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+    } else {
+        printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+    }
+
+    pthread_t workers[NUM_THREADS];
+    for (i = 0; i < NUM_THREADS; i++) {
+        struct v *data = (struct v *) malloc(sizeof(struct v));
+        data->number = i;
+        data->data = NULL;
+
+        if (parallel) {
+            /* Now we will create the thread passing it data as an argument */
+            pthread_create(&workers[thread_counter], NULL, runner, data);
+            /*pthread_join(workers[thread_counter], NULL);*/
+            thread_counter++;
+        } else {
+            do_stuff(data);
+        }
+    }
+
+    if (parallel) {
+        for (i = 0; i < NUM_THREADS; i++)  {
+            pthread_join(workers[i], NULL);
+        }
+    }
+
+    return 0;
+}
+
+void *runner(void *ptr)
+{
+    do_stuff(ptr);
+    pthread_exit(0);
+}
+
+void do_stuff(void *ptr)
+{
+    /* Cast argument to struct v pointer */
+    struct v *data = ptr;
+    size_t i;
+
+    for (i=0; i<FILES_PER_ITERATION;i++) {
+        decode_file(INPUT_FILE);
+    }
+
+    time_t ltime;
+    struct tm result;
+    char stime[32];
+
+    ltime = time(NULL);
+    localtime_r(&ltime, &result);
+    strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
+    /* asctime_r(&result, stime); */
+
+    printf("%s: Worker %ld finished.\n", stime, data->number);
+}
diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index 3f0c3bf3f..d3e39f1f3 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -31,17 +31,24 @@ rm -fr $temp_dir
 mkdir -p $temp_dir
 cd $temp_dir
 
-# Test 01
+# Test 01: clone + output
+# ------------------------
 mkdir -p $OUTPUT
 time ${test_dir}/grib_ecc-604-1 par $input $NUM_THREADS $NUM_ITER
 validate
 
-# Test 02
+# Test 02: No clone + output
+# --------------------------
 rm -fr $OUTPUT
 mkdir -p $OUTPUT
 time ${test_dir}/grib_ecc-604-2 par $input $NUM_THREADS $NUM_ITER
 validate
 
+# Test 03: clone + no output
+# ---------------------------
+rm -fr $OUTPUT
+time ${test_dir}/grib_ecc-604-3 par $input $NUM_THREADS $NUM_ITER
+
 
 # Clean up
 cd $test_dir

From 2a16a080c07a3f51c4889ac66e239d2a090955c7 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 8 May 2018 18:01:14 +0100
Subject: [PATCH 12/66] ECC-604: Print out what each test does

---
 tests/grib_ecc-604-1.c | 3 ++-
 tests/grib_ecc-604-2.c | 4 +++-
 tests/grib_ecc-604-3.c | 4 +++-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/tests/grib_ecc-604-1.c b/tests/grib_ecc-604-1.c
index fb4f72982..d50ec7958 100644
--- a/tests/grib_ecc-604-1.c
+++ b/tests/grib_ecc-604-1.c
@@ -1,5 +1,5 @@
 /*
- * Test for ECC-604:
+ * Test for ECC-604: Each thread creates a new GRIB handle, clones it and writes it out
  */
 #include <time.h>
 #include <pthread.h>
@@ -99,6 +99,7 @@ int main(int argc, char **argv)
     }
     if (parallel) {
         printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+        printf("Each thread creates a new GRIB handle, clones it and writes it out\n");
     } else {
         printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
     }
diff --git a/tests/grib_ecc-604-2.c b/tests/grib_ecc-604-2.c
index 331edbf5d..fc15d1206 100644
--- a/tests/grib_ecc-604-2.c
+++ b/tests/grib_ecc-604-2.c
@@ -1,5 +1,6 @@
 /*
- * Test for ECC-604: This version does not clone the GRIB handle
+ * Test for ECC-604: Each thread creates a new GRIB handle and writes it out.
+ *                   It does not clone the handle.
  */
 
 #include <time.h>
@@ -96,6 +97,7 @@ int main(int argc, char **argv)
     }
     if (parallel) {
         printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+        printf("Each thread creates a new GRIB handle and writes it out. No cloning the handle\n");
     } else {
         printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
     }
diff --git a/tests/grib_ecc-604-3.c b/tests/grib_ecc-604-3.c
index a84d4ad4e..84b1b5654 100644
--- a/tests/grib_ecc-604-3.c
+++ b/tests/grib_ecc-604-3.c
@@ -1,5 +1,6 @@
 /*
- * Test for ECC-604: No output writing
+ * Test for ECC-604: Each thread creates a new GRIB handle and clones it.
+ *                   No output writing.
  */
 #include <time.h>
 #include <pthread.h>
@@ -94,6 +95,7 @@ int main(int argc, char **argv)
     }
     if (parallel) {
         printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+        printf("Each thread creates a new GRIB handle and clones it. No output writing\n");
     } else {
         printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
     }

From 9006a680c3ea2c8ebbaf534a80d8ba0b21f106d9 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 11 May 2018 11:48:23 +0100
Subject: [PATCH 13/66] ECC-604: Add more operations in threads

---
 tests/grib_ecc-604-1.c | 16 +++++++++++-----
 tests/grib_ecc-604-2.c | 16 +++++++++++-----
 tests/grib_ecc-604-3.c | 16 +++++++++++-----
 tests/grib_ecc-604.sh  | 15 ++++++++++-----
 4 files changed, 43 insertions(+), 20 deletions(-)

diff --git a/tests/grib_ecc-604-1.c b/tests/grib_ecc-604-1.c
index d50ec7958..2ba11d628 100644
--- a/tests/grib_ecc-604-1.c
+++ b/tests/grib_ecc-604-1.c
@@ -7,9 +7,10 @@
 
 #include "grib_api.h"
 
-static size_t NUM_THREADS         = 8;
-static size_t FILES_PER_ITERATION = 300;
-static char* INPUT_FILE           = NULL;
+/* These are passed in via argv */
+static size_t NUM_THREADS         = 0;
+static size_t FILES_PER_ITERATION = 0;
+static char*  INPUT_FILE          = NULL;
 
 static int encode_file(char *template_file, char *output_file)
 {
@@ -28,7 +29,8 @@ static int encode_file(char *template_file, char *output_file)
         int i;
         long count;
         double d,e;
-        size_t values_len= 0;
+        size_t values_len = 0;
+        size_t str_len = 20;
 
         grib_handle *clone_handle = grib_handle_clone(source_handle);
         assert(clone_handle);
@@ -44,10 +46,14 @@ static int encode_file(char *template_file, char *output_file)
             if (count>100) {e*=10; count=1;}
             values[i]=d;
             d+=e;
+            if (d > 10000) d = 0;
             count++;
         }
 
-        GRIB_CHECK(grib_set_long(clone_handle,"bitsPerValue",16),0);
+        GRIB_CHECK(grib_set_string(clone_handle,"stepUnits", "s", &str_len),0);
+        GRIB_CHECK(grib_set_long(clone_handle, "startStep", 43200), 0);
+        GRIB_CHECK(grib_set_long(clone_handle, "endStep", 86400), 0);
+        GRIB_CHECK(grib_set_long(clone_handle, "bitsPerValue", 16),0);
 
         /* set data values */
         GRIB_CHECK(grib_set_double_array(clone_handle,"values",values,values_len),0);
diff --git a/tests/grib_ecc-604-2.c b/tests/grib_ecc-604-2.c
index fc15d1206..183a6ca13 100644
--- a/tests/grib_ecc-604-2.c
+++ b/tests/grib_ecc-604-2.c
@@ -9,9 +9,10 @@
 
 #include "grib_api.h"
 
-static size_t NUM_THREADS         = 8;
-static size_t FILES_PER_ITERATION = 300;
-static char* INPUT_FILE           = NULL;
+/* These are passed in via argv */
+static size_t NUM_THREADS         = 0;
+static size_t FILES_PER_ITERATION = 0;
+static char*  INPUT_FILE          = NULL;
 
 static int encode_file(char *template_file, char *output_file)
 {
@@ -30,7 +31,8 @@ static int encode_file(char *template_file, char *output_file)
         int i;
         long count;
         double d,e;
-        size_t values_len= 0;
+        size_t values_len = 0;
+        size_t str_len = 20;
 
         /*GRIB_CHECK(grib_set_long(source_handle, "centre", 250),0);*/
         GRIB_CHECK(grib_get_size(source_handle, "values", &values_len),0);
@@ -43,10 +45,14 @@ static int encode_file(char *template_file, char *output_file)
             if (count>100) {e*=10; count=1;}
             values[i]=d;
             d+=e;
+            if (d > 10000) d = 0;
             count++;
         }
 
-        GRIB_CHECK(grib_set_long(source_handle,"bitsPerValue",16),0);
+        GRIB_CHECK(grib_set_string(source_handle,"stepUnits", "s", &str_len),0);
+        GRIB_CHECK(grib_set_long(source_handle, "startStep", 43200), 0);
+        GRIB_CHECK(grib_set_long(source_handle, "endStep", 86400), 0);
+        GRIB_CHECK(grib_set_long(source_handle, "bitsPerValue", 16),0);
 
         /* set data values */
         GRIB_CHECK(grib_set_double_array(source_handle,"values",values,values_len),0);
diff --git a/tests/grib_ecc-604-3.c b/tests/grib_ecc-604-3.c
index 84b1b5654..8e9c4075d 100644
--- a/tests/grib_ecc-604-3.c
+++ b/tests/grib_ecc-604-3.c
@@ -8,9 +8,10 @@
 
 #include "grib_api.h"
 
-static size_t NUM_THREADS         = 8;
-static size_t FILES_PER_ITERATION = 300;
-static char* INPUT_FILE           = NULL;
+/* These are passed in via argv */
+static size_t NUM_THREADS         = 0;
+static size_t FILES_PER_ITERATION = 0;
+static char*  INPUT_FILE          = NULL;
 
 static int decode_file(char *template_file)
 {
@@ -28,7 +29,8 @@ static int decode_file(char *template_file)
         int i;
         long count;
         double d,e;
-        size_t values_len= 0;
+        size_t values_len = 0;
+        size_t str_len = 20;
 
         grib_handle *clone_handle = grib_handle_clone(source_handle);
         assert(clone_handle);
@@ -44,10 +46,14 @@ static int decode_file(char *template_file)
             if (count>100) {e*=10; count=1;}
             values[i]=d;
             d+=e;
+            if (d > 10000) d = 0;
             count++;
         }
 
-        GRIB_CHECK(grib_set_long(clone_handle,"bitsPerValue",16),0);
+        GRIB_CHECK(grib_set_string(clone_handle,"stepUnits", "s", &str_len),0);
+        GRIB_CHECK(grib_set_long(clone_handle, "startStep", 43200), 0);
+        GRIB_CHECK(grib_set_long(clone_handle, "endStep", 86400), 0);
+        GRIB_CHECK(grib_set_long(clone_handle, "bitsPerValue", 16),0);
 
         /* set data values */
         GRIB_CHECK(grib_set_double_array(clone_handle,"values",values,values_len),0);
diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index d3e39f1f3..6147c0217 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -16,22 +16,27 @@ temp_dir=tempdir.${label}
 validate()
 {
     echo "Checking every output file is identical..."
+    # Get checksum of first file
+    ck1=`cksum $OUTPUT/output_file_0-0.grib | awk '{print $1}'`
     set +x
+    # Get checksum of all of them and sort unique
     res=`cksum $OUTPUT/output_file_* | awk '{print $1}' | sort -u`
     set -x
-    [ "$res" = "2572910830" ]
+    # Should be the same as the first
+    [ "$res" = "$ck1" ]
 }
 
-NUM_THREADS=8
-NUM_ITER=300
+NUM_THREADS=6
+NUM_ITER=200
 OUTPUT=output
 input=$ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl
+# input=$ECCODES_SAMPLES_PATH/gg_sfc_grib1.tmpl
 
 rm -fr $temp_dir
 mkdir -p $temp_dir
 cd $temp_dir
 
-# Test 01: clone + output
+# Test 01: Clone + output
 # ------------------------
 mkdir -p $OUTPUT
 time ${test_dir}/grib_ecc-604-1 par $input $NUM_THREADS $NUM_ITER
@@ -44,7 +49,7 @@ mkdir -p $OUTPUT
 time ${test_dir}/grib_ecc-604-2 par $input $NUM_THREADS $NUM_ITER
 validate
 
-# Test 03: clone + no output
+# Test 03: Clone + no output
 # ---------------------------
 rm -fr $OUTPUT
 time ${test_dir}/grib_ecc-604-3 par $input $NUM_THREADS $NUM_ITER

From 880f3b843700b9222f6f5a8de064b9371d8186bc Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 11 May 2018 12:44:46 +0100
Subject: [PATCH 14/66] ECC-604: Replace global variable with data member

---
 src/grib_accessor_class_g1fcperiod.c   |  1 +
 src/grib_accessor_class_g1step_range.c | 15 ++++++++-------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/grib_accessor_class_g1fcperiod.c b/src/grib_accessor_class_g1fcperiod.c
index 4f5ca5147..53a46d061 100644
--- a/src/grib_accessor_class_g1fcperiod.c
+++ b/src/grib_accessor_class_g1fcperiod.c
@@ -48,6 +48,7 @@ typedef struct grib_accessor_g1fcperiod {
 	const char *step_unit;
 	const char *stepType;
 	const char *patch_fp_precip;
+	int error_on_units;
 /* Members defined in g1fcperiod */
 } grib_accessor_g1fcperiod;
 
diff --git a/src/grib_accessor_class_g1step_range.c b/src/grib_accessor_class_g1step_range.c
index 502e43102..7fa4990eb 100644
--- a/src/grib_accessor_class_g1step_range.c
+++ b/src/grib_accessor_class_g1step_range.c
@@ -30,6 +30,7 @@
    MEMBERS    = const char *step_unit
    MEMBERS    = const char *stepType
    MEMBERS    = const char *patch_fp_precip
+   MEMBERS    = int error_on_units
    END_CLASS_DEF
 
  */
@@ -71,6 +72,7 @@ typedef struct grib_accessor_g1step_range {
 	const char *step_unit;
 	const char *stepType;
 	const char *patch_fp_precip;
+	int error_on_units;
 } grib_accessor_g1step_range;
 
 extern grib_accessor_class* grib_accessor_class_abstract_long_vector;
@@ -164,6 +166,7 @@ static void init(grib_accessor* a,const long l, grib_arguments* c)
     self->step_unit           = grib_arguments_get_name(h,c,n++);
     self->stepType            = grib_arguments_get_name(h,c,n++);
     self->patch_fp_precip     = grib_arguments_get_name(h,c,n++);
+    self->error_on_units = 1;
 
     self->number_of_elements=2;
     self->v=(long*)grib_context_malloc_clear(h->context,
@@ -180,8 +183,6 @@ static void dump(grib_accessor* a, grib_dumper* dumper)
 
 }
 
-static int error_on_units=1;
-
 static int u2s1[] =  {
         60,      /* (0) minutes */
         3600,    /* (1) hour    */
@@ -324,7 +325,7 @@ static int unpack_string(grib_accessor* a, char* val, size_t *len)
         else
             sprintf(step_unit_string,"h");
 
-        if (error_on_units) {
+        if (self->error_on_units) {
             grib_get_long_internal(hand,self->unit,&unit);
             if (unit==254) {
                 unit=15; /* See ECC-316 */
@@ -640,9 +641,9 @@ static int pack_long(grib_accessor* a, const long* val, size_t *len)
         return pack_string( a,buff,&bufflen);
     case  0 :
         self->pack_index=-1;
-        error_on_units=0;
+        self->error_on_units=0;
         unpack_string(a,sval,&svallen);
-        error_on_units=1;
+        self->error_on_units=1;
         while (*p != '-' && *p != '\0' ) p++;
         if (*p=='-') {
             sprintf(buff,"%ld-%s",*val,++p);
@@ -656,9 +657,9 @@ static int pack_long(grib_accessor* a, const long* val, size_t *len)
         return pack_string( a,buff,&bufflen);
     case  1 :
         self->pack_index=-1;
-        error_on_units=0;
+        self->error_on_units=0;
         unpack_string(a,sval,&svallen);
-        error_on_units=1;
+        self->error_on_units=1;
         while (*p != '-' && *p != '\0' ) p++;
         if (*p=='-') {
             *p='\0';

From 6900ce19b90301997a8af8796d060e9ae53fa47c Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 11 May 2018 16:51:17 +0100
Subject: [PATCH 15/66] ECC-689: GRIB decode performance: remove redundant keys
 from GRIB2 messages

---
 definitions/boot.def       |  8 +-------
 definitions/grib1/boot.def |  6 ++++++
 tests/grib1to2.sh          | 10 ++++++++++
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/definitions/boot.def b/definitions/boot.def
index f41ab73cb..abebf4f09 100644
--- a/definitions/boot.def
+++ b/definitions/boot.def
@@ -24,19 +24,13 @@ constant zero=0 : hidden;
 constant one=1 : hidden;
 constant two=1 : hidden;
 constant three=1 : hidden;
-constant eight=8 : hidden;
 constant hundred=100 : hidden;
-constant eleven=11 : hidden;
 constant true="true" :hidden;
 constant false="false" : hidden;
 transient truncateLaplacian=0 : hidden;
 constant marsDir="mars" : no_copy,hidden;
 constant present=1 : hidden;
-constant epsPoint=1 : hidden;
-constant epsContinous=11 : hidden;
-constant epsStatisticsPoint=2 : hidden;
-constant epsStatisticsContinous=12 : hidden;
-alias epsStatistics=zero : hidden;
+# alias epsStatistics=zero : hidden;
 
 constant defaultParameter = 0 : hidden;
 constant defaultName="unknown" :hidden;
diff --git a/definitions/grib1/boot.def b/definitions/grib1/boot.def
index ffb46fa6f..6f8e8d66a 100644
--- a/definitions/grib1/boot.def
+++ b/definitions/grib1/boot.def
@@ -9,6 +9,12 @@
 
 constant ieeeFloats   = 0 : hidden, edition_specific;
 transient eps=0;
+constant eight=8 : hidden;
+constant eleven=11 : hidden;
+constant epsPoint=1 : hidden;
+constant epsContinous=11 : hidden;
+constant epsStatisticsPoint=2 : hidden;
+constant epsStatisticsContinous=12 : hidden;
 
 meta headersOnly headers_only();
 
diff --git a/tests/grib1to2.sh b/tests/grib1to2.sh
index b6b868c38..f4829ef58 100755
--- a/tests/grib1to2.sh
+++ b/tests/grib1to2.sh
@@ -51,6 +51,7 @@ do
 done
 
 # ECC-457 ECMWF total precipitation
+# ----------------------------------
 input=${data_dir}/tp_ecmwf.grib
 output=temp.grib1to2.grib
 ${tools_dir}/grib_set -s edition=2 $input $output
@@ -58,6 +59,15 @@ res=`${tools_dir}/grib_get -w count=1 -p edition,paramId,units $output`
 [ "$res" = "2 228228 kg m**-2" ]
 rm -f $output
 
+# Local Definition 30
+# --------------------
+sample_g1=$ECCODES_SAMPLES_PATH/GRIB1.tmpl
+${tools_dir}/grib_set -s setLocalDefinition=1,localDefinitionNumber=30,stepType=accum,edition=2 $sample_g1 $output
+grib_check_key_equals $output productDefinitionTemplateNumber 11
+${tools_dir}/grib_set -s setLocalDefinition=1,localDefinitionNumber=30,type=em,stepType=accum,edition=2 $sample_g1 $output
+grib_check_key_equals $output productDefinitionTemplateNumber 12
+rm -f $output
+
 #sed "s:toolsdir:${tools_dir}/:" ${tools_dir}/grib1to2.txt > ${tools_dir}/grib1to2.test
 #chmod +x ${tools_dir}/grib1to2.test
 #${tools_dir}/grib1to2.test -f ${data_dir}/test.grib1 ${data_dir}/test.grib2

From d9a9ce5fe49d81e76a76683361f51e6b037352da Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 18 May 2018 14:36:06 +0100
Subject: [PATCH 16/66] Tests: do both decoding and encoding of data values

---
 tests/grib_ecc-604-1.c | 13 +++----------
 tests/grib_ecc-604-2.c | 13 +++----------
 tests/grib_ecc-604-3.c | 13 +++----------
 3 files changed, 9 insertions(+), 30 deletions(-)

diff --git a/tests/grib_ecc-604-1.c b/tests/grib_ecc-604-1.c
index 2ba11d628..fb1fbb96f 100644
--- a/tests/grib_ecc-604-1.c
+++ b/tests/grib_ecc-604-1.c
@@ -27,8 +27,6 @@ static int encode_file(char *template_file, char *output_file)
     /* loop over the messages in the source GRIB and clone them */
     while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
         int i;
-        long count;
-        double d,e;
         size_t values_len = 0;
         size_t str_len = 20;
 
@@ -39,15 +37,10 @@ static int encode_file(char *template_file, char *output_file)
         GRIB_CHECK(grib_get_size(clone_handle, "values", &values_len),0);
 
         values = (double*)malloc(values_len*sizeof(double));
-        d=10e-8;
-        e=d;
-        count=1;
+        GRIB_CHECK(grib_get_double_array(clone_handle, "values", values, &values_len),0);
+
         for (i=0;i<values_len;i++) {
-            if (count>100) {e*=10; count=1;}
-            values[i]=d;
-            d+=e;
-            if (d > 10000) d = 0;
-            count++;
+            values[i] *= 0.9;
         }
 
         GRIB_CHECK(grib_set_string(clone_handle,"stepUnits", "s", &str_len),0);
diff --git a/tests/grib_ecc-604-2.c b/tests/grib_ecc-604-2.c
index 183a6ca13..31bcc9246 100644
--- a/tests/grib_ecc-604-2.c
+++ b/tests/grib_ecc-604-2.c
@@ -29,8 +29,6 @@ static int encode_file(char *template_file, char *output_file)
     /* loop over the messages in the source GRIB */
     while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
         int i;
-        long count;
-        double d,e;
         size_t values_len = 0;
         size_t str_len = 20;
 
@@ -38,15 +36,10 @@ static int encode_file(char *template_file, char *output_file)
         GRIB_CHECK(grib_get_size(source_handle, "values", &values_len),0);
 
         values = (double*)malloc(values_len*sizeof(double));
-        d=10e-8;
-        e=d;
-        count=1;
+        GRIB_CHECK(grib_get_double_array(source_handle, "values", values, &values_len),0);
+
         for (i=0;i<values_len;i++) {
-            if (count>100) {e*=10; count=1;}
-            values[i]=d;
-            d+=e;
-            if (d > 10000) d = 0;
-            count++;
+            values[i] *= 0.9;
         }
 
         GRIB_CHECK(grib_set_string(source_handle,"stepUnits", "s", &str_len),0);
diff --git a/tests/grib_ecc-604-3.c b/tests/grib_ecc-604-3.c
index 8e9c4075d..6ab10b29f 100644
--- a/tests/grib_ecc-604-3.c
+++ b/tests/grib_ecc-604-3.c
@@ -27,8 +27,6 @@ static int decode_file(char *template_file)
     /* loop over the messages in the source GRIB and clone them */
     while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
         int i;
-        long count;
-        double d,e;
         size_t values_len = 0;
         size_t str_len = 20;
 
@@ -39,15 +37,10 @@ static int decode_file(char *template_file)
         GRIB_CHECK(grib_get_size(clone_handle, "values", &values_len),0);
 
         values = (double*)malloc(values_len*sizeof(double));
-        d=10e-8;
-        e=d;
-        count=1;
+        GRIB_CHECK(grib_get_double_array(clone_handle, "values", values, &values_len),0);
+
         for (i=0;i<values_len;i++) {
-            if (count>100) {e*=10; count=1;}
-            values[i]=d;
-            d+=e;
-            if (d > 10000) d = 0;
-            count++;
+            values[i] *= 0.9;
         }
 
         GRIB_CHECK(grib_set_string(clone_handle,"stepUnits", "s", &str_len),0);

From d43a19a9aed905bb66bcf33ef686d3863bc84f9f Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 18 May 2018 16:01:38 +0100
Subject: [PATCH 17/66] Tests: Run with different input GRIB files

---
 tests/grib_ecc-604.sh | 54 ++++++++++++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 21 deletions(-)

diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index 6147c0217..641964519 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -13,6 +13,10 @@
 label="grib_ecc-604"
 temp_dir=tempdir.${label}
 
+NUM_THREADS=6
+NUM_ITER=100
+OUTPUT=output
+
 validate()
 {
     echo "Checking every output file is identical..."
@@ -25,34 +29,42 @@ validate()
     # Should be the same as the first
     [ "$res" = "$ck1" ]
 }
+process()
+{
+    input=$1 # The input GRIB file
 
-NUM_THREADS=6
-NUM_ITER=200
-OUTPUT=output
-input=$ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl
-# input=$ECCODES_SAMPLES_PATH/gg_sfc_grib1.tmpl
+    # Test 01: Clone + output
+    # ------------------------
+    rm -fr $OUTPUT;   mkdir -p $OUTPUT
+    time ${test_dir}/grib_ecc-604-1 par $input $NUM_THREADS $NUM_ITER
+    validate
 
+    # Test 02: No clone + output
+    # --------------------------
+    rm -fr $OUTPUT;   mkdir -p $OUTPUT
+    time ${test_dir}/grib_ecc-604-2 par $input $NUM_THREADS $NUM_ITER
+    validate
+
+    # Test 03: Clone + no output
+    # ---------------------------
+    rm -fr $OUTPUT
+    time ${test_dir}/grib_ecc-604-3 par $input $NUM_THREADS $NUM_ITER
+    # Nothing to validate as there is no output
+}
+###################################################
 rm -fr $temp_dir
 mkdir -p $temp_dir
 cd $temp_dir
 
-# Test 01: Clone + output
-# ------------------------
-mkdir -p $OUTPUT
-time ${test_dir}/grib_ecc-604-1 par $input $NUM_THREADS $NUM_ITER
-validate
+# GRIB1 inputs
+process $ECCODES_SAMPLES_PATH/gg_sfc_grib1.tmpl
+process ${data_dir}/gen_bitmap.grib
+process ${data_dir}/spectral_complex.grib1
+#process ${data_dir}/gen.grib
 
-# Test 02: No clone + output
-# --------------------------
-rm -fr $OUTPUT
-mkdir -p $OUTPUT
-time ${test_dir}/grib_ecc-604-2 par $input $NUM_THREADS $NUM_ITER
-validate
-
-# Test 03: Clone + no output
-# ---------------------------
-rm -fr $OUTPUT
-time ${test_dir}/grib_ecc-604-3 par $input $NUM_THREADS $NUM_ITER
+# GRIB2 inputs
+process $ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl
+process ${data_dir}/sample.grib2
 
 
 # Clean up

From 20936572c5dc6d166048acbaf7e7ea006c23afcc Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 18 May 2018 18:03:55 +0100
Subject: [PATCH 18/66] ECC-604: Use one mutex to lock both concept file and
 template

---
 src/grib_parse_utils.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/grib_parse_utils.c b/src/grib_parse_utils.c
index eaf3ec5e4..cd3a606fa 100644
--- a/src/grib_parse_utils.c
+++ b/src/grib_parse_utils.c
@@ -9,7 +9,7 @@
  */
 
 /***************************************************************************
- *   Jean Baptiste Filippi - 01.11.2005                                                           *
+ *   Jean Baptiste Filippi - 01.11.2005                                    *
  *   Enrico Fucile
  *                                                                         *
  ***************************************************************************/
@@ -728,16 +728,16 @@ static grib_action* grib_parse_stream(grib_context* gc, const char* filename)
 grib_concept_value* grib_parse_concept_file( grib_context* gc,const char* filename)
 {
     GRIB_MUTEX_INIT_ONCE(&once,&init);
-    GRIB_MUTEX_LOCK(&mutex_concept);
+    GRIB_MUTEX_LOCK(&mutex_file);
 
     gc = gc ? gc : grib_context_get_default();
     grib_parser_context = gc;
 
     if(parse(gc,filename) == 0) {
-        GRIB_MUTEX_UNLOCK(&mutex_concept);
+        GRIB_MUTEX_UNLOCK(&mutex_file);
         return grib_parser_concept;
     } else {
-        GRIB_MUTEX_UNLOCK(&mutex_concept);
+        GRIB_MUTEX_UNLOCK(&mutex_file);
         return NULL;
     }
 }

From a37f6e39d735ff2e932178b4f744a8ebda2baf50 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Mon, 21 May 2018 15:01:18 +0100
Subject: [PATCH 19/66] Compiler warnings

---
 src/grib_accessor_class.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/grib_accessor_class.c b/src/grib_accessor_class.c
index ef2e22beb..64d4d2558 100644
--- a/src/grib_accessor_class.c
+++ b/src/grib_accessor_class.c
@@ -51,6 +51,7 @@ struct table_entry
     grib_accessor_class   **cclass;
 };
 
+#ifdef ACCESSOR_FACTORY_USE_TRIE
 /* Note: A fast cut-down version of strcmp which does NOT return -1 */
 /* 0 means input strings are equal and 1 means not equal */
 static GRIB_INLINE int grib_inline_strcmp(const char* a,const char* b)
@@ -65,6 +66,7 @@ static struct table_entry table[] =
 /* This file is generated by ./make_class.pl */
 #include "grib_accessor_factory.h"
 };
+#endif
 
 #define NUMBER(x) (sizeof(x)/sizeof(x[0]))
 
@@ -98,6 +100,7 @@ grib_section* grib_create_root_section(const grib_context *context, grib_handle
 }
 
 /* Only used if ACCESSOR_FACTORY_USE_TRIE */
+#ifdef ACCESSOR_FACTORY_USE_TRIE
 static GRIB_INLINE grib_accessor_class* get_class(grib_context* c,char* type)
 {
     int i;
@@ -120,6 +123,7 @@ static GRIB_INLINE grib_accessor_class* get_class(grib_context* c,char* type)
     grib_context_log(c,GRIB_LOG_FATAL,"unable to create class %s",type);
     return NULL;
 }
+#endif
 
 grib_accessor* grib_accessor_factory(grib_section* p, grib_action* creator,
         const long len, grib_arguments* params)

From 41b9ea99db2f0da7eeb69d76be44b8ee67e5fda4 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Mon, 21 May 2018 17:17:15 +0100
Subject: [PATCH 20/66] Tests: Run with 2nd order packing

---
 tests/grib_ecc-604.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index 641964519..d76f4af33 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -60,7 +60,8 @@ cd $temp_dir
 process $ECCODES_SAMPLES_PATH/gg_sfc_grib1.tmpl
 process ${data_dir}/gen_bitmap.grib
 process ${data_dir}/spectral_complex.grib1
-#process ${data_dir}/gen.grib
+process ${data_dir}/gen_ext.grib
+process ${data_dir}/gen_ext_spd_2.grib
 
 # GRIB2 inputs
 process $ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl

From 0cd0a36f5483785176b3216ac33ed54f9d0d1ba4 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 22 May 2018 12:23:08 +0100
Subject: [PATCH 21/66] Bring in changes in develop (ecbuild etc)

---
 CMakeLists.txt                    | 36 +++++++++++-
 cmake/eccodes_test_endiness.cmake | 93 +++++++++++++++++++++++++++++++
 eccodes_config.h.in               | 35 ++++++++----
 fortran/CMakeLists.txt            |  4 +-
 4 files changed, 152 insertions(+), 16 deletions(-)
 create mode 100644 cmake/eccodes_test_endiness.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index fbc103b7c..243894498 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -31,6 +31,37 @@ ecbuild_requires_macro_version( 2.7.2 )
 
 ecbuild_declare_project()
 
+###############################################################################
+# system checks needed for eccodes_config.h and some options like MEMFS
+
+check_type_size( int            ECCODES_SIZEOF_INT    )
+check_type_size( long           ECCODES_SIZEOF_LONG   )
+check_type_size( size_t         ECCODES_SIZEOF_SIZE_T )
+
+check_include_files( assert.h         ECCODES_HAVE_ASSERT_H      )
+check_include_files( string.h         ECCODES_HAVE_STRING_H      )
+check_include_files( sys/types.h      ECCODES_HAVE_SYS_TYPES_H   )
+check_include_files( sys/stat.h       ECCODES_HAVE_SYS_STAT_H    )
+check_include_files( fcntl.h          ECCODES_HAVE_FCNTL_H      )
+check_include_files( unistd.h         ECCODES_HAVE_UNISTD_H      )
+
+check_symbol_exists( fseeko           "stdio.h"    ECCODES_HAVE_FSEEKO )
+check_symbol_exists( posix_memalign   "stdlib.h"   ECCODES_HAVE_POSIX_MEMALIGN )
+check_symbol_exists( fmemopen         "stdio.h"    ECCODES_HAVE_FMEMOPEN )
+check_symbol_exists( funopen          "stdio.h"    ECCODES_HAVE_FUNOPEN )
+
+check_c_source_compiles(
+      " typedef int foo_t;
+      static inline foo_t static_foo(){return 0;}
+      foo_t foo(){return 0;}
+      int main(int argc, char *argv[]){return 0;}
+      " ECCODES_HAVE_C_INLINE )
+
+include(eccodes_test_endiness)
+
+ecbuild_debug("ECCODES_BIG_ENDIAN=${ECCODES_BIG_ENDIAN}")
+ecbuild_debug("ECCODES_LITTLE_ENDIAN=${ECCODES_LITTLE_ENDIAN}")
+
 ###############################################################################
 # some variables/options of this project
 
@@ -96,7 +127,7 @@ endif()
 ecbuild_add_option( FEATURE MEMFS
     DESCRIPTION "Memory based access to definitions/samples"
     DEFAULT OFF
-    CONDITION EC_HAVE_FMEMOPEN OR EC_HAVE_FUNOPEN
+    CONDITION ECCODES_HAVE_FMEMOPEN OR ECCODES_HAVE_FUNOPEN
     REQUIRED_PACKAGES PythonInterp
 )
 
@@ -192,7 +223,7 @@ else()
 endif()
 
 set( IS_BIG_ENDIAN 0 )
-if( EC_BIG_ENDIAN )
+if( ECCODES_BIG_ENDIAN )
     set( IS_BIG_ENDIAN 1 )
 endif()
 
@@ -273,6 +304,7 @@ set( ECCODES_DEFINITION_PATH  ${the_default_data_prefix}/${ECCODES_DEFINITION_SU
 set( ECCODES_SAMPLES_PATH     ${the_default_data_prefix}/${ECCODES_SAMPLES_SUFF} )
 set( ECCODES_IFS_SAMPLES_PATH ${the_default_data_prefix}/${ECCODES_IFS_SAMPLES_SUFF} )
 
+###############################################################################
 ### config header
 
 ecbuild_generate_config_headers()
diff --git a/cmake/eccodes_test_endiness.cmake b/cmake/eccodes_test_endiness.cmake
new file mode 100644
index 000000000..634de4227
--- /dev/null
+++ b/cmake/eccodes_test_endiness.cmake
@@ -0,0 +1,93 @@
+# (C) Copyright 2011- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation nor
+# does it submit to any jurisdiction.
+
+test_big_endian( _BIG_ENDIAN )
+
+if( _BIG_ENDIAN )
+        set( ECCODES_BIG_ENDIAN    1 )
+        set( ECCODES_LITTLE_ENDIAN 0 )
+else()
+        set( ECCODES_BIG_ENDIAN    0 )
+        set( ECCODES_LITTLE_ENDIAN 1 )
+endif()
+
+set( ECCODES_BIG_ENDIAN    ${ECCODES_BIG_ENDIAN}    )
+set( ECCODES_LITTLE_ENDIAN ${ECCODES_LITTLE_ENDIAN} )
+
+if( NOT DEFINED IEEE_BE )
+    check_c_source_runs(
+       "int compare(unsigned char* a,unsigned char* b) {
+         while(*a != 0) if (*(b++)!=*(a++)) return 1;
+         return 0;
+       }
+       int main(int argc,char** argv) {
+         unsigned char dc[]={0x30,0x61,0xDE,0x80,0x93,0x67,0xCC,0xD9,0};
+         double da=1.23456789e-75;
+         unsigned char* ca;
+
+         unsigned char fc[]={0x05,0x83,0x48,0x22,0};
+         float fa=1.23456789e-35;
+
+         if (sizeof(double)!=8) return 1;
+
+         ca=(unsigned char*)&da;
+         if (compare(dc,ca)) return 1;
+
+         if (sizeof(float)!=4) return 1;
+
+         ca=(unsigned char*)&fa;
+         if (compare(fc,ca)) return 1;
+
+         return 0;
+       }" IEEE_BE )
+
+    if( "${IEEE_BE}" STREQUAL "" )
+      set( IEEE_BE 0 CACHE INTERNAL "Test IEEE_BE")
+    endif()
+
+endif()
+
+if( ECCODES_BIG_ENDIAN AND NOT IEEE_BE )
+    ecbuild_critical("Failed to sanity check on endiness: OS should be Big-Endian but compiled code runs differently -- to ignore this pass -DIEEE_BE=0 to CMake/ecBuild")
+endif()
+
+if( NOT DEFINED IEEE_LE )
+    check_c_source_runs(
+       "int compare(unsigned char* a,unsigned char* b) {
+         while(*a != 0) if (*(b++)!=*(a++)) return 1;
+         return 0;
+       }
+       int main(int argc,char** argv) {
+         unsigned char dc[]={0xD9,0xCC,0x67,0x93,0x80,0xDE,0x61,0x30,0};
+         double da=1.23456789e-75;
+         unsigned char* ca;
+
+         unsigned char fc[]={0x22,0x48,0x83,0x05,0};
+         float fa=1.23456789e-35;
+
+         if (sizeof(double)!=8) return 1;
+
+         ca=(unsigned char*)&da;
+         if (compare(dc,ca)) return 1;
+
+         if (sizeof(float)!=4) return 1;
+
+         ca=(unsigned char*)&fa;
+         if (compare(fc,ca)) return 1;
+
+         return 0;
+       }" IEEE_LE )
+
+    if( "${IEEE_LE}" STREQUAL "" )
+      set( IEEE_LE 0 CACHE INTERNAL "Test IEEE_LE")
+    endif()
+endif()
+
+if( ECCODES_LITTLE_ENDIAN AND NOT IEEE_LE )
+    ecbuild_critical("Failed to sanity check on endiness: OS should be Little-Endian but compiled code runs differently -- to ignore this pass -DIEEE_LE=0 to CMake/ecBuild")
+endif()
diff --git a/eccodes_config.h.in b/eccodes_config.h.in
index 4b14f898f..e9a068d07 100644
--- a/eccodes_config.h.in
+++ b/eccodes_config.h.in
@@ -23,7 +23,7 @@
 
 #define ECCODES_DEFINITION_PATH      "@ECCODES_DEFINITION_PATH@"
 
-#ifdef EC_HAVE_C_INLINE
+#ifdef ECCODES_HAVE_C_INLINE
 #define GRIB_INLINE inline
 #endif
 
@@ -40,35 +40,46 @@
 
 /* headers */
 
-#ifdef EC_HAVE_ASSERT_H
+#cmakedefine ECCODES_HAVE_ASSERT_H
+#cmakedefine ECCODES_HAVE_STRING_H
+#cmakedefine ECCODES_HAVE_SYS_TYPES_H
+#cmakedefine ECCODES_HAVE_SYS_STAT_H
+#cmakedefine ECCODES_HAVE_FCNTL_H
+#cmakedefine ECCODES_HAVE_UNISTD_H
+#cmakedefine ECCODES_HAVE_FSEEKO
+#cmakedefine ECCODES_HAVE_POSIX_MEMALIGN
+#cmakedefine ECCODES_HAVE_FMEMOPEN
+#cmakedefine ECCODES_HAVE_FUNOPEN
+
+#if defined(EC_HAVE_ASSERT_H) || defined(ECCODES_HAVE_ASSERT_H)
 #define   HAVE_ASSERT_H 1
 #endif
 
-#ifdef EC_HAVE_STRING_H
+#if defined(EC_HAVE_STRING_H) || defined(ECCODES_HAVE_STRING_H)
 #define   HAVE_STRING_H 1
 #endif
 
-#ifdef EC_HAVE_FSEEKO
-#define   HAVE_FSEEKO 1
-#endif
-
-#ifdef EC_HAVE_SYS_TYPES_H
+#if defined(EC_HAVE_SYS_TYPES_H) || defined(ECCODES_HAVE_SYS_TYPES_H)
 #define   HAVE_SYS_TYPES_H 1
 #endif
 
-#ifdef  EC_HAVE_SYS_STAT_H
+#if defined(EC_HAVE_SYS_STAT_H) || defined(ECCODES_HAVE_SYS_STAT_H)
 #define    HAVE_SYS_STAT_H 1
 #endif
 
-#ifdef  EC_HAVE_FCNTL_H
+#if defined(EC_HAVE_FCNTL_H) || defined(ECCODES_HAVE_FCNTL_H)
 #define    HAVE_FCNTL_H 1
 #endif
 
-#ifdef  EC_HAVE_UNISTD_H
+#if defined(EC_HAVE_UNISTD_H) || defined(ECCODES_HAVE_UNISTD_H)
 #define    HAVE_UNISTD_H 1
 #endif
 
-#ifdef  EC_HAVE_POSIX_MEMALIGN
+#if defined(EC_HAVE_FSEEKO) || defined(ECCODES_HAVE_FSEEKO)
+#define   HAVE_FSEEKO 1
+#endif
+
+#if defined(EC_HAVE_POSIX_MEMALIGN) || defined(ECCODES_HAVE_POSIX_MEMALIGN)
 #define         POSIX_MEMALIGN 1
 #endif
 
diff --git a/fortran/CMakeLists.txt b/fortran/CMakeLists.txt
index ff5822aad..119f09c2c 100644
--- a/fortran/CMakeLists.txt
+++ b/fortran/CMakeLists.txt
@@ -13,7 +13,7 @@ if( HAVE_FORTRAN )
                         COMMAND grib_types > grib_kinds.h
                         DEPENDS grib_types )
 
-    if( ${EC_SIZEOF_INT} EQUAL ${EC_SIZEOF_LONG} )
+    if( ${ECCODES_SIZEOF_INT} EQUAL ${ECCODES_SIZEOF_LONG} )
         set( _long_int_interface    grib_f90_int.f90 )
         set( _long_int_interface_ec eccodes_f90_int.f90 )
     else()
@@ -21,7 +21,7 @@ if( HAVE_FORTRAN )
         set( _long_int_interface_ec eccodes_f90_long_int.f90 )
     endif()
 
-    if( ${EC_SIZEOF_INT} EQUAL ${EC_SIZEOF_SIZE_T} )
+    if( ${ECCODES_SIZEOF_INT} EQUAL ${ECCODES_SIZEOF_SIZE_T} )
         set( _sizet_int_interface    grib_f90_int_size_t.f90 )
         set( _sizet_int_interface_ec eccodes_f90_int_size_t.f90 )
     else()

From c098dfe125f8e9b8af5e0cae72c2ae3734cfdf9b Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 29 May 2018 15:47:26 +0100
Subject: [PATCH 22/66] Mark read-only global shared data as const

---
 src/grib_accessor_class_validity_date.c | 2 +-
 src/grib_accessor_class_validity_time.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/grib_accessor_class_validity_date.c b/src/grib_accessor_class_validity_date.c
index 9aebf24f1..696160d14 100644
--- a/src/grib_accessor_class_validity_date.c
+++ b/src/grib_accessor_class_validity_date.c
@@ -144,7 +144,7 @@ static void init_class(grib_accessor_class* c)
 /* END_CLASS_IMP */
 
 /* Table of multipliers to convert step units to minutes */
-static double u2m[] =  {
+static const double u2m[] =  {
         1,         /* index 0:  minutes  */
         60,        /* index 1:  hour     */
         24*60,     /* index 2:  day      */
diff --git a/src/grib_accessor_class_validity_time.c b/src/grib_accessor_class_validity_time.c
index 9da09b736..3666cf71d 100644
--- a/src/grib_accessor_class_validity_time.c
+++ b/src/grib_accessor_class_validity_time.c
@@ -142,7 +142,7 @@ static void init_class(grib_accessor_class* c)
 /* END_CLASS_IMP */
 
 /* Table of multipliers to convert step units to minutes */
-static double u2m[] =  {
+static const double u2m[] =  {
         1,         /* index 0:  minutes  */
         60,        /* index 1:  hour     */
         24*60,     /* index 2:  day      */

From 029e7367e22eed77d0be7966dacb6d26b9c0ee16 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 29 May 2018 17:44:13 +0100
Subject: [PATCH 23/66] Mark read-only global shared data as const

---
 src/grib_accessor_class_bufr_data_array.c      | 18 ++++++++++++++----
 src/grib_accessor_class_count_missing.c        |  2 +-
 ...a_g1second_order_general_extended_packing.c |  4 ++--
 src/grib_accessor_class_data_simple_packing.c  |  4 ++--
 src/grib_accessor_class_g1step_range.c         |  7 +++----
 src/grib_accessor_class_g2end_step.c           |  4 ++--
 src/grib_accessor_class_g2step.c               |  2 +-
 ...ccessor_class_second_order_bits_per_value.c |  6 +++---
 src/grib_accessor_class_signed.c               |  4 +---
 src/grib_accessor_class_step_in_units.c        |  4 ++--
 src/grib_accessor_class_unsigned.c             |  2 +-
 src/grib_bits.c                                |  4 ++--
 src/grib_box_class.c                           |  2 +-
 src/grib_hash_keys.c                           |  2 +-
 src/grib_iterator_class.c                      |  2 +-
 src/grib_itrie.c                               |  2 +-
 src/grib_itrie_keys.c                          |  2 +-
 src/grib_nearest_class.c                       |  2 +-
 src/grib_trie.c                                |  2 +-
 src/grib_trie_with_rank.c                      |  2 +-
 src/md5.c                                      |  4 ++--
 21 files changed, 44 insertions(+), 37 deletions(-)

diff --git a/src/grib_accessor_class_bufr_data_array.c b/src/grib_accessor_class_bufr_data_array.c
index 0023051d3..c3d22d2b4 100644
--- a/src/grib_accessor_class_bufr_data_array.c
+++ b/src/grib_accessor_class_bufr_data_array.c
@@ -256,11 +256,21 @@ static void restart_bitmap(grib_accessor_bufr_data_array *self)
     self->bitmapCurrentElementsDescriptorsIndex=self->bitmapStartElementsDescriptorsIndex-1;
 }
 
-static void cancel_bitmap(grib_accessor_bufr_data_array *self) { self->bitmapCurrent=-1;self->bitmapStart=-1; }
+static void cancel_bitmap(grib_accessor_bufr_data_array *self)
+{
+    self->bitmapCurrent=-1;
+    self->bitmapStart=-1;
+}
 
-static int is_bitmap_start_defined(grib_accessor_bufr_data_array *self) { return self->bitmapStart==-1 ? 0 : 1; }
+static int is_bitmap_start_defined(grib_accessor_bufr_data_array *self)
+{
+    return self->bitmapStart==-1 ? 0 : 1;
+}
 
-int accessor_bufr_data_array_create_keys(grib_accessor* a,long onlySubset,long startSubset,long endSubset) { return create_keys(a,onlySubset,startSubset,endSubset); }
+int accessor_bufr_data_array_create_keys(grib_accessor* a,long onlySubset,long startSubset,long endSubset)
+{
+    return create_keys(a,onlySubset,startSubset,endSubset);
+}
 
 int accessor_bufr_data_array_process_elements(grib_accessor* a,int flag,long onlySubset,long startSubset,long endSubset)
 {
@@ -1891,7 +1901,7 @@ static grib_accessor* create_accessor_from_descriptor(grib_accessor* a,grib_acce
 #define NUMBER_OF_QUALIFIERS_CATEGORIES 7
 #define MAX_NUMBER_OF_BITMAPS 5
 
-static int number_of_qualifiers=NUMBER_OF_QUALIFIERS_PER_CATEGORY*NUMBER_OF_QUALIFIERS_CATEGORIES;
+static const int number_of_qualifiers=NUMBER_OF_QUALIFIERS_PER_CATEGORY*NUMBER_OF_QUALIFIERS_CATEGORIES;
 
 static GRIB_INLINE int significanceQualifierIndex(int X,int Y)
 {
diff --git a/src/grib_accessor_class_count_missing.c b/src/grib_accessor_class_count_missing.c
index 8e39d4445..caa9d03d4 100644
--- a/src/grib_accessor_class_count_missing.c
+++ b/src/grib_accessor_class_count_missing.c
@@ -185,7 +185,7 @@ static void init(grib_accessor* a, const long len , grib_arguments* arg )
   self->numberOfDataPoints = grib_arguments_get_name(grib_handle_of_accessor(a),arg,n++);
 }
 
-static int used[] ={ 0,1,3,7,15,31,63,127,255};
+static const int used[] ={ 0,1,3,7,15,31,63,127,255};
 
 static int    unpack_long   (grib_accessor* a, long* val, size_t *len)
 {
diff --git a/src/grib_accessor_class_data_g1second_order_general_extended_packing.c b/src/grib_accessor_class_data_g1second_order_general_extended_packing.c
index b2ee0cd99..2d1b28c5a 100644
--- a/src/grib_accessor_class_data_g1second_order_general_extended_packing.c
+++ b/src/grib_accessor_class_data_g1second_order_general_extended_packing.c
@@ -204,7 +204,7 @@ static void init_class(grib_accessor_class* c)
 #define MAX_NUMBER_OF_GROUPS 65534
 #define EFDEBUG 0
 
-static unsigned long nbits[64]={
+static const unsigned long nbits[64]={
         0x1,                 0x2,                 0x4,                 0x8, 
         0x10,                0x20,                0x40,                0x80, 
         0x100,               0x200,               0x400,               0x800,
@@ -225,7 +225,7 @@ static unsigned long nbits[64]={
 
 static long number_of_bits(grib_handle*h, unsigned long x)
 {
-    unsigned long *n=nbits;
+    const unsigned long *n=nbits;
     const int count = sizeof(nbits)/sizeof(nbits[0]);
     long i=0;
     while (x>=*n) {
diff --git a/src/grib_accessor_class_data_simple_packing.c b/src/grib_accessor_class_data_simple_packing.c
index 905b0e067..5e485c205 100644
--- a/src/grib_accessor_class_data_simple_packing.c
+++ b/src/grib_accessor_class_data_simple_packing.c
@@ -181,7 +181,7 @@ static void init(grib_accessor* a,const long v, grib_arguments* args)
     self->dirty=1;
 }
 
-static unsigned long nbits[32]={
+static const unsigned long nbits[32]={
         0x1, 0x2, 0x4, 0x8, 0x10, 0x20,
         0x40, 0x80, 0x100, 0x200, 0x400, 0x800,
         0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000,
@@ -193,7 +193,7 @@ static unsigned long nbits[32]={
 static int number_of_bits(unsigned long x, long* result)
 {
     const int count = sizeof(nbits)/sizeof(nbits[0]);
-    unsigned long *n=nbits;
+    const unsigned long *n=nbits;
     *result=0;
     while (x >= *n) {
         n++;
diff --git a/src/grib_accessor_class_g1step_range.c b/src/grib_accessor_class_g1step_range.c
index 7fa4990eb..09540778b 100644
--- a/src/grib_accessor_class_g1step_range.c
+++ b/src/grib_accessor_class_g1step_range.c
@@ -180,10 +180,9 @@ static void init(grib_accessor* a,const long l, grib_arguments* c)
 static void dump(grib_accessor* a, grib_dumper* dumper)
 {
     grib_dump_string(dumper,a,NULL);
-
 }
 
-static int u2s1[] =  {
+static const int u2s1[] =  {
         60,      /* (0) minutes */
         3600,    /* (1) hour    */
         86400,   /* (2) day     */
@@ -202,11 +201,11 @@ static int u2s1[] =  {
         1        /* (15) seconds  */   /* See ECC-316 */
 };
 
-static int units_index[] = {
+static const int units_index[] = {
         1,0,10,11,12,2,0,13,14,15
 };
 
-static int u2s[] =  {
+static const int u2s[] =  {
         60,      /* (0) minutes */
         3600,    /* (1) hour    */
         86400,   /* (2) day     */
diff --git a/src/grib_accessor_class_g2end_step.c b/src/grib_accessor_class_g2end_step.c
index 04a85121e..e3f1ea888 100644
--- a/src/grib_accessor_class_g2end_step.c
+++ b/src/grib_accessor_class_g2end_step.c
@@ -202,7 +202,7 @@ static void dump(grib_accessor* a, grib_dumper* dumper)
     grib_dump_double(dumper,a,NULL);
 }
 
-static int u2s2[] =  {
+static const int u2s2[] =  {
         60,      /* (0) minutes */
         3600,    /* (1) hour    */
         86400,   /* (2) day     */
@@ -219,7 +219,7 @@ static int u2s2[] =  {
         1        /* (13) seconds  */
 };
 
-static int u2s[] =  {
+static const int u2s[] =  {
         60,      /* (0) minutes */
         3600,    /* (1) hour    */
         86400,   /* (2) day     */
diff --git a/src/grib_accessor_class_g2step.c b/src/grib_accessor_class_g2step.c
index 725fbca09..9e2709f86 100644
--- a/src/grib_accessor_class_g2step.c
+++ b/src/grib_accessor_class_g2step.c
@@ -146,7 +146,7 @@ static void dump(grib_accessor* a, grib_dumper* dumper)
   grib_dump_double(dumper,a,NULL);
 
 }
-static int units[] =  {
+static const int units[] =  {
   0,      /* (0) minutes */
   1,      /* (1) hour    */
   24,     /* (2) day */
diff --git a/src/grib_accessor_class_second_order_bits_per_value.c b/src/grib_accessor_class_second_order_bits_per_value.c
index 460bd2ff7..ef0628b67 100644
--- a/src/grib_accessor_class_second_order_bits_per_value.c
+++ b/src/grib_accessor_class_second_order_bits_per_value.c
@@ -141,7 +141,7 @@ static void init_class(grib_accessor_class* c)
 /* END_CLASS_IMP */
 
 /*
-static unsigned long nbits[32]={
+static const unsigned long nbits[32]={
         0x1, 0x2, 0x4, 0x8, 0x10, 0x20,
         0x40, 0x80, 0x100, 0x200, 0x400, 0x800,
         0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000,
@@ -150,7 +150,7 @@ static unsigned long nbits[32]={
         0x40000000, 0x80000000
 };
 */
-static unsigned long nbits[64]={
+static const unsigned long nbits[64]={
         0x1,                 0x2,                 0x4,                 0x8,
         0x10,                0x20,                0x40,                0x80,
         0x100,               0x200,               0x400,               0x800,
@@ -171,7 +171,7 @@ static unsigned long nbits[64]={
 
 static int number_of_bits(unsigned long x, long* result)
 {
-    unsigned long *n=nbits;
+    const unsigned long *n=nbits;
     const int count = sizeof(nbits)/sizeof(nbits[0]);
     *result=0;
     while (x>=*n) {
diff --git a/src/grib_accessor_class_signed.c b/src/grib_accessor_class_signed.c
index 16c675521..1f417b142 100644
--- a/src/grib_accessor_class_signed.c
+++ b/src/grib_accessor_class_signed.c
@@ -148,7 +148,6 @@ static void init(grib_accessor* a, const long len , grib_arguments* arg )
     a->length = len * count;
     self->nbytes = len;
     Assert(a->length>=0);
-
 }
 
 static void dump(grib_accessor* a, grib_dumper* dumper)
@@ -161,7 +160,7 @@ static void dump(grib_accessor* a, grib_dumper* dumper)
         grib_dump_values(dumper,a);
 }
 
-static long ones[] = {
+static const long ones[] = {
         0,
         -0x7f,
         -0x7fff,
@@ -169,7 +168,6 @@ static long ones[] = {
         -0x7fffffff,
 };
 
-
 static int unpack_long(grib_accessor* a, long* val, size_t *len)
 {
 
diff --git a/src/grib_accessor_class_step_in_units.c b/src/grib_accessor_class_step_in_units.c
index 29274bbbb..ace38ece8 100644
--- a/src/grib_accessor_class_step_in_units.c
+++ b/src/grib_accessor_class_step_in_units.c
@@ -158,7 +158,7 @@ static void dump(grib_accessor* a, grib_dumper* dumper)
 }
 
 /* Conversion of to seconds - Grib edition 2 table 4.4 */
-static int u2s2[] =  {
+static const int u2s2[] =  {
   60,      /* (0)  minutes   */
   3600,    /* (1)  hour      */
   86400,   /* (2)  day       */
@@ -176,7 +176,7 @@ static int u2s2[] =  {
 };
 
 /* Note: 'stepUnits' has a different table with extra entries e.g. 15 and 30 mins */
-static int u2s[] =  {
+static const int u2s[] =  {
   60,      /* (0)  minutes    */
   3600,    /* (1)  hour       */
   86400,   /* (2)  day        */
diff --git a/src/grib_accessor_class_unsigned.c b/src/grib_accessor_class_unsigned.c
index 21e8fce02..01b1b8b03 100644
--- a/src/grib_accessor_class_unsigned.c
+++ b/src/grib_accessor_class_unsigned.c
@@ -170,7 +170,7 @@ static void dump(grib_accessor* a, grib_dumper* dumper)
         grib_dump_values(dumper,a);
 }
 
-static unsigned long ones[] = {
+static const unsigned long ones[] = {
         0,
         0xff,
         0xffff,
diff --git a/src/grib_bits.c b/src/grib_bits.c
index 832b62471..335fcc97d 100644
--- a/src/grib_bits.c
+++ b/src/grib_bits.c
@@ -30,9 +30,9 @@ long GRIB_MASK = -1;       /* Mask of sword bits */
  ((b)==max_nbits ? GRIB_MASK : (~(GRIB_MASK<<(b))<<(max_nbits-((q)+(b)))))
 
 
-static unsigned long dmasks[] = { 0xFF, 0xFE, 0xFC, 0xF8, 0xF0, 0xE0, 0xC0, 0x80, 0x00, };
+static const unsigned long dmasks[] = { 0xFF, 0xFE, 0xFC, 0xF8, 0xF0, 0xE0, 0xC0, 0x80, 0x00, };
 
-static int max_nbits = sizeof(unsigned long)*8;
+static const int max_nbits = sizeof(unsigned long)*8;
 
 unsigned long grib_decode_unsigned_byte_long(const unsigned char* p, long o, int l)
 {
diff --git a/src/grib_box_class.c b/src/grib_box_class.c
index de2d8d720..12eb5672d 100644
--- a/src/grib_box_class.c
+++ b/src/grib_box_class.c
@@ -21,7 +21,7 @@ struct table_entry
   grib_box_class   **cclass;
 };
 
-static struct table_entry table[] =
+static const struct table_entry table[] =
 {
   /* This file is generated by ./make_class.pl */
 #include "grib_box_factory.h"
diff --git a/src/grib_hash_keys.c b/src/grib_hash_keys.c
index c2ccbf974..feee0e466 100644
--- a/src/grib_hash_keys.c
+++ b/src/grib_hash_keys.c
@@ -10673,7 +10673,7 @@ grib_keys_hash_get (const char *str, unsigned int len)
  *  Enrico Fucile
  **************************************/
 
-static int mapping[] = {
+static const int mapping[] = {
 0, /* 00 */
 0, /* 01 */
 0, /* 02 */
diff --git a/src/grib_iterator_class.c b/src/grib_iterator_class.c
index fbab42026..30005c86d 100644
--- a/src/grib_iterator_class.c
+++ b/src/grib_iterator_class.c
@@ -27,7 +27,7 @@ struct table_entry
   grib_iterator_class   **cclass;
 };
 
-static struct table_entry table[] =
+static const struct table_entry table[] =
 {
   /* This file is generated by ./make_class.pl */
 #include "grib_iterator_factory.h"
diff --git a/src/grib_itrie.c b/src/grib_itrie.c
index 2aea8b312..9457abe7c 100644
--- a/src/grib_itrie.c
+++ b/src/grib_itrie.c
@@ -11,7 +11,7 @@
 #include "grib_api_internal.h"
 
 /* Note: all non-alpha are mapped to 0 */
-static int mapping[] = {
+static const int mapping[] = {
 0, /* 00 */
 0, /* 01 */
 0, /* 02 */
diff --git a/src/grib_itrie_keys.c b/src/grib_itrie_keys.c
index 0b3220d57..2346497a5 100644
--- a/src/grib_itrie_keys.c
+++ b/src/grib_itrie_keys.c
@@ -12,7 +12,7 @@
  *  Enrico Fucile
  **************************************/
 
-static int mapping[] = {
+static const int mapping[] = {
 0, /* 00 */
 0, /* 01 */
 0, /* 02 */
diff --git a/src/grib_nearest_class.c b/src/grib_nearest_class.c
index 0991b47a7..14fa16279 100644
--- a/src/grib_nearest_class.c
+++ b/src/grib_nearest_class.c
@@ -26,7 +26,7 @@ struct table_entry
   grib_nearest_class   **cclass;
 };
 
-static struct table_entry table[] =
+static const struct table_entry table[] =
 {
   /* This file is generated by ./make_class.pl */
 #include "grib_nearest_factory.h"
diff --git a/src/grib_trie.c b/src/grib_trie.c
index 117e30193..13d935c27 100644
--- a/src/grib_trie.c
+++ b/src/grib_trie.c
@@ -11,7 +11,7 @@
 #include "grib_api_internal.h"
 
 /* Note: all non-alpha are mapped to 0 */
-static int mapping[] = {
+static const int mapping[] = {
 0, /* 00 */
 0, /* 01 */
 0, /* 02 */
diff --git a/src/grib_trie_with_rank.c b/src/grib_trie_with_rank.c
index 980a5923e..117342b99 100644
--- a/src/grib_trie_with_rank.c
+++ b/src/grib_trie_with_rank.c
@@ -11,7 +11,7 @@
 #include "grib_api_internal.h"
 
 /* Note: all non-alpha are mapped to 0 */
-static int mapping[] = {
+static const int mapping[] = {
 0, /* 00 */
 0, /* 01 */
 0, /* 02 */
diff --git a/src/md5.c b/src/md5.c
index b4822e698..907e394f2 100644
--- a/src/md5.c
+++ b/src/md5.c
@@ -19,13 +19,13 @@
     #pragma _CRI noopt
 #endif
 
-static unsigned long r[] = {
+static const unsigned long r[] = {
     7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22,
     5,  9, 14, 20, 5,  9, 14, 20, 5,  9, 14, 20, 5,  9, 14, 20,
     4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23,
     6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21};
 
-static unsigned long k[] = {
+static const unsigned long k[] = {
     0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501,
     0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821,
     0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8,

From d01d1e13aa71403d572a0927dad02d650688c495 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 30 May 2018 14:40:24 +0100
Subject: [PATCH 24/66] Data race on global shared variable

---
 src/grib_accessor_class_data_jpeg2000_packing.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/grib_accessor_class_data_jpeg2000_packing.c b/src/grib_accessor_class_data_jpeg2000_packing.c
index 90a394d8c..e71cad092 100644
--- a/src/grib_accessor_class_data_jpeg2000_packing.c
+++ b/src/grib_accessor_class_data_jpeg2000_packing.c
@@ -215,11 +215,11 @@ static void init(grib_accessor* a,const long v, grib_arguments* args)
     }
 
     self->dump_jpg = codes_getenv("ECCODES_GRIB_DUMP_JPG_FILE");
-
-    if(first) {
-        if(self->dump_jpg)
+    if(self->dump_jpg) {
+        if (first) {
             printf("GRIB JPEG dumping to %s\n",self->dump_jpg);
-        first = 0;
+            first = 0;
+        }
     }
 }
 

From 6bbdee4c79b963e5ae33a6b1d09c3d57baec8591 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 30 May 2018 14:40:55 +0100
Subject: [PATCH 25/66] Thread test needs data downloaded

---
 tests/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index e84844a9c..324b34ce8 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -227,6 +227,7 @@ if( ENABLE_EXTRA_TESTS AND HAVE_ECCODES_THREADS )
     endforeach()
     ecbuild_add_test( TARGET eccodes_t_grib_ecc-604
                       TYPE SCRIPT
+                      TEST_DEPENDS eccodes_download_gribs
                       COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_ecc-604.sh
     )
 endif()

From 386b1eb39826b814d1845d79f835220caa5fef37 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 30 May 2018 14:41:45 +0100
Subject: [PATCH 26/66] Test: Added grib2 JPEG packing too

---
 tests/grib_ecc-604.sh | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index d76f4af33..4facc9393 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -13,8 +13,8 @@
 label="grib_ecc-604"
 temp_dir=tempdir.${label}
 
-NUM_THREADS=6
-NUM_ITER=100
+NUM_THREADS=8
+NUM_ITER=50
 OUTPUT=output
 
 validate()
@@ -56,17 +56,30 @@ rm -fr $temp_dir
 mkdir -p $temp_dir
 cd $temp_dir
 
-# GRIB1 inputs
-process $ECCODES_SAMPLES_PATH/gg_sfc_grib1.tmpl
-process ${data_dir}/gen_bitmap.grib
-process ${data_dir}/spectral_complex.grib1
-process ${data_dir}/gen_ext.grib
-process ${data_dir}/gen_ext_spd_2.grib
+GRIB1_INPUTS="
+  $ECCODES_SAMPLES_PATH/gg_sfc_grib1.tmpl
+  ${data_dir}/gen_bitmap.grib
+  ${data_dir}/spectral_complex.grib1
+  ${data_dir}/gen_ext.grib
+  ${data_dir}/gen_ext_spd_2.grib"
 
-# GRIB2 inputs
-process $ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl
-process ${data_dir}/sample.grib2
+GRIB2_INPUTS="
+  $ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl
+  ${data_dir}/reduced_gaussian_sub_area.grib2
+  ${data_dir}/test_file.grib2
+  ${data_dir}/sample.grib2"
 
+# Check HAVE_JPEG is defined and is equal to 1
+if [ "x$HAVE_JPEG" != x ]; then
+    if [ $HAVE_JPEG -eq 1 ]; then
+        # Include files which have messages with grid_jpeg packing
+        GRIB2_INPUTS="${data_dir}/jpeg.grib2  ${data_dir}/reduced_gaussian_surface_jpeg.grib2  ${data_dir}/v.grib2 "$GRIB2_INPUTS
+    fi
+fi
+
+for g1 in $GRIB1_INPUTS $GRIB2_INPUTS; do
+    process $g1
+done
 
 # Clean up
 cd $test_dir

From 594eb2b1348a14df4ca41745da9749af70ee7519 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 30 May 2018 16:33:40 +0100
Subject: [PATCH 27/66] Do a dump on each handle to fully decode message in
 parallel

---
 tests/grib_ecc-604-1.c | 4 ++++
 tests/grib_ecc-604-2.c | 4 ++++
 tests/grib_ecc-604-3.c | 5 +++++
 3 files changed, 13 insertions(+)

diff --git a/tests/grib_ecc-604-1.c b/tests/grib_ecc-604-1.c
index fb1fbb96f..8349080c4 100644
--- a/tests/grib_ecc-604-1.c
+++ b/tests/grib_ecc-604-1.c
@@ -56,6 +56,10 @@ static int encode_file(char *template_file, char *output_file)
             perror(output_file);
             return 1;
         }
+        {
+            FILE *devnull = fopen("/dev/null", "w");
+            grib_dump_content(source_handle,devnull,"wmo",0,NULL);
+        }
         grib_handle_delete(clone_handle);
         grib_handle_delete(source_handle);
         free(values);
diff --git a/tests/grib_ecc-604-2.c b/tests/grib_ecc-604-2.c
index 31bcc9246..fb41d95c7 100644
--- a/tests/grib_ecc-604-2.c
+++ b/tests/grib_ecc-604-2.c
@@ -55,6 +55,10 @@ static int encode_file(char *template_file, char *output_file)
             perror(output_file);
             return 1;
         }
+        {
+            FILE *devnull = fopen("/dev/null", "w");
+            grib_dump_content(source_handle,devnull,"wmo",0,NULL);
+        }
         grib_handle_delete(source_handle);
         free(values);
     }
diff --git a/tests/grib_ecc-604-3.c b/tests/grib_ecc-604-3.c
index 6ab10b29f..f773f2c5f 100644
--- a/tests/grib_ecc-604-3.c
+++ b/tests/grib_ecc-604-3.c
@@ -53,6 +53,11 @@ static int decode_file(char *template_file)
 
         GRIB_CHECK(grib_get_message(clone_handle,&buffer,&size),0);
 
+        {
+            FILE *devnull = fopen("/dev/null", "w");
+            grib_dump_content(source_handle,devnull,"wmo",0,NULL);
+        }
+
         grib_handle_delete(clone_handle);
         grib_handle_delete(source_handle);
         free(values);

From 4f9fb235250b0b55e6b94408a8d94d9c2381d6cb Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 30 May 2018 17:44:32 +0100
Subject: [PATCH 28/66] Clean up

---
 tests/grib_ecc-604.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index 4facc9393..0a3421abe 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -13,8 +13,8 @@
 label="grib_ecc-604"
 temp_dir=tempdir.${label}
 
-NUM_THREADS=8
-NUM_ITER=50
+NUM_THREADS=3
+NUM_ITER=10
 OUTPUT=output
 
 validate()
@@ -73,12 +73,13 @@ GRIB2_INPUTS="
 if [ "x$HAVE_JPEG" != x ]; then
     if [ $HAVE_JPEG -eq 1 ]; then
         # Include files which have messages with grid_jpeg packing
-        GRIB2_INPUTS="${data_dir}/jpeg.grib2  ${data_dir}/reduced_gaussian_surface_jpeg.grib2  ${data_dir}/v.grib2 "$GRIB2_INPUTS
+        GRIB2_INPUTS="${data_dir}/jpeg.grib2 ${data_dir}/reduced_gaussian_surface_jpeg.grib2 "$GRIB2_INPUTS
     fi
 fi
+#GRIB2_INPUTS=$GRIB2_INPUTS" ${data_dir}/ccsds.grib2 "
 
-for g1 in $GRIB1_INPUTS $GRIB2_INPUTS; do
-    process $g1
+for gf in $GRIB1_INPUTS $GRIB2_INPUTS; do
+    process $gf
 done
 
 # Clean up

From e7e4a35b181e913289f6181d387915db12d14034 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 30 May 2018 18:12:45 +0100
Subject: [PATCH 29/66] Add mutex for dump and iterator

---
 src/grib_dumper.c   | 266 ++++++++++++++++++++++++--------------------
 src/grib_iterator.c |  41 ++++++-
 2 files changed, 181 insertions(+), 126 deletions(-)

diff --git a/src/grib_dumper.c b/src/grib_dumper.c
index 34c52c972..129332891 100644
--- a/src/grib_dumper.c
+++ b/src/grib_dumper.c
@@ -11,204 +11,230 @@
 /**************************************
  *  Enrico Fucile
  **************************************/
-
-
 #include "grib_api_internal.h"
 
+#if GRIB_PTHREADS
+static pthread_once_t once  = PTHREAD_ONCE_INIT;
+static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static void init_mutex() {
+    pthread_mutexattr_t attr;
+    pthread_mutexattr_init(&attr);
+    pthread_mutexattr_settype(&attr,PTHREAD_MUTEX_RECURSIVE);
+    pthread_mutex_init(&mutex,&attr);
+    pthread_mutexattr_destroy(&attr);
+}
+#elif GRIB_OMP_THREADS
+static int once = 0;
+static omp_nest_lock_t mutex;
+
+static void init_mutex()
+{
+    GRIB_OMP_CRITICAL(lock_dumper_c)
+    {
+        if (once == 0)
+        {
+            omp_init_nest_lock(&mutex);
+            once = 1;
+        }
+    }
+}
+#endif
 
 static void init_dumpers(grib_dumper_class* c,grib_dumper* d)
 {
-  if(c) {
-    grib_dumper_class *s = c->super ? *(c->super) : NULL;
-    if(!c->inited)
-    {
-      if(c->init_class) c->init_class(c);
-      c->inited = 1;
+    GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
+    GRIB_MUTEX_LOCK(&mutex);
+    if(c) {
+        grib_dumper_class *s = c->super ? *(c->super) : NULL;
+        if(!c->inited)
+        {
+            if(c->init_class) c->init_class(c);
+            c->inited = 1;
+        }
+        init_dumpers(s,d);
+        if(c->init) c->init(d);
     }
-    init_dumpers(s,d);
-    if(c->init) c->init(d);
-  }
+    GRIB_MUTEX_UNLOCK(&mutex);
 }
 
 void grib_init_dumper(grib_dumper* d)
 {
-  init_dumpers(d->cclass,d);
+    init_dumpers(d->cclass,d);
 }
 
 void grib_dumper_delete(grib_dumper* d)
 {
-  grib_dumper_class *c = d->cclass;
-  grib_context *ctx = d->handle->context;
-  while(c)
-  {
-    grib_dumper_class *s = c->super ? *(c->super) : NULL;
-    if(c->destroy) c->destroy(d);
-    c = s;
-  }
-  grib_context_free(ctx,d);
+    grib_dumper_class *c = d->cclass;
+    grib_context *ctx = d->handle->context;
+    while(c)
+    {
+        grib_dumper_class *s = c->super ? *(c->super) : NULL;
+        if(c->destroy) c->destroy(d);
+        c = s;
+    }
+    grib_context_free(ctx,d);
 }
 
-
 void  grib_dump_long(grib_dumper* d, grib_accessor* a, const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
-  while(c)
-  {
-    if(c->dump_long)
+    grib_dumper_class *c = d->cclass;
+    while(c)
     {
-      c->dump_long(d, a, comment);
-      return;
+        if(c->dump_long)
+        {
+            c->dump_long(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
-  Assert(0);
+    Assert(0);
 }
 
 void  grib_dump_double(grib_dumper* d,grib_accessor* a,const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
-  while(c)
-  {
-    if(c->dump_double)
+    grib_dumper_class *c = d->cclass;
+    while(c)
     {
-      c->dump_double(d, a, comment);
-      return;
+        if(c->dump_double)
+        {
+            c->dump_double(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
-  Assert(0);
+    Assert(0);
 }
 
 void  grib_dump_string(grib_dumper* d,grib_accessor* a,const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
-  while(c)
-  {
-    if(c->dump_string)
+    grib_dumper_class *c = d->cclass;
+    while(c)
     {
-      c->dump_string(d, a, comment);
-      return;
+        if(c->dump_string)
+        {
+            c->dump_string(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
-  Assert(0);
+    Assert(0);
 }
 
 void  grib_dump_string_array(grib_dumper* d,grib_accessor* a,const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
-  while(c)
-  {
-    if(c->dump_string_array)
+    grib_dumper_class *c = d->cclass;
+    while(c)
     {
-      c->dump_string_array(d, a, comment);
-      return;
+        if(c->dump_string_array)
+        {
+            c->dump_string_array(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
-  Assert(0);
+    Assert(0);
 }
 
 void  grib_dump_label(grib_dumper* d,grib_accessor* a,const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
-  while(c)
-  {
-    if(c->dump_label)
+    grib_dumper_class *c = d->cclass;
+    while(c)
     {
-      c->dump_label(d, a, comment);
-      return;
+        if(c->dump_label)
+        {
+            c->dump_label(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
-  Assert(0);
+    Assert(0);
 }
 
 void  grib_dump_bytes(grib_dumper* d,grib_accessor* a,const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
-  while(c)
-  {
-    if(c->dump_bytes)
+    grib_dumper_class *c = d->cclass;
+    while(c)
     {
-      c->dump_bytes(d, a, comment);
-      return;
+        if(c->dump_bytes)
+        {
+            c->dump_bytes(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
-  Assert(0);
+    Assert(0);
 }
 
 void  grib_dump_bits(grib_dumper* d,grib_accessor* a,const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
-  while(c)
-  {
-    if(c->dump_bits)
+    grib_dumper_class *c = d->cclass;
+    while(c)
     {
-      c->dump_bits(d, a, comment);
-      return;
+        if(c->dump_bits)
+        {
+            c->dump_bits(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
-  Assert(0);
+    Assert(0);
 }
 
-
 void  grib_dump_section(grib_dumper* d,grib_accessor* a,grib_block_of_accessors* block)
 {
-  grib_dumper_class *c = d->cclass;
-  while(c)
-  {
-    if(c->dump_section)
+    grib_dumper_class *c = d->cclass;
+    while(c)
     {
-      c->dump_section(d, a, block);
-      return;
+        if(c->dump_section)
+        {
+            c->dump_section(d, a, block);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
-  Assert(0);
+    Assert(0);
 }
 
 void  grib_dump_values(grib_dumper* d,grib_accessor* a)
 {
-  grib_dumper_class *c = d->cclass;
-  while(c)
-  {
-    if(c->dump_values)
+    grib_dumper_class *c = d->cclass;
+    while(c)
     {
-      c->dump_values(d, a);
-      return;
+        if(c->dump_values)
+        {
+            c->dump_values(d, a);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
-  Assert(0);
+    Assert(0);
 }
 
 void  grib_dump_header(grib_dumper* d,grib_handle* h)
 {
-  grib_dumper_class *c = d->cclass;
-  while(c)
-  {
-    if(c->header)
+    grib_dumper_class *c = d->cclass;
+    while(c)
     {
-      c->header(d,h);
-      return;
+        if(c->header)
+        {
+            c->header(d,h);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
 }
 
 void  grib_dump_footer(grib_dumper* d,grib_handle* h)
 {
-  grib_dumper_class *c = d->cclass;
-  while(c)
-  {
-    if(c->footer)
+    grib_dumper_class *c = d->cclass;
+    while(c)
     {
-      c->footer(d,h);
-      return;
+        if(c->footer)
+        {
+            c->footer(d,h);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
 }
diff --git a/src/grib_iterator.c b/src/grib_iterator.c
index 97233ba59..928f4c62e 100644
--- a/src/grib_iterator.c
+++ b/src/grib_iterator.c
@@ -15,6 +15,35 @@
  ***************************************************************************/
 #include "grib_api_internal.h"
 
+#if GRIB_PTHREADS
+static pthread_once_t once  = PTHREAD_ONCE_INIT;
+static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static void init_mutex() {
+    pthread_mutexattr_t attr;
+    pthread_mutexattr_init(&attr);
+    pthread_mutexattr_settype(&attr,PTHREAD_MUTEX_RECURSIVE);
+    pthread_mutex_init(&mutex,&attr);
+    pthread_mutexattr_destroy(&attr);
+}
+#elif GRIB_OMP_THREADS
+static int once = 0;
+static omp_nest_lock_t mutex;
+
+static void init_mutex()
+{
+    GRIB_OMP_CRITICAL(lock_iterator_c)
+    {
+        if (once == 0)
+        {
+            omp_init_nest_lock(&mutex);
+            once = 1;
+        }
+    }
+}
+#endif
+
+
 int grib_get_data(grib_handle* h,double* lats, double* lons,double* values)
 {
     int err=0;
@@ -71,7 +100,6 @@ int grib_iterator_previous(grib_iterator *i,double* lat,double* lon,double* valu
     return 0;
 }
 
-
 int grib_iterator_reset(grib_iterator *i)
 {
     grib_iterator_class *c = i->cclass;
@@ -85,12 +113,9 @@ int grib_iterator_reset(grib_iterator *i)
     return 0;
 }
 
-
 /* For this one, ALL init are called */
-
 static int init_iterator(grib_iterator_class* c,grib_iterator* i, grib_handle *h, grib_arguments* args)
 {
-
     if(c) {
         int ret = GRIB_SUCCESS;
         grib_iterator_class *s = c->super ? *(c->super) : NULL;
@@ -110,11 +135,15 @@ static int init_iterator(grib_iterator_class* c,grib_iterator* i, grib_handle *h
 
 int grib_iterator_init(grib_iterator* i, grib_handle *h, grib_arguments* args)
 {
-    return init_iterator(i->cclass,i,h,args);
+    int r = 0;
+    GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
+    GRIB_MUTEX_LOCK(&mutex);
+    r = init_iterator(i->cclass,i,h,args);
+    GRIB_MUTEX_UNLOCK(&mutex);
+    return r;
 }
 
 /* For this one, ALL destroy are called */
-
 int grib_iterator_delete(grib_iterator *i)
 {
     grib_iterator_class *c = i->cclass;

From 68dd0b3b873ad63e87315729a9506301d2a0e8c2 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 30 May 2018 18:13:06 +0100
Subject: [PATCH 30/66] Context shared resource needs mutex

---
 src/grib_accessor_class_count_file.c  | 13 ++++++-------
 src/grib_accessor_class_count_total.c | 11 +++++------
 src/grib_api_prototypes.h             |  2 ++
 src/grib_context.c                    | 23 +++++++++++++++++++++--
 4 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/src/grib_accessor_class_count_file.c b/src/grib_accessor_class_count_file.c
index 602ef2c01..280a9fc4c 100644
--- a/src/grib_accessor_class_count_file.c
+++ b/src/grib_accessor_class_count_file.c
@@ -133,14 +133,13 @@ static void init_class(grib_accessor_class* c)
 
 static void init(grib_accessor* a,const long l, grib_arguments* c)
 {
-  a->flags  |= GRIB_ACCESSOR_FLAG_READ_ONLY;
-  a->length=0;
+    a->flags  |= GRIB_ACCESSOR_FLAG_READ_ONLY;
+    a->length=0;
 }
 
-static int  unpack_long(grib_accessor* a, long* val, size_t *len)
+static int unpack_long(grib_accessor* a, long* val, size_t *len)
 {
-  *val = a->context->handle_file_count;
-  *len =1;
-  return 0;
+    *val = grib_context_get_handle_file_count(a->context);
+    *len =1;
+    return 0;
 }
-
diff --git a/src/grib_accessor_class_count_total.c b/src/grib_accessor_class_count_total.c
index 70d6054ff..89572f37d 100644
--- a/src/grib_accessor_class_count_total.c
+++ b/src/grib_accessor_class_count_total.c
@@ -135,14 +135,13 @@ static void init_class(grib_accessor_class* c)
 
 static void init(grib_accessor* a,const long l, grib_arguments* c)
 {
-  a->flags  |= GRIB_ACCESSOR_FLAG_READ_ONLY;
-  a->length=0;
+    a->flags  |= GRIB_ACCESSOR_FLAG_READ_ONLY;
+    a->length=0;
 }
 
 static int  unpack_long(grib_accessor* a, long* val, size_t *len)
 {
-  *val = a->context->handle_total_count;
-  *len =1;
-  return 0;
+    *val = grib_context_get_handle_total_count(a->context);
+    *len =1;
+    return 0;
 }
-
diff --git a/src/grib_api_prototypes.h b/src/grib_api_prototypes.h
index 93d92a1ca..615d45e56 100644
--- a/src/grib_api_prototypes.h
+++ b/src/grib_api_prototypes.h
@@ -994,6 +994,8 @@ void grib_context_set_buffer_memory_proc(grib_context *c, grib_malloc_proc m, gr
 void grib_context_set_data_accessing_proc(grib_context *c, grib_data_read_proc read, grib_data_write_proc write, grib_data_tell_proc tell);
 void grib_context_log(const grib_context *c, int level, const char *fmt, ...);
 void grib_context_print(const grib_context *c, void *descriptor, const char *fmt, ...);
+int grib_context_get_handle_file_count(grib_context *c);
+int grib_context_get_handle_total_count(grib_context *c);
 void grib_context_set_handle_file_count(grib_context *c, int new_count);
 void grib_context_set_handle_total_count(grib_context *c, int new_count);
 void grib_context_increment_handle_file_count(grib_context *c);
diff --git a/src/grib_context.c b/src/grib_context.c
index d73ac1b34..c130920be 100644
--- a/src/grib_context.c
+++ b/src/grib_context.c
@@ -946,6 +946,27 @@ void grib_context_print(const grib_context *c, void* descriptor,const char* fmt,
     c->print(c,descriptor,msg);
 }
 
+int grib_context_get_handle_file_count(grib_context *c)
+{
+    int r = 0;
+    if (!c) c=grib_context_get_default();
+    GRIB_MUTEX_INIT_ONCE(&once,&init);
+    GRIB_MUTEX_LOCK(&mutex_c);
+    r = c->handle_file_count;
+    GRIB_MUTEX_UNLOCK(&mutex_c);
+    return r;
+}
+int grib_context_get_handle_total_count(grib_context *c)
+{
+    int r = 0;
+    if (!c) c=grib_context_get_default();
+    GRIB_MUTEX_INIT_ONCE(&once,&init);
+    GRIB_MUTEX_LOCK(&mutex_c);
+    r=c->handle_total_count;
+    GRIB_MUTEX_UNLOCK(&mutex_c);
+    return r;
+}
+
 void grib_context_set_handle_file_count(grib_context *c, int new_count)
 {
     if (!c) c=grib_context_get_default();
@@ -954,7 +975,6 @@ void grib_context_set_handle_file_count(grib_context *c, int new_count)
     c->handle_file_count = new_count;
     GRIB_MUTEX_UNLOCK(&mutex_c);
 }
-
 void grib_context_set_handle_total_count(grib_context *c, int new_count)
 {
     if (!c) c=grib_context_get_default();
@@ -972,7 +992,6 @@ void grib_context_increment_handle_file_count(grib_context *c)
     c->handle_file_count++;
     GRIB_MUTEX_UNLOCK(&mutex_c);
 }
-
 void grib_context_increment_handle_total_count(grib_context *c)
 {
     if (!c) c=grib_context_get_default();

From e788a28814411866887644b234f5db96074f0abf Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 1 Jun 2018 11:55:49 +0100
Subject: [PATCH 31/66] Use debug dump mode to decode all keys in message

---
 tests/grib_ecc-604-1.c | 3 ++-
 tests/grib_ecc-604-2.c | 2 +-
 tests/grib_ecc-604-3.c | 3 ++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/grib_ecc-604-1.c b/tests/grib_ecc-604-1.c
index 8349080c4..18f51347e 100644
--- a/tests/grib_ecc-604-1.c
+++ b/tests/grib_ecc-604-1.c
@@ -58,7 +58,8 @@ static int encode_file(char *template_file, char *output_file)
         }
         {
             FILE *devnull = fopen("/dev/null", "w");
-            grib_dump_content(source_handle,devnull,"wmo",0,NULL);
+            grib_dump_content(source_handle,devnull, "debug", 0, NULL);
+            grib_dump_content(clone_handle, devnull, "debug", 0, NULL);
         }
         grib_handle_delete(clone_handle);
         grib_handle_delete(source_handle);
diff --git a/tests/grib_ecc-604-2.c b/tests/grib_ecc-604-2.c
index fb41d95c7..de25768bf 100644
--- a/tests/grib_ecc-604-2.c
+++ b/tests/grib_ecc-604-2.c
@@ -57,7 +57,7 @@ static int encode_file(char *template_file, char *output_file)
         }
         {
             FILE *devnull = fopen("/dev/null", "w");
-            grib_dump_content(source_handle,devnull,"wmo",0,NULL);
+            grib_dump_content(source_handle,devnull,"debug",0,NULL);
         }
         grib_handle_delete(source_handle);
         free(values);
diff --git a/tests/grib_ecc-604-3.c b/tests/grib_ecc-604-3.c
index f773f2c5f..07b1a0d29 100644
--- a/tests/grib_ecc-604-3.c
+++ b/tests/grib_ecc-604-3.c
@@ -55,7 +55,8 @@ static int decode_file(char *template_file)
 
         {
             FILE *devnull = fopen("/dev/null", "w");
-            grib_dump_content(source_handle,devnull,"wmo",0,NULL);
+            grib_dump_content(source_handle, devnull, "debug", 0, NULL);
+            grib_dump_content(clone_handle,  devnull, "debug", 0, NULL);
         }
 
         grib_handle_delete(clone_handle);

From eb7e4e0a58db0792ef04d11ce3bd172799880832 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 20 Jul 2018 14:16:37 +0100
Subject: [PATCH 32/66] Compiler warnings

---
 examples/C/grib_print_data.c     |   3 +-
 src/grib_accessor_classes_hash.c |   3 +-
 tests/grib_ecc-604-1.c           |  43 +--
 tests/grib_ecc-604-2.c           |  43 +--
 tests/grib_ecc-604-3.c           |  44 +--
 tests/sh_ieee64.c                |   2 +-
 tools/metar_ls.c                 | 498 +++++++++++++++----------------
 7 files changed, 319 insertions(+), 317 deletions(-)

diff --git a/examples/C/grib_print_data.c b/examples/C/grib_print_data.c
index 464daeb11..61087cc9c 100644
--- a/examples/C/grib_print_data.c
+++ b/examples/C/grib_print_data.c
@@ -71,8 +71,7 @@ int main(int argc, char** argv)
     printf("max=%.10e min=%.10e average=%.10e\n",max,min,average);
 
     {
-        /* Example of accessing specific elements from data values */
-        i=0;
+        /* Example of accessing specific elements from data values  */
         double vals_arr[3] = {0, 0, 0};
         const int NUM = 3;
         int index_arr[3];
diff --git a/src/grib_accessor_classes_hash.c b/src/grib_accessor_classes_hash.c
index 51deed99a..b7384476e 100644
--- a/src/grib_accessor_classes_hash.c
+++ b/src/grib_accessor_classes_hash.c
@@ -708,9 +708,10 @@ grib_accessor_classes_hash (const char *str, unsigned int len)
 
 #ifdef DEBUG
     {
+        const char *s;
         Assert( len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH );
         Assert( key <= MAX_HASH_VALUE && key >= 0 );
-        const char *s = classes[key].name;
+        s = classes[key].name;
         Assert( *str == *s && strcmp(str + 1, s + 1)==0 );
     }
 #endif
diff --git a/tests/grib_ecc-604-1.c b/tests/grib_ecc-604-1.c
index 18f51347e..d5e0ff3ae 100644
--- a/tests/grib_ecc-604-1.c
+++ b/tests/grib_ecc-604-1.c
@@ -108,25 +108,27 @@ int main(int argc, char **argv)
         printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
     }
 
-    pthread_t workers[NUM_THREADS];
-    for (i = 0; i < NUM_THREADS; i++) {
-        struct v *data = (struct v *) malloc(sizeof(struct v));
-        data->number = i;
-        data->data = NULL;
+    {
+        pthread_t workers[NUM_THREADS];
+        for (i = 0; i < NUM_THREADS; i++) {
+            struct v *data = (struct v *) malloc(sizeof(struct v));
+            data->number = i;
+            data->data = NULL;
+
+            if (parallel) {
+                /* Now we will create the thread passing it data as an argument */
+                pthread_create(&workers[thread_counter], NULL, runner, data);
+                /*pthread_join(workers[thread_counter], NULL);*/
+                thread_counter++;
+            } else {
+                do_stuff(data);
+            }
+        }
 
         if (parallel) {
-            /* Now we will create the thread passing it data as an argument */
-            pthread_create(&workers[thread_counter], NULL, runner, data);
-            /*pthread_join(workers[thread_counter], NULL);*/
-            thread_counter++;
-        } else {
-            do_stuff(data);
-        }
-    }
-
-    if (parallel) {
-        for (i = 0; i < NUM_THREADS; i++)  {
-            pthread_join(workers[i], NULL);
+            for (i = 0; i < NUM_THREADS; i++)  {
+                pthread_join(workers[i], NULL);
+            }
         }
     }
 
@@ -145,16 +147,15 @@ void do_stuff(void *ptr)
     struct v *data = ptr;
     size_t i;
     char output_file[50];
+    time_t ltime;
+    struct tm result;
+    char stime[32];
 
     for (i=0; i<FILES_PER_ITERATION;i++) {
         sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
         encode_file(INPUT_FILE,output_file);
     }
 
-    time_t ltime;
-    struct tm result;
-    char stime[32];
-
     ltime = time(NULL);
     localtime_r(&ltime, &result);
     strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
diff --git a/tests/grib_ecc-604-2.c b/tests/grib_ecc-604-2.c
index de25768bf..9af8aa832 100644
--- a/tests/grib_ecc-604-2.c
+++ b/tests/grib_ecc-604-2.c
@@ -105,25 +105,27 @@ int main(int argc, char **argv)
         printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
     }
 
-    pthread_t workers[NUM_THREADS];
-    for (i = 0; i < NUM_THREADS; i++) {
-        struct v *data = (struct v *) malloc(sizeof(struct v));
-        data->number = i;
-        data->data = NULL;
+    {
+        pthread_t workers[NUM_THREADS];
+        for (i = 0; i < NUM_THREADS; i++) {
+            struct v *data = (struct v *) malloc(sizeof(struct v));
+            data->number = i;
+            data->data = NULL;
+
+            if (parallel) {
+                /* Now we will create the thread passing it data as an argument */
+                pthread_create(&workers[thread_counter], NULL, runner, data);
+                /*pthread_join(workers[thread_counter], NULL);*/
+                thread_counter++;
+            } else {
+                do_stuff(data);
+            }
+        }
 
         if (parallel) {
-            /* Now we will create the thread passing it data as an argument */
-            pthread_create(&workers[thread_counter], NULL, runner, data);
-            /*pthread_join(workers[thread_counter], NULL);*/
-            thread_counter++;
-        } else {
-            do_stuff(data);
-        }
-    }
-
-    if (parallel) {
-        for (i = 0; i < NUM_THREADS; i++)  {
-            pthread_join(workers[i], NULL);
+            for (i = 0; i < NUM_THREADS; i++)  {
+                pthread_join(workers[i], NULL);
+            }
         }
     }
 
@@ -142,16 +144,15 @@ void do_stuff(void *ptr)
     struct v *data = ptr;
     size_t i;
     char output_file[50];
+    time_t ltime;
+    struct tm result;
+    char stime[32];
 
     for (i=0; i<FILES_PER_ITERATION;i++) {
         sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
         encode_file(INPUT_FILE,output_file);
     }
 
-    time_t ltime;
-    struct tm result;
-    char stime[32];
-
     ltime = time(NULL);
     localtime_r(&ltime, &result);
     strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
diff --git a/tests/grib_ecc-604-3.c b/tests/grib_ecc-604-3.c
index 07b1a0d29..e484e1253 100644
--- a/tests/grib_ecc-604-3.c
+++ b/tests/grib_ecc-604-3.c
@@ -105,28 +105,29 @@ int main(int argc, char **argv)
         printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
     }
 
-    pthread_t workers[NUM_THREADS];
-    for (i = 0; i < NUM_THREADS; i++) {
-        struct v *data = (struct v *) malloc(sizeof(struct v));
-        data->number = i;
-        data->data = NULL;
+    {
+        pthread_t workers[NUM_THREADS];
+        for (i = 0; i < NUM_THREADS; i++) {
+            struct v *data = (struct v *) malloc(sizeof(struct v));
+            data->number = i;
+            data->data = NULL;
+
+            if (parallel) {
+                /* Now we will create the thread passing it data as an argument */
+                pthread_create(&workers[thread_counter], NULL, runner, data);
+                /*pthread_join(workers[thread_counter], NULL);*/
+                thread_counter++;
+            } else {
+                do_stuff(data);
+            }
+        }
 
         if (parallel) {
-            /* Now we will create the thread passing it data as an argument */
-            pthread_create(&workers[thread_counter], NULL, runner, data);
-            /*pthread_join(workers[thread_counter], NULL);*/
-            thread_counter++;
-        } else {
-            do_stuff(data);
+            for (i = 0; i < NUM_THREADS; i++)  {
+                pthread_join(workers[i], NULL);
+            }
         }
     }
-
-    if (parallel) {
-        for (i = 0; i < NUM_THREADS; i++)  {
-            pthread_join(workers[i], NULL);
-        }
-    }
-
     return 0;
 }
 
@@ -141,15 +142,14 @@ void do_stuff(void *ptr)
     /* Cast argument to struct v pointer */
     struct v *data = ptr;
     size_t i;
+    time_t ltime;
+    struct tm result;
+    char stime[32];
 
     for (i=0; i<FILES_PER_ITERATION;i++) {
         decode_file(INPUT_FILE);
     }
 
-    time_t ltime;
-    struct tm result;
-    char stime[32];
-
     ltime = time(NULL);
     localtime_r(&ltime, &result);
     strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
diff --git a/tests/sh_ieee64.c b/tests/sh_ieee64.c
index f5788ddfc..3c7fdb5db 100644
--- a/tests/sh_ieee64.c
+++ b/tests/sh_ieee64.c
@@ -230,6 +230,7 @@ int main (int argc, char * argv[])
     size_t len;
     grib_handle * h;
     double zval[ILCHAM];
+    int m, n, k;
 
     GRIB_CHECK (((h = grib_handle_new_from_samples (NULL, "sh_ml_grib2")) == NULL), 0);
 
@@ -258,7 +259,6 @@ int main (int argc, char * argv[])
     len = ILCHAM;
     GRIB_CHECK (grib_get_double_array (h, "values", zval, &len), 0);
 
-    int m, n, k;
     for (m = 0, k = 0; m < MTRONC+1; m++)
         for (n = m; n < MTRONC+1; k++, n++)
         {
diff --git a/tools/metar_ls.c b/tools/metar_ls.c
index edbfa6ce4..6222ff051 100644
--- a/tools/metar_ls.c
+++ b/tools/metar_ls.c
@@ -18,34 +18,34 @@
 #include "grib_tools.h"
 
 grib_option grib_options[]={
-/*  {id, args, help}, on, command_line, value*/
-    {"f",0,0,1,0,0},
-    {"p:",0,0,0,1,0},
-    {"F:",0,0,1,1,"%g"},
-    {"P:",0,0,0,1,0},
-    {"w:",0,0,0,1,0},
-	{"j",0,"json output\n",0,1,0},
-    {"B:",0,0,0,1,0},
-    {"l:",0,0,0,1,0},
-	{"s:",0,0,0,1,0},
-    {"i:",0,0,0,1,0},
-    {"n:",0,0,1,1,"ls"},
-    {"m",0,0,0,1,0},
-    {"V",0,0,0,1,0},
-    {"W:",0,0,1,1,"10"},
-    {"S",0,0,1,0,0},
-    {"M",0,0,0,1,0},
-    {"H",0,0,1,0,0},
-    {"g",0,0,0,1,0},
-    {"P",0,0,1,0,0},
-    {"T:",0,0,1,0,"M"},
-    {"7",0,0,0,1,0},
-    {"v",0,0,1,0,0},
-    {"x",0,0,0,1,0}
+        /*  {id, args, help}, on, command_line, value*/
+        {"f",0,0,1,0,0},
+        {"p:",0,0,0,1,0},
+        {"F:",0,0,1,1,"%g"},
+        {"P:",0,0,0,1,0},
+        {"w:",0,0,0,1,0},
+        {"j",0,"json output\n",0,1,0},
+        {"B:",0,0,0,1,0},
+        {"l:",0,0,0,1,0},
+        {"s:",0,0,0,1,0},
+        {"i:",0,0,0,1,0},
+        {"n:",0,0,1,1,"ls"},
+        {"m",0,0,0,1,0},
+        {"V",0,0,0,1,0},
+        {"W:",0,0,1,1,"10"},
+        {"S",0,0,1,0,0},
+        {"M",0,0,0,1,0},
+        {"H",0,0,1,0,0},
+        {"g",0,0,0,1,0},
+        {"P",0,0,1,0,0},
+        {"T:",0,0,1,0,"M"},
+        {"7",0,0,0,1,0},
+        {"v",0,0,1,0,0},
+        {"x",0,0,0,1,0}
 };
 
 char* grib_tool_description="List content of metar files printing values of "
-                  "some keys.\n\tIt does not fail when a key is not found.";
+        "some keys.\n\tIt does not fail when a key is not found.";
 char* grib_tool_name="metar_ls";
 char* grib_tool_usage="[options] file file ...";
 static char* new_handle="";
@@ -65,288 +65,288 @@ int main(int argc, char *argv[]) { return grib_tool(argc,argv);}
 This is executed before processing the options with i
 getopt and therfore it is the right place for hacking 
 the arguments if needed
-*/
+ */
 int grib_tool_before_getopt(grib_runtime_options* options) {
-  return 0;
+    return 0;
 }
 
 /*
 The options have been parsed and the structure 
 grib_runtime_options* options has been loaded. 
 Initialization and startup can be done here
-*/
+ */
 int grib_tool_init(grib_runtime_options* options) {
-  char  *end = NULL, *end1=NULL;
-  size_t size=4;
-  int ret=0;
-  double min=0,max=0;
-  int i=0;
-  char* p=NULL;
-  if (grib_options_on("j")) {
-	options->verbose=0;
-	json=1;
-  }
-  
-  if (options->latlon) {
-	  
-    lat = strtod(options->latlon,&end);
-    if (*end != ',') {
-      printf("ERROR: wrong latitude value\n");
-      exit(1);
+    char  *end = NULL, *end1=NULL;
+    size_t size=4;
+    int ret=0;
+    double min=0,max=0;
+    int i=0;
+    char* p=NULL;
+    if (grib_options_on("j")) {
+        options->verbose=0;
+        json=1;
     }
-    lon= strtod(++end,&end1);
-    
-    mode=GRIB_NEAREST_SAME_POINT | GRIB_NEAREST_SAME_GRID;
 
-    if (end1 && *end1 == ',') {
-      end1++;
-      if (*end1 != '0') {
-        p=end1;
-        while (*p != ',' && *p !='\0') p++;
-        if (*end1 == '4') {
-            options->latlon_mode=4;
-        } else if (*end1 == '1') {
-            options->latlon_mode=1;
-        } else {
-          printf("ERROR %s: wrong mode given in option -l\n",grib_tool_name);
-          exit(1);
+    if (options->latlon) {
+
+        lat = strtod(options->latlon,&end);
+        if (*end != ',') {
+            printf("ERROR: wrong latitude value\n");
+            exit(1);
         }
-      }
-      if (*p == ',') {
-        p++;
-        options->latlon_mask=strdup(p);
-      }
+        lon= strtod(++end,&end1);
+
+        mode=GRIB_NEAREST_SAME_POINT | GRIB_NEAREST_SAME_GRID;
+
+        if (end1 && *end1 == ',') {
+            end1++;
+            if (*end1 != '0') {
+                p=end1;
+                while (*p != ',' && *p !='\0') p++;
+                if (*end1 == '4') {
+                    options->latlon_mode=4;
+                } else if (*end1 == '1') {
+                    options->latlon_mode=1;
+                } else {
+                    printf("ERROR %s: wrong mode given in option -l\n",grib_tool_name);
+                    exit(1);
+                }
+            }
+            if (*p == ',') {
+                p++;
+                options->latlon_mask=strdup(p);
+            }
+        }
+
     }
 
-  }
+    if (options->latlon && options->latlon_mask) {
+        FILE* f=NULL;
+        grib_handle* hh;
+        f=fopen(options->latlon_mask,"r");
+        if(!f) {
+            perror(options->latlon_mask);
+            exit(1);
+        }
+        hh=grib_handle_new_from_file(0,f,&ret);
+        fclose(f);
+        GRIB_CHECK_NOLINE(ret,0);
+        n=grib_nearest_new(hh,&ret);
+        GRIB_CHECK_NOLINE(ret,0);
+        GRIB_CHECK_NOLINE(grib_nearest_find(n,hh,lat,lon,mode,
+                options->lats,options->lons,options->mask_values,options->distances,options->indexes,&size),0);
+        grib_nearest_delete(n);
+        n=NULL;
+        grib_handle_delete( hh);
 
-  if (options->latlon && options->latlon_mask) {
-    FILE* f=NULL;
-    grib_handle* hh;
-    f=fopen(options->latlon_mask,"r");
-    if(!f) {
-      perror(options->latlon_mask);
-      exit(1);
-    }
-    hh=grib_handle_new_from_file(0,f,&ret);
-    fclose(f);
-    GRIB_CHECK_NOLINE(ret,0);
-    n=grib_nearest_new(hh,&ret);
-    GRIB_CHECK_NOLINE(ret,0);
-    GRIB_CHECK_NOLINE(grib_nearest_find(n,hh,lat,lon,mode,
-        options->lats,options->lons,options->mask_values,options->distances,options->indexes,&size),0);
-    grib_nearest_delete(n);
-    n=NULL;
-    grib_handle_delete( hh);
+        options->latlon_idx=-1;
+        max=options->distances[0];
+        for (i=0;i<4;i++)
+            if (max<options->distances[i]) {max=options->distances[i];}
+        min=max;
+        for (i=0;i<4;i++) {
+            if ((min >= options->distances[i]) && (options->mask_values[i] >= 0.5)) {
+                options->latlon_idx=i;
+                min = options->distances[i];
+            }
+        }
 
-    options->latlon_idx=-1;
-    max=options->distances[0];
-    for (i=0;i<4;i++)
-      if (max<options->distances[i]) {max=options->distances[i];}
-    min=max;
-    for (i=0;i<4;i++) {
-      if ((min >= options->distances[i]) && (options->mask_values[i] >= 0.5)) {
-        options->latlon_idx=i;
-        min = options->distances[i];
-      }
-    }
-
-    if (options->latlon_idx<0){
-      min=0;
-      options->latlon_idx=0;
-      for (i=1;i<4;i++)
-        if (min>options->distances[i]) {
-          min = options->distances[i];
-          options->latlon_idx=i;
+        if (options->latlon_idx<0){
+            min=0;
+            options->latlon_idx=0;
+            for (i=1;i<4;i++)
+                if (min>options->distances[i]) {
+                    min = options->distances[i];
+                    options->latlon_idx=i;
+                }
         }
     }
-  }
-  if (json) printf("[\n");
+    if (json) printf("[\n");
 
-  return 0;
+    return 0;
 }
 
 /*
 A new file is being parsed. The file name is file. This function is called every time
 a new input file name is processed, before opening the file.
-*/
+ */
 int grib_tool_new_filename_action(grib_runtime_options* options,const char* file) {
-   return 0;
-   }
+    return 0;
+}
 
 
 int grib_tool_new_file_action(grib_runtime_options* options,grib_tools_file* file) {
-   return 0;
+    return 0;
 }
 
 static void print_key_values(grib_runtime_options* options,grib_handle* h) {
-	int i;
-	int ret=0;
-	char* s="\"keys\" : {";
-	double dvalue=0;
-	long lvalue=0;
-	char value[MAX_STRING_LEN];
-	size_t len=MAX_STRING_LEN;
-	for (i=0;i<options->print_keys_count;i++) {
-		ret=GRIB_SUCCESS;
-		printf("%s",s);
-		len=MAX_STRING_LEN;
-		printf("\"%s\" : ",options->print_keys[i].name);
-		if (grib_is_missing(h,options->print_keys[i].name,&ret) && ret==GRIB_SUCCESS)
-			printf("\"missing\"");
-		else if ( ret == GRIB_SUCCESS ) {
-			if (options->print_keys[i].type == GRIB_TYPE_UNDEFINED)
-				grib_get_native_type(h,options->print_keys[i].name,&(options->print_keys[i].type));
-			switch (options->print_keys[i].type) {
-				case GRIB_TYPE_STRING:
-					ret=grib_get_string( h,options->print_keys[i].name,value,&len);
-					printf("\"%s\"",value);
-					break;
-				case GRIB_TYPE_DOUBLE:
-					ret=grib_get_double( h,options->print_keys[i].name,&dvalue);
-					printf("%g",dvalue);
-					break;
-				case GRIB_TYPE_LONG:
-					ret=grib_get_long( h,options->print_keys[i].name,&lvalue);
-					printf("%ld",lvalue);
-					break;
-				default:
-					printf("invalid_type");
-			}
-		}
-		if (ret == GRIB_NOT_FOUND) printf("null");
-		s=", ";
-	}
-	printf("}");
+    int i;
+    int ret=0;
+    char* s="\"keys\" : {";
+    double dvalue=0;
+    long lvalue=0;
+    char value[MAX_STRING_LEN];
+    size_t len=MAX_STRING_LEN;
+    for (i=0;i<options->print_keys_count;i++) {
+        ret=GRIB_SUCCESS;
+        printf("%s",s);
+        len=MAX_STRING_LEN;
+        printf("\"%s\" : ",options->print_keys[i].name);
+        if (grib_is_missing(h,options->print_keys[i].name,&ret) && ret==GRIB_SUCCESS)
+            printf("\"missing\"");
+        else if ( ret == GRIB_SUCCESS ) {
+            if (options->print_keys[i].type == GRIB_TYPE_UNDEFINED)
+                grib_get_native_type(h,options->print_keys[i].name,&(options->print_keys[i].type));
+            switch (options->print_keys[i].type) {
+            case GRIB_TYPE_STRING:
+                ret=grib_get_string( h,options->print_keys[i].name,value,&len);
+                printf("\"%s\"",value);
+                break;
+            case GRIB_TYPE_DOUBLE:
+                ret=grib_get_double( h,options->print_keys[i].name,&dvalue);
+                printf("%g",dvalue);
+                break;
+            case GRIB_TYPE_LONG:
+                ret=grib_get_long( h,options->print_keys[i].name,&lvalue);
+                printf("%ld",lvalue);
+                break;
+            default:
+                printf("invalid_type");
+            }
+        }
+        if (ret == GRIB_NOT_FOUND) printf("null");
+        s=", ";
+    }
+    printf("}");
 }
 
 /* A new handle is available from the current input file and can be processed here.
 The handle available in this function is in the set of messages satisfying the constrant of the
 -w option. They are not to be skipped.
-*/
+ */
 int grib_tool_new_handle_action(grib_runtime_options* options, grib_handle* h) {
-  size_t size=4;
-  double v=0;
-  int err=0;
-  int i;
-  
-  if (!options->skip) {
+    size_t size=4;
+    double v=0;
+    int err=0;
+    int i;
 
-	  if (options->set_values_count != 0)
-		  err=grib_set_values(h,options->set_values,options->set_values_count);
+    if (!options->skip) {
 
-	  if( err != GRIB_SUCCESS && options->fail) exit(err);
-  }
-  
-  if (options->latlon) {
-    err=0;
-	double min;
-    if (!n) n=grib_nearest_new(h,&err);
-    GRIB_CHECK_NOLINE(err,0);
-    GRIB_CHECK_NOLINE(grib_nearest_find(n,h,lat,lon,0,
-        options->lats,options->lons,options->values,
-        options->distances,options->indexes,&size),0);
+        if (options->set_values_count != 0)
+            err=grib_set_values(h,options->set_values,options->set_values_count);
 
-	if (!options->latlon_mask) {
-		min=options->distances[0];
-		options->latlon_idx=0;
-		i=0;
-		for (i=1;i<4;i++) {
-			if (min>options->distances[i]) {
-					min=options->distances[i];
-					options->latlon_idx=i;
-			}
-		}
-	}
-	
-	if (json) {
-		char* s="\n[\n";
-		double missingValue=9999;
-		char value[MAX_STRING_LEN];
-		size_t len=MAX_STRING_LEN;
-		printf("%s",new_handle);
-		printf(	"{\n");
-		print_key_values(options,h);
-		printf("\n, \"selected\" : %d",options->latlon_idx);
-		printf(", \"method\" : ");
-		if (options->latlon_mask) printf("\"nearest_land\"");
-		else printf("\"nearest\"");
-		printf("\n, \"neighbours\" : ");
-		for (i=0;i<4;i++) {
-			printf("%s",s);
-			len=MAX_STRING_LEN;
-			printf("{\"index\" : %d, \"latitude\" : %g, \"longitude\" : %g, \"distance\" : %g, "
-				   "\"distance_unit\" : \"km\", ",
-	   				(int)options->indexes[i],options->lats[i],options->lons[i],
-					options->distances[i]);
-			if (grib_get_double_element(h,"values",options->indexes[i],&v) == GRIB_SUCCESS) {
-				if (v==missingValue) printf("\"value\" : null ");
-				else printf("\"value\" : %g ",v);
-			}
+        if( err != GRIB_SUCCESS && options->fail) exit(err);
+    }
 
-			if (grib_get_string( h,"units",value,&len)==GRIB_SUCCESS)
-				printf(", \"unit\" : \"%s\"",value);
-			
-			if (options->latlon_mask) 
-				printf(", \"mask_value\"=%.2f",options->mask_values[i]);
-			printf("}");
-			s="\n,";
-		}
+    if (options->latlon) {
+        double min;
+        err=0;
+        if (!n) n=grib_nearest_new(h,&err);
+        GRIB_CHECK_NOLINE(err,0);
+        GRIB_CHECK_NOLINE(grib_nearest_find(n,h,lat,lon,0,
+                options->lats,options->lons,options->values,
+                options->distances,options->indexes,&size),0);
 
-		printf("\n]");
-		printf("\n}");
-	}
+        if (!options->latlon_mask) {
+            min=options->distances[0];
+            options->latlon_idx=0;
+            i=0;
+            for (i=1;i<4;i++) {
+                if (min>options->distances[i]) {
+                    min=options->distances[i];
+                    options->latlon_idx=i;
+                }
+            }
+        }
 
-  }
-  new_handle="\n,";
-  return 0;
+        if (json) {
+            char* s="\n[\n";
+            double missingValue=9999;
+            char value[MAX_STRING_LEN];
+            size_t len=MAX_STRING_LEN;
+            printf("%s",new_handle);
+            printf(	"{\n");
+            print_key_values(options,h);
+            printf("\n, \"selected\" : %d",options->latlon_idx);
+            printf(", \"method\" : ");
+            if (options->latlon_mask) printf("\"nearest_land\"");
+            else printf("\"nearest\"");
+            printf("\n, \"neighbours\" : ");
+            for (i=0;i<4;i++) {
+                printf("%s",s);
+                len=MAX_STRING_LEN;
+                printf("{\"index\" : %d, \"latitude\" : %g, \"longitude\" : %g, \"distance\" : %g, "
+                        "\"distance_unit\" : \"km\", ",
+                        (int)options->indexes[i],options->lats[i],options->lons[i],
+                        options->distances[i]);
+                if (grib_get_double_element(h,"values",options->indexes[i],&v) == GRIB_SUCCESS) {
+                    if (v==missingValue) printf("\"value\" : null ");
+                    else printf("\"value\" : %g ",v);
+                }
+
+                if (grib_get_string( h,"units",value,&len)==GRIB_SUCCESS)
+                    printf(", \"unit\" : \"%s\"",value);
+
+                if (options->latlon_mask)
+                    printf(", \"mask_value\"=%.2f",options->mask_values[i]);
+                printf("}");
+                s="\n,";
+            }
+
+            printf("\n]");
+            printf("\n}");
+        }
+
+    }
+    new_handle="\n,";
+    return 0;
 }
 
 /* A new handle to skip is available. At this point something can be done
 with the message to be skipped before deleting the handle. */
 int grib_tool_skip_handle(grib_runtime_options* options, grib_handle* h) {
-  grib_handle_delete(h);
-  return 0;
+    grib_handle_delete(h);
+    return 0;
 }
 
 /* key values can be printed in this function. Headers are already printed if requested.*/
 void grib_tool_print_key_values(grib_runtime_options* options,grib_handle* h) {
-  grib_print_key_values(options,h);
+    grib_print_key_values(options,h);
 }
 
 /* this is executed after the last message in the last file is processed */
 int grib_tool_finalise_action(grib_runtime_options* options) {
-  int i=0;
-  if (options->latlon && options->verbose) {
- 
-		printf("Input Point: latitude=%.2f  longitude=%.2f\n",lat,lon);
-		printf("Grid Point chosen #%d index=%d latitude=%.2f longitude=%.2f distance=%.2f (Km)\n",
-			   options->latlon_idx+1,(int)options->indexes[options->latlon_idx],
-									  options->lats[options->latlon_idx],
-		   	options->lons[options->latlon_idx],
-	 		options->distances[options->latlon_idx]);
+    int i=0;
+    if (options->latlon && options->verbose) {
 
-		if (options->latlon_mask) {
-			printf("Mask values:\n");
-			for (i=0;i<4;i++) {
-				printf("- %d - index=%d latitude=%.2f longitude=%.2f distance=%.2f (Km) value=%.2f\n",
-					   i+1,(int)options->indexes[i],options->lats[i],options->lons[i],
-							options->distances[i],options->mask_values[i]);
-			}
-		} else {
-			printf("Other grid Points\n");
-			for (i=0;i<4;i++) {
-				printf("- %d - index=%d latitude=%.2f longitude=%.2f distance=%.2f (Km)\n",
-					   i+1,(int)options->indexes[i],options->lats[i],options->lons[i],
-							options->distances[i]);
-			}
-		}
-	}
+        printf("Input Point: latitude=%.2f  longitude=%.2f\n",lat,lon);
+        printf("Grid Point chosen #%d index=%d latitude=%.2f longitude=%.2f distance=%.2f (Km)\n",
+                options->latlon_idx+1,(int)options->indexes[options->latlon_idx],
+                options->lats[options->latlon_idx],
+                options->lons[options->latlon_idx],
+                options->distances[options->latlon_idx]);
+
+        if (options->latlon_mask) {
+            printf("Mask values:\n");
+            for (i=0;i<4;i++) {
+                printf("- %d - index=%d latitude=%.2f longitude=%.2f distance=%.2f (Km) value=%.2f\n",
+                        i+1,(int)options->indexes[i],options->lats[i],options->lons[i],
+                        options->distances[i],options->mask_values[i]);
+            }
+        } else {
+            printf("Other grid Points\n");
+            for (i=0;i<4;i++) {
+                printf("- %d - index=%d latitude=%.2f longitude=%.2f distance=%.2f (Km)\n",
+                        i+1,(int)options->indexes[i],options->lats[i],options->lons[i],
+                        options->distances[i]);
+            }
+        }
+    }
 
     if (n) grib_nearest_delete(n);
-	if (json) printf("\n]\n");
-  
-  return 0;
+    if (json) printf("\n]\n");
+
+    return 0;
 }
 
 int grib_no_handle_action(grib_runtime_options* options, int err)

From 6c2f7fde521bcf9e2c3ef3a3051d49e6e2eb31e0 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 20 Jul 2018 14:45:30 +0100
Subject: [PATCH 33/66] Merge fix from develop: Clean up entry in table

---
 definitions/bufr/tables/0/local/19-1/98/0/element.table | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/definitions/bufr/tables/0/local/19-1/98/0/element.table b/definitions/bufr/tables/0/local/19-1/98/0/element.table
index 4a3467b89..063d74532 100644
--- a/definitions/bufr/tables/0/local/19-1/98/0/element.table
+++ b/definitions/bufr/tables/0/local/19-1/98/0/element.table
@@ -14,15 +14,10 @@
 022191|numberOfValidPointsForSpecificBand|long|NUMBER OF VALID POINTS FOR SPECIFIC BAND|Numeric|0|0|10|Numeric|0|4
 022192|rmsSpecificBandSignificnatWaveHeight|double|RMS SPECIFIC BAND SIGNIFICANT WAVE HEIGHT|m|3|0|16|m|3|5
 022193|numberOfValidPointsForSpecificBandSing|long|NUMBER OF VALID POINTS FOR SPECIFIC BAND SIGN. WAVE HEIGHT|NUMERIC|0|0|10|NUMERIC|0|4
-022194|specificBandNetInstrCorrectionForSigWaveHeight|double|SPECIFIC BAND NET INSTR. CORRECTION FOR SIGNIFICANT WAVE HEIGHT|M|3|-1000|11|M                         3         4
+022194|specificBandNetInstrCorrectionForSigWaveHeight|double|SPECIFIC BAND NET INSTR. CORRECTION FOR SIGNIFICANT WAVE HEIGHT|M|3|-1000|11|M|3|4
 022195|numberOfValidPointsForSpecificBandBackscatter|long|NUMBER OF VALID POINTS FOR SPECIFIC BAND BACKSCATTER|NUMERIC|0|0|10|NUMERIC|0|4
 025112|bandSpecificAltimeterDataQualityFlag|flag|BAND SPECIFIC ALTIMETER DATA QUALITY FLAG|FLAG TABLE|0|0|9|FLAG TABLE|0|3
 025113|bandSpecificAltimeterCorrectionQualityFlag|flag|BAND SPECIFIC ALTIMETER CORRECTION QUALITY FLAG|FLAG TABLE|0|0|9|FLAG TABLE|0|3
 025165|ionosphericCorrectionFromModelOnSpecificBand|double|IONOSPHERIC CORRECTION FROM MODEL ON SPECIFIC BAND|m|4|-32768|16|m|4|5
 025166|seaStateBiasCorrectionOnSpecificBand|double|SEA STATE BIAS CORRECTION ON SPECIFIC BAND|m|4|-32768|16|m|4|5
 025167|specificBandNetInstrumentalCorrection|double|SPECIFIC BAND NET INSTRUMENTAL CORRECTION|m|4|-120000|18|m|4|6
-
-
-
-
-

From 834b2304721b8eab99ff908669489fdb32b9b1c9 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 20 Jul 2018 15:35:15 +0100
Subject: [PATCH 34/66] Test for JPEG

---
 tests/grib_ecc-604.sh | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index 0a3421abe..a11f97dcb 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -69,12 +69,9 @@ GRIB2_INPUTS="
   ${data_dir}/test_file.grib2
   ${data_dir}/sample.grib2"
 
-# Check HAVE_JPEG is defined and is equal to 1
-if [ "x$HAVE_JPEG" != x ]; then
-    if [ $HAVE_JPEG -eq 1 ]; then
-        # Include files which have messages with grid_jpeg packing
-        GRIB2_INPUTS="${data_dir}/jpeg.grib2 ${data_dir}/reduced_gaussian_surface_jpeg.grib2 "$GRIB2_INPUTS
-    fi
+if [ $HAVE_JPEG -eq 1 ]; then
+    echo "Adding extra files (HAVE_JPEG=1)"
+    GRIB2_INPUTS="${data_dir}/jpeg.grib2 ${data_dir}/reduced_gaussian_surface_jpeg.grib2 "$GRIB2_INPUTS
 fi
 #GRIB2_INPUTS=$GRIB2_INPUTS" ${data_dir}/ccsds.grib2 "
 

From 63fddf86ffe10c9d602acbb2be93cbfb08986f1e Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 25 Jul 2018 14:48:42 +0100
Subject: [PATCH 35/66] Testing: unified the 3 programs to test clone/write etc

---
 tests/CMakeLists.txt                       |   2 +-
 tests/grib_ecc-604-2.c                     | 162 ---------------------
 tests/grib_ecc-604-3.c                     | 159 --------------------
 tests/{grib_ecc-604-1.c => grib_ecc-604.c} |  78 ++++++----
 tests/grib_ecc-604.sh                      |   6 +-
 5 files changed, 53 insertions(+), 354 deletions(-)
 delete mode 100644 tests/grib_ecc-604-2.c
 delete mode 100644 tests/grib_ecc-604-3.c
 rename tests/{grib_ecc-604-1.c => grib_ecc-604.c} (62%)

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 324b34ce8..bd062ed40 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -218,7 +218,7 @@ if( ENABLE_EXTRA_TESTS AND HAVE_ECCODES_THREADS )
                       COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_encode_pthreads.sh
     )
 
-    foreach( test grib_ecc-604-1 grib_ecc-604-2 grib_ecc-604-3 )
+    foreach( test grib_ecc-604 )
         ecbuild_add_executable( TARGET    ${test}
                                 NOINSTALL
                                 SOURCES   ${test}.c
diff --git a/tests/grib_ecc-604-2.c b/tests/grib_ecc-604-2.c
deleted file mode 100644
index 9af8aa832..000000000
--- a/tests/grib_ecc-604-2.c
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Test for ECC-604: Each thread creates a new GRIB handle and writes it out.
- *                   It does not clone the handle.
- */
-
-#include <time.h>
-#include <pthread.h>
-#include <assert.h>
-
-#include "grib_api.h"
-
-/* These are passed in via argv */
-static size_t NUM_THREADS         = 0;
-static size_t FILES_PER_ITERATION = 0;
-static char*  INPUT_FILE          = NULL;
-
-static int encode_file(char *template_file, char *output_file)
-{
-    FILE *in, *out;
-    grib_handle *source_handle = NULL;
-    const void *buffer = NULL;
-    size_t size = 0;
-    int err = 0;
-    double *values;
-
-    in = fopen(template_file,"r"); assert(in);
-    out = fopen(output_file,"w");  assert(out);
-
-    /* loop over the messages in the source GRIB */
-    while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
-        int i;
-        size_t values_len = 0;
-        size_t str_len = 20;
-
-        /*GRIB_CHECK(grib_set_long(source_handle, "centre", 250),0);*/
-        GRIB_CHECK(grib_get_size(source_handle, "values", &values_len),0);
-
-        values = (double*)malloc(values_len*sizeof(double));
-        GRIB_CHECK(grib_get_double_array(source_handle, "values", values, &values_len),0);
-
-        for (i=0;i<values_len;i++) {
-            values[i] *= 0.9;
-        }
-
-        GRIB_CHECK(grib_set_string(source_handle,"stepUnits", "s", &str_len),0);
-        GRIB_CHECK(grib_set_long(source_handle, "startStep", 43200), 0);
-        GRIB_CHECK(grib_set_long(source_handle, "endStep", 86400), 0);
-        GRIB_CHECK(grib_set_long(source_handle, "bitsPerValue", 16),0);
-
-        /* set data values */
-        GRIB_CHECK(grib_set_double_array(source_handle,"values",values,values_len),0);
-
-        GRIB_CHECK(grib_get_message(source_handle,&buffer,&size),0);
-        if(fwrite(buffer,1,size,out) != size) {
-            perror(output_file);
-            return 1;
-        }
-        {
-            FILE *devnull = fopen("/dev/null", "w");
-            grib_dump_content(source_handle,devnull,"debug",0,NULL);
-        }
-        grib_handle_delete(source_handle);
-        free(values);
-    }
-
-    fclose(out);
-    fclose(in);
-
-    return 0;
-}
-
-void do_stuff(void *data);
-
-/* Structure for passing data to threads */
-struct v {
-    size_t number;
-    char *data;
-};
-
-void *runner(void *ptr); /* the thread */
-
-int main(int argc, char **argv)
-{
-    size_t i;
-    int thread_counter = 0;
-    int parallel = 1;
-    const char* prog = argv[0];
-    char* mode;
-    if (argc!=5) {
-        fprintf(stderr, "Usage:\n\t%s seq file numRuns numIter\nOr\n\t%s par file numThreads numIter\n", prog, prog);
-        return 1;
-    }
-    mode = argv[1];
-    INPUT_FILE= argv[2];
-    NUM_THREADS = atol(argv[3]);
-    FILES_PER_ITERATION = atol(argv[4]);
-
-    if (strcmp(mode,"seq")==0) {
-        parallel = 0;
-    }
-    if (parallel) {
-        printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
-        printf("Each thread creates a new GRIB handle and writes it out. No cloning the handle\n");
-    } else {
-        printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
-    }
-
-    {
-        pthread_t workers[NUM_THREADS];
-        for (i = 0; i < NUM_THREADS; i++) {
-            struct v *data = (struct v *) malloc(sizeof(struct v));
-            data->number = i;
-            data->data = NULL;
-
-            if (parallel) {
-                /* Now we will create the thread passing it data as an argument */
-                pthread_create(&workers[thread_counter], NULL, runner, data);
-                /*pthread_join(workers[thread_counter], NULL);*/
-                thread_counter++;
-            } else {
-                do_stuff(data);
-            }
-        }
-
-        if (parallel) {
-            for (i = 0; i < NUM_THREADS; i++)  {
-                pthread_join(workers[i], NULL);
-            }
-        }
-    }
-
-    return 0;
-}
-
-void *runner(void *ptr)
-{
-    do_stuff(ptr);
-    pthread_exit(0);
-}
-
-void do_stuff(void *ptr)
-{
-    /* Cast argument to struct v pointer */
-    struct v *data = ptr;
-    size_t i;
-    char output_file[50];
-    time_t ltime;
-    struct tm result;
-    char stime[32];
-
-    for (i=0; i<FILES_PER_ITERATION;i++) {
-        sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
-        encode_file(INPUT_FILE,output_file);
-    }
-
-    ltime = time(NULL);
-    localtime_r(&ltime, &result);
-    strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
-    /* asctime_r(&result, stime); */
-
-    printf("%s: Worker %ld finished.\n", stime, data->number);
-}
diff --git a/tests/grib_ecc-604-3.c b/tests/grib_ecc-604-3.c
deleted file mode 100644
index e484e1253..000000000
--- a/tests/grib_ecc-604-3.c
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Test for ECC-604: Each thread creates a new GRIB handle and clones it.
- *                   No output writing.
- */
-#include <time.h>
-#include <pthread.h>
-#include <assert.h>
-
-#include "grib_api.h"
-
-/* These are passed in via argv */
-static size_t NUM_THREADS         = 0;
-static size_t FILES_PER_ITERATION = 0;
-static char*  INPUT_FILE          = NULL;
-
-static int decode_file(char *template_file)
-{
-    FILE *in;
-    grib_handle *source_handle = NULL;
-    const void *buffer = NULL;
-    size_t size = 0;
-    int err = 0;
-    double *values;
-
-    in = fopen(template_file,"r"); assert(in);
-
-    /* loop over the messages in the source GRIB and clone them */
-    while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
-        int i;
-        size_t values_len = 0;
-        size_t str_len = 20;
-
-        grib_handle *clone_handle = grib_handle_clone(source_handle);
-        assert(clone_handle);
-
-        /*GRIB_CHECK(grib_set_long(clone_handle, "centre", 250),0);*/
-        GRIB_CHECK(grib_get_size(clone_handle, "values", &values_len),0);
-
-        values = (double*)malloc(values_len*sizeof(double));
-        GRIB_CHECK(grib_get_double_array(clone_handle, "values", values, &values_len),0);
-
-        for (i=0;i<values_len;i++) {
-            values[i] *= 0.9;
-        }
-
-        GRIB_CHECK(grib_set_string(clone_handle,"stepUnits", "s", &str_len),0);
-        GRIB_CHECK(grib_set_long(clone_handle, "startStep", 43200), 0);
-        GRIB_CHECK(grib_set_long(clone_handle, "endStep", 86400), 0);
-        GRIB_CHECK(grib_set_long(clone_handle, "bitsPerValue", 16),0);
-
-        /* set data values */
-        GRIB_CHECK(grib_set_double_array(clone_handle,"values",values,values_len),0);
-
-        GRIB_CHECK(grib_get_message(clone_handle,&buffer,&size),0);
-
-        {
-            FILE *devnull = fopen("/dev/null", "w");
-            grib_dump_content(source_handle, devnull, "debug", 0, NULL);
-            grib_dump_content(clone_handle,  devnull, "debug", 0, NULL);
-        }
-
-        grib_handle_delete(clone_handle);
-        grib_handle_delete(source_handle);
-        free(values);
-    }
-
-    fclose(in);
-
-    return 0;
-}
-
-void do_stuff(void *data);
-
-/* Structure for passing data to threads */
-struct v {
-    size_t number;
-    char *data;
-};
-
-void *runner(void *ptr); /* the thread */
-
-int main(int argc, char **argv)
-{
-    size_t i;
-    int thread_counter = 0;
-    int parallel = 1;
-    const char* prog = argv[0];
-    char* mode;
-    if (argc!=5) {
-        fprintf(stderr, "Usage:\n\t%s seq file numRuns numIter\nOr\n\t%s par file numThreads numIter\n", prog, prog);
-        return 1;
-    }
-    mode = argv[1];
-    INPUT_FILE= argv[2];
-    NUM_THREADS = atol(argv[3]);
-    FILES_PER_ITERATION = atol(argv[4]);
-
-    if (strcmp(mode,"seq")==0) {
-        parallel = 0;
-    }
-    if (parallel) {
-        printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
-        printf("Each thread creates a new GRIB handle and clones it. No output writing\n");
-    } else {
-        printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
-    }
-
-    {
-        pthread_t workers[NUM_THREADS];
-        for (i = 0; i < NUM_THREADS; i++) {
-            struct v *data = (struct v *) malloc(sizeof(struct v));
-            data->number = i;
-            data->data = NULL;
-
-            if (parallel) {
-                /* Now we will create the thread passing it data as an argument */
-                pthread_create(&workers[thread_counter], NULL, runner, data);
-                /*pthread_join(workers[thread_counter], NULL);*/
-                thread_counter++;
-            } else {
-                do_stuff(data);
-            }
-        }
-
-        if (parallel) {
-            for (i = 0; i < NUM_THREADS; i++)  {
-                pthread_join(workers[i], NULL);
-            }
-        }
-    }
-    return 0;
-}
-
-void *runner(void *ptr)
-{
-    do_stuff(ptr);
-    pthread_exit(0);
-}
-
-void do_stuff(void *ptr)
-{
-    /* Cast argument to struct v pointer */
-    struct v *data = ptr;
-    size_t i;
-    time_t ltime;
-    struct tm result;
-    char stime[32];
-
-    for (i=0; i<FILES_PER_ITERATION;i++) {
-        decode_file(INPUT_FILE);
-    }
-
-    ltime = time(NULL);
-    localtime_r(&ltime, &result);
-    strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
-    /* asctime_r(&result, stime); */
-
-    printf("%s: Worker %ld finished.\n", stime, data->number);
-}
diff --git a/tests/grib_ecc-604-1.c b/tests/grib_ecc-604.c
similarity index 62%
rename from tests/grib_ecc-604-1.c
rename to tests/grib_ecc-604.c
index d5e0ff3ae..8e2de7252 100644
--- a/tests/grib_ecc-604-1.c
+++ b/tests/grib_ecc-604.c
@@ -4,6 +4,7 @@
 #include <time.h>
 #include <pthread.h>
 #include <assert.h>
+#include <unistd.h>
 
 #include "grib_api.h"
 
@@ -11,6 +12,8 @@
 static size_t NUM_THREADS         = 0;
 static size_t FILES_PER_ITERATION = 0;
 static char*  INPUT_FILE          = NULL;
+int opt_clone = 0; /* If 1 then clone source handle */
+int opt_write = 0; /* If 1 write handle to file */
 
 static int encode_file(char *template_file, char *output_file)
 {
@@ -22,51 +25,56 @@ static int encode_file(char *template_file, char *output_file)
     double *values;
 
     in = fopen(template_file,"r"); assert(in);
-    out = fopen(output_file,"w");  assert(out);
+    if (output_file) {
+        out = fopen(output_file,"w");  assert(out);
+    }
 
     /* loop over the messages in the source GRIB and clone them */
     while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
         int i;
         size_t values_len = 0;
         size_t str_len = 20;
+        grib_handle *h = source_handle;
+        
+        if (opt_clone) {
+            h = grib_handle_clone(source_handle); assert(h);
+        }
 
-        grib_handle *clone_handle = grib_handle_clone(source_handle);
-        assert(clone_handle);
-
-        /*GRIB_CHECK(grib_set_long(clone_handle, "centre", 250),0);*/
-        GRIB_CHECK(grib_get_size(clone_handle, "values", &values_len),0);
+        GRIB_CHECK(grib_get_size(h, "values", &values_len),0);
 
         values = (double*)malloc(values_len*sizeof(double));
-        GRIB_CHECK(grib_get_double_array(clone_handle, "values", values, &values_len),0);
+        GRIB_CHECK(grib_get_double_array(h, "values", values, &values_len),0);
 
         for (i=0;i<values_len;i++) {
             values[i] *= 0.9;
         }
 
-        GRIB_CHECK(grib_set_string(clone_handle,"stepUnits", "s", &str_len),0);
-        GRIB_CHECK(grib_set_long(clone_handle, "startStep", 43200), 0);
-        GRIB_CHECK(grib_set_long(clone_handle, "endStep", 86400), 0);
-        GRIB_CHECK(grib_set_long(clone_handle, "bitsPerValue", 16),0);
+        GRIB_CHECK(grib_set_string(h,"stepUnits", "s", &str_len),0);
+        GRIB_CHECK(grib_set_long(h, "startStep", 43200), 0);
+        GRIB_CHECK(grib_set_long(h, "endStep", 86400), 0);
+        GRIB_CHECK(grib_set_long(h, "bitsPerValue", 16),0);
 
         /* set data values */
-        GRIB_CHECK(grib_set_double_array(clone_handle,"values",values,values_len),0);
+        GRIB_CHECK(grib_set_double_array(h,"values",values,values_len),0);
 
-        GRIB_CHECK(grib_get_message(clone_handle,&buffer,&size),0);
-        if(fwrite(buffer,1,size,out) != size) {
-            perror(output_file);
-            return 1;
+        GRIB_CHECK(grib_get_message(h,&buffer,&size),0);
+        if (output_file) {
+            if(fwrite(buffer,1,size,out) != size) {
+                perror(output_file);
+                return 1;
+            }
         }
         {
             FILE *devnull = fopen("/dev/null", "w");
             grib_dump_content(source_handle,devnull, "debug", 0, NULL);
-            grib_dump_content(clone_handle, devnull, "debug", 0, NULL);
         }
-        grib_handle_delete(clone_handle);
+
         grib_handle_delete(source_handle);
+        if(opt_clone) grib_handle_delete(h);
         free(values);
     }
 
-    fclose(out);
+    if (output_file) fclose(out);
     fclose(in);
 
     return 0;
@@ -86,24 +94,32 @@ int main(int argc, char **argv)
 {
     size_t i;
     int thread_counter = 0;
-    int parallel = 1;
+    int parallel=1, index=0, c=0;
     const char* prog = argv[0];
     char* mode;
-    if (argc!=5) {
-        fprintf(stderr, "Usage:\n\t%s seq file numRuns numIter\nOr\n\t%s par file numThreads numIter\n", prog, prog);
+    if (argc<5 || argc>7) {
+        fprintf(stderr, "Usage:\n\t%s [options] seq file numRuns numIter\nOr\n\t%s [options] par file numThreads numIter\n", prog, prog);
         return 1;
     }
-    mode = argv[1];
-    INPUT_FILE= argv[2];
-    NUM_THREADS = atol(argv[3]);
-    FILES_PER_ITERATION = atol(argv[4]);
+    
+    while ((c = getopt (argc, argv, "cw")) != -1) {
+        switch (c) {
+            case 'c': opt_clone=1; break;
+            case 'w': opt_write=1; break;
+        }
+    }
+    index = optind;
+    mode = argv[index];
+    INPUT_FILE = argv[index+1];
+    NUM_THREADS = atol(argv[index+2]);
+    FILES_PER_ITERATION = atol(argv[index+3]);
 
     if (strcmp(mode,"seq")==0) {
         parallel = 0;
     }
     if (parallel) {
         printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
-        printf("Each thread creates a new GRIB handle, clones it and writes it out\n");
+        printf("Options: clone=%d, write=%d\n", opt_clone, opt_write);
     } else {
         printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
     }
@@ -152,8 +168,12 @@ void do_stuff(void *ptr)
     char stime[32];
 
     for (i=0; i<FILES_PER_ITERATION;i++) {
-        sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
-        encode_file(INPUT_FILE,output_file);
+        if (opt_write) {
+            sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
+            encode_file(INPUT_FILE,output_file);
+        } else {
+            encode_file(INPUT_FILE,NULL);
+        }
     }
 
     ltime = time(NULL);
diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index a11f97dcb..4a22eab98 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -36,19 +36,19 @@ process()
     # Test 01: Clone + output
     # ------------------------
     rm -fr $OUTPUT;   mkdir -p $OUTPUT
-    time ${test_dir}/grib_ecc-604-1 par $input $NUM_THREADS $NUM_ITER
+    time ${test_dir}/grib_ecc-604 -c -w par $input $NUM_THREADS $NUM_ITER
     validate
 
     # Test 02: No clone + output
     # --------------------------
     rm -fr $OUTPUT;   mkdir -p $OUTPUT
-    time ${test_dir}/grib_ecc-604-2 par $input $NUM_THREADS $NUM_ITER
+    time ${test_dir}/grib_ecc-604 -w par $input $NUM_THREADS $NUM_ITER
     validate
 
     # Test 03: Clone + no output
     # ---------------------------
     rm -fr $OUTPUT
-    time ${test_dir}/grib_ecc-604-3 par $input $NUM_THREADS $NUM_ITER
+    time ${test_dir}/grib_ecc-604 -c par $input $NUM_THREADS $NUM_ITER
     # Nothing to validate as there is no output
 }
 ###################################################

From 449f94d2077e6b437af29f1be73be74550fcf5e1 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 27 Jul 2018 17:39:24 +0100
Subject: [PATCH 36/66] Remove unused static global

---
 src/grib_dumper_class_bufr_simple.c | 24 ------------------------
 1 file changed, 24 deletions(-)

diff --git a/src/grib_dumper_class_bufr_simple.c b/src/grib_dumper_class_bufr_simple.c
index 95d68a752..cc7d57cc8 100644
--- a/src/grib_dumper_class_bufr_simple.c
+++ b/src/grib_dumper_class_bufr_simple.c
@@ -110,8 +110,6 @@ struct string_count {
     string_count* next;
 };
 
-static int depth=0;
-
 static void init_class      (grib_dumper_class* c){}
 
 static int init(grib_dumper* d)
@@ -188,7 +186,6 @@ static void dump_values(grib_dumper* d, grib_accessor* a)
         if (icount>cols || i==0) {fprintf(self->dumper.out,"\n      ");icount=0;}
         fprintf(self->dumper.out, "%g",values[i]);
 
-        depth-=2;
         fprintf(self->dumper.out,"}\n");
         grib_context_free(c,values);
     } else {
@@ -217,7 +214,6 @@ static void dump_values(grib_dumper* d, grib_accessor* a)
 
         dump_attributes(d,a,prefix);
         if (dofree) grib_context_free(c,prefix);
-        depth-=2;
     }
 
     (void)err; /* TODO */
@@ -260,7 +256,6 @@ static void dump_values_attribute(grib_dumper* d,grib_accessor* a, const char* p
         if (icount>cols || i==0) {fprintf(self->dumper.out,"\n      ");icount=0;}
         fprintf(self->dumper.out,"%g", values[i]);
 
-        depth-=2;
         fprintf(self->dumper.out,"}\n");
         grib_context_free(c,values);
     } else {
@@ -281,7 +276,6 @@ static void dump_values_attribute(grib_dumper* d,grib_accessor* a, const char* p
         dump_attributes(d,a,prefix1);
 
         grib_context_free(c,prefix1);
-        depth-=2;
     }
 
     (void)err; /* TODO */
@@ -319,7 +313,6 @@ static void dump_long(grib_dumper* d, grib_accessor* a, const char* comment)
             
             dump_attributes(d,a,prefix);
             if (dofree) grib_context_free(c,prefix);
-            depth-=2;
         }
         return;
     }
@@ -360,7 +353,6 @@ static void dump_long(grib_dumper* d, grib_accessor* a, const char* comment)
         else
             fprintf(self->dumper.out,"%ld ",values[i]);
 
-        depth-=2;
         fprintf(self->dumper.out,"}\n");
         grib_context_free(a->context,values);
     } else {
@@ -389,7 +381,6 @@ static void dump_long(grib_dumper* d, grib_accessor* a, const char* comment)
 
         dump_attributes(d,a,prefix);
         if (dofree) grib_context_free(c,prefix);
-        depth-=2;
     }
     (void)err; /* TODO */
 }
@@ -430,7 +421,6 @@ static void dump_long_attribute(grib_dumper* d, grib_accessor* a, const char* pr
         }
         if (icount>cols || i==0) {fprintf(self->dumper.out,"\n      ");icount=0;}
         fprintf(self->dumper.out,"%ld ",values[i]);
-        depth-=2;
         fprintf(self->dumper.out,"}\n");
         grib_context_free(a->context,values);
 
@@ -453,7 +443,6 @@ static void dump_long_attribute(grib_dumper* d, grib_accessor* a, const char* pr
         dump_attributes(d,a,prefix1);
 
         grib_context_free(c,prefix1);
-        depth-=2;
     }
     (void)err; /* TODO */
 }
@@ -502,7 +491,6 @@ static void dump_double(grib_dumper* d, grib_accessor* a, const char* comment)
 
         dump_attributes(d,a,prefix);
         if (dofree) grib_context_free(c,prefix);
-        depth-=2;
     }
 }
 
@@ -530,7 +518,6 @@ static void dump_string_array(grib_dumper* d, grib_accessor* a, const char* comm
     self->begin=0;
 
     if (self->isLeaf==0) {
-        depth+=2;
         if ((r=compute_bufr_key_rank(h,self->keys,a->name))!=0)
             fprintf(self->dumper.out,"#%d#%s=",r,a->name);
         else
@@ -548,14 +535,11 @@ static void dump_string_array(grib_dumper* d, grib_accessor* a, const char* comm
     err = grib_unpack_string_array(a,values,&size);
 
     fprintf(self->dumper.out, "{");
-    depth+=2;
     for  (i=0;i<size-1;i++) {
         fprintf(self->dumper.out,"    \"%s\",\n",values[i]);
     }
     fprintf(self->dumper.out,"    \"%s\"\n",values[i]);
 
-    depth-=2;
-
     fprintf(self->dumper.out, "}\n");
 
     if (self->isLeaf==0) {
@@ -570,7 +554,6 @@ static void dump_string_array(grib_dumper* d, grib_accessor* a, const char* comm
 
         dump_attributes(d,a,prefix);
         if (dofree) grib_context_free(c,prefix);
-        depth-=2;
     }
 
     grib_context_free(c,values);
@@ -615,7 +598,6 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
     while(*p) { if(!isprint(*p)) *p = '.'; p++; }
 
     if (self->isLeaf==0) {
-        depth+=2;
         if (r!=0)
             fprintf(self->dumper.out,"#%d#%s=",r,a->name);
         else
@@ -636,7 +618,6 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
 
         dump_attributes(d,a,prefix);
         if (dofree) grib_context_free(c,prefix);
-        depth-=2;
     }
 
     grib_context_free(c,value);
@@ -682,10 +663,8 @@ static void dump_section(grib_dumper* d, grib_accessor* a, grib_block_of_accesso
             !grib_inline_strcmp(a->name,"META")
     ) {
         grib_handle* h=grib_handle_of_accessor(a);
-        depth=2;
         self->begin=1;
         self->empty=1;
-        depth+=2;
         _dump_long_array(h,self->dumper.out,"dataPresentIndicator");
         _dump_long_array(h,self->dumper.out,"delayedDescriptorReplicationFactor");
         _dump_long_array(h,self->dumper.out,"shortDelayedDescriptorReplicationFactor");
@@ -693,15 +672,12 @@ static void dump_section(grib_dumper* d, grib_accessor* a, grib_block_of_accesso
         /* Do not show the inputOverriddenReferenceValues array. That's more for ENCODING */
         /*_dump_long_array(h,self->dumper.out,"inputOverriddenReferenceValues","inputOverriddenReferenceValues");*/
         grib_dump_accessors_block(d,block);
-        depth-=2;
     } else if (!grib_inline_strcmp(a->name,"groupNumber")) {
         if ( (a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0)
             return;
         self->begin=1;
         self->empty=1;
-        depth+=2;
         grib_dump_accessors_block(d,block);
-        depth-=2;
     } else {
         grib_dump_accessors_block(d,block);
     }

From b3b9baf006f288e043208609c947b902c482a1cd Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 27 Jul 2018 17:40:13 +0100
Subject: [PATCH 37/66] ECC-604: BUFR decoding in parallel

---
 tests/CMakeLists.txt  |   7 +-
 tests/bufr_ecc-604.c  | 174 ++++++++++++++++++++++++++++++++++++++++++
 tests/bufr_ecc-604.sh |  66 ++++++++++++++++
 3 files changed, 246 insertions(+), 1 deletion(-)
 create mode 100644 tests/bufr_ecc-604.c
 create mode 100755 tests/bufr_ecc-604.sh

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index bd062ed40..ac4e5bfa1 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -218,7 +218,7 @@ if( ENABLE_EXTRA_TESTS AND HAVE_ECCODES_THREADS )
                       COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_encode_pthreads.sh
     )
 
-    foreach( test grib_ecc-604 )
+    foreach( test grib_ecc-604 bufr_ecc-604 )
         ecbuild_add_executable( TARGET    ${test}
                                 NOINSTALL
                                 SOURCES   ${test}.c
@@ -230,6 +230,11 @@ if( ENABLE_EXTRA_TESTS AND HAVE_ECCODES_THREADS )
                       TEST_DEPENDS eccodes_download_gribs
                       COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_ecc-604.sh
     )
+    ecbuild_add_test( TARGET eccodes_t_bufr_ecc-604
+                      TYPE SCRIPT
+                      TEST_DEPENDS eccodes_download_bufrs
+                      COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bufr_ecc-604.sh
+    )
 endif()
 
 ecbuild_add_test( TARGET eccodes_t_grib_to_netcdf
diff --git a/tests/bufr_ecc-604.c b/tests/bufr_ecc-604.c
new file mode 100644
index 000000000..a4ee6e573
--- /dev/null
+++ b/tests/bufr_ecc-604.c
@@ -0,0 +1,174 @@
+/*
+ * Test for ECC-604: Each thread creates a new BUFR handle, optionaly clone it and/or write it out
+ */
+#include <time.h>
+#include <pthread.h>
+#include <assert.h>
+#include <unistd.h>
+
+#include "eccodes.h"
+#include "grib_api_internal.h"
+
+/* These are passed in via argv */
+static size_t NUM_THREADS         = 0;
+static size_t FILES_PER_ITERATION = 0;
+static char*  INPUT_FILE          = NULL;
+int opt_clone = 0; /* If 1 then clone source handle */
+int opt_write = 0; /* If 1 write handle to file */
+
+static int encode_file(char *template_file, char *output_file)
+{
+    FILE *in, *out;
+    codes_handle *source_handle = NULL;
+    const void *buffer = NULL;
+    size_t size = 0;
+    int err = 0;
+    long numSubsets = 0;
+
+    in = fopen(template_file,"r"); assert(in);
+    if (output_file) {
+        out = fopen(output_file,"w");  assert(out);
+    }
+
+    /* loop over the messages in the source BUFR and clone them */
+    while ((source_handle = codes_handle_new_from_file(NULL, in, PRODUCT_BUFR, &err)) != NULL || err != CODES_SUCCESS) {
+        codes_handle *h = source_handle;
+        
+        if (opt_clone) {
+            h = codes_handle_clone(source_handle); assert(h);
+        }
+
+        CODES_CHECK(codes_get_long(h,"numberOfSubsets", &numSubsets),0);
+        CODES_CHECK(codes_set_long(h, "unpack", 1),0);
+
+        CODES_CHECK(codes_get_message(h,&buffer,&size),0);
+        if (output_file) {
+            if(fwrite(buffer,1,size,out) != size) {
+                perror(output_file);
+                return 1;
+            }
+        }
+        {
+            /* JSON dump with all attributes */
+            FILE *devnull = fopen("/dev/null", "w");
+            grib_dumper* dumper = NULL;
+            const char* dumper_name = "bufr_simple";
+            unsigned long dump_flags = 512;
+            //codes_dump_content(source_handle,devnull, "json", 1024, NULL);
+            dumper=grib_dump_content_with_dumper(source_handle, dumper, devnull, dumper_name, dump_flags,0);
+            assert(dumper);
+        }
+
+        codes_handle_delete(source_handle);
+        if(opt_clone) codes_handle_delete(h);
+    }
+
+    if (output_file) fclose(out);
+    fclose(in);
+
+    return 0;
+}
+
+void do_stuff(void *data);
+
+/* Structure for passing data to threads */
+struct v {
+    size_t number;
+    char *data;
+};
+
+void *runner(void *ptr); /* the thread */
+
+int main(int argc, char **argv)
+{
+    size_t i;
+    int thread_counter = 0;
+    int parallel=1, index=0, c=0;
+    const char* prog = argv[0];
+    char* mode;
+    if (argc<5 || argc>7) {
+        fprintf(stderr, "Usage:\n\t%s [options] seq file numRuns numIter\nOr\n\t%s [options] par file numThreads numIter\n", prog, prog);
+        return 1;
+    }
+    
+    while ((c = getopt (argc, argv, "cw")) != -1) {
+        switch (c) {
+            case 'c': opt_clone=1; break;
+            case 'w': opt_write=1; break;
+        }
+    }
+    index = optind;
+    mode = argv[index];
+    INPUT_FILE = argv[index+1];
+    NUM_THREADS = atol(argv[index+2]);
+    FILES_PER_ITERATION = atol(argv[index+3]);
+
+    if (strcmp(mode,"seq")==0) {
+        parallel = 0;
+    }
+    if (parallel) {
+        printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+        printf("Options: clone=%d, write=%d\n", opt_clone, opt_write);
+    } else {
+        printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+    }
+
+    {
+        pthread_t workers[NUM_THREADS];
+        for (i = 0; i < NUM_THREADS; i++) {
+            struct v *data = (struct v *) malloc(sizeof(struct v));
+            data->number = i;
+            data->data = NULL;
+
+            if (parallel) {
+                /* Now we will create the thread passing it data as an argument */
+                pthread_create(&workers[thread_counter], NULL, runner, data);
+                /*pthread_join(workers[thread_counter], NULL);*/
+                thread_counter++;
+            } else {
+                do_stuff(data);
+            }
+        }
+
+        if (parallel) {
+            for (i = 0; i < NUM_THREADS; i++)  {
+                pthread_join(workers[i], NULL);
+            }
+        }
+    }
+
+    return 0;
+}
+
+void *runner(void *ptr)
+{
+    do_stuff(ptr);
+    pthread_exit(0);
+}
+
+void do_stuff(void *ptr)
+{
+    /* Cast argument to struct v pointer */
+    struct v *data = ptr;
+    size_t i;
+    char output_file[50];
+    time_t ltime;
+    struct tm result;
+    char stime[32];
+
+    for (i=0; i<FILES_PER_ITERATION;i++) {
+        if (opt_write) {
+            sprintf(output_file,"output/output_file_%ld-%ld.bufr", data->number, i);
+            encode_file(INPUT_FILE,output_file);
+        } else {
+            encode_file(INPUT_FILE,NULL);
+        }
+    }
+
+    ltime = time(NULL);
+    localtime_r(&ltime, &result);
+    strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
+    /* asctime_r(&result, stime); */
+
+    printf("%s: Worker %ld finished.\n", stime, data->number);
+}
diff --git a/tests/bufr_ecc-604.sh b/tests/bufr_ecc-604.sh
new file mode 100755
index 000000000..482b4413d
--- /dev/null
+++ b/tests/bufr_ecc-604.sh
@@ -0,0 +1,66 @@
+#!/bin/sh
+# Copyright 2005-2018 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+
+label="bufr_ecc-604"
+temp_dir=tempdir.${label}
+
+NUM_THREADS=3
+NUM_ITER=10
+OUTPUT=output
+
+validate()
+{
+    echo "Checking every output file is identical..."
+    # Get checksum of first file
+    ck1=`cksum $OUTPUT/output_file_0-0.bufr | awk '{print $1}'`
+    set +x
+    # Get checksum of all of them and sort unique
+    res=`cksum $OUTPUT/output_file_* | awk '{print $1}' | sort -u`
+    set -x
+    # Should be the same as the first
+    [ "$res" = "$ck1" ]
+}
+process()
+{
+    input=$1 # The input BUFR file
+
+    # Test 01: Clone + output
+    # ------------------------
+    rm -fr $OUTPUT;   mkdir -p $OUTPUT
+    time ${test_dir}/bufr_ecc-604 -c -w par $input $NUM_THREADS $NUM_ITER
+    validate
+
+    # Test 02: No clone + output
+    # --------------------------
+    rm -fr $OUTPUT;   mkdir -p $OUTPUT
+    time ${test_dir}/bufr_ecc-604 -w par $input $NUM_THREADS $NUM_ITER
+    validate
+
+    # Test 03: Clone + no output
+    # ---------------------------
+    rm -fr $OUTPUT
+    time ${test_dir}/bufr_ecc-604 -c par $input $NUM_THREADS $NUM_ITER
+    # Nothing to validate as there is no output
+}
+###################################################
+rm -fr $temp_dir
+mkdir -p $temp_dir
+cd $temp_dir
+
+for bf in $data_dir/bufr/*.bufr; do
+    echo "Doing $bf"
+    process $bf
+done
+
+# Clean up
+cd $test_dir
+rm -fr $temp_dir

From 5b794f3c35182a8119fc54e815cc20de2dbea771 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Mon, 30 Jul 2018 16:11:00 +0100
Subject: [PATCH 38/66] Test: Simplify

---
 tests/bufr_ecc-604.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/bufr_ecc-604.c b/tests/bufr_ecc-604.c
index a4ee6e573..3ee07634f 100644
--- a/tests/bufr_ecc-604.c
+++ b/tests/bufr_ecc-604.c
@@ -49,14 +49,14 @@ static int encode_file(char *template_file, char *output_file)
             }
         }
         {
-            /* JSON dump with all attributes */
             FILE *devnull = fopen("/dev/null", "w");
             grib_dumper* dumper = NULL;
             const char* dumper_name = "bufr_simple";
-            unsigned long dump_flags = 512;
-            //codes_dump_content(source_handle,devnull, "json", 1024, NULL);
-            dumper=grib_dump_content_with_dumper(source_handle, dumper, devnull, dumper_name, dump_flags,0);
+            unsigned long dump_flags = CODES_DUMP_FLAG_ALL_DATA;
+            //codes_dump_content(source_handle,devnull, "json", 1024, NULL); /* JSON dump with all attributes */
+            dumper=grib_dump_content_with_dumper(source_handle, dumper, devnull, dumper_name, dump_flags, NULL);
             assert(dumper);
+            fclose(devnull);
         }
 
         codes_handle_delete(source_handle);

From 21f477dd30c95b83ea2a31a919c239d900b83faf Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Mon, 30 Jul 2018 17:29:53 +0100
Subject: [PATCH 39/66] Test: Add option to do invoke the dumper (decode all
 values)

---
 tests/bufr_ecc-604.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/bufr_ecc-604.c b/tests/bufr_ecc-604.c
index 3ee07634f..27f340b7e 100644
--- a/tests/bufr_ecc-604.c
+++ b/tests/bufr_ecc-604.c
@@ -13,6 +13,7 @@
 static size_t NUM_THREADS         = 0;
 static size_t FILES_PER_ITERATION = 0;
 static char*  INPUT_FILE          = NULL;
+int opt_dump  = 0; /* If 1 then dump handle to /dev/null */
 int opt_clone = 0; /* If 1 then clone source handle */
 int opt_write = 0; /* If 1 write handle to file */
 
@@ -48,7 +49,7 @@ static int encode_file(char *template_file, char *output_file)
                 return 1;
             }
         }
-        {
+        if (opt_dump) {
             FILE *devnull = fopen("/dev/null", "w");
             grib_dumper* dumper = NULL;
             const char* dumper_name = "bufr_simple";
@@ -91,8 +92,9 @@ int main(int argc, char **argv)
         return 1;
     }
     
-    while ((c = getopt (argc, argv, "cw")) != -1) {
+    while ((c = getopt (argc, argv, "dcw")) != -1) {
         switch (c) {
+            case 'd': opt_dump=1; break;
             case 'c': opt_clone=1; break;
             case 'w': opt_write=1; break;
         }
@@ -108,7 +110,7 @@ int main(int argc, char **argv)
     }
     if (parallel) {
         printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
-        printf("Options: clone=%d, write=%d\n", opt_clone, opt_write);
+        printf("Options: dump=%d, clone=%d, write=%d\n", opt_dump, opt_clone, opt_write);
     } else {
         printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
     }

From 4ac5e3691e004027d16f94f09d65f39e13d67da2 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Mon, 30 Jul 2018 17:30:19 +0100
Subject: [PATCH 40/66] Test: Process all BUFR test data using list

---
 tests/bufr_ecc-604.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/bufr_ecc-604.sh b/tests/bufr_ecc-604.sh
index 482b4413d..7bd6200ee 100755
--- a/tests/bufr_ecc-604.sh
+++ b/tests/bufr_ecc-604.sh
@@ -56,9 +56,11 @@ rm -fr $temp_dir
 mkdir -p $temp_dir
 cd $temp_dir
 
-for bf in $data_dir/bufr/*.bufr; do
-    echo "Doing $bf"
-    process $bf
+bufr_files=`cat ${data_dir}/bufr/bufr_data_files.txt`
+for bf in ${bufr_files}; do
+    b=${data_dir}/bufr/$bf
+    echo "Doing $b"
+    process $b
 done
 
 # Clean up

From 3826c9ddf26d1d48d9f51a15ea137242e012e6ef Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Mon, 6 Aug 2018 13:44:56 +0100
Subject: [PATCH 41/66] Tests: Add option for dump

---
 tests/bufr_ecc-604.c  |  6 +++---
 tests/grib_ecc-604.c  | 14 ++++++++------
 tests/grib_ecc-604.sh |  6 +++---
 3 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/tests/bufr_ecc-604.c b/tests/bufr_ecc-604.c
index 27f340b7e..3f0c5b577 100644
--- a/tests/bufr_ecc-604.c
+++ b/tests/bufr_ecc-604.c
@@ -27,7 +27,7 @@ static int encode_file(char *template_file, char *output_file)
     long numSubsets = 0;
 
     in = fopen(template_file,"r"); assert(in);
-    if (output_file) {
+    if (opt_write) {
         out = fopen(output_file,"w");  assert(out);
     }
 
@@ -43,7 +43,7 @@ static int encode_file(char *template_file, char *output_file)
         CODES_CHECK(codes_set_long(h, "unpack", 1),0);
 
         CODES_CHECK(codes_get_message(h,&buffer,&size),0);
-        if (output_file) {
+        if (opt_write) {
             if(fwrite(buffer,1,size,out) != size) {
                 perror(output_file);
                 return 1;
@@ -64,7 +64,7 @@ static int encode_file(char *template_file, char *output_file)
         if(opt_clone) codes_handle_delete(h);
     }
 
-    if (output_file) fclose(out);
+    if (opt_write) fclose(out);
     fclose(in);
 
     return 0;
diff --git a/tests/grib_ecc-604.c b/tests/grib_ecc-604.c
index 8e2de7252..be7be4643 100644
--- a/tests/grib_ecc-604.c
+++ b/tests/grib_ecc-604.c
@@ -12,6 +12,7 @@
 static size_t NUM_THREADS         = 0;
 static size_t FILES_PER_ITERATION = 0;
 static char*  INPUT_FILE          = NULL;
+int opt_dump  = 0; /* If 1 then dump handle to /dev/null */
 int opt_clone = 0; /* If 1 then clone source handle */
 int opt_write = 0; /* If 1 write handle to file */
 
@@ -25,7 +26,7 @@ static int encode_file(char *template_file, char *output_file)
     double *values;
 
     in = fopen(template_file,"r"); assert(in);
-    if (output_file) {
+    if (opt_write) {
         out = fopen(output_file,"w");  assert(out);
     }
 
@@ -58,13 +59,13 @@ static int encode_file(char *template_file, char *output_file)
         GRIB_CHECK(grib_set_double_array(h,"values",values,values_len),0);
 
         GRIB_CHECK(grib_get_message(h,&buffer,&size),0);
-        if (output_file) {
+        if (opt_write) {
             if(fwrite(buffer,1,size,out) != size) {
                 perror(output_file);
                 return 1;
             }
         }
-        {
+        if (opt_dump) {
             FILE *devnull = fopen("/dev/null", "w");
             grib_dump_content(source_handle,devnull, "debug", 0, NULL);
         }
@@ -74,7 +75,7 @@ static int encode_file(char *template_file, char *output_file)
         free(values);
     }
 
-    if (output_file) fclose(out);
+    if (opt_write) fclose(out);
     fclose(in);
 
     return 0;
@@ -102,8 +103,9 @@ int main(int argc, char **argv)
         return 1;
     }
     
-    while ((c = getopt (argc, argv, "cw")) != -1) {
+    while ((c = getopt (argc, argv, "dcw")) != -1) {
         switch (c) {
+            case 'd': opt_dump=1; break;
             case 'c': opt_clone=1; break;
             case 'w': opt_write=1; break;
         }
@@ -119,7 +121,7 @@ int main(int argc, char **argv)
     }
     if (parallel) {
         printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
-        printf("Options: clone=%d, write=%d\n", opt_clone, opt_write);
+        printf("Options: dump=%d, clone=%d, write=%d\n", opt_dump, opt_clone, opt_write);
     } else {
         printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
     }
diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index 4a22eab98..f85d35042 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -45,10 +45,10 @@ process()
     time ${test_dir}/grib_ecc-604 -w par $input $NUM_THREADS $NUM_ITER
     validate
 
-    # Test 03: Clone + no output
-    # ---------------------------
+    # Test 03: Clone + dump + no output
+    # ---------------------------------
     rm -fr $OUTPUT
-    time ${test_dir}/grib_ecc-604 -c par $input $NUM_THREADS $NUM_ITER
+    time ${test_dir}/grib_ecc-604 -c -d par $input $NUM_THREADS $NUM_ITER
     # Nothing to validate as there is no output
 }
 ###################################################

From b9963a1e0c4fba4193f4e24b28651a150659eff3 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Thu, 4 Jul 2019 13:11:33 +0100
Subject: [PATCH 42/66] Test: print total count

---
 tests/grib_ecc-604.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/grib_ecc-604.c b/tests/grib_ecc-604.c
index be7be4643..055d1e8ac 100644
--- a/tests/grib_ecc-604.c
+++ b/tests/grib_ecc-604.c
@@ -120,7 +120,7 @@ int main(int argc, char **argv)
         parallel = 0;
     }
     if (parallel) {
-        printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+        printf("Running parallel in %ld threads. %ld iterations (prod=%ld)\n", NUM_THREADS, FILES_PER_ITERATION, NUM_THREADS*FILES_PER_ITERATION);
         printf("Options: dump=%d, clone=%d, write=%d\n", opt_dump, opt_clone, opt_write);
     } else {
         printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);

From d5c1ca51b3b207a89ee9af3a293329910f8a195b Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Thu, 4 Jul 2019 16:58:13 +0100
Subject: [PATCH 43/66] Test: Added new program for pure encoding

---
 tests/CMakeLists.txt        |   2 +-
 tests/grib_ecc-604-encode.c | 145 ++++++++++++++++++++++++++++++++++++
 tests/grib_ecc-604.c        |   2 +-
 3 files changed, 147 insertions(+), 2 deletions(-)
 create mode 100644 tests/grib_ecc-604-encode.c

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 6e9a7039d..db3afea3d 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -246,7 +246,7 @@ if( ENABLE_EXTRA_TESTS AND HAVE_ECCODES_THREADS )
                       COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_encode_pthreads.sh
     )
 
-    foreach( test grib_ecc-604 bufr_ecc-604 )
+    foreach( test grib_ecc-604 grib_ecc-604-encode bufr_ecc-604 )
         ecbuild_add_executable( TARGET    ${test}
                                 NOINSTALL
                                 SOURCES   ${test}.c
diff --git a/tests/grib_ecc-604-encode.c b/tests/grib_ecc-604-encode.c
new file mode 100644
index 000000000..82e245457
--- /dev/null
+++ b/tests/grib_ecc-604-encode.c
@@ -0,0 +1,145 @@
+/*
+ * Test for ECC-604: GRIB decoding/encoding sequentially and parallel with POSIX threads
+ */
+#include <time.h>
+#include <pthread.h>
+#include <assert.h>
+#include <unistd.h>
+
+#include "grib_api.h"
+
+/* These are passed in via argv */
+static size_t NUM_THREADS         = 0;
+static size_t FILES_PER_ITERATION = 0;
+static char*  INPUT_FILE          = NULL;
+int opt_dump  = 0; /* If 1 then dump handle to /dev/null */
+int opt_clone = 0; /* If 1 then clone source handle */
+int opt_write = 0; /* If 1 write handle to file */
+
+static int encode_values(grib_handle* h, char *output_file)
+{
+    double *values;
+    size_t size = 1000 * 1000;
+    size_t i = 0;
+    values = (double*)malloc(size*sizeof(double));
+    for (i=0; i<size; ++i) {
+        double v = i;
+        if (i % 1000 == 0) v = 0;
+        values[i] = v;
+    }
+    GRIB_CHECK(grib_set_long(h,"bitsPerValue",16),0);
+    GRIB_CHECK(grib_set_double_array(h,"values",values,size), 0);
+    free(values);
+    return GRIB_SUCCESS;
+}
+
+void do_encode(void *data);
+
+/* Structure for passing data to threads */
+struct v {
+    size_t number;
+    char *data;
+};
+
+void *runner(void *ptr); /* the thread */
+
+int main(int argc, char **argv)
+{
+    size_t i;
+    int thread_counter = 0;
+    int parallel=1, index=0, c=0;
+    const char* prog = argv[0];
+    char* mode;
+    if (argc<5 || argc>7) {
+        fprintf(stderr, "Usage:\n\t%s [options] seq sample numRuns numIter\nOr\n\t%s [options] par sample numThreads numIter\n", prog, prog);
+        return 1;
+    }
+    
+    while ((c = getopt (argc, argv, "dcw")) != -1) {
+        switch (c) {
+            case 'd': opt_dump=1; break;
+            case 'c': opt_clone=1; break;
+            case 'w': opt_write=1; break;
+        }
+    }
+    index = optind;
+    mode = argv[index];
+    INPUT_FILE = argv[index+1]; /* Has to be the name of a sample file (without tmpl extension) */
+    NUM_THREADS = atol(argv[index+2]);
+    FILES_PER_ITERATION = atol(argv[index+3]);
+
+    if (strcmp(mode,"seq")==0) {
+        parallel = 0;
+    }
+    if (parallel) {
+        printf("Running parallel in %ld threads. %ld iterations (prod=%ld)\n", NUM_THREADS, FILES_PER_ITERATION, NUM_THREADS*FILES_PER_ITERATION);
+        printf("Options: dump=%d, clone=%d, write=%d\n", opt_dump, opt_clone, opt_write);
+    } else {
+        printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+    }
+
+    {
+        pthread_t workers[NUM_THREADS];
+        for (i = 0; i < NUM_THREADS; i++) {
+            struct v *data = (struct v *) malloc(sizeof(struct v));
+            data->number = i;
+            data->data = NULL;
+
+            if (parallel) {
+                /* Now we will create the thread passing it data as an argument */
+                pthread_create(&workers[thread_counter], NULL, runner, data);
+                /*pthread_join(workers[thread_counter], NULL);*/
+                thread_counter++;
+            } else {
+                do_encode(data);
+            }
+        }
+
+        if (parallel) {
+            for (i = 0; i < NUM_THREADS; i++)  {
+                pthread_join(workers[i], NULL);
+            }
+        }
+    }
+
+    return 0;
+}
+
+void *runner(void *ptr)
+{
+    do_encode(ptr);
+    pthread_exit(0);
+}
+
+void do_encode(void *ptr)
+{
+    /* Cast argument to struct v pointer */
+    struct v *data = ptr;
+    size_t i;
+    char output_file[50];
+    time_t ltime;
+    struct tm result;
+    char stime[32];
+    grib_handle *hs = NULL;
+
+    hs = grib_handle_new_from_samples(0, INPUT_FILE);
+
+    for (i=0; i<FILES_PER_ITERATION;i++) {
+        grib_handle *h = grib_handle_clone(hs);
+        if (opt_write) {
+            sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
+            encode_values(h,output_file);
+        } else {
+            encode_values(h,NULL);
+        }
+        grib_handle_delete(h);
+    }
+
+    ltime = time(NULL);
+    localtime_r(&ltime, &result);
+    strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
+    /* asctime_r(&result, stime); */
+
+    printf("%s: Worker %ld finished.\n", stime, data->number);
+    grib_handle_delete(hs);
+}
diff --git a/tests/grib_ecc-604.c b/tests/grib_ecc-604.c
index 055d1e8ac..2f2d268d7 100644
--- a/tests/grib_ecc-604.c
+++ b/tests/grib_ecc-604.c
@@ -1,5 +1,5 @@
 /*
- * Test for ECC-604: Each thread creates a new GRIB handle, clones it and writes it out
+ * Test for ECC-604: GRIB decoding/encoding sequentially and parallel with POSIX threads
  */
 #include <time.h>
 #include <pthread.h>

From f171627e10b0df2708c87934bc8931d6a8bc6086 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Fri, 5 Jul 2019 11:09:01 +0100
Subject: [PATCH 44/66] Test: Add variable to tune number of values

---
 tests/grib_ecc-604-encode.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/grib_ecc-604-encode.c b/tests/grib_ecc-604-encode.c
index 82e245457..bd9046469 100644
--- a/tests/grib_ecc-604-encode.c
+++ b/tests/grib_ecc-604-encode.c
@@ -19,12 +19,13 @@ int opt_write = 0; /* If 1 write handle to file */
 static int encode_values(grib_handle* h, char *output_file)
 {
     double *values;
-    size_t size = 1000 * 1000;
+    const size_t DIM = 1000;
+    size_t size = DIM * DIM;
     size_t i = 0;
     values = (double*)malloc(size*sizeof(double));
     for (i=0; i<size; ++i) {
         double v = i;
-        if (i % 1000 == 0) v = 0;
+        if (i % DIM == 0) v = 0;
         values[i] = v;
     }
     GRIB_CHECK(grib_set_long(h,"bitsPerValue",16),0);
@@ -88,7 +89,6 @@ int main(int argc, char **argv)
             if (parallel) {
                 /* Now we will create the thread passing it data as an argument */
                 pthread_create(&workers[thread_counter], NULL, runner, data);
-                /*pthread_join(workers[thread_counter], NULL);*/
                 thread_counter++;
             } else {
                 do_encode(data);

From 628c60ed1ed62c39f95dc047f15aec44a96b48b9 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Thu, 11 Jul 2019 16:39:07 +0100
Subject: [PATCH 45/66] ECC-604: unnecessary locks when arg is NULL

---
 src/action.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/action.c b/src/action.c
index 6337a03f1..2ab3a523a 100644
--- a/src/action.c
+++ b/src/action.c
@@ -47,11 +47,15 @@ static void init_mutex()
 
 static void init(grib_action_class *c)
 {
+    if (!c) return;
+
     GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
     GRIB_MUTEX_LOCK(&mutex1);
     if(c && !c->inited)
     {
-        init(c->super ? *(c->super) : NULL);
+        if (c->super) {
+            init( *(c->super) );
+        }
         c->init_class(c);
         c->inited = 1;
     }

From 6dd035fdb0fcf14bc7693a321818632d306048b2 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Thu, 11 Jul 2019 17:53:16 +0100
Subject: [PATCH 46/66] ECC-604: non-recursive test version

---
 src/action.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/src/action.c b/src/action.c
index 2ab3a523a..857e67fe5 100644
--- a/src/action.c
+++ b/src/action.c
@@ -62,6 +62,31 @@ static void init(grib_action_class *c)
     GRIB_MUTEX_UNLOCK(&mutex1);
 }
 
+#if 0
+/* A non-recursive version */
+static void init(grib_action_class *c)
+{
+    if (!c) return;
+
+    GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
+    GRIB_MUTEX_LOCK(&mutex1);
+    if(!c->inited)
+    {
+        if(c->super) {
+            grib_action_class *g = *(c->super);
+            if (g && !g->inited) {
+                Assert(g->super == NULL);
+                g->init_class(g);
+                g->inited = 1;
+            }
+        }
+        c->init_class(c);
+        c->inited = 1;
+    }
+    GRIB_MUTEX_UNLOCK(&mutex1);
+}
+#endif
+
 void grib_dump(grib_action* a, FILE* f, int l)
 {
     grib_action_class *c = a->cclass;

From 099ea3d1cf34af71cf131469c802d9009399815d Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Thu, 18 Jul 2019 12:34:18 +0100
Subject: [PATCH 47/66] Performance: remove unnecessary check for NULL

---
 src/action.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/action.c b/src/action.c
index 857e67fe5..e5a81b167 100644
--- a/src/action.c
+++ b/src/action.c
@@ -51,7 +51,7 @@ static void init(grib_action_class *c)
 
     GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
     GRIB_MUTEX_LOCK(&mutex1);
-    if(c && !c->inited)
+    if(!c->inited)
     {
         if (c->super) {
             init( *(c->super) );

From 3bb602977d1766f705c277200136267f897d28ef Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Thu, 18 Jul 2019 12:34:51 +0100
Subject: [PATCH 48/66] Fix usage error

---
 tests/grib_ecc-604.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/grib_ecc-604.c b/tests/grib_ecc-604.c
index 2f2d268d7..4ae31c52a 100644
--- a/tests/grib_ecc-604.c
+++ b/tests/grib_ecc-604.c
@@ -98,7 +98,7 @@ int main(int argc, char **argv)
     int parallel=1, index=0, c=0;
     const char* prog = argv[0];
     char* mode;
-    if (argc<5 || argc>7) {
+    if (argc<5 || argc>8) {
         fprintf(stderr, "Usage:\n\t%s [options] seq file numRuns numIter\nOr\n\t%s [options] par file numThreads numIter\n", prog, prog);
         return 1;
     }

From 29d6b84d51896cbc319bbb646fab29f94fa858ef Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 23 Jul 2019 17:11:31 +0100
Subject: [PATCH 49/66] ECC-604: lock only used in DEBUG mode for self->loop
 test

---
 src/action_class_when.c | 65 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 58 insertions(+), 7 deletions(-)

diff --git a/src/action_class_when.c b/src/action_class_when.c
index 420620cbe..463e45c02 100644
--- a/src/action_class_when.c
+++ b/src/action_class_when.c
@@ -82,6 +82,37 @@ static void init_class(grib_action_class* c)
 }
 /* END_CLASS_IMP */
 
+#ifdef DEBUG
+  #if GRIB_PTHREADS
+    static pthread_once_t once  = PTHREAD_ONCE_INIT;
+    static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+
+    static void init() {
+        pthread_mutexattr_t attr;
+        pthread_mutexattr_init(&attr);
+        pthread_mutexattr_settype(&attr,PTHREAD_MUTEX_RECURSIVE);
+        pthread_mutex_init(&mutex,&attr);
+        pthread_mutexattr_destroy(&attr);
+    }
+  #elif GRIB_OMP_THREADS
+    static int once = 0;
+    static omp_nest_lock_t mutex;
+
+    static void init()
+    {
+        GRIB_OMP_CRITICAL(lock_action_class_when_c)
+        {
+            if (once == 0)
+            {
+                omp_init_nest_lock(&mutex);
+                once = 1;
+            }
+        }
+    }
+  #endif /*GRIB_PTHREADS*/
+#endif /*DEBUG*/
+
+
 grib_action* grib_action_create_when( grib_context* context,
         grib_expression* expression,
         grib_action* block_true,grib_action* block_false)
@@ -152,18 +183,39 @@ static void dump(grib_action* act, FILE* f, int lvl)
     printf("\n");
 }
 
+#ifdef DEBUG
+  #define SET_LOOP(s,v) set_self_loop(s,v);
+#else
+  #define SET_LOOP(s,v)
+#endif
+
+#ifdef DEBUG
+static int get_self_loop(grib_action_when* self) {
+    GRIB_MUTEX_INIT_ONCE(&once,&init);
+    GRIB_MUTEX_LOCK(&mutex);
+    return self->loop;
+    GRIB_MUTEX_UNLOCK(&mutex);
+}
+static void set_self_loop(grib_action_when* self, int loop_value) {
+    GRIB_MUTEX_INIT_ONCE(&once,&init);
+    GRIB_MUTEX_LOCK(&mutex);
+    self->loop = loop_value;
+    GRIB_MUTEX_UNLOCK(&mutex);
+}
+#endif
+
 static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor* observed)
 {
     grib_action_when* self = (grib_action_when*) a;
     grib_action *b = NULL;
     int ret = GRIB_SUCCESS;
     long lres;
+    grib_handle* hand = grib_handle_of_accessor(observed);
 
     if ((ret = grib_expression_evaluate_long(grib_handle_of_accessor(observed), self->expression,&lres))
             != GRIB_SUCCESS) return ret;
 #ifdef DEBUG
-    if(self->loop)
-    {
+    if (get_self_loop(self)==1) {
         printf("LOOP detected...\n");
         printf("WHEN triggered by %s %ld\n",observed->name,lres);
         grib_expression_print(observed->context,self->expression,0);
@@ -171,7 +223,7 @@ static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor*
         return ret;
     }
 #endif
-    self->loop = 1;
+    SET_LOOP(self, 1);
 
     if(lres)
         b=self->block_true;
@@ -179,15 +231,14 @@ static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor*
         b=self->block_false;
 
     while(b) {
-        ret = grib_action_execute(b,grib_handle_of_accessor(observed));
+        ret = grib_action_execute(b,hand);
         if(ret != GRIB_SUCCESS) {
-            self->loop = 0;
+            SET_LOOP(self, 0);
             return ret;
         }
         b = b->next;
     }
-
-    self->loop = 0;
+    SET_LOOP(self, 0);
 
     return GRIB_SUCCESS;
 }

From bc67a0cadd8e0f34546914701b6149beb2c83aab Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 23 Jul 2019 18:23:46 +0100
Subject: [PATCH 50/66] Fix comments

---
 examples/C/grib_pthreads.c | 2 +-
 tests/bufr_ecc-604.c       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/C/grib_pthreads.c b/examples/C/grib_pthreads.c
index 3b497f67c..f08f8a12c 100644
--- a/examples/C/grib_pthreads.c
+++ b/examples/C/grib_pthreads.c
@@ -38,7 +38,7 @@ static void* process_grib(void* threadID)
     CODES_CHECK(codes_set_string(h,"indicatorOfUnitOfTimeRange", "s", &str_len),0);
     CODES_CHECK(codes_set_string(h,"stepUnits", "s", &str_len),0);
     CODES_CHECK(codes_set_long(h, "endStep", 86400), 0);
-    //CODES_CHECK(codes_set_long(h,"centre", 80),0);
+    /* CODES_CHECK(codes_set_long(h,"centre", 80),0); */
 
     CODES_CHECK(codes_get_long(h,"endStep", &step),0);
     CODES_CHECK(codes_get_string(h, "indicatorOfUnitOfTimeRange", mystring, &str_len),0);
diff --git a/tests/bufr_ecc-604.c b/tests/bufr_ecc-604.c
index 3f0c5b577..cc2572ef2 100644
--- a/tests/bufr_ecc-604.c
+++ b/tests/bufr_ecc-604.c
@@ -54,7 +54,7 @@ static int encode_file(char *template_file, char *output_file)
             grib_dumper* dumper = NULL;
             const char* dumper_name = "bufr_simple";
             unsigned long dump_flags = CODES_DUMP_FLAG_ALL_DATA;
-            //codes_dump_content(source_handle,devnull, "json", 1024, NULL); /* JSON dump with all attributes */
+            /* codes_dump_content(source_handle,devnull, "json", 1024, NULL); */ /* JSON dump with all attributes */
             dumper=grib_dump_content_with_dumper(source_handle, dumper, devnull, dumper_name, dump_flags, NULL);
             assert(dumper);
             fclose(devnull);

From 4db723657e3e76e18dc1039a3a7ad7334e4b39f6 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 23 Jul 2019 18:24:37 +0100
Subject: [PATCH 51/66] ECC-604: remove lock on grib_action_notify_change

---
 src/action.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/action.c b/src/action.c
index e5a81b167..545161094 100644
--- a/src/action.c
+++ b/src/action.c
@@ -162,20 +162,20 @@ int grib_action_notify_change( grib_action* a, grib_accessor *observer, grib_acc
 {
     grib_action_class *c = a->cclass;
 
-    GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
-    GRIB_MUTEX_LOCK(&mutex1);
+    /*GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);*/
+    /*GRIB_MUTEX_LOCK(&mutex1);*/
 
     init(c);
     while(c)
     {
         if(c->notify_change) {
             int result = c->notify_change(a,observer,observed);
-            GRIB_MUTEX_UNLOCK(&mutex1);
+            /*GRIB_MUTEX_UNLOCK(&mutex1);*/
             return result;
         }
         c = c->super ? *(c->super) : NULL;
     }
-    GRIB_MUTEX_UNLOCK(&mutex1);
+    /*GRIB_MUTEX_UNLOCK(&mutex1);*/
     Assert(0);
     return 0;
 }

From 6b1212cee473bc8eb3199c5fd3911117d93af6ca Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 23 Jul 2019 18:25:40 +0100
Subject: [PATCH 52/66] ECC-604: Enable checks when DEBUG_LOOP is defined. Also
 show thread ID

---
 src/action_class_when.c | 42 ++++++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/src/action_class_when.c b/src/action_class_when.c
index 463e45c02..d1ad7b859 100644
--- a/src/action_class_when.c
+++ b/src/action_class_when.c
@@ -82,7 +82,8 @@ static void init_class(grib_action_class* c)
 }
 /* END_CLASS_IMP */
 
-#ifdef DEBUG
+#ifdef DEBUG_LOOP
+  #include <syscall.h>
   #if GRIB_PTHREADS
     static pthread_once_t once  = PTHREAD_ONCE_INIT;
     static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
@@ -110,7 +111,7 @@ static void init_class(grib_action_class* c)
         }
     }
   #endif /*GRIB_PTHREADS*/
-#endif /*DEBUG*/
+#endif /*DEBUG_LOOP*/
 
 
 grib_action* grib_action_create_when( grib_context* context,
@@ -183,26 +184,29 @@ static void dump(grib_action* act, FILE* f, int lvl)
     printf("\n");
 }
 
-#ifdef DEBUG
+#ifdef DEBUG_LOOP
   #define SET_LOOP(s,v) set_self_loop(s,v);
+  static int get_self_loop(grib_action_when* self) {
+    int result = 0;
+    GRIB_MUTEX_INIT_ONCE(&once,&init);
+    GRIB_MUTEX_LOCK(&mutex);
+    /*printf("    get_self_loop::  tid = %ld\n", syscall(SYS_gettid));*/
+    result = self->loop;
+    GRIB_MUTEX_UNLOCK(&mutex);
+    return result;
+  }
+  static void set_self_loop(grib_action_when* self, int loop_value) {
+    pthread_t tid = pthread_self();
+    GRIB_MUTEX_INIT_ONCE(&once,&init);
+    GRIB_MUTEX_LOCK(&mutex);
+    /*if(loop_value==1)printf("set_self_loop::  tid = %ld\n", syscall(SYS_gettid));*/
+    self->loop = loop_value;
+    GRIB_MUTEX_UNLOCK(&mutex);
+  }
 #else
   #define SET_LOOP(s,v)
 #endif
 
-#ifdef DEBUG
-static int get_self_loop(grib_action_when* self) {
-    GRIB_MUTEX_INIT_ONCE(&once,&init);
-    GRIB_MUTEX_LOCK(&mutex);
-    return self->loop;
-    GRIB_MUTEX_UNLOCK(&mutex);
-}
-static void set_self_loop(grib_action_when* self, int loop_value) {
-    GRIB_MUTEX_INIT_ONCE(&once,&init);
-    GRIB_MUTEX_LOCK(&mutex);
-    self->loop = loop_value;
-    GRIB_MUTEX_UNLOCK(&mutex);
-}
-#endif
 
 static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor* observed)
 {
@@ -214,9 +218,9 @@ static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor*
 
     if ((ret = grib_expression_evaluate_long(grib_handle_of_accessor(observed), self->expression,&lres))
             != GRIB_SUCCESS) return ret;
-#ifdef DEBUG
+#ifdef DEBUG_LOOP
     if (get_self_loop(self)==1) {
-        printf("LOOP detected...\n");
+        printf("LOOP detected... tid=%ld\n", syscall(SYS_gettid));
         printf("WHEN triggered by %s %ld\n",observed->name,lres);
         grib_expression_print(observed->context,self->expression,0);
         printf("\n");

From ba23150b0757ca30b4b9341a741aed69ffb45c13 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 24 Jul 2019 12:51:08 +0100
Subject: [PATCH 53/66] ECC-604: Disable check in threaded mode

---
 src/action_class_when.c | 68 ++++++++---------------------------------
 1 file changed, 12 insertions(+), 56 deletions(-)

diff --git a/src/action_class_when.c b/src/action_class_when.c
index d1ad7b859..c8b087303 100644
--- a/src/action_class_when.c
+++ b/src/action_class_when.c
@@ -82,37 +82,10 @@ static void init_class(grib_action_class* c)
 }
 /* END_CLASS_IMP */
 
-#ifdef DEBUG_LOOP
-  #include <syscall.h>
-  #if GRIB_PTHREADS
-    static pthread_once_t once  = PTHREAD_ONCE_INIT;
-    static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
-
-    static void init() {
-        pthread_mutexattr_t attr;
-        pthread_mutexattr_init(&attr);
-        pthread_mutexattr_settype(&attr,PTHREAD_MUTEX_RECURSIVE);
-        pthread_mutex_init(&mutex,&attr);
-        pthread_mutexattr_destroy(&attr);
-    }
-  #elif GRIB_OMP_THREADS
-    static int once = 0;
-    static omp_nest_lock_t mutex;
-
-    static void init()
-    {
-        GRIB_OMP_CRITICAL(lock_action_class_when_c)
-        {
-            if (once == 0)
-            {
-                omp_init_nest_lock(&mutex);
-                once = 1;
-            }
-        }
-    }
-  #endif /*GRIB_PTHREADS*/
-#endif /*DEBUG_LOOP*/
-
+/* The check on self->loop can only be done in non-threaded mode */
+#if defined(DEBUG) && GRIB_PTHREADS == 0 && GRIB_OMP_THREADS == 0
+  #define CHECK_LOOP 1
+#endif
 
 grib_action* grib_action_create_when( grib_context* context,
         grib_expression* expression,
@@ -133,7 +106,6 @@ grib_action* grib_action_create_when( grib_context* context,
     a->block_true       = block_true;
     a->block_false      = block_false;
 
-
     sprintf(name,"_when%p",(void*)expression);
 
     act->name      = grib_context_strdup_persistent(context,name);
@@ -184,30 +156,12 @@ static void dump(grib_action* act, FILE* f, int lvl)
     printf("\n");
 }
 
-#ifdef DEBUG_LOOP
-  #define SET_LOOP(s,v) set_self_loop(s,v);
-  static int get_self_loop(grib_action_when* self) {
-    int result = 0;
-    GRIB_MUTEX_INIT_ONCE(&once,&init);
-    GRIB_MUTEX_LOCK(&mutex);
-    /*printf("    get_self_loop::  tid = %ld\n", syscall(SYS_gettid));*/
-    result = self->loop;
-    GRIB_MUTEX_UNLOCK(&mutex);
-    return result;
-  }
-  static void set_self_loop(grib_action_when* self, int loop_value) {
-    pthread_t tid = pthread_self();
-    GRIB_MUTEX_INIT_ONCE(&once,&init);
-    GRIB_MUTEX_LOCK(&mutex);
-    /*if(loop_value==1)printf("set_self_loop::  tid = %ld\n", syscall(SYS_gettid));*/
-    self->loop = loop_value;
-    GRIB_MUTEX_UNLOCK(&mutex);
-  }
+#ifdef CHECK_LOOP
+  #define SET_LOOP(self,v)  self->loop=v;
 #else
-  #define SET_LOOP(s,v)
+  #define SET_LOOP(self,v)
 #endif
 
-
 static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor* observed)
 {
     grib_action_when* self = (grib_action_when*) a;
@@ -218,9 +172,10 @@ static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor*
 
     if ((ret = grib_expression_evaluate_long(grib_handle_of_accessor(observed), self->expression,&lres))
             != GRIB_SUCCESS) return ret;
-#ifdef DEBUG_LOOP
-    if (get_self_loop(self)==1) {
-        printf("LOOP detected... tid=%ld\n", syscall(SYS_gettid));
+#ifdef CHECK_LOOP
+    if(self->loop)
+    {
+        printf("LOOP detected...\n");
         printf("WHEN triggered by %s %ld\n",observed->name,lres);
         grib_expression_print(observed->context,self->expression,0);
         printf("\n");
@@ -242,6 +197,7 @@ static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor*
         }
         b = b->next;
     }
+
     SET_LOOP(self, 0);
 
     return GRIB_SUCCESS;

From d837d6b4aa568afbb1b931e8da52e0f7bfee7951 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 30 Jul 2019 12:27:30 +0100
Subject: [PATCH 54/66] Thread-safety: disable statics unless DEBUG mode

---
 src/grib_loader_from_handle.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/grib_loader_from_handle.c b/src/grib_loader_from_handle.c
index 83fa154b7..ae27d5793 100644
--- a/src/grib_loader_from_handle.c
+++ b/src/grib_loader_from_handle.c
@@ -73,8 +73,10 @@ int grib_init_accessor_from_handle(grib_loader* loader,grib_accessor* ga,grib_ar
     unsigned char*   uval = NULL;
     long*   lval = NULL;
     double* dval = NULL;
+#ifdef DEBUG
     static int first  = 1;
     static const char* missing = 0;
+#endif
     const char* name = NULL;
     int k = 0;
     grib_handle *g;
@@ -137,16 +139,16 @@ int grib_init_accessor_from_handle(grib_loader* loader,grib_accessor* ga,grib_ar
     if(ret != GRIB_SUCCESS)
     {
         name = ga->name;
-
+#ifdef DEBUG
         if(first)
         {
             missing = codes_getenv("ECCODES_PRINT_MISSING");
             first = 0;
         }
-
+#endif
         grib_context_log(h->context,GRIB_LOG_DEBUG, "Copying [%s] failed: %s",
                 name,grib_get_error_message(ret));
-
+#ifdef DEBUG
         if(missing)
         {
             fprintf(stdout,"REPARSE: no value for %s",name);
@@ -154,7 +156,7 @@ int grib_init_accessor_from_handle(grib_loader* loader,grib_accessor* ga,grib_ar
                 fprintf(stdout," (default value)");
             fprintf(stdout,"\n");
         }
-
+#endif
         return GRIB_SUCCESS;
     }
 

From f4ba853635dd2b1bf589c1da56090dfbf6ad7ec8 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Mon, 12 Aug 2019 13:19:21 +0100
Subject: [PATCH 55/66] Testing: check encoded values

---
 examples/C/grib_pthreads.c | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/examples/C/grib_pthreads.c b/examples/C/grib_pthreads.c
index f08f8a12c..e6d5aec27 100644
--- a/examples/C/grib_pthreads.c
+++ b/examples/C/grib_pthreads.c
@@ -15,15 +15,27 @@
 #include "eccodes.h"
 #define NUM_THREADS 4
 
+/* Return 0 if numbers considered equal, otherwise 1 */
+static int compare_doubles(double a,double b,double tolerance)
+{
+    int ret=0;
+    double d=fabs(a-b);
+    if (d > tolerance) {
+        ret=1;
+    }
+    return ret;
+}
+
 static void* process_grib(void* threadID)
 {
     const long tid = (long)threadID;
-    size_t str_len = 20;
+    size_t str_len = 20, i = 0;
     long indicatorOfUnitOfTimeRange = 1, step = 0;
     char mystring[100];
     double* values = NULL;
     size_t values_len = 0;
-    double min=0,max=0;
+    double min=0,max=0,avg=0;
+    const double tol = 1e-6;
     double pv[4]={1,2,3,4};
     const size_t pvsize=4;
     ProductKind prod_kind = 0;
@@ -38,7 +50,7 @@ static void* process_grib(void* threadID)
     CODES_CHECK(codes_set_string(h,"indicatorOfUnitOfTimeRange", "s", &str_len),0);
     CODES_CHECK(codes_set_string(h,"stepUnits", "s", &str_len),0);
     CODES_CHECK(codes_set_long(h, "endStep", 86400), 0);
-    /* CODES_CHECK(codes_set_long(h,"centre", 80),0); */
+    CODES_CHECK(codes_set_long(h,"centre", 80),0);
 
     CODES_CHECK(codes_get_long(h,"endStep", &step),0);
     CODES_CHECK(codes_get_string(h, "indicatorOfUnitOfTimeRange", mystring, &str_len),0);
@@ -49,10 +61,20 @@ static void* process_grib(void* threadID)
     CODES_CHECK(codes_get_size(h,"values",&values_len),0);
     values = (double*)malloc(values_len*sizeof(double));
     CODES_CHECK(codes_get_double_array(h, "values", values, &values_len),0);
+    for (i=0;i<values_len;i++) {
+        if (i%2) values[i] *= 0.94;
+        else if (i%3) values[i] *= 0.84;
+    }
+    GRIB_CHECK(grib_set_double_array(h,"values",values,values_len),0);
     free(values);
 
     CODES_CHECK(codes_get_double(h, "min", &min),0);
     CODES_CHECK(codes_get_double(h, "max", &max),0);
+    CODES_CHECK(codes_get_double(h, "avg", &avg),0);
+    printf("Thread %ld: min=%g max=%g avg=%g\n", tid, min, max, avg);
+    assert( compare_doubles(min, 0.84, tol)==0 );
+    assert( compare_doubles(max, 1.00, tol)==0 );
+    assert( compare_doubles(avg, 0.916774, tol)==0 );
 
     codes_handle_delete(h);
     pthread_exit(NULL);

From f48f3a2ffc1229bbedf159e2217fbb4301d8c11f Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Thu, 12 Sep 2019 12:54:11 +0100
Subject: [PATCH 56/66] ECC-990: Crash: grib_copy -r on packingType of
 grid_second_order_general_grib1

---
 ...lass_data_g1second_order_general_packing.c | 38 ++++++++++---------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/src/grib_accessor_class_data_g1second_order_general_packing.c b/src/grib_accessor_class_data_g1second_order_general_packing.c
index 747bc329c..58a052a56 100644
--- a/src/grib_accessor_class_data_g1second_order_general_packing.c
+++ b/src/grib_accessor_class_data_g1second_order_general_packing.c
@@ -183,23 +183,24 @@ static void init_class(grib_accessor_class* c)
 static void init(grib_accessor* a,const long v, grib_arguments* args)
 {
     grib_accessor_data_g1second_order_general_packing *self =(grib_accessor_data_g1second_order_general_packing*)a;
+    grib_handle* hand = grib_handle_of_accessor(a);
 
-    self->half_byte    = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->packingType    = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->ieee_packing    = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->precision    = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->widthOfFirstOrderValues    = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->N1 = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->N2 = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->numberOfGroups = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->numberOfSecondOrderPackedValues = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->extraValues = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->Ni = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->Nj = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->pl = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->jPointsAreConsecutive = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->bitmap = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
-    self->groupWidths = grib_arguments_get_name(grib_handle_of_accessor(a),args,self->carg++);
+    self->half_byte    = grib_arguments_get_name(hand,args,self->carg++);
+    self->packingType    = grib_arguments_get_name(hand,args,self->carg++);
+    self->ieee_packing    = grib_arguments_get_name(hand,args,self->carg++);
+    self->precision    = grib_arguments_get_name(hand,args,self->carg++);
+    self->widthOfFirstOrderValues    = grib_arguments_get_name(hand,args,self->carg++);
+    self->N1 = grib_arguments_get_name(hand,args,self->carg++);
+    self->N2 = grib_arguments_get_name(hand,args,self->carg++);
+    self->numberOfGroups = grib_arguments_get_name(hand,args,self->carg++);
+    self->numberOfSecondOrderPackedValues = grib_arguments_get_name(hand,args,self->carg++);
+    self->extraValues = grib_arguments_get_name(hand,args,self->carg++);
+    self->Ni = grib_arguments_get_name(hand,args,self->carg++);
+    self->Nj = grib_arguments_get_name(hand,args,self->carg++);
+    self->pl = grib_arguments_get_name(hand,args,self->carg++);
+    self->jPointsAreConsecutive = grib_arguments_get_name(hand,args,self->carg++);
+    self->bitmap = grib_arguments_get_name(hand,args,self->carg++);
+    self->groupWidths = grib_arguments_get_name(hand,args,self->carg++);
     self->edition=1;
     a->flags |= GRIB_ACCESSOR_FLAG_DATA;
 }
@@ -317,9 +318,10 @@ static int pack_double(grib_accessor* a, const double* cval, size_t *len)
     int err = 0;
     char type[]="grid_second_order";
     size_t size=strlen(type);
+    grib_handle* hand = grib_handle_of_accessor(a);
 
-    err = grib_set_string(grib_handle_of_accessor(a),"packingType",type,&size);
+    err = grib_set_string(hand,"packingType",type,&size);
     if (err) return err;
 
-    return grib_set_double_array(grib_handle_of_accessor(a),"values",cval,*len);
+    return grib_set_double_array(hand,"values",cval,*len);
 }

From 9625da847bf31195dfa69f2b3689290b8526bea8 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Thu, 12 Sep 2019 15:37:39 +0100
Subject: [PATCH 57/66] Add extra tests (CCSDS and grib1 2nd order)

---
 tests/grib_ecc-604.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/grib_ecc-604.sh b/tests/grib_ecc-604.sh
index f85d35042..ad87d8695 100755
--- a/tests/grib_ecc-604.sh
+++ b/tests/grib_ecc-604.sh
@@ -61,6 +61,7 @@ GRIB1_INPUTS="
   ${data_dir}/gen_bitmap.grib
   ${data_dir}/spectral_complex.grib1
   ${data_dir}/gen_ext.grib
+  ${data_dir}/gen.grib
   ${data_dir}/gen_ext_spd_2.grib"
 
 GRIB2_INPUTS="
@@ -73,7 +74,9 @@ if [ $HAVE_JPEG -eq 1 ]; then
     echo "Adding extra files (HAVE_JPEG=1)"
     GRIB2_INPUTS="${data_dir}/jpeg.grib2 ${data_dir}/reduced_gaussian_surface_jpeg.grib2 "$GRIB2_INPUTS
 fi
-#GRIB2_INPUTS=$GRIB2_INPUTS" ${data_dir}/ccsds.grib2 "
+if [ $HAVE_AEC -eq 1 ]; then
+    GRIB2_INPUTS=$GRIB2_INPUTS" ${data_dir}/ccsds.grib2 "
+fi
 
 for gf in $GRIB1_INPUTS $GRIB2_INPUTS; do
     process $gf

From 791062b53b03e759e84e427329cdde0148d48e3d Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Mon, 16 Sep 2019 15:15:03 +0100
Subject: [PATCH 58/66] Direct error messages to stderr not stdout

---
 src/grib_util.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/grib_util.c b/src/grib_util.c
index 34a12cd74..63c7f3a03 100644
--- a/src/grib_util.c
+++ b/src/grib_util.c
@@ -931,7 +931,7 @@ grib_handle* grib_util_set_spec2(grib_handle* h,
                 setSecondOrder=1;
                 break;
             default :
-                printf("invalid packing_spec->packing_type = %ld\n",(long)packing_spec->packing_type);
+                fprintf(stderr, "invalid packing_spec->packing_type = %ld\n",(long)packing_spec->packing_type);
                 *err = GRIB_INTERNAL_ERROR;
                 goto cleanup;
                 break;
@@ -957,7 +957,7 @@ grib_handle* grib_util_set_spec2(grib_handle* h,
             break;
 
         default:
-            printf("invalid packing_spec->accuracy = %ld\n",(long)packing_spec->accuracy);
+            fprintf(stderr, "invalid packing_spec->accuracy = %ld\n",(long)packing_spec->accuracy);
             *err = GRIB_INTERNAL_ERROR;
             goto cleanup;
             break;
@@ -1333,7 +1333,7 @@ grib_handle* grib_util_set_spec2(grib_handle* h,
             setSecondOrder=1;
             break;
         default :
-            printf("invalid packing_spec->packing_type = %ld\n",(long)packing_spec->packing_type);
+            fprintf(stderr,"invalid packing_spec->packing_type = %ld\n",(long)packing_spec->packing_type);
             *err = GRIB_INTERNAL_ERROR;
             goto cleanup;
             break;
@@ -1390,7 +1390,7 @@ grib_handle* grib_util_set_spec2(grib_handle* h,
         break;
 
     default:
-        printf("invalid packing_spec->accuracy = %ld\n",(long)packing_spec->accuracy);
+        fprintf(stderr, "invalid packing_spec->accuracy = %ld\n",(long)packing_spec->accuracy);
         *err = GRIB_INTERNAL_ERROR;
         goto cleanup;
         break;
@@ -1453,7 +1453,7 @@ grib_handle* grib_util_set_spec2(grib_handle* h,
         if (global_grid) {
             size_t sum = sum_of_pl_array(spec->pl, spec->pl_size);
             if (data_values_count != sum) {
-                printf("invalid reduced gaussian grid: specified as global, data_values_count=%ld but sum of pl array=%ld\n",
+                fprintf(stderr, "invalid reduced gaussian grid: specified as global, data_values_count=%ld but sum of pl array=%ld\n",
                         (long)data_values_count, (long)sum);
                 *err = GRIB_WRONG_GRID;
                 goto cleanup;

From 8e9cecd702eeb672c3bf0feaa850e0a1eb2290d6 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 17 Sep 2019 10:52:19 +0100
Subject: [PATCH 59/66] ECC-673: bufr_filter: Arguments not parsed correctly on
 CLI

---
 tools/bufr_filter | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/tools/bufr_filter b/tools/bufr_filter
index 6e34b3168..e631b87a0 100755
--- a/tools/bufr_filter
+++ b/tools/bufr_filter
@@ -6,6 +6,7 @@ ECCODES_TOOL=codes_bufr_filter
 result=0 # return code from function
 ERR_TOOL_NOT_FOUND=666
 
+the_tool=$ECCODES_TOOL
 is_emoslib=0
 
 #########################################################
@@ -15,15 +16,12 @@ is_emoslib=0
 #  sets the global variable 'result'
 try_tool()
 {
-    the_tool=$1
-    the_args=$args
-
     if [ -f "${script_dir}/$the_tool" ]; then
-        ${script_dir}/$the_tool $the_args
+        ${script_dir}/$the_tool "${@}"
         result=$?
     else
         if command -v $the_tool >/dev/null 2>&1; then
-            $the_tool $the_args
+            $the_tool "${@}"
             result=$?
         else
             #echo "Could not find $the_tool. Return error"
@@ -36,32 +34,30 @@ try_tool()
 # Deal with case where no arguments are provided e.g. usage
 if [ $# -eq 0 ]; then
     # Give priority to ecCodes over emoslib
-    try_tool $ECCODES_TOOL
+    the_tool=$ECCODES_TOOL
+    try_tool "${@}"
     if [ $result -eq $ERR_TOOL_NOT_FOUND ]; then
-        try_tool $EMOSLIB_TOOL
+        the_tool=$EMOSLIB_TOOL
+        try_tool "${@}"
     fi
     exit 0
 fi
 
 # Now process arguments. The "-i" switch is specific to emoslib
-args="$@"
 for i in "$@" ; do
-   case $i in
-     -i) is_emoslib=1; shift ;;
-     *)  shift ;;
-   esac
+   if [ "$i" = "-i" ]; then is_emoslib=1; fi
 done
 
 #########################################################
 # set -x
 if [ $is_emoslib -eq 1 ]; then
     pkg=emoslib
-    tool=$EMOSLIB_TOOL
-    try_tool $tool
+    the_tool=$EMOSLIB_TOOL
+    try_tool "${@}"
 else
     pkg=ecCodes
-    tool=$ECCODES_TOOL
-    try_tool $tool
+    the_tool=$ECCODES_TOOL
+    try_tool "${@}"
 fi
 if [ $result -eq $ERR_TOOL_NOT_FOUND ]; then
     echo "ERROR: Could not find the executable: $tool. Aborting!"  2>&1

From 7a6ad1c886500b433d2509abb944570690cdc138 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 17 Sep 2019 15:09:06 +0100
Subject: [PATCH 60/66] ECC-673: bufr_filter: Arguments not parsed correctly on
 CLI

---
 tools/bufr_filter | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/tools/bufr_filter b/tools/bufr_filter
index e631b87a0..b390bfa44 100755
--- a/tools/bufr_filter
+++ b/tools/bufr_filter
@@ -11,9 +11,9 @@ is_emoslib=0
 
 #########################################################
 # Arguments:
-#  the executable name
+#  arguments from the script command line
 # Return Value:
-#  sets the global variable 'result'
+#  the global variable 'result' holds the exit code
 try_tool()
 {
     if [ -f "${script_dir}/$the_tool" ]; then
@@ -49,20 +49,19 @@ for i in "$@" ; do
 done
 
 #########################################################
-# set -x
 if [ $is_emoslib -eq 1 ]; then
-    pkg=emoslib
+    pkg='emoslib/bufrdc'
     the_tool=$EMOSLIB_TOOL
     try_tool "${@}"
 else
-    pkg=ecCodes
+    pkg='ecCodes'
     the_tool=$ECCODES_TOOL
     try_tool "${@}"
 fi
 if [ $result -eq $ERR_TOOL_NOT_FOUND ]; then
-    echo "ERROR: Could not find the executable: $tool. Aborting!"  2>&1
-    echo "       The arguments you passed in are relevant to $pkg" 2>&1
-    echo "       Please make sure you have $pkg installed in your path"  2>&1
+    echo "ERROR: Could not find the executable: $the_tool. Aborting!"  2>&1
+    echo "       The arguments you passed in are relevant to $pkg." 2>&1
+    echo "       Please make sure you have $pkg installed in your path."  2>&1
     exit 1
 fi
 exit $result

From 5373158709b374ea70591154d427a16226308cea Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 17 Sep 2019 15:24:46 +0100
Subject: [PATCH 61/66] ECC-673: bufr_filter: Arguments not parsed correctly on
 CLI

---
 tests/bufr_ecc-428.sh | 18 +++++++++---------
 tools/bufr_filter     |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/bufr_ecc-428.sh b/tests/bufr_ecc-428.sh
index 88bd4e2be..a1691ff3d 100755
--- a/tests/bufr_ecc-428.sh
+++ b/tests/bufr_ecc-428.sh
@@ -35,21 +35,21 @@ cat > $tempRules <<EOF
 EOF
 
 cat $tempRules
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText
 echo "784" > $tempRef1
 diff $tempRef1 $tempText
 
 export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=1
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText
 echo "784 784 784 784 784 784 784 784 784 784 784 784 784 784 784" > $tempRef2
 diff $tempRef2 $tempText
 
 unset ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText
 diff $tempRef1 $tempText
 
 export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=0
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText
 diff $tempRef1 $tempText
 
 
@@ -64,7 +64,7 @@ EOF
 cat $tempRules
 
 export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=1
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText 2>$tempErrs
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText 2>$tempErrs
 grep -q '^48 54 59.*91 97' $tempText
 
 
@@ -79,7 +79,7 @@ EOF
 cat $tempRules
 
 export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=1
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText 2>$tempErrs
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText 2>$tempErrs
 grep -q '^LBG LBG LBG LBG.*LBG$' $tempText
 
 
@@ -94,7 +94,7 @@ EOF
 cat $tempRules
 
 export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=1
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText 2>$tempErrs
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText 2>$tempErrs
 grep -q '^6 6 6.*24 24.*6 6' $tempText
 
 # --------------------------------------------------------
@@ -107,11 +107,11 @@ cat > $tempRules <<EOF
 EOF
 
 cat $tempRules
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile >/dev/null
+${tools_dir}/bufr_filter $tempRules $bufrFile >/dev/null
 
 echo "TODO: Searching for backscatter ... currently failing"
 export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=1
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile 2>$tempErrs
+${tools_dir}/bufr_filter $tempRules $bufrFile 2>$tempErrs
 cat $tempErrs
 
 # Clean up
diff --git a/tools/bufr_filter b/tools/bufr_filter
index b390bfa44..3057c33cc 100755
--- a/tools/bufr_filter
+++ b/tools/bufr_filter
@@ -1,5 +1,5 @@
 #!/bin/sh
-
+set -u
 script_dir=`dirname $0`
 EMOSLIB_TOOL=emoslib_bufr_filter
 ECCODES_TOOL=codes_bufr_filter

From 987d51f4e766d5042d301179ce3c43b7dc5b8d0a Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 17 Sep 2019 17:06:21 +0100
Subject: [PATCH 62/66] bufr_filter script: rename executable

---
 tools/bufr_filter | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/bufr_filter b/tools/bufr_filter
index 3057c33cc..645485334 100755
--- a/tools/bufr_filter
+++ b/tools/bufr_filter
@@ -1,7 +1,7 @@
 #!/bin/sh
 set -u
 script_dir=`dirname $0`
-EMOSLIB_TOOL=emoslib_bufr_filter
+EMOSLIB_TOOL=bufrdc_emoslib_bufr_filter
 ECCODES_TOOL=codes_bufr_filter
 result=0 # return code from function
 ERR_TOOL_NOT_FOUND=666

From 0598254de2e00d0e3f0773af3d3d1cf5f96d2f66 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Tue, 17 Sep 2019 17:35:00 +0100
Subject: [PATCH 63/66] ECC-673: Added test

---
 tests/CMakeLists.txt  |  1 +
 tests/bufr_ecc-673.sh | 45 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100755 tests/bufr_ecc-673.sh

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index f349916a8..6a50bcb44 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -92,6 +92,7 @@ list( APPEND tests_data_reqd
     bufr_keys_iter
     bufr_get_element
     bufr_wmo_tables
+    bufr_ecc-673
     bufr_ecc-428
     bufr_ecc-197
     bufr_ecc-286
diff --git a/tests/bufr_ecc-673.sh b/tests/bufr_ecc-673.sh
new file mode 100755
index 000000000..ecb49511e
--- /dev/null
+++ b/tests/bufr_ecc-673.sh
@@ -0,0 +1,45 @@
+#!/bin/sh
+# Copyright 2005-2019 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# 
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+
+# ---------------------------------------------------------
+# This is the test for the JIRA issue ECC-673
+# bufr_filter script and paths with spaces
+# ---------------------------------------------------------
+cd ${data_dir}/bufr
+label="bufr_ecc_673_test"
+
+tempBufr=temp.${label}.bufr
+tempRules=temp.${label}.filter
+
+# --------------------------------------------------------
+# Test 1: input bufr with spaces in name
+# --------------------------------------------------------
+tempBufr="temp.${label}.airs 5 7.bufr"
+cp airs_57.bufr "${tempBufr}"
+cat > $tempRules <<EOF
+ set unpack=1;
+ print "[satelliteIdentifier!0]";
+EOF
+${tools_dir}/bufr_filter $tempRules "${tempBufr}"
+
+# --------------------------------------------------------
+# Test 2: input bufr and rules file with spaces in name
+# --------------------------------------------------------
+tempRules2="temp.${label} . filter"
+cat > "$tempRules2" <<EOF
+ set unpack=1;
+ print "[satelliteIdentifier!0]";
+EOF
+${tools_dir}/bufr_filter "${tempRules2}" "${tempBufr}"
+
+# Clean up
+rm -rf "${tempRules}" "${tempRules2}" "${tempBufr}"

From 73f3e1c8714eaa15dd829023b5a4958f46c8eb9d Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 18 Sep 2019 17:14:24 +0100
Subject: [PATCH 64/66] DGOV-201: rename the key

---
 definitions/grib2/carra_suiteName.table | 6 +++---
 definitions/grib2/local.98.43.def       | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/definitions/grib2/carra_suiteName.table b/definitions/grib2/carra_suiteName.table
index 93bc03191..a0cae5341 100644
--- a/definitions/grib2/carra_suiteName.table
+++ b/definitions/grib2/carra_suiteName.table
@@ -1,5 +1,5 @@
 # CARRA/CERRA suite names
 0 unknown Unknown
-1 enmi-haro-aar      HARMONIE-AROME reanalysis  by MetNorway on AROME-Artic domain
-2 enmi-haro-igb      HARMONIE-AROME reanalysis by MetNorway on IGB domain
-3 enmi-haro-par      HARMONIE-AROME reanalysis by MetNorway on Pan-Arctic domain
+1 enmi-haro-aar HARMONIE-AROME reanalysis by MetNorway on AROME-Artic domain
+2 enmi-haro-igb HARMONIE-AROME reanalysis by MetNorway on IGB domain
+3 enmi-haro-par HARMONIE-AROME reanalysis by MetNorway on Pan-Arctic domain
diff --git a/definitions/grib2/local.98.43.def b/definitions/grib2/local.98.43.def
index 8212f32b2..4e95bc71f 100644
--- a/definitions/grib2/local.98.43.def
+++ b/definitions/grib2/local.98.43.def
@@ -1,4 +1,4 @@
 # Local definition 43: Copernicus Arctic/European Regional ReAnalysis (CARRA/CERRA)
-codetable[2] carraSuiteName "grib2/carra_suiteName.table" : dump;
-alias mars.origin = carraSuiteName;
+codetable[2] suiteName "grib2/carra_suiteName.table" : dump;
+alias mars.origin = suiteName;
 unalias mars.domain; 

From 23452a32ecaa4763f61043b6c2f5077083bfefa8 Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Wed, 18 Sep 2019 17:26:13 +0100
Subject: [PATCH 65/66] DGOV-201: rename the code table

---
 definitions/grib2/local.98.43.def                               | 2 +-
 .../{carra_suiteName.table => regional_reanal_suiteName.table}  | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename definitions/grib2/{carra_suiteName.table => regional_reanal_suiteName.table} (100%)

diff --git a/definitions/grib2/local.98.43.def b/definitions/grib2/local.98.43.def
index 4e95bc71f..5cf36c86f 100644
--- a/definitions/grib2/local.98.43.def
+++ b/definitions/grib2/local.98.43.def
@@ -1,4 +1,4 @@
 # Local definition 43: Copernicus Arctic/European Regional ReAnalysis (CARRA/CERRA)
-codetable[2] suiteName "grib2/carra_suiteName.table" : dump;
+codetable[2] suiteName "grib2/regional_reanal_suiteName.table" : dump;
 alias mars.origin = suiteName;
 unalias mars.domain; 
diff --git a/definitions/grib2/carra_suiteName.table b/definitions/grib2/regional_reanal_suiteName.table
similarity index 100%
rename from definitions/grib2/carra_suiteName.table
rename to definitions/grib2/regional_reanal_suiteName.table

From aedba47268da0dadfa4d327a240bcd8b8d24097e Mon Sep 17 00:00:00 2001
From: Shahram Najm <masn@ecmwf.int>
Date: Thu, 19 Sep 2019 12:43:28 +0100
Subject: [PATCH 66/66] ECC-987: MARS: new class for CARRA/CERRA

---
 definitions/mars/class.table | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/definitions/mars/class.table b/definitions/mars/class.table
index 2260eff29..8a0537758 100644
--- a/definitions/mars/class.table
+++ b/definitions/mars/class.table
@@ -35,8 +35,7 @@
 34 lw WMO Lead Centre Wave Forecast Verification
 35 ce Copernicus Emergency Management Service (CEMS)
 36 cr Copernicus Atmosphere Monitoring Service (CAMS) Research
-37 ca Copernicus Arctic Regional ReAnalysis (CARRA)
-38 cu Copernicus European Regional ReAnalysis (CERRA)
+37 rr Copernicus Regional ReAnalysis (CARRA/CERRA)
 99 te Test
 100 at Austria
 101 be Belgium