Merge branch 'feature/DGOV-201-CARRA-TEST' of ssh://git.ecmwf.int:7999/eccodes/eccodes into feature/DGOV-201-CARRA-TEST

CARRA/CERRA origins update

definitions/grib2/carra_suiteName.table

@@ -1,5 +0,0 @@
-# CARRA/CERRA suite names
-0 unknown Unknown
-1 enmi-haro-cae      HARMONIE-AROME reanalysis by MetNorway on CARRA-East domain
-2 enmi-haro-caw      HARMONIE-AROME reanalysis by MetNorway on CARRA-West domain
-3 enmi-haro-par      HARMONIE-AROME reanalysis by MetNorway on Pan-Arctic domain

definitions/grib2/local.98.43.def

@@ -1,4 +1,4 @@
 # Local definition 43: Copernicus Arctic/European Regional ReAnalysis (CARRA/CERRA)
-codetable[2] carraSuiteName "grib2/carra_suiteName.table" : dump;
+codetable[2] suiteName "grib2/regional_reanal_suiteName.table" : dump;
-alias mars.origin = carraSuiteName;
+alias mars.origin = suiteName;
 unalias mars.domain; 

definitions/grib2/regional_reanal_suiteName.table

@@ -0,0 +1,7 @@
+# CARRA/CERRA suite names
+0 unknown Unknown
+1 enmi-haro-cae HARMONIE-AROME reanalysis by MetNorway on CARRA-East domain
+2 enmi-haro-caw HARMONIE-AROME reanalysis by MetNorway on CARRA-West domain
+3 enmi-haro-par HARMONIE-AROME reanalysis by MetNorway on Pan-Arctic domain
+4 eswi-hald-ecx HARMONIE-ALADIN reanalysis by SMHI on EURO-CORDEX domain
+5 lfpw-mesu-ecx MESCAN-SURFEX surface reanalysis by Meteo-France on EURO-CORDEX domain

definitions/mars/class.table

@@ -35,8 +35,7 @@
 34 lw WMO Lead Centre Wave Forecast Verification
 35 ce Copernicus Emergency Management Service (CEMS)
 36 cr Copernicus Atmosphere Monitoring Service (CAMS) Research
-37 ca Copernicus Arctic Regional ReAnalysis (CARRA)
+37 rr Copernicus Regional ReAnalysis (CARRA/CERRA)
-38 cu Copernicus European Regional ReAnalysis (CERRA)
 99 te Test
 100 at Austria
 101 be Belgium

examples/C/grib_pthreads.c

@@ -15,15 +15,27 @@
 #include "eccodes.h"
 #define NUM_THREADS 4
 
+/* Return 0 if numbers considered equal, otherwise 1 */
+static int compare_doubles(double a,double b,double tolerance)
+{
+    int ret=0;
+    double d=fabs(a-b);
+    if (d > tolerance) {
+        ret=1;
+    }
+    return ret;
+}
+
 static void* process_grib(void* threadID)
 {
     const long tid = (long)threadID;
-    size_t str_len = 20;
+    size_t str_len = 20, i = 0;
     long indicatorOfUnitOfTimeRange = 1, step = 0;
     char mystring[100];
     double* values = NULL;
     size_t values_len = 0;
-    double min=0,max=0;
+    double min=0,max=0,avg=0;
+    const double tol = 1e-6;
     double pv[4]={1,2,3,4};
     const size_t pvsize=4;
     ProductKind prod_kind = 0;
@@ -49,10 +61,20 @@ static void* process_grib(void* threadID)
     CODES_CHECK(codes_get_size(h,"values",&values_len),0);
     values = (double*)malloc(values_len*sizeof(double));
     CODES_CHECK(codes_get_double_array(h, "values", values, &values_len),0);
+    for (i=0;i<values_len;i++) {
+        if (i%2) values[i] *= 0.94;
+        else if (i%3) values[i] *= 0.84;
+    }
+    GRIB_CHECK(grib_set_double_array(h,"values",values,values_len),0);
     free(values);
 
     CODES_CHECK(codes_get_double(h, "min", &min),0);
     CODES_CHECK(codes_get_double(h, "max", &max),0);
+    CODES_CHECK(codes_get_double(h, "avg", &avg),0);
+    printf("Thread %ld: min=%g max=%g avg=%g\n", tid, min, max, avg);
+    assert( compare_doubles(min, 0.84, tol)==0 );
+    assert( compare_doubles(max, 1.00, tol)==0 );
+    assert( compare_doubles(avg, 0.916774, tol)==0 );
 
     codes_handle_delete(h);
     pthread_exit(NULL);
@@ -67,6 +89,7 @@ int main(int argc, char** argv)
         printf("Creating thread %ld\n", i);
         error = pthread_create(&threads[i], NULL, process_grib, (void *)i);
         if (error) {
+            assert(0);
             return 1;
         }
     }

src/action.c

@@ -62,6 +62,31 @@ static void init(grib_action_class *c)
     GRIB_MUTEX_UNLOCK(&mutex1);
 }
 
+#if 0
+/* A non-recursive version */
+static void init(grib_action_class *c)
+{
+    if (!c) return;
+
+    GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
+    GRIB_MUTEX_LOCK(&mutex1);
+    if(!c->inited)
+    {
+        if(c->super) {
+            grib_action_class *g = *(c->super);
+            if (g && !g->inited) {
+                Assert(g->super == NULL);
+                g->init_class(g);
+                g->inited = 1;
+            }
+        }
+        c->init_class(c);
+        c->inited = 1;
+    }
+    GRIB_MUTEX_UNLOCK(&mutex1);
+}
+#endif
+
 void grib_dump(grib_action* a, FILE* f, int l)
 {
     grib_action_class *c = a->cclass;
@@ -119,10 +144,11 @@ int grib_create_accessor(grib_section* p, grib_action* a,  grib_loader* h)
     {
         if(c->create_accessor) {
             int ret;
-            GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
+            /* ECC-604: Do not lock excessively */
-            GRIB_MUTEX_LOCK(&mutex1);
+            /*GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);*/
+            /*GRIB_MUTEX_LOCK(&mutex1);*/
             ret=c->create_accessor(p, a, h);
-            GRIB_MUTEX_UNLOCK(&mutex1);
+            /*GRIB_MUTEX_UNLOCK(&mutex1);*/
             return ret;
         }
         c = c->super ? *(c->super) : NULL;
@@ -136,20 +162,20 @@ int grib_action_notify_change( grib_action* a, grib_accessor *observer, grib_acc
 {
     grib_action_class *c = a->cclass;
 
-    GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
+    /*GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);*/
-    GRIB_MUTEX_LOCK(&mutex1);
+    /*GRIB_MUTEX_LOCK(&mutex1);*/
 
     init(c);
     while(c)
     {
         if(c->notify_change) {
             int result = c->notify_change(a,observer,observed);
-            GRIB_MUTEX_UNLOCK(&mutex1);
+            /*GRIB_MUTEX_UNLOCK(&mutex1);*/
             return result;
         }
         c = c->super ? *(c->super) : NULL;
     }
-    GRIB_MUTEX_UNLOCK(&mutex1);
+    /*GRIB_MUTEX_UNLOCK(&mutex1);*/
     Assert(0);
     return 0;
 }

src/action_class_when.c

@@ -82,6 +82,11 @@ static void init_class(grib_action_class* c)
 }
 /* END_CLASS_IMP */
 
+/* The check on self->loop can only be done in non-threaded mode */
+#if defined(DEBUG) && GRIB_PTHREADS == 0 && GRIB_OMP_THREADS == 0
+  #define CHECK_LOOP 1
+#endif
+
 grib_action* grib_action_create_when( grib_context* context,
         grib_expression* expression,
         grib_action* block_true,grib_action* block_false)
@@ -101,7 +106,6 @@ grib_action* grib_action_create_when( grib_context* context,
     a->block_true       = block_true;
     a->block_false      = block_false;
 
-
     sprintf(name,"_when%p",(void*)expression);
 
     act->name      = grib_context_strdup_persistent(context,name);
@@ -152,12 +156,19 @@ static void dump(grib_action* act, FILE* f, int lvl)
     printf("\n");
 }
 
+#ifdef CHECK_LOOP
+  #define SET_LOOP(self,v)  self->loop=v;
+#else
+  #define SET_LOOP(self,v)
+#endif
+
 static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor* observed)
 {
     grib_action_when* self = (grib_action_when*) a;
     grib_action *b = NULL;
     int ret = GRIB_SUCCESS;
     long lres;
+
     /* ECC-974: observed->parent will change as a result of the execute
      * so must store the handle once here (in 'hand') rather than call
      * grib_handle_of_accessor(observed) later
@@ -166,7 +177,7 @@ static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor*
 
     if ((ret = grib_expression_evaluate_long(hand, self->expression,&lres))
             != GRIB_SUCCESS) return ret;
-#ifdef DEBUG
+#ifdef CHECK_LOOP
     if(self->loop)
     {
         printf("LOOP detected...\n");
@@ -176,7 +187,7 @@ static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor*
         return ret;
     }
 #endif
-    self->loop = 1;
+    SET_LOOP(self, 1);
 
     if(lres)
         b=self->block_true;
@@ -186,13 +197,13 @@ static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor*
     while(b) {
         ret = grib_action_execute(b,hand);
         if(ret != GRIB_SUCCESS) {
-            self->loop = 0;
+            SET_LOOP(self, 0);
             return ret;
         }
         b = b->next;
     }
 
-    self->loop = 0;
+    SET_LOOP(self, 0);
 
     return GRIB_SUCCESS;
 }

src/grib_context.c

@@ -623,7 +623,9 @@ char *grib_context_full_defs_path(grib_context* c,const char* basename)
     if(*basename == '/' || *basename ==  '.') {
         return (char*)basename;
     } else {
+        GRIB_MUTEX_LOCK(&mutex_c);  /* See ECC-604 */
         fullpath=(grib_string_list*)grib_trie_get(c->def_files,basename);
+        GRIB_MUTEX_UNLOCK(&mutex_c);
         if (fullpath!=NULL) {
             return fullpath->value;
         }

src/grib_dumper.c

@@ -11,204 +11,230 @@
 /**************************************
  *  Enrico Fucile
  **************************************/
-
-
 #include "grib_api_internal.h"
 
+#if GRIB_PTHREADS
+static pthread_once_t once  = PTHREAD_ONCE_INIT;
+static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static void init_mutex() {
+    pthread_mutexattr_t attr;
+    pthread_mutexattr_init(&attr);
+    pthread_mutexattr_settype(&attr,PTHREAD_MUTEX_RECURSIVE);
+    pthread_mutex_init(&mutex,&attr);
+    pthread_mutexattr_destroy(&attr);
+}
+#elif GRIB_OMP_THREADS
+static int once = 0;
+static omp_nest_lock_t mutex;
+
+static void init_mutex()
+{
+    GRIB_OMP_CRITICAL(lock_dumper_c)
+    {
+        if (once == 0)
+        {
+            omp_init_nest_lock(&mutex);
+            once = 1;
+        }
+    }
+}
+#endif
 
 static void init_dumpers(grib_dumper_class* c,grib_dumper* d)
 {
-  if(c) {
+    GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
-    grib_dumper_class *s = c->super ? *(c->super) : NULL;
+    GRIB_MUTEX_LOCK(&mutex);
-    if(!c->inited)
+    if(c) {
-    {
+        grib_dumper_class *s = c->super ? *(c->super) : NULL;
-      if(c->init_class) c->init_class(c);
+        if(!c->inited)
-      c->inited = 1;
+        {
+            if(c->init_class) c->init_class(c);
+            c->inited = 1;
+        }
+        init_dumpers(s,d);
+        if(c->init) c->init(d);
     }
-    init_dumpers(s,d);
+    GRIB_MUTEX_UNLOCK(&mutex);
-    if(c->init) c->init(d);
-  }
 }
 
 void grib_init_dumper(grib_dumper* d)
 {
-  init_dumpers(d->cclass,d);
+    init_dumpers(d->cclass,d);
 }
 
 void grib_dumper_delete(grib_dumper* d)
 {
-  grib_dumper_class *c = d->cclass;
+    grib_dumper_class *c = d->cclass;
-  grib_context *ctx = d->handle->context;
+    grib_context *ctx = d->handle->context;
-  while(c)
+    while(c)
-  {
+    {
-    grib_dumper_class *s = c->super ? *(c->super) : NULL;
+        grib_dumper_class *s = c->super ? *(c->super) : NULL;
-    if(c->destroy) c->destroy(d);
+        if(c->destroy) c->destroy(d);
-    c = s;
+        c = s;
-  }
+    }
-  grib_context_free(ctx,d);
+    grib_context_free(ctx,d);
 }
 
-
 void  grib_dump_long(grib_dumper* d, grib_accessor* a, const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
+    grib_dumper_class *c = d->cclass;
-  while(c)
+    while(c)
-  {
-    if(c->dump_long)
     {
-      c->dump_long(d, a, comment);
+        if(c->dump_long)
-      return;
+        {
+            c->dump_long(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
+    Assert(0);
-  }
-  Assert(0);
 }
 
 void  grib_dump_double(grib_dumper* d,grib_accessor* a,const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
+    grib_dumper_class *c = d->cclass;
-  while(c)
+    while(c)
-  {
-    if(c->dump_double)
     {
-      c->dump_double(d, a, comment);
+        if(c->dump_double)
-      return;
+        {
+            c->dump_double(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
+    Assert(0);
-  }
-  Assert(0);
 }
 
 void  grib_dump_string(grib_dumper* d,grib_accessor* a,const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
+    grib_dumper_class *c = d->cclass;
-  while(c)
+    while(c)
-  {
-    if(c->dump_string)
     {
-      c->dump_string(d, a, comment);
+        if(c->dump_string)
-      return;
+        {
+            c->dump_string(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
+    Assert(0);
-  }
-  Assert(0);
 }
 
 void  grib_dump_string_array(grib_dumper* d,grib_accessor* a,const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
+    grib_dumper_class *c = d->cclass;
-  while(c)
+    while(c)
-  {
-    if(c->dump_string_array)
     {
-      c->dump_string_array(d, a, comment);
+        if(c->dump_string_array)
-      return;
+        {
+            c->dump_string_array(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
+    Assert(0);
-  }
-  Assert(0);
 }
 
 void  grib_dump_label(grib_dumper* d,grib_accessor* a,const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
+    grib_dumper_class *c = d->cclass;
-  while(c)
+    while(c)
-  {
-    if(c->dump_label)
     {
-      c->dump_label(d, a, comment);
+        if(c->dump_label)
-      return;
+        {
+            c->dump_label(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
+    Assert(0);
-  }
-  Assert(0);
 }
 
 void  grib_dump_bytes(grib_dumper* d,grib_accessor* a,const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
+    grib_dumper_class *c = d->cclass;
-  while(c)
+    while(c)
-  {
-    if(c->dump_bytes)
     {
-      c->dump_bytes(d, a, comment);
+        if(c->dump_bytes)
-      return;
+        {
+            c->dump_bytes(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
+    Assert(0);
-  }
-  Assert(0);
 }
 
 void  grib_dump_bits(grib_dumper* d,grib_accessor* a,const char* comment)
 {
-  grib_dumper_class *c = d->cclass;
+    grib_dumper_class *c = d->cclass;
-  while(c)
+    while(c)
-  {
-    if(c->dump_bits)
     {
-      c->dump_bits(d, a, comment);
+        if(c->dump_bits)
-      return;
+        {
+            c->dump_bits(d, a, comment);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
+    Assert(0);
-  }
-  Assert(0);
 }
 
-
 void  grib_dump_section(grib_dumper* d,grib_accessor* a,grib_block_of_accessors* block)
 {
-  grib_dumper_class *c = d->cclass;
+    grib_dumper_class *c = d->cclass;
-  while(c)
+    while(c)
-  {
-    if(c->dump_section)
     {
-      c->dump_section(d, a, block);
+        if(c->dump_section)
-      return;
+        {
+            c->dump_section(d, a, block);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
+    Assert(0);
-  }
-  Assert(0);
 }
 
 void  grib_dump_values(grib_dumper* d,grib_accessor* a)
 {
-  grib_dumper_class *c = d->cclass;
+    grib_dumper_class *c = d->cclass;
-  while(c)
+    while(c)
-  {
-    if(c->dump_values)
     {
-      c->dump_values(d, a);
+        if(c->dump_values)
-      return;
+        {
+            c->dump_values(d, a);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
+    Assert(0);
-  }
-  Assert(0);
 }
 
 void  grib_dump_header(grib_dumper* d,grib_handle* h)
 {
-  grib_dumper_class *c = d->cclass;
+    grib_dumper_class *c = d->cclass;
-  while(c)
+    while(c)
-  {
-    if(c->header)
     {
-      c->header(d,h);
+        if(c->header)
-      return;
+        {
+            c->header(d,h);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
 }
 
 void  grib_dump_footer(grib_dumper* d,grib_handle* h)
 {
-  grib_dumper_class *c = d->cclass;
+    grib_dumper_class *c = d->cclass;
-  while(c)
+    while(c)
-  {
-    if(c->footer)
     {
-      c->footer(d,h);
+        if(c->footer)
-      return;
+        {
+            c->footer(d,h);
+            return;
+        }
+        c = c->super ? *(c->super) : NULL;
     }
-    c = c->super ? *(c->super) : NULL;
-  }
 }

src/grib_iterator.c

@@ -15,6 +15,35 @@
  ***************************************************************************/
 #include "grib_api_internal.h"
 
+#if GRIB_PTHREADS
+static pthread_once_t once  = PTHREAD_ONCE_INIT;
+static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static void init_mutex() {
+    pthread_mutexattr_t attr;
+    pthread_mutexattr_init(&attr);
+    pthread_mutexattr_settype(&attr,PTHREAD_MUTEX_RECURSIVE);
+    pthread_mutex_init(&mutex,&attr);
+    pthread_mutexattr_destroy(&attr);
+}
+#elif GRIB_OMP_THREADS
+static int once = 0;
+static omp_nest_lock_t mutex;
+
+static void init_mutex()
+{
+    GRIB_OMP_CRITICAL(lock_iterator_c)
+    {
+        if (once == 0)
+        {
+            omp_init_nest_lock(&mutex);
+            once = 1;
+        }
+    }
+}
+#endif
+
+
 int grib_get_data(grib_handle* h,double* lats, double* lons,double* values)
 {
     int err=0;
@@ -71,7 +100,6 @@ int grib_iterator_previous(grib_iterator *i,double* lat,double* lon,double* valu
     return 0;
 }
 
-
 int grib_iterator_reset(grib_iterator *i)
 {
     grib_iterator_class *c = i->cclass;
@@ -85,12 +113,9 @@ int grib_iterator_reset(grib_iterator *i)
     return 0;
 }
 
-
 /* For this one, ALL init are called */
-
 static int init_iterator(grib_iterator_class* c,grib_iterator* i, grib_handle *h, grib_arguments* args)
 {
-
     if(c) {
         int ret = GRIB_SUCCESS;
         grib_iterator_class *s = c->super ? *(c->super) : NULL;
@@ -110,7 +135,12 @@ static int init_iterator(grib_iterator_class* c,grib_iterator* i, grib_handle *h
 
 int grib_iterator_init(grib_iterator* i, grib_handle *h, grib_arguments* args)
 {
-    return init_iterator(i->cclass,i,h,args);
+    int r = 0;
+    GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
+    GRIB_MUTEX_LOCK(&mutex);
+    r = init_iterator(i->cclass,i,h,args);
+    GRIB_MUTEX_UNLOCK(&mutex);
+    return r;
 }
 
 /* For this one, ALL destroy are called */

src/grib_parse_utils.c

@@ -9,7 +9,7 @@
  */
 
 /***************************************************************************
- *   Jean Baptiste Filippi - 01.11.2005                                                           *
+ *   Jean Baptiste Filippi - 01.11.2005                                    *
  *   Enrico Fucile
  *                                                                         *
  ***************************************************************************/
@@ -729,16 +729,16 @@ static grib_action* grib_parse_stream(grib_context* gc, const char* filename)
 grib_concept_value* grib_parse_concept_file( grib_context* gc,const char* filename)
 {
     GRIB_MUTEX_INIT_ONCE(&once,&init);
-    GRIB_MUTEX_LOCK(&mutex_concept);
+    GRIB_MUTEX_LOCK(&mutex_file);
 
     gc = gc ? gc : grib_context_get_default();
     grib_parser_context = gc;
 
     if(parse(gc,filename) == 0) {
-        GRIB_MUTEX_UNLOCK(&mutex_concept);
+        GRIB_MUTEX_UNLOCK(&mutex_file);
         return grib_parser_concept;
     } else {
-        GRIB_MUTEX_UNLOCK(&mutex_concept);
+        GRIB_MUTEX_UNLOCK(&mutex_file);
         return NULL;
     }
 }

src/grib_util.c

@@ -931,7 +931,7 @@ grib_handle* grib_util_set_spec2(grib_handle* h,
                 setSecondOrder=1;
                 break;
             default :
-                printf("invalid packing_spec->packing_type = %ld\n",(long)packing_spec->packing_type);
+                fprintf(stderr, "invalid packing_spec->packing_type = %ld\n",(long)packing_spec->packing_type);
                 *err = GRIB_INTERNAL_ERROR;
                 goto cleanup;
                 break;
@@ -957,7 +957,7 @@ grib_handle* grib_util_set_spec2(grib_handle* h,
             break;
 
         default:
-            printf("invalid packing_spec->accuracy = %ld\n",(long)packing_spec->accuracy);
+            fprintf(stderr, "invalid packing_spec->accuracy = %ld\n",(long)packing_spec->accuracy);
             *err = GRIB_INTERNAL_ERROR;
             goto cleanup;
             break;
@@ -1333,7 +1333,7 @@ grib_handle* grib_util_set_spec2(grib_handle* h,
             setSecondOrder=1;
             break;
         default :
-            printf("invalid packing_spec->packing_type = %ld\n",(long)packing_spec->packing_type);
+            fprintf(stderr,"invalid packing_spec->packing_type = %ld\n",(long)packing_spec->packing_type);
             *err = GRIB_INTERNAL_ERROR;
             goto cleanup;
             break;
@@ -1390,7 +1390,7 @@ grib_handle* grib_util_set_spec2(grib_handle* h,
         break;
 
     default:
-        printf("invalid packing_spec->accuracy = %ld\n",(long)packing_spec->accuracy);
+        fprintf(stderr, "invalid packing_spec->accuracy = %ld\n",(long)packing_spec->accuracy);
         *err = GRIB_INTERNAL_ERROR;
         goto cleanup;
         break;
@@ -1453,7 +1453,7 @@ grib_handle* grib_util_set_spec2(grib_handle* h,
         if (global_grid) {
             size_t sum = sum_of_pl_array(spec->pl, spec->pl_size);
             if (data_values_count != sum) {
-                printf("invalid reduced gaussian grid: specified as global, data_values_count=%ld but sum of pl array=%ld\n",
+                fprintf(stderr, "invalid reduced gaussian grid: specified as global, data_values_count=%ld but sum of pl array=%ld\n",
                         (long)data_values_count, (long)sum);
                 *err = GRIB_WRONG_GRID;
                 goto cleanup;

tests/CMakeLists.txt

@@ -92,6 +92,7 @@ list( APPEND tests_data_reqd
     bufr_keys_iter
     bufr_get_element
     bufr_wmo_tables
+    bufr_ecc-673
     bufr_ecc-428
     bufr_ecc-197
     bufr_ecc-286
@@ -254,6 +255,24 @@ if( ENABLE_EXTRA_TESTS AND HAVE_ECCODES_THREADS )
                       TYPE SCRIPT
                       COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_encode_pthreads.sh
     )
+
+    foreach( test grib_ecc-604 grib_ecc-604-encode bufr_ecc-604 )
+        ecbuild_add_executable( TARGET    ${test}
+                                NOINSTALL
+                                SOURCES   ${test}.c
+                                LIBS      eccodes
+        )
+    endforeach()
+    ecbuild_add_test( TARGET eccodes_t_grib_ecc-604
+                      TYPE SCRIPT
+                      TEST_DEPENDS eccodes_download_gribs
+                      COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_ecc-604.sh
+    )
+    ecbuild_add_test( TARGET eccodes_t_bufr_ecc-604
+                      TYPE SCRIPT
+                      TEST_DEPENDS eccodes_download_bufrs
+                      COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bufr_ecc-604.sh
+    )
 endif()
 
 ecbuild_add_test( TARGET eccodes_t_grib_to_netcdf

tests/bufr_ecc-428.sh

@@ -35,21 +35,21 @@ cat > $tempRules <<EOF
 EOF
 
 cat $tempRules
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText
 echo "784" > $tempRef1
 diff $tempRef1 $tempText
 
 export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=1
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText
 echo "784 784 784 784 784 784 784 784 784 784 784 784 784 784 784" > $tempRef2
 diff $tempRef2 $tempText
 
 unset ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText
 diff $tempRef1 $tempText
 
 export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=0
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText
 diff $tempRef1 $tempText
 
 
@@ -64,7 +64,7 @@ EOF
 cat $tempRules
 
 export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=1
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText 2>$tempErrs
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText 2>$tempErrs
 grep -q '^48 54 59.*91 97' $tempText
 
 
@@ -79,7 +79,7 @@ EOF
 cat $tempRules
 
 export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=1
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText 2>$tempErrs
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText 2>$tempErrs
 grep -q '^LBG LBG LBG LBG.*LBG$' $tempText
 
 
@@ -94,7 +94,7 @@ EOF
 cat $tempRules
 
 export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=1
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile > $tempText 2>$tempErrs
+${tools_dir}/bufr_filter $tempRules $bufrFile > $tempText 2>$tempErrs
 grep -q '^6 6 6.*24 24.*6 6' $tempText
 
 # --------------------------------------------------------
@@ -107,11 +107,11 @@ cat > $tempRules <<EOF
 EOF
 
 cat $tempRules
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile >/dev/null
+${tools_dir}/bufr_filter $tempRules $bufrFile >/dev/null
 
 echo "TODO: Searching for backscatter ... currently failing"
 export ECCODES_BUFR_MULTI_ELEMENT_CONSTANT_ARRAYS=1
-${tools_dir}/codes_bufr_filter $tempRules $bufrFile 2>$tempErrs
+${tools_dir}/bufr_filter $tempRules $bufrFile 2>$tempErrs
 cat $tempErrs
 
 # Clean up

tests/bufr_ecc-604.c

@@ -0,0 +1,176 @@
+/*
+ * Test for ECC-604: Each thread creates a new BUFR handle, optionaly clone it and/or write it out
+ */
+#include <time.h>
+#include <pthread.h>
+#include <assert.h>
+#include <unistd.h>
+
+#include "eccodes.h"
+#include "grib_api_internal.h"
+
+/* These are passed in via argv */
+static size_t NUM_THREADS         = 0;
+static size_t FILES_PER_ITERATION = 0;
+static char*  INPUT_FILE          = NULL;
+int opt_dump  = 0; /* If 1 then dump handle to /dev/null */
+int opt_clone = 0; /* If 1 then clone source handle */
+int opt_write = 0; /* If 1 write handle to file */
+
+static int encode_file(char *template_file, char *output_file)
+{
+    FILE *in, *out;
+    codes_handle *source_handle = NULL;
+    const void *buffer = NULL;
+    size_t size = 0;
+    int err = 0;
+    long numSubsets = 0;
+
+    in = fopen(template_file,"r"); assert(in);
+    if (opt_write) {
+        out = fopen(output_file,"w");  assert(out);
+    }
+
+    /* loop over the messages in the source BUFR and clone them */
+    while ((source_handle = codes_handle_new_from_file(NULL, in, PRODUCT_BUFR, &err)) != NULL || err != CODES_SUCCESS) {
+        codes_handle *h = source_handle;
+        
+        if (opt_clone) {
+            h = codes_handle_clone(source_handle); assert(h);
+        }
+
+        CODES_CHECK(codes_get_long(h,"numberOfSubsets", &numSubsets),0);
+        CODES_CHECK(codes_set_long(h, "unpack", 1),0);
+
+        CODES_CHECK(codes_get_message(h,&buffer,&size),0);
+        if (opt_write) {
+            if(fwrite(buffer,1,size,out) != size) {
+                perror(output_file);
+                return 1;
+            }
+        }
+        if (opt_dump) {
+            FILE *devnull = fopen("/dev/null", "w");
+            grib_dumper* dumper = NULL;
+            const char* dumper_name = "bufr_simple";
+            unsigned long dump_flags = CODES_DUMP_FLAG_ALL_DATA;
+            /* codes_dump_content(source_handle,devnull, "json", 1024, NULL); */ /* JSON dump with all attributes */
+            dumper=grib_dump_content_with_dumper(source_handle, dumper, devnull, dumper_name, dump_flags, NULL);
+            assert(dumper);
+            fclose(devnull);
+        }
+
+        codes_handle_delete(source_handle);
+        if(opt_clone) codes_handle_delete(h);
+    }
+
+    if (opt_write) fclose(out);
+    fclose(in);
+
+    return 0;
+}
+
+void do_stuff(void *data);
+
+/* Structure for passing data to threads */
+struct v {
+    size_t number;
+    char *data;
+};
+
+void *runner(void *ptr); /* the thread */
+
+int main(int argc, char **argv)
+{
+    size_t i;
+    int thread_counter = 0;
+    int parallel=1, index=0, c=0;
+    const char* prog = argv[0];
+    char* mode;
+    if (argc<5 || argc>7) {
+        fprintf(stderr, "Usage:\n\t%s [options] seq file numRuns numIter\nOr\n\t%s [options] par file numThreads numIter\n", prog, prog);
+        return 1;
+    }
+    
+    while ((c = getopt (argc, argv, "dcw")) != -1) {
+        switch (c) {
+            case 'd': opt_dump=1; break;
+            case 'c': opt_clone=1; break;
+            case 'w': opt_write=1; break;
+        }
+    }
+    index = optind;
+    mode = argv[index];
+    INPUT_FILE = argv[index+1];
+    NUM_THREADS = atol(argv[index+2]);
+    FILES_PER_ITERATION = atol(argv[index+3]);
+
+    if (strcmp(mode,"seq")==0) {
+        parallel = 0;
+    }
+    if (parallel) {
+        printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+        printf("Options: dump=%d, clone=%d, write=%d\n", opt_dump, opt_clone, opt_write);
+    } else {
+        printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+    }
+
+    {
+        pthread_t workers[NUM_THREADS];
+        for (i = 0; i < NUM_THREADS; i++) {
+            struct v *data = (struct v *) malloc(sizeof(struct v));
+            data->number = i;
+            data->data = NULL;
+
+            if (parallel) {
+                /* Now we will create the thread passing it data as an argument */
+                pthread_create(&workers[thread_counter], NULL, runner, data);
+                /*pthread_join(workers[thread_counter], NULL);*/
+                thread_counter++;
+            } else {
+                do_stuff(data);
+            }
+        }
+
+        if (parallel) {
+            for (i = 0; i < NUM_THREADS; i++)  {
+                pthread_join(workers[i], NULL);
+            }
+        }
+    }
+
+    return 0;
+}
+
+void *runner(void *ptr)
+{
+    do_stuff(ptr);
+    pthread_exit(0);
+}
+
+void do_stuff(void *ptr)
+{
+    /* Cast argument to struct v pointer */
+    struct v *data = ptr;
+    size_t i;
+    char output_file[50];
+    time_t ltime;
+    struct tm result;
+    char stime[32];
+
+    for (i=0; i<FILES_PER_ITERATION;i++) {
+        if (opt_write) {
+            sprintf(output_file,"output/output_file_%ld-%ld.bufr", data->number, i);
+            encode_file(INPUT_FILE,output_file);
+        } else {
+            encode_file(INPUT_FILE,NULL);
+        }
+    }
+
+    ltime = time(NULL);
+    localtime_r(&ltime, &result);
+    strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
+    /* asctime_r(&result, stime); */
+
+    printf("%s: Worker %ld finished.\n", stime, data->number);
+}

tests/bufr_ecc-604.sh

@@ -0,0 +1,68 @@
+#!/bin/sh
+# Copyright 2005-2018 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+
+label="bufr_ecc-604"
+temp_dir=tempdir.${label}
+
+NUM_THREADS=3
+NUM_ITER=10
+OUTPUT=output
+
+validate()
+{
+    echo "Checking every output file is identical..."
+    # Get checksum of first file
+    ck1=`cksum $OUTPUT/output_file_0-0.bufr | awk '{print $1}'`
+    set +x
+    # Get checksum of all of them and sort unique
+    res=`cksum $OUTPUT/output_file_* | awk '{print $1}' | sort -u`
+    set -x
+    # Should be the same as the first
+    [ "$res" = "$ck1" ]
+}
+process()
+{
+    input=$1 # The input BUFR file
+
+    # Test 01: Clone + output
+    # ------------------------
+    rm -fr $OUTPUT;   mkdir -p $OUTPUT
+    time ${test_dir}/bufr_ecc-604 -c -w par $input $NUM_THREADS $NUM_ITER
+    validate
+
+    # Test 02: No clone + output
+    # --------------------------
+    rm -fr $OUTPUT;   mkdir -p $OUTPUT
+    time ${test_dir}/bufr_ecc-604 -w par $input $NUM_THREADS $NUM_ITER
+    validate
+
+    # Test 03: Clone + no output
+    # ---------------------------
+    rm -fr $OUTPUT
+    time ${test_dir}/bufr_ecc-604 -c par $input $NUM_THREADS $NUM_ITER
+    # Nothing to validate as there is no output
+}
+###################################################
+rm -fr $temp_dir
+mkdir -p $temp_dir
+cd $temp_dir
+
+bufr_files=`cat ${data_dir}/bufr/bufr_data_files.txt`
+for bf in ${bufr_files}; do
+    b=${data_dir}/bufr/$bf
+    echo "Doing $b"
+    process $b
+done
+
+# Clean up
+cd $test_dir
+rm -fr $temp_dir

tests/bufr_ecc-673.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+# Copyright 2005-2019 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# 
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+
+# ---------------------------------------------------------
+# This is the test for the JIRA issue ECC-673
+# bufr_filter script and paths with spaces
+# ---------------------------------------------------------
+cd ${data_dir}/bufr
+label="bufr_ecc_673_test"
+
+tempBufr=temp.${label}.bufr
+tempRules=temp.${label}.filter
+
+# --------------------------------------------------------
+# Test 1: input bufr with spaces in name
+# --------------------------------------------------------
+tempBufr="temp.${label}.airs 5 7.bufr"
+cp airs_57.bufr "${tempBufr}"
+cat > $tempRules <<EOF
+ set unpack=1;
+ print "[satelliteIdentifier!0]";
+EOF
+${tools_dir}/bufr_filter $tempRules "${tempBufr}"
+
+# --------------------------------------------------------
+# Test 2: input bufr and rules file with spaces in name
+# --------------------------------------------------------
+tempRules2="temp.${label} . filter"
+cat > "$tempRules2" <<EOF
+ set unpack=1;
+ print "[satelliteIdentifier!0]";
+EOF
+${tools_dir}/bufr_filter "${tempRules2}" "${tempBufr}"
+
+# Clean up
+rm -rf "${tempRules}" "${tempRules2}" "${tempBufr}"

tests/grib_ecc-604-encode.c

@@ -0,0 +1,145 @@
+/*
+ * Test for ECC-604: GRIB decoding/encoding sequentially and parallel with POSIX threads
+ */
+#include <time.h>
+#include <pthread.h>
+#include <assert.h>
+#include <unistd.h>
+
+#include "grib_api.h"
+
+/* These are passed in via argv */
+static size_t NUM_THREADS         = 0;
+static size_t FILES_PER_ITERATION = 0;
+static char*  INPUT_FILE          = NULL;
+int opt_dump  = 0; /* If 1 then dump handle to /dev/null */
+int opt_clone = 0; /* If 1 then clone source handle */
+int opt_write = 0; /* If 1 write handle to file */
+
+static int encode_values(grib_handle* h, char *output_file)
+{
+    double *values;
+    const size_t DIM = 1000;
+    size_t size = DIM * DIM;
+    size_t i = 0;
+    values = (double*)malloc(size*sizeof(double));
+    for (i=0; i<size; ++i) {
+        double v = i;
+        if (i % DIM == 0) v = 0;
+        values[i] = v;
+    }
+    GRIB_CHECK(grib_set_long(h,"bitsPerValue",16),0);
+    GRIB_CHECK(grib_set_double_array(h,"values",values,size), 0);
+    free(values);
+    return GRIB_SUCCESS;
+}
+
+void do_encode(void *data);
+
+/* Structure for passing data to threads */
+struct v {
+    size_t number;
+    char *data;
+};
+
+void *runner(void *ptr); /* the thread */
+
+int main(int argc, char **argv)
+{
+    size_t i;
+    int thread_counter = 0;
+    int parallel=1, index=0, c=0;
+    const char* prog = argv[0];
+    char* mode;
+    if (argc<5 || argc>7) {
+        fprintf(stderr, "Usage:\n\t%s [options] seq sample numRuns numIter\nOr\n\t%s [options] par sample numThreads numIter\n", prog, prog);
+        return 1;
+    }
+    
+    while ((c = getopt (argc, argv, "dcw")) != -1) {
+        switch (c) {
+            case 'd': opt_dump=1; break;
+            case 'c': opt_clone=1; break;
+            case 'w': opt_write=1; break;
+        }
+    }
+    index = optind;
+    mode = argv[index];
+    INPUT_FILE = argv[index+1]; /* Has to be the name of a sample file (without tmpl extension) */
+    NUM_THREADS = atol(argv[index+2]);
+    FILES_PER_ITERATION = atol(argv[index+3]);
+
+    if (strcmp(mode,"seq")==0) {
+        parallel = 0;
+    }
+    if (parallel) {
+        printf("Running parallel in %ld threads. %ld iterations (prod=%ld)\n", NUM_THREADS, FILES_PER_ITERATION, NUM_THREADS*FILES_PER_ITERATION);
+        printf("Options: dump=%d, clone=%d, write=%d\n", opt_dump, opt_clone, opt_write);
+    } else {
+        printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+    }
+
+    {
+        pthread_t workers[NUM_THREADS];
+        for (i = 0; i < NUM_THREADS; i++) {
+            struct v *data = (struct v *) malloc(sizeof(struct v));
+            data->number = i;
+            data->data = NULL;
+
+            if (parallel) {
+                /* Now we will create the thread passing it data as an argument */
+                pthread_create(&workers[thread_counter], NULL, runner, data);
+                thread_counter++;
+            } else {
+                do_encode(data);
+            }
+        }
+
+        if (parallel) {
+            for (i = 0; i < NUM_THREADS; i++)  {
+                pthread_join(workers[i], NULL);
+            }
+        }
+    }
+
+    return 0;
+}
+
+void *runner(void *ptr)
+{
+    do_encode(ptr);
+    pthread_exit(0);
+}
+
+void do_encode(void *ptr)
+{
+    /* Cast argument to struct v pointer */
+    struct v *data = ptr;
+    size_t i;
+    char output_file[50];
+    time_t ltime;
+    struct tm result;
+    char stime[32];
+    grib_handle *hs = NULL;
+
+    hs = grib_handle_new_from_samples(0, INPUT_FILE);
+
+    for (i=0; i<FILES_PER_ITERATION;i++) {
+        grib_handle *h = grib_handle_clone(hs);
+        if (opt_write) {
+            sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
+            encode_values(h,output_file);
+        } else {
+            encode_values(h,NULL);
+        }
+        grib_handle_delete(h);
+    }
+
+    ltime = time(NULL);
+    localtime_r(&ltime, &result);
+    strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
+    /* asctime_r(&result, stime); */
+
+    printf("%s: Worker %ld finished.\n", stime, data->number);
+    grib_handle_delete(hs);
+}

tests/grib_ecc-604.c

@@ -0,0 +1,187 @@
+/*
+ * Test for ECC-604: GRIB decoding/encoding sequentially and parallel with POSIX threads
+ */
+#include <time.h>
+#include <pthread.h>
+#include <assert.h>
+#include <unistd.h>
+
+#include "grib_api.h"
+
+/* These are passed in via argv */
+static size_t NUM_THREADS         = 0;
+static size_t FILES_PER_ITERATION = 0;
+static char*  INPUT_FILE          = NULL;
+int opt_dump  = 0; /* If 1 then dump handle to /dev/null */
+int opt_clone = 0; /* If 1 then clone source handle */
+int opt_write = 0; /* If 1 write handle to file */
+
+static int encode_file(char *template_file, char *output_file)
+{
+    FILE *in, *out;
+    grib_handle *source_handle = NULL;
+    const void *buffer = NULL;
+    size_t size = 0;
+    int err = 0;
+    double *values;
+
+    in = fopen(template_file,"r"); assert(in);
+    if (opt_write) {
+        out = fopen(output_file,"w");  assert(out);
+    }
+
+    /* loop over the messages in the source GRIB and clone them */
+    while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
+        int i;
+        size_t values_len = 0;
+        size_t str_len = 20;
+        grib_handle *h = source_handle;
+        
+        if (opt_clone) {
+            h = grib_handle_clone(source_handle); assert(h);
+        }
+
+        GRIB_CHECK(grib_get_size(h, "values", &values_len),0);
+
+        values = (double*)malloc(values_len*sizeof(double));
+        GRIB_CHECK(grib_get_double_array(h, "values", values, &values_len),0);
+
+        for (i=0;i<values_len;i++) {
+            values[i] *= 0.9;
+        }
+
+        GRIB_CHECK(grib_set_string(h,"stepUnits", "s", &str_len),0);
+        GRIB_CHECK(grib_set_long(h, "startStep", 43200), 0);
+        GRIB_CHECK(grib_set_long(h, "endStep", 86400), 0);
+        GRIB_CHECK(grib_set_long(h, "bitsPerValue", 16),0);
+
+        /* set data values */
+        GRIB_CHECK(grib_set_double_array(h,"values",values,values_len),0);
+
+        GRIB_CHECK(grib_get_message(h,&buffer,&size),0);
+        if (opt_write) {
+            if(fwrite(buffer,1,size,out) != size) {
+                perror(output_file);
+                return 1;
+            }
+        }
+        if (opt_dump) {
+            FILE *devnull = fopen("/dev/null", "w");
+            grib_dump_content(source_handle,devnull, "debug", 0, NULL);
+        }
+
+        grib_handle_delete(source_handle);
+        if(opt_clone) grib_handle_delete(h);
+        free(values);
+    }
+
+    if (opt_write) fclose(out);
+    fclose(in);
+
+    return 0;
+}
+
+void do_stuff(void *data);
+
+/* Structure for passing data to threads */
+struct v {
+    size_t number;
+    char *data;
+};
+
+void *runner(void *ptr); /* the thread */
+
+int main(int argc, char **argv)
+{
+    size_t i;
+    int thread_counter = 0;
+    int parallel=1, index=0, c=0;
+    const char* prog = argv[0];
+    char* mode;
+    if (argc<5 || argc>8) {
+        fprintf(stderr, "Usage:\n\t%s [options] seq file numRuns numIter\nOr\n\t%s [options] par file numThreads numIter\n", prog, prog);
+        return 1;
+    }
+    
+    while ((c = getopt (argc, argv, "dcw")) != -1) {
+        switch (c) {
+            case 'd': opt_dump=1; break;
+            case 'c': opt_clone=1; break;
+            case 'w': opt_write=1; break;
+        }
+    }
+    index = optind;
+    mode = argv[index];
+    INPUT_FILE = argv[index+1];
+    NUM_THREADS = atol(argv[index+2]);
+    FILES_PER_ITERATION = atol(argv[index+3]);
+
+    if (strcmp(mode,"seq")==0) {
+        parallel = 0;
+    }
+    if (parallel) {
+        printf("Running parallel in %ld threads. %ld iterations (prod=%ld)\n", NUM_THREADS, FILES_PER_ITERATION, NUM_THREADS*FILES_PER_ITERATION);
+        printf("Options: dump=%d, clone=%d, write=%d\n", opt_dump, opt_clone, opt_write);
+    } else {
+        printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
+    }
+
+    {
+        pthread_t workers[NUM_THREADS];
+        for (i = 0; i < NUM_THREADS; i++) {
+            struct v *data = (struct v *) malloc(sizeof(struct v));
+            data->number = i;
+            data->data = NULL;
+
+            if (parallel) {
+                /* Now we will create the thread passing it data as an argument */
+                pthread_create(&workers[thread_counter], NULL, runner, data);
+                /*pthread_join(workers[thread_counter], NULL);*/
+                thread_counter++;
+            } else {
+                do_stuff(data);
+            }
+        }
+
+        if (parallel) {
+            for (i = 0; i < NUM_THREADS; i++)  {
+                pthread_join(workers[i], NULL);
+            }
+        }
+    }
+
+    return 0;
+}
+
+void *runner(void *ptr)
+{
+    do_stuff(ptr);
+    pthread_exit(0);
+}
+
+void do_stuff(void *ptr)
+{
+    /* Cast argument to struct v pointer */
+    struct v *data = ptr;
+    size_t i;
+    char output_file[50];
+    time_t ltime;
+    struct tm result;
+    char stime[32];
+
+    for (i=0; i<FILES_PER_ITERATION;i++) {
+        if (opt_write) {
+            sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
+            encode_file(INPUT_FILE,output_file);
+        } else {
+            encode_file(INPUT_FILE,NULL);
+        }
+    }
+
+    ltime = time(NULL);
+    localtime_r(&ltime, &result);
+    strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
+    /* asctime_r(&result, stime); */
+
+    printf("%s: Worker %ld finished.\n", stime, data->number);
+}

tests/grib_ecc-604.sh

@@ -0,0 +1,87 @@
+#!/bin/sh
+# Copyright 2005-2018 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.sh
+
+label="grib_ecc-604"
+temp_dir=tempdir.${label}
+
+NUM_THREADS=3
+NUM_ITER=10
+OUTPUT=output
+
+validate()
+{
+    echo "Checking every output file is identical..."
+    # Get checksum of first file
+    ck1=`cksum $OUTPUT/output_file_0-0.grib | awk '{print $1}'`
+    set +x
+    # Get checksum of all of them and sort unique
+    res=`cksum $OUTPUT/output_file_* | awk '{print $1}' | sort -u`
+    set -x
+    # Should be the same as the first
+    [ "$res" = "$ck1" ]
+}
+process()
+{
+    input=$1 # The input GRIB file
+
+    # Test 01: Clone + output
+    # ------------------------
+    rm -fr $OUTPUT;   mkdir -p $OUTPUT
+    time ${test_dir}/grib_ecc-604 -c -w par $input $NUM_THREADS $NUM_ITER
+    validate
+
+    # Test 02: No clone + output
+    # --------------------------
+    rm -fr $OUTPUT;   mkdir -p $OUTPUT
+    time ${test_dir}/grib_ecc-604 -w par $input $NUM_THREADS $NUM_ITER
+    validate
+
+    # Test 03: Clone + dump + no output
+    # ---------------------------------
+    rm -fr $OUTPUT
+    time ${test_dir}/grib_ecc-604 -c -d par $input $NUM_THREADS $NUM_ITER
+    # Nothing to validate as there is no output
+}
+###################################################
+rm -fr $temp_dir
+mkdir -p $temp_dir
+cd $temp_dir
+
+GRIB1_INPUTS="
+  $ECCODES_SAMPLES_PATH/gg_sfc_grib1.tmpl
+  ${data_dir}/gen_bitmap.grib
+  ${data_dir}/spectral_complex.grib1
+  ${data_dir}/gen_ext.grib
+  ${data_dir}/gen.grib
+  ${data_dir}/gen_ext_spd_2.grib"
+
+GRIB2_INPUTS="
+  $ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl
+  ${data_dir}/reduced_gaussian_sub_area.grib2
+  ${data_dir}/test_file.grib2
+  ${data_dir}/sample.grib2"
+
+if [ $HAVE_JPEG -eq 1 ]; then
+    echo "Adding extra files (HAVE_JPEG=1)"
+    GRIB2_INPUTS="${data_dir}/jpeg.grib2 ${data_dir}/reduced_gaussian_surface_jpeg.grib2 "$GRIB2_INPUTS
+fi
+if [ $HAVE_AEC -eq 1 ]; then
+    GRIB2_INPUTS=$GRIB2_INPUTS" ${data_dir}/ccsds.grib2 "
+fi
+
+for gf in $GRIB1_INPUTS $GRIB2_INPUTS; do
+    process $gf
+done
+
+# Clean up
+cd $test_dir
+rm -fr $temp_dir

tools/bufr_filter

@@ -1,29 +1,27 @@
 #!/bin/sh
-
+set -u
 script_dir=`dirname $0`
-EMOSLIB_TOOL=emoslib_bufr_filter
+EMOSLIB_TOOL=bufrdc_emoslib_bufr_filter
 ECCODES_TOOL=codes_bufr_filter
 result=0 # return code from function
 ERR_TOOL_NOT_FOUND=666
 
+the_tool=$ECCODES_TOOL
 is_emoslib=0
 
 #########################################################
 # Arguments:
-#  the executable name
+#  arguments from the script command line
 # Return Value:
-#  sets the global variable 'result'
+#  the global variable 'result' holds the exit code
 try_tool()
 {
-    the_tool=$1
-    the_args=$args
-
     if [ -f "${script_dir}/$the_tool" ]; then
-        ${script_dir}/$the_tool $the_args
+        ${script_dir}/$the_tool "${@}"
         result=$?
     else
         if command -v $the_tool >/dev/null 2>&1; then
-            $the_tool $the_args
+            $the_tool "${@}"
             result=$?
         else
             #echo "Could not find $the_tool. Return error"
@@ -36,37 +34,34 @@ try_tool()
 # Deal with case where no arguments are provided e.g. usage
 if [ $# -eq 0 ]; then
     # Give priority to ecCodes over emoslib
-    try_tool $ECCODES_TOOL
+    the_tool=$ECCODES_TOOL
+    try_tool "${@}"
     if [ $result -eq $ERR_TOOL_NOT_FOUND ]; then
-        try_tool $EMOSLIB_TOOL
+        the_tool=$EMOSLIB_TOOL
+        try_tool "${@}"
     fi
     exit 0
 fi
 
 # Now process arguments. The "-i" switch is specific to emoslib
-args="$@"
 for i in "$@" ; do
-   case $i in
+   if [ "$i" = "-i" ]; then is_emoslib=1; fi
-     -i) is_emoslib=1; shift ;;
-     *)  shift ;;
-   esac
 done
 
 #########################################################
-# set -x
 if [ $is_emoslib -eq 1 ]; then
-    pkg=emoslib
+    pkg='emoslib/bufrdc'
-    tool=$EMOSLIB_TOOL
+    the_tool=$EMOSLIB_TOOL
-    try_tool $tool
+    try_tool "${@}"
 else
-    pkg=ecCodes
+    pkg='ecCodes'
-    tool=$ECCODES_TOOL
+    the_tool=$ECCODES_TOOL
-    try_tool $tool
+    try_tool "${@}"
 fi
 if [ $result -eq $ERR_TOOL_NOT_FOUND ]; then
-    echo "ERROR: Could not find the executable: $tool. Aborting!"  2>&1
+    echo "ERROR: Could not find the executable: $the_tool. Aborting!"  2>&1
-    echo "       The arguments you passed in are relevant to $pkg" 2>&1
+    echo "       The arguments you passed in are relevant to $pkg." 2>&1
-    echo "       Please make sure you have $pkg installed in your path"  2>&1
+    echo "       Please make sure you have $pkg installed in your path."  2>&1
     exit 1
 fi
 exit $result