core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECC-604: Improve multi-threaded performance of ecCodes

This commit is contained in:
Shahram Najm 2019-09-16 15:48:52 +01:00
parent 791062b53b e2743bb76a
commit cf9d512ab7
13 changed files with 942 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
examples/C/grib_pthreads.c
View File

@ -15,15 +15,27 @@
#include "eccodes.h"
#define NUM_THREADS 4
/* Return 0 if numbers considered equal, otherwise 1 */
static int compare_doubles(double a,double b,double tolerance)
{
int ret=0;
double d=fabs(a-b);
if (d > tolerance) {
ret=1;
}
return ret;
}
static void* process_grib(void* threadID)
{
const long tid = (long)threadID;
size_t str_len = 20;
size_t str_len = 20, i = 0;
long indicatorOfUnitOfTimeRange = 1, step = 0;
char mystring[100];
double* values = NULL;
size_t values_len = 0;
double min=0,max=0;
double min=0,max=0,avg=0;
const double tol = 1e-6;
double pv[4]={1,2,3,4};
const size_t pvsize=4;
ProductKind prod_kind = 0;
@ -49,10 +61,20 @@ static void* process_grib(void* threadID)
CODES_CHECK(codes_get_size(h,"values",&values_len),0);
values = (double*)malloc(values_len*sizeof(double));
CODES_CHECK(codes_get_double_array(h, "values", values, &values_len),0);
for (i=0;i<values_len;i++) {
if (i%2) values[i] *= 0.94;
else if (i%3) values[i] *= 0.84;
}
GRIB_CHECK(grib_set_double_array(h,"values",values,values_len),0);
free(values);
CODES_CHECK(codes_get_double(h, "min", &min),0);
CODES_CHECK(codes_get_double(h, "max", &max),0);
CODES_CHECK(codes_get_double(h, "avg", &avg),0);
printf("Thread %ld: min=%g max=%g avg=%g\n", tid, min, max, avg);
assert( compare_doubles(min, 0.84, tol)==0 );
assert( compare_doubles(max, 1.00, tol)==0 );
assert( compare_doubles(avg, 0.916774, tol)==0 );
codes_handle_delete(h);
pthread_exit(NULL);
@ -67,6 +89,7 @@ int main(int argc, char** argv)
printf("Creating thread %ld\n", i);
error = pthread_create(&threads[i], NULL, process_grib, (void *)i);
if (error) {
assert(0);
return 1;
}
}

40
src/action.c
View File

@ -62,6 +62,31 @@ static void init(grib_action_class *c)
GRIB_MUTEX_UNLOCK(&mutex1);
}
#if 0
/* A non-recursive version */
static void init(grib_action_class *c)
{
if (!c) return;
GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
GRIB_MUTEX_LOCK(&mutex1);
if(!c->inited)
{
if(c->super) {
grib_action_class *g = *(c->super);
if (g && !g->inited) {
Assert(g->super == NULL);
g->init_class(g);
g->inited = 1;
}
}
c->init_class(c);
c->inited = 1;
}
GRIB_MUTEX_UNLOCK(&mutex1);
}
#endif
void grib_dump(grib_action* a, FILE* f, int l)
{
grib_action_class *c = a->cclass;
@ -119,10 +144,11 @@ int grib_create_accessor(grib_section* p, grib_action* a, grib_loader* h)
{
if(c->create_accessor) {
int ret;
GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
GRIB_MUTEX_LOCK(&mutex1);
/* ECC-604: Do not lock excessively */
/*GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);*/
/*GRIB_MUTEX_LOCK(&mutex1);*/
ret=c->create_accessor(p, a, h);
GRIB_MUTEX_UNLOCK(&mutex1);
/*GRIB_MUTEX_UNLOCK(&mutex1);*/
return ret;
}
c = c->super ? *(c->super) : NULL;
@ -136,20 +162,20 @@ int grib_action_notify_change( grib_action* a, grib_accessor *observer, grib_acc
{
grib_action_class *c = a->cclass;
GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
GRIB_MUTEX_LOCK(&mutex1);
/*GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);*/
/*GRIB_MUTEX_LOCK(&mutex1);*/
init(c);
while(c)
{
if(c->notify_change) {
int result = c->notify_change(a,observer,observed);
GRIB_MUTEX_UNLOCK(&mutex1);
/*GRIB_MUTEX_UNLOCK(&mutex1);*/
return result;
}
c = c->super ? *(c->super) : NULL;
}
GRIB_MUTEX_UNLOCK(&mutex1);
/*GRIB_MUTEX_UNLOCK(&mutex1);*/
Assert(0);
return 0;
}

21
src/action_class_when.c
View File

@ -82,6 +82,11 @@ static void init_class(grib_action_class* c)
}
/* END_CLASS_IMP */
/* The check on self->loop can only be done in non-threaded mode */
#if defined(DEBUG) && GRIB_PTHREADS == 0 && GRIB_OMP_THREADS == 0
#define CHECK_LOOP 1
#endif
grib_action* grib_action_create_when( grib_context* context,
grib_expression* expression,
grib_action* block_true,grib_action* block_false)
@ -101,7 +106,6 @@ grib_action* grib_action_create_when( grib_context* context,
a->block_true = block_true;
a->block_false = block_false;
sprintf(name,"_when%p",(void*)expression);
act->name = grib_context_strdup_persistent(context,name);
@ -152,12 +156,19 @@ static void dump(grib_action* act, FILE* f, int lvl)
printf("\n");
}
#ifdef CHECK_LOOP
#define SET_LOOP(self,v) self->loop=v;
#else
#define SET_LOOP(self,v)
#endif
static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor* observed)
{
grib_action_when* self = (grib_action_when*) a;
grib_action *b = NULL;
int ret = GRIB_SUCCESS;
long lres;
/* ECC-974: observed->parent will change as a result of the execute
* so must store the handle once here (in 'hand') rather than call
* grib_handle_of_accessor(observed) later
@ -166,7 +177,7 @@ static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor*
if ((ret = grib_expression_evaluate_long(hand, self->expression,&lres))
!= GRIB_SUCCESS) return ret;
#ifdef DEBUG
#ifdef CHECK_LOOP
if(self->loop)
{
printf("LOOP detected...\n");
@ -176,7 +187,7 @@ static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor*
return ret;
}
#endif
self->loop = 1;
SET_LOOP(self, 1);
if(lres)
b=self->block_true;
@ -186,13 +197,13 @@ static int notify_change(grib_action* a, grib_accessor* observer,grib_accessor*
while(b) {
ret = grib_action_execute(b,hand);
if(ret != GRIB_SUCCESS) {
self->loop = 0;
SET_LOOP(self, 0);
return ret;
}
b = b->next;
}
self->loop = 0;
SET_LOOP(self, 0);
return GRIB_SUCCESS;
}

2
src/grib_context.c
View File

@ -623,7 +623,9 @@ char *grib_context_full_defs_path(grib_context* c,const char* basename)
if(*basename == '/' || *basename == '.') {
return (char*)basename;
} else {
GRIB_MUTEX_LOCK(&mutex_c); /* See ECC-604 */
fullpath=(grib_string_list*)grib_trie_get(c->def_files,basename);
GRIB_MUTEX_UNLOCK(&mutex_c);
if (fullpath!=NULL) {
return fullpath->value;
}

266
src/grib_dumper.c
View File

@ -11,204 +11,230 @@
/**************************************
* Enrico Fucile
**************************************/
#include "grib_api_internal.h"
#if GRIB_PTHREADS
static pthread_once_t once = PTHREAD_ONCE_INIT;
static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
static void init_mutex() {
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr,PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&mutex,&attr);
pthread_mutexattr_destroy(&attr);
}
#elif GRIB_OMP_THREADS
static int once = 0;
static omp_nest_lock_t mutex;
static void init_mutex()
{
GRIB_OMP_CRITICAL(lock_dumper_c)
{
if (once == 0)
{
omp_init_nest_lock(&mutex);
once = 1;
}
}
}
#endif
static void init_dumpers(grib_dumper_class* c,grib_dumper* d)
{
if(c) {
grib_dumper_class *s = c->super ? *(c->super) : NULL;
if(!c->inited)
{
if(c->init_class) c->init_class(c);
c->inited = 1;
GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
GRIB_MUTEX_LOCK(&mutex);
if(c) {
grib_dumper_class *s = c->super ? *(c->super) : NULL;
if(!c->inited)
{
if(c->init_class) c->init_class(c);
c->inited = 1;
}
init_dumpers(s,d);
if(c->init) c->init(d);
}
init_dumpers(s,d);
if(c->init) c->init(d);
}
GRIB_MUTEX_UNLOCK(&mutex);
}
void grib_init_dumper(grib_dumper* d)
{
init_dumpers(d->cclass,d);
init_dumpers(d->cclass,d);
}
void grib_dumper_delete(grib_dumper* d)
{
grib_dumper_class *c = d->cclass;
grib_context *ctx = d->handle->context;
while(c)
{
grib_dumper_class *s = c->super ? *(c->super) : NULL;
if(c->destroy) c->destroy(d);
c = s;
}
grib_context_free(ctx,d);
grib_dumper_class *c = d->cclass;
grib_context *ctx = d->handle->context;
while(c)
{
grib_dumper_class *s = c->super ? *(c->super) : NULL;
if(c->destroy) c->destroy(d);
c = s;
}
grib_context_free(ctx,d);
}
void grib_dump_long(grib_dumper* d, grib_accessor* a, const char* comment)
{
grib_dumper_class *c = d->cclass;
while(c)
{
if(c->dump_long)
grib_dumper_class *c = d->cclass;
while(c)
{
c->dump_long(d, a, comment);
return;
if(c->dump_long)
{
c->dump_long(d, a, comment);
return;
}
c = c->super ? *(c->super) : NULL;
}
c = c->super ? *(c->super) : NULL;
}
Assert(0);
Assert(0);
}
void grib_dump_double(grib_dumper* d,grib_accessor* a,const char* comment)
{
grib_dumper_class *c = d->cclass;
while(c)
{
if(c->dump_double)
grib_dumper_class *c = d->cclass;
while(c)
{
c->dump_double(d, a, comment);
return;
if(c->dump_double)
{
c->dump_double(d, a, comment);
return;
}
c = c->super ? *(c->super) : NULL;
}
c = c->super ? *(c->super) : NULL;
}
Assert(0);
Assert(0);
}
void grib_dump_string(grib_dumper* d,grib_accessor* a,const char* comment)
{
grib_dumper_class *c = d->cclass;
while(c)
{
if(c->dump_string)
grib_dumper_class *c = d->cclass;
while(c)
{
c->dump_string(d, a, comment);
return;
if(c->dump_string)
{
c->dump_string(d, a, comment);
return;
}
c = c->super ? *(c->super) : NULL;
}
c = c->super ? *(c->super) : NULL;
}
Assert(0);
Assert(0);
}
void grib_dump_string_array(grib_dumper* d,grib_accessor* a,const char* comment)
{
grib_dumper_class *c = d->cclass;
while(c)
{
if(c->dump_string_array)
grib_dumper_class *c = d->cclass;
while(c)
{
c->dump_string_array(d, a, comment);
return;
if(c->dump_string_array)
{
c->dump_string_array(d, a, comment);
return;
}
c = c->super ? *(c->super) : NULL;
}
c = c->super ? *(c->super) : NULL;
}
Assert(0);
Assert(0);
}
void grib_dump_label(grib_dumper* d,grib_accessor* a,const char* comment)
{
grib_dumper_class *c = d->cclass;
while(c)
{
if(c->dump_label)
grib_dumper_class *c = d->cclass;
while(c)
{
c->dump_label(d, a, comment);
return;
if(c->dump_label)
{
c->dump_label(d, a, comment);
return;
}
c = c->super ? *(c->super) : NULL;
}
c = c->super ? *(c->super) : NULL;
}
Assert(0);
Assert(0);
}
void grib_dump_bytes(grib_dumper* d,grib_accessor* a,const char* comment)
{
grib_dumper_class *c = d->cclass;
while(c)
{
if(c->dump_bytes)
grib_dumper_class *c = d->cclass;
while(c)
{
c->dump_bytes(d, a, comment);
return;
if(c->dump_bytes)
{
c->dump_bytes(d, a, comment);
return;
}
c = c->super ? *(c->super) : NULL;
}
c = c->super ? *(c->super) : NULL;
}
Assert(0);
Assert(0);
}
void grib_dump_bits(grib_dumper* d,grib_accessor* a,const char* comment)
{
grib_dumper_class *c = d->cclass;
while(c)
{
if(c->dump_bits)
grib_dumper_class *c = d->cclass;
while(c)
{
c->dump_bits(d, a, comment);
return;
if(c->dump_bits)
{
c->dump_bits(d, a, comment);
return;
}
c = c->super ? *(c->super) : NULL;
}
c = c->super ? *(c->super) : NULL;
}
Assert(0);
Assert(0);
}
void grib_dump_section(grib_dumper* d,grib_accessor* a,grib_block_of_accessors* block)
{
grib_dumper_class *c = d->cclass;
while(c)
{
if(c->dump_section)
grib_dumper_class *c = d->cclass;
while(c)
{
c->dump_section(d, a, block);
return;
if(c->dump_section)
{
c->dump_section(d, a, block);
return;
}
c = c->super ? *(c->super) : NULL;
}
c = c->super ? *(c->super) : NULL;
}
Assert(0);
Assert(0);
}
void grib_dump_values(grib_dumper* d,grib_accessor* a)
{
grib_dumper_class *c = d->cclass;
while(c)
{
if(c->dump_values)
grib_dumper_class *c = d->cclass;
while(c)
{
c->dump_values(d, a);
return;
if(c->dump_values)
{
c->dump_values(d, a);
return;
}
c = c->super ? *(c->super) : NULL;
}
c = c->super ? *(c->super) : NULL;
}
Assert(0);
Assert(0);
}
void grib_dump_header(grib_dumper* d,grib_handle* h)
{
grib_dumper_class *c = d->cclass;
while(c)
{
if(c->header)
grib_dumper_class *c = d->cclass;
while(c)
{
c->header(d,h);
return;
if(c->header)
{
c->header(d,h);
return;
}
c = c->super ? *(c->super) : NULL;
}
c = c->super ? *(c->super) : NULL;
}
}
void grib_dump_footer(grib_dumper* d,grib_handle* h)
{
grib_dumper_class *c = d->cclass;
while(c)
{
if(c->footer)
grib_dumper_class *c = d->cclass;
while(c)
{
c->footer(d,h);
return;
if(c->footer)
{
c->footer(d,h);
return;
}
c = c->super ? *(c->super) : NULL;
}
c = c->super ? *(c->super) : NULL;
}
}

40
src/grib_iterator.c
View File

@ -15,6 +15,35 @@
***************************************************************************/
#include "grib_api_internal.h"
#if GRIB_PTHREADS
static pthread_once_t once = PTHREAD_ONCE_INIT;
static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
static void init_mutex() {
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr,PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&mutex,&attr);
pthread_mutexattr_destroy(&attr);
}
#elif GRIB_OMP_THREADS
static int once = 0;
static omp_nest_lock_t mutex;
static void init_mutex()
{
GRIB_OMP_CRITICAL(lock_iterator_c)
{
if (once == 0)
{
omp_init_nest_lock(&mutex);
once = 1;
}
}
}
#endif
int grib_get_data(grib_handle* h,double* lats, double* lons,double* values)
{
int err=0;
@ -71,7 +100,6 @@ int grib_iterator_previous(grib_iterator *i,double* lat,double* lon,double* valu
return 0;
}
int grib_iterator_reset(grib_iterator *i)
{
grib_iterator_class *c = i->cclass;
@ -85,12 +113,9 @@ int grib_iterator_reset(grib_iterator *i)
return 0;
}
/* For this one, ALL init are called */
static int init_iterator(grib_iterator_class* c,grib_iterator* i, grib_handle *h, grib_arguments* args)
{
if(c) {
int ret = GRIB_SUCCESS;
grib_iterator_class *s = c->super ? *(c->super) : NULL;
@ -110,7 +135,12 @@ static int init_iterator(grib_iterator_class* c,grib_iterator* i, grib_handle *h
int grib_iterator_init(grib_iterator* i, grib_handle *h, grib_arguments* args)
{
return init_iterator(i->cclass,i,h,args);
int r = 0;
GRIB_MUTEX_INIT_ONCE(&once,&init_mutex);
GRIB_MUTEX_LOCK(&mutex);
r = init_iterator(i->cclass,i,h,args);
GRIB_MUTEX_UNLOCK(&mutex);
return r;
}
/* For this one, ALL destroy are called */

8
src/grib_parse_utils.c
View File

@ -9,7 +9,7 @@
*/
/***************************************************************************
* Jean Baptiste Filippi - 01.11.2005 *
* Jean Baptiste Filippi - 01.11.2005 *
* Enrico Fucile
* *
***************************************************************************/
@ -729,16 +729,16 @@ static grib_action* grib_parse_stream(grib_context* gc, const char* filename)
grib_concept_value* grib_parse_concept_file( grib_context* gc,const char* filename)
{
GRIB_MUTEX_INIT_ONCE(&once,&init);
GRIB_MUTEX_LOCK(&mutex_concept);
GRIB_MUTEX_LOCK(&mutex_file);
gc = gc ? gc : grib_context_get_default();
grib_parser_context = gc;
if(parse(gc,filename) == 0) {
GRIB_MUTEX_UNLOCK(&mutex_concept);
GRIB_MUTEX_UNLOCK(&mutex_file);
return grib_parser_concept;
} else {
GRIB_MUTEX_UNLOCK(&mutex_concept);
GRIB_MUTEX_UNLOCK(&mutex_file);
return NULL;
}
}

18
tests/CMakeLists.txt
View File

@ -254,6 +254,24 @@ if( ENABLE_EXTRA_TESTS AND HAVE_ECCODES_THREADS )
TYPE SCRIPT
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_encode_pthreads.sh
)
foreach( test grib_ecc-604 grib_ecc-604-encode bufr_ecc-604 )
ecbuild_add_executable( TARGET ${test}
NOINSTALL
SOURCES ${test}.c
LIBS eccodes
)
endforeach()
ecbuild_add_test( TARGET eccodes_t_grib_ecc-604
TYPE SCRIPT
TEST_DEPENDS eccodes_download_gribs
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_ecc-604.sh
)
ecbuild_add_test( TARGET eccodes_t_bufr_ecc-604
TYPE SCRIPT
TEST_DEPENDS eccodes_download_bufrs
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bufr_ecc-604.sh
)
endif()
ecbuild_add_test( TARGET eccodes_t_grib_to_netcdf

176
tests/bufr_ecc-604.c Normal file
View File

@ -0,0 +1,176 @@
/*
* Test for ECC-604: Each thread creates a new BUFR handle, optionaly clone it and/or write it out
*/
#include <time.h>
#include <pthread.h>
#include <assert.h>
#include <unistd.h>
#include "eccodes.h"
#include "grib_api_internal.h"
/* These are passed in via argv */
static size_t NUM_THREADS = 0;
static size_t FILES_PER_ITERATION = 0;
static char* INPUT_FILE = NULL;
int opt_dump = 0; /* If 1 then dump handle to /dev/null */
int opt_clone = 0; /* If 1 then clone source handle */
int opt_write = 0; /* If 1 write handle to file */
static int encode_file(char *template_file, char *output_file)
{
FILE *in, *out;
codes_handle *source_handle = NULL;
const void *buffer = NULL;
size_t size = 0;
int err = 0;
long numSubsets = 0;
in = fopen(template_file,"r"); assert(in);
if (opt_write) {
out = fopen(output_file,"w"); assert(out);
}
/* loop over the messages in the source BUFR and clone them */
while ((source_handle = codes_handle_new_from_file(NULL, in, PRODUCT_BUFR, &err)) != NULL || err != CODES_SUCCESS) {
codes_handle *h = source_handle;
if (opt_clone) {
h = codes_handle_clone(source_handle); assert(h);
}
CODES_CHECK(codes_get_long(h,"numberOfSubsets", &numSubsets),0);
CODES_CHECK(codes_set_long(h, "unpack", 1),0);
CODES_CHECK(codes_get_message(h,&buffer,&size),0);
if (opt_write) {
if(fwrite(buffer,1,size,out) != size) {
perror(output_file);
return 1;
}
}
if (opt_dump) {
FILE *devnull = fopen("/dev/null", "w");
grib_dumper* dumper = NULL;
const char* dumper_name = "bufr_simple";
unsigned long dump_flags = CODES_DUMP_FLAG_ALL_DATA;
/* codes_dump_content(source_handle,devnull, "json", 1024, NULL); */ /* JSON dump with all attributes */
dumper=grib_dump_content_with_dumper(source_handle, dumper, devnull, dumper_name, dump_flags, NULL);
assert(dumper);
fclose(devnull);
}
codes_handle_delete(source_handle);
if(opt_clone) codes_handle_delete(h);
}
if (opt_write) fclose(out);
fclose(in);
return 0;
}
void do_stuff(void *data);
/* Structure for passing data to threads */
struct v {
size_t number;
char *data;
};
void *runner(void *ptr); /* the thread */
int main(int argc, char **argv)
{
size_t i;
int thread_counter = 0;
int parallel=1, index=0, c=0;
const char* prog = argv[0];
char* mode;
if (argc<5 || argc>7) {
fprintf(stderr, "Usage:\n\t%s [options] seq file numRuns numIter\nOr\n\t%s [options] par file numThreads numIter\n", prog, prog);
return 1;
}
while ((c = getopt (argc, argv, "dcw")) != -1) {
switch (c) {
case 'd': opt_dump=1; break;
case 'c': opt_clone=1; break;
case 'w': opt_write=1; break;
}
}
index = optind;
mode = argv[index];
INPUT_FILE = argv[index+1];
NUM_THREADS = atol(argv[index+2]);
FILES_PER_ITERATION = atol(argv[index+3]);
if (strcmp(mode,"seq")==0) {
parallel = 0;
}
if (parallel) {
printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
printf("Options: dump=%d, clone=%d, write=%d\n", opt_dump, opt_clone, opt_write);
} else {
printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
}
{
pthread_t workers[NUM_THREADS];
for (i = 0; i < NUM_THREADS; i++) {
struct v *data = (struct v *) malloc(sizeof(struct v));
data->number = i;
data->data = NULL;
if (parallel) {
/* Now we will create the thread passing it data as an argument */
pthread_create(&workers[thread_counter], NULL, runner, data);
/*pthread_join(workers[thread_counter], NULL);*/
thread_counter++;
} else {
do_stuff(data);
}
}
if (parallel) {
for (i = 0; i < NUM_THREADS; i++) {
pthread_join(workers[i], NULL);
}
}
}
return 0;
}
void *runner(void *ptr)
{
do_stuff(ptr);
pthread_exit(0);
}
void do_stuff(void *ptr)
{
/* Cast argument to struct v pointer */
struct v *data = ptr;
size_t i;
char output_file[50];
time_t ltime;
struct tm result;
char stime[32];
for (i=0; i<FILES_PER_ITERATION;i++) {
if (opt_write) {
sprintf(output_file,"output/output_file_%ld-%ld.bufr", data->number, i);
encode_file(INPUT_FILE,output_file);
} else {
encode_file(INPUT_FILE,NULL);
}
}
ltime = time(NULL);
localtime_r(&ltime, &result);
strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
/* asctime_r(&result, stime); */
printf("%s: Worker %ld finished.\n", stime, data->number);
}

68
tests/bufr_ecc-604.sh Executable file
View File

@ -0,0 +1,68 @@
#!/bin/sh
# Copyright 2005-2018 ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
#
. ./include.sh
label="bufr_ecc-604"
temp_dir=tempdir.${label}
NUM_THREADS=3
NUM_ITER=10
OUTPUT=output
validate()
{
echo "Checking every output file is identical..."
# Get checksum of first file
ck1=`cksum $OUTPUT/output_file_0-0.bufr | awk '{print $1}'`
set +x
# Get checksum of all of them and sort unique
res=`cksum $OUTPUT/output_file_* | awk '{print $1}' | sort -u`
set -x
# Should be the same as the first
[ "$res" = "$ck1" ]
}
process()
{
input=$1 # The input BUFR file
# Test 01: Clone + output
# ------------------------
rm -fr $OUTPUT; mkdir -p $OUTPUT
time ${test_dir}/bufr_ecc-604 -c -w par $input $NUM_THREADS $NUM_ITER
validate
# Test 02: No clone + output
# --------------------------
rm -fr $OUTPUT; mkdir -p $OUTPUT
time ${test_dir}/bufr_ecc-604 -w par $input $NUM_THREADS $NUM_ITER
validate
# Test 03: Clone + no output
# ---------------------------
rm -fr $OUTPUT
time ${test_dir}/bufr_ecc-604 -c par $input $NUM_THREADS $NUM_ITER
# Nothing to validate as there is no output
}
###################################################
rm -fr $temp_dir
mkdir -p $temp_dir
cd $temp_dir
bufr_files=`cat ${data_dir}/bufr/bufr_data_files.txt`
for bf in ${bufr_files}; do
b=${data_dir}/bufr/$bf
echo "Doing $b"
process $b
done
# Clean up
cd $test_dir
rm -fr $temp_dir

145
tests/grib_ecc-604-encode.c Normal file
View File

@ -0,0 +1,145 @@
/*
* Test for ECC-604: GRIB decoding/encoding sequentially and parallel with POSIX threads
*/
#include <time.h>
#include <pthread.h>
#include <assert.h>
#include <unistd.h>
#include "grib_api.h"
/* These are passed in via argv */
static size_t NUM_THREADS = 0;
static size_t FILES_PER_ITERATION = 0;
static char* INPUT_FILE = NULL;
int opt_dump = 0; /* If 1 then dump handle to /dev/null */
int opt_clone = 0; /* If 1 then clone source handle */
int opt_write = 0; /* If 1 write handle to file */
static int encode_values(grib_handle* h, char *output_file)
{
double *values;
const size_t DIM = 1000;
size_t size = DIM * DIM;
size_t i = 0;
values = (double*)malloc(size*sizeof(double));
for (i=0; i<size; ++i) {
double v = i;
if (i % DIM == 0) v = 0;
values[i] = v;
}
GRIB_CHECK(grib_set_long(h,"bitsPerValue",16),0);
GRIB_CHECK(grib_set_double_array(h,"values",values,size), 0);
free(values);
return GRIB_SUCCESS;
}
void do_encode(void *data);
/* Structure for passing data to threads */
struct v {
size_t number;
char *data;
};
void *runner(void *ptr); /* the thread */
int main(int argc, char **argv)
{
size_t i;
int thread_counter = 0;
int parallel=1, index=0, c=0;
const char* prog = argv[0];
char* mode;
if (argc<5 || argc>7) {
fprintf(stderr, "Usage:\n\t%s [options] seq sample numRuns numIter\nOr\n\t%s [options] par sample numThreads numIter\n", prog, prog);
return 1;
}
while ((c = getopt (argc, argv, "dcw")) != -1) {
switch (c) {
case 'd': opt_dump=1; break;
case 'c': opt_clone=1; break;
case 'w': opt_write=1; break;
}
}
index = optind;
mode = argv[index];
INPUT_FILE = argv[index+1]; /* Has to be the name of a sample file (without tmpl extension) */
NUM_THREADS = atol(argv[index+2]);
FILES_PER_ITERATION = atol(argv[index+3]);
if (strcmp(mode,"seq")==0) {
parallel = 0;
}
if (parallel) {
printf("Running parallel in %ld threads. %ld iterations (prod=%ld)\n", NUM_THREADS, FILES_PER_ITERATION, NUM_THREADS*FILES_PER_ITERATION);
printf("Options: dump=%d, clone=%d, write=%d\n", opt_dump, opt_clone, opt_write);
} else {
printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
}
{
pthread_t workers[NUM_THREADS];
for (i = 0; i < NUM_THREADS; i++) {
struct v *data = (struct v *) malloc(sizeof(struct v));
data->number = i;
data->data = NULL;
if (parallel) {
/* Now we will create the thread passing it data as an argument */
pthread_create(&workers[thread_counter], NULL, runner, data);
thread_counter++;
} else {
do_encode(data);
}
}
if (parallel) {
for (i = 0; i < NUM_THREADS; i++) {
pthread_join(workers[i], NULL);
}
}
}
return 0;
}
void *runner(void *ptr)
{
do_encode(ptr);
pthread_exit(0);
}
void do_encode(void *ptr)
{
/* Cast argument to struct v pointer */
struct v *data = ptr;
size_t i;
char output_file[50];
time_t ltime;
struct tm result;
char stime[32];
grib_handle *hs = NULL;
hs = grib_handle_new_from_samples(0, INPUT_FILE);
for (i=0; i<FILES_PER_ITERATION;i++) {
grib_handle *h = grib_handle_clone(hs);
if (opt_write) {
sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
encode_values(h,output_file);
} else {
encode_values(h,NULL);
}
grib_handle_delete(h);
}
ltime = time(NULL);
localtime_r(&ltime, &result);
strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
/* asctime_r(&result, stime); */
printf("%s: Worker %ld finished.\n", stime, data->number);
grib_handle_delete(hs);
}

187
tests/grib_ecc-604.c Normal file
View File

@ -0,0 +1,187 @@
/*
* Test for ECC-604: GRIB decoding/encoding sequentially and parallel with POSIX threads
*/
#include <time.h>
#include <pthread.h>
#include <assert.h>
#include <unistd.h>
#include "grib_api.h"
/* These are passed in via argv */
static size_t NUM_THREADS = 0;
static size_t FILES_PER_ITERATION = 0;
static char* INPUT_FILE = NULL;
int opt_dump = 0; /* If 1 then dump handle to /dev/null */
int opt_clone = 0; /* If 1 then clone source handle */
int opt_write = 0; /* If 1 write handle to file */
static int encode_file(char *template_file, char *output_file)
{
FILE *in, *out;
grib_handle *source_handle = NULL;
const void *buffer = NULL;
size_t size = 0;
int err = 0;
double *values;
in = fopen(template_file,"r"); assert(in);
if (opt_write) {
out = fopen(output_file,"w"); assert(out);
}
/* loop over the messages in the source GRIB and clone them */
while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
int i;
size_t values_len = 0;
size_t str_len = 20;
grib_handle *h = source_handle;
if (opt_clone) {
h = grib_handle_clone(source_handle); assert(h);
}
GRIB_CHECK(grib_get_size(h, "values", &values_len),0);
values = (double*)malloc(values_len*sizeof(double));
GRIB_CHECK(grib_get_double_array(h, "values", values, &values_len),0);
for (i=0;i<values_len;i++) {
values[i] *= 0.9;
}
GRIB_CHECK(grib_set_string(h,"stepUnits", "s", &str_len),0);
GRIB_CHECK(grib_set_long(h, "startStep", 43200), 0);
GRIB_CHECK(grib_set_long(h, "endStep", 86400), 0);
GRIB_CHECK(grib_set_long(h, "bitsPerValue", 16),0);
/* set data values */
GRIB_CHECK(grib_set_double_array(h,"values",values,values_len),0);
GRIB_CHECK(grib_get_message(h,&buffer,&size),0);
if (opt_write) {
if(fwrite(buffer,1,size,out) != size) {
perror(output_file);
return 1;
}
}
if (opt_dump) {
FILE *devnull = fopen("/dev/null", "w");
grib_dump_content(source_handle,devnull, "debug", 0, NULL);
}
grib_handle_delete(source_handle);
if(opt_clone) grib_handle_delete(h);
free(values);
}
if (opt_write) fclose(out);
fclose(in);
return 0;
}
void do_stuff(void *data);
/* Structure for passing data to threads */
struct v {
size_t number;
char *data;
};
void *runner(void *ptr); /* the thread */
int main(int argc, char **argv)
{
size_t i;
int thread_counter = 0;
int parallel=1, index=0, c=0;
const char* prog = argv[0];
char* mode;
if (argc<5 || argc>8) {
fprintf(stderr, "Usage:\n\t%s [options] seq file numRuns numIter\nOr\n\t%s [options] par file numThreads numIter\n", prog, prog);
return 1;
}
while ((c = getopt (argc, argv, "dcw")) != -1) {
switch (c) {
case 'd': opt_dump=1; break;
case 'c': opt_clone=1; break;
case 'w': opt_write=1; break;
}
}
index = optind;
mode = argv[index];
INPUT_FILE = argv[index+1];
NUM_THREADS = atol(argv[index+2]);
FILES_PER_ITERATION = atol(argv[index+3]);
if (strcmp(mode,"seq")==0) {
parallel = 0;
}
if (parallel) {
printf("Running parallel in %ld threads. %ld iterations (prod=%ld)\n", NUM_THREADS, FILES_PER_ITERATION, NUM_THREADS*FILES_PER_ITERATION);
printf("Options: dump=%d, clone=%d, write=%d\n", opt_dump, opt_clone, opt_write);
} else {
printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
}
{
pthread_t workers[NUM_THREADS];
for (i = 0; i < NUM_THREADS; i++) {
struct v *data = (struct v *) malloc(sizeof(struct v));
data->number = i;
data->data = NULL;
if (parallel) {
/* Now we will create the thread passing it data as an argument */
pthread_create(&workers[thread_counter], NULL, runner, data);
/*pthread_join(workers[thread_counter], NULL);*/
thread_counter++;
} else {
do_stuff(data);
}
}
if (parallel) {
for (i = 0; i < NUM_THREADS; i++) {
pthread_join(workers[i], NULL);
}
}
}
return 0;
}
void *runner(void *ptr)
{
do_stuff(ptr);
pthread_exit(0);
}
void do_stuff(void *ptr)
{
/* Cast argument to struct v pointer */
struct v *data = ptr;
size_t i;
char output_file[50];
time_t ltime;
struct tm result;
char stime[32];
for (i=0; i<FILES_PER_ITERATION;i++) {
if (opt_write) {
sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
encode_file(INPUT_FILE,output_file);
} else {
encode_file(INPUT_FILE,NULL);
}
}
ltime = time(NULL);
localtime_r(&ltime, &result);
strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
/* asctime_r(&result, stime); */
printf("%s: Worker %ld finished.\n", stime, data->number);
}

87
tests/grib_ecc-604.sh Executable file
View File

@ -0,0 +1,87 @@
#!/bin/sh
# Copyright 2005-2018 ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
#
. ./include.sh
label="grib_ecc-604"
temp_dir=tempdir.${label}
NUM_THREADS=3
NUM_ITER=10
OUTPUT=output
validate()
{
echo "Checking every output file is identical..."
# Get checksum of first file
ck1=`cksum $OUTPUT/output_file_0-0.grib | awk '{print $1}'`
set +x
# Get checksum of all of them and sort unique
res=`cksum $OUTPUT/output_file_* | awk '{print $1}' | sort -u`
set -x
# Should be the same as the first
[ "$res" = "$ck1" ]
}
process()
{
input=$1 # The input GRIB file
# Test 01: Clone + output
# ------------------------
rm -fr $OUTPUT; mkdir -p $OUTPUT
time ${test_dir}/grib_ecc-604 -c -w par $input $NUM_THREADS $NUM_ITER
validate
# Test 02: No clone + output
# --------------------------
rm -fr $OUTPUT; mkdir -p $OUTPUT
time ${test_dir}/grib_ecc-604 -w par $input $NUM_THREADS $NUM_ITER
validate
# Test 03: Clone + dump + no output
# ---------------------------------
rm -fr $OUTPUT
time ${test_dir}/grib_ecc-604 -c -d par $input $NUM_THREADS $NUM_ITER
# Nothing to validate as there is no output
}
###################################################
rm -fr $temp_dir
mkdir -p $temp_dir
cd $temp_dir
GRIB1_INPUTS="
$ECCODES_SAMPLES_PATH/gg_sfc_grib1.tmpl
${data_dir}/gen_bitmap.grib
${data_dir}/spectral_complex.grib1
${data_dir}/gen_ext.grib
${data_dir}/gen.grib
${data_dir}/gen_ext_spd_2.grib"
GRIB2_INPUTS="
$ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl
${data_dir}/reduced_gaussian_sub_area.grib2
${data_dir}/test_file.grib2
${data_dir}/sample.grib2"
if [ $HAVE_JPEG -eq 1 ]; then
echo "Adding extra files (HAVE_JPEG=1)"
GRIB2_INPUTS="${data_dir}/jpeg.grib2 ${data_dir}/reduced_gaussian_surface_jpeg.grib2 "$GRIB2_INPUTS
fi
if [ $HAVE_AEC -eq 1 ]; then
GRIB2_INPUTS=$GRIB2_INPUTS" ${data_dir}/ccsds.grib2 "
fi
for gf in $GRIB1_INPUTS $GRIB2_INPUTS; do
process $gf
done
# Clean up
cd $test_dir
rm -fr $temp_dir