core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Testing: unified the 3 programs to test clone/write etc

This commit is contained in:
Shahram Najm 2018-07-25 14:48:42 +01:00
parent 834b230472
commit 63fddf86ff
5 changed files with 53 additions and 354 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/CMakeLists.txt
View File

@ -218,7 +218,7 @@ if( ENABLE_EXTRA_TESTS AND HAVE_ECCODES_THREADS )
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/grib_encode_pthreads.sh
)
foreach( test grib_ecc-604-1 grib_ecc-604-2 grib_ecc-604-3 )
foreach( test grib_ecc-604 )
ecbuild_add_executable( TARGET ${test}
NOINSTALL
SOURCES ${test}.c

162
tests/grib_ecc-604-2.c
View File

@ -1,162 +0,0 @@
/*
* Test for ECC-604: Each thread creates a new GRIB handle and writes it out.
* It does not clone the handle.
*/
#include <time.h>
#include <pthread.h>
#include <assert.h>
#include "grib_api.h"
/* These are passed in via argv */
static size_t NUM_THREADS = 0;
static size_t FILES_PER_ITERATION = 0;
static char* INPUT_FILE = NULL;
static int encode_file(char *template_file, char *output_file)
{
FILE *in, *out;
grib_handle *source_handle = NULL;
const void *buffer = NULL;
size_t size = 0;
int err = 0;
double *values;
in = fopen(template_file,"r"); assert(in);
out = fopen(output_file,"w"); assert(out);
/* loop over the messages in the source GRIB */
while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
int i;
size_t values_len = 0;
size_t str_len = 20;
/*GRIB_CHECK(grib_set_long(source_handle, "centre", 250),0);*/
GRIB_CHECK(grib_get_size(source_handle, "values", &values_len),0);
values = (double*)malloc(values_len*sizeof(double));
GRIB_CHECK(grib_get_double_array(source_handle, "values", values, &values_len),0);
for (i=0;i<values_len;i++) {
values[i] *= 0.9;
}
GRIB_CHECK(grib_set_string(source_handle,"stepUnits", "s", &str_len),0);
GRIB_CHECK(grib_set_long(source_handle, "startStep", 43200), 0);
GRIB_CHECK(grib_set_long(source_handle, "endStep", 86400), 0);
GRIB_CHECK(grib_set_long(source_handle, "bitsPerValue", 16),0);
/* set data values */
GRIB_CHECK(grib_set_double_array(source_handle,"values",values,values_len),0);
GRIB_CHECK(grib_get_message(source_handle,&buffer,&size),0);
if(fwrite(buffer,1,size,out) != size) {
perror(output_file);
return 1;
}
{
FILE *devnull = fopen("/dev/null", "w");
grib_dump_content(source_handle,devnull,"debug",0,NULL);
}
grib_handle_delete(source_handle);
free(values);
}
fclose(out);
fclose(in);
return 0;
}
void do_stuff(void *data);
/* Structure for passing data to threads */
struct v {
size_t number;
char *data;
};
void *runner(void *ptr); /* the thread */
int main(int argc, char **argv)
{
size_t i;
int thread_counter = 0;
int parallel = 1;
const char* prog = argv[0];
char* mode;
if (argc!=5) {
fprintf(stderr, "Usage:\n\t%s seq file numRuns numIter\nOr\n\t%s par file numThreads numIter\n", prog, prog);
return 1;
}
mode = argv[1];
INPUT_FILE= argv[2];
NUM_THREADS = atol(argv[3]);
FILES_PER_ITERATION = atol(argv[4]);
if (strcmp(mode,"seq")==0) {
parallel = 0;
}
if (parallel) {
printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
printf("Each thread creates a new GRIB handle and writes it out. No cloning the handle\n");
} else {
printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
}
{
pthread_t workers[NUM_THREADS];
for (i = 0; i < NUM_THREADS; i++) {
struct v *data = (struct v *) malloc(sizeof(struct v));
data->number = i;
data->data = NULL;
if (parallel) {
/* Now we will create the thread passing it data as an argument */
pthread_create(&workers[thread_counter], NULL, runner, data);
/*pthread_join(workers[thread_counter], NULL);*/
thread_counter++;
} else {
do_stuff(data);
}
}
if (parallel) {
for (i = 0; i < NUM_THREADS; i++) {
pthread_join(workers[i], NULL);
}
}
}
return 0;
}
void *runner(void *ptr)
{
do_stuff(ptr);
pthread_exit(0);
}
void do_stuff(void *ptr)
{
/* Cast argument to struct v pointer */
struct v *data = ptr;
size_t i;
char output_file[50];
time_t ltime;
struct tm result;
char stime[32];
for (i=0; i<FILES_PER_ITERATION;i++) {
sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
encode_file(INPUT_FILE,output_file);
}
ltime = time(NULL);
localtime_r(&ltime, &result);
strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
/* asctime_r(&result, stime); */
printf("%s: Worker %ld finished.\n", stime, data->number);
}

159
tests/grib_ecc-604-3.c
View File

@ -1,159 +0,0 @@
/*
* Test for ECC-604: Each thread creates a new GRIB handle and clones it.
* No output writing.
*/
#include <time.h>
#include <pthread.h>
#include <assert.h>
#include "grib_api.h"
/* These are passed in via argv */
static size_t NUM_THREADS = 0;
static size_t FILES_PER_ITERATION = 0;
static char* INPUT_FILE = NULL;
static int decode_file(char *template_file)
{
FILE *in;
grib_handle *source_handle = NULL;
const void *buffer = NULL;
size_t size = 0;
int err = 0;
double *values;
in = fopen(template_file,"r"); assert(in);
/* loop over the messages in the source GRIB and clone them */
while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
int i;
size_t values_len = 0;
size_t str_len = 20;
grib_handle *clone_handle = grib_handle_clone(source_handle);
assert(clone_handle);
/*GRIB_CHECK(grib_set_long(clone_handle, "centre", 250),0);*/
GRIB_CHECK(grib_get_size(clone_handle, "values", &values_len),0);
values = (double*)malloc(values_len*sizeof(double));
GRIB_CHECK(grib_get_double_array(clone_handle, "values", values, &values_len),0);
for (i=0;i<values_len;i++) {
values[i] *= 0.9;
}
GRIB_CHECK(grib_set_string(clone_handle,"stepUnits", "s", &str_len),0);
GRIB_CHECK(grib_set_long(clone_handle, "startStep", 43200), 0);
GRIB_CHECK(grib_set_long(clone_handle, "endStep", 86400), 0);
GRIB_CHECK(grib_set_long(clone_handle, "bitsPerValue", 16),0);
/* set data values */
GRIB_CHECK(grib_set_double_array(clone_handle,"values",values,values_len),0);
GRIB_CHECK(grib_get_message(clone_handle,&buffer,&size),0);
{
FILE *devnull = fopen("/dev/null", "w");
grib_dump_content(source_handle, devnull, "debug", 0, NULL);
grib_dump_content(clone_handle, devnull, "debug", 0, NULL);
}
grib_handle_delete(clone_handle);
grib_handle_delete(source_handle);
free(values);
}
fclose(in);
return 0;
}
void do_stuff(void *data);
/* Structure for passing data to threads */
struct v {
size_t number;
char *data;
};
void *runner(void *ptr); /* the thread */
int main(int argc, char **argv)
{
size_t i;
int thread_counter = 0;
int parallel = 1;
const char* prog = argv[0];
char* mode;
if (argc!=5) {
fprintf(stderr, "Usage:\n\t%s seq file numRuns numIter\nOr\n\t%s par file numThreads numIter\n", prog, prog);
return 1;
}
mode = argv[1];
INPUT_FILE= argv[2];
NUM_THREADS = atol(argv[3]);
FILES_PER_ITERATION = atol(argv[4]);
if (strcmp(mode,"seq")==0) {
parallel = 0;
}
if (parallel) {
printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
printf("Each thread creates a new GRIB handle and clones it. No output writing\n");
} else {
printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
}
{
pthread_t workers[NUM_THREADS];
for (i = 0; i < NUM_THREADS; i++) {
struct v *data = (struct v *) malloc(sizeof(struct v));
data->number = i;
data->data = NULL;
if (parallel) {
/* Now we will create the thread passing it data as an argument */
pthread_create(&workers[thread_counter], NULL, runner, data);
/*pthread_join(workers[thread_counter], NULL);*/
thread_counter++;
} else {
do_stuff(data);
}
}
if (parallel) {
for (i = 0; i < NUM_THREADS; i++) {
pthread_join(workers[i], NULL);
}
}
}
return 0;
}
void *runner(void *ptr)
{
do_stuff(ptr);
pthread_exit(0);
}
void do_stuff(void *ptr)
{
/* Cast argument to struct v pointer */
struct v *data = ptr;
size_t i;
time_t ltime;
struct tm result;
char stime[32];
for (i=0; i<FILES_PER_ITERATION;i++) {
decode_file(INPUT_FILE);
}
ltime = time(NULL);
localtime_r(&ltime, &result);
strftime(stime, 32, "%H:%M:%S", &result); /* Try to get milliseconds here too*/
/* asctime_r(&result, stime); */
printf("%s: Worker %ld finished.\n", stime, data->number);
}

76
tests/grib_ecc-604-1.c → tests/grib_ecc-604.c
View File

@ -4,6 +4,7 @@
#include <time.h>
#include <pthread.h>
#include <assert.h>
#include <unistd.h>
#include "grib_api.h"
@ -11,6 +12,8 @@
static size_t NUM_THREADS = 0;
static size_t FILES_PER_ITERATION = 0;
static char* INPUT_FILE = NULL;
int opt_clone = 0; /* If 1 then clone source handle */
int opt_write = 0; /* If 1 write handle to file */
static int encode_file(char *template_file, char *output_file)
{
@ -22,51 +25,56 @@ static int encode_file(char *template_file, char *output_file)
double *values;
in = fopen(template_file,"r"); assert(in);
out = fopen(output_file,"w"); assert(out);
if (output_file) {
out = fopen(output_file,"w"); assert(out);
}
/* loop over the messages in the source GRIB and clone them */
while ((source_handle = grib_handle_new_from_file(0, in, &err))!=NULL) {
int i;
size_t values_len = 0;
size_t str_len = 20;
grib_handle *h = source_handle;
grib_handle *clone_handle = grib_handle_clone(source_handle);
assert(clone_handle);
if (opt_clone) {
h = grib_handle_clone(source_handle); assert(h);
}
/*GRIB_CHECK(grib_set_long(clone_handle, "centre", 250),0);*/
GRIB_CHECK(grib_get_size(clone_handle, "values", &values_len),0);
GRIB_CHECK(grib_get_size(h, "values", &values_len),0);
values = (double*)malloc(values_len*sizeof(double));
GRIB_CHECK(grib_get_double_array(clone_handle, "values", values, &values_len),0);
GRIB_CHECK(grib_get_double_array(h, "values", values, &values_len),0);
for (i=0;i<values_len;i++) {
values[i] *= 0.9;
}
GRIB_CHECK(grib_set_string(clone_handle,"stepUnits", "s", &str_len),0);
GRIB_CHECK(grib_set_long(clone_handle, "startStep", 43200), 0);
GRIB_CHECK(grib_set_long(clone_handle, "endStep", 86400), 0);
GRIB_CHECK(grib_set_long(clone_handle, "bitsPerValue", 16),0);
GRIB_CHECK(grib_set_string(h,"stepUnits", "s", &str_len),0);
GRIB_CHECK(grib_set_long(h, "startStep", 43200), 0);
GRIB_CHECK(grib_set_long(h, "endStep", 86400), 0);
GRIB_CHECK(grib_set_long(h, "bitsPerValue", 16),0);
/* set data values */
GRIB_CHECK(grib_set_double_array(clone_handle,"values",values,values_len),0);
GRIB_CHECK(grib_set_double_array(h,"values",values,values_len),0);
GRIB_CHECK(grib_get_message(clone_handle,&buffer,&size),0);
if(fwrite(buffer,1,size,out) != size) {
perror(output_file);
return 1;
GRIB_CHECK(grib_get_message(h,&buffer,&size),0);
if (output_file) {
if(fwrite(buffer,1,size,out) != size) {
perror(output_file);
return 1;
}
}
{
FILE *devnull = fopen("/dev/null", "w");
grib_dump_content(source_handle,devnull, "debug", 0, NULL);
grib_dump_content(clone_handle, devnull, "debug", 0, NULL);
}
grib_handle_delete(clone_handle);
grib_handle_delete(source_handle);
if(opt_clone) grib_handle_delete(h);
free(values);
}
fclose(out);
if (output_file) fclose(out);
fclose(in);
return 0;
@ -86,24 +94,32 @@ int main(int argc, char **argv)
{
size_t i;
int thread_counter = 0;
int parallel = 1;
int parallel=1, index=0, c=0;
const char* prog = argv[0];
char* mode;
if (argc!=5) {
fprintf(stderr, "Usage:\n\t%s seq file numRuns numIter\nOr\n\t%s par file numThreads numIter\n", prog, prog);
if (argc<5 || argc>7) {
fprintf(stderr, "Usage:\n\t%s [options] seq file numRuns numIter\nOr\n\t%s [options] par file numThreads numIter\n", prog, prog);
return 1;
}
mode = argv[1];
INPUT_FILE= argv[2];
NUM_THREADS = atol(argv[3]);
FILES_PER_ITERATION = atol(argv[4]);
while ((c = getopt (argc, argv, "cw")) != -1) {
switch (c) {
case 'c': opt_clone=1; break;
case 'w': opt_write=1; break;
}
}
index = optind;
mode = argv[index];
INPUT_FILE = argv[index+1];
NUM_THREADS = atol(argv[index+2]);
FILES_PER_ITERATION = atol(argv[index+3]);
if (strcmp(mode,"seq")==0) {
parallel = 0;
}
if (parallel) {
printf("Running parallel in %ld threads. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
printf("Each thread creates a new GRIB handle, clones it and writes it out\n");
printf("Options: clone=%d, write=%d\n", opt_clone, opt_write);
} else {
printf("Running sequentially in %ld runs. %ld iterations\n", NUM_THREADS, FILES_PER_ITERATION);
}
@ -152,8 +168,12 @@ void do_stuff(void *ptr)
char stime[32];
for (i=0; i<FILES_PER_ITERATION;i++) {
sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
encode_file(INPUT_FILE,output_file);
if (opt_write) {
sprintf(output_file,"output/output_file_%ld-%ld.grib", data->number, i);
encode_file(INPUT_FILE,output_file);
} else {
encode_file(INPUT_FILE,NULL);
}
}
ltime = time(NULL);

6
tests/grib_ecc-604.sh
View File

@ -36,19 +36,19 @@ process()
# Test 01: Clone + output
# ------------------------
rm -fr $OUTPUT; mkdir -p $OUTPUT
time ${test_dir}/grib_ecc-604-1 par $input $NUM_THREADS $NUM_ITER
time ${test_dir}/grib_ecc-604 -c -w par $input $NUM_THREADS $NUM_ITER
validate
# Test 02: No clone + output
# --------------------------
rm -fr $OUTPUT; mkdir -p $OUTPUT
time ${test_dir}/grib_ecc-604-2 par $input $NUM_THREADS $NUM_ITER
time ${test_dir}/grib_ecc-604 -w par $input $NUM_THREADS $NUM_ITER
validate
# Test 03: Clone + no output
# ---------------------------
rm -fr $OUTPUT
time ${test_dir}/grib_ecc-604-3 par $input $NUM_THREADS $NUM_ITER
time ${test_dir}/grib_ecc-604 -c par $input $NUM_THREADS $NUM_ITER
# Nothing to validate as there is no output
}
###################################################