core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECC-320: bufr_dump -Dpython

This commit is contained in:
Shahram Najm 2016-09-14 11:54:57 +01:00
parent aa7f64319e
commit c3af7e550a
10 changed files with 108 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
examples/python/bufr_get_keys.py
View File

@ -33,15 +33,15 @@ def example():
# loop for the messages in the file
while 1:
# get handle for message
gid = codes_bufr_new_from_file(f)
if gid is None:
bufr = codes_bufr_new_from_file(f)
if bufr is None:
break
print "message: %s" % cnt
# we need to instruct ecCodes to expand all the descriptors
# i.e. unpack the data values
codes_set(gid, 'unpack', 1)
codes_set(bufr, 'unpack', 1)
# ---------------------------------------------
# get values for keys holding a single value
@ -50,28 +50,28 @@ def example():
key = 'blockNumber'
try:
print ' %s: %s' % (key, codes_get(gid, key))
print ' %s: %s' % (key, codes_get(bufr, key))
except CodesInternalError as err:
print 'Error with key="%s" : %s' % (key, err.msg)
# Native type integer
key = 'stationNumber'
try:
print ' %s: %s' % (key, codes_get(gid, key))
print ' %s: %s' % (key, codes_get(bufr, key))
except CodesInternalError as err:
print 'Error with key="%s" : %s' % (key, err.msg)
# Native type float
key = 'airTemperatureAt2M'
try:
print ' %s: %s' % (key, codes_get(gid, key))
print ' %s: %s' % (key, codes_get(bufr, key))
except CodesInternalError as err:
print 'Error with key="%s" : %s' % (key, err.msg)
# Native type string
key = 'typicalDate'
try:
print ' %s: %s' % (key, codes_get(gid, key))
print ' %s: %s' % (key, codes_get(bufr, key))
except CodesInternalError as err:
print 'Error with key="%s" : %s' % (key, err.msg)
@ -82,11 +82,11 @@ def example():
key = 'bufrdcExpandedDescriptors'
# get size
num = codes_get_size(gid, key)
num = codes_get_size(bufr, key)
print ' size of %s is: %s' % (key, num)
# get values
values = codes_get_array(gid, key)
values = codes_get_array(bufr, key)
for i in xrange(len(values)):
print " %d %06d" % (i + 1, values[i])
@ -94,18 +94,18 @@ def example():
key = 'numericValues'
# get size
num = codes_get_size(gid, key)
num = codes_get_size(bufr, key)
print ' size of %s is: %s' % (key, num)
# get values
values = codes_get_array(gid, key)
values = codes_get_array(bufr, key)
for i in xrange(len(values)):
print " %d %.10e" % (i + 1, values[i])
cnt += 1
# delete handle
codes_release(gid)
codes_release(bufr)
# close the file
f.close()

13
python/eccodes/eccodes.py
View File

@ -1,19 +1,19 @@
from gribapi import __version__
from gribapi import GRIB_CHECK as CODES_CHECK
from gribapi import CODES_PRODUCT_GRIB
from gribapi import CODES_PRODUCT_BUFR
from gribapi import CODES_PRODUCT_ANY
from gribapi import GRIB_MISSING_DOUBLE as CODES_MISSING_DOUBLE
from gribapi import GRIB_MISSING_LONG as CODES_MISSING_LONG
from gribapi import grib_new_from_file as codes_grib_new_from_file
from gribapi import bufr_new_from_file as codes_bufr_new_from_file
from gribapi import metar_new_from_file as codes_metar_new_from_file
from gribapi import gts_new_from_file as codes_gts_new_from_file
from gribapi import any_new_from_file as codes_any_new_from_file
from gribapi import metar_new_from_file as codes_metar_new_from_file
from gribapi import codes_new_from_file
from gribapi import any_new_from_file as codes_any_new_from_file
from gribapi import bufr_new_from_file as codes_bufr_new_from_file
from gribapi import grib_new_from_file as codes_grib_new_from_file
from gribapi import grib_count_in_file as codes_count_in_file
from gribapi import grib_multi_support_on as codes_grib_multi_support_on
@ -21,7 +21,6 @@ from gribapi import grib_multi_support_off as codes_grib_multi_support_off
from gribapi import grib_release as codes_release
from gribapi import grib_get_string as codes_get_string
from gribapi import grib_set_string as codes_set_string
from gribapi import grib_set_string_array as codes_set_string_array
from gribapi import grib_gribex_mode_on as codes_gribex_mode_on
from gribapi import grib_gribex_mode_off as codes_gribex_mode_off
from gribapi import grib_write as codes_write
@ -52,6 +51,8 @@ from gribapi import codes_bufr_new_from_samples
from gribapi import grib_clone as codes_clone
from gribapi import grib_set_double_array as codes_set_double_array
from gribapi import grib_get_double_array as codes_get_double_array
from gribapi import grib_get_string_array as codes_get_string_array
from gribapi import grib_set_string_array as codes_set_string_array
from gribapi import grib_set_long_array as codes_set_long_array
from gribapi import grib_get_long_array as codes_get_long_array
from gribapi import grib_multi_new as codes_grib_multi_new

13
src/grib_dumper_class_bufr_decode_fortran.c
View File

@ -601,13 +601,9 @@ static void dump_attributes(grib_dumper* d,grib_accessor* a, const char* prefix)
static void header(grib_dumper* d, grib_handle* h)
{
grib_dumper_bufr_decode_fortran *self = (grib_dumper_bufr_decode_fortran*)d;
long localSectionPresent,edition,bufrHeaderCentre;
grib_get_long(h,"localSectionPresent",&localSectionPresent);
grib_get_long(h,"bufrHeaderCentre",&bufrHeaderCentre);
grib_get_long(h,"edition",&edition);
if (d->count<2) {
/* This is the first message being processed */
fprintf(self->dumper.out,"! This program was automatically generated with bufr_dump -Dfortran\n");
fprintf(self->dumper.out,"! Using ecCodes version: ");
grib_print_api_version(self->dumper.out);
@ -639,13 +635,6 @@ static void header(grib_dumper* d, grib_handle* h)
static void footer(grib_dumper* d, grib_handle* h)
{
grib_dumper_bufr_decode_fortran *self = (grib_dumper_bufr_decode_fortran*)d;
/*if (d->count==1)
fprintf(self->dumper.out," call codes_open_file(outfile,'outfile.bufr','w')\n");
else
fprintf(self->dumper.out," call codes_open_file(outfile,'outfile.bufr','a')\n");
*/
/*fprintf(self->dumper.out," call codes_close_file(ifile)\n");*/
fprintf(self->dumper.out," call codes_release(ibufr)\n");
}

149
src/grib_dumper_class_bufr_decode_python.c
View File

@ -155,8 +155,7 @@ static void dump_values(grib_dumper* d,grib_accessor* a)
double value; size_t size = 0;
double *values=NULL;
int err = 0;
int i,r,icount;
int cols=2;
int r=0;
long count=0;
char* sval;
grib_context* c=a->context;
@ -178,39 +177,22 @@ static void dump_values(grib_dumper* d,grib_accessor* a)
self->empty=0;
if (size>1) {
fprintf(self->dumper.out," rvalues = (");
icount=0;
for (i=0; i<size-1; ++i) {
if (icount>cols || i==0) {fprintf(self->dumper.out,"\n ");icount=0;}
sval=dval_to_string(c,values[i]);
fprintf(self->dumper.out,"%s, ", sval);
grib_context_free(c,sval);
icount++;
}
if (icount>cols || i==0) {fprintf(self->dumper.out,"\n ");icount=0;}
sval=dval_to_string(c,values[i]);
fprintf(self->dumper.out,"%s", sval);
grib_context_free(c,sval);
depth-=2;
/* Note: In python to make a tuple with one element, you need the trailing comma */
fprintf(self->dumper.out,",)\n");
grib_context_free(c,values);
if ((r=compute_key_rank(h,self->keys,a->name))!=0)
fprintf(self->dumper.out," codes_set_array(ibufr, '#%d#%s', rvalues)\n",r,a->name);
fprintf(self->dumper.out," dVals = codes_get_array(ibufr, '#%d#%s')\n",r,a->name);
else
fprintf(self->dumper.out," codes_set_array(ibufr, '%s', rvalues)\n",a->name);
fprintf(self->dumper.out," dVals = codes_get_array(ibufr, '%s')\n",a->name);
} else {
r=compute_key_rank(h,self->keys,a->name);
if( !grib_is_missing_double(a,value) ) {
sval=dval_to_string(c,value);
if (r!=0)
fprintf(self->dumper.out," codes_set(ibufr, '#%d#%s', %s)\n",r,a->name,sval);
fprintf(self->dumper.out," dVal = codes_get(ibufr, '#%d#%s')\n",r,a->name);
else
fprintf(self->dumper.out," codes_set(ibufr, '%s', %s)\n",a->name,sval);
fprintf(self->dumper.out," dVal = codes_get(ibufr, '%s')\n",a->name);
grib_context_free(c,sval);
}
@ -316,8 +298,7 @@ static void dump_long(grib_dumper* d, grib_accessor* a, const char* comment)
long value; size_t size = 0;
long *values=NULL;
int err = 0;
int i,r,icount;
int cols=4;
int r=0;
long count=0;
grib_context* c=a->context;
grib_handle* h=grib_handle_of_accessor(a);
@ -356,35 +337,21 @@ static void dump_long(grib_dumper* d, grib_accessor* a, const char* comment)
self->empty=0;
if (size>1) {
fprintf(self->dumper.out," ivalues = (");
icount=0;
for (i=0;i<size-1;i++) {
if (icount>cols || i==0) {fprintf(self->dumper.out,"\n ");icount=0;}
fprintf(self->dumper.out,"%ld, ",values[i]);
icount++;
}
if (icount>cols || i==0) {fprintf(self->dumper.out,"\n ");icount=0;}
fprintf(self->dumper.out,"%ld",values[i]);
depth-=2;
/* Note: In python to make a tuple with one element, you need the trailing comma */
fprintf(self->dumper.out,",)\n");
grib_context_free(a->context,values);
if ((r=compute_key_rank(h,self->keys,a->name))!=0)
fprintf(self->dumper.out," codes_set_array(ibufr, '#%d#%s', ivalues)\n",r,a->name);
fprintf(self->dumper.out," iValues = codes_get_array(ibufr, '#%d#%s')\n",r,a->name);
else
fprintf(self->dumper.out," codes_set_array(ibufr, '%s', ivalues)\n",a->name);
fprintf(self->dumper.out," iValues = codes_get_array(ibufr, '%s')\n",a->name);
} else {
r=compute_key_rank(h,self->keys,a->name);
if( !grib_is_missing_long(a,value) ) {
if (r!=0)
fprintf(self->dumper.out," codes_set(ibufr, '#%d#%s', ",r,a->name);
fprintf(self->dumper.out," iVal = codes_get(ibufr, '#%d#%s')\n",r,a->name);
else
fprintf(self->dumper.out," codes_set(ibufr, '%s', ",a->name);
fprintf(self->dumper.out,"%ld)\n",value);
fprintf(self->dumper.out," iVal = codes_get(ibufr, '%s')\n",a->name);
}
}
@ -411,8 +378,6 @@ static void dump_long_attribute(grib_dumper* d, grib_accessor* a, const char* pr
long value; size_t size = 0;
long *values=NULL;
int err = 0;
int i,icount;
int cols=4;
long count=0;
grib_context* c=a->context;
@ -432,28 +397,14 @@ static void dump_long_attribute(grib_dumper* d, grib_accessor* a, const char* pr
self->empty=0;
if (size>1) {
fprintf(self->dumper.out," ivalues = (");
icount=0;
for (i=0;i<size-1;i++) {
if (icount>cols || i==0) {fprintf(self->dumper.out," \n ");icount=0;}
fprintf(self->dumper.out,"%ld, ",values[i]);
icount++;
}
if (icount>cols || i==0) {fprintf(self->dumper.out," \n ");icount=0;}
fprintf(self->dumper.out,"%ld ",values[i]);
depth-=2;
/* Note: In python to make a tuple with one element, you need the trailing comma */
fprintf(self->dumper.out,",)\n");
grib_context_free(a->context,values);
fprintf(self->dumper.out," codes_set_array(ibufr, '%s->%s' \n,ivalues)\n",prefix,a->name);
fprintf(self->dumper.out," iVals = codes_get_array(ibufr, '%s->%s')\n",prefix,a->name);
} else {
/* int r=compute_key_rank(h,self->keys,a->name); */
if( !grib_is_missing_long(a,value) ) {
fprintf(self->dumper.out," codes_set(ibufr, '%s->%s'\n,",prefix,a->name);
fprintf(self->dumper.out,"%ld)\n",value);
fprintf(self->dumper.out," iVal = codes_get(ibufr, '%s->%s')\n",prefix,a->name);
}
}
@ -494,9 +445,9 @@ static void dump_double(grib_dumper* d,grib_accessor* a,const char* comment)
if( !grib_is_missing_double(a,value) ) {
sval=dval_to_string(c,value);
if (r!=0)
fprintf(self->dumper.out," codes_set(ibufr, '#%d#%s', %s)\n",r,a->name,sval);
fprintf(self->dumper.out," dVal = codes_get(ibufr, '#%d#%s')\n",r,a->name);
else
fprintf(self->dumper.out," codes_set(ibufr, '%s', %s)\n",a->name,sval);
fprintf(self->dumper.out," dVal = codes_get(ibufr, '%s')\n",a->name);
grib_context_free(c,sval);
}
@ -521,7 +472,7 @@ static void dump_string_array(grib_dumper* d,grib_accessor* a,const char* commen
{
grib_dumper_bufr_decode_python *self = (grib_dumper_bufr_decode_python*)d;
char **values;
size_t size = 0,i=0;
size_t size = 0;
grib_context* c=NULL;
int err = 0;
long count=0;
@ -540,8 +491,6 @@ static void dump_string_array(grib_dumper* d,grib_accessor* a,const char* commen
return;
}
fprintf(self->dumper.out," svalues = (");
self->empty=0;
values=(char**)grib_context_malloc_clear(c,size*sizeof(char*));
@ -552,16 +501,11 @@ static void dump_string_array(grib_dumper* d,grib_accessor* a,const char* commen
err = grib_unpack_string_array(a,values,&size);
for (i=0;i<size-1;i++) {
fprintf(self->dumper.out," \"%s\", \n",values[i]);
}
fprintf(self->dumper.out," \"%s\", )\n",values[i]);
if (self->isLeaf==0) {
if ((r=compute_key_rank(h,self->keys,a->name))!=0)
fprintf(self->dumper.out," codes_set_string_array(ibufr, '#%d#%s', svalues)\n",r,a->name);
fprintf(self->dumper.out," sVals = codes_get_string_array(ibufr, '#%d#%s')\n",r,a->name);
else
fprintf(self->dumper.out," codes_set_string_array(ibufr, '%s', svalues)\n",a->name);
fprintf(self->dumper.out," sVals = codes_get_string_array(ibufr, '%s')\n",a->name);
}
if (self->isLeaf==0) {
@ -619,12 +563,10 @@ static void dump_string(grib_dumper* d,grib_accessor* a,const char* comment)
if (self->isLeaf==0) {
depth+=2;
if (r!=0)
fprintf(self->dumper.out," codes_set(ibufr, '#%d#%s',",r,a->name);
fprintf(self->dumper.out," sVal = codes_get(ibufr, '#%d#%s')\n", r,a->name);
else
fprintf(self->dumper.out," codes_set(ibufr, '%s',",a->name);
fprintf(self->dumper.out," sVal = codes_get(ibufr, '%s')\n", a->name);
}
fprintf(self->dumper.out,"\'%s\')\n",value);
if (self->isLeaf==0) {
char* prefix;
@ -655,27 +597,10 @@ static void dump_label(grib_dumper* d,grib_accessor* a,const char* comment)
static void _dump_long_array(grib_handle* h, FILE* f, const char* key, const char* print_key)
{
long* val;
size_t size=0,i;
int cols=9,icount=0;
size_t size=0;
if (grib_get_size(h,key,&size)==GRIB_NOT_FOUND) return;
fprintf(f," ivalues = (");
val=grib_context_malloc_clear(h->context,sizeof(long)*size);
grib_get_long_array(h,key,val,&size);
for (i=0;i<size-1;i++) {
if (icount>cols || i==0) {fprintf(f," \n ");icount=0;}
fprintf(f,"%ld, ",val[i]);
icount++;
}
if (icount>cols) {fprintf(f," \n ");}
/* Note: In python to make a tuple with one element, you need the trailing comma */
fprintf(f,"%ld ,)\n",val[size-1]);
grib_context_free(h->context,val);
fprintf(f," codes_set_array(ibufr, '%s', ivalues)\n",print_key);
fprintf(f," iVals = codes_get_array(ibufr, '%s')\n",print_key);
}
static void dump_section(grib_dumper* d, grib_accessor* a, grib_block_of_accessors* block)
@ -743,22 +668,6 @@ static void dump_attributes(grib_dumper* d,grib_accessor* a, const char* prefix)
static void header(grib_dumper* d, grib_handle* h)
{
grib_dumper_bufr_decode_python *self = (grib_dumper_bufr_decode_python*)d;
char sampleName[200]={0};
long localSectionPresent,edition,bufrHeaderCentre,isSatellite;
grib_get_long(h,"localSectionPresent",&localSectionPresent);
grib_get_long(h,"bufrHeaderCentre",&bufrHeaderCentre);
grib_get_long(h,"edition",&edition);
if (localSectionPresent && bufrHeaderCentre==98 ) {
grib_get_long(h,"isSatellite",&isSatellite);
if (isSatellite)
sprintf(sampleName,"BUFR%ld_local_satellite",edition);
else
sprintf(sampleName,"BUFR%ld_local",edition);
} else {
sprintf(sampleName,"BUFR%ld",edition);
}
if (d->count<2) {
/* This is the first message being processed */
@ -769,20 +678,18 @@ static void header(grib_dumper* d, grib_handle* h)
fprintf(self->dumper.out,"import traceback\n");
fprintf(self->dumper.out,"import sys\n");
fprintf(self->dumper.out,"from eccodes import *\n\n\n");
fprintf(self->dumper.out,"def bufr_decode():\n");
fprintf(self->dumper.out,"def bufr_decode(input):\n");
fprintf(self->dumper.out," f = open(input)\n");
}
fprintf(self->dumper.out," ibufr = codes_bufr_new_from_samples('%s')\n",sampleName);
fprintf(self->dumper.out," # Message number %ld\n # -----------------\n", d->count);
fprintf(self->dumper.out," print 'Decoding message number %ld'\n", d->count);
fprintf(self->dumper.out," ibufr = codes_bufr_new_from_file(f)\n");
fprintf(self->dumper.out," codes_set(ibufr, 'unpack', 1)\n");
}
static void footer(grib_dumper* d, grib_handle* h)
{
grib_dumper_bufr_decode_python *self = (grib_dumper_bufr_decode_python*)d;
fprintf(self->dumper.out," codes_set(ibufr, 'pack', 1)\n");
if (d->count==1)
fprintf(self->dumper.out," outfile = open('outfile.bufr', 'w')\n");
else
fprintf(self->dumper.out," outfile = open('outfile.bufr', 'a')\n");
fprintf(self->dumper.out," codes_write(ibufr, outfile)\n");
fprintf(self->dumper.out," codes_release(ibufr)\n");
}

16
tests/CMakeLists.txt
View File

@ -154,13 +154,15 @@ endif()
if( HAVE_PYTHON AND ENABLE_EXTRA_TESTS )
set(test bufr_dump_encode_python)
ecbuild_add_test( TARGET eccodes_t_${test}
TYPE SCRIPT
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${test}.sh
ENVIRONMENT PYTHON=${PYTHON_EXECUTABLE} PYTHONPATH=@PROJECT_BINARY_DIR@/python
TEST_DEPENDS eccodes_download_bufrs
)
list( APPEND ptests bufr_dump_encode_python bufr_dump_decode_python )
foreach( test ${ptests} )
ecbuild_add_test( TARGET eccodes_t_${test}
TYPE SCRIPT
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${test}.sh
ENVIRONMENT PYTHON=${PYTHON_EXECUTABLE} PYTHONPATH=@PROJECT_BINARY_DIR@/python
TEST_DEPENDS eccodes_download_bufrs
)
endforeach()
endif()
###########################################

3
tests/bufr_dump_decode_fortran.sh
View File

@ -74,3 +74,6 @@ do
rm -f $tempExe $tempSrc
done
cd $test_dir
rm -fr $tempDir

40
tests/bufr_dump_decode_python.sh Executable file
View File

@ -0,0 +1,40 @@
#!/bin/sh
# Copyright 2005-2016 ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
#
# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
#
. ./include.sh
#Define a common label for all the tmp files
label="bufr_dump_decode_python_test"
#Create log file
fLog=${label}".log"
rm -f $fLog
tempDir=${label}.dir
mkdir -p $tempDir
cd $tempDir
bufr_files=`cat ${data_dir}/bufr/bufr_data_files.txt`
for file in ${bufr_files}
do
inputBufr=${data_dir}/bufr/$file
tempSrc=$label.$file.py
# Generate Python code from BUFR file
${tools_dir}bufr_dump -Dpython $inputBufr > $tempSrc
$PYTHON $tempSrc $inputBufr
rm -f $tempSrc
done
cd $test_dir
rm -fr $tempDir

3
tests/bufr_dump_encode_C.sh
View File

@ -84,3 +84,6 @@ do
rm -f $tempExe $tempSrc $tempBufr
done
cd $test_dir
rm -fr $tempDir

3
tests/bufr_dump_encode_fortran.sh
View File

@ -78,3 +78,6 @@ do
rm -f $tempExe $tempSrc $tempBufr
done
cd $test_dir
rm -fr $tempDir

3
tools/bufr_dump.c
View File

@ -412,9 +412,10 @@ int grib_tool_finalise_action(grib_runtime_options* options)
}
if (!strcmp(options->dump_mode,"python")) {
fprintf(stdout,"\n\n");
fprintf(stdout," f.close()\n\n");
fprintf(stdout,"def main():\n");
fprintf(stdout," try:\n");
fprintf(stdout," bufr_decode()\n");
fprintf(stdout," bufr_decode(sys.argv[1])\n");
fprintf(stdout," except CodesInternalError as err:\n");
fprintf(stdout," traceback.print_exc(file=sys.stderr)\n");
fprintf(stdout," return 1\n");