core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ECC-394: bufr_dump -E option: include keys whose values are MISSING

This commit is contained in:
Shahram Najm 2018-01-02 16:32:55 +00:00
parent 30b705b8da
commit 62cfacc0ae
10 changed files with 193 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/grib_dumper_class_bufr_encode_C.c
View File

@ -144,10 +144,18 @@ static int destroy(grib_dumper* d)
return GRIB_SUCCESS;
}
static char* lval_to_string(grib_context* c, long v)
{
char* sval=grib_context_malloc_clear(c,sizeof(char)*40);
if (v == GRIB_MISSING_LONG) sprintf(sval,"CODES_MISSING_LONG");
else sprintf(sval,"%ld",v);
return sval;
}
static char* dval_to_string(grib_context* c, double v)
{
char* sval=(char*)grib_context_malloc_clear(c,sizeof(char)*40);
sprintf(sval,"%.18e",v);
if (v == GRIB_MISSING_DOUBLE) sprintf(sval,"CODES_MISSING_DOUBLE");
else sprintf(sval,"%.18e",v);
return sval;
}
@ -208,17 +216,13 @@ static void dump_values(grib_dumper* d, grib_accessor* a)
fprintf(self->dumper.out," CODES_CHECK(codes_set_double_array(h, \"%s\", rvalues, size), 0);\n",a->name);
} else {
r=compute_bufr_key_rank(h,self->keys,a->name);
if( !grib_is_missing_double(a,value) ) {
sval=dval_to_string(c,value);
if (r!=0)
fprintf(self->dumper.out," CODES_CHECK(codes_set_double(h, \"#%d#%s\", %s), 0);\n", r, a->name, sval);
else
fprintf(self->dumper.out," CODES_CHECK(codes_set_double(h, \"%s\", %s), 0);\n", a->name, sval);
grib_context_free(c,sval);
}
}
if (self->isLeaf==0) {
char* prefix;
@ -290,15 +294,9 @@ static void dump_values_attribute(grib_dumper* d, grib_accessor* a, const char*
fprintf(self->dumper.out," CODES_CHECK(codes_set_double_array(h, \"%s->%s\", rvalues, size), 0);\n", prefix,a->name);
} else {
/* int r=compute_bufr_key_rank(h,self->keys,a->name); */
if( !grib_is_missing_double(a,value) ) {
sval=dval_to_string(c,value);
fprintf(self->dumper.out," CODES_CHECK(codes_set_double(h, \"%s->%s\", %s), 0);\n", prefix,a->name, sval);
grib_context_free(c,sval);
}
}
if (self->isLeaf==0) {
@ -325,6 +323,7 @@ static void dump_long(grib_dumper* d,grib_accessor* a, const char* comment)
int i,r,icount;
int cols=4;
long count=0;
char* sval = NULL;
grib_context* c=a->context;
grib_handle* h=grib_handle_of_accessor(a);
@ -388,7 +387,7 @@ static void dump_long(grib_dumper* d,grib_accessor* a, const char* comment)
} else {
r=compute_bufr_key_rank(h,self->keys,a->name);
if( !grib_is_missing_long(a,value) ) {
sval=lval_to_string(c,value);
int doing_unexpandedDescriptors=0;
if (r!=0) {
fprintf(self->dumper.out," CODES_CHECK(codes_set_long(h, \"#%d#%s\", ", r,a->name);
@ -400,11 +399,11 @@ static void dump_long(grib_dumper* d,grib_accessor* a, const char* comment)
fprintf(self->dumper.out," CODES_CHECK(codes_set_long(h, \"%s\", ", a->name);
}
fprintf(self->dumper.out,"%ld), 0);\n",value);
fprintf(self->dumper.out,"%s), 0);\n",sval);
grib_context_free(c,sval);
if (doing_unexpandedDescriptors)
fprintf(self->dumper.out,"\n");
}
}
if (self->isLeaf==0) {
char* prefix;
@ -471,11 +470,10 @@ static void dump_long_attribute(grib_dumper* d, grib_accessor* a, const char* pr
fprintf(self->dumper.out," CODES_CHECK(codes_set_long_array(h, \"%s->%s\", ivalues, size), 0);\n", prefix,a->name);
} else {
/* int r=compute_bufr_key_rank(h,self->keys,a->name); */
if( !grib_is_missing_long(a,value) ) {
char* sval=lval_to_string(c,value);
fprintf(self->dumper.out," CODES_CHECK(codes_set_long(h, \"%s->%s\", ", prefix,a->name);
fprintf(self->dumper.out,"%ld), 0);\n",value);
}
fprintf(self->dumper.out,"%s), 0);\n",sval);
grib_context_free(c,sval);
}
if (self->isLeaf==0) {
@ -512,15 +510,13 @@ static void dump_double(grib_dumper* d, grib_accessor* a, const char* comment)
self->empty=0;
r=compute_bufr_key_rank(h,self->keys,a->name);
if( !grib_is_missing_double(a,value) ) {
sval=dval_to_string(c,value);
if (r!=0)
fprintf(self->dumper.out," CODES_CHECK(codes_set_double(h, \"#%d#%s\", %s), 0);\n", r,a->name, sval);
else
fprintf(self->dumper.out," CODES_CHECK(codes_set_double(h, \"%s\", %s), 0);\n", a->name, sval);
grib_context_free(c,sval);
}
if (self->isLeaf==0) {
char* prefix;
@ -630,8 +626,9 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
err = grib_unpack_string(a,value,&size);
p=value;
r=compute_bufr_key_rank(h,self->keys,a->name);
if (grib_is_missing_string(a,(unsigned char *)value,size))
return;
if (grib_is_missing_string(a,(unsigned char *)value,size)) {
strcpy(value, ""); /* Empty string means MISSING string */
}
while(*p) { if(!isprint(*p)) *p = '.'; p++; }

5
src/grib_dumper_class_bufr_encode_filter.c
View File

@ -590,8 +590,9 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
err = grib_unpack_string(a,value,&size);
p=value;
r=compute_bufr_key_rank(h,self->keys,a->name);
if (grib_is_missing_string(a,(unsigned char *)value,size))
return;
if (grib_is_missing_string(a,(unsigned char *)value,size)) {
strcpy(value, ""); /* Empty string means MISSING string */
}
while(*p) { if(!isprint(*p)) *p = '.'; p++; }

44
src/grib_dumper_class_bufr_encode_fortran.c
View File

@ -144,9 +144,19 @@ static int destroy(grib_dumper* d)
return GRIB_SUCCESS;
}
static char* lval_to_string(grib_context* c, long v)
{
char* sval=grib_context_malloc_clear(c,sizeof(char)*40);
if (v == GRIB_MISSING_LONG) sprintf(sval,"CODES_MISSING_LONG");
else sprintf(sval,"%ld",v);
return sval;
}
static char* dval_to_string(grib_context* c, double v)
{
char* sval=(char*)grib_context_malloc_clear(c,sizeof(char)*40);
if (v == GRIB_MISSING_DOUBLE) {
sprintf(sval,"CODES_MISSING_DOUBLE");
} else {
char* p;
sprintf(sval,"%.18e",v);
p=sval;
@ -154,6 +164,7 @@ static char* dval_to_string(grib_context* c, double v)
if (*p == 'e') *p='d';
p++;
}
}
return sval;
}
@ -215,17 +226,12 @@ static void dump_values(grib_dumper* d, grib_accessor* a)
fprintf(self->dumper.out," call codes_set(ibufr,'%s',rvalues)\n",a->name);
} else {
r=compute_bufr_key_rank(h,self->keys,a->name);
if( !grib_is_missing_double(a,value) ) {
sval=dval_to_string(c,value);
if (r!=0)
fprintf(self->dumper.out," call codes_set(ibufr,'#%d#%s',%s)\n",r,a->name,sval);
else
fprintf(self->dumper.out," call codes_set(ibufr,'%s',%s)\n",a->name,sval);
grib_context_free(c,sval);
}
}
if (self->isLeaf==0) {
@ -299,14 +305,9 @@ static void dump_values_attribute(grib_dumper* d, grib_accessor* a, const char*
fprintf(self->dumper.out," call codes_set(ibufr,'%s->%s' &\n,rvalues)\n",prefix,a->name);
} else {
if( !grib_is_missing_double(a,value) ) {
sval=dval_to_string(c,value);
fprintf(self->dumper.out," call codes_set(ibufr,'%s->%s' &\n,%s)\n",prefix,a->name,sval);
grib_context_free(c,sval);
}
}
if (self->isLeaf==0) {
@ -333,6 +334,7 @@ static void dump_long(grib_dumper* d,grib_accessor* a, const char* comment)
int i,r,icount;
int cols=4;
long count=0;
char* sval = NULL;
grib_context* c=a->context;
grib_handle* h=grib_handle_of_accessor(a);
@ -394,7 +396,7 @@ static void dump_long(grib_dumper* d,grib_accessor* a, const char* comment)
} else {
r=compute_bufr_key_rank(h,self->keys,a->name);
if( !grib_is_missing_long(a,value) ) {
sval=lval_to_string(c,value);
int doing_unexpandedDescriptors=0;
if (r!=0) {
fprintf(self->dumper.out," call codes_set(ibufr,'#%d#%s',",r,a->name);
@ -406,11 +408,11 @@ static void dump_long(grib_dumper* d,grib_accessor* a, const char* comment)
fprintf(self->dumper.out," call codes_set(ibufr,'%s',",a->name);
}
fprintf(self->dumper.out,"%ld)\n",value);
fprintf(self->dumper.out,"%s)\n",sval);
grib_context_free(c,sval);
if (doing_unexpandedDescriptors)
fprintf(self->dumper.out,"\n");
}
}
if (self->isLeaf==0) {
char* prefix;
@ -476,11 +478,10 @@ static void dump_long_attribute(grib_dumper* d, grib_accessor* a, const char* pr
fprintf(self->dumper.out," call codes_set(ibufr,'%s->%s' &\n,ivalues)\n",prefix,a->name);
} else {
/* int r=compute_bufr_key_rank(h,self->keys,a->name); */
if( !grib_is_missing_long(a,value) ) {
char* sval=lval_to_string(c,value);
fprintf(self->dumper.out," call codes_set(ibufr,'%s->%s'&\n,",prefix,a->name);
fprintf(self->dumper.out,"%ld)\n",value);
}
fprintf(self->dumper.out,"%s)\n",sval);
grib_context_free(c,sval);
}
if (self->isLeaf==0) {
@ -517,15 +518,13 @@ static void dump_double(grib_dumper* d, grib_accessor* a, const char* comment)
self->empty=0;
r=compute_bufr_key_rank(h,self->keys,a->name);
if( !grib_is_missing_double(a,value) ) {
sval=dval_to_string(c,value);
if (r!=0)
fprintf(self->dumper.out," call codes_set(ibufr,'#%d#%s',%s)\n",r,a->name,sval);
else
fprintf(self->dumper.out," call codes_set(ibufr,'%s',%s)\n",a->name,sval);
grib_context_free(c,sval);
}
if (self->isLeaf==0) {
char* prefix;
@ -640,8 +639,9 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
err = grib_unpack_string(a,value,&size);
p=value;
r=compute_bufr_key_rank(h,self->keys,a->name);
if (grib_is_missing_string(a,(unsigned char *)value,size))
return;
if (grib_is_missing_string(a,(unsigned char *)value,size)) {
strcpy(value, ""); /* Empty string means MISSING string */
}
while(*p) { if(!isprint(*p)) *p = '.'; p++; }

44
src/grib_dumper_class_bufr_encode_python.c
View File

@ -144,10 +144,18 @@ static int destroy(grib_dumper* d)
return GRIB_SUCCESS;
}
static char* lval_to_string(grib_context* c, long v)
{
char* sval=grib_context_malloc_clear(c,sizeof(char)*40);
if (v == GRIB_MISSING_LONG) sprintf(sval,"CODES_MISSING_LONG");
else sprintf(sval,"%ld",v);
return sval;
}
static char* dval_to_string(const grib_context* c,double v)
{
char* sval=(char*)grib_context_malloc_clear(c,sizeof(char)*40);
sprintf(sval,"%.18e",v);
if (v == GRIB_MISSING_DOUBLE) sprintf(sval,"CODES_MISSING_DOUBLE");
else sprintf(sval,"%.18e",v);
return sval;
}
@ -196,7 +204,7 @@ static void dump_values(grib_dumper* d,grib_accessor* a)
grib_context_free(c,sval);
depth-=2;
/* Note: In python to make a tuple with one element, you need the trailing comma */
/* Note: In Python to make a tuple with one element, you need the trailing comma */
fprintf(self->dumper.out,",)\n");
grib_context_free(c,values);
@ -206,17 +214,13 @@ static void dump_values(grib_dumper* d,grib_accessor* a)
fprintf(self->dumper.out," codes_set_array(ibufr, '%s', rvalues)\n",a->name);
} else {
r=compute_bufr_key_rank(h,self->keys,a->name);
if( !grib_is_missing_double(a,value) ) {
sval=dval_to_string(c,value);
if (r!=0)
fprintf(self->dumper.out," codes_set(ibufr, '#%d#%s', %s)\n",r,a->name,sval);
else
fprintf(self->dumper.out," codes_set(ibufr, '%s', %s)\n",a->name,sval);
grib_context_free(c,sval);
}
}
if (self->isLeaf==0) {
char* prefix;
@ -287,15 +291,10 @@ static void dump_values_attribute(grib_dumper* d,grib_accessor* a, const char* p
fprintf(self->dumper.out," codes_set_array(ibufr, '%s->%s' \n, rvalues)\n",prefix,a->name);
} else {
/* int r=compute_bufr_key_rank(h,self->keys,a->name); */
if( !grib_is_missing_double(a,value) ) {
sval=dval_to_string(c,value);
fprintf(self->dumper.out," codes_set(ibufr, '%s->%s' \n,%s)\n",prefix,a->name,sval);
grib_context_free(c,sval);
}
}
if (self->isLeaf==0) {
char* prefix1;
@ -321,6 +320,7 @@ static void dump_long(grib_dumper* d, grib_accessor* a, const char* comment)
int i,r,icount;
int cols=4;
long count=0;
char* sval = NULL;
grib_context* c=a->context;
grib_handle* h=grib_handle_of_accessor(a);
@ -380,7 +380,7 @@ static void dump_long(grib_dumper* d, grib_accessor* a, const char* comment)
} else {
r=compute_bufr_key_rank(h,self->keys,a->name);
if( !grib_is_missing_long(a,value) ) {
sval=lval_to_string(c,value);
int doing_unexpandedDescriptors=0;
if (r!=0) {
fprintf(self->dumper.out," codes_set(ibufr, '#%d#%s', ",r,a->name);
@ -392,11 +392,11 @@ static void dump_long(grib_dumper* d, grib_accessor* a, const char* comment)
fprintf(self->dumper.out," codes_set(ibufr, '%s', ",a->name);
}
fprintf(self->dumper.out,"%ld)\n",value);
fprintf(self->dumper.out,"%s)\n",sval);
grib_context_free(c,sval);
if (doing_unexpandedDescriptors)
fprintf(self->dumper.out,"\n");
}
}
if (self->isLeaf==0) {
char* prefix;
@ -460,11 +460,10 @@ static void dump_long_attribute(grib_dumper* d, grib_accessor* a, const char* pr
fprintf(self->dumper.out," codes_set_array(ibufr, '%s->%s' \n,ivalues)\n",prefix,a->name);
} else {
/* int r=compute_bufr_key_rank(h,self->keys,a->name); */
if( !grib_is_missing_long(a,value) ) {
char* sval=lval_to_string(c,value);
fprintf(self->dumper.out," codes_set(ibufr, '%s->%s'\n,",prefix,a->name);
fprintf(self->dumper.out,"%ld)\n",value);
}
fprintf(self->dumper.out,"%s)\n",sval);
grib_context_free(c,sval);
}
if (self->isLeaf==0) {
@ -501,15 +500,13 @@ static void dump_double(grib_dumper* d,grib_accessor* a,const char* comment)
self->empty=0;
r=compute_bufr_key_rank(h,self->keys,a->name);
if( !grib_is_missing_double(a,value) ) {
sval=dval_to_string(c,value);
if (r!=0)
fprintf(self->dumper.out," codes_set(ibufr, '#%d#%s', %s)\n",r,a->name,sval);
else
fprintf(self->dumper.out," codes_set(ibufr, '%s', %s)\n",a->name,sval);
grib_context_free(c,sval);
}
if (self->isLeaf==0) {
char* prefix;
@ -621,8 +618,9 @@ static void dump_string(grib_dumper* d,grib_accessor* a,const char* comment)
err = grib_unpack_string(a,value,&size);
p=value;
r=compute_bufr_key_rank(h,self->keys,a->name);
if (grib_is_missing_string(a,(unsigned char *)value,size))
return;
if (grib_is_missing_string(a,(unsigned char *)value,size)) {
strcpy(value, ""); /* Empty string means MISSING string */
}
while(*p) { if(!isprint(*p)) *p = '.'; p++; }

3
src/grib_value.c
View File

@ -551,10 +551,11 @@ int grib_is_missing_string(grib_accessor* a, unsigned char* x, size_t len)
{
/* For a string value to be missing, every character has to be */
/* all 1's (i.e. 0xFF) */
/* Note: An empty string is also classified as missing */
int ret;
size_t i=0;
if (len==0) return 0;
if (len==0) return 1; /* empty string */
ret=1;
for (i=0;i<len;i++) {
if (x[i] != 0xFF ) {

12
tests/bufr_dump_encode_C.sh
View File

@ -84,12 +84,12 @@ do
fi
${tools_dir}/bufr_compare ${data_dir}/bufr/$file $tempBufr
TEMP_JSON1=${label}.$file.json
TEMP_JSON2=${label}.$tempBufr.json
${tools_dir}/bufr_dump ${data_dir}/bufr/$file > $TEMP_JSON1
${tools_dir}/bufr_dump $tempBufr > $TEMP_JSON2
diff $TEMP_JSON1 $TEMP_JSON2
rm -f $TEMP_JSON1 $TEMP_JSON2
TEMP_OUT1=${label}.$file.dump.out
TEMP_OUT2=${label}.$tempBufr.dump.out
${tools_dir}/bufr_dump -p ${data_dir}/bufr/$file > $TEMP_OUT1
${tools_dir}/bufr_dump -p $tempBufr > $TEMP_OUT2
diff $TEMP_OUT1 $TEMP_OUT2
rm -f $TEMP_OUT1 $TEMP_OUT2
fi
rm -f $tempExe $tempSrc $tempBufr

12
tests/bufr_dump_encode_filter.sh
View File

@ -54,12 +54,12 @@ do
${tools_dir}/codes_bufr_filter -o $fBufrTmp $fRules $f
${tools_dir}/bufr_compare $fBufrTmp $f
TEMP_JSON1=${label}.$f.json
TEMP_JSON2=${label}.$fBufrTmp.json
${tools_dir}/bufr_dump $f > $TEMP_JSON1
${tools_dir}/bufr_dump $fBufrTmp > $TEMP_JSON2
diff $TEMP_JSON1 $TEMP_JSON2
rm -f $TEMP_JSON1 $TEMP_JSON2
TEMP_OUT1=${label}.$f.dump.out
TEMP_OUT2=${label}.$fBufrTmp.dump.out
${tools_dir}/bufr_dump -p $f > $TEMP_OUT1
${tools_dir}/bufr_dump -p $fBufrTmp > $TEMP_OUT2
diff $TEMP_OUT1 $TEMP_OUT2
rm -f $TEMP_OUT1 $TEMP_OUT2
rm -f $fBufrTmp $fRules
fi

13
tests/bufr_dump_encode_fortran.sh
View File

@ -41,6 +41,7 @@ if command -v pkg-config >/dev/null 2>&1; then
INSTALL_DIR=`grep -w CMAKE_INSTALL_PREFIX $CACHE_FILE | cut -d'=' -f2`
FLAGS_LINKER=`echo $FLAGS_LINKER | sed -e "s:$INSTALL_DIR:$BUILD_DIR:g"`
FLAGS_COMPILER=`echo $FLAGS_COMPILER | sed -e "s:$INSTALL_DIR:$BUILD_DIR:g"`
FLAGS_COMPILER="$FLAGS_COMPILER -ffree-line-length-0"
# TODO: For now only support when shared libs enabled
SHARED_LIBS=`grep -w BUILD_SHARED_LIBS $CACHE_FILE | cut -d'=' -f2`
@ -73,12 +74,12 @@ do
./$tempExe
${tools_dir}/bufr_compare ${data_dir}/bufr/$file $tempBufr
TEMP_JSON1=${label}.$file.json
TEMP_JSON2=${label}.$tempBufr.json
${tools_dir}/bufr_dump ${data_dir}/bufr/$file > $TEMP_JSON1
${tools_dir}/bufr_dump $tempBufr > $TEMP_JSON2
diff $TEMP_JSON1 $TEMP_JSON2
rm -f $TEMP_JSON1 $TEMP_JSON2
TEMP_OUT1=${label}.$file.dump.out
TEMP_OUT2=${label}.$tempBufr.dump.out
${tools_dir}/bufr_dump -p ${data_dir}/bufr/$file > $TEMP_OUT1
${tools_dir}/bufr_dump -p $tempBufr > $TEMP_OUT2
diff $TEMP_OUT1 $TEMP_OUT2
rm -f $TEMP_OUT1 $TEMP_OUT2
fi
rm -f $tempExe $tempSrc $tempBufr

12
tests/bufr_dump_encode_python.sh
View File

@ -42,12 +42,12 @@ do
# Check original BUFR file against one we generated from sample
${tools_dir}/bufr_compare $inputBufr $tempBufr
TEMP_JSON1=${label}.$file.json
TEMP_JSON2=${label}.$tempBufr.json
${tools_dir}/bufr_dump $inputBufr > $TEMP_JSON1
${tools_dir}/bufr_dump $tempBufr > $TEMP_JSON2
diff $TEMP_JSON1 $TEMP_JSON2 2>/dev/null
rm -f $TEMP_JSON1 $TEMP_JSON2
TEMP_OUT1=${label}.$file.dump.out
TEMP_OUT2=${label}.$tempBufr.dump.out
${tools_dir}/bufr_dump -p $inputBufr > $TEMP_OUT1
${tools_dir}/bufr_dump -p $tempBufr > $TEMP_OUT2
diff $TEMP_OUT1 $TEMP_OUT2
rm -f $TEMP_OUT1 $TEMP_OUT2
rm -f $tempSrc $tempBufr
done

12
tools/bufr_compare.c
View File

@ -741,12 +741,24 @@ static int compare_values(grib_runtime_options* options, grib_handle* handle1, g
{
if(grib_inline_strcmp(sval1,sval2) != 0)
{
/* Check if strings are 'missing'.
* Note: one string could have all its bits=1 and the other empty */
int equal = 0;
grib_accessor* a1 = grib_find_accessor(handle1, name);
grib_accessor* a2 = grib_find_accessor(handle2, name);
int is_miss_1 = grib_is_missing_string(a1, (unsigned char *)sval1, len1);
int is_miss_2 = grib_is_missing_string(a2, (unsigned char *)sval2, len2);
if ( is_miss_1 && is_miss_2 ) {
equal = 1;
}
if (!equal) {
printInfo(handle1);
printf("string [%s]: [%s] != [%s]\n", name, sval1, sval2);
err1 = GRIB_VALUE_MISMATCH;
save_error(c,name);
}
}
}
grib_context_free(handle1->context,sval1);
grib_context_free(handle2->context,sval2);