ECC-53 ECC-163

This commit is contained in:
Enrico Fucile 2015-10-28 11:46:53 +00:00
parent dfa054154c
commit bd4385fce0
13 changed files with 256 additions and 130 deletions

View File

@ -12,13 +12,9 @@ section_length[3] section4Length ;
unsigned[1] reserved = 0;
position offsetBeforeData;
#meta subsetNumber bufr_subset_number();
transient subsetNumber=0;
template dataKeys "bufr/dataKeys.def";
meta numericValues bufr_data_array(offsetSection4,offsetBeforeData,offsetEndSection4,section4Length, numberOfSubsets,subsetNumber,expandedCodes,expandedFlags, elementsDescriptorsIndex,compressedData,dataKeys);
meta numericValues bufr_data_array(offsetSection4,offsetBeforeData,offsetEndSection4,section4Length, numberOfSubsets,expandedCodes,expandedFlags, elementsDescriptorsIndex,compressedData,dataKeys);
meta stringValues bufr_string_values(numericValues);
meta unpack unpack_bufr_values(numericValues) ;

View File

@ -24,6 +24,7 @@ void usage(char* prog) {
int main(int argc,char* argv[])
{
char key[200]={0,};
FILE* in = NULL;
/* message handle. Required in all the eccodes calls acting on a message.*/
@ -63,18 +64,17 @@ int main(int argc,char* argv[])
/* loop over the subsets */
for(i=1; i <= numberOfSubsets; i++)
{
/* specify the subset number */
CODES_CHECK(codes_set_long(h,"subsetNumber",0),0);
sprintf(key,"/subsetNumber=%d/blockNumber",i);
printf(" subsetNumber=%ld",i);
/* read and print some data values */
CODES_CHECK(codes_get_long(h,"blockNumber",&longVal),0);
printf(" blockNumber: %ld\n",longVal);
CODES_CHECK(codes_get_long(h,key,&longVal),0);
printf(" blockNumber=%ld",longVal);
CODES_CHECK(codes_get_long(h,"stationNumber",&longVal),0);
printf(" stationNumber: %ld\n",longVal);
sprintf(key,"/subsetNumber=%d/stationNumber",i);
CODES_CHECK(codes_get_long(h,key,&longVal),0);
printf(" stationNumber=%ld\n",longVal);
/*CODES_CHECK(codes_get_double(h,"airTemperatureAt2M",&doubleVal),0);
printf(" airTemperatureAt2M %f\n",doubleVal);*/
}
/* delete handle */

View File

@ -17,20 +17,35 @@ label="bufr_subset_test_c"
fTmp=${label}.tmp.txt
rm -f $fTmp | true
#Prepare ref file
fRef=${label}.ref
cat > $fRef <<EOF
message: 0
numberOfSubsets: 12
subsetNumber=1 blockNumber=1 stationNumber=27
subsetNumber=2 blockNumber=1 stationNumber=84
subsetNumber=3 blockNumber=1 stationNumber=270
subsetNumber=4 blockNumber=1 stationNumber=272
subsetNumber=5 blockNumber=1 stationNumber=308
subsetNumber=6 blockNumber=1 stationNumber=371
subsetNumber=7 blockNumber=1 stationNumber=381
subsetNumber=8 blockNumber=1 stationNumber=382
subsetNumber=9 blockNumber=1 stationNumber=387
subsetNumber=10 blockNumber=1 stationNumber=413
subsetNumber=11 blockNumber=1 stationNumber=464
subsetNumber=12 blockNumber=1 stationNumber=485
EOF
#We check "synop_multi_subset.bufr". The path is
#hardcoded in the example
REDIRECT=/dev/null
#Write the values into a file and compare with reference
${examples_dir}c_bufr_subset #2> $REDIRECT > $fTmp
#TODO: add a proper check when subsets are properly implemented
${examples_dir}c_bufr_subset 2> $REDIRECT > $fTmp
#We compare output to the reference by ignoring the whitespaces
#diff -w $fRef $fTmp >$REDIRECT 2> $REDIRECT
#cat $fTmp
diff -w $fRef $fTmp >$REDIRECT 2> $REDIRECT
#Clean up
rm -f $fTmp
rm -f $fTmp $fRef

View File

@ -22,6 +22,7 @@ integer :: ibufr
integer :: i, count=0
integer(kind=4) :: numberOfSubsets
integer(kind=4) :: blockNumber,stationNumber
character(100) :: key
!real(kind=8) :: t2m
call codes_open_file(ifile,'../../data/bufr/synop_multi_subset.bufr','r')
@ -46,20 +47,20 @@ integer(kind=4) :: blockNumber,stationNumber
! loop over the subsets
do i=1,numberOfSubsets
! specify the subset number
call codes_set(ibufr,'subsetNumber',0)
100 format('/subsetNumber=',I5.5,'/blockNumber')
write(key,100) I
write(*,*) key
write(*,*) ' subsetNumber:',i
! read and print some data values
call codes_get(ibufr,'blockNumber',blockNumber);
call codes_get(ibufr,key,blockNumber);
write(*,*) ' blockNumber:',blockNumber
write(key,*) '/subsetNumber=',I,'/stationNumber'
call codes_get(ibufr,'stationNumber',stationNumber);
write(*,*) ' stationNumber:',stationNumber
!call codes_get(ibufr,'airTemperatureAt2M',t2m);
!write(*,*) ' airTemperatureAt2M:',t2m
end do
! release the bufr message

View File

@ -17,20 +17,72 @@ label="bufr_subset_test_f"
fTmp=${label}.tmp.txt
rm -f $fTmp | true
#Prepare ref file
fRef=${label}.ref
cat > $fRef <<EOF
message: 0
numberOfSubsets: 12
/subsetNumber=00001/blockNumber
subsetNumber: 1
blockNumber: 1
stationNumber: 485
/subsetNumber=00002/blockNumber
subsetNumber: 2
blockNumber: 1
stationNumber: 485
/subsetNumber=00003/blockNumber
subsetNumber: 3
blockNumber: 1
stationNumber: 485
/subsetNumber=00004/blockNumber
subsetNumber: 4
blockNumber: 1
stationNumber: 485
/subsetNumber=00005/blockNumber
subsetNumber: 5
blockNumber: 1
stationNumber: 485
/subsetNumber=00006/blockNumber
subsetNumber: 6
blockNumber: 1
stationNumber: 485
/subsetNumber=00007/blockNumber
subsetNumber: 7
blockNumber: 1
stationNumber: 485
/subsetNumber=00008/blockNumber
subsetNumber: 8
blockNumber: 1
stationNumber: 485
/subsetNumber=00009/blockNumber
subsetNumber: 9
blockNumber: 1
stationNumber: 485
/subsetNumber=00010/blockNumber
subsetNumber: 10
blockNumber: 1
stationNumber: 485
/subsetNumber=00011/blockNumber
subsetNumber: 11
blockNumber: 1
stationNumber: 485
/subsetNumber=00012/blockNumber
subsetNumber: 12
blockNumber: 1
stationNumber: 485
EOF
#We check "synop_multi_subset.bufr". The path is
#hardcoded in the example
REDIRECT=/dev/null
REDIRECT=/dev/stdout
#Write the values into a file and compare with reference
${examples_dir}/eccodes_f_bufr_subset 2> $REDIRECT > $fTmp
#TODO: add a proper check when subsets are properly implemented
#We compare output to the reference by ignoring the whitespaces
#diff -w $fRef $fTmp >$REDIRECT 2> $REDIRECT
diff -w $fRef $fTmp >$REDIRECT 2> $REDIRECT
#cat $fTmp
#Clean up
rm -f $fTmp
rm -f $fTmp $fRef

View File

@ -46,25 +46,19 @@ def example():
print ' %s: %d' % (key,numberOfSubsets)
# loop over the subsets
for i in range(numberOfSubsets) :
#specify the subset number
codes_set(gid,'subsetNumber',0)
for i in range(1,numberOfSubsets+1) :
# read and print some data values
key='blockNumber'
val=codes_get(gid,key)
print ' %s: %d' % (key,val)
key='/subsetNumber=%d/blockNumber' % i
print key;
val=codes_get_long(gid,key)
print ' %s= %d' % (key,val)
key='stationNumber'
val=codes_get(gid,key)
key='/subsetNumber=%d/stationNumber' % i
val=codes_get_long(gid,key)
print ' %s: %d' % (key,val)
#key='airTemperatureAt2M'
#val=codes_get(gid,key)
#print ' %d: %d' % (key,val)
cnt+=1
# delete handle

View File

@ -17,18 +17,59 @@ label="bufr_subset_test_p"
fTmp=${label}.tmp.txt
rm -f $fTmp | true
#Prepare ref file
fRef=${label}.ref
cat > $fRef<<EOF
message: 0
numberOfSubsets: 12
/subsetNumber=1/blockNumber
/subsetNumber=1/blockNumber= 1
/subsetNumber=1/stationNumber: 27
/subsetNumber=2/blockNumber
/subsetNumber=2/blockNumber= 1
/subsetNumber=2/stationNumber: 84
/subsetNumber=3/blockNumber
/subsetNumber=3/blockNumber= 1
/subsetNumber=3/stationNumber: 270
/subsetNumber=4/blockNumber
/subsetNumber=4/blockNumber= 1
/subsetNumber=4/stationNumber: 272
/subsetNumber=5/blockNumber
/subsetNumber=5/blockNumber= 1
/subsetNumber=5/stationNumber: 308
/subsetNumber=6/blockNumber
/subsetNumber=6/blockNumber= 1
/subsetNumber=6/stationNumber: 371
/subsetNumber=7/blockNumber
/subsetNumber=7/blockNumber= 1
/subsetNumber=7/stationNumber: 381
/subsetNumber=8/blockNumber
/subsetNumber=8/blockNumber= 1
/subsetNumber=8/stationNumber: 382
/subsetNumber=9/blockNumber
/subsetNumber=9/blockNumber= 1
/subsetNumber=9/stationNumber: 387
/subsetNumber=10/blockNumber
/subsetNumber=10/blockNumber= 1
/subsetNumber=10/stationNumber: 413
/subsetNumber=11/blockNumber
/subsetNumber=11/blockNumber= 1
/subsetNumber=11/stationNumber: 464
/subsetNumber=12/blockNumber
/subsetNumber=12/blockNumber= 1
/subsetNumber=12/stationNumber: 485
EOF
#We check "synop_multi_subset.bufr". The path is
#hardcoded in the example
REDIRECT=/dev/null
REDIRECT=/dev/stdout
#
$PYTHON $examples_src/bufr_subset.py 2> $REDIRECT > $fTmp
#TODO: add a proper check when subsets are properly implemented
#cat $fTmp
diff $fTmp $fRef
#Clean up
rm -f $fTmp
rm -f $fTmp $fRef

View File

@ -30,7 +30,6 @@
MEMBERS = const char* offsetEndSection4Name
MEMBERS = const char* section4LengthName
MEMBERS = const char* numberOfSubsetsName
MEMBERS = const char* subsetNumberName
MEMBERS = const char* expandedDescriptorsName
MEMBERS = const char* flagsName
MEMBERS = const char* unitsName
@ -39,7 +38,6 @@
MEMBERS = bufr_descriptors_array* expanded
MEMBERS = grib_accessor* expandedAccessor
MEMBERS = int* canBeMissing
MEMBERS = long subsetNumber
MEMBERS = long numberOfSubsets
MEMBERS = long compressedData
MEMBERS = grib_vdarray* numericValues
@ -94,7 +92,6 @@ typedef struct grib_accessor_bufr_data_array {
const char* offsetEndSection4Name;
const char* section4LengthName;
const char* numberOfSubsetsName;
const char* subsetNumberName;
const char* expandedDescriptorsName;
const char* flagsName;
const char* unitsName;
@ -103,7 +100,6 @@ typedef struct grib_accessor_bufr_data_array {
bufr_descriptors_array* expanded;
grib_accessor* expandedAccessor;
int* canBeMissing;
long subsetNumber;
long numberOfSubsets;
long compressedData;
grib_vdarray* numericValues;
@ -247,7 +243,6 @@ static void init(grib_accessor* a,const long v, grib_arguments* params)
self->offsetEndSection4Name = grib_arguments_get_name(a->parent->h,params,n++);
self->section4LengthName = grib_arguments_get_name(a->parent->h,params,n++);
self->numberOfSubsetsName = grib_arguments_get_name(a->parent->h,params,n++);
self->subsetNumberName = grib_arguments_get_name(a->parent->h,params,n++);
self->expandedDescriptorsName = grib_arguments_get_name(a->parent->h,params,n++);
self->flagsName = grib_arguments_get_name(a->parent->h,params,n++);
self->elementsDescriptorsIndexName = grib_arguments_get_name(a->parent->h,params,n++);
@ -388,7 +383,6 @@ static int get_descriptors(grib_accessor* a)
ret=grib_get_long(h,self->numberOfSubsetsName,&(self->numberOfSubsets));
ret=grib_get_long(h,self->compressedDataName,&(self->compressedData));
ret=grib_get_long(h,self->subsetNumberName,&(self->subsetNumber));
return ret;
}
@ -804,7 +798,7 @@ static int encode_new_element(grib_context* c,grib_accessor_bufr_data_array* sel
if (self->expanded->v[i]->type==BUFR_DESCRIPTOR_TYPE_STRING) {
/* string */
slen=self->expanded->v[i]->width/8;
csval=grib_context_malloc_clear(c,slen);
csval=grib_context_malloc_clear(c,slen+1);
for (ii=0;ii<slen;ii++) csval[ii]=missingChar;
grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data encoding: \t %s = %s",
self->expanded->v[i]->shortName,csval);
@ -876,7 +870,7 @@ static int encode_element(grib_context* c,grib_accessor_bufr_data_array* self,in
err=encode_string_array(c,buff,pos,i,self,self->stringValues->v[idx]);
} else {
idx=((int)self->numericValues->v[0]->v[elementIndex]/1000-1)/self->numberOfSubsets;
err=encode_string_value(c,buff,pos,i,self,self->stringValues->v[idx]->v[self->subsetNumber]);
err=encode_string_value(c,buff,pos,i,self,self->stringValues->v[idx]->v[subsetIndex]);
}
} else {
/* numeric or codetable or flagtable */
@ -1510,6 +1504,24 @@ static int create_keys(grib_accessor* a)
/* reset_qualifiers(significanceQualifierGroup); */
}
if (ide==0 && !self->compressedData) {
grib_accessor* asn=NULL;
long subsetNumber=iss+1;
size_t len=1;
grib_action creatorsn = {0, };
creatorsn.op = "variable";
creatorsn.name_space = "";
creatorsn.flags = GRIB_ACCESSOR_FLAG_READ_ONLY | GRIB_ACCESSOR_FLAG_DUMP ;
creatorsn.set = 0;
creatorsn.name="subsetNumber";
asn=grib_accessor_factory(section, &creatorsn, 0, NULL);
accessor_variable_set_type(asn,GRIB_TYPE_LONG);
grib_pack_long(asn,&subsetNumber,&len);
grib_push_accessor(asn,section->block);
grib_accessors_list_push(self->dataAccessors,asn);
}
count++;
elementAccessor=create_accessor_from_descriptor(a,associatedFieldAccessor,section,ide,iss,dump,count);
associatedFieldAccessor=NULL;
@ -1854,46 +1866,30 @@ static void dump(grib_accessor* a, grib_dumper* dumper)
static int value_count(grib_accessor* a,long* count)
{
int err=0,l;
long i,subsetNumber=0;
grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a;
grib_context* c=a->parent->h->context;
int err=0,l;
long i,subsetNumber=0;
grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a;
grib_context* c=a->parent->h->context;
err=process_elements(a,PROCESS_DECODE);
if (err) return err;
err=process_elements(a,PROCESS_DECODE);
if (err) return err;
err=grib_get_long(a->parent->h,self->subsetNumberName,&subsetNumber);
if (err) return err;
if (subsetNumber>self->numberOfSubsets) {
err=GRIB_INVALID_KEY_VALUE;
grib_context_log(c,GRIB_LOG_ERROR,"%s=%ld is too big, %s=%ld",self->subsetNumberName,self->numberOfSubsetsName);
return err;
}
if (self->compressedData) {
l=grib_vdarray_used_size(self->numericValues);
if (self->compressedData) {
l=grib_vdarray_used_size(self->numericValues);
*count=l*self->numberOfSubsets;
} else {
*count=0;
for (i=0;i<self->numberOfSubsets;i++)
*count+=grib_iarray_used_size(self->elementsDescriptorsIndex->v[i]);
}
*count=l;
if (subsetNumber<=0) {
*count *= self->numberOfSubsets;
}
} else {
if (subsetNumber>0) {
*count=grib_iarray_used_size(self->elementsDescriptorsIndex->v[subsetNumber-1]);
} else {
*count=0;
for (i=0;i<self->numberOfSubsets;i++)
*count+=grib_iarray_used_size(self->elementsDescriptorsIndex->v[i]);
}
}
return err;
return err;
}
static int unpack_double(grib_accessor* a, double* val, size_t *len)
{
int err=0,i,k,ii;
long n=0;
int proc_flag=PROCESS_DECODE;
size_t l=0,elementsInSubset;
long numberOfSubsets=0;
@ -1907,42 +1903,24 @@ static int unpack_double(grib_accessor* a, double* val, size_t *len)
if (!val) return err;
l=grib_vdarray_used_size(self->numericValues);
err=grib_get_long(a->parent->h,self->subsetNumberName,&n);
if (err) return err;
err=grib_get_long(a->parent->h,self->numberOfSubsetsName,&numberOfSubsets);
if (err) return err;
if (n>numberOfSubsets) {
err=GRIB_INVALID_KEY_VALUE;
grib_context_log(c,GRIB_LOG_ERROR,"%s=%ld is too big, %s=%ld",self->subsetNumberName,self->numberOfSubsetsName);
return err;
}
if (self->compressedData) {
if (n>0) {
for (i=0;i<l;i++) {
val[i]=self->numericValues->v[i]->n > 1 ? self->numericValues->v[i]->v[n-1] : self->numericValues->v[i]->v[0];
}
} else {
ii=0;
for (k=0;k<numberOfSubsets;k++) {
for (i=0;i<l;i++) {
val[ii++]=self->numericValues->v[i]->n > 1 ? self->numericValues->v[i]->v[k] : self->numericValues->v[i]->v[0];
}
}
ii=0;
for (k=0;k<numberOfSubsets;k++) {
for (i=0;i<l;i++) {
val[ii++]=self->numericValues->v[i]->n > 1 ? self->numericValues->v[i]->v[k] : self->numericValues->v[i]->v[0];
}
}
} else {
if (n>0) {
elementsInSubset=grib_iarray_used_size(self->elementsDescriptorsIndex->v[n]);
for (i=0;i<elementsInSubset;i++) val[i]=self->numericValues->v[n-1]->v[i];
} else {
ii=0;
for (k=0;k<numberOfSubsets;k++) {
elementsInSubset=grib_iarray_used_size(self->elementsDescriptorsIndex->v[k]);
for (i=0;i<elementsInSubset;i++) {
val[ii++]=self->numericValues->v[k]->v[i];
}
}
ii=0;
for (k=0;k<numberOfSubsets;k++) {
elementsInSubset=grib_iarray_used_size(self->elementsDescriptorsIndex->v[k]);
for (i=0;i<elementsInSubset;i++) {
val[ii++]=self->numericValues->v[k]->v[i];
}
}
}
return GRIB_SUCCESS;

View File

@ -48,9 +48,14 @@ int grib_encode_string(const unsigned char* bitStream, long *bitOffset, size_t n
int remainder = *bitOffset % 8;
unsigned char c;
unsigned char* p;
char* s=string;
unsigned char mask[] ={ 0, 0x80, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE };
int remainderComplement=8-remainder;
char str[512]={0,};
char *s=str;
Assert(numberOfCharacters<512);
if (string) memcpy(s,string,strlen(string));
/* if (remainder) byteOffset++; */
@ -59,7 +64,7 @@ int grib_encode_string(const unsigned char* bitStream, long *bitOffset, size_t n
p=(unsigned char*)bitStream+byteOffset;
if ( remainder == 0 ) {
memcpy(p,string,numberOfCharacters);
memcpy(p,str,numberOfCharacters);
*bitOffset+=numberOfCharacters*8;
return err;
}

View File

@ -287,6 +287,7 @@ static grib_accessors_list* search_by_condition(grib_handle* h,const char* name,
grib_accessor* data=search_and_cache(h,"dataAccessors",0);
if (data && condition->left) {
al=accessor_bufr_data_array_get_dataAccessors(data);
if (!al) return NULL;
result=(grib_accessors_list*)grib_context_malloc_clear(al->accessor->parent->h->context,sizeof(grib_accessors_list));
search_accessors_list_by_condition(al,name,condition,result);
if (!result->accessor) {

View File

@ -796,16 +796,22 @@ int grib_attributes_count(grib_accessor* a, size_t* size) {
int grib_get_long(grib_handle* h, const char* name, long* val)
{
grib_accessor* act = NULL;
size_t l = 1;
int ret=0;
char* attribute_name=NULL;
size_t length = 1;
grib_accessor* a = NULL;
grib_accessors_list* al=NULL;
int ret=0;
act = grib_find_accessor(h, name);
ret = act ? grib_unpack_long(act, val, &l) : GRIB_NOT_FOUND;
return ret;
if (name[0] == '/' ) {
al=grib_find_accessors_list(h,name);
if (!al) return GRIB_NOT_FOUND;
ret=grib_unpack_long(al->accessor, val , &length);
grib_context_free(h->context,al);
} else {
a=grib_find_accessor(h, name);
if(!a) return GRIB_NOT_FOUND;
ret=grib_unpack_long(a, val , &length);
}
return ret;
}
int grib_get_double_internal(grib_handle* h, const char* name, double* val)

View File

@ -960,3 +960,41 @@ ${tools_dir}bufr_compare -b relativeHumidity,horizontalVisibility ${f}.out $f
rm -f ${f}.out
rm -f $fRules ${fout} $fLog
#-----------------------------------------------------------
# Test: access subsets by condition
#-----------------------------------------------------------
cat > $fRules <<EOF
set unpack=1;
print "stationId=[/subsetNumber=6/blockNumber!%.2d][/subsetNumber=6/stationNumber!%.3d]";
print "latitude=[/subsetNumber=6/latitude]";
print "longitude=[/subsetNumber=6/longitude]";
print "airTemperature=[/subsetNumber=6/airTemperature]";
print "--------";
print "stationId=[/subsetNumber=9/blockNumber!%.2d][/subsetNumber=9/stationNumber!%.3d]";
print "latitude=[/subsetNumber=9/latitude]";
print "longitude=[/subsetNumber=9/longitude]";
print "airTemperature=[/subsetNumber=9/airTemperature]";
EOF
f="synop_multi_subset.bufr"
echo "Test: access subsets by condition" >> $fLog
echo "file: $f" >> $fLog
${tools_dir}bufr_filter $fRules $f 2>> $fLog 1>> $fLog
${tools_dir}bufr_filter $fRules $f 2>> ${f}.log 1>> ${f}.log
cat > ${f}.ref <<EOF
stationId=01371
latitude=61.122
longitude=9.063
airTemperature=265.35
--------
stationId=01387
latitude=61.455
longitude=10.1857
airTemperature=267.55
EOF
diff ${f}.ref ${f}.log
rm -f ${f}.ref ${f}.log

View File

@ -14,7 +14,6 @@ set -x
REDIRECT=/dev/null
cat > bufrdc_num_ref.filter<<EOF
set subsetNumber=0;
print "[numericValues!1%23.14e]";
EOF