mirror of https://github.com/ecmwf/eccodes.git
ECC-219 implemented for compressed data and added test
This commit is contained in:
Enrico Fucile
parent
a3b724b1c7
commit
a3942899a6
|
|
@ -58,6 +58,8 @@
|
|||
MEMBERS = long* inputShortReplications
|
||||
MEMBERS = int nInputShortReplications
|
||||
MEMBERS = int iInputShortReplications
|
||||
MEMBERS = int start
|
||||
MEMBERS = int end
|
||||
|
||||
END_CLASS_DEF
|
||||
|
||||
|
|
@ -125,6 +127,8 @@ typedef struct grib_accessor_bufr_data_array {
|
|||
long* inputShortReplications;
|
||||
int nInputShortReplications;
|
||||
int iInputShortReplications;
|
||||
int start;
|
||||
int end;
|
||||
} grib_accessor_bufr_data_array;
|
||||
|
||||
extern grib_accessor_class* grib_accessor_class_gen;
|
||||
|
|
@ -495,22 +499,29 @@ static grib_darray* decode_double_array(grib_context* c,unsigned char* data,long
|
|||
static int encode_string_array(grib_context* c,grib_buffer* buff,long* pos, int i,
|
||||
grib_accessor_bufr_data_array* self,grib_sarray* stringValues)
|
||||
{
|
||||
int err=0,end,start;
|
||||
int err=0,end,start,n;
|
||||
int j,modifiedWidth,width;
|
||||
|
||||
start=0;
|
||||
end=grib_sarray_used_size(stringValues);
|
||||
start=self->start;
|
||||
end=self->end;
|
||||
n=end-start;
|
||||
if (n<=0) return GRIB_NO_VALUES;
|
||||
|
||||
if (grib_sarray_used_size(stringValues)==1) n=1;
|
||||
|
||||
if (n>grib_sarray_used_size(stringValues))
|
||||
return GRIB_ARRAY_TOO_SMALL;
|
||||
|
||||
modifiedWidth= self->expanded->v[i]->width;
|
||||
|
||||
grib_buffer_set_ulength_bits(c,buff,buff->ulength_bits+modifiedWidth);
|
||||
grib_encode_string(buff->data,pos,modifiedWidth/8,stringValues->v[0]);
|
||||
width= end > 1 ? modifiedWidth : 0;
|
||||
width= n > 1 ? modifiedWidth : 0;
|
||||
|
||||
grib_buffer_set_ulength_bits(c,buff,buff->ulength_bits+6);
|
||||
grib_encode_unsigned_longb(buff->data,width/8,pos,6);
|
||||
if (width) {
|
||||
grib_buffer_set_ulength_bits(c,buff,buff->ulength_bits+width*end);
|
||||
grib_buffer_set_ulength_bits(c,buff,buff->ulength_bits+width*n);
|
||||
for (j=start;j<end;j++) {
|
||||
grib_encode_string(buff->data,pos,width/8,stringValues->v[j]);
|
||||
}
|
||||
|
|
@ -527,7 +538,7 @@ static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos,int i
|
|||
long localReference=0,localWidth=0,modifiedWidth,modifiedReference;
|
||||
long reference,allone;
|
||||
double localRange,modifiedFactor,inverseFactor;
|
||||
size_t nvals,ii;
|
||||
size_t nvals,start,ii;
|
||||
double min,max,maxAllowed,minAllowed;
|
||||
double* v=NULL;
|
||||
int thereIsAMissing=0;
|
||||
|
|
@ -540,9 +551,32 @@ static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos,int i
|
|||
maxAllowed=(grib_power(modifiedWidth,2)+modifiedReference)*modifiedFactor;
|
||||
minAllowed=modifiedReference*modifiedFactor;
|
||||
|
||||
nvals=grib_darray_used_size(dvalues);
|
||||
|
||||
start=self->start;
|
||||
nvals=self->end-self->start;
|
||||
if (nvals<=0) return GRIB_NO_VALUES;
|
||||
|
||||
v=dvalues->v;
|
||||
|
||||
/* is constant */
|
||||
if (grib_darray_used_size(dvalues)==1) {
|
||||
localWidth=0;
|
||||
grib_buffer_set_ulength_bits(c,buff,buff->ulength_bits+modifiedWidth);
|
||||
if (*v == GRIB_MISSING_DOUBLE) {
|
||||
grib_set_bits_on(buff->data,pos,modifiedWidth);
|
||||
} else {
|
||||
lval=round(*v * inverseFactor)-modifiedReference;
|
||||
grib_encode_unsigned_longb(buff->data,lval,pos,modifiedWidth);
|
||||
}
|
||||
grib_buffer_set_ulength_bits(c,buff,buff->ulength_bits+6);
|
||||
grib_encode_unsigned_longb(buff->data,localWidth,pos,6);
|
||||
return err;
|
||||
}
|
||||
|
||||
if (nvals>grib_darray_used_size(dvalues)) return GRIB_ARRAY_TOO_SMALL;
|
||||
v=dvalues->v+start;
|
||||
|
||||
/* encoding only one value out of many*/
|
||||
if (nvals==1) {
|
||||
localWidth=0;
|
||||
grib_buffer_set_ulength_bits(c,buff,buff->ulength_bits+modifiedWidth);
|
||||
|
|
@ -609,8 +643,8 @@ static int encode_double_array(grib_context* c,grib_buffer* buff,long* pos,int i
|
|||
grib_encode_unsigned_longb(buff->data,localWidth,pos,6);
|
||||
|
||||
if (localWidth) {
|
||||
grib_buffer_set_ulength_bits(c,buff,buff->ulength_bits+self->numberOfSubsets*localWidth);
|
||||
for (j=0;j<self->numberOfSubsets;j++) {
|
||||
grib_buffer_set_ulength_bits(c,buff,buff->ulength_bits+nvals*localWidth);
|
||||
for (j=self->start;j<self->end;j++) {
|
||||
if (dvalues->v[j]==GRIB_MISSING_DOUBLE) {
|
||||
grib_set_bits_on(buff->data,pos,localWidth);
|
||||
} else {
|
||||
|
|
@ -1387,21 +1421,22 @@ static void grib_convert_to_attribute(grib_accessor* a)
|
|||
|
||||
static void set_subset_start_end(grib_accessor_bufr_data_array *self,long *onlySubset,long *startSubset,long *endSubset,long *ret_start,long *ret_end)
|
||||
{
|
||||
if (self->compressedData == 1 ) {
|
||||
*ret_start=0;
|
||||
*ret_end=1 ;
|
||||
} else {
|
||||
if (*startSubset>0 && *endSubset>=*startSubset) {
|
||||
*ret_start=*startSubset-1;
|
||||
*ret_end= *endSubset;
|
||||
} else if (*onlySubset>0) {
|
||||
*ret_start=*onlySubset-1;
|
||||
*ret_end= *onlySubset;
|
||||
} else {
|
||||
*ret_start=0;
|
||||
*ret_end= self->numberOfSubsets;
|
||||
}
|
||||
}
|
||||
if (*startSubset>0 && *endSubset>=*startSubset) {
|
||||
*ret_start=*startSubset-1;
|
||||
*ret_end= *endSubset;
|
||||
} else if (*onlySubset>0) {
|
||||
*ret_start=*onlySubset-1;
|
||||
*ret_end= *onlySubset;
|
||||
} else {
|
||||
*ret_start=0;
|
||||
*ret_end= self->numberOfSubsets;
|
||||
}
|
||||
self->start=*ret_start;
|
||||
self->end=*ret_end;
|
||||
if (self->compressedData==1) {
|
||||
*ret_start=0;
|
||||
*ret_end=1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -2029,8 +2064,8 @@ static int process_elements(grib_accessor* a,int flag,long onlySubset,long start
|
|||
self->bitsToEndData=buffer->ulength*8;
|
||||
grib_set_bytes(grib_handle_of_accessor(a),self->bufrDataEncodedName,buffer->data,&(buffer->ulength));
|
||||
grib_buffer_delete(c,buffer);
|
||||
if (self->compressedData==0 && self->numberOfSubsets != (end-start) ) {
|
||||
grib_set_long(h,self->numberOfSubsetsName,end-start);
|
||||
if (self->numberOfSubsets != (self->end-self->start) ) {
|
||||
grib_set_long(h,self->numberOfSubsetsName,self->end-self->start);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -918,7 +918,7 @@ rm -f ${fOut}.log
|
|||
rm -f $fLog $fRules ${fOut}
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# Test: extract subsets
|
||||
# Test: extract subsets uncompressed data
|
||||
#-----------------------------------------------------------
|
||||
cat > $fRules <<EOF
|
||||
set unpack=1;
|
||||
|
|
@ -940,7 +940,7 @@ EOF
|
|||
f="synop_multi_subset.bufr"
|
||||
fOut="extract.bufr"
|
||||
|
||||
echo "Test: extract subsets" >> $fLog
|
||||
echo "Test: extract subsets uncompressed data" >> $fLog
|
||||
echo "file: $f" >> $fLog
|
||||
${tools_dir}bufr_filter -o ${fOut} $fRules $f 2>> $fLog 1>> $fLog
|
||||
|
||||
|
|
@ -989,3 +989,58 @@ diff ${f}.log.ref ${f}.log
|
|||
rm -f ${f}.log ${f}.log.ref
|
||||
rm -f $fLog $fRules
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# Test: extract subsets compressed data
|
||||
#-----------------------------------------------------------
|
||||
cat > $fRules <<EOF
|
||||
set unpack=1;
|
||||
|
||||
set extractSubset=10;
|
||||
set doExtractSubsets=1;
|
||||
write;
|
||||
|
||||
set extractSubsetIntervalStart=3;
|
||||
set extractSubsetIntervalEnd=8;
|
||||
set doExtractSubsets=1;
|
||||
write;
|
||||
EOF
|
||||
|
||||
f="g2nd_208.bufr"
|
||||
fOut="g2nd_208.bufr.out"
|
||||
|
||||
echo "Test: extract subsets compressed data" >> $fLog
|
||||
echo "file: $f" >> $fLog
|
||||
${tools_dir}bufr_filter -o $fOut $fRules $f 2>> $fLog 1>> $fLog
|
||||
|
||||
cat > ${fRules} <<EOF
|
||||
set unpack=1;
|
||||
print "=== message number [count]";
|
||||
print "numberOfSubsets=[numberOfSubsets]";
|
||||
print "solarElevation=[solarElevation!10]";
|
||||
print "fieldOfViewNumber=[fieldOfViewNumber!10]";
|
||||
print "orbitNumber=[orbitNumber!10]";
|
||||
print "casRegistryNumber=[casRegistryNumber!10]";
|
||||
EOF
|
||||
|
||||
${tools_dir}bufr_filter $fRules $fOut > ${f}.log
|
||||
|
||||
cat > ${f}.log.ref <<EOF
|
||||
=== message number 1
|
||||
numberOfSubsets=1
|
||||
solarElevation=33.2
|
||||
fieldOfViewNumber=1
|
||||
orbitNumber=2147483647
|
||||
casRegistryNumber=10102-44-0
|
||||
=== message number 2
|
||||
numberOfSubsets=6
|
||||
solarElevation=29.71 29.23 37.21 36.78 36.34 35.46
|
||||
fieldOfViewNumber=2 2 0 0 0 1
|
||||
orbitNumber=2147483647
|
||||
casRegistryNumber=10102-44-0
|
||||
EOF
|
||||
|
||||
diff ${f}.log.ref ${f}.log
|
||||
|
||||
rm -f ${f}.log ${f}.log.ref
|
||||
rm -f $fLog $fOut $fRules
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue