mirror of https://github.com/ecmwf/eccodes.git
Merge branch 'develop' of ssh://software.ecmwf.int:7999/eccodes/eccodes into develop
This commit is contained in:
Shahram Najm
commit
913b0e55bd
|
|
@ -17,5 +17,7 @@ transient inputExtendedDelayedDescriptorReplicationFactor={-1};
|
|||
#negative value=not used
|
||||
transient inputShortDelayedDescriptorReplicationFactor={-1};
|
||||
|
||||
transient extractSubset=-1;
|
||||
|
||||
template boot_edition "bufr/boot_edition_[ed:l].def";
|
||||
|
||||
|
|
|
|||
|
|
@ -25,13 +25,18 @@ unsigned[1] dataSubCategory : dump;
|
|||
unsigned[1] masterTablesVersionNumber : dump;
|
||||
unsigned[1] localTablesVersionNumber : dump;
|
||||
|
||||
transient typicalCentury=21;
|
||||
|
||||
unsigned[1] typicalYearOfCentury : dump;
|
||||
transient typicalYear = (typicalCentury - 1 ) *100 + typicalYearOfCentury : no_copy;
|
||||
|
||||
unsigned[1] typicalMonth : dump;
|
||||
unsigned[1] typicalDay : dump;
|
||||
unsigned[1] typicalHour : dump;
|
||||
unsigned[1] typicalMinute : dump;
|
||||
unsigned[1] spare; # ???
|
||||
|
||||
alias ls.centre=bufrHeaderCentre;
|
||||
alias ls.masterTablesVersionNumber=masterTablesVersionNumber;
|
||||
alias ls.localTablesVersionNumber=localTablesVersionNumber;
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,9 @@ unsigned[1] masterTablesVersionNumber : dump;
|
|||
unsigned[1] localTablesVersionNumber : dump;
|
||||
|
||||
unsigned[2] typicalYear : dump;
|
||||
transient typicalYearOfCentury = typicalYear % 100;
|
||||
transient typicalCentury = ( typicalYear - typicalYearOfCentury ) / 100 + 1 ;
|
||||
|
||||
unsigned[1] typicalMonth : dump;
|
||||
unsigned[1] typicalDay : dump;
|
||||
unsigned[1] typicalHour : dump;
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ hash_array sequences (defaultSequence,"sequence.def",tablesMasterDir,tablesLocal
|
|||
#smart_table codeFlags ("codeFlag.def",tablesMasterDir,tablesLocalDir,...);
|
||||
|
||||
position offsetSection3;
|
||||
section_length[3] section3Length ;
|
||||
section_length[3] section3Length = 9 ;
|
||||
unsigned[1] reservedSection3 = 0;
|
||||
unsigned[2] numberOfSubsets : dump;
|
||||
alias ls.numberOfSubsets=numberOfSubsets;
|
||||
|
|
@ -72,12 +72,12 @@ flagbit observedData(section3Flags,7) : dump,no_copy;
|
|||
flagbit compressedData(section3Flags,6) : dump,no_copy;
|
||||
position offsetDescriptors;
|
||||
|
||||
#meta unexpandedDescriptorsEncoded raw(section3Length - 7,totalLength,section3Length) : hidden;
|
||||
|
||||
transient createNewData=1;
|
||||
meta numberOfUnexpandedDescriptors evaluate( (section3Length - 7) / 2 ) : no_copy;
|
||||
#meta unexpandedDescriptors unexpanded_descriptors(unexpandedDescriptorsEncoded,createNewData) :dump,no_copy;
|
||||
meta unexpandedDescriptors unexpanded_descriptors(numberOfUnexpandedDescriptors,createNewData) :dump,no_copy;
|
||||
|
||||
meta unexpandedDescriptorsEncoded raw(totalLength,section3Length,7) : hidden;
|
||||
meta unexpandedDescriptors unexpanded_descriptors(unexpandedDescriptorsEncoded,createNewData) :dump,no_copy;
|
||||
|
||||
concept BufrTemplate (unknown,"BufrTemplate.def",templatesMasterDir,templatesLocalDir) : no_copy;
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ section_length[3] section4Length ;
|
|||
unsigned[1] reservedSection4 = 0;
|
||||
position offsetBeforeData;
|
||||
|
||||
meta bufrDataEncoded raw(section4Length - 4,totalLength,section4Length) : hidden;
|
||||
meta bufrDataEncoded raw(totalLength,section4Length,4) : hidden;
|
||||
|
||||
template dataKeys "bufr/dataKeys.def";
|
||||
|
||||
|
|
|
|||
|
|
@ -1664,7 +1664,7 @@ static int process_elements(grib_accessor* a,int flag)
|
|||
grib_iarray* elementsDescriptorsIndex=0;
|
||||
|
||||
long pos=0,dataOffset=0;
|
||||
int iss,end,elementIndex,index;
|
||||
int iss,end,start,elementIndex,index;
|
||||
long numberOfDescriptors;
|
||||
long totalSize;
|
||||
bufr_descriptor** descriptors=0;
|
||||
|
|
@ -1674,6 +1674,7 @@ static int process_elements(grib_accessor* a,int flag)
|
|||
codec_element_proc codec_element;
|
||||
codec_replication_proc codec_replication;
|
||||
grib_accessor* dataAccessor=NULL;
|
||||
long extractSubset=-1;
|
||||
|
||||
grib_darray* dval = NULL;
|
||||
grib_sarray* sval = NULL;
|
||||
|
|
@ -1718,6 +1719,7 @@ static int process_elements(grib_accessor* a,int flag)
|
|||
pos=0;
|
||||
codec_element=&encode_element;
|
||||
codec_replication=&encode_replication;
|
||||
grib_get_long(grib_handle_of_accessor(a),"extractSubset",&extractSubset);
|
||||
break;
|
||||
default :
|
||||
return GRIB_NOT_IMPLEMENTED;
|
||||
|
|
@ -1744,11 +1746,22 @@ static int process_elements(grib_accessor* a,int flag)
|
|||
self->elementsDescriptorsIndex=grib_viarray_new(c,100,100);
|
||||
}
|
||||
|
||||
end= self->compressedData ? 1 : self->numberOfSubsets;
|
||||
if (self->compressedData == 1 ) {
|
||||
start=0;
|
||||
end=1 ;
|
||||
} else {
|
||||
if (extractSubset<=0) {
|
||||
start=0;
|
||||
end= self->numberOfSubsets;
|
||||
} else {
|
||||
start=extractSubset-1;
|
||||
end= extractSubset;
|
||||
}
|
||||
}
|
||||
|
||||
numberOfDescriptors=grib_bufr_descriptors_array_used_size(self->expanded);
|
||||
|
||||
for (iss=0;iss<end;iss++) {
|
||||
for (iss=start;iss<end;iss++) {
|
||||
icount=1;
|
||||
grib_context_log(c, GRIB_LOG_DEBUG,"BUFR data processing: subsetNumber=%ld", iss+1);
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
IMPLEMENTS = compare;unpack_bytes;byte_count;value_count
|
||||
MEMBERS = const char* totalLength
|
||||
MEMBERS = const char* sectionLength
|
||||
MEMBERS = long relativeOffset
|
||||
END_CLASS_DEF
|
||||
|
||||
*/
|
||||
|
|
@ -49,6 +50,7 @@ typedef struct grib_accessor_raw {
|
|||
/* Members defined in raw */
|
||||
const char* totalLength;
|
||||
const char* sectionLength;
|
||||
long relativeOffset;
|
||||
} grib_accessor_raw;
|
||||
|
||||
extern grib_accessor_class* grib_accessor_class_gen;
|
||||
|
|
@ -134,15 +136,25 @@ static void init_class(grib_accessor_class* c)
|
|||
static void init(grib_accessor* a, const long len , grib_arguments* arg )
|
||||
{
|
||||
int n=0;
|
||||
int err=0;
|
||||
long sectionLength;
|
||||
grib_accessor_raw *self =(grib_accessor_raw*)a;
|
||||
grib_expression* e=grib_arguments_get_expression(grib_handle_of_accessor(a), arg,n++);
|
||||
grib_expression* e;
|
||||
|
||||
a->length=0;
|
||||
grib_expression_evaluate_long(grib_handle_of_accessor(a),e,&(a->length));
|
||||
self->totalLength = grib_arguments_get_name(grib_handle_of_accessor(a),arg,n++);
|
||||
self->sectionLength = grib_arguments_get_name(grib_handle_of_accessor(a),arg,n++);
|
||||
|
||||
Assert(a->length>=0);
|
||||
e=grib_arguments_get_expression(grib_handle_of_accessor(a), arg,n++);
|
||||
grib_expression_evaluate_long(grib_handle_of_accessor(a),e,&(self->relativeOffset));
|
||||
if (err) grib_context_log(grib_handle_of_accessor(a)->context,GRIB_LOG_FATAL,"unable to evaluate relativeOffset");
|
||||
|
||||
grib_get_long(grib_handle_of_accessor(a),self->sectionLength,§ionLength);
|
||||
|
||||
a->length=sectionLength-self->relativeOffset;
|
||||
if (a->length<0) a->length=0;
|
||||
|
||||
/* Assert(a->length>=0); */
|
||||
}
|
||||
|
||||
static int get_native_type(grib_accessor* a){
|
||||
|
|
@ -201,20 +213,21 @@ static int pack_bytes(grib_accessor* a, const unsigned char* val, size_t *len)
|
|||
{
|
||||
size_t length = *len;
|
||||
long totalLength;
|
||||
long section4Length;
|
||||
/* grib_handle* h=grib_handle_of_accessor(a); */
|
||||
long sectionLength;
|
||||
grib_handle* h=grib_handle_of_accessor(a);
|
||||
grib_accessor_raw *self =(grib_accessor_raw*)a;
|
||||
long dlen=length-a->length;
|
||||
|
||||
grib_get_long(grib_handle_of_accessor(a),self->totalLength,&totalLength);
|
||||
totalLength+=length-a->length;
|
||||
grib_get_long(grib_handle_of_accessor(a),self->sectionLength,§ion4Length);
|
||||
section4Length+=length-a->length;
|
||||
grib_get_long(h,self->totalLength,&totalLength);
|
||||
totalLength+=dlen;
|
||||
grib_get_long(h,self->sectionLength,§ionLength);
|
||||
sectionLength+=dlen;
|
||||
|
||||
grib_buffer_replace(a, val, length,1,1);
|
||||
|
||||
grib_set_long(grib_handle_of_accessor(a),self->totalLength,totalLength);
|
||||
grib_set_long(grib_handle_of_accessor(a),self->sectionLength,section4Length);
|
||||
a->length = section4Length-4;
|
||||
grib_set_long(h,self->totalLength,totalLength);
|
||||
grib_set_long(h,self->sectionLength,sectionLength);
|
||||
a->length = length;
|
||||
|
||||
return GRIB_SUCCESS;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,11 +18,10 @@
|
|||
IMPLEMENTS = unpack_long;pack_long
|
||||
IMPLEMENTS = init;dump
|
||||
IMPLEMENTS = next_offset
|
||||
IMPLEMENTS = byte_count
|
||||
IMPLEMENTS = value_count
|
||||
IMPLEMENTS = byte_offset
|
||||
IMPLEMENTS = update_size
|
||||
MEMBERS = const char* numberOfUnexpandedDescriptors
|
||||
MEMBERS = grib_accessor* unexpandedDescriptorsEncoded
|
||||
MEMBERS = const char* createNewData
|
||||
|
||||
END_CLASS_DEF
|
||||
|
|
@ -41,7 +40,6 @@ or edit "accessor.class" and rerun ./make_class.pl
|
|||
|
||||
static int pack_long(grib_accessor*, const long* val,size_t *len);
|
||||
static int unpack_long(grib_accessor*, long* val,size_t *len);
|
||||
static long byte_count(grib_accessor*);
|
||||
static long byte_offset(grib_accessor*);
|
||||
static long next_offset(grib_accessor*);
|
||||
static int value_count(grib_accessor*,long*);
|
||||
|
|
@ -55,7 +53,7 @@ typedef struct grib_accessor_unexpanded_descriptors {
|
|||
/* Members defined in gen */
|
||||
/* Members defined in long */
|
||||
/* Members defined in unexpanded_descriptors */
|
||||
const char* numberOfUnexpandedDescriptors;
|
||||
grib_accessor* unexpandedDescriptorsEncoded;
|
||||
const char* createNewData;
|
||||
} grib_accessor_unexpanded_descriptors;
|
||||
|
||||
|
|
@ -74,7 +72,7 @@ static grib_accessor_class _grib_accessor_class_unexpanded_descriptors = {
|
|||
&next_offset, /* get length of section */
|
||||
0, /* get length of string */
|
||||
&value_count, /* get number of values */
|
||||
&byte_count, /* get number of bytes */
|
||||
0, /* get number of bytes */
|
||||
&byte_offset, /* get offset to bytes */
|
||||
0, /* get native type */
|
||||
0, /* get sub_section */
|
||||
|
|
@ -111,6 +109,7 @@ grib_accessor_class* grib_accessor_class_unexpanded_descriptors = &_grib_accesso
|
|||
static void init_class(grib_accessor_class* c)
|
||||
{
|
||||
c->string_length = (*(c->super))->string_length;
|
||||
c->byte_count = (*(c->super))->byte_count;
|
||||
c->get_native_type = (*(c->super))->get_native_type;
|
||||
c->sub_section = (*(c->super))->sub_section;
|
||||
c->pack_missing = (*(c->super))->pack_missing;
|
||||
|
|
@ -138,28 +137,13 @@ static void init_class(grib_accessor_class* c)
|
|||
|
||||
/* END_CLASS_IMP */
|
||||
|
||||
static long compute_byte_count(grib_accessor* a){
|
||||
grib_accessor_unexpanded_descriptors* self = (grib_accessor_unexpanded_descriptors*)a;
|
||||
long numberOfUnexpandedDescriptors;
|
||||
int ret=0;
|
||||
|
||||
ret=grib_get_long(grib_handle_of_accessor(a),self->numberOfUnexpandedDescriptors,&numberOfUnexpandedDescriptors);
|
||||
if (ret) {
|
||||
grib_context_log(a->context,GRIB_LOG_ERROR,
|
||||
"%s unable to get %s to compute size",a->name,self->numberOfUnexpandedDescriptors);
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 2*numberOfUnexpandedDescriptors;
|
||||
}
|
||||
|
||||
static void init(grib_accessor* a, const long len , grib_arguments* args )
|
||||
{
|
||||
grib_accessor_unexpanded_descriptors* self = (grib_accessor_unexpanded_descriptors*)a;
|
||||
int n=0;
|
||||
self->numberOfUnexpandedDescriptors=grib_arguments_get_name(grib_handle_of_accessor(a),args,n++);
|
||||
self->unexpandedDescriptorsEncoded=grib_find_accessor(grib_handle_of_accessor(a),grib_arguments_get_name(grib_handle_of_accessor(a),args,n++));
|
||||
self->createNewData=grib_arguments_get_name(grib_handle_of_accessor(a),args,n++);
|
||||
a->length = compute_byte_count(a);
|
||||
a->length = 0;
|
||||
}
|
||||
|
||||
static void dump(grib_accessor* a, grib_dumper* dumper)
|
||||
|
|
@ -171,13 +155,15 @@ static int unpack_long (grib_accessor* a, long* val, size_t *len)
|
|||
{
|
||||
grib_accessor_unexpanded_descriptors* self = (grib_accessor_unexpanded_descriptors*)a;
|
||||
int ret=0;
|
||||
long pos = a->offset*8;
|
||||
long pos = 0;
|
||||
long rlen =0;
|
||||
long numberOfUnexpandedDescriptors = 0;
|
||||
long f,x,y;
|
||||
long *v=val;
|
||||
int i;
|
||||
|
||||
pos=accessor_raw_get_offset(self->unexpandedDescriptorsEncoded)*8;
|
||||
|
||||
ret=value_count(a,&rlen);
|
||||
if (ret) return ret;
|
||||
|
||||
|
|
@ -195,9 +181,6 @@ static int unpack_long (grib_accessor* a, long* val, size_t *len)
|
|||
return GRIB_ARRAY_TOO_SMALL;
|
||||
}
|
||||
|
||||
ret=grib_get_long(grib_handle_of_accessor(a),self->numberOfUnexpandedDescriptors,&numberOfUnexpandedDescriptors);
|
||||
if (ret) return ret;
|
||||
|
||||
for (i=0;i<rlen;i++) {
|
||||
f=grib_decode_unsigned_long(grib_handle_of_accessor(a)->buffer->data,&pos,2);
|
||||
x=grib_decode_unsigned_long(grib_handle_of_accessor(a)->buffer->data,&pos,6);
|
||||
|
|
@ -233,15 +216,9 @@ static int pack_long (grib_accessor* a, const long* val, size_t *len)
|
|||
grib_encode_unsigned_longb(buf,y,&pos,8);
|
||||
}
|
||||
|
||||
grib_get_long(grib_handle_of_accessor(a),"section3Length",§ion3Length);
|
||||
section3Length+=buflen-a->length;
|
||||
grib_get_long(grib_handle_of_accessor(a),"totalLength",&totalLength);
|
||||
totalLength+=buflen-a->length;
|
||||
grib_pack_bytes(self->unexpandedDescriptorsEncoded,buf,&buflen);
|
||||
grib_context_free(grib_handle_of_accessor(a)->context,buf);
|
||||
|
||||
grib_buffer_replace(a,buf,buflen,1,1);
|
||||
|
||||
grib_set_long(grib_handle_of_accessor(a),"totalLength",totalLength);
|
||||
grib_set_long(grib_handle_of_accessor(a),"section3Length",section3Length);
|
||||
if (createNewData==0) return ret;
|
||||
|
||||
expanded=grib_find_accessor(grib_handle_of_accessor(a),"expandedCodes");
|
||||
|
|
@ -253,16 +230,16 @@ static int pack_long (grib_accessor* a, const long* val, size_t *len)
|
|||
return ret;
|
||||
}
|
||||
|
||||
static long byte_count(grib_accessor* a){
|
||||
return a->length;
|
||||
}
|
||||
|
||||
static int value_count(grib_accessor* a,long* numberOfUnexpandedDescriptors)
|
||||
{
|
||||
grib_accessor_unexpanded_descriptors* self = (grib_accessor_unexpanded_descriptors*)a;
|
||||
*numberOfUnexpandedDescriptors=0;
|
||||
long n=0;
|
||||
|
||||
return grib_get_long(grib_handle_of_accessor(a),self->numberOfUnexpandedDescriptors,numberOfUnexpandedDescriptors);
|
||||
grib_value_count(self->unexpandedDescriptorsEncoded,&n);
|
||||
*numberOfUnexpandedDescriptors=n/2;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static long byte_offset(grib_accessor* a){
|
||||
|
|
|
|||
|
|
@ -1437,3 +1437,4 @@ int grib_encode_double_array_complex(size_t n_vals, double *val, long nbits, dou
|
|||
|
||||
|
||||
long accessor_raw_get_offset(grib_accessor* a);
|
||||
void accessor_raw_set_length(grib_accessor* a,size_t len);
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ int grib_init_accessor_from_handle(grib_loader* loader,grib_accessor* ga,grib_ar
|
|||
const char* name = NULL;
|
||||
int k = 0;
|
||||
grib_handle *g;
|
||||
grib_accessor* ao=NULL;
|
||||
int e, pack_missing = 0;
|
||||
grib_context_log(h->context,GRIB_LOG_DEBUG, "XXXXX Copying %s", ga->name);
|
||||
|
||||
|
|
@ -246,9 +247,11 @@ int grib_init_accessor_from_handle(grib_loader* loader,grib_accessor* ga,grib_ar
|
|||
|
||||
case GRIB_TYPE_BYTES:
|
||||
|
||||
len=grib_byte_count(ga);
|
||||
ao=grib_find_accessor(h,name);
|
||||
len=grib_byte_count(ao);
|
||||
uval = (unsigned char*)grib_context_malloc(h->context,len*sizeof(char));
|
||||
ret = grib_get_bytes_internal(h,name,uval,&len);
|
||||
ret=grib_unpack_bytes(ao,uval,&len);
|
||||
/* ret = grib_get_bytes_internal(h,name,uval,&len); */
|
||||
if(ret == GRIB_SUCCESS)
|
||||
{
|
||||
grib_context_log(h->context,GRIB_LOG_DEBUG, "Copying %d byte(s) to %s", len, name);
|
||||
|
|
|
|||
1947
src/grib_yacc.c
1947
src/grib_yacc.c
File diff suppressed because it is too large
Load Diff
|
|
@ -500,6 +500,9 @@ simple : UNSIGNED '[' INTEGER ']' IDENT default flags
|
|||
| SECTION_LENGTH '[' INTEGER ']' IDENT
|
||||
{ $$ = grib_action_create_gen(grib_parser_context,$5,"section_length",$3,NULL,NULL,0,NULL,NULL);free($5); }
|
||||
|
||||
| SECTION_LENGTH '[' INTEGER ']' IDENT default
|
||||
{ $$ = grib_action_create_gen(grib_parser_context,$5,"section_length",$3,NULL,$6,0,NULL,NULL);free($5); }
|
||||
|
||||
| G1_MESSAGE_LENGTH '[' INTEGER ']' IDENT '(' argument_list ')'
|
||||
{ $$ = grib_action_create_gen(grib_parser_context,$5,"g1_message_length",$3,$7,NULL,0,NULL,NULL);free($5); }
|
||||
|
||||
|
|
|
|||
|
|
@ -400,6 +400,7 @@ rm -f ${f}.ref ${f}.log
|
|||
fRulesReady="set_unexpandedDescriptors.filter"
|
||||
f="syno_1.bufr"
|
||||
testScript="set_unexpandedDescriptors_test.sh"
|
||||
testScript1="set_unexpandedDescriptors_test_1.sh"
|
||||
echo "Test: set unexpandedDescriptors big test" >> $fLog
|
||||
echo "file: $f" >> $fLog
|
||||
|
||||
|
|
@ -407,14 +408,17 @@ cat >$testScript <<EOF
|
|||
set -e
|
||||
|
||||
EOF
|
||||
chmod +x $testScript
|
||||
echo "set -x" > $testScript1
|
||||
chmod +x $testScript1
|
||||
|
||||
${tools_dir}bufr_filter $fRulesReady $f 2>> $fLog 1>> $testScript
|
||||
|
||||
./$testScript
|
||||
sed -e "s:diff:${tools_dir}bufr_compare:" < $testScript >> $testScript1
|
||||
|
||||
./$testScript1
|
||||
|
||||
rm -f new_*bufr
|
||||
rm -f $testScript
|
||||
rm -f $testScript $testScript1
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# Test: packing
|
||||
|
|
|
|||
Loading…
Reference in New Issue