From fffcfdbab4dc81f5c5f523a8d655c115f7b19dc2 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Wed, 31 May 2017 17:21:18 +0100 Subject: [PATCH] Merge of ECC-420-performance. Without the last set of commits (less mallocs) --- definitions/bufr/section.3.def | 5 + src/CMakeLists.txt | 3 + src/Makefile.am | 3 + src/eccodes.h | 1 + src/grib_accessor.c | 5 - src/grib_accessor_class.h | 1 + src/grib_accessor_class_bufr_clear_tables.c | 143 +++++ src/grib_accessor_class_bufr_data_array.c | 41 +- src/grib_accessor_class_bufr_group.c | 9 +- ...grib_accessor_class_expanded_descriptors.c | 94 +++- src/grib_accessor_class_forward.c | 10 +- src/grib_accessor_class_library_version.c | 8 +- src/grib_accessor_factory.h | 1 + src/grib_accessor_factory_hash_list | 1 + src/grib_api.h | 2 + src/grib_api_internal.h | 18 + src/grib_api_prototypes.h | 21 +- src/grib_buffer.c | 12 +- src/grib_context.c | 3 +- src/grib_expression_class_accessor.c | 1 - src/grib_expression_class_binop.c | 1 - src/grib_expression_class_column.c | 1 - src/grib_expression_class_constant.c | 1 - src/grib_expression_class_double.c | 1 - src/grib_expression_class_functor.c | 1 - src/grib_expression_class_is_in_dict.c | 1 - src/grib_expression_class_is_in_list.c | 1 - src/grib_expression_class_is_integer.c | 1 - src/grib_expression_class_length.c | 1 - src/grib_expression_class_long.c | 1 - src/grib_expression_class_string.c | 1 - src/grib_expression_class_string_compare.c | 1 - src/grib_expression_class_sub_string.c | 1 - src/grib_expression_class_true.c | 1 - src/grib_expression_class_unop.c | 1 - src/grib_oarray.c | 108 ++++ src/grib_query.c | 14 +- src/grib_trie_with_rank.c | 515 ++++++++++++++++++ tools/grib_tools.c | 12 +- 39 files changed, 947 insertions(+), 99 deletions(-) create mode 100644 src/grib_accessor_class_bufr_clear_tables.c create mode 100644 src/grib_oarray.c create mode 100644 src/grib_trie_with_rank.c diff --git a/definitions/bufr/section.3.def b/definitions/bufr/section.3.def index 9a070d524..522d75992 100644 --- a/definitions/bufr/section.3.def +++ b/definitions/bufr/section.3.def @@ -12,6 +12,11 @@ constant tablesMasterDir="bufr/tables/[masterTableNumber]/wmo/[masterTablesVersi constant tablesLocalDir="bufr/tables/[masterTableNumber]/local/[localTablesVersionNumber]/[bufrHeaderCentre:l]/[bufrHeaderSubCentre]" : hidden; constant rootTablesDir="bufr/tables" : hidden; +meta clearTables bufr_clear_tables() = 0 : hidden,no_copy; + +when (changed(tablesMasterDir)) { set clearTables=1;} +when (changed(tablesLocalDir)) { set clearTables=1;} + # we should implement some table version dependency #constant templatesMasterDir="bufr/templates/[masterTableNumber]/[masterTablesVersionNumber]"; #constant templatesLocalDir="bufr/templates/local/[masterTableNumber]/[bufrHeaderCentre:l]/[localTablesVersionNumber]"; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d162de05e..04b8c752c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -51,6 +51,7 @@ list( APPEND grib_api_srcs grib_bufr_descriptors_array.c grib_darray.c grib_sarray.c + grib_oarray.c grib_vdarray.c grib_vsarray.c grib_iarray.c @@ -63,6 +64,7 @@ list( APPEND grib_api_srcs grib_accessor_class_bitmap.c grib_accessor_class_bits.c grib_accessor_class_bits_per_value.c + grib_accessor_class_bufr_clear_tables.c grib_accessor_class_bufr_data.c grib_accessor_class_bufr_elements_table.c grib_accessor_class_bufr_data_array.c @@ -299,6 +301,7 @@ list( APPEND grib_api_srcs grib_hash_keys.c grib_io.c grib_trie.c + grib_trie_with_rank.c grib_itrie.c grib_rules.c grib_keys_iterator.c diff --git a/src/Makefile.am b/src/Makefile.am index ad5f441d7..4125d7af3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -52,6 +52,7 @@ libeccodes_la_prototypes= \ grib_bufr_descriptors_array.c \ grib_darray.c \ grib_sarray.c \ + grib_oarray.c \ grib_iarray.c \ grib_vdarray.c \ grib_vsarray.c \ @@ -64,6 +65,7 @@ libeccodes_la_prototypes= \ grib_accessor_class_bits.c \ grib_accessor_class_bits_per_value.c \ grib_accessor_class_bufr_data.c \ + grib_accessor_class_bufr_clear_tables.c \ grib_accessor_class_bufr_data_array.c \ grib_accessor_class_bufr_data_element.c \ grib_accessor_class_bufr_elements_table.c \ @@ -314,6 +316,7 @@ libeccodes_la_prototypes= \ grib_hash_keys.c \ grib_io.c \ grib_trie.c \ + grib_trie_with_rank.c \ grib_itrie.c \ grib_rules.c \ grib_keys_iterator.c \ diff --git a/src/eccodes.h b/src/eccodes.h index 2e5d1e407..2e16d5d0f 100644 --- a/src/eccodes.h +++ b/src/eccodes.h @@ -170,6 +170,7 @@ typedef struct grib_fieldset codes_fieldset; typedef struct grib_order_by codes_order_by; typedef struct grib_where codes_where; typedef struct grib_sarray codes_sarray; +typedef struct grib_oarray codes_oarray; typedef struct grib_darray codes_darray; typedef struct grib_iarray codes_iarray; typedef struct grib_vdarray codes_vdarray; diff --git a/src/grib_accessor.c b/src/grib_accessor.c index 1242ea8f1..831db6077 100644 --- a/src/grib_accessor.c +++ b/src/grib_accessor.c @@ -463,11 +463,6 @@ static void init_accessor(grib_accessor_class* c,grib_accessor* a, const long le { if(c) { grib_accessor_class *s = c->super ? *(c->super) : NULL; - if(!c->inited) - { - if(c->init_class) c->init_class(c); - c->inited = 1; - } init_accessor(s,a,len,args); if(c->init) c->init(a,len, args); } diff --git a/src/grib_accessor_class.h b/src/grib_accessor_class.h index 95f59228a..a1fbbcdcd 100644 --- a/src/grib_accessor_class.h +++ b/src/grib_accessor_class.h @@ -12,6 +12,7 @@ extern grib_accessor_class* grib_accessor_class_bits_per_value; extern grib_accessor_class* grib_accessor_class_blob; extern grib_accessor_class* grib_accessor_class_box; extern grib_accessor_class* grib_accessor_class_budgdate; +extern grib_accessor_class* grib_accessor_class_bufr_clear_tables; extern grib_accessor_class* grib_accessor_class_bufr_data; extern grib_accessor_class* grib_accessor_class_bufr_data_array; extern grib_accessor_class* grib_accessor_class_bufr_data_element; diff --git a/src/grib_accessor_class_bufr_clear_tables.c b/src/grib_accessor_class_bufr_clear_tables.c new file mode 100644 index 000000000..eb0fce072 --- /dev/null +++ b/src/grib_accessor_class_bufr_clear_tables.c @@ -0,0 +1,143 @@ +/* + * Copyright 2005-2017 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by + * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. + */ + + +/************************************* + * Enrico Fucile + **************************************/ + +#include "grib_api_internal.h" +/* + This is used by make_class.pl + + START_CLASS_DEF + CLASS = accessor + SUPER = grib_accessor_class_gen + IMPLEMENTS = pack_long; get_native_type + END_CLASS_DEF + + */ + +/* START_CLASS_IMP */ + +/* + +Don't edit anything between START_CLASS_IMP and END_CLASS_IMP +Instead edit values between START_CLASS_DEF and END_CLASS_DEF +or edit "accessor.class" and rerun ./make_class.pl + +*/ + +static int get_native_type(grib_accessor*); +static int pack_long(grib_accessor*, const long* val,size_t *len); +static void init_class(grib_accessor_class*); + +typedef struct grib_accessor_bufr_clear_tables { + grib_accessor att; +/* Members defined in gen */ +/* Members defined in bufr_clear_tables */ +} grib_accessor_bufr_clear_tables; + +extern grib_accessor_class* grib_accessor_class_gen; + +static grib_accessor_class _grib_accessor_class_bufr_clear_tables = { + &grib_accessor_class_gen, /* super */ + "bufr_clear_tables", /* name */ + sizeof(grib_accessor_bufr_clear_tables), /* size */ + 0, /* inited */ + &init_class, /* init_class */ + 0, /* init */ + 0, /* post_init */ + 0, /* free mem */ + 0, /* describes himself */ + 0, /* get length of section */ + 0, /* get length of string */ + 0, /* get number of values */ + 0, /* get number of bytes */ + 0, /* get offset to bytes */ + &get_native_type, /* get native type */ + 0, /* get sub_section */ + 0, /* grib_pack procedures long */ + 0, /* grib_pack procedures long */ + &pack_long, /* grib_pack procedures long */ + 0, /* grib_unpack procedures long */ + 0, /* grib_pack procedures double */ + 0, /* grib_unpack procedures double */ + 0, /* grib_pack procedures string */ + 0, /* grib_unpack procedures string */ + 0, /* grib_pack array procedures string */ + 0, /* grib_unpack array procedures string */ + 0, /* grib_pack procedures bytes */ + 0, /* grib_unpack procedures bytes */ + 0, /* pack_expression */ + 0, /* notify_change */ + 0, /* update_size */ + 0, /* preferred_size */ + 0, /* resize */ + 0, /* nearest_smaller_value */ + 0, /* next accessor */ + 0, /* compare vs. another accessor */ + 0, /* unpack only ith value */ + 0, /* unpack a subarray */ + 0, /* clear */ + 0, /* clone accessor */ +}; + + +grib_accessor_class* grib_accessor_class_bufr_clear_tables = &_grib_accessor_class_bufr_clear_tables; + + +static void init_class(grib_accessor_class* c) +{ + c->dump = (*(c->super))->dump; + c->next_offset = (*(c->super))->next_offset; + c->string_length = (*(c->super))->string_length; + c->value_count = (*(c->super))->value_count; + c->byte_count = (*(c->super))->byte_count; + c->byte_offset = (*(c->super))->byte_offset; + c->sub_section = (*(c->super))->sub_section; + c->pack_missing = (*(c->super))->pack_missing; + c->is_missing = (*(c->super))->is_missing; + c->unpack_long = (*(c->super))->unpack_long; + c->pack_double = (*(c->super))->pack_double; + c->unpack_double = (*(c->super))->unpack_double; + c->pack_string = (*(c->super))->pack_string; + c->unpack_string = (*(c->super))->unpack_string; + c->pack_string_array = (*(c->super))->pack_string_array; + c->unpack_string_array = (*(c->super))->unpack_string_array; + c->pack_bytes = (*(c->super))->pack_bytes; + c->unpack_bytes = (*(c->super))->unpack_bytes; + c->pack_expression = (*(c->super))->pack_expression; + c->notify_change = (*(c->super))->notify_change; + c->update_size = (*(c->super))->update_size; + c->preferred_size = (*(c->super))->preferred_size; + c->resize = (*(c->super))->resize; + c->nearest_smaller_value = (*(c->super))->nearest_smaller_value; + c->next = (*(c->super))->next; + c->compare = (*(c->super))->compare; + c->unpack_double_element = (*(c->super))->unpack_double_element; + c->unpack_double_subarray = (*(c->super))->unpack_double_subarray; + c->clear = (*(c->super))->clear; + c->make_clone = (*(c->super))->make_clone; +} + +/* END_CLASS_IMP */ + +static int pack_long(grib_accessor* a, const long* val,size_t *len) +{ + grib_handle* h=grib_handle_of_accessor(a); + h->bufr_elements_table=NULL; + return GRIB_SUCCESS; +} + +static int get_native_type(grib_accessor* a) +{ + return GRIB_TYPE_UNDEFINED; +} diff --git a/src/grib_accessor_class_bufr_data_array.c b/src/grib_accessor_class_bufr_data_array.c index ea52758c3..0cb4d05bb 100644 --- a/src/grib_accessor_class_bufr_data_array.c +++ b/src/grib_accessor_class_bufr_data_array.c @@ -62,8 +62,7 @@ MEMBERS = int nInputShortReplications MEMBERS = int iInputShortReplications MEMBERS = grib_iarray* iss_list - MEMBERS = grib_trie* dataAccessorsTrie - MEMBERS = grib_trie* dataAccessorsRank + MEMBERS = grib_trie_with_rank* dataAccessorsTrie MEMBERS = grib_sarray* tempStrings MEMBERS = int change_ref_value_operand MEMBERS = bufr_tableb_override* tableb_override @@ -138,8 +137,7 @@ typedef struct grib_accessor_bufr_data_array { int nInputShortReplications; int iInputShortReplications; grib_iarray* iss_list; - grib_trie* dataAccessorsTrie; - grib_trie* dataAccessorsRank; + grib_trie_with_rank* dataAccessorsTrie; grib_sarray* tempStrings; int change_ref_value_operand; bufr_tableb_override* tableb_override; @@ -338,7 +336,6 @@ static void init(grib_accessor* a,const long v, grib_arguments* params) self->expanded=0; self->expandedAccessor=0; self->dataAccessorsTrie=0; - self->dataAccessorsRank=0; self->change_ref_value_operand=0; /* 0, 255 or YYY */ self->tableb_override = NULL; @@ -442,7 +439,7 @@ grib_accessors_list* accessor_bufr_data_array_get_dataAccessors(grib_accessor* a return self->dataAccessors; } -grib_trie* accessor_bufr_data_array_get_dataAccessorsTrie(grib_accessor* a) +grib_trie_with_rank* accessor_bufr_data_array_get_dataAccessorsTrie(grib_accessor* a) { grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a; return self->dataAccessorsTrie; @@ -1899,7 +1896,7 @@ static grib_accessor* accessor_or_attribute_with_same_name(grib_accessor* a,cons return ok; } } - +#if 0 static int get_key_rank(grib_trie* accessorsRank,grib_accessor* a) { int* r=(int*)grib_trie_get(accessorsRank,a->name); @@ -1912,13 +1909,10 @@ static int get_key_rank(grib_trie* accessorsRank,grib_accessor* a) } return *r; } - -static void grib_data_accessors_trie_push(grib_trie* accessorsTrie, grib_accessor* a, int r) +#endif +static int grib_data_accessors_trie_push(grib_trie_with_rank* accessorsTrie, grib_accessor* a) { - char* name=(char*)grib_context_malloc_clear(a->context, strlen(a->name)+20); - sprintf(name,"#%d#%s", r, a->name); - grib_trie_insert(accessorsTrie, name, a); - grib_context_free(a->context, name); + return grib_trie_with_rank_insert(accessorsTrie, a->name, a); } static int create_keys(grib_accessor* a,long onlySubset,long startSubset,long endSubset) @@ -1973,14 +1967,9 @@ static int create_keys(grib_accessor* a,long onlySubset,long startSubset,long en self->dataAccessors=grib_accessors_list_create(c); if (self->dataAccessorsTrie) { - grib_trie_delete(self->dataAccessorsTrie); + grib_trie_with_rank_delete(self->dataAccessorsTrie); } - self->dataAccessorsTrie=grib_trie_new(c); - - if (self->dataAccessorsRank) { - grib_trie_delete(self->dataAccessorsRank); - } - self->dataAccessorsRank=grib_trie_new(c); + self->dataAccessorsTrie=grib_trie_with_rank_new(c); if (self->tempStrings) { grib_sarray_delete_content(c, self->tempStrings); @@ -2141,9 +2130,8 @@ static int create_keys(grib_accessor* a,long onlySubset,long startSubset,long en grib_pack_long(asn,&subsetNumber,&len); grib_push_accessor(asn,section->block); - rank=get_key_rank(self->dataAccessorsRank,asn); + rank=grib_data_accessors_trie_push(self->dataAccessorsTrie,asn); grib_accessors_list_push(self->dataAccessors,asn,rank); - grib_data_accessors_trie_push(self->dataAccessorsTrie,asn,rank); } count++; elementAccessor=create_accessor_from_descriptor(a,associatedFieldAccessor,section,ide,iss,dump,count); @@ -2158,9 +2146,8 @@ static int create_keys(grib_accessor* a,long onlySubset,long startSubset,long en newAccessor->parent=groupSection; newAccessor->name=grib_context_strdup(c,elementFromBitmap->name); grib_push_accessor(newAccessor,groupSection->block); - rank=get_key_rank(self->dataAccessorsRank,newAccessor); + rank=grib_data_accessors_trie_push(self->dataAccessorsTrie,newAccessor); grib_accessors_list_push(self->dataAccessors,newAccessor,rank); - grib_data_accessors_trie_push(self->dataAccessorsTrie,newAccessor,rank); } err=grib_accessor_add_attribute(accessor_or_attribute_with_same_name(elementFromBitmap,elementAccessor->name),elementAccessor,1); @@ -2186,10 +2173,9 @@ static int create_keys(grib_accessor* a,long onlySubset,long startSubset,long en break; default: grib_push_accessor(elementAccessor,section->block); - rank=get_key_rank(self->dataAccessorsRank,elementAccessor); + rank=grib_data_accessors_trie_push(self->dataAccessorsTrie,elementAccessor); grib_accessors_list_push(self->dataAccessors,elementAccessor,rank); lastAccessorInList=grib_accessors_list_last(self->dataAccessors); - grib_data_accessors_trie_push(self->dataAccessorsTrie,elementAccessor,rank); } } } @@ -2749,8 +2735,7 @@ static void destroy(grib_context* c,grib_accessor* a) grib_accessor_bufr_data_array *self =(grib_accessor_bufr_data_array*)a; self_clear(c,self); if (self->dataAccessors) grib_accessors_list_delete(c,self->dataAccessors); - if (self->dataAccessorsTrie) grib_trie_delete_container(self->dataAccessorsTrie); - if (self->dataAccessorsRank) grib_trie_delete(self->dataAccessorsRank); + if (self->dataAccessorsTrie) grib_trie_with_rank_delete_container(self->dataAccessorsTrie); if (self->tempStrings) { grib_sarray_delete_content(c, self->tempStrings); grib_sarray_delete (c, self->tempStrings); diff --git a/src/grib_accessor_class_bufr_group.c b/src/grib_accessor_class_bufr_group.c index 015af4bc8..2992f7940 100644 --- a/src/grib_accessor_class_bufr_group.c +++ b/src/grib_accessor_class_bufr_group.c @@ -16,7 +16,7 @@ START_CLASS_DEF CLASS = accessor SUPER = grib_accessor_class_variable - IMPLEMENTS = init;dump;next + IMPLEMENTS = dump;next END_CLASS_DEF */ @@ -32,7 +32,6 @@ or edit "accessor.class" and rerun ./make_class.pl */ static void dump(grib_accessor*, grib_dumper*); -static void init(grib_accessor*,const long, grib_arguments* ); static void init_class(grib_accessor_class*); static grib_accessor* next(grib_accessor*, int); @@ -54,7 +53,7 @@ static grib_accessor_class _grib_accessor_class_bufr_group = { sizeof(grib_accessor_bufr_group), /* size */ 0, /* inited */ &init_class, /* init_class */ - &init, /* init */ + 0, /* init */ 0, /* post_init */ 0, /* free mem */ &dump, /* describes himself */ @@ -131,10 +130,6 @@ static void init_class(grib_accessor_class* c) /* END_CLASS_IMP */ -static void init(grib_accessor* a, const long length , grib_arguments* args ) -{ -} - static void dump(grib_accessor* a, grib_dumper* dumper) { grib_dump_section(dumper,a,a->sub_section->block); diff --git a/src/grib_accessor_class_expanded_descriptors.c b/src/grib_accessor_class_expanded_descriptors.c index e66e447e4..cb0ffc29f 100644 --- a/src/grib_accessor_class_expanded_descriptors.c +++ b/src/grib_accessor_class_expanded_descriptors.c @@ -519,33 +519,82 @@ static bufr_descriptors_array* do_expand(grib_accessor* a,bufr_descriptors_array return expanded; } +static bufr_descriptors_array* expanded_descriptors_list_get(grib_context* c,const char* key,long* u,size_t size) { + bufr_descriptors_map_list* expandedUnexpandedMapList; + size_t i=0; + int found=0; + + if (!c->expanded_descriptors) { + c->expanded_descriptors=(grib_trie*)grib_trie_new(c); + return NULL; + } + expandedUnexpandedMapList=grib_trie_get(c->expanded_descriptors,key); + found=0; + while (expandedUnexpandedMapList) { + if (expandedUnexpandedMapList->unexpanded->n==size) { + found=1; + for (i=0;iunexpanded->v[i]->code!=u[i]) { + found=0; + break; + } + } + } + if (found) return expandedUnexpandedMapList->expanded; + expandedUnexpandedMapList=expandedUnexpandedMapList->next; + } + + return NULL; +} + +static void expanded_descriptor_list_push(grib_context* c,grib_trie* expanded_descriptors,const char* key,bufr_descriptors_array* expanded,bufr_descriptors_array* unexpanded) { + bufr_descriptors_map_list* descriptorsList=NULL; + bufr_descriptors_map_list* next=NULL; + bufr_descriptors_map_list* newdescriptorsList=NULL; + + newdescriptorsList=(bufr_descriptors_map_list*)grib_context_malloc_clear(c,sizeof(bufr_descriptors_map_list)); + newdescriptorsList->expanded=expanded; + newdescriptorsList->unexpanded=unexpanded; + + descriptorsList=grib_trie_get(expanded_descriptors,key); + if (descriptorsList) { + next=descriptorsList; + while(next->next) { + next=next->next; + } + next->next=newdescriptorsList; + } else { + grib_trie_insert(expanded_descriptors,key,newdescriptorsList); + } +} + static int expand(grib_accessor* a) { grib_accessor_expanded_descriptors* self = (grib_accessor_expanded_descriptors*)a; int err=0; size_t unexpandedSize=0; + /* grib_iarray* unexp=0; */ int i; long* u=0; + char key[50]={0,}; + long centre,masterTablesVersionNumber,localTablesVersionNumber,masterTablesNumber; change_coding_params ccp; bufr_descriptors_array* unexpanded=NULL; + bufr_descriptors_array* unexpanded_copy=NULL; + bufr_descriptors_array* expanded=NULL; grib_context* c=a->context; + grib_handle* h=grib_handle_of_accessor(a); if (!self->do_expand) { return err; } self->do_expand=0; - if (!self->tablesAccessor) { - self->tablesAccessor=grib_find_accessor(grib_handle_of_accessor(a),self->tablesAccessorName); - Assert(self->tablesAccessor); - } - if (self->rank!=0) { err=expand(self->expandedAccessor); self->expanded=((grib_accessor_expanded_descriptors*)self->expandedAccessor)->expanded; return err; } - grib_bufr_descriptors_array_delete(self->expanded); err=grib_get_size(grib_handle_of_accessor(a),self->unexpandedDescriptors,&unexpandedSize); if (err) return err; if (unexpandedSize==0) { @@ -557,9 +606,33 @@ static int expand(grib_accessor* a) err=grib_get_long_array(grib_handle_of_accessor(a),self->unexpandedDescriptors,u,&unexpandedSize); if (err) return err; + err=grib_get_long(h,"bufrHeaderCentre",¢re); + if (err) return err; + err=grib_get_long(h,"masterTablesVersionNumber",&masterTablesVersionNumber); + if (err) return err; + err=grib_get_long(h,"localTablesVersionNumber",&localTablesVersionNumber); + if (err) return err; + err=grib_get_long(h,"masterTableNumber",&masterTablesNumber); + if (err) return err; + + sprintf(key,"%ld_%ld_%ld_%ld_%ld",centre,masterTablesVersionNumber,localTablesVersionNumber,masterTablesNumber,u[0]); + expanded=expanded_descriptors_list_get(c,key,u,unexpandedSize); + if (expanded) { + self->expanded=expanded; + grib_context_free(c,u); + return err; + } + + if (!self->tablesAccessor) { + self->tablesAccessor=grib_find_accessor(grib_handle_of_accessor(a),self->tablesAccessorName); + Assert(self->tablesAccessor); + } + unexpanded=grib_bufr_descriptors_array_new(c,unexpandedSize,100); + unexpanded_copy=grib_bufr_descriptors_array_new(c,unexpandedSize,100); for (i=0;itablesAccessor,u[i],&err)); + grib_bufr_descriptors_array_push(unexpanded_copy,grib_bufr_descriptor_new(self->tablesAccessor,u[i],&err)); } grib_context_free(c,u); @@ -571,6 +644,7 @@ static int expand(grib_accessor* a) ccp.associatedFieldWidth=0; ccp.newStringWidth=0; self->expanded=do_expand(a,unexpanded,&ccp,&err); + expanded_descriptor_list_push(c,c->expanded_descriptors,key,self->expanded,unexpanded_copy); grib_bufr_descriptors_array_delete(unexpanded); return err; @@ -688,10 +762,10 @@ static int value_count(grib_accessor* a,long* rlen) static void destroy(grib_context* c,grib_accessor* a) { - grib_accessor_expanded_descriptors* self = (grib_accessor_expanded_descriptors*)a; - if (self->rank==0 && self->expanded) { - grib_bufr_descriptors_array_delete(self->expanded); - } + /* grib_accessor_expanded_descriptors* self = (grib_accessor_expanded_descriptors*)a; */ + /* if (self->rank==0 && self->expanded) { */ + /* grib_bufr_descriptors_array_delete(self->expanded); */ + /* } */ } static int get_native_type(grib_accessor* a) diff --git a/src/grib_accessor_class_forward.c b/src/grib_accessor_class_forward.c index d03bcb9cc..4b0e9d2d2 100644 --- a/src/grib_accessor_class_forward.c +++ b/src/grib_accessor_class_forward.c @@ -16,7 +16,7 @@ START_CLASS_DEF SUPER= grib_accessor_class_gen CLASS = accessor - IMPLEMENTS = init;dump + IMPLEMENTS = dump END_CLASS_DEF */ @@ -32,7 +32,6 @@ or edit "accessor.class" and rerun ./make_class.pl */ static void dump(grib_accessor*, grib_dumper*); -static void init(grib_accessor*,const long, grib_arguments* ); static void init_class(grib_accessor_class*); typedef struct grib_accessor_forward { @@ -49,7 +48,7 @@ static grib_accessor_class _grib_accessor_class_forward = { sizeof(grib_accessor_forward), /* size */ 0, /* inited */ &init_class, /* init_class */ - &init, /* init */ + 0, /* init */ 0, /* post_init */ 0, /* free mem */ &dump, /* describes himself */ @@ -127,11 +126,6 @@ static void init_class(grib_accessor_class* c) /* END_CLASS_IMP */ -static void init(grib_accessor* a,const long l, grib_arguments* c) -{ - /* a->alias = grib_arguments_get_name(grib_handle_of_accessor(a),c,0); */ -} - static void dump(grib_accessor* a, grib_dumper* dumper) { diff --git a/src/grib_accessor_class_library_version.c b/src/grib_accessor_class_library_version.c index 5db21bcb7..bf21aa3e3 100644 --- a/src/grib_accessor_class_library_version.c +++ b/src/grib_accessor_class_library_version.c @@ -16,7 +16,6 @@ CLASS = accessor SUPER = grib_accessor_class_ascii IMPLEMENTS = unpack_string;value_count;string_length - IMPLEMENTS = init END_CLASS_DEF */ @@ -34,7 +33,6 @@ or edit "accessor.class" and rerun ./make_class.pl static int unpack_string (grib_accessor*, char*, size_t *len); static size_t string_length(grib_accessor*); static int value_count(grib_accessor*,long*); -static void init(grib_accessor*,const long, grib_arguments* ); static void init_class(grib_accessor_class*); typedef struct grib_accessor_library_version { @@ -52,7 +50,7 @@ static grib_accessor_class _grib_accessor_class_library_version = { sizeof(grib_accessor_library_version), /* size */ 0, /* inited */ &init_class, /* init_class */ - &init, /* init */ + 0, /* init */ 0, /* post_init */ 0, /* free mem */ 0, /* describes himself */ @@ -128,10 +126,6 @@ static void init_class(grib_accessor_class* c) /* END_CLASS_IMP */ -static void init(grib_accessor* a,const long l, grib_arguments* c) -{ -} - static int unpack_string(grib_accessor* a, char* val, size_t *len) { char result[30]={0,}; diff --git a/src/grib_accessor_factory.h b/src/grib_accessor_factory.h index f344af2cf..97205e513 100644 --- a/src/grib_accessor_factory.h +++ b/src/grib_accessor_factory.h @@ -12,6 +12,7 @@ { "blob", &grib_accessor_class_blob, }, { "box", &grib_accessor_class_box, }, { "budgdate", &grib_accessor_class_budgdate, }, +{ "bufr_clear_tables", &grib_accessor_class_bufr_clear_tables, }, { "bufr_data", &grib_accessor_class_bufr_data, }, { "bufr_data_array", &grib_accessor_class_bufr_data_array, }, { "bufr_data_element", &grib_accessor_class_bufr_data_element, }, diff --git a/src/grib_accessor_factory_hash_list b/src/grib_accessor_factory_hash_list index b516727d6..4ffcf7775 100644 --- a/src/grib_accessor_factory_hash_list +++ b/src/grib_accessor_factory_hash_list @@ -12,6 +12,7 @@ bits_per_value, &grib_accessor_class_bits_per_value blob, &grib_accessor_class_blob box, &grib_accessor_class_box budgdate, &grib_accessor_class_budgdate +bufr_clear_tables, &grib_accessor_class_bufr_clear_tables bufr_data, &grib_accessor_class_bufr_data bufr_data_array, &grib_accessor_class_bufr_data_array bufr_data_element, &grib_accessor_class_bufr_data_element diff --git a/src/grib_api.h b/src/grib_api.h index c3c810a51..df2f1b305 100644 --- a/src/grib_api.h +++ b/src/grib_api.h @@ -217,6 +217,7 @@ typedef struct grib_order_by grib_order_by; typedef struct grib_where grib_where; typedef struct grib_sarray grib_sarray; +typedef struct grib_oarray grib_oarray; typedef struct grib_darray grib_darray; typedef struct grib_iarray grib_iarray; typedef struct grib_vdarray grib_vdarray; @@ -224,6 +225,7 @@ typedef struct grib_vsarray grib_vsarray; typedef struct grib_viarray grib_viarray; typedef struct bufr_descriptor bufr_descriptor; typedef struct bufr_descriptors_array bufr_descriptors_array; +typedef struct bufr_descriptors_map_list bufr_descriptors_map_list; grib_fieldset *grib_fieldset_new_from_files(grib_context *c, char *filenames[], int nfiles, char **keys, int nkeys, const char *where_string, const char *order_by_string, int *err); void grib_fieldset_delete(grib_fieldset* set); diff --git a/src/grib_api_internal.h b/src/grib_api_internal.h index d3c0c35f6..ee33e0872 100644 --- a/src/grib_api_internal.h +++ b/src/grib_api_internal.h @@ -713,6 +713,8 @@ struct grib_block_of_accessors typedef struct grib_trie grib_trie; +typedef struct grib_trie_with_rank_list grib_trie_with_rank_list; +typedef struct grib_trie_with_rank grib_trie_with_rank; typedef struct grib_itrie grib_itrie; @@ -724,6 +726,14 @@ struct grib_sarray { grib_context* context; } ; +struct grib_oarray { + void** v; + size_t size; + size_t n; + size_t incsize; + grib_context* context; +} ; + struct grib_darray { double* v; size_t size; @@ -803,6 +813,12 @@ struct bufr_descriptors_array { grib_context* context; } ; +struct bufr_descriptors_map_list { + bufr_descriptors_array* unexpanded; + bufr_descriptors_array* expanded; + bufr_descriptors_map_list* next; +}; + /* BUFR: operator 203: Table B changed reference values */ typedef struct bufr_tableb_override bufr_tableb_override; struct bufr_tableb_override { @@ -857,6 +873,7 @@ struct grib_handle long missingValueLong; double missingValueDouble; ProductKind product_kind; + grib_trie* bufr_elements_table; }; struct grib_multi_handle { @@ -1050,6 +1067,7 @@ struct grib_context FILE* log_stream; grib_trie* classes; grib_trie* lists; + grib_trie* expanded_descriptors; #if GRIB_PTHREADS pthread_mutex_t mutex; #elif GRIB_OMP_THREADS diff --git a/src/grib_api_prototypes.h b/src/grib_api_prototypes.h index 15616bea9..302e02ecc 100644 --- a/src/grib_api_prototypes.h +++ b/src/grib_api_prototypes.h @@ -214,6 +214,16 @@ void grib_sarray_delete_content(grib_context *c, grib_sarray *v); char **grib_sarray_get_array(grib_context *c, grib_sarray *v); size_t grib_sarray_used_size(grib_sarray *v); +/* grib_oarray.c */ +grib_oarray *grib_oarray_new(grib_context *c, size_t size, size_t incsize); +grib_oarray *grib_oarray_resize(grib_context *c, grib_oarray *v); +grib_oarray *grib_oarray_push(grib_context *c, grib_oarray *v, void *val); +void grib_oarray_delete(grib_context *c, grib_oarray *v); +void grib_oarray_delete_content(grib_context *c, grib_oarray *v); +void** grib_oarray_get_array(grib_context *c, grib_oarray *v); +size_t grib_oarray_used_size(grib_oarray *v); +void* grib_oarray_get(grib_oarray* v,int i); + /* grib_iarray.c */ grib_iarray *grib_iarray_new_from_array(grib_context *c, long *a, size_t size); grib_iarray *grib_iarray_new(grib_context *c, size_t size, size_t incsize); @@ -280,7 +290,7 @@ int accessor_bufr_data_array_create_keys(grib_accessor *a, long onlySubset, long int accessor_bufr_data_array_process_elements(grib_accessor *a, int flag, long onlySubset, long startSubset, long endSubset); grib_vsarray *accessor_bufr_data_array_get_stringValues(grib_accessor *a); grib_accessors_list *accessor_bufr_data_array_get_dataAccessors(grib_accessor *a); -grib_trie *accessor_bufr_data_array_get_dataAccessorsTrie(grib_accessor *a); +grib_trie_with_rank *accessor_bufr_data_array_get_dataAccessorsTrie(grib_accessor *a); void accessor_bufr_data_array_set_unpackMode(grib_accessor *a, int unpackMode); /* grib_accessor_class_bufr_data_element.c */ @@ -1118,6 +1128,15 @@ void *grib_trie_insert(grib_trie *t, const char *key, void *data); void *grib_trie_insert_no_replace(grib_trie *t, const char *key, void *data); void *grib_trie_get(grib_trie *t, const char *key); +/* grib_trie_with_rank.c */ +grib_trie_with_rank *grib_trie_with_rank_new(grib_context *c); +void grib_trie_with_rank_delete_container(grib_trie_with_rank *t); +void grib_trie_with_rank_delete(grib_trie_with_rank *t); +void grib_trie_with_rank_clear(grib_trie_with_rank *t); +int grib_trie_with_rank_insert(grib_trie_with_rank *t, const char *key, void *data); +void *grib_trie_with_rank_get(grib_trie_with_rank *t, const char *key,int rank); +void *grib_trie_with_rank_get_from_list(grib_trie_with_rank_list* list,int rank); + /* grib_itrie.c */ grib_itrie *grib_itrie_new(grib_context *c, int *count); void grib_itrie_delete(grib_itrie *t); diff --git a/src/grib_buffer.c b/src/grib_buffer.c index 84e0364f0..823e4931d 100644 --- a/src/grib_buffer.c +++ b/src/grib_buffer.c @@ -97,22 +97,26 @@ static void grib_grow_buffer_to(const grib_context *c, grib_buffer *b, size_t ns void grib_grow_buffer(const grib_context *c, grib_buffer *b, size_t new_size) { - size_t len = ((new_size + 1023)/1024)*1024; - grib_grow_buffer_to(c,b,len); + if (new_size > b->length) { + size_t len = 0; + size_t inc= b->length > 2048 ? b->length : 2048; + len = ((new_size + 2*inc)/1024) * 1024; + grib_grow_buffer_to(c,b,len); + } } void grib_buffer_set_ulength_bits(const grib_context *c, grib_buffer *b, size_t length_bits) { size_t length=length_bits/8; if (length_bits%8) length++; - grib_grow_buffer_to(c,b,length); + grib_grow_buffer(c,b,length); b->ulength_bits = length_bits; b->ulength = length; } void grib_buffer_set_ulength(const grib_context *c, grib_buffer *b, size_t length) { - grib_grow_buffer_to(c,b,length); + grib_grow_buffer(c,b,length); b->ulength = length; b->ulength_bits = length*8; } diff --git a/src/grib_context.c b/src/grib_context.c index ee8821ed7..584b05bf7 100644 --- a/src/grib_context.c +++ b/src/grib_context.c @@ -362,7 +362,8 @@ static grib_context default_grib_context = { 0, /* bufr_set_to_missing_if_out_of_range */ 0, /* log_stream */ 0, /* classes */ - 0 /* lists */ + 0, /* lists */ + 0 /* expanded_descriptors */ #if GRIB_PTHREADS ,PTHREAD_MUTEX_INITIALIZER /* mutex */ #endif diff --git a/src/grib_expression_class_accessor.c b/src/grib_expression_class_accessor.c index 0671b8c8d..38c1795b6 100644 --- a/src/grib_expression_class_accessor.c +++ b/src/grib_expression_class_accessor.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = destroy IMPLEMENTS = native_type IMPLEMENTS = get_name diff --git a/src/grib_expression_class_binop.c b/src/grib_expression_class_binop.c index dd71051c3..1f836a50e 100644 --- a/src/grib_expression_class_binop.c +++ b/src/grib_expression_class_binop.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = destroy IMPLEMENTS = native_type IMPLEMENTS = evaluate_long diff --git a/src/grib_expression_class_column.c b/src/grib_expression_class_column.c index 1b823edcd..e4faf7f4b 100644 --- a/src/grib_expression_class_column.c +++ b/src/grib_expression_class_column.c @@ -20,7 +20,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = destroy IMPLEMENTS = native_type IMPLEMENTS = get_name diff --git a/src/grib_expression_class_constant.c b/src/grib_expression_class_constant.c index 43bd199e4..f9bda9c55 100644 --- a/src/grib_expression_class_constant.c +++ b/src/grib_expression_class_constant.c @@ -20,7 +20,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = destroy IMPLEMENTS = native_type IMPLEMENTS = evaluate;print diff --git a/src/grib_expression_class_double.c b/src/grib_expression_class_double.c index d8d7129c6..6211e62f6 100644 --- a/src/grib_expression_class_double.c +++ b/src/grib_expression_class_double.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = destroy IMPLEMENTS = native_type IMPLEMENTS = evaluate_long diff --git a/src/grib_expression_class_functor.c b/src/grib_expression_class_functor.c index 68b4e2b6f..1156e9a41 100644 --- a/src/grib_expression_class_functor.c +++ b/src/grib_expression_class_functor.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = destroy IMPLEMENTS = native_type IMPLEMENTS = evaluate_long diff --git a/src/grib_expression_class_is_in_dict.c b/src/grib_expression_class_is_in_dict.c index df4ca7b77..77a823d35 100644 --- a/src/grib_expression_class_is_in_dict.c +++ b/src/grib_expression_class_is_in_dict.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = native_type IMPLEMENTS = add_dependency IMPLEMENTS = get_name diff --git a/src/grib_expression_class_is_in_list.c b/src/grib_expression_class_is_in_list.c index 05c3903a1..c26f803d8 100644 --- a/src/grib_expression_class_is_in_list.c +++ b/src/grib_expression_class_is_in_list.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = destroy IMPLEMENTS = native_type IMPLEMENTS = get_name diff --git a/src/grib_expression_class_is_integer.c b/src/grib_expression_class_is_integer.c index 8446d1dae..79e5b597d 100644 --- a/src/grib_expression_class_is_integer.c +++ b/src/grib_expression_class_is_integer.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = destroy IMPLEMENTS = native_type IMPLEMENTS = get_name diff --git a/src/grib_expression_class_length.c b/src/grib_expression_class_length.c index 84e7eff7c..c385d7480 100644 --- a/src/grib_expression_class_length.c +++ b/src/grib_expression_class_length.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = destroy IMPLEMENTS = native_type IMPLEMENTS = get_name diff --git a/src/grib_expression_class_long.c b/src/grib_expression_class_long.c index e0081273e..40a5887bb 100644 --- a/src/grib_expression_class_long.c +++ b/src/grib_expression_class_long.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = native_type;pack_missing IMPLEMENTS = destroy IMPLEMENTS = evaluate_long diff --git a/src/grib_expression_class_string.c b/src/grib_expression_class_string.c index 5e3195b83..0e587daf5 100644 --- a/src/grib_expression_class_string.c +++ b/src/grib_expression_class_string.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = native_type IMPLEMENTS = destroy IMPLEMENTS = evaluate_string diff --git a/src/grib_expression_class_string_compare.c b/src/grib_expression_class_string_compare.c index fb60b8b0f..4962553aa 100644 --- a/src/grib_expression_class_string_compare.c +++ b/src/grib_expression_class_string_compare.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = destroy IMPLEMENTS = native_type IMPLEMENTS = evaluate_long diff --git a/src/grib_expression_class_sub_string.c b/src/grib_expression_class_sub_string.c index b06213e4c..ceff97967 100644 --- a/src/grib_expression_class_sub_string.c +++ b/src/grib_expression_class_sub_string.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = native_type IMPLEMENTS = destroy IMPLEMENTS = evaluate_string diff --git a/src/grib_expression_class_true.c b/src/grib_expression_class_true.c index ac2639cc0..9ab717e77 100644 --- a/src/grib_expression_class_true.c +++ b/src/grib_expression_class_true.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = destroy IMPLEMENTS = native_type IMPLEMENTS = evaluate_long diff --git a/src/grib_expression_class_unop.c b/src/grib_expression_class_unop.c index 90bbec212..ec0d121bd 100644 --- a/src/grib_expression_class_unop.c +++ b/src/grib_expression_class_unop.c @@ -15,7 +15,6 @@ START_CLASS_DEF CLASS = expression - IMPLEMENTS = init_class IMPLEMENTS = destroy IMPLEMENTS = native_type IMPLEMENTS = evaluate_long diff --git a/src/grib_oarray.c b/src/grib_oarray.c new file mode 100644 index 000000000..a07d9662d --- /dev/null +++ b/src/grib_oarray.c @@ -0,0 +1,108 @@ +/* + * Copyright 2005-2017 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by + * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. + */ + +/*************************************************************************** + * + * Enrico Fucile + * + ***************************************************************************/ + +#include "grib_api_internal.h" + +grib_oarray* grib_oarray_new(grib_context* c,size_t size,size_t incsize) +{ + grib_oarray* v=NULL; + if (!c) c=grib_context_get_default(); + v=(grib_oarray*)grib_context_malloc_clear(c,sizeof(grib_oarray)); + if (!v) { + grib_context_log(c,GRIB_LOG_ERROR, + "grib_oarray_new unable to allocate %d bytes\n",sizeof(grib_oarray)); + return NULL; + } + v->size=size; + v->n=0; + v->incsize=incsize; + v->v=(void**)grib_context_malloc_clear(c,sizeof(char*)*size); + if (!v->v) { + grib_context_log(c,GRIB_LOG_ERROR, + "grib_oarray_new unable to allocate %d bytes\n",sizeof(char*)*size); + return NULL; + } + return v; +} + +grib_oarray* grib_oarray_resize(grib_context* c,grib_oarray* v) +{ + int newsize=v->incsize+v->size; + + if (!c) c=grib_context_get_default(); + + v->v=(void**)grib_context_realloc(c,v->v,newsize*sizeof(char*)); + v->size=newsize; + if (!v->v) { + grib_context_log(c,GRIB_LOG_ERROR, + "grib_oarray_resize unable to allocate %d bytes\n",sizeof(char*)*newsize); + return NULL; + } + return v; +} + +grib_oarray* grib_oarray_push(grib_context* c,grib_oarray* v,void* val) +{ + size_t start_size=100; + size_t start_incsize=100; + if (!v) v=grib_oarray_new(c,start_size,start_incsize); + + if (v->n >= v->size) v=grib_oarray_resize(c,v); + v->v[v->n]=val; + v->n++; + return v; +} + +void grib_oarray_delete(grib_context* c,grib_oarray* v) +{ + if (!v) return; + if (!c) grib_context_get_default(); + if (v->v) grib_context_free(c,v->v); + grib_context_free(c,v); +} + +void grib_oarray_delete_content(grib_context* c,grib_oarray* v) +{ + int i; + if (!v | !v->v) return; + if (!c) grib_context_get_default(); + for (i=0;in;i++) { + if (v->v[i]) grib_context_free(c,v->v[i]); + v->v[i]=0; + } + v->n=0; +} + +void** grib_oarray_get_array(grib_context* c,grib_oarray* v) +{ + void** ret; + int i; + if (!v) return NULL; + ret=(void**)grib_context_malloc_clear(c,sizeof(char*)*v->n); + for (i=0;in;i++) ret[i]=v->v[i]; + return ret; +} + +void* grib_oarray_get(grib_oarray* v,int i) +{ + if (v==NULL || i>v->n-1) return NULL; + return v->v[i]; +} + +size_t grib_oarray_used_size(grib_oarray* v) +{ + return v->n; +} diff --git a/src/grib_query.c b/src/grib_query.c index a27b78ed3..272f7e9b9 100644 --- a/src/grib_query.c +++ b/src/grib_query.c @@ -200,11 +200,11 @@ static char* get_condition(const char* name,codes_condition* condition) return str; } -static grib_accessor* _search_by_rank(grib_accessor* a,const char* name) { +static grib_accessor* _search_by_rank(grib_accessor* a,const char* name,int rank) { grib_accessor* ret=NULL; - grib_trie* t=accessor_bufr_data_array_get_dataAccessorsTrie(a); + grib_trie_with_rank* t=accessor_bufr_data_array_get_dataAccessorsTrie(a); - ret=(grib_accessor*)grib_trie_get(t,name); + ret=(grib_accessor*)grib_trie_with_rank_get(t,name,rank); return ret; } @@ -225,11 +225,11 @@ static grib_accessor* _search_by_rank(grib_accessor* a,const char* name,long ran } */ -static grib_accessor* search_by_rank(grib_handle* h, const char* name,const char *the_namespace) +static grib_accessor* search_by_rank(grib_handle* h, const char* name,int rank,const char *the_namespace) { grib_accessor* data=search_and_cache(h,"dataAccessors",the_namespace); if (data) { - return _search_by_rank(data,name); + return _search_by_rank(data,name,rank); } else { grib_accessor* ret=NULL; int rank; @@ -390,7 +390,9 @@ static grib_accessor* search_and_cache(grib_handle* h, const char* name,const ch grib_accessor* a=NULL; if (name[0]=='#') { - a=search_by_rank(h,name,the_namespace); + int rank=-1; + char* basename=get_rank(name,&rank); + a=search_by_rank(h,basename,rank,the_namespace); } else { a=_search_and_cache(h,name,the_namespace); } diff --git a/src/grib_trie_with_rank.c b/src/grib_trie_with_rank.c new file mode 100644 index 000000000..6f9133e92 --- /dev/null +++ b/src/grib_trie_with_rank.c @@ -0,0 +1,515 @@ +/* + * Copyright 2005-2017 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by + * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. + */ + +#include "grib_api_internal.h" + +/* Note: all non-alpha are mapped to 0 */ +static int mapping[] = { +0, /* 00 */ +0, /* 01 */ +0, /* 02 */ +0, /* 03 */ +0, /* 04 */ +0, /* 05 */ +0, /* 06 */ +0, /* 07 */ +0, /* 08 */ +0, /* 09 */ +0, /* 0a */ +0, /* 0b */ +0, /* 0c */ +0, /* 0d */ +0, /* 0e */ +0, /* 0f */ +0, /* 10 */ +0, /* 11 */ +0, /* 12 */ +0, /* 13 */ +0, /* 14 */ +0, /* 15 */ +0, /* 16 */ +0, /* 17 */ +0, /* 18 */ +0, /* 19 */ +0, /* 1a */ +0, /* 1b */ +0, /* 1c */ +0, /* 1d */ +0, /* 1e */ +0, /* 1f */ +0, /* 20 */ +0, /* 21 */ +0, /* 22 */ +38, /* # */ +0, /* 24 */ +0, /* 25 */ +0, /* 26 */ +0, /* 27 */ +0, /* 28 */ +0, /* 29 */ +0, /* 2a */ +0, /* 2b */ +0, /* 2c */ +0, /* 2d */ +0, /* 2e */ +0, /* 2f */ +1, /* 0 */ +2, /* 1 */ +3, /* 2 */ +4, /* 3 */ +5, /* 4 */ +6, /* 5 */ +7, /* 6 */ +8, /* 7 */ +9, /* 8 */ +10, /* 9 */ +0, /* 3a */ +0, /* 3b */ +0, /* 3c */ +0, /* 3d */ +0, /* 3e */ +0, /* 3f */ +0, /* 40 */ +11, /* A */ +12, /* B */ +13, /* C */ +14, /* D */ +15, /* E */ +16, /* F */ +17, /* G */ +18, /* H */ +19, /* I */ +20, /* J */ +21, /* K */ +22, /* L */ +23, /* M */ +24, /* N */ +25, /* O */ +26, /* P */ +27, /* Q */ +28, /* R */ +29, /* S */ +30, /* T */ +31, /* U */ +32, /* V */ +33, /* W */ +34, /* X */ +35, /* Y */ +36, /* Z */ +0, /* 5b */ +0, /* 5c */ +0, /* 5d */ +0, /* 5e */ +37, /* _ */ +0, /* 60 */ +11, /* a */ +12, /* b */ +13, /* c */ +14, /* d */ +15, /* e */ +16, /* f */ +17, /* g */ +18, /* h */ +19, /* i */ +20, /* j */ +21, /* k */ +22, /* l */ +23, /* m */ +24, /* n */ +25, /* o */ +26, /* p */ +27, /* q */ +28, /* r */ +29, /* s */ +30, /* t */ +31, /* u */ +32, /* v */ +33, /* w */ +34, /* x */ +35, /* y */ +36, /* z */ +0, /* 7b */ +0, /* 7c */ +0, /* 7d */ +0, /* 7e */ +0, /* 7f */ +0, /* 80 */ +0, /* 81 */ +0, /* 82 */ +0, /* 83 */ +0, /* 84 */ +0, /* 85 */ +0, /* 86 */ +0, /* 87 */ +0, /* 88 */ +0, /* 89 */ +0, /* 8a */ +0, /* 8b */ +0, /* 8c */ +0, /* 8d */ +0, /* 8e */ +0, /* 8f */ +0, /* 90 */ +0, /* 91 */ +0, /* 92 */ +0, /* 93 */ +0, /* 94 */ +0, /* 95 */ +0, /* 96 */ +0, /* 97 */ +0, /* 98 */ +0, /* 99 */ +0, /* 9a */ +0, /* 9b */ +0, /* 9c */ +0, /* 9d */ +0, /* 9e */ +0, /* 9f */ +0, /* a0 */ +0, /* a1 */ +0, /* a2 */ +0, /* a3 */ +0, /* a4 */ +0, /* a5 */ +0, /* a6 */ +0, /* a7 */ +0, /* a8 */ +0, /* a9 */ +0, /* aa */ +0, /* ab */ +0, /* ac */ +0, /* ad */ +0, /* ae */ +0, /* af */ +0, /* b0 */ +0, /* b1 */ +0, /* b2 */ +0, /* b3 */ +0, /* b4 */ +0, /* b5 */ +0, /* b6 */ +0, /* b7 */ +0, /* b8 */ +0, /* b9 */ +0, /* ba */ +0, /* bb */ +0, /* bc */ +0, /* bd */ +0, /* be */ +0, /* bf */ +0, /* c0 */ +0, /* c1 */ +0, /* c2 */ +0, /* c3 */ +0, /* c4 */ +0, /* c5 */ +0, /* c6 */ +0, /* c7 */ +0, /* c8 */ +0, /* c9 */ +0, /* ca */ +0, /* cb */ +0, /* cc */ +0, /* cd */ +0, /* ce */ +0, /* cf */ +0, /* d0 */ +0, /* d1 */ +0, /* d2 */ +0, /* d3 */ +0, /* d4 */ +0, /* d5 */ +0, /* d6 */ +0, /* d7 */ +0, /* d8 */ +0, /* d9 */ +0, /* da */ +0, /* db */ +0, /* dc */ +0, /* dd */ +0, /* de */ +0, /* df */ +0, /* e0 */ +0, /* e1 */ +0, /* e2 */ +0, /* e3 */ +0, /* e4 */ +0, /* e5 */ +0, /* e6 */ +0, /* e7 */ +0, /* e8 */ +0, /* e9 */ +0, /* ea */ +0, /* eb */ +0, /* ec */ +0, /* ed */ +0, /* ee */ +0, /* ef */ +0, /* f0 */ +0, /* f1 */ +0, /* f2 */ +0, /* f3 */ +0, /* f4 */ +0, /* f5 */ +0, /* f6 */ +0, /* f7 */ +0, /* f8 */ +0, /* f9 */ +0, /* fa */ +0, /* fb */ +0, /* fc */ +0, /* fd */ +0, /* fe */ +0, /* ff */ +}; + +/* ECC-388 */ +#ifdef DEBUG + static const size_t NUM_MAPPINGS = sizeof(mapping)/sizeof(mapping[0]); + + #define DebugCheckBounds(index, value) \ + do { \ + if (!((index) >= 0 && (index) < NUM_MAPPINGS) ) {printf("ERROR: string='%s' index=%ld @ %s +%d \n", value, (long)index, __FILE__, __LINE__); abort();} \ + } while(0) +#else + #define DebugCheckBounds(index, value) +#endif + + +#define SIZE 39 + +#if GRIB_PTHREADS +static pthread_once_t once = PTHREAD_ONCE_INIT; +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + +static void init() { + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr,PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&mutex,&attr); + pthread_mutexattr_destroy(&attr); +} +#elif GRIB_OMP_THREADS +static int once = 0; +static omp_nest_lock_t mutex; + +static void init() +{ + GRIB_OMP_CRITICAL(lock_grib_trie_with_rank_c) + { + if (once == 0) + { + omp_init_nest_lock(&mutex); + once = 1; + } + } +} +#endif + +/* +struct grib_trie_with_rank_list { + grib_trie_with_rank_list* next; + int rank; + void* data; +}; +*/ + +struct grib_trie_with_rank { + grib_trie_with_rank* next[SIZE]; + grib_context *context; + int first; + int last; + grib_oarray* objs; +}; + +grib_trie_with_rank *grib_trie_with_rank_new(grib_context* c) +{ +#ifdef RECYCLE_TRIE + grib_trie_with_rank* t = grib_context_malloc_clear_persistent(c,sizeof(grib_trie_with_rank)); +#else + grib_trie_with_rank* t = (grib_trie_with_rank*)grib_context_malloc_clear(c,sizeof(grib_trie_with_rank)); +#endif + t->context = c; + t->first=SIZE; + t->last=-1; + return t; +} + +/* +static void grib_trie_with_rank_delete_container_list(grib_context* c,grib_trie_with_rank_list *list) { + grib_trie_with_rank_list* next=list; + grib_trie_with_rank_list* p; + while (next) { + p=next; + next=next->next; + grib_context_free( c, p ); + } +} +*/ + +void grib_trie_with_rank_delete_container(grib_trie_with_rank *t) +{ + GRIB_MUTEX_INIT_ONCE(&once,&init); + GRIB_MUTEX_LOCK(&mutex); + if(t) { + int i; + for(i = t->first; i <= t->last; i++) + if (t->next[i]) { + grib_trie_with_rank_delete_container(t->next[i]); + } + grib_oarray_delete(t->context,t->objs); + /* grib_trie_with_rank_delete_container_list(t->context,t->list); */ +#ifdef RECYCLE_TRIE + grib_context_free_persistent(t->context,t); +#else + grib_context_free(t->context,t); +#endif + } + GRIB_MUTEX_UNLOCK(&mutex); +} + +/* +static void grib_trie_with_rank_delete_list(grib_context* c,grib_trie_with_rank_list *list) { + grib_trie_with_rank_list* next=list; + grib_trie_with_rank_list* p; + while (next) { + grib_context_free( c, next->data ); + p=next; + next=next->next; + grib_context_free( c, p ); + } +} +*/ + +void grib_trie_with_rank_delete(grib_trie_with_rank *t) +{ + GRIB_MUTEX_INIT_ONCE(&once,&init); + GRIB_MUTEX_LOCK(&mutex); + if(t) { + int i; + for(i = t->first; i <= t->last; i++) + if (t->next[i]) { + if (t->objs) { + grib_oarray_delete_content(t->context,t->objs); + grib_oarray_delete(t->context,t->objs); + } + /* grib_trie_with_rank_delete_list(t->context, t->next[i]->list ); */ + grib_trie_with_rank_delete(t->next[i]); + } +#ifdef RECYCLE_TRIE + grib_context_free_persistent(t->context,t); +#else + grib_context_free(t->context,t); +#endif + } + GRIB_MUTEX_UNLOCK(&mutex); +} + +void grib_trie_with_rank_clear(grib_trie_with_rank *t) +{ + if(t) { + int i; + if (t->objs) { + grib_oarray_delete_content(t->context,t->objs); + grib_oarray_delete(t->context,t->objs); + } + + for(i = t->first; i <= t->last; i++) + if (t->next[i]) + grib_trie_with_rank_clear(t->next[i]); + } +} + +/* +static void grib_trie_with_rank_insert_in_list(grib_trie_with_rank* t,void* data) { + if (t->list==NULL) { + t->list=grib_context_malloc_clear(t->context,sizeof(grib_trie_with_rank_list)); + t->list->data=data; + t->list->rank=1; + t->last_list=t->list; + } else { + t->last_list->next=grib_context_malloc_clear(t->context,sizeof(grib_trie_with_rank_list)); + t->last_list=t->last_list->next; + t->last_list->data=data; + t->last_list->rank++; + } +} +*/ + +int grib_trie_with_rank_insert(grib_trie_with_rank* t,const char* key,void* data) +{ + grib_trie_with_rank *last = t; + const char *k = key; + + GRIB_MUTEX_INIT_ONCE(&once,&init); + GRIB_MUTEX_LOCK(&mutex); + + while(*k && t) { + last = t; + DebugCheckBounds((int)*k, key); + t = t->next[mapping[(int)*k]]; + if(t) k++; + } + + if(*k != 0) { + t = last; + while(*k) { + int j = 0; + DebugCheckBounds((int)*k, key); + j = mapping[(int)*k++]; + if(j < t->first) t->first = j; + if(j > t->last) t->last = j; + t = t->next[j] =grib_trie_with_rank_new(t->context); + } + } + if (t->objs==NULL) t->objs=grib_oarray_new(t->context,100,1000); + grib_oarray_push(t->context,t->objs,data); + /* grib_trie_with_rank_insert_in_list(t,data); */ + GRIB_MUTEX_UNLOCK(&mutex); + return grib_oarray_used_size(t->objs); +} + +/* +void *grib_trie_with_rank_get_from_list(grib_trie_with_rank_list* list,int rank) { + grib_trie_with_rank_list* next=list; + int r=1; + + while(next) { + if (r==rank) return next->data; + next=next->next; + r++; + } + return NULL; +} +*/ + +void *grib_trie_with_rank_get(grib_trie_with_rank* t,const char* key,int rank) +{ + const char *k = key; + void* data; + GRIB_MUTEX_INIT_ONCE(&once,&init); + + if (rank<0) return NULL; + + GRIB_MUTEX_LOCK(&mutex); + + while(*k && t) { + DebugCheckBounds((int)*k, key); + t = t->next[mapping[(int)*k++]]; + } + + if(*k == 0 && t != NULL ) { + data=grib_oarray_get(t->objs,rank-1); + GRIB_MUTEX_UNLOCK(&mutex); + return data; + } + GRIB_MUTEX_UNLOCK(&mutex); + return NULL; +} diff --git a/tools/grib_tools.c b/tools/grib_tools.c index 32fb5404a..64bda062a 100644 --- a/tools/grib_tools.c +++ b/tools/grib_tools.c @@ -116,24 +116,24 @@ grib_runtime_options global_options={ static grib_handle* grib_handle_new_from_file_x(grib_context* c,FILE* f,int mode,int headers_only,int *err) { + if (mode==MODE_GRIB) + return grib_new_from_file(c,f,headers_only,err); + + if (mode==MODE_BUFR) + return bufr_new_from_file(c,f,err); + if (mode==MODE_ANY) return any_new_from_file(c,f,err); if (mode==MODE_GTS) return gts_new_from_file(c,f,err); - if (mode==MODE_BUFR) - return bufr_new_from_file(c,f,err); - if (mode==MODE_METAR) return metar_new_from_file(c,f,err); if (mode==MODE_TAF) return taf_new_from_file(c,f,err); - if (mode==MODE_GRIB) - return grib_new_from_file(c,f,headers_only,err); - Assert(!"grib_handle_new_from_file_x: unknown mode"); return NULL; }