diff --git a/src/grib_api.h b/src/grib_api.h index c5fd1747c..9c6b87e9e 100644 --- a/src/grib_api.h +++ b/src/grib_api.h @@ -281,6 +281,8 @@ grib_index* grib_index_new_from_file(grib_context* c, */ grib_index* grib_index_new(grib_context* c, const char* keys, int* err); +int codes_index_set_product_kind(grib_index* index, ProductKind product_kind); +int codes_index_set_unpack_bufr(grib_index* index, int unpack); /** * Indexes the file given in argument in the index given in argument. * diff --git a/src/grib_api_internal.h b/src/grib_api_internal.h index 05ddfd81a..0f0564542 100644 --- a/src/grib_api_internal.h +++ b/src/grib_api_internal.h @@ -1393,6 +1393,8 @@ struct grib_index grib_field_list* current; grib_file* files; int count; + ProductKind product_kind; + int unpack_bufr; /* For meaningful for BUFR */ }; /* header compute */ diff --git a/src/grib_api_prototypes.h b/src/grib_api_prototypes.h index 8039c468b..129df97dd 100644 --- a/src/grib_api_prototypes.h +++ b/src/grib_api_prototypes.h @@ -717,7 +717,7 @@ int grib_write_short(FILE* fh, short val); int grib_write_long(FILE* fh, long val); int grib_write_unsigned_long(FILE* fh, unsigned long val); int grib_write_string(FILE* fh, const char* s); -int grib_write_identifier(FILE* fh); +int grib_write_identifier(FILE* fh, const char* id); int grib_write_null_marker(FILE* fh); int grib_write_not_null_marker(FILE* fh); char* grib_read_string(grib_context* c, FILE* fh, int* err); @@ -728,7 +728,6 @@ int grib_index_write(grib_index* index, const char* filename); grib_index* grib_index_read(grib_context* c, const char* filename, int* err); int grib_index_search_same(grib_index* index, grib_handle* h); int grib_index_add_file(grib_index* index, const char* filename); -grib_handle* new_message_from_file(int message_type, grib_context* c, FILE* f, int* error); int _codes_index_add_file(grib_index* index, const char* filename, int message_type); int grib_index_add_file(grib_index* index, const char* filename); grib_index* grib_index_new_from_file(grib_context* c, const char* filename, const char* keys, int* err); @@ -747,6 +746,9 @@ grib_handle* grib_handle_new_from_index(grib_index* index, int* err); grib_handle* codes_new_from_index(grib_index* index, int message_type, int* err); void grib_index_rewind(grib_index* index); int grib_index_search(grib_index* index, grib_index_key* keys); +int codes_index_set_product_kind(grib_index* index, ProductKind product_kind); +int codes_index_set_unpack_bufr(grib_index* index, int unpack); +int is_index_file(const char* filename); /* grib_accessor_class_number_of_points_gaussian.c */ @@ -1477,7 +1479,6 @@ int grib2_is_PDTN_ChemicalDistFunc(long productDefinitionTemplateNumber); int grib2_is_PDTN_Aerosol(long productDefinitionTemplateNumber); int grib2_is_PDTN_AerosolOptical(long productDefinitionTemplateNumber); int grib2_select_PDTN(int is_eps, int is_instant, int is_chemical, int is_chemical_distfn, int is_aerosol, int is_aerosol_optical); -int is_grib_index_file(const char* filename); size_t sum_of_pl_array(const long* pl, size_t plsize); int grib_is_earth_oblate(grib_handle* h); int grib_util_grib_data_quality_check(grib_handle* h, double min_val, double max_val); diff --git a/src/grib_index.c b/src/grib_index.c index db4b935a8..dfe523335 100644 --- a/src/grib_index.c +++ b/src/grib_index.c @@ -366,9 +366,9 @@ int grib_write_string(FILE* fh, const char* s) return GRIB_SUCCESS; } -int grib_write_identifier(FILE* fh) +int grib_write_identifier(FILE* fh, const char* ID) { - return grib_write_string(fh, "GRBIDX1"); + return grib_write_string(fh, ID); } int grib_write_null_marker(FILE* fh) @@ -557,6 +557,8 @@ grib_index* grib_index_new(grib_context* c, const char* key, int* err) return NULL; } index->context = c; + index->product_kind = PRODUCT_GRIB; + index->unpack_bufr = 0; while ((key = get_key(&p, &type)) != NULL) { keys = grib_index_new_key(c, keys, key, type, err); @@ -835,6 +837,7 @@ int grib_index_write(grib_index* index, const char* filename) int err = 0; FILE* fh; grib_file* files; + char* identifier = NULL; fh = fopen(filename, "w"); if (!fh) { @@ -844,7 +847,10 @@ int grib_index_write(grib_index* index, const char* filename) return GRIB_IO_PROBLEM; } - err = grib_write_identifier(fh); + if (index->product_kind == PRODUCT_GRIB) identifier = "GRBIDX1"; + if (index->product_kind == PRODUCT_BUFR) identifier = "BFRIDX1"; + Assert(identifier); + err = grib_write_identifier(fh, identifier); if (err) { grib_context_log(index->context, (GRIB_LOG_ERROR) | (GRIB_LOG_PERROR), "Unable to write in file %s", filename); @@ -905,6 +911,7 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err) char* identifier = NULL; int max = 0; FILE* fh = NULL; + ProductKind product_kind = PRODUCT_GRIB; if (!c) c = grib_context_get_default(); @@ -923,6 +930,8 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err) fclose(fh); return NULL; } + + if (strcmp(identifier, "BFRIDX1")==0) product_kind = PRODUCT_BUFR; grib_context_free(c, identifier); *err = grib_read_uchar(fh, &marker); @@ -967,6 +976,7 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err) index = (grib_index*)grib_context_malloc_clear(c, sizeof(grib_index)); index->context = c; + index->product_kind = product_kind; index->keys = grib_read_index_keys(c, fh, err); if (*err) @@ -1046,16 +1056,21 @@ int grib_index_search_same(grib_index* index, grib_handle* h) int grib_index_add_file(grib_index* index, const char* filename) { - return _codes_index_add_file(index, filename, CODES_GRIB); + int message_type = 0; + if (index->product_kind == PRODUCT_GRIB) message_type = CODES_GRIB; + else if (index->product_kind == PRODUCT_BUFR) message_type = CODES_BUFR; + else return GRIB_INVALID_ARGUMENT; + + return _codes_index_add_file(index, filename, message_type); } -grib_handle* new_message_from_file(int message_type, grib_context* c, FILE* f, int* error) +static grib_handle* new_message_from_file(int message_type, grib_context* c, FILE* f, int* error) { if (message_type == CODES_GRIB) return grib_new_from_file(c, f, 0, error); /* headers_only=0 */ if (message_type == CODES_BUFR) return bufr_new_from_file(c, f, error); - Assert(0); + Assert(!"new_message_from_file: invalid message type"); return NULL; } @@ -1120,6 +1135,15 @@ int _codes_index_add_file(grib_index* index, const char* filename, int message_t field_tree = index->fields; index_key->value[0] = 0; message_count++; + + if (index->product_kind == PRODUCT_BUFR && index->unpack_bufr) { + err = grib_set_long(h, "unpack", 1); + if (err) { + grib_context_log(c, GRIB_LOG_ERROR, "unable to unpack BUFR to create index. \"%s\": %s", + index_key->name, grib_get_error_message(err)); + return err; + } + } while (index_key) { if (index_key->type == GRIB_TYPE_UNDEFINED) { @@ -1222,7 +1246,7 @@ int _codes_index_add_file(grib_index* index, const char* filename, int message_t field_tree->field = field; grib_handle_delete(h); - } + }/*foreach message*/ grib_file_close(file->name, 0, &err); @@ -1611,7 +1635,7 @@ int grib_index_select_string(grib_index* index, const char* skey, const char* va grib_handle* codes_index_get_handle(grib_field* field, int message_type, int* err) { grib_handle* h = NULL; - typedef grib_handle* (*message_new_proc)(grib_context*, FILE*, int, int*); + typedef grib_handle* (*message_new_proc)(grib_context*, FILE*, int*); message_new_proc message_new = NULL; if (!field->file) { @@ -1626,13 +1650,10 @@ grib_handle* codes_index_get_handle(grib_field* field, int message_type, int* er return NULL; switch (message_type) { case CODES_GRIB: - message_new = grib_new_from_file; + message_new = codes_grib_handle_new_from_file; break; case CODES_BUFR: - grib_context_log(grib_context_get_default(), GRIB_LOG_ERROR, "codes_index_get_handle: indexing not implemented for BUFR"); - /* message_new=bufr_new_from_file; */ - *err = GRIB_NOT_IMPLEMENTED; - return NULL; + message_new = codes_bufr_handle_new_from_file; break; default: grib_context_log(grib_context_get_default(), GRIB_LOG_ERROR, "codes_index_get_handle: invalid message type"); @@ -1641,7 +1662,7 @@ grib_handle* codes_index_get_handle(grib_field* field, int message_type, int* er } fseeko(field->file->handle, field->offset, SEEK_SET); - h = message_new(0, field->file->handle, 0, err); + h = message_new(0, field->file->handle, err); if (*err != GRIB_SUCCESS) return NULL; @@ -1829,7 +1850,12 @@ char* grib_get_field_file(grib_index* index, off_t* offset) grib_handle* grib_handle_new_from_index(grib_index* index, int* err) { - return codes_new_from_index(index, CODES_GRIB, err); + ProductKind pkind = index->product_kind; + if (pkind == PRODUCT_GRIB) + return codes_new_from_index(index, CODES_GRIB, err); + if (pkind == PRODUCT_BUFR) + return codes_new_from_index(index, CODES_BUFR, err); + return NULL; } grib_handle* codes_new_from_index(grib_index* index, int message_type, int* err) @@ -1931,3 +1957,58 @@ int grib_index_search(grib_index* index, grib_index_key* keys) grib_index_rewind(index); return 0; } + +int codes_index_set_product_kind(grib_index* index, ProductKind product_kind) +{ + if (!index) + return GRIB_INVALID_ARGUMENT; + + if (product_kind == PRODUCT_GRIB || product_kind == PRODUCT_BUFR) { + index->product_kind = product_kind; + } else { + return GRIB_INVALID_ARGUMENT; + } + return GRIB_SUCCESS; +} + +int codes_index_set_unpack_bufr(grib_index* index, int unpack) +{ + if (!index) + return GRIB_INVALID_ARGUMENT; + if (index->product_kind != PRODUCT_BUFR) + return GRIB_INVALID_ARGUMENT; + index->unpack_bufr = unpack; + return GRIB_SUCCESS; +} + +/* Return 1 if the file is an index file. 0 otherwise */ +int is_index_file(const char* filename) +{ + FILE* fh; + char buf[8] = {0,}; + const char* id_grib = "GRBIDX"; + const char* id_bufr = "BFRIDX"; + int ret = 0; + size_t size = 0; + + fh = fopen(filename, "r"); + if (!fh) + return 0; + + size = fread(buf, 1, 1, fh); + if (size != 1) { + fclose(fh); + return 0; + } + size = fread(buf, 6, 1, fh); + if (size != 1) { + fclose(fh); + return 0; + } + + ret = (strcmp(buf, id_grib)==0 || strcmp(buf, id_bufr)==0); + + fclose(fh); + + return ret; +} diff --git a/src/grib_util.c b/src/grib_util.c index 608019a43..8366d5531 100644 --- a/src/grib_util.c +++ b/src/grib_util.c @@ -2140,36 +2140,6 @@ int grib2_select_PDTN(int is_eps, int is_instant, } } -int is_grib_index_file(const char* filename) -{ - FILE* fh; - char buf[8] = {0,}; - const char* str = "GRBIDX"; - int ret = 0; - size_t size = 0; - - fh = fopen(filename, "r"); - if (!fh) - return 0; - - size = fread(buf, 1, 1, fh); - if (size != 1) { - fclose(fh); - return 0; - } - size = fread(buf, 6, 1, fh); - if (size != 1) { - fclose(fh); - return 0; - } - - ret = !strcmp(buf, str); - - fclose(fh); - - return ret; -} - size_t sum_of_pl_array(const long* pl, size_t plsize) { long i, count = 0; diff --git a/tests/grib_tigge_conversions.sh b/tests/grib_tigge_conversions.sh index 6e8519363..e63f1da40 100755 --- a/tests/grib_tigge_conversions.sh +++ b/tests/grib_tigge_conversions.sh @@ -16,7 +16,7 @@ REDIRECT=/dev/null -dir="${data_dir}/tigge/" +dir="${data_dir}/tigge" temp1="temp.grib1_" temp2="temp.grib2_" diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 634cbf8bf..2a958708a 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -30,7 +30,7 @@ list( APPEND ecc_tools_binaries codes_info codes_count codes_split_file grib_histogram grib_filter grib_ls grib_dump grib_merge grib2ppm grib_set grib_get grib_get_data grib_copy - grib_compare codes_parser grib_index_build + grib_compare codes_parser grib_index_build bufr_index_build bufr_ls bufr_dump bufr_set bufr_get bufr_copy bufr_compare gts_get gts_compare gts_copy gts_dump gts_filter gts_ls diff --git a/tools/bufr_index_build.c b/tools/bufr_index_build.c index 119727330..7e564ce91 100644 --- a/tools/bufr_index_build.c +++ b/tools/bufr_index_build.c @@ -41,7 +41,7 @@ grib_option grib_options[] = { 0, 1, 0 } }; -int compress_index; +static int compress_index; int grib_options_count = sizeof(grib_options) / sizeof(grib_option); @@ -73,6 +73,8 @@ int grib_tool_init(grib_runtime_options* options) options->onlyfiles = 1; idx = grib_index_new(c, keys, &ret); + codes_index_set_product_kind(idx, PRODUCT_BUFR); + codes_index_set_unpack_bufr(idx, 1); if (!idx || ret) grib_context_log(c, GRIB_LOG_FATAL, @@ -85,7 +87,7 @@ int grib_tool_new_filename_action(grib_runtime_options* options, const char* fil { int ret = 0; printf("--- %s: processing %s\n", grib_tool_name, file); - ret = _codes_index_add_file(idx, file, CODES_BUFR); + ret = grib_index_add_file(idx, file); if (ret) { printf("error: %s\n", grib_get_error_message(ret)); exit(ret); diff --git a/tools/grib_dump.c b/tools/grib_dump.c index 9541e8908..19f96525b 100644 --- a/tools/grib_dump.c +++ b/tools/grib_dump.c @@ -139,7 +139,7 @@ int grib_tool_new_file_action(grib_runtime_options* options, grib_tools_file* fi * In debug dump mode, allow dumping of GRIB index files */ if (strcmp(options->dump_mode, "debug") == 0) { - if (is_grib_index_file(options->current_infile->name)) { + if (is_index_file(options->current_infile->name)) { int err = 0; grib_context* c = grib_context_get_default(); const char* filename = options->current_infile->name; diff --git a/tools/grib_tools.c b/tools/grib_tools.c index 2f2023596..100b63d02 100644 --- a/tools/grib_tools.c +++ b/tools/grib_tools.c @@ -180,8 +180,8 @@ int grib_tool(int argc, char** argv) /* ECC-926: Currently only GRIB indexing works. Disable the through_index if BUFR, GTS etc */ if (global_options.mode == MODE_GRIB && - is_grib_index_file(global_options.infile->name) && - (global_options.infile_extra && is_grib_index_file(global_options.infile_extra->name))) { + is_index_file(global_options.infile->name) && + (global_options.infile_extra && is_index_file(global_options.infile_extra->name))) { global_options.through_index = 1; return grib_tool_index(&global_options); }