From b2cfc4d60c3b88777a19dbcdb03139faaee365b5 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 13 Jul 2020 19:09:48 +0100 Subject: [PATCH] BUFR indexing: first draft --- src/grib_api.h | 2 ++ src/grib_api_internal.h | 2 ++ src/grib_api_prototypes.h | 4 +-- src/grib_index.c | 76 ++++++++++++++++++++++++++++++++------- src/grib_util.c | 7 ++-- tools/CMakeLists.txt | 2 +- tools/bufr_index_build.c | 6 ++-- tools/grib_dump.c | 2 +- tools/grib_tools.c | 4 +-- 9 files changed, 81 insertions(+), 24 deletions(-) diff --git a/src/grib_api.h b/src/grib_api.h index 34e3a49fb..4db5152b1 100644 --- a/src/grib_api.h +++ b/src/grib_api.h @@ -281,6 +281,8 @@ grib_index* grib_index_new_from_file(grib_context* c, */ grib_index* grib_index_new(grib_context* c, const char* keys, int* err); +int codes_index_set_product_kind(grib_index* index, ProductKind product_kind); +int codes_index_set_unpack_bufr(grib_index* index, int unpack); /** * Indexes the file given in argument in the index given in argument. * diff --git a/src/grib_api_internal.h b/src/grib_api_internal.h index 05ddfd81a..0f0564542 100644 --- a/src/grib_api_internal.h +++ b/src/grib_api_internal.h @@ -1393,6 +1393,8 @@ struct grib_index grib_field_list* current; grib_file* files; int count; + ProductKind product_kind; + int unpack_bufr; /* For meaningful for BUFR */ }; /* header compute */ diff --git a/src/grib_api_prototypes.h b/src/grib_api_prototypes.h index 8039c468b..adffb24d7 100644 --- a/src/grib_api_prototypes.h +++ b/src/grib_api_prototypes.h @@ -717,7 +717,7 @@ int grib_write_short(FILE* fh, short val); int grib_write_long(FILE* fh, long val); int grib_write_unsigned_long(FILE* fh, unsigned long val); int grib_write_string(FILE* fh, const char* s); -int grib_write_identifier(FILE* fh); +int grib_write_identifier(FILE* fh, const char* id); int grib_write_null_marker(FILE* fh); int grib_write_not_null_marker(FILE* fh); char* grib_read_string(grib_context* c, FILE* fh, int* err); @@ -1477,7 +1477,7 @@ int grib2_is_PDTN_ChemicalDistFunc(long productDefinitionTemplateNumber); int grib2_is_PDTN_Aerosol(long productDefinitionTemplateNumber); int grib2_is_PDTN_AerosolOptical(long productDefinitionTemplateNumber); int grib2_select_PDTN(int is_eps, int is_instant, int is_chemical, int is_chemical_distfn, int is_aerosol, int is_aerosol_optical); -int is_grib_index_file(const char* filename); +int is_index_file(const char* filename); size_t sum_of_pl_array(const long* pl, size_t plsize); int grib_is_earth_oblate(grib_handle* h); int grib_util_grib_data_quality_check(grib_handle* h, double min_val, double max_val); diff --git a/src/grib_index.c b/src/grib_index.c index db4b935a8..1df0fca87 100644 --- a/src/grib_index.c +++ b/src/grib_index.c @@ -366,9 +366,9 @@ int grib_write_string(FILE* fh, const char* s) return GRIB_SUCCESS; } -int grib_write_identifier(FILE* fh) +int grib_write_identifier(FILE* fh, const char* ID) { - return grib_write_string(fh, "GRBIDX1"); + return grib_write_string(fh, ID); } int grib_write_null_marker(FILE* fh) @@ -557,6 +557,8 @@ grib_index* grib_index_new(grib_context* c, const char* key, int* err) return NULL; } index->context = c; + index->product_kind = PRODUCT_GRIB; + index->unpack_bufr = 0; while ((key = get_key(&p, &type)) != NULL) { keys = grib_index_new_key(c, keys, key, type, err); @@ -835,6 +837,7 @@ int grib_index_write(grib_index* index, const char* filename) int err = 0; FILE* fh; grib_file* files; + char* identifier = NULL; fh = fopen(filename, "w"); if (!fh) { @@ -844,7 +847,10 @@ int grib_index_write(grib_index* index, const char* filename) return GRIB_IO_PROBLEM; } - err = grib_write_identifier(fh); + if (index->product_kind == PRODUCT_GRIB) identifier = "GRBIDX1"; + if (index->product_kind == PRODUCT_BUFR) identifier = "BFRIDX1"; + Assert(identifier); + err = grib_write_identifier(fh, identifier); if (err) { grib_context_log(index->context, (GRIB_LOG_ERROR) | (GRIB_LOG_PERROR), "Unable to write in file %s", filename); @@ -905,6 +911,7 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err) char* identifier = NULL; int max = 0; FILE* fh = NULL; + ProductKind product_kind = PRODUCT_GRIB; if (!c) c = grib_context_get_default(); @@ -923,6 +930,8 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err) fclose(fh); return NULL; } + if (strcmp(identifier, "GRBIDX1")==0) product_kind = PRODUCT_GRIB; + if (strcmp(identifier, "BFRIDX1")==0) product_kind = PRODUCT_BUFR; grib_context_free(c, identifier); *err = grib_read_uchar(fh, &marker); @@ -967,6 +976,7 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err) index = (grib_index*)grib_context_malloc_clear(c, sizeof(grib_index)); index->context = c; + index->product_kind = product_kind; index->keys = grib_read_index_keys(c, fh, err); if (*err) @@ -1046,7 +1056,13 @@ int grib_index_search_same(grib_index* index, grib_handle* h) int grib_index_add_file(grib_index* index, const char* filename) { - return _codes_index_add_file(index, filename, CODES_GRIB); + const ProductKind pkind = index->product_kind; + if (pkind == PRODUCT_GRIB) + return _codes_index_add_file(index, filename, CODES_GRIB); + else if (pkind == PRODUCT_BUFR) + return _codes_index_add_file(index, filename, CODES_BUFR); + Assert(!"grib_index_add_file: Only GRIB and BUFR are supported"); + return GRIB_INVALID_ARGUMENT; } grib_handle* new_message_from_file(int message_type, grib_context* c, FILE* f, int* error) @@ -1120,6 +1136,15 @@ int _codes_index_add_file(grib_index* index, const char* filename, int message_t field_tree = index->fields; index_key->value[0] = 0; message_count++; + + if (index->product_kind == PRODUCT_BUFR && index->unpack_bufr) { + err = grib_set_long(h, "unpack", 1); + if (err) { + grib_context_log(c, GRIB_LOG_ERROR, "BUFR: unable to create index. \"%s\": %s", + index_key->name, grib_get_error_message(err)); + return err; + } + } while (index_key) { if (index_key->type == GRIB_TYPE_UNDEFINED) { @@ -1222,7 +1247,7 @@ int _codes_index_add_file(grib_index* index, const char* filename, int message_t field_tree->field = field; grib_handle_delete(h); - } + }/*foreach message*/ grib_file_close(file->name, 0, &err); @@ -1611,7 +1636,7 @@ int grib_index_select_string(grib_index* index, const char* skey, const char* va grib_handle* codes_index_get_handle(grib_field* field, int message_type, int* err) { grib_handle* h = NULL; - typedef grib_handle* (*message_new_proc)(grib_context*, FILE*, int, int*); + typedef grib_handle* (*message_new_proc)(grib_context*, FILE*, int*); message_new_proc message_new = NULL; if (!field->file) { @@ -1626,13 +1651,10 @@ grib_handle* codes_index_get_handle(grib_field* field, int message_type, int* er return NULL; switch (message_type) { case CODES_GRIB: - message_new = grib_new_from_file; + message_new = codes_grib_handle_new_from_file; break; case CODES_BUFR: - grib_context_log(grib_context_get_default(), GRIB_LOG_ERROR, "codes_index_get_handle: indexing not implemented for BUFR"); - /* message_new=bufr_new_from_file; */ - *err = GRIB_NOT_IMPLEMENTED; - return NULL; + message_new = codes_bufr_handle_new_from_file; break; default: grib_context_log(grib_context_get_default(), GRIB_LOG_ERROR, "codes_index_get_handle: invalid message type"); @@ -1641,7 +1663,7 @@ grib_handle* codes_index_get_handle(grib_field* field, int message_type, int* er } fseeko(field->file->handle, field->offset, SEEK_SET); - h = message_new(0, field->file->handle, 0, err); + h = message_new(0, field->file->handle, err); if (*err != GRIB_SUCCESS) return NULL; @@ -1829,7 +1851,12 @@ char* grib_get_field_file(grib_index* index, off_t* offset) grib_handle* grib_handle_new_from_index(grib_index* index, int* err) { - return codes_new_from_index(index, CODES_GRIB, err); + ProductKind pkind = index->product_kind; + if (pkind == PRODUCT_GRIB) + return codes_new_from_index(index, CODES_GRIB, err); + if (pkind == PRODUCT_BUFR) + return codes_new_from_index(index, CODES_BUFR, err); + return NULL; } grib_handle* codes_new_from_index(grib_index* index, int message_type, int* err) @@ -1931,3 +1958,26 @@ int grib_index_search(grib_index* index, grib_index_key* keys) grib_index_rewind(index); return 0; } + +int codes_index_set_product_kind(grib_index* index, ProductKind product_kind) +{ + if (!index) + return GRIB_INVALID_ARGUMENT; + + if (product_kind == PRODUCT_GRIB || product_kind == PRODUCT_BUFR) { + index->product_kind = product_kind; + } else { + return GRIB_INVALID_ARGUMENT; + } + return GRIB_SUCCESS; +} + +int codes_index_set_unpack_bufr(grib_index* index, int unpack) +{ + if (!index) + return GRIB_INVALID_ARGUMENT; + if (index->product_kind != PRODUCT_BUFR) + return GRIB_INVALID_ARGUMENT; + index->unpack_bufr = unpack; + return GRIB_SUCCESS; +} diff --git a/src/grib_util.c b/src/grib_util.c index 9e76d22f4..b8052f190 100644 --- a/src/grib_util.c +++ b/src/grib_util.c @@ -2133,11 +2133,12 @@ int grib2_select_PDTN(int is_eps, int is_instant, } } -int is_grib_index_file(const char* filename) +int is_index_file(const char* filename) { FILE* fh; char buf[8] = {0,}; - const char* str = "GRBIDX"; + const char* id_grib = "GRBIDX"; + const char* id_bufr = "BFRIDX"; int ret = 0; size_t size = 0; @@ -2156,7 +2157,7 @@ int is_grib_index_file(const char* filename) return 0; } - ret = !strcmp(buf, str); + ret = (strcmp(buf, id_grib)==0 || strcmp(buf, id_bufr)==0); fclose(fh); diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 634cbf8bf..2a958708a 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -30,7 +30,7 @@ list( APPEND ecc_tools_binaries codes_info codes_count codes_split_file grib_histogram grib_filter grib_ls grib_dump grib_merge grib2ppm grib_set grib_get grib_get_data grib_copy - grib_compare codes_parser grib_index_build + grib_compare codes_parser grib_index_build bufr_index_build bufr_ls bufr_dump bufr_set bufr_get bufr_copy bufr_compare gts_get gts_compare gts_copy gts_dump gts_filter gts_ls diff --git a/tools/bufr_index_build.c b/tools/bufr_index_build.c index 119727330..7e564ce91 100644 --- a/tools/bufr_index_build.c +++ b/tools/bufr_index_build.c @@ -41,7 +41,7 @@ grib_option grib_options[] = { 0, 1, 0 } }; -int compress_index; +static int compress_index; int grib_options_count = sizeof(grib_options) / sizeof(grib_option); @@ -73,6 +73,8 @@ int grib_tool_init(grib_runtime_options* options) options->onlyfiles = 1; idx = grib_index_new(c, keys, &ret); + codes_index_set_product_kind(idx, PRODUCT_BUFR); + codes_index_set_unpack_bufr(idx, 1); if (!idx || ret) grib_context_log(c, GRIB_LOG_FATAL, @@ -85,7 +87,7 @@ int grib_tool_new_filename_action(grib_runtime_options* options, const char* fil { int ret = 0; printf("--- %s: processing %s\n", grib_tool_name, file); - ret = _codes_index_add_file(idx, file, CODES_BUFR); + ret = grib_index_add_file(idx, file); if (ret) { printf("error: %s\n", grib_get_error_message(ret)); exit(ret); diff --git a/tools/grib_dump.c b/tools/grib_dump.c index 9541e8908..19f96525b 100644 --- a/tools/grib_dump.c +++ b/tools/grib_dump.c @@ -139,7 +139,7 @@ int grib_tool_new_file_action(grib_runtime_options* options, grib_tools_file* fi * In debug dump mode, allow dumping of GRIB index files */ if (strcmp(options->dump_mode, "debug") == 0) { - if (is_grib_index_file(options->current_infile->name)) { + if (is_index_file(options->current_infile->name)) { int err = 0; grib_context* c = grib_context_get_default(); const char* filename = options->current_infile->name; diff --git a/tools/grib_tools.c b/tools/grib_tools.c index 2f2023596..100b63d02 100644 --- a/tools/grib_tools.c +++ b/tools/grib_tools.c @@ -180,8 +180,8 @@ int grib_tool(int argc, char** argv) /* ECC-926: Currently only GRIB indexing works. Disable the through_index if BUFR, GTS etc */ if (global_options.mode == MODE_GRIB && - is_grib_index_file(global_options.infile->name) && - (global_options.infile_extra && is_grib_index_file(global_options.infile_extra->name))) { + is_index_file(global_options.infile->name) && + (global_options.infile_extra && is_index_file(global_options.infile_extra->name))) { global_options.through_index = 1; return grib_tool_index(&global_options); }