core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

BUFR indexing: first draft

This commit is contained in:
Shahram Najm 2020-07-13 19:09:48 +01:00
parent 1a260c942b
commit b2cfc4d60c
9 changed files with 81 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/grib_api.h
View File

@ -281,6 +281,8 @@ grib_index* grib_index_new_from_file(grib_context* c,
*/
grib_index* grib_index_new(grib_context* c, const char* keys, int* err);
int codes_index_set_product_kind(grib_index* index, ProductKind product_kind);
int codes_index_set_unpack_bufr(grib_index* index, int unpack);
/**
* Indexes the file given in argument in the index given in argument.
*

2
src/grib_api_internal.h
View File

@ -1393,6 +1393,8 @@ struct grib_index
grib_field_list* current;
grib_file* files;
int count;
ProductKind product_kind;
int unpack_bufr; /* For meaningful for BUFR */
};
/* header compute */

4
src/grib_api_prototypes.h
View File

@ -717,7 +717,7 @@ int grib_write_short(FILE* fh, short val);
int grib_write_long(FILE* fh, long val);
int grib_write_unsigned_long(FILE* fh, unsigned long val);
int grib_write_string(FILE* fh, const char* s);
int grib_write_identifier(FILE* fh);
int grib_write_identifier(FILE* fh, const char* id);
int grib_write_null_marker(FILE* fh);
int grib_write_not_null_marker(FILE* fh);
char* grib_read_string(grib_context* c, FILE* fh, int* err);
@ -1477,7 +1477,7 @@ int grib2_is_PDTN_ChemicalDistFunc(long productDefinitionTemplateNumber);
int grib2_is_PDTN_Aerosol(long productDefinitionTemplateNumber);
int grib2_is_PDTN_AerosolOptical(long productDefinitionTemplateNumber);
int grib2_select_PDTN(int is_eps, int is_instant, int is_chemical, int is_chemical_distfn, int is_aerosol, int is_aerosol_optical);
int is_grib_index_file(const char* filename);
int is_index_file(const char* filename);
size_t sum_of_pl_array(const long* pl, size_t plsize);
int grib_is_earth_oblate(grib_handle* h);
int grib_util_grib_data_quality_check(grib_handle* h, double min_val, double max_val);

76
src/grib_index.c
View File

@ -366,9 +366,9 @@ int grib_write_string(FILE* fh, const char* s)
return GRIB_SUCCESS;
}
int grib_write_identifier(FILE* fh)
int grib_write_identifier(FILE* fh, const char* ID)
{
return grib_write_string(fh, "GRBIDX1");
return grib_write_string(fh, ID);
}
int grib_write_null_marker(FILE* fh)
@ -557,6 +557,8 @@ grib_index* grib_index_new(grib_context* c, const char* key, int* err)
return NULL;
}
index->context = c;
index->product_kind = PRODUCT_GRIB;
index->unpack_bufr = 0;
while ((key = get_key(&p, &type)) != NULL) {
keys = grib_index_new_key(c, keys, key, type, err);
@ -835,6 +837,7 @@ int grib_index_write(grib_index* index, const char* filename)
int err = 0;
FILE* fh;
grib_file* files;
char* identifier = NULL;
fh = fopen(filename, "w");
if (!fh) {
@ -844,7 +847,10 @@ int grib_index_write(grib_index* index, const char* filename)
return GRIB_IO_PROBLEM;
}
err = grib_write_identifier(fh);
if (index->product_kind == PRODUCT_GRIB) identifier = "GRBIDX1";
if (index->product_kind == PRODUCT_BUFR) identifier = "BFRIDX1";
Assert(identifier);
err = grib_write_identifier(fh, identifier);
if (err) {
grib_context_log(index->context, (GRIB_LOG_ERROR) | (GRIB_LOG_PERROR),
"Unable to write in file %s", filename);
@ -905,6 +911,7 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err)
char* identifier = NULL;
int max = 0;
FILE* fh = NULL;
ProductKind product_kind = PRODUCT_GRIB;
if (!c)
c = grib_context_get_default();
@ -923,6 +930,8 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err)
fclose(fh);
return NULL;
}
if (strcmp(identifier, "GRBIDX1")==0) product_kind = PRODUCT_GRIB;
if (strcmp(identifier, "BFRIDX1")==0) product_kind = PRODUCT_BUFR;
grib_context_free(c, identifier);
*err = grib_read_uchar(fh, &marker);
@ -967,6 +976,7 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err)
index = (grib_index*)grib_context_malloc_clear(c, sizeof(grib_index));
index->context = c;
index->product_kind = product_kind;
index->keys = grib_read_index_keys(c, fh, err);
if (*err)
@ -1046,7 +1056,13 @@ int grib_index_search_same(grib_index* index, grib_handle* h)
int grib_index_add_file(grib_index* index, const char* filename)
{
return _codes_index_add_file(index, filename, CODES_GRIB);
const ProductKind pkind = index->product_kind;
if (pkind == PRODUCT_GRIB)
return _codes_index_add_file(index, filename, CODES_GRIB);
else if (pkind == PRODUCT_BUFR)
return _codes_index_add_file(index, filename, CODES_BUFR);
Assert(!"grib_index_add_file: Only GRIB and BUFR are supported");
return GRIB_INVALID_ARGUMENT;
}
grib_handle* new_message_from_file(int message_type, grib_context* c, FILE* f, int* error)
@ -1120,6 +1136,15 @@ int _codes_index_add_file(grib_index* index, const char* filename, int message_t
field_tree = index->fields;
index_key->value[0] = 0;
message_count++;
if (index->product_kind == PRODUCT_BUFR && index->unpack_bufr) {
err = grib_set_long(h, "unpack", 1);
if (err) {
grib_context_log(c, GRIB_LOG_ERROR, "BUFR: unable to create index. \"%s\": %s",
index_key->name, grib_get_error_message(err));
return err;
}
}
while (index_key) {
if (index_key->type == GRIB_TYPE_UNDEFINED) {
@ -1222,7 +1247,7 @@ int _codes_index_add_file(grib_index* index, const char* filename, int message_t
field_tree->field = field;
grib_handle_delete(h);
}
}/*foreach message*/
grib_file_close(file->name, 0, &err);
@ -1611,7 +1636,7 @@ int grib_index_select_string(grib_index* index, const char* skey, const char* va
grib_handle* codes_index_get_handle(grib_field* field, int message_type, int* err)
{
grib_handle* h = NULL;
typedef grib_handle* (*message_new_proc)(grib_context*, FILE*, int, int*);
typedef grib_handle* (*message_new_proc)(grib_context*, FILE*, int*);
message_new_proc message_new = NULL;
if (!field->file) {
@ -1626,13 +1651,10 @@ grib_handle* codes_index_get_handle(grib_field* field, int message_type, int* er
return NULL;
switch (message_type) {
case CODES_GRIB:
message_new = grib_new_from_file;
message_new = codes_grib_handle_new_from_file;
break;
case CODES_BUFR:
grib_context_log(grib_context_get_default(), GRIB_LOG_ERROR, "codes_index_get_handle: indexing not implemented for BUFR");
/* message_new=bufr_new_from_file; */
*err = GRIB_NOT_IMPLEMENTED;
return NULL;
message_new = codes_bufr_handle_new_from_file;
break;
default:
grib_context_log(grib_context_get_default(), GRIB_LOG_ERROR, "codes_index_get_handle: invalid message type");
@ -1641,7 +1663,7 @@ grib_handle* codes_index_get_handle(grib_field* field, int message_type, int* er
}
fseeko(field->file->handle, field->offset, SEEK_SET);
h = message_new(0, field->file->handle, 0, err);
h = message_new(0, field->file->handle, err);
if (*err != GRIB_SUCCESS)
return NULL;
@ -1829,7 +1851,12 @@ char* grib_get_field_file(grib_index* index, off_t* offset)
grib_handle* grib_handle_new_from_index(grib_index* index, int* err)
{
return codes_new_from_index(index, CODES_GRIB, err);
ProductKind pkind = index->product_kind;
if (pkind == PRODUCT_GRIB)
return codes_new_from_index(index, CODES_GRIB, err);
if (pkind == PRODUCT_BUFR)
return codes_new_from_index(index, CODES_BUFR, err);
return NULL;
}
grib_handle* codes_new_from_index(grib_index* index, int message_type, int* err)
@ -1931,3 +1958,26 @@ int grib_index_search(grib_index* index, grib_index_key* keys)
grib_index_rewind(index);
return 0;
}
int codes_index_set_product_kind(grib_index* index, ProductKind product_kind)
{
if (!index)
return GRIB_INVALID_ARGUMENT;
if (product_kind == PRODUCT_GRIB || product_kind == PRODUCT_BUFR) {
index->product_kind = product_kind;
} else {
return GRIB_INVALID_ARGUMENT;
}
return GRIB_SUCCESS;
}
int codes_index_set_unpack_bufr(grib_index* index, int unpack)
{
if (!index)
return GRIB_INVALID_ARGUMENT;
if (index->product_kind != PRODUCT_BUFR)
return GRIB_INVALID_ARGUMENT;
index->unpack_bufr = unpack;
return GRIB_SUCCESS;
}

7
src/grib_util.c
View File

@ -2133,11 +2133,12 @@ int grib2_select_PDTN(int is_eps, int is_instant,
}
}
int is_grib_index_file(const char* filename)
int is_index_file(const char* filename)
{
FILE* fh;
char buf[8] = {0,};
const char* str = "GRBIDX";
const char* id_grib = "GRBIDX";
const char* id_bufr = "BFRIDX";
int ret = 0;
size_t size = 0;
@ -2156,7 +2157,7 @@ int is_grib_index_file(const char* filename)
return 0;
}
ret = !strcmp(buf, str);
ret = (strcmp(buf, id_grib)==0 || strcmp(buf, id_bufr)==0);
fclose(fh);

2
tools/CMakeLists.txt
View File

@ -30,7 +30,7 @@ list( APPEND ecc_tools_binaries
codes_info codes_count codes_split_file
grib_histogram grib_filter grib_ls grib_dump grib_merge
grib2ppm grib_set grib_get grib_get_data grib_copy
grib_compare codes_parser grib_index_build
grib_compare codes_parser grib_index_build bufr_index_build
bufr_ls bufr_dump bufr_set bufr_get
bufr_copy bufr_compare
gts_get gts_compare gts_copy gts_dump gts_filter gts_ls

6
tools/bufr_index_build.c
View File

@ -41,7 +41,7 @@ grib_option grib_options[] = {
0, 1, 0 }
};
int compress_index;
static int compress_index;
int grib_options_count = sizeof(grib_options) / sizeof(grib_option);
@ -73,6 +73,8 @@ int grib_tool_init(grib_runtime_options* options)
options->onlyfiles = 1;
idx = grib_index_new(c, keys, &ret);
codes_index_set_product_kind(idx, PRODUCT_BUFR);
codes_index_set_unpack_bufr(idx, 1);
if (!idx || ret)
grib_context_log(c, GRIB_LOG_FATAL,
@ -85,7 +87,7 @@ int grib_tool_new_filename_action(grib_runtime_options* options, const char* fil
{
int ret = 0;
printf("--- %s: processing %s\n", grib_tool_name, file);
ret = _codes_index_add_file(idx, file, CODES_BUFR);
ret = grib_index_add_file(idx, file);
if (ret) {
printf("error: %s\n", grib_get_error_message(ret));
exit(ret);

2
tools/grib_dump.c
View File

@ -139,7 +139,7 @@ int grib_tool_new_file_action(grib_runtime_options* options, grib_tools_file* fi
* In debug dump mode, allow dumping of GRIB index files
*/
if (strcmp(options->dump_mode, "debug") == 0) {
if (is_grib_index_file(options->current_infile->name)) {
if (is_index_file(options->current_infile->name)) {
int err = 0;
grib_context* c = grib_context_get_default();
const char* filename = options->current_infile->name;

4
tools/grib_tools.c
View File

@ -180,8 +180,8 @@ int grib_tool(int argc, char** argv)
/* ECC-926: Currently only GRIB indexing works. Disable the through_index if BUFR, GTS etc */
if (global_options.mode == MODE_GRIB &&
is_grib_index_file(global_options.infile->name) &&
(global_options.infile_extra && is_grib_index_file(global_options.infile_extra->name))) {
is_index_file(global_options.infile->name) &&
(global_options.infile_extra && is_index_file(global_options.infile_extra->name))) {
global_options.through_index = 1;
return grib_tool_index(&global_options);
}