BUFR indexing: first draft

src/grib_api.h

@@ -281,6 +281,8 @@ grib_index* grib_index_new_from_file(grib_context* c,
  */
 grib_index* grib_index_new(grib_context* c, const char* keys, int* err);
 
+int codes_index_set_product_kind(grib_index* index, ProductKind product_kind);
+int codes_index_set_unpack_bufr(grib_index* index, int unpack);
 /**
  *  Indexes the file given in argument in the index given in argument.
  *

src/grib_api_internal.h

@@ -1393,6 +1393,8 @@ struct grib_index
     grib_field_list* current;
     grib_file* files;
     int count;
+    ProductKind product_kind;
+    int unpack_bufr; /* For meaningful for BUFR */
 };
 
 /* header compute */

src/grib_api_prototypes.h

@@ -717,7 +717,7 @@ int grib_write_short(FILE* fh, short val);
 int grib_write_long(FILE* fh, long val);
 int grib_write_unsigned_long(FILE* fh, unsigned long val);
 int grib_write_string(FILE* fh, const char* s);
-int grib_write_identifier(FILE* fh);
+int grib_write_identifier(FILE* fh, const char* id);
 int grib_write_null_marker(FILE* fh);
 int grib_write_not_null_marker(FILE* fh);
 char* grib_read_string(grib_context* c, FILE* fh, int* err);
@@ -1477,7 +1477,7 @@ int grib2_is_PDTN_ChemicalDistFunc(long productDefinitionTemplateNumber);
 int grib2_is_PDTN_Aerosol(long productDefinitionTemplateNumber);
 int grib2_is_PDTN_AerosolOptical(long productDefinitionTemplateNumber);
 int grib2_select_PDTN(int is_eps, int is_instant, int is_chemical, int is_chemical_distfn, int is_aerosol, int is_aerosol_optical);
-int is_grib_index_file(const char* filename);
+int is_index_file(const char* filename);
 size_t sum_of_pl_array(const long* pl, size_t plsize);
 int grib_is_earth_oblate(grib_handle* h);
 int grib_util_grib_data_quality_check(grib_handle* h, double min_val, double max_val);

src/grib_index.c

@@ -366,9 +366,9 @@ int grib_write_string(FILE* fh, const char* s)
     return GRIB_SUCCESS;
 }
 
-int grib_write_identifier(FILE* fh)
+int grib_write_identifier(FILE* fh, const char* ID)
 {
-    return grib_write_string(fh, "GRBIDX1");
+    return grib_write_string(fh, ID);
 }
 
 int grib_write_null_marker(FILE* fh)
@@ -557,6 +557,8 @@ grib_index* grib_index_new(grib_context* c, const char* key, int* err)
         return NULL;
     }
     index->context = c;
+    index->product_kind = PRODUCT_GRIB;
+    index->unpack_bufr = 0;
 
     while ((key = get_key(&p, &type)) != NULL) {
         keys = grib_index_new_key(c, keys, key, type, err);
@@ -835,6 +837,7 @@ int grib_index_write(grib_index* index, const char* filename)
     int err = 0;
     FILE* fh;
     grib_file* files;
+    char* identifier = NULL;
 
     fh = fopen(filename, "w");
     if (!fh) {
@@ -844,7 +847,10 @@ int grib_index_write(grib_index* index, const char* filename)
         return GRIB_IO_PROBLEM;
     }
 
-    err = grib_write_identifier(fh);
+    if (index->product_kind == PRODUCT_GRIB) identifier = "GRBIDX1";
+    if (index->product_kind == PRODUCT_BUFR) identifier = "BFRIDX1";
+    Assert(identifier);
+    err = grib_write_identifier(fh, identifier);
     if (err) {
         grib_context_log(index->context, (GRIB_LOG_ERROR) | (GRIB_LOG_PERROR),
                          "Unable to write in file %s", filename);
@@ -905,6 +911,7 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err)
     char* identifier     = NULL;
     int max              = 0;
     FILE* fh             = NULL;
+    ProductKind product_kind = PRODUCT_GRIB;
 
     if (!c)
         c = grib_context_get_default();
@@ -923,6 +930,8 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err)
         fclose(fh);
         return NULL;
     }
+    if (strcmp(identifier, "GRBIDX1")==0) product_kind = PRODUCT_GRIB;
+    if (strcmp(identifier, "BFRIDX1")==0) product_kind = PRODUCT_BUFR;
     grib_context_free(c, identifier);
 
     *err = grib_read_uchar(fh, &marker);
@@ -967,6 +976,7 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err)
 
     index          = (grib_index*)grib_context_malloc_clear(c, sizeof(grib_index));
     index->context = c;
+    index->product_kind = product_kind;
 
     index->keys = grib_read_index_keys(c, fh, err);
     if (*err)
@@ -1046,7 +1056,13 @@ int grib_index_search_same(grib_index* index, grib_handle* h)
 
 int grib_index_add_file(grib_index* index, const char* filename)
 {
-    return _codes_index_add_file(index, filename, CODES_GRIB);
+    const ProductKind pkind = index->product_kind;
+    if (pkind == PRODUCT_GRIB)
+        return _codes_index_add_file(index, filename, CODES_GRIB);
+    else if (pkind == PRODUCT_BUFR)
+        return _codes_index_add_file(index, filename, CODES_BUFR);
+    Assert(!"grib_index_add_file: Only GRIB and BUFR are supported");
+    return GRIB_INVALID_ARGUMENT;
 }
 
 grib_handle* new_message_from_file(int message_type, grib_context* c, FILE* f, int* error)
@@ -1120,6 +1136,15 @@ int _codes_index_add_file(grib_index* index, const char* filename, int message_t
         field_tree          = index->fields;
         index_key->value[0] = 0;
         message_count++;
+        
+        if (index->product_kind == PRODUCT_BUFR && index->unpack_bufr) {
+            err = grib_set_long(h, "unpack", 1);
+            if (err) {
+                grib_context_log(c, GRIB_LOG_ERROR, "BUFR: unable to create index. \"%s\": %s",
+                                 index_key->name, grib_get_error_message(err));
+                return err;
+            }
+        }
 
         while (index_key) {
             if (index_key->type == GRIB_TYPE_UNDEFINED) {
@@ -1222,7 +1247,7 @@ int _codes_index_add_file(grib_index* index, const char* filename, int message_t
             field_tree->field = field;
 
         grib_handle_delete(h);
-    }
+    }/*foreach message*/
 
     grib_file_close(file->name, 0, &err);
 
@@ -1611,7 +1636,7 @@ int grib_index_select_string(grib_index* index, const char* skey, const char* va
 grib_handle* codes_index_get_handle(grib_field* field, int message_type, int* err)
 {
     grib_handle* h = NULL;
-    typedef grib_handle* (*message_new_proc)(grib_context*, FILE*, int, int*);
+    typedef grib_handle* (*message_new_proc)(grib_context*, FILE*, int*);
     message_new_proc message_new = NULL;
 
     if (!field->file) {
@@ -1626,13 +1651,10 @@ grib_handle* codes_index_get_handle(grib_field* field, int message_type, int* er
         return NULL;
     switch (message_type) {
         case CODES_GRIB:
-            message_new = grib_new_from_file;
+            message_new = codes_grib_handle_new_from_file;
             break;
         case CODES_BUFR:
-            grib_context_log(grib_context_get_default(), GRIB_LOG_ERROR, "codes_index_get_handle: indexing not implemented for BUFR");
+            message_new = codes_bufr_handle_new_from_file;
-            /* message_new=bufr_new_from_file; */
-            *err = GRIB_NOT_IMPLEMENTED;
-            return NULL;
             break;
         default:
             grib_context_log(grib_context_get_default(), GRIB_LOG_ERROR, "codes_index_get_handle: invalid message type");
@@ -1641,7 +1663,7 @@ grib_handle* codes_index_get_handle(grib_field* field, int message_type, int* er
     }
 
     fseeko(field->file->handle, field->offset, SEEK_SET);
-    h = message_new(0, field->file->handle, 0, err);
+    h = message_new(0, field->file->handle, err);
     if (*err != GRIB_SUCCESS)
         return NULL;
 
@@ -1829,7 +1851,12 @@ char* grib_get_field_file(grib_index* index, off_t* offset)
 
 grib_handle* grib_handle_new_from_index(grib_index* index, int* err)
 {
-    return codes_new_from_index(index, CODES_GRIB, err);
+    ProductKind pkind = index->product_kind;
+    if (pkind == PRODUCT_GRIB)
+        return codes_new_from_index(index, CODES_GRIB, err);
+    if (pkind == PRODUCT_BUFR)
+        return codes_new_from_index(index, CODES_BUFR, err);
+    return NULL;
 }
 
 grib_handle* codes_new_from_index(grib_index* index, int message_type, int* err)
@@ -1931,3 +1958,26 @@ int grib_index_search(grib_index* index, grib_index_key* keys)
     grib_index_rewind(index);
     return 0;
 }
+
+int codes_index_set_product_kind(grib_index* index, ProductKind product_kind)
+{
+    if (!index)
+        return GRIB_INVALID_ARGUMENT;
+
+    if (product_kind == PRODUCT_GRIB || product_kind == PRODUCT_BUFR) {
+        index->product_kind = product_kind;
+    } else {
+        return GRIB_INVALID_ARGUMENT;
+    }
+    return GRIB_SUCCESS;
+}
+
+int codes_index_set_unpack_bufr(grib_index* index, int unpack)
+{
+    if (!index)
+        return GRIB_INVALID_ARGUMENT;
+    if (index->product_kind != PRODUCT_BUFR)
+        return GRIB_INVALID_ARGUMENT;
+    index->unpack_bufr = unpack;
+    return GRIB_SUCCESS;
+}

src/grib_util.c

@@ -2133,11 +2133,12 @@ int grib2_select_PDTN(int is_eps, int is_instant,
     }
 }
 
-int is_grib_index_file(const char* filename)
+int is_index_file(const char* filename)
 {
     FILE* fh;
     char buf[8] = {0,};
-    const char* str = "GRBIDX";
+    const char* id_grib = "GRBIDX";
+    const char* id_bufr = "BFRIDX";
     int ret         = 0;
     size_t size     = 0;
 
@@ -2156,7 +2157,7 @@ int is_grib_index_file(const char* filename)
         return 0;
     }
 
-    ret = !strcmp(buf, str);
+    ret = (strcmp(buf, id_grib)==0 || strcmp(buf, id_bufr)==0);
 
     fclose(fh);
 

tools/CMakeLists.txt

@@ -30,7 +30,7 @@ list( APPEND ecc_tools_binaries
              codes_info codes_count codes_split_file
              grib_histogram grib_filter grib_ls grib_dump grib_merge
              grib2ppm grib_set grib_get grib_get_data grib_copy
-             grib_compare codes_parser grib_index_build
+             grib_compare codes_parser grib_index_build bufr_index_build
              bufr_ls bufr_dump bufr_set bufr_get
              bufr_copy bufr_compare
              gts_get gts_compare gts_copy gts_dump gts_filter gts_ls

tools/bufr_index_build.c

@@ -41,7 +41,7 @@ grib_option grib_options[] = {
       0, 1, 0 }
 };
 
-int compress_index;
+static int compress_index;
 
 int grib_options_count = sizeof(grib_options) / sizeof(grib_option);
 
@@ -73,6 +73,8 @@ int grib_tool_init(grib_runtime_options* options)
     options->onlyfiles = 1;
 
     idx = grib_index_new(c, keys, &ret);
+    codes_index_set_product_kind(idx, PRODUCT_BUFR);
+    codes_index_set_unpack_bufr(idx, 1);
 
     if (!idx || ret)
         grib_context_log(c, GRIB_LOG_FATAL,
@@ -85,7 +87,7 @@ int grib_tool_new_filename_action(grib_runtime_options* options, const char* fil
 {
     int ret = 0;
     printf("--- %s: processing %s\n", grib_tool_name, file);
-    ret = _codes_index_add_file(idx, file, CODES_BUFR);
+    ret = grib_index_add_file(idx, file);
     if (ret) {
         printf("error: %s\n", grib_get_error_message(ret));
         exit(ret);

tools/grib_dump.c

@@ -139,7 +139,7 @@ int grib_tool_new_file_action(grib_runtime_options* options, grib_tools_file* fi
      * In debug dump mode, allow dumping of GRIB index files
      */
     if (strcmp(options->dump_mode, "debug") == 0) {
-        if (is_grib_index_file(options->current_infile->name)) {
+        if (is_index_file(options->current_infile->name)) {
             int err              = 0;
             grib_context* c      = grib_context_get_default();
             const char* filename = options->current_infile->name;

tools/grib_tools.c

@@ -180,8 +180,8 @@ int grib_tool(int argc, char** argv)
 
     /* ECC-926: Currently only GRIB indexing works. Disable the through_index if BUFR, GTS etc */
     if (global_options.mode == MODE_GRIB &&
-        is_grib_index_file(global_options.infile->name) &&
+        is_index_file(global_options.infile->name) &&
-        (global_options.infile_extra && is_grib_index_file(global_options.infile_extra->name))) {
+        (global_options.infile_extra && is_index_file(global_options.infile_extra->name))) {
         global_options.through_index = 1;
         return grib_tool_index(&global_options);
     }