eccodes/tools/grib_tools.cc

/*
 * (C) Copyright 2005- ECMWF.
 *
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 *
 * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
 * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
 */

#include "grib_tools.h"

#include <stdlib.h>
#include <string>

#ifdef HAVE_ECKIT_GEO
    #include "eckit/runtime/Main.h"
#endif

#if HAVE_LIBJASPER
/* Remove compiler warnings re macros being redefined */
#undef PACKAGE_BUGREPORT
#undef PACKAGE_NAME
#undef PACKAGE_STRING
#undef PACKAGE_TARNAME
#undef PACKAGE_VERSION
#endif

#ifdef ENABLE_FLOATING_POINT_EXCEPTIONS
 #include <fenv.h>
 int feenableexcept(int excepts);
#endif

GRIB_INLINE static int grib_inline_strcmp(const char* a, const char* b)
{
    if (*a != *b)
        return 1;
    while ((*a != 0 && *b != 0) && *(a) == *(b)) {
        a++;
        b++;
    }
    return (*a == 0 && *b == 0) ? 0 : 1;
}

static void grib_print_header(grib_runtime_options* options, grib_handle* h);
static void grib_tools_set_print_keys(grib_runtime_options* options, grib_handle* h, const char* ns);
static int grib_tool_with_orderby(grib_runtime_options* options);
static int grib_tool_without_orderby(grib_runtime_options* options);
static int grib_tool_onlyfiles(grib_runtime_options* options);
static int grib_tool_index(grib_runtime_options* options);
static int process(grib_context* c, grib_runtime_options* options, const char* path);
static int scan(grib_context* c, grib_runtime_options* options, const char* dir);

FILE* dump_file;

static grib_runtime_options global_options = {
    0,  /* verbose       */
    0,  /* fail          */
    0,  /* skip          */
    12, /* default_print_width */
    0,  /* print_header */
    0,  /* name_space */
    0,  /* print_number */
    1,  /* print_statistics */
    {{0,},}, /* grib_values requested_print_keys[MAX_KEYS] */
    0, /* requested_print_keys_count */
    {{0,},}, /* grib_values print_keys[MAX_KEYS] */
    0, /* print_keys_count  */
    0, /* strict            */
    0, /* multi_support     */
    0, /* set_values_count  */
    {{0,},}, /* grib_values set_values[MAX_KEYS] */
    {{0,},}, /* grib_values constraints[MAX_KEYS] */
    0, /* constraints_count */
    {{0,},}, /* grib_values compare[MAX_KEYS] */
    0, /* compare_count */
    0, /* handle_count      */
    0, /* filter_handle_count */
    0, /* file_count     */
    0, /* grib_tools_file* infile_extra */
    0, /* grib_tools_file* current_infile */
    0, /* grib_tools_file* infile */
    0, /* grib_tools_file* outfile */
    0, /* grib_action* action */
    0, /* int dump_flags; */
    0, /* char* dump_mode; */
    0, /* repack    */
    0, /* error    */
    0, /* gts    */
    0, /* orderby    */
    0, /* latlon    */
    {0,}, /* double lats[4] */
    {0,}, /* double lons[4] */
    {0,}, /* double values[4] */
    {0,}, /* double distances[4] */
    {0,},  /* int indexes[4] */
    4,  /* int latlon_mode */
    0,  /* char* latlon_mask */
    -1, /* int latlon_idx */
    {0,}, /* double mask_values[4] */
    0, /* index */
    0, /* index_on */
    0, /* constant */
    0, /* dump_filename*/
    0, /* grib_fieldset* idx */
    0, /* random */
    0, /* format */
    0, /* onlyfiles */
    0, /* tolerance_count  */
    0, /* through_index */
    0, /* index1  */
    0, /* index2  */
    0, /* context  */
    0, /* stop  */
    0, /* mode  */
    0, /* headers_only  */
    0, /* skip_all  */
    {{0,},}, /* grib_values tolerance[MAX_KEYS] */
    0, /* infile_offset */
    0  /* JSON output */
};

static grib_handle* grib_handle_new_from_file_x(grib_context* c, FILE* f, int mode, int headers_only, int* err)
{
    if (mode == MODE_GRIB)
        return grib_new_from_file(c, f, headers_only, err);

    if (mode == MODE_BUFR)
        return bufr_new_from_file(c, f, err);

    if (mode == MODE_ANY)
        return any_new_from_file(c, f, err);

    if (mode == MODE_GTS)
        return gts_new_from_file(c, f, err);

    if (mode == MODE_METAR)
        return metar_new_from_file(c, f, err);

    if (mode == MODE_TAF)
        return taf_new_from_file(c, f, err);

    ECCODES_ASSERT(!"grib_handle_new_from_file_x: unknown mode");
    return NULL;
}

int grib_tool(int argc, char** argv)
{
#ifdef HAVE_ECKIT_GEO
    eckit::Main::initialise(argc, argv);
#endif

    int ret                = 0;
    int i = 0;
    grib_context* c        = grib_context_get_default();
    global_options.context = c;

    /* This is a consequence of ECC-440.
     * We want to keep the output file(s) opened as various
     * messages are appended to them. Otherwise they will be opened/closed
     * multiple times.
     */
    if (c->file_pool_max_opened_files == 0)
        c->file_pool_max_opened_files = 200;

#ifdef ENABLE_FLOATING_POINT_EXCEPTIONS
    feenableexcept(FE_ALL_EXCEPT & ~FE_INEXACT);
#endif

    if (getenv("DOXYGEN_USAGE") && argc == 1)
        usage_doxygen();

    grib_get_runtime_options(argc, argv, &global_options);

    grib_tool_before_getopt(&global_options);

    grib_process_runtime_options(c, argc, argv, &global_options);

    grib_tool_init(&global_options);

    ECCODES_ASSERT(global_options.dump_filename == NULL);
    dump_file = stdout;
    // if (global_options.dump_filename) {
    //     dump_file = fopen(global_options.dump_filename, "w");
    //     if (!dump_file) {
    //         perror(global_options.dump_filename);
    //         exit(1);
    //     }
    // }

    /* ECC-926: Currently only GRIB and BUFR indexing work. Disable the through_index if GTS etc */
    if ((global_options.mode == MODE_GRIB || global_options.mode == MODE_BUFR) &&
        is_index_file(global_options.infile->name) &&
        (global_options.infile_extra && is_index_file(global_options.infile_extra->name)))
    {
        global_options.through_index = 1;
        return grib_tool_index(&global_options);
    }

    if (global_options.onlyfiles)
        ret = grib_tool_onlyfiles(&global_options);
    else {
        if (global_options.orderby)
            ret = grib_tool_with_orderby(&global_options);
        else
            ret = grib_tool_without_orderby(&global_options);
    }

    if (global_options.dump_filename)
        fclose(dump_file);

    /* Free memory */
    for (i = 0; i < global_options.print_keys_count; i++) {
        if (global_options.print_keys[i].name) {
            free((char*)global_options.print_keys[i].name);
        }
    }
    return ret;
}

static int grib_tool_with_orderby(grib_runtime_options* options)
{
    int err             = 0;
    grib_failed *failed = NULL, *p = NULL;
    grib_handle* h          = NULL;
    grib_tools_file* infile = options->infile;
    const char** filenames = NULL;
    int files_count    = 0;
    grib_fieldset* set = NULL;
    int i              = 0;
    grib_context* c    = grib_context_get_default();

    if (infile)
        infile->failed = NULL;

    files_count = 0;
    while (infile) {
        files_count++;
        infile = infile->next;
    }

    filenames = (const char**)grib_context_malloc_clear(c, files_count * sizeof(char*));

    infile = options->infile;
    for (i = 0; i < files_count; i++) {
        filenames[i] = infile->name;
        infile       = infile->next;
    }

    if (grib_options_on("7"))
        c->no_fail_on_wrong_length = 1;

    set = grib_fieldset_new_from_files(0, filenames, files_count, 0, 0, 0, options->orderby, &err);
    if (err) {
        grib_context_log(c, GRIB_LOG_ERROR, "unable to create index for input file %s (%s)",
                         filenames[0], grib_get_error_message(err));
        exit(err);
    }

    options->handle_count = 0;
    grib_context_set_handle_file_count(c, 0);  /* ECC-873 */
    grib_context_set_handle_total_count(c, 0); /* ECC-873 */
    while (!options->skip_all && ((h = grib_fieldset_next_handle(set, &err)) != NULL || err != GRIB_SUCCESS)) {
        options->handle_count++;
        grib_context_set_handle_file_count(c, options->handle_count);  /* ECC-873 */
        grib_context_set_handle_total_count(c, options->handle_count); /* ECC-873 */
        options->error = err;

        if (!h) {
            grib_no_handle_action(options, err);

            failed        = (grib_failed*)grib_context_malloc_clear(c, sizeof(grib_failed));
            failed->count = infile->handle_count;
            failed->error = err;
            failed->next  = NULL;

            if (!infile->failed) {
                infile->failed = failed;
            }
            else {
                p = infile->failed;
                while (p->next)
                    p = p->next;
                p->next = failed;
            }

            continue;
        }

        if (options->json_output == 0 || options->latlon)
            grib_print_header(options, h);
        else
            grib_tools_set_print_keys(options, h, options->name_space);

        grib_skip_check(options, h);

        if (options->skip && options->strict) {
            grib_tool_skip_handle(options, h);
            continue;
        }

        grib_tool_new_handle_action(options, h);

        grib_print_key_values(options, h);

        grib_handle_delete(h);
    }

    if (set->size==0) fprintf(stderr, "%s: No messages found in fieldset\n", tool_name);
    grib_tool_finalise_action(options);

    grib_fieldset_delete(set);
    free(filenames);

    return 0;
}

static char iobuf[1024 * 1024];

// Read the first few bytes of the file to guess what kind of product
// it could be. Returns an empty string if it fails
static std::string guess_file_product(const std::string& filename)
{
    std::string result;
    char buffer[5] = {0,};
    FILE* fin = fopen(filename.c_str(), "rb");
    if (fin) {
        size_t bytes = fread(buffer, 1, sizeof(buffer), fin);
        if (bytes == sizeof(buffer)) {
            if (strncmp(buffer, "GRIB", 4)==0) {
                result = "GRIB";
            } else if (strncmp(buffer, "BUDG", 4)==0) {
                result = "GRIB";
            } else if (strncmp(buffer, "BUFR", 4)==0) {
                result = "BUFR";
            }
        }
        fclose(fin);
    }
    return result;
}

static int grib_tool_without_orderby(grib_runtime_options* options)
{
    int err = 0;
    /*int nofail=0;*/
    grib_failed *failed = NULL, *p = NULL;
    grib_handle* h          = NULL;
    grib_tools_file* infile = options->infile;

    grib_context* c              = grib_context_get_default();
    options->file_count          = 0;
    options->handle_count        = 0;
    options->filter_handle_count = 0;
    options->current_infile      = options->infile;
    infile->failed               = NULL;

    if (grib_options_on("7"))
        c->no_fail_on_wrong_length = 1;

    while (infile != NULL && infile->name != NULL) {
        if (options->print_statistics && options->verbose && !options->json_output)
            fprintf(dump_file, "%s\n", infile->name);
        if (strcmp(infile->name, "-") == 0)
            infile->file = stdin;
        else
            infile->file = fopen(infile->name, "rb");
        if (!infile->file) {
            perror(infile->name);
            exit(1);
        }
        if (options->infile_offset) {
#ifndef ECCODES_ON_WINDOWS
            /* Check at compile time to ensure our file offset is at least 64 bits */
            static_assert(sizeof(options->infile_offset) >= 8);
#endif
            err = fseeko(infile->file, options->infile_offset, SEEK_SET);
            if (err) {
                fprintf(stderr, "%s: Invalid file offset: %ld\n", tool_name, (long)options->infile_offset);
                exit(1);
            }
        }

        setvbuf(infile->file, iobuf, _IOFBF, sizeof(iobuf));

        options->file_count++;
        infile->handle_count        = 0;
        infile->filter_handle_count = 0;

        grib_tool_new_file_action(options, infile);
        /*nofail=grib_options_on("f");*/

        while (!options->skip_all && ((h = grib_handle_new_from_file_x(c, infile->file, options->mode,
                                                                       options->headers_only, &err)) != NULL ||
                                      err != GRIB_SUCCESS)) {
            infile->handle_count++;
            options->handle_count++;

            if (c->no_fail_on_wrong_length && (err == GRIB_PREMATURE_END_OF_FILE || err == GRIB_WRONG_LENGTH))
                err = 0;
            if (!options->error) {
                /* ECC-1086: Do not clear a previous error */
                options->error = err;
            }

            if (!h) {
                /* fprintf(dump_file,"\t\t\"ERROR: unreadable message\"\n"); */
                grib_no_handle_action(options, err);

                failed        = (grib_failed*)grib_context_malloc_clear(c, sizeof(grib_failed));
                failed->count = infile->handle_count;
                failed->error = err;
                failed->next  = NULL;

                if (!infile->failed) {
                    infile->failed = failed;
                }
                else {
                    p = infile->failed;
                    while (p->next)
                        p = p->next;
                    p->next = failed;
                }

                continue;
            }

            if (options->json_output == 0 || options->latlon)
                grib_print_header(options, h);
            else
                grib_tools_set_print_keys(options, h, options->name_space);

            grib_skip_check(options, h);

            if (options->skip && options->strict) {
                grib_tool_skip_handle(options, h);
                continue;
            }

            grib_tool_new_handle_action(options, h);

            grib_print_key_values(options, h);

            grib_handle_delete(h);
        }

        grib_print_file_statistics(options, infile);

        if (infile->file)
            fclose(infile->file);

        if (infile->handle_count == 0) {
            fprintf(stderr, "%s: No messages found in %s\n", tool_name, infile->name);
            std::string product = guess_file_product(infile->name);
            if (!product.empty()) {
                fprintf(stderr, "%s: Input file seems to be %s\n", tool_name, product.c_str());
            }
            if (options->fail)
                exit(1);
        }

        infile                  = infile->next;
        options->current_infile = infile;
    }

    grib_print_full_statistics(options);

    grib_tool_finalise_action(options);

    return options->error;
}

static int navigate(grib_field_tree* fields, grib_runtime_options* options)
{
    int err          = 0;
    int message_type = 0;

    if (!fields || options->stop)
        return 0;

    switch (options->mode) {
        case MODE_GRIB:
            message_type = CODES_GRIB;
            break;
        case MODE_BUFR:
            message_type = CODES_BUFR;
            break;
        default:
            fprintf(stderr, "%s %s: Invalid mode", tool_name, __func__);
            exit(1);
    }

    if (fields->field) {
        grib_handle* h = codes_index_get_handle(fields->field, message_type, &err);
        if (!options->index2->current)
            options->index2->current = (grib_field_list*)grib_context_malloc_clear(options->context, sizeof(grib_field_list));
        options->index2->current->field = fields->field;
        if (!h)
            return err;
        grib_skip_check(options, h);
        if (options->skip && options->strict) {
            grib_tool_skip_handle(options, h);
        }
        else {
            grib_tool_new_handle_action(options, h);
            grib_handle_delete(h);
        }
    }

    err = navigate(fields->next_level, options);
    if (err)
        return err;

    err = navigate(fields->next, options);

    return err;
}

static int grib_tool_index(grib_runtime_options* options)
{
    int err  = 0;
    char* f1 = options->infile->name;
    char* f2 = options->infile_extra->name;
    grib_index_key *k1, *k2;
    int found = 0;

    grib_context* c = grib_context_get_default();

    options->index1 = grib_index_read(c, f1, &err);
    if (err)
        grib_context_log(c, (GRIB_LOG_FATAL) | (GRIB_LOG_PERROR),
                         "unable to read index from %s", f1);

    options->index2 = grib_index_read(c, f2, &err);
    if (err)
        grib_context_log(c, (GRIB_LOG_FATAL) | (GRIB_LOG_PERROR),
                         "unable to read index from %s", f2);

    k1 = options->index1->keys;
    while (k1) {
        k2    = options->index2->keys;
        found = 0;
        while (k2) {
            if (!strcmp(k1->name, k2->name)) {
                found = 1;
                break;
            }
            k2 = k2->next;
        }
        if (!found) {
            fprintf(stderr, "Indexes contained in the input files have different keys!\n");
            fprintf(stderr, "keys in file %s:\n", f1);
            k1 = options->index1->keys;
            while (k1) {
                fprintf(stderr, "\t%s\n", k1->name);
                k1 = k1->next;
            }
            fprintf(stderr, "keys in file %s:\n", f2);
            k2 = options->index2->keys;
            while (k2) {
                fprintf(stderr, "\t%s\n", k2->name);
                k2 = k2->next;
            }
            exit(1);
        }

        k1->value[0] = 0;
        k1           = k1->next;
    }

    k2 = options->index2->keys;
    while (k2) {
        k1    = options->index1->keys;
        found = 0;
        while (k1) {
            if (!strcmp(k1->name, k2->name)) {
                found = 1;
                break;
            }
            k1 = k1->next;
        }
        if (!found) {
            fprintf(stderr,"Indexes contained in the input files have different keys!\n");
            fprintf(stderr, "keys in file %s:\n", f2);
            k2 = options->index2->keys;
            while (k2) {
                fprintf(stderr, "\t%s\n", k2->name);
                k2 = k2->next;
            }
            fprintf(stderr, "keys in file %s:\n", f1);
            k1 = options->index1->keys;
            while (k1) {
                fprintf(stderr, "\t%s\n", k1->name);
                k1 = k1->next;
            }

            exit(1);
        }

        k2 = k2->next;
    }

    navigate(options->index2->fields, options);

    grib_context_free(c, options->index2->current);

    grib_tool_finalise_action(options);

    return 0;
}

#ifndef ECCODES_ON_WINDOWS
static int scan(grib_context* c, grib_runtime_options* options, const char* dir)
{
    struct dirent* s;
    DIR* d;
    int err = 0;

    d = opendir(dir);
    if (!d) {
        grib_context_log(c, (GRIB_LOG_ERROR) | (GRIB_LOG_PERROR), "opendir %s", dir);
        return GRIB_IO_PROBLEM;
    }

    while ((s = readdir(d)) && (err == 0)) {
        if (strcmp(s->d_name, ".") != 0 && strcmp(s->d_name, "..") != 0) {
            char buf[1024];
            snprintf(buf, sizeof(buf), "%s/%s", dir, s->d_name);
            err = process(c, options, buf);
        }
    }
    closedir(d);
    return err;
}
#else
static int scan(grib_context* c, grib_runtime_options* options, const char* dir)
{
    struct _finddata_t fileinfo;
    intptr_t handle;
    char buffer[1024];
    snprintf(buffer, sizeof(buffer), "%s/*", dir);
    if ((handle = _findfirst(buffer, &fileinfo)) != -1) {
        do {
            if (strcmp(fileinfo.name, ".") != 0 && strcmp(fileinfo.name, "..") != 0) {
                char buf[1024];
                snprintf(buf, sizeof(buf), "%s/%s", dir, fileinfo.name);
                process(c, options, buf);
            }
        } while (!_findnext(handle, &fileinfo));

        _findclose(handle);
    }
    else {
        grib_context_log(c, (GRIB_LOG_ERROR) | (GRIB_LOG_PERROR), "opendir %s", dir);
        return GRIB_IO_PROBLEM;
    }
    return 0;
}
#endif


static int process(grib_context* c, grib_runtime_options* options, const char* path)
{
    struct stat s;
    int stat_val = 0;
    int err = 0;

#ifndef ECCODES_ON_WINDOWS
    stat_val = lstat(path, &s);
#else
    stat_val = stat(path, &s);
#endif

    if (stat_val != 0) {
        grib_context_log(c, (GRIB_LOG_ERROR) | (GRIB_LOG_PERROR), "Cannot stat %s", path);
        return GRIB_IO_PROBLEM;
    }

    if (S_ISDIR(s.st_mode) && !S_ISLNK(s.st_mode)) {
        err = scan(c, options, path);
    }
    else {
        err = grib_tool_new_filename_action(options, path);
    }
    return err;
}

static int grib_tool_onlyfiles(grib_runtime_options* options)
{
    grib_context* c         = grib_context_get_default();
    grib_tools_file* infile = options->infile;
    int err = 0;

    while (infile != NULL && infile->name != NULL) {
        err = process(c, options, infile->name);
        if (err) return err;
        infile = infile->next;
    }

    err = grib_tool_finalise_action(options);

    return err;
}

static void grib_print_header(grib_runtime_options* options, grib_handle* h)
{
    size_t strlenkey = 0;
    int width;
    int written_to_dump = 0; /* boolean */
    if (options->json_output && !options->latlon)
        return; // For JSON output we do not print a single header for all msgs
    if (options->handle_count != 1)
        return;

    grib_tools_set_print_keys(options, h, options->name_space);

    if (options->verbose && options->print_header) {
        int j = 0;
        for (j = 0; j < options->print_keys_count; j++) {
            strlenkey = strlen(options->print_keys[j].name);
            width     = strlenkey < options->default_print_width ? options->default_print_width + 2 : strlenkey + 2;
            if (options->default_print_width < 0)
                width = strlenkey + 1;

            fprintf(dump_file, "%-*s", (int)width, options->print_keys[j].name);
            written_to_dump = 1;
        }
        if (options->latlon) {
            if (options->latlon_mode == 4) {
                fprintf(dump_file, "       value1 ");
                fprintf(dump_file, " value2 ");
                fprintf(dump_file, " value3 ");
                fprintf(dump_file, " value4 ");
            }
            else {
                fprintf(dump_file, " value ");
            }
            written_to_dump = 1;
        }
        if (options->index_on) {
            fprintf(dump_file, "        value(%d) ", (int)options->index);
            written_to_dump = 1;
        }
        if (written_to_dump) {
            fprintf(dump_file, "\n");
        }
    }
}

static int cmpstringp(const void* p1, const void* p2)
{
    // The actual arguments to this function are "pointers to
    // pointers to char", but strcmp(3) arguments are "pointers
    // to char", hence the following cast plus dereference
    return strcmp(*(char* const*)p1, *(char* const*)p2);
}

static void grib_tools_set_print_keys(grib_runtime_options* options, grib_handle* h, const char* ns)
{
    int i                     = 0;
    grib_keys_iterator* kiter = NULL;

    options->print_keys_count = 0;

    for (i = 0; i < options->requested_print_keys_count; i++) {
        options->print_keys[options->print_keys_count].name = options->requested_print_keys[i].name;
        if (strlen(options->requested_print_keys[i].name) > options->default_print_width)
            options->default_print_width = (int)strlen(options->requested_print_keys[i].name);
        options->print_keys[options->print_keys_count].type = options->requested_print_keys[i].type;
        options->print_keys_count++;
    }

    if (ns) {
        kiter = grib_keys_iterator_new(h, 0, ns);
        if (!kiter) {
            fprintf(stderr, "%s: Unable to create keys iterator\n", tool_name);
            exit(1);
        }

        while (grib_keys_iterator_next(kiter)) {
            const char* name = grib_keys_iterator_get_name(kiter);

            if (options->print_keys_count >= MAX_KEYS) {
                fprintf(stderr, "%s: Keys list too long (more than %d keys)\n",
                        tool_name, options->print_keys_count);
                exit(1);
            }
            if (options->print_keys[options->print_keys_count].name) {
                free((char*)options->print_keys[options->print_keys_count].name);
            }
            options->print_keys[options->print_keys_count].name = strdup(name);
            if (strlen(name) > options->default_print_width)
                options->default_print_width = (int)strlen(name);
            options->print_keys[options->print_keys_count].type = GRIB_TYPE_STRING;
            // For the statistics namespace, do not force the type to be string.
            // Setting it to undefined will use the keys' native type i.e. GRIB_TYPE_DOUBLE
            if (strcmp(ns,"statistics")==0)
                options->print_keys[options->print_keys_count].type = GRIB_TYPE_UNDEFINED;
            options->print_keys_count++;
        }

        grib_keys_iterator_delete(kiter);
        if (options->print_keys_count == 0 && options->latlon == 0) {
            int j = 0, k = 0, ns_count = 0;
            const char* all_namespace_vals[1024] = {NULL,}; // sorted array containing all namespaces
            printf("ERROR: namespace \"%s\" does not contain any key.\n", ns);
            printf("Here are the available namespaces in this message:\n");
            for (i = 0; i < ACCESSORS_ARRAY_SIZE; i++) {
                grib_accessor* anAccessor = h->accessors[i];
                if (anAccessor) {
                    for (j = 0; j < MAX_ACCESSOR_NAMES; j++) {
                        const char* a_namespace = anAccessor->all_name_spaces_[j];
                        if (a_namespace) {
                            all_namespace_vals[k++] = a_namespace;
                            ns_count++;
                        }
                    }
                }
            }
            qsort(&all_namespace_vals, ns_count, sizeof(char*), cmpstringp);
            for (i = 0; i < ns_count; ++i) {
                if (all_namespace_vals[i]) {
                    int print_it = 1;
                    if (i > 0 && strcmp(all_namespace_vals[i], all_namespace_vals[i - 1]) == 0) {
                        print_it = 0; // skip duplicate entries
                    }
                    if (print_it)
                        printf("\t%s\n", all_namespace_vals[i]);
                }
            }
            exit(1);
        }
    }
}

static int to_skip(grib_runtime_options* options, grib_handle* h, grib_values* v, int* err)
{
    double dvalue              = 0;
    int ret                    = 0;
    long lvalue                = 0;
    char value[MAX_STRING_LEN] = {0,};
    size_t len = MAX_STRING_LEN;
    *err       = 0;

    ECCODES_ASSERT(options->constraints_count > 0);

    if (strcmp(v->name, "count")==0 && v->long_value < 1) {
        fprintf(dump_file, "ERROR: Invalid value for key '%s' (must be an integer greater than 0)\n", v->name);
        exit(1);
    }

    switch (v->type) {
        case GRIB_TYPE_STRING:
            *err = grib_get_string(h, v->name, value, &len);
            ret  = v->equal ? grib_inline_strcmp(value, v->string_value) : !grib_inline_strcmp(value, v->string_value);
            break;
        case GRIB_TYPE_DOUBLE:
            *err = grib_get_double(h, v->name, &dvalue);
            ret  = v->equal ? (dvalue != v->double_value) : (dvalue == v->double_value);
            break;
        case GRIB_TYPE_LONG:
            *err = grib_get_long(h, v->name, &lvalue);
            ret  = v->equal ? (lvalue != v->long_value) : (lvalue == v->long_value);
            break;
        case GRIB_TYPE_MISSING:
            lvalue = grib_is_missing(h, v->name, err);
            ret    = (lvalue == v->equal) ? 0 : 1;
            break;
        default:
            fprintf(dump_file, "ERROR: Invalid type for %s\n", v->name);
            exit(1);
    }

    return ret;
}

void grib_skip_check(grib_runtime_options* options, grib_handle* h)
{
    int i, ret = 0;
    grib_values* v = NULL;

    /* ECC-1179: bufr_copy/bufr_ls: Allow 'where' clause with Data Section keys */
    if (options->constraints_count > 0 && h->product_kind == PRODUCT_BUFR) {
        for (i = 0; i < options->set_values_count; i++) {
            if (strcmp(options->set_values[i].name, "unpack")==0) {
                grib_set_long(h, "unpack", 1);
                break;
            }
        }
    }

    for (i = 0; i < options->constraints_count; i++) {
        v = &(options->constraints[i]);
        if (v->equal) {
            options->skip = 1;
            while (v) {
                if (!to_skip(options, h, v, &ret)) {
                    if (!strcmp(v->name, "count") && !v->next) {
                        /* We have count=XX and nothing after that so we can
                         * skip every other message after it
                         */
                        options->skip_all = 1;
                    }
                    options->skip = 0;
                    break;
                }
                if (ret != GRIB_SUCCESS && options->fail) {
                    grib_context_log(h->context, GRIB_LOG_ERROR, "unable to get \"%s\" (%s)",
                                     v->name, grib_get_error_message(ret));
                    exit(ret);
                }
                v = v->next;
            }
        }
        else {
            options->skip = 0;
            while (v) {
                if (to_skip(options, h, v, &ret)) {
                    options->skip = 1;
                    break;
                }
                if (ret != GRIB_SUCCESS && options->fail) {
                    grib_context_log(h->context, GRIB_LOG_ERROR, "unable to get \"%s\" (%s)",
                                     v->name, grib_get_error_message(ret));
                    exit(ret);
                }
                v = v->next;
            }
        }
        if (options->skip == 1)
            break;
    }

    if (!options->skip) {
        options->filter_handle_count++;
        if (options->current_infile)
            options->current_infile->filter_handle_count++;
    }
}

/* TODO: Does not work for 2.7e+01 */
static int is_valid_JSON_number(const char* input)
{
    const char* p = input;
    size_t len    = 0;
    int is_float  = 0;
    if (p == 0 || *p == '\0')
        return 0;
    if (*p == '-')
        p++;

    while (*p) {
        if (*p == '.') {
            if (is_float) return 0; /*more than 1 dot*/
            is_float = 1;
        }
        if (*p != '.' && !isdigit(*p))
            return 0;
        p++;
        len++;
    }
    /*
     * Note: BUFR keys like typicalTime/rdbtimetime can have values
     * like 000000 or 013329 which are invalid JSON numbers.
     * In JSON a leading zero must not be followed by another digit
     */
    if (!is_float && len > 2 && input[0] == '0' && isdigit(input[1]))
        return 0; /* Not a valid JSON number */
    return 1;
}

static int get_initial_element_of_array(grib_handle* h, const char* keyName, size_t num_vals, char* value)
{
    int err = 0, type = 0;
    size_t len          = num_vals;
    char* sval          = NULL;
    unsigned char* uval = NULL;
    long* lval          = NULL;
    double* dval        = NULL;
    grib_context* c     = h->context;

    ECCODES_ASSERT(num_vals > 1); /* This is for array keys */
    if ((err = grib_get_native_type(h, keyName, &type)) != GRIB_SUCCESS)
        return err;
    switch (type) {
        case GRIB_TYPE_STRING:
            grib_get_string_length(h, keyName, &len);
            sval = (char*)grib_context_malloc(c, len * sizeof(char));
            if (!sval)
                return GRIB_OUT_OF_MEMORY;
            if ((err = grib_get_string(h, keyName, sval, &len)) != GRIB_SUCCESS) {
                free(sval);
                return err;
            }
            snprintf(value, len, "%s", sval);
            free(sval);
            break;
        case GRIB_TYPE_LONG:
            lval = (long*)grib_context_malloc(c, num_vals * sizeof(long));
            if (!lval)
                return GRIB_OUT_OF_MEMORY;
            if ((err = grib_get_long_array(h, keyName, lval, &len)) != GRIB_SUCCESS)
                return err;
            snprintf(value, 32, "%ld...", lval[0]);
            free(lval);
            break;
        case GRIB_TYPE_DOUBLE:
            dval = (double*)grib_context_malloc(c, num_vals * sizeof(double));
            if (!dval)
                return GRIB_OUT_OF_MEMORY;
            if ((err = grib_get_double_array(h, keyName, dval, &len)) != GRIB_SUCCESS)
                return err;
            snprintf(value, 32, "%g...", dval[0]);
            free(dval);
            break;
        case GRIB_TYPE_BYTES:
            uval = (unsigned char*)grib_context_malloc(c, num_vals * sizeof(unsigned char));
            if (!uval)
                return GRIB_OUT_OF_MEMORY;
            if ((err = grib_get_bytes(h, keyName, uval, &len)) != GRIB_SUCCESS)
                return err;
            snprintf(value, 32, "%d...", (short)uval[0]);
            free(uval);
            break;
        default:
            snprintf(value, 32, "%s...", "");
    }
    return GRIB_SUCCESS;
}

static void get_value_for_key(grib_handle* h, const char* key_name, int key_type, char* value_str,
                              const char* format, int fix_lsdate, int fix_lstime)
{
    int ret = 0, type = key_type;
    double dvalue = 0;
    long lvalue   = 0;
    size_t len    = MAX_STRING_LEN;
    if (grib_is_missing(h, key_name, &ret) && ret == GRIB_SUCCESS) {
        snprintf(value_str, 32, "MISSING");
        return;
    }
    if (ret == GRIB_NOT_FOUND) {
        snprintf(value_str, 32, "not_found");
        return;
    }

    if (type == GRIB_TYPE_UNDEFINED) {
        ret = grib_get_native_type(h, key_name, &type);
        if (ret != GRIB_SUCCESS) {
            fprintf(dump_file, "ERROR: Could not determine type for %s\n", key_name);
            exit(1);
        }
    }

    if (type == GRIB_TYPE_STRING) {
        const char* pName = key_name;
        size_t num_vals = 0;
        /* ECC-707 */
        if (fix_lsdate && strcmp(pName, "date") == 0) pName = "ls.date";
        if (fix_lstime && strcmp(pName, "time") == 0) pName = "ls.time";
        ret = grib_get_size(h, pName, &num_vals);
        if (ret == GRIB_SUCCESS && num_vals > 1) { /* See ECC-278 */
            ret = get_initial_element_of_array(h, pName, num_vals, value_str);
        } else {
            ret = grib_get_string(h, pName, value_str, &len);
        }
    }
    else if (type == GRIB_TYPE_DOUBLE) {
        ret = grib_get_double(h, key_name, &dvalue);
        snprintf(value_str, 32, format, dvalue);
    }
    else if (type == GRIB_TYPE_LONG) {
        ret = grib_get_long(h, key_name, &lvalue);
        snprintf(value_str, 32, "%ld", lvalue);
    }
    else if (type == GRIB_TYPE_BYTES) {
        ret = grib_get_string(h, key_name, value_str, &len);
    }
    else {
        fprintf(dump_file, "ERROR: Invalid format option for %s\n", key_name);
        exit(1);
    }

    if (ret != GRIB_SUCCESS) {
        if (ret == GRIB_NOT_FOUND) {
            snprintf(value_str, 32, "not_found");
        } else {
            fprintf(dump_file, "ERROR: Failed to get value for key '%s' (%s)\n", key_name, grib_get_error_message(ret));
            if (ret == GRIB_ARRAY_TOO_SMALL)
                fprintf(dump_file, "\tHint: Tool %s cannot print keys of array type. Use grib_filter.\n", tool_name);
            exit(1);
        }
    }
}

/* See ECC-707 */
static int fix_for_lsdate_needed(grib_handle* h)
{
    long lsdate_bug = 0;
    int err         = grib_get_long(h, "lsdate_bug", &lsdate_bug);
    if (!err && lsdate_bug == 1) return 1;
    return 0;
}
static int fix_for_lstime_needed(grib_handle* h)
{
    long lstime_bug = 0;
    int err         = grib_get_long(h, "lstime_bug", &lstime_bug);
    if (!err && lstime_bug == 1) return 1;
    return 0;
}

void grib_print_key_values(grib_runtime_options* options, grib_handle* h)
{
    int i   = 0;
    int ret = 0, width = 0;
    int strlenvalue = 0;
    double dvalue   = 0;
    long lvalue     = 0;
    char value[MAX_STRING_LEN] = {0,};
    const char* notfound = "not_found";
    int written_to_dump  = 0; /* boolean */
    grib_accessor* acc   = NULL;
    size_t num_vals      = 0;
    int fix_lsdate       = 0;
    int fix_lstime       = 0;

    if (!options->verbose)
        return;

    fix_lsdate = (options->name_space && strcmp(options->name_space, "ls") == 0 && fix_for_lsdate_needed(h));
    fix_lstime = (options->name_space && strcmp(options->name_space, "ls") == 0 && fix_for_lstime_needed(h));

    if (options->json_output && !options->latlon) {
        /* fprintf(dump_file, "\"message %d\" : {\n", options->handle_count); */
        fprintf(dump_file, "  {\n");
        for (i = 0; i < options->print_keys_count; i++) {
            fprintf(dump_file, "    \"%s\": ", options->print_keys[i].name);
            get_value_for_key(h, options->print_keys[i].name, options->print_keys[i].type, value,
                              options->format, fix_lsdate, fix_lstime);
            if (is_valid_JSON_number(value))
                fprintf(dump_file, "%s", value);
            else
                fprintf(dump_file, "\"%s\"", value);
            if (i != options->print_keys_count - 1)
                fprintf(dump_file, ",\n");
            else
                fprintf(dump_file, "\n");
        }
        fprintf(dump_file, "  }");
        return;
    }

    for (i = 0; i < options->print_keys_count; i++) {
        size_t len  = MAX_STRING_LEN;
        int keyType = options->print_keys[i].type;
        ret         = GRIB_SUCCESS;

        if (h->product_kind == PRODUCT_BUFR) {
            /* ECC-236: Do not use grib_is_missing for BUFR */
            if (!grib_is_defined(h, options->print_keys[i].name))
                ret = GRIB_NOT_FOUND;
            if (ret == GRIB_SUCCESS) {
                ret = grib_get_size(h, options->print_keys[i].name, &num_vals);
            }
            if (ret == GRIB_SUCCESS) {
                if (keyType == GRIB_TYPE_UNDEFINED)
                    grib_get_native_type(h, options->print_keys[i].name, &keyType);
                switch (keyType) {
                    case GRIB_TYPE_STRING:
                        acc = grib_find_accessor(h, options->print_keys[i].name);
                        ret = grib_get_string(h, options->print_keys[i].name, value, &len);
                        if (grib_is_missing_string(acc, (unsigned char*)value, len))
                            snprintf(value, 32, "MISSING");
                        break;
                    case GRIB_TYPE_DOUBLE:
                        if (num_vals > 1) {
                            ret = GRIB_ARRAY_TOO_SMALL;
                        } else {
                            ret = grib_get_double(h, options->print_keys[i].name, &dvalue);
                            if (dvalue == GRIB_MISSING_DOUBLE) snprintf(value, 32, "MISSING");
                            else                               snprintf(value, 32, options->format, dvalue);
                        }
                        break;
                    case GRIB_TYPE_LONG:
                        if (num_vals > 1) {
                            ret = GRIB_ARRAY_TOO_SMALL;
                        } else {
                            ret = grib_get_long(h, options->print_keys[i].name, &lvalue);
                            if (lvalue == GRIB_MISSING_LONG) snprintf(value, 32, "MISSING");
                            else                             snprintf(value, 32, "%ld", lvalue);
                        }
                        break;
                    case GRIB_TYPE_BYTES:
                        ret = grib_get_string(h, options->print_keys[i].name, value, &len);
                        break;
                    default:
                        fprintf(dump_file, "ERROR: Could not determine type for %s\n", options->print_keys[i].name);
                        exit(1);
                }
            }
        }
        else {
            /* Other products e.g. GRIB */
            if (grib_is_missing(h, options->print_keys[i].name, &ret) && ret == GRIB_SUCCESS) {
                snprintf(value, 32, "MISSING");
            }
            else if (ret == GRIB_SUCCESS) {
                const char* pName = NULL;
                if (keyType == GRIB_TYPE_UNDEFINED)
                    grib_get_native_type(h, options->print_keys[i].name, &keyType);
                switch (keyType) {
                    case GRIB_TYPE_STRING:
                        pName = options->print_keys[i].name;
                        if (fix_lsdate && strcmp(pName, "date") == 0) { /* ECC-707 */
                            pName = "ls.date";
                        }
                        if (fix_lstime && strcmp(pName, "time") == 0) {
                            pName = "ls.time";
                        }
                        ret = grib_get_size(h, pName, &num_vals);
                        if (ret == GRIB_SUCCESS && num_vals > 1) { /* See ECC-278 */
                            ret = get_initial_element_of_array(h, pName, num_vals, value);
                        }
                        else {
                            ret = grib_get_string(h, pName, value, &len);
                        }
                        break;
                    case GRIB_TYPE_DOUBLE:
                        ret = grib_get_double(h, options->print_keys[i].name, &dvalue);
                        snprintf(value, 32, options->format, dvalue);
                        break;
                    case GRIB_TYPE_LONG:
                        ret = grib_get_long(h, options->print_keys[i].name, &lvalue);
                        snprintf(value, 32, "%ld", lvalue);
                        break;
                    case GRIB_TYPE_BYTES:
                        ret = grib_get_string(h, options->print_keys[i].name, value, &len);
                        break;
                    default:
                        fprintf(dump_file, "ERROR: Invalid format option for %s\n", options->print_keys[i].name);
                        exit(1);
                }
            }
        }

        if (ret != GRIB_SUCCESS) {
            if (options->fail) { // ECC-1551
                //GRIB_CHECK_NOLINE(ret, options->print_keys[i].name);
                grib_context_log(h->context, GRIB_LOG_ERROR, "%s (%s)",
                                options->print_keys[i].name, grib_get_error_message(ret));
                if (ret == GRIB_ARRAY_TOO_SMALL || ret == GRIB_BUFFER_TOO_SMALL) {
                    fprintf(dump_file, "\tHint: Tool %s cannot print keys of array type. Use grib_filter.\n", tool_name);
                }
                exit(ret);
            }
            if (ret == GRIB_NOT_FOUND) {
                strcpy(value, notfound);
            } else {
                fprintf(dump_file, "%s (%s)\n", options->print_keys[i].name, grib_get_error_message(ret));
                if (ret == GRIB_ARRAY_TOO_SMALL || ret == GRIB_BUFFER_TOO_SMALL) {
                    fprintf(dump_file, "\tHint: Tool %s cannot print keys of array type. Use grib_filter.\n", tool_name);
                }
                exit(ret);
            }
        }

        strlenvalue = (int)strlen(value);

        width = strlenvalue < options->default_print_width ? options->default_print_width + 2 : strlenvalue + 2;

        if (options->default_print_width < 0)
            width = strlenvalue + 1;

        if (options->print_keys_count == i + 1 && options->latlon == 0)
            width--;

        fprintf(dump_file, "%-*s", (int)width, value);
        written_to_dump = 1;
    }

    if (options->latlon) {
        if (options->latlon_mode == 4) {
            int ii = 0;
            for (ii = 0; ii < LATLON_SIZE; ii++) {
                fprintf(dump_file, options->format, options->values[ii]);
                fprintf(dump_file, " ");
            }
            written_to_dump = 1;
        }
        else if (options->latlon_mode == 1) {
            snprintf(value, sizeof(value), options->format, options->values[options->latlon_idx]);
            strlenvalue = (int)strlen(value);
            width       = strlenvalue < options->default_print_width ? options->default_print_width + 2 : strlenvalue + 2;
            fprintf(dump_file, "%-*s", (int)width, value);
            written_to_dump = 1;
        }
    }
    if (options->index_on) {
        double v = 0;
        /*if (grib_get_double_element(h,"values",options->index,&v) != GRIB_SUCCESS) {*/
        if (1) {
            size_t size, the_index = 0;
            double* values;
            int err = 0;

            err = grib_get_size(h, "values", &size);
            if (err) {
                snprintf(value, 32, "unknown");
                if (!options->fail)
                    exit(err);
                return;
            }
            values = (double*)grib_context_malloc_clear(h->context, size * sizeof(double));
            grib_get_double_array(h, "values", values, &size);
            the_index = options->index;
            if (the_index >= size) {
                fprintf(dump_file, "\n");
                fprintf(stderr, "%s: Invalid index value %d (should be between 0 and %d)\n",
                        tool_name, options->index, (int)(size - 1));
                exit(1);
            }
            v = values[options->index];
            grib_context_free(h->context, values);
        }

        snprintf(value, 32, options->format, v);
        strlenvalue = (int)strlen(value);
        width       = strlenvalue < options->default_print_width ? options->default_print_width + 2 : strlenvalue + 2;
        fprintf(dump_file, "%s%-*s", (written_to_dump?" ":""),  (int)width, value);
        written_to_dump = 1;
    }
    if (written_to_dump) {
        fprintf(dump_file, "\n");
    }
}

void grib_print_file_statistics(grib_runtime_options* options, grib_tools_file* file)
{
    grib_failed* failed = NULL;
    ECCODES_ASSERT(file);
    if (options->json_output && !options->latlon)
        return;

    failed = file->failed;

    if (!options->print_statistics || !options->verbose)
        return;

    fprintf(dump_file, "%d of %d messages in %s\n\n",
            file->filter_handle_count,
            file->handle_count,
            file->name);
    if (!failed)
        return;

    //  fprintf(dump_file,"Following bad messages found in %s\n", file->name);
    //  fprintf(dump_file,"N      Error\n");
    //  while (failed){
    //      fprintf(dump_file,"%-*d    %s\n", 7,failed->count,
    //      grib_get_error_message(failed->error));
    //      failed=failed->next;
    //  }
    //  fprintf(dump_file,"\n");
}

void grib_print_full_statistics(grib_runtime_options* options)
{
    if (options->json_output && !options->latlon)
        return;
    if (options->print_statistics && options->verbose)
        fprintf(dump_file, "%d of %d total messages in %d files\n",
                options->filter_handle_count, options->handle_count, options->file_count);
}

static int filenames_equal(const char* f1, const char* f2)
{
    int eq = 0;
    grib_context* c = grib_context_get_default();
    char* resolved1 = codes_resolve_path(c, f1);
    char* resolved2 = codes_resolve_path(c, f2);
    eq = (strcmp(resolved1, resolved2)==0);
    grib_context_free(c, resolved1);
    grib_context_free(c, resolved2);
    return eq;
}

void grib_tools_write_message(grib_runtime_options* options, grib_handle* h)
{
    const void* buffer;
    size_t size;
    grib_file* of       = NULL;
    int err             = 0;
    char filename[1024] = {0,};
    ECCODES_ASSERT(options->outfile != NULL && options->outfile->name != NULL);

    /* See ECC-1086
     * if (options->error == GRIB_WRONG_LENGTH)
     *   return;
     */
    if ((err = grib_get_message(h, &buffer, &size)) != GRIB_SUCCESS) {
        grib_context_log(h->context, GRIB_LOG_ERROR, "unable to get binary message\n");
        exit(err);
    }

    err = grib_recompose_name(h, NULL, options->outfile->name, filename, 0);

    /* Check outfile is not same as infile */
    if (filenames_equal(options->infile->name, filename)) {
        grib_context_log(h->context, GRIB_LOG_ERROR,
                "output file '%s' is the same as input file. Aborting\n", filename);
        exit(GRIB_IO_PROBLEM);
    }

    of = grib_file_open(filename, "w", &err);

    if (!of || !of->handle) {
        grib_context_log(h->context, (GRIB_LOG_ERROR) | (GRIB_LOG_PERROR),
                         "unable to open file %s\n", filename);
        exit(GRIB_IO_PROBLEM);
    }

    if (options->gts && h->gts_header) {
        if (fwrite(h->gts_header, 1, h->gts_header_len, of->handle) != h->gts_header_len) {
            grib_context_log(h->context, (GRIB_LOG_ERROR) | (GRIB_LOG_PERROR),
                             "Error writing GTS header to %s", filename);
            exit(GRIB_IO_PROBLEM);
        }
    }

    if (fwrite(buffer, 1, size, of->handle) != size) {
        grib_context_log(h->context, (GRIB_LOG_ERROR) | (GRIB_LOG_PERROR),
                         "Error writing to %s", filename);
        exit(GRIB_IO_PROBLEM);
    }

    if (options->gts && h->gts_header) {
        const char gts_trailer[4] = { '\x0D', '\x0D', '\x0A', '\x03' };
        if (fwrite(gts_trailer, 1, 4, of->handle) != 4) {
            grib_context_log(h->context, (GRIB_LOG_ERROR) | (GRIB_LOG_PERROR),
                             "Error writing GTS trailer to %s", filename);
            exit(GRIB_IO_PROBLEM);
        }
    }

    grib_file_close(filename, 0, &err);

    if (err != GRIB_SUCCESS) {
        grib_context_log(h->context, GRIB_LOG_ERROR, "unable to write message\n");
        exit(err);
    }

    options->outfile->file = NULL;

#if 0
    if (!options->outfile->file)  {
        options->outfile->file = fopen(options->outfile->name,"w");
        if(!options->outfile->file) {
            perror(options->outfile->name);
            exit(1);
        }
    }
    GRIB_CHECK_NOLINE(grib_get_message(h,&buffer,&size),0);
    if (options->gts && h->gts_header)
        fwrite(h->gts_header,1,h->gts_header_len,options->outfile->file);

    if(fwrite(buffer,1,size,options->outfile->file) != size)
    {
        perror(options->outfile->name);
        exit(1);
    }

    if (options->gts && h->gts_header) {
        char gts_trailer[4]={'\x0D','\x0D','\x0A','\x03'};
        fwrite(gts_trailer,1,4,options->outfile->file);
    }
#endif
}
int exit_if_input_is_directory(const char* toolname, const char* filename)
{
    if (path_is_directory(filename)) {
        fprintf(stderr, "%s: ERROR: \"%s\": Is a directory\n", toolname, filename);
        exit(1);
    }
    return 0;
}