From 211c22ca37b784beceeea9fa72839d8cddde3884 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sat, 12 Mar 2022 12:52:00 +0000 Subject: [PATCH 1/6] Tools: codes_export_resource --- src/codes_memfs.c | 7 ++-- tools/CMakeLists.txt | 1 + tools/codes_export_resource.c | 69 +++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 tools/codes_export_resource.c diff --git a/src/codes_memfs.c b/src/codes_memfs.c index 5360393a2..a51f4a012 100644 --- a/src/codes_memfs.c +++ b/src/codes_memfs.c @@ -11,7 +11,7 @@ #include "grib_api_internal.h" #ifdef HAVE_MEMFS -/* These two functions are implemented in the generated C file memfs.c in the build area */ +/* These two functions are implemented in the generated C file memfs_gen_final.c in the build area */ /* See the memfs.py Python generator */ int codes_memfs_exists(const char* path); FILE* codes_memfs_open(const char* path); @@ -20,7 +20,7 @@ FILE* codes_fopen(const char* name, const char* mode) { FILE* f; - if (strcmp(mode, "r") != 0) { + if (strcmp(mode, "r") != 0) { /* Not reading */ return fopen(name, mode); } @@ -34,6 +34,7 @@ FILE* codes_fopen(const char* name, const char* mode) int codes_access(const char* name, int mode) { + /* F_OK tests for the existence of the file */ if (mode != F_OK) { return access(name, mode); } @@ -46,7 +47,7 @@ int codes_access(const char* name, int mode) } #else - +/* No MEMFS */ FILE* codes_fopen(const char* name, const char* mode) { return fopen(name, mode); diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index b8b0c41ba..afb57a18a 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -41,6 +41,7 @@ list( APPEND ecc_tools_binaries_extra gg_sub_area_check grib_repair grib_to_json + codes_export_resource grib_check_gaussian_grid bufr_split_by_rdbSubtype ) diff --git a/tools/codes_export_resource.c b/tools/codes_export_resource.c new file mode 100644 index 000000000..85e95ae45 --- /dev/null +++ b/tools/codes_export_resource.c @@ -0,0 +1,69 @@ +/* + * (C) Copyright 2005- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by + * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. + */ + +#include "grib_api_internal.h" +#include + +static int fail_on_error = 1; +static const char* toolname = NULL; + +static void usage(const char* prog) +{ + printf("Usage: %s [-s | -d] path\n", prog); + exit(1); +} + +int main(int argc, char* argv[]) +{ + char* path = NULL; + char* full = NULL; + char* option = NULL; + char* resource = NULL; + int verbose = 0; + int export_sample = 0, export_definition = 0; + grib_context* c = grib_context_get_default(); + + toolname = argv[0]; + option = argv[1]; + path = argv[2]; + + if (argc != 3) usage(toolname); + + if (strcmp(option, "-s") == 0) { + export_sample = 1; + resource = "sample"; + } else if (strcmp(option, "-d") == 0) { + export_definition = 1; + resource = "definition"; + } else { + fprintf(stderr, "Invalid option: Specify either -s or -d\n"); + return 1; + } + + if (export_sample) { + char* t = strstr(path, ".tmpl"); + if (t) { + *t = 0; // get rid of the sample file extension + } + full = grib_external_template_path(c, path); + } + else if (export_definition) { + full = grib_context_full_defs_path(c, path); + } + if (!full) { + fprintf(stderr, "Failed to export %s: %s\n", resource, path); + return 1; + } + printf("full = %s\n", full); + assert(path_is_regular_file(full)); + grib_context_free(c, full); + + return 0; +} From 95184606413b69e8f00181f40e9c74a8d1d1c46a Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sat, 12 Mar 2022 14:23:23 +0000 Subject: [PATCH 2/6] Tools: codes_export_resource --- src/grib_templates.c | 2 +- tools/codes_export_resource.c | 95 +++++++++++++++++++++++------------ 2 files changed, 65 insertions(+), 32 deletions(-) diff --git a/src/grib_templates.c b/src/grib_templates.c index 60a94fc62..f847c7409 100644 --- a/src/grib_templates.c +++ b/src/grib_templates.c @@ -111,7 +111,7 @@ static char* try_template_path(grib_context* c, const char* dir, const char* nam sprintf(path, "%s/%s.tmpl", dir, name); - if (codes_access(path, R_OK) == 0) { + if (codes_access(path, F_OK) == 0) { return grib_context_strdup(c, path); } diff --git a/tools/codes_export_resource.c b/tools/codes_export_resource.c index 85e95ae45..40ffb0e21 100644 --- a/tools/codes_export_resource.c +++ b/tools/codes_export_resource.c @@ -9,61 +9,94 @@ */ #include "grib_api_internal.h" -#include -static int fail_on_error = 1; -static const char* toolname = NULL; +typedef enum ResourceType +{ + UNKNOWN, + SAMPLE, + DEFINITION +} ResourceType; static void usage(const char* prog) { - printf("Usage: %s [-s | -d] path\n", prog); + printf("Usage: %s [-s | -d] resource_path out_file\n", prog); exit(1); } +#define SIZE (1024 * 1024) int main(int argc, char* argv[]) { - char* path = NULL; - char* full = NULL; - char* option = NULL; - char* resource = NULL; - int verbose = 0; - int export_sample = 0, export_definition = 0; - grib_context* c = grib_context_get_default(); + char* resource_path = NULL; + char* resource_name = NULL; + ResourceType resource_type = UNKNOWN; + char* full_path = NULL; + char* out_file = NULL; + char* option = NULL; + grib_context* c = grib_context_get_default(); + FILE* fin = NULL; + FILE* fout = NULL; + char buffer[SIZE] = {0,}; + size_t bytes = 0; - toolname = argv[0]; - option = argv[1]; - path = argv[2]; + if (argc != 4) usage(argv[0]); - if (argc != 3) usage(toolname); + option = argv[1]; + resource_path = argv[2]; + out_file = argv[3]; if (strcmp(option, "-s") == 0) { - export_sample = 1; - resource = "sample"; - } else if (strcmp(option, "-d") == 0) { - export_definition = 1; - resource = "definition"; - } else { + resource_type = SAMPLE; + resource_name = "sample"; + } + else if (strcmp(option, "-d") == 0) { + resource_type = DEFINITION; + resource_name = "definition"; + } + else { fprintf(stderr, "Invalid option: Specify either -s or -d\n"); return 1; } - if (export_sample) { - char* t = strstr(path, ".tmpl"); + if (resource_type == SAMPLE) { + char* t = strstr(resource_path, ".tmpl"); if (t) { *t = 0; // get rid of the sample file extension } - full = grib_external_template_path(c, path); + full_path = grib_external_template_path(c, resource_path); } - else if (export_definition) { - full = grib_context_full_defs_path(c, path); + else if (resource_type == DEFINITION) { + full_path = grib_context_full_defs_path(c, resource_path); } - if (!full) { - fprintf(stderr, "Failed to export %s: %s\n", resource, path); + if (!full_path) { + fprintf(stderr, "Failed to export %s: %s\n", resource_name, resource_path); return 1; } - printf("full = %s\n", full); - assert(path_is_regular_file(full)); - grib_context_free(c, full); + + fout = fopen(out_file, "wb"); + if (!fout) { + perror(out_file); + fprintf(stderr, "Failed to open output file %s\n", out_file); + return 1; + } + fin = codes_fopen(full_path, "r"); + if (!fin) { + fprintf(stderr, "Failed to open resource %s\n", full_path); + return 1; + } + /* write resource to fout */ + while (0 < (bytes = fread(buffer, 1, sizeof(buffer), fin))) + fwrite(buffer, 1, bytes, fout); + + if (fclose(fin) != 0) { + fprintf(stderr, "Call to fclose failed (input)\n"); + return 1; + } + if (fclose(fout) != 0) { + fprintf(stderr, "Call to fclose failed (output)\n"); + return 1; + } + + grib_context_free(c, full_path); return 0; } From 5ee4186fda6ae9235e99091ef15375827447847f Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sat, 12 Mar 2022 14:45:57 +0000 Subject: [PATCH 3/6] Tools: codes_export_resource --- tools/codes_export_resource.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tools/codes_export_resource.c b/tools/codes_export_resource.c index 40ffb0e21..4db164b4f 100644 --- a/tools/codes_export_resource.c +++ b/tools/codes_export_resource.c @@ -60,7 +60,7 @@ int main(int argc, char* argv[]) if (resource_type == SAMPLE) { char* t = strstr(resource_path, ".tmpl"); if (t) { - *t = 0; // get rid of the sample file extension + *t = '\0'; // get rid of sample file extension (if there) } full_path = grib_external_template_path(c, resource_path); } @@ -68,24 +68,27 @@ int main(int argc, char* argv[]) full_path = grib_context_full_defs_path(c, resource_path); } if (!full_path) { - fprintf(stderr, "Failed to export %s: %s\n", resource_name, resource_path); + fprintf(stderr, "Failed to access %s: '%s'\n", resource_name, resource_path); return 1; } fout = fopen(out_file, "wb"); if (!fout) { - perror(out_file); - fprintf(stderr, "Failed to open output file %s\n", out_file); + fprintf(stderr, "Failed to open output file '%s'\n", out_file); return 1; } fin = codes_fopen(full_path, "r"); if (!fin) { - fprintf(stderr, "Failed to open resource %s\n", full_path); + fprintf(stderr, "Failed to open resource '%s'\n", full_path); return 1; } - /* write resource to fout */ - while (0 < (bytes = fread(buffer, 1, sizeof(buffer), fin))) - fwrite(buffer, 1, bytes, fout); + /* write resource bytes to fout */ + while (0 < (bytes = fread(buffer, 1, sizeof(buffer), fin))) { + if (fwrite(buffer, 1, bytes, fout) != bytes) { + fprintf(stderr, "Failed to write out bytes\n"); + return 1; + } + } if (fclose(fin) != 0) { fprintf(stderr, "Call to fclose failed (input)\n"); From 549372d608c0a3715a28ded8adfd9b9dc56c89e2 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sat, 12 Mar 2022 17:24:38 +0000 Subject: [PATCH 4/6] Tools: codes_export_resource test --- tests/CMakeLists.txt | 6 ++++++ tests/codes_export_resource.sh | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100755 tests/codes_export_resource.sh diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 526ec0651..fc22747d6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -400,6 +400,12 @@ if( HAVE_BUILD_TOOLS ) COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bufr_split_by_rdbSubtype.sh TEST_DEPENDS eccodes_download_bufrs ) + ecbuild_add_test( TARGET eccodes_t_codes_export_resource + TYPE SCRIPT + CONDITION ECCODES_INSTALL_EXTRA_TOOLS + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/codes_export_resource.sh ) + + # Performance test. Must have -DENABLE_TIMER=ON ecbuild_add_test( TARGET eccodes_t_grib_ecc-386 TYPE SCRIPT diff --git a/tests/codes_export_resource.sh b/tests/codes_export_resource.sh new file mode 100755 index 000000000..4a5c2f322 --- /dev/null +++ b/tests/codes_export_resource.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# (C) Copyright 2005- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by +# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. +# + +. ./include.sh + +label="codes_export_resource_test" +temp=temp.$label + +# Sample files +# -------------- +f='GRIB2.tmpl' +${tools_dir}/codes_export_resource -s $f $temp +cmp $ECCODES_SAMPLES_PATH/$f $temp + +# Use the shortened form +${tools_dir}/codes_export_resource -s GRIB2 $temp +cmp $ECCODES_SAMPLES_PATH/GRIB2.tmpl $temp + +# Definition files +# ---------------- +f='boot.def' +${tools_dir}/codes_export_resource -d $f $temp +cmp $ECCODES_DEFINITION_PATH/$f $temp + +f='common/c-11.table' +${tools_dir}/codes_export_resource -d $f $temp +cmp $ECCODES_DEFINITION_PATH/$f $temp + + +# Clean up +rm -f $temp From 470173a32c8fef4d5cee6411c7aed866438ae6f4 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sat, 12 Mar 2022 18:51:00 +0000 Subject: [PATCH 5/6] Tools: codes_export_resource test --- tests/codes_export_resource.sh | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/codes_export_resource.sh b/tests/codes_export_resource.sh index 4a5c2f322..02774ce58 100755 --- a/tests/codes_export_resource.sh +++ b/tests/codes_export_resource.sh @@ -14,7 +14,7 @@ label="codes_export_resource_test" temp=temp.$label # Sample files -# -------------- +# ---------------- f='GRIB2.tmpl' ${tools_dir}/codes_export_resource -s $f $temp cmp $ECCODES_SAMPLES_PATH/$f $temp @@ -23,6 +23,11 @@ cmp $ECCODES_SAMPLES_PATH/$f $temp ${tools_dir}/codes_export_resource -s GRIB2 $temp cmp $ECCODES_SAMPLES_PATH/GRIB2.tmpl $temp +# IFS Samples +# ---------------- +ECCODES_SAMPLES_PATH=/MEMFS/ifs_samples/grib1_mlgrib2 ${tools_dir}/codes_export_resource -s gg_ml.tmpl $temp + + # Definition files # ---------------- f='boot.def' @@ -34,5 +39,21 @@ ${tools_dir}/codes_export_resource -d $f $temp cmp $ECCODES_DEFINITION_PATH/$f $temp +# Failing cases +# ---------------- +set +e +${tools_dir}/codes_export_resource -d nonexistent $temp +status=$? +set -e +[ $status -eq 1 ] + +set +e +${tools_dir}/codes_export_resource -s nonexistent $temp +status=$? +set -e +[ $status -eq 1 ] + + + # Clean up rm -f $temp From d33b0247b95b04e9419e3322b7a7a469ef6a2e50 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Sat, 12 Mar 2022 18:57:53 +0000 Subject: [PATCH 6/6] Tools: codes_export_resource tests --- tests/codes_export_resource.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/codes_export_resource.sh b/tests/codes_export_resource.sh index 02774ce58..b8444ebd1 100755 --- a/tests/codes_export_resource.sh +++ b/tests/codes_export_resource.sh @@ -25,8 +25,9 @@ cmp $ECCODES_SAMPLES_PATH/GRIB2.tmpl $temp # IFS Samples # ---------------- -ECCODES_SAMPLES_PATH=/MEMFS/ifs_samples/grib1_mlgrib2 ${tools_dir}/codes_export_resource -s gg_ml.tmpl $temp - +if [ $HAVE_MEMFS -eq 1 ]; then + ECCODES_SAMPLES_PATH=/MEMFS/ifs_samples/grib1_mlgrib2 ${tools_dir}/codes_export_resource -s gg_ml.tmpl $temp +fi # Definition files # ----------------