diff --git a/data/bitmap.diff b/data/bitmap.diff index a9d7527d0..e4ee14a1d 100644 --- a/data/bitmap.diff +++ b/data/bitmap.diff @@ -15,12 +15,9 @@ GRIB { # Unknown code table entry (grib1/0.ecmf.table) subCentre = 0; paramId = 130; - #-READ ONLY- cfNameECMF = air_temperature; #-READ ONLY- cfName = air_temperature; - #-READ ONLY- cfVarNameECMF = t; #-READ ONLY- cfVarName = t; #-READ ONLY- units = K; - #-READ ONLY- nameECMF = Temperature; #-READ ONLY- name = Temperature; decimalScaleFactor = 0; dataDate = 20080206; diff --git a/data/no_bitmap.diff b/data/no_bitmap.diff index a79a11110..d3a5d0471 100644 --- a/data/no_bitmap.diff +++ b/data/no_bitmap.diff @@ -15,12 +15,9 @@ GRIB { # Unknown code table entry (grib1/0.ecmf.table) subCentre = 0; paramId = 130; - #-READ ONLY- cfNameECMF = air_temperature; #-READ ONLY- cfName = air_temperature; - #-READ ONLY- cfVarNameECMF = t; #-READ ONLY- cfVarName = t; #-READ ONLY- units = K; - #-READ ONLY- nameECMF = Temperature; #-READ ONLY- name = Temperature; decimalScaleFactor = 0; dataDate = 20080206; diff --git a/definitions/grib1/section.1.def b/definitions/grib1/section.1.def index ff89dcd09..974977dfd 100644 --- a/definitions/grib1/section.1.def +++ b/definitions/grib1/section.1.def @@ -160,16 +160,16 @@ concept paramIdECMF (defaultParameter,"paramId.def",conceptsMasterDir,conceptsLo concept paramId (paramIdECMF,"paramId.def",conceptsDir2,conceptsDir1): long_type,dump; # transient pid = paramId : hidden; -concept cfNameECMF(defaultName,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only; +concept cfNameECMF(defaultName,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only; concept cfName(cfNameECMF,"cfName.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only; -concept cfVarNameECMF(defaultName,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only; +concept cfVarNameECMF(defaultName,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only; concept cfVarName(cfVarNameECMF,"cfVarName.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only; concept unitsECMF(defaultName,"units.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only; concept units(unitsECMF,"units.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only; -concept nameECMF(defaultName,"name.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only; +concept nameECMF(defaultName,"name.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only; concept name(nameECMF,"name.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only; signed[2] decimalScaleFactor :dump; diff --git a/definitions/grib2/parameters.def b/definitions/grib2/parameters.def index 2c6b703c8..6ffe2fe0d 100644 --- a/definitions/grib2/parameters.def +++ b/definitions/grib2/parameters.def @@ -7,24 +7,24 @@ concept paramIdLegacyECMF(defaultParameter,"paramId.legacy.def",conceptsMasterDi concept paramIdECMF (paramIdLegacyECMF,"paramId.def",conceptsMasterDir,conceptsLocalDirECMF): long_type,no_copy; concept paramId (paramIdECMF,"paramId.def",conceptsDir2,conceptsDir1): long_type; -concept shortNameLegacyECMF(defaultShortName,"shortName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden; -concept shortNameECMF (shortNameLegacyECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump; +concept shortNameLegacyECMF(defaultShortName,"shortName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden; +concept shortNameECMF (shortNameLegacyECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy; concept ls.shortName (shortNameECMF,"shortName.def",conceptsDir2,conceptsDir1): no_copy,dump; concept unitsLegacyECMF(defaultName,"units.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden,read_only; concept unitsECMF (unitsLegacyECMF,"units.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy, read_only; concept units (unitsECMF,"units.def",conceptsDir2,conceptsDir1): no_copy, read_only; -concept nameLegacyECMF(defaultName,"name.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden; -concept nameECMF(nameLegacyECMF,"name.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump; +concept nameLegacyECMF(defaultName,"name.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden; +concept nameECMF(nameLegacyECMF,"name.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy; concept name(nameECMF,"name.def",conceptsDir2,conceptsDir1): no_copy,dump; -concept cfNameLegacyECMF(defaultShortName,"cfName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden; -concept cfNameECMF(cfNameLegacyECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,dump; +concept cfNameLegacyECMF(defaultShortName,"cfName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden; +concept cfNameECMF(cfNameLegacyECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy; concept cfName(cfNameECMF,"cfName.def",conceptsDir2,conceptsDir1) : no_copy,dump; -concept cfVarNameLegacyECMF(defaultShortName,"cfVarName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden; -concept cfVarNameECMF (cfVarNameLegacyECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump; +concept cfVarNameLegacyECMF(defaultShortName,"cfVarName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden; +concept cfVarNameECMF (cfVarNameLegacyECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy; concept cfVarName (cfVarNameECMF,"cfVarName.def",conceptsDir2,conceptsDir1): no_copy,dump; # modelName: Contribution from Daniel Lee @ DWD diff --git a/definitions/grib2/template.7.2.def b/definitions/grib2/template.7.2.def index 8c13829aa..c131daf02 100644 --- a/definitions/grib2/template.7.2.def +++ b/definitions/grib2/template.7.2.def @@ -45,3 +45,9 @@ meta values data_apply_bitmap(codedValues, alias data.packedValues = codedValues; template statistics "common/statistics_grid.def"; +if(missingValueManagementUsed) { + #remove numberOfMissing; + #remove numberOfMissingValues; + meta numberOfMissing count_missing(bitmap,unusedBitsInBitmap,numberOfDataPoints,missingValueManagementUsed) : dump; + alias numberOfMissingValues=numberOfMissing; +} diff --git a/definitions/grib2/template.7.3.def b/definitions/grib2/template.7.3.def index dd2a40be0..afada6614 100644 --- a/definitions/grib2/template.7.3.def +++ b/definitions/grib2/template.7.3.def @@ -42,3 +42,9 @@ meta values data_apply_bitmap(codedValues, alias data.packedValues=codedValues; template statistics "common/statistics_grid.def"; +if(missingValueManagementUsed) { + #remove numberOfMissingValues; + #remove numberOfMissing; + meta numberOfMissing count_missing(bitmap,unusedBitsInBitmap,numberOfDataPoints,missingValueManagementUsed) : dump; + alias numberOfMissingValues=numberOfMissing; +} diff --git a/src/grib_accessor_class.c b/src/grib_accessor_class.c index b72feb037..52419470a 100644 --- a/src/grib_accessor_class.c +++ b/src/grib_accessor_class.c @@ -300,7 +300,9 @@ int grib_section_adjust_sizes(grib_section* s, int update, int depth) if (offset != a->offset) { grib_context_log(a->context, GRIB_LOG_ERROR, - "Offset mismatch %s A->offset %ld offset %ld\n", a->name, (long)a->offset, (long)offset); + "Offset mismatch accessor=%s: accessor's offset=%ld, but actual offset=%ld", + a->name, (long)a->offset, (long)offset); + grib_context_log(a->context, GRIB_LOG_ERROR, "Hint: Check section lengths are in sync with their contents"); a->offset = offset; return GRIB_DECODING_ERROR; } diff --git a/src/grib_accessor_class_count_missing.c b/src/grib_accessor_class_count_missing.c index 3c9ec36fd..99b7d0943 100644 --- a/src/grib_accessor_class_count_missing.c +++ b/src/grib_accessor_class_count_missing.c @@ -21,6 +21,7 @@ MEMBERS = const char* bitmap MEMBERS = const char* unusedBitsInBitmap MEMBERS = const char* numberOfDataPoints + MEMBERS = const char* missingValueManagementUsed END_CLASS_DEF */ @@ -49,6 +50,7 @@ typedef struct grib_accessor_count_missing const char* bitmap; const char* unusedBitsInBitmap; const char* numberOfDataPoints; + const char* missingValueManagementUsed; } grib_accessor_count_missing; extern grib_accessor_class* grib_accessor_class_long; @@ -181,15 +183,41 @@ static void init(grib_accessor* a, const long len, grib_arguments* arg) { int n = 0; grib_accessor_count_missing* self = (grib_accessor_count_missing*)a; + grib_handle* h = grib_handle_of_accessor(a); a->length = 0; a->flags |= GRIB_ACCESSOR_FLAG_READ_ONLY; - self->bitmap = grib_arguments_get_name(grib_handle_of_accessor(a), arg, n++); - self->unusedBitsInBitmap = grib_arguments_get_name(grib_handle_of_accessor(a), arg, n++); - self->numberOfDataPoints = grib_arguments_get_name(grib_handle_of_accessor(a), arg, n++); + self->bitmap = grib_arguments_get_name(h, arg, n++); + self->unusedBitsInBitmap = grib_arguments_get_name(h, arg, n++); + self->numberOfDataPoints = grib_arguments_get_name(h, arg, n++); + self->missingValueManagementUsed = grib_arguments_get_name(h, arg, n++); /* Can be NULL */ } static const int used[] = { 0, 1, 3, 7, 15, 31, 63, 127, 255 }; +static int get_count_of_missing_values(grib_handle* h, long* p_count_of_missing) +{ + int err = 0; + long count_of_missing = 0; + size_t vsize = 0, ii = 0; + double* values = NULL; + double mv = 0; + if ((err = grib_get_double(h, "missingValue", &mv)) != GRIB_SUCCESS) + return err; + if ((err = grib_get_size(h, "values", &vsize)) != GRIB_SUCCESS) + return err; + values = (double*)grib_context_malloc(h->context, vsize * sizeof(double)); + if (!values) + return GRIB_OUT_OF_MEMORY; + if ((err = grib_get_double_array(h, "values", values, &vsize)) != GRIB_SUCCESS) + return err; + for (ii = 0; ii < vsize; ii++) { + if (values[ii] == mv) ++count_of_missing; + } + grib_context_free(h->context, values); + *p_count_of_missing = count_of_missing; + + return GRIB_SUCCESS; +} static int unpack_long(grib_accessor* a, long* val, size_t* len) { grib_accessor_count_missing* self = (grib_accessor_count_missing*)a; @@ -202,10 +230,23 @@ static int unpack_long(grib_accessor* a, long* val, size_t* len) grib_handle* h = grib_handle_of_accessor(a); grib_accessor* bitmap = grib_find_accessor(h, self->bitmap); - *val = 0; + *val = 0; /* By default assume none are missing */ *len = 1; - if (!bitmap) + if (!bitmap) { + long mvmu = 0; + if (self->missingValueManagementUsed && + grib_get_long(h, self->missingValueManagementUsed, &mvmu) == GRIB_SUCCESS && mvmu != 0) + { + /* ECC-523: No bitmap. Missing values are encoded in the Data Section. + * So we must decode all the data values and count how many are missing + */ + long count_of_missing = 0; + if (get_count_of_missing_values(h, &count_of_missing) == GRIB_SUCCESS) { + *val = count_of_missing; + } + } return GRIB_SUCCESS; + } size = grib_byte_count(bitmap); offset = grib_byte_offset(bitmap); diff --git a/src/grib_accessor_class_md5.c b/src/grib_accessor_class_md5.c index ed5474be5..14f2f9bc5 100644 --- a/src/grib_accessor_class_md5.c +++ b/src/grib_accessor_class_md5.c @@ -271,6 +271,6 @@ static void destroy(grib_context* c, grib_accessor* a) static int value_count(grib_accessor* a, long* count) { - *count = 16; + *count = 1; /* ECC-1475 */ return 0; } diff --git a/src/grib_context.c b/src/grib_context.c index ed2f62850..c02c25000 100644 --- a/src/grib_context.c +++ b/src/grib_context.c @@ -242,20 +242,22 @@ static void default_print(const grib_context* c, void* descriptor, const char* m void grib_context_set_print_proc(grib_context* c, grib_print_proc p) { - c = c ? c : grib_context_get_default(); - c->print = p; + c = c ? c : grib_context_get_default(); + /* Set logging back to the default if p is NULL */ + c->print = (p ? p : &default_print); } void grib_context_set_debug(grib_context* c, int mode) { - c = c ? c : grib_context_get_default(); + c = c ? c : grib_context_get_default(); c->debug = mode; } void grib_context_set_logging_proc(grib_context* c, grib_log_proc p) { - c = c ? c : grib_context_get_default(); - c->output_log = p; + c = c ? c : grib_context_get_default(); + /* Set logging back to the default if p is NULL */ + c->output_log = (p ? p : &default_log); } long grib_get_api_version() diff --git a/src/grib_jasper_encoding.c b/src/grib_jasper_encoding.c index 1f9e3bd88..609991a0a 100644 --- a/src/grib_jasper_encoding.c +++ b/src/grib_jasper_encoding.c @@ -22,7 +22,7 @@ static int ecc_jasper_initialise() { -#if JASPER_VERSION_MAJOR == 3 +#if JASPER_VERSION_MAJOR >= 3 int jaserr = 0; jas_conf_clear(); jas_conf_set_max_mem_usage(jas_get_total_mem_size()); @@ -36,7 +36,7 @@ static int ecc_jasper_initialise() static jas_image_t* ecc_jasper_decode(jas_stream_t *in) { -#if JASPER_VERSION_MAJOR == 3 +#if JASPER_VERSION_MAJOR >= 3 /* Second argument (=fmt) < 0 means "If possible, try to determine the format of the input data" */ return jas_image_decode(in, -1, 0); #else @@ -46,7 +46,7 @@ static jas_image_t* ecc_jasper_decode(jas_stream_t *in) static int ecc_jasper_encode(jas_image_t *image, jas_stream_t *jpcstream, char *optstr) { -#if JASPER_VERSION_MAJOR == 3 +#if JASPER_VERSION_MAJOR >= 3 const int fmt = jas_image_strtofmt("jpc"); return jas_image_encode(image, jpcstream, fmt, optstr); #else @@ -56,7 +56,7 @@ static int ecc_jasper_encode(jas_image_t *image, jas_stream_t *jpcstream, char * static void ecc_jasper_cleanup() { -#if JASPER_VERSION_MAJOR == 3 +#if JASPER_VERSION_MAJOR >= 3 jas_cleanup_thread(); jas_cleanup_library(); #endif diff --git a/tests/grib_complex.sh b/tests/grib_complex.sh index 05f5663e8..9fba2c28b 100755 --- a/tests/grib_complex.sh +++ b/tests/grib_complex.sh @@ -29,3 +29,11 @@ for file in $files; do rm -f $temp $temp1 $temp2 done + +# ECC-523 +infile=${data_dir}/gfs.complex.mvmu.grib2 +grib_check_key_equals $infile 'missingValueManagementUsed,bitmapPresent' '1 0' +grib_check_key_equals $infile 'numberOfMissing,numberOfMissingValues' '556901 556901' +infile=${data_dir}/gfs.c255.grib2 +grib_check_key_equals $infile 'missingValueManagementUsed,bitmapPresent' '0 1' +grib_check_key_equals $infile 'numberOfMissing,numberOfMissingValues' '7665 7665' diff --git a/tests/unit_tests.c b/tests/unit_tests.c index 43870f7ef..6075baa75 100644 --- a/tests/unit_tests.c +++ b/tests/unit_tests.c @@ -14,6 +14,7 @@ #define NUMBER(x) (sizeof(x) / sizeof(x[0])) int assertion_caught = 0; +int logging_caught = 0; typedef enum { @@ -1468,6 +1469,28 @@ static void test_assertion_catching() free(list); } + +static void my_logging_proc(const grib_context* c, int level, const char* mesg) +{ + logging_caught = 1; +} +static void test_logging_proc() +{ + grib_context* context = grib_context_get_default(); + Assert(logging_caught == 0); + + /* Override default behaviour */ + grib_context_set_logging_proc(context, my_logging_proc); + grib_context_log(context, GRIB_LOG_ERROR, "This error will be handled by me"); + Assert(logging_caught == 1); + + /* Restore the logging proc */ + logging_caught = 0; + grib_context_set_logging_proc(context, NULL); + grib_context_log(context, GRIB_LOG_ERROR, "This will come out as normal"); + Assert(logging_caught == 0); +} + static void test_concept_condition_strings() { int err = 0; @@ -1631,10 +1654,12 @@ static void test_parse_keyval_string() free( (void*)values3[0].name ); } + int main(int argc, char** argv) { printf("Doing unit tests. ecCodes version = %ld\n", grib_get_api_version()); + test_logging_proc(); test_grib_binary_search(); test_parse_keyval_string(); diff --git a/tools/codes_export_resource.c b/tools/codes_export_resource.c index 4736869e4..1511ae762 100644 --- a/tools/codes_export_resource.c +++ b/tools/codes_export_resource.c @@ -27,7 +27,7 @@ static void usage(const char* prog) int main(int argc, char* argv[]) { char* resource_path = NULL; - char* resource_name = NULL; + const char* resource_name = NULL; ResourceType resource_type = UNKNOWN; char* full_path = NULL; char* out_file = NULL; diff --git a/tools/gts_dump.c b/tools/gts_dump.c index e0f7ad411..4eac49809 100644 --- a/tools/gts_dump.c +++ b/tools/gts_dump.c @@ -58,7 +58,7 @@ int grib_tool_init(grib_runtime_options* options) { int opt = grib_options_on("C") + grib_options_on("O") + grib_options_on("D"); - options->dump_mode = "default"; + options->dump_mode = (char*)"default"; if (opt > 1) { @@ -67,12 +67,12 @@ int grib_tool_init(grib_runtime_options* options) } if (grib_options_on("O")) { - options->dump_mode = "wmo"; + options->dump_mode = (char*)"wmo"; options->dump_flags = GRIB_DUMP_FLAG_CODED | GRIB_DUMP_FLAG_OCTET | GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY; } if (grib_options_on("D")) { - options->dump_mode = "debug"; + options->dump_mode = (char*)"debug"; options->dump_flags = GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY; } diff --git a/tools/metar_dump.c b/tools/metar_dump.c index 16007f837..869159297 100644 --- a/tools/metar_dump.c +++ b/tools/metar_dump.c @@ -56,7 +56,7 @@ int grib_tool_init(grib_runtime_options* options) { int opt = grib_options_on("C") + grib_options_on("O") + grib_options_on("D"); - options->dump_mode = "default"; + options->dump_mode = (char*)"default"; if (opt > 1) { printf("%s: simultaneous O/D options not allowed\n", tool_name); @@ -64,22 +64,22 @@ int grib_tool_init(grib_runtime_options* options) } if (grib_options_on("O")) { - options->dump_mode = "wmo"; + options->dump_mode = (char*)"wmo"; options->dump_flags = GRIB_DUMP_FLAG_CODED | GRIB_DUMP_FLAG_OCTET | GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY; } if (grib_options_on("D")) { - options->dump_mode = "debug"; + options->dump_mode = (char*)"debug"; options->dump_flags = GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY; } if (grib_options_on("J")) { - options->dump_mode = "json"; + options->dump_mode = (char*)"json"; options->dump_flags = GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY; } if (grib_options_on("X")) { - options->dump_mode = "xml"; + options->dump_mode = (char*)"xml"; options->dump_flags = GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY; }