Merge branch 'develop' into feature/ECC-1466-GRIB2Defs

This commit is contained in:
Shahram Najm 2022-11-29 12:07:28 +00:00
commit a2fed197a2
16 changed files with 126 additions and 42 deletions

View File

@ -15,12 +15,9 @@ GRIB {
# Unknown code table entry (grib1/0.ecmf.table)
subCentre = 0;
paramId = 130;
#-READ ONLY- cfNameECMF = air_temperature;
#-READ ONLY- cfName = air_temperature;
#-READ ONLY- cfVarNameECMF = t;
#-READ ONLY- cfVarName = t;
#-READ ONLY- units = K;
#-READ ONLY- nameECMF = Temperature;
#-READ ONLY- name = Temperature;
decimalScaleFactor = 0;
dataDate = 20080206;

View File

@ -15,12 +15,9 @@ GRIB {
# Unknown code table entry (grib1/0.ecmf.table)
subCentre = 0;
paramId = 130;
#-READ ONLY- cfNameECMF = air_temperature;
#-READ ONLY- cfName = air_temperature;
#-READ ONLY- cfVarNameECMF = t;
#-READ ONLY- cfVarName = t;
#-READ ONLY- units = K;
#-READ ONLY- nameECMF = Temperature;
#-READ ONLY- name = Temperature;
decimalScaleFactor = 0;
dataDate = 20080206;

View File

@ -160,16 +160,16 @@ concept paramIdECMF (defaultParameter,"paramId.def",conceptsMasterDir,conceptsLo
concept paramId (paramIdECMF,"paramId.def",conceptsDir2,conceptsDir1): long_type,dump;
# transient pid = paramId : hidden;
concept cfNameECMF(defaultName,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only;
concept cfNameECMF(defaultName,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only;
concept cfName(cfNameECMF,"cfName.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only;
concept cfVarNameECMF(defaultName,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only;
concept cfVarNameECMF(defaultName,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only;
concept cfVarName(cfVarNameECMF,"cfVarName.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only;
concept unitsECMF(defaultName,"units.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only;
concept units(unitsECMF,"units.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only;
concept nameECMF(defaultName,"name.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only;
concept nameECMF(defaultName,"name.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only;
concept name(nameECMF,"name.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only;
signed[2] decimalScaleFactor :dump;

View File

@ -7,24 +7,24 @@ concept paramIdLegacyECMF(defaultParameter,"paramId.legacy.def",conceptsMasterDi
concept paramIdECMF (paramIdLegacyECMF,"paramId.def",conceptsMasterDir,conceptsLocalDirECMF): long_type,no_copy;
concept paramId (paramIdECMF,"paramId.def",conceptsDir2,conceptsDir1): long_type;
concept shortNameLegacyECMF(defaultShortName,"shortName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden;
concept shortNameECMF (shortNameLegacyECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump;
concept shortNameLegacyECMF(defaultShortName,"shortName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden;
concept shortNameECMF (shortNameLegacyECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy;
concept ls.shortName (shortNameECMF,"shortName.def",conceptsDir2,conceptsDir1): no_copy,dump;
concept unitsLegacyECMF(defaultName,"units.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden,read_only;
concept unitsECMF (unitsLegacyECMF,"units.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy, read_only;
concept units (unitsECMF,"units.def",conceptsDir2,conceptsDir1): no_copy, read_only;
concept nameLegacyECMF(defaultName,"name.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden;
concept nameECMF(nameLegacyECMF,"name.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump;
concept nameLegacyECMF(defaultName,"name.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden;
concept nameECMF(nameLegacyECMF,"name.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy;
concept name(nameECMF,"name.def",conceptsDir2,conceptsDir1): no_copy,dump;
concept cfNameLegacyECMF(defaultShortName,"cfName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden;
concept cfNameECMF(cfNameLegacyECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,dump;
concept cfNameLegacyECMF(defaultShortName,"cfName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden;
concept cfNameECMF(cfNameLegacyECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy;
concept cfName(cfNameECMF,"cfName.def",conceptsDir2,conceptsDir1) : no_copy,dump;
concept cfVarNameLegacyECMF(defaultShortName,"cfVarName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden;
concept cfVarNameECMF (cfVarNameLegacyECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump;
concept cfVarNameLegacyECMF(defaultShortName,"cfVarName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden;
concept cfVarNameECMF (cfVarNameLegacyECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy;
concept cfVarName (cfVarNameECMF,"cfVarName.def",conceptsDir2,conceptsDir1): no_copy,dump;
# modelName: Contribution from Daniel Lee @ DWD

View File

@ -45,3 +45,9 @@ meta values data_apply_bitmap(codedValues,
alias data.packedValues = codedValues;
template statistics "common/statistics_grid.def";
if(missingValueManagementUsed) {
#remove numberOfMissing;
#remove numberOfMissingValues;
meta numberOfMissing count_missing(bitmap,unusedBitsInBitmap,numberOfDataPoints,missingValueManagementUsed) : dump;
alias numberOfMissingValues=numberOfMissing;
}

View File

@ -42,3 +42,9 @@ meta values data_apply_bitmap(codedValues,
alias data.packedValues=codedValues;
template statistics "common/statistics_grid.def";
if(missingValueManagementUsed) {
#remove numberOfMissingValues;
#remove numberOfMissing;
meta numberOfMissing count_missing(bitmap,unusedBitsInBitmap,numberOfDataPoints,missingValueManagementUsed) : dump;
alias numberOfMissingValues=numberOfMissing;
}

View File

@ -300,7 +300,9 @@ int grib_section_adjust_sizes(grib_section* s, int update, int depth)
if (offset != a->offset) {
grib_context_log(a->context, GRIB_LOG_ERROR,
"Offset mismatch %s A->offset %ld offset %ld\n", a->name, (long)a->offset, (long)offset);
"Offset mismatch accessor=%s: accessor's offset=%ld, but actual offset=%ld",
a->name, (long)a->offset, (long)offset);
grib_context_log(a->context, GRIB_LOG_ERROR, "Hint: Check section lengths are in sync with their contents");
a->offset = offset;
return GRIB_DECODING_ERROR;
}

View File

@ -21,6 +21,7 @@
MEMBERS = const char* bitmap
MEMBERS = const char* unusedBitsInBitmap
MEMBERS = const char* numberOfDataPoints
MEMBERS = const char* missingValueManagementUsed
END_CLASS_DEF
*/
@ -49,6 +50,7 @@ typedef struct grib_accessor_count_missing
const char* bitmap;
const char* unusedBitsInBitmap;
const char* numberOfDataPoints;
const char* missingValueManagementUsed;
} grib_accessor_count_missing;
extern grib_accessor_class* grib_accessor_class_long;
@ -181,15 +183,41 @@ static void init(grib_accessor* a, const long len, grib_arguments* arg)
{
int n = 0;
grib_accessor_count_missing* self = (grib_accessor_count_missing*)a;
grib_handle* h = grib_handle_of_accessor(a);
a->length = 0;
a->flags |= GRIB_ACCESSOR_FLAG_READ_ONLY;
self->bitmap = grib_arguments_get_name(grib_handle_of_accessor(a), arg, n++);
self->unusedBitsInBitmap = grib_arguments_get_name(grib_handle_of_accessor(a), arg, n++);
self->numberOfDataPoints = grib_arguments_get_name(grib_handle_of_accessor(a), arg, n++);
self->bitmap = grib_arguments_get_name(h, arg, n++);
self->unusedBitsInBitmap = grib_arguments_get_name(h, arg, n++);
self->numberOfDataPoints = grib_arguments_get_name(h, arg, n++);
self->missingValueManagementUsed = grib_arguments_get_name(h, arg, n++); /* Can be NULL */
}
static const int used[] = { 0, 1, 3, 7, 15, 31, 63, 127, 255 };
static int get_count_of_missing_values(grib_handle* h, long* p_count_of_missing)
{
int err = 0;
long count_of_missing = 0;
size_t vsize = 0, ii = 0;
double* values = NULL;
double mv = 0;
if ((err = grib_get_double(h, "missingValue", &mv)) != GRIB_SUCCESS)
return err;
if ((err = grib_get_size(h, "values", &vsize)) != GRIB_SUCCESS)
return err;
values = (double*)grib_context_malloc(h->context, vsize * sizeof(double));
if (!values)
return GRIB_OUT_OF_MEMORY;
if ((err = grib_get_double_array(h, "values", values, &vsize)) != GRIB_SUCCESS)
return err;
for (ii = 0; ii < vsize; ii++) {
if (values[ii] == mv) ++count_of_missing;
}
grib_context_free(h->context, values);
*p_count_of_missing = count_of_missing;
return GRIB_SUCCESS;
}
static int unpack_long(grib_accessor* a, long* val, size_t* len)
{
grib_accessor_count_missing* self = (grib_accessor_count_missing*)a;
@ -202,10 +230,23 @@ static int unpack_long(grib_accessor* a, long* val, size_t* len)
grib_handle* h = grib_handle_of_accessor(a);
grib_accessor* bitmap = grib_find_accessor(h, self->bitmap);
*val = 0;
*val = 0; /* By default assume none are missing */
*len = 1;
if (!bitmap)
if (!bitmap) {
long mvmu = 0;
if (self->missingValueManagementUsed &&
grib_get_long(h, self->missingValueManagementUsed, &mvmu) == GRIB_SUCCESS && mvmu != 0)
{
/* ECC-523: No bitmap. Missing values are encoded in the Data Section.
* So we must decode all the data values and count how many are missing
*/
long count_of_missing = 0;
if (get_count_of_missing_values(h, &count_of_missing) == GRIB_SUCCESS) {
*val = count_of_missing;
}
}
return GRIB_SUCCESS;
}
size = grib_byte_count(bitmap);
offset = grib_byte_offset(bitmap);

View File

@ -271,6 +271,6 @@ static void destroy(grib_context* c, grib_accessor* a)
static int value_count(grib_accessor* a, long* count)
{
*count = 16;
*count = 1; /* ECC-1475 */
return 0;
}

View File

@ -242,20 +242,22 @@ static void default_print(const grib_context* c, void* descriptor, const char* m
void grib_context_set_print_proc(grib_context* c, grib_print_proc p)
{
c = c ? c : grib_context_get_default();
c->print = p;
c = c ? c : grib_context_get_default();
/* Set logging back to the default if p is NULL */
c->print = (p ? p : &default_print);
}
void grib_context_set_debug(grib_context* c, int mode)
{
c = c ? c : grib_context_get_default();
c = c ? c : grib_context_get_default();
c->debug = mode;
}
void grib_context_set_logging_proc(grib_context* c, grib_log_proc p)
{
c = c ? c : grib_context_get_default();
c->output_log = p;
c = c ? c : grib_context_get_default();
/* Set logging back to the default if p is NULL */
c->output_log = (p ? p : &default_log);
}
long grib_get_api_version()

View File

@ -22,7 +22,7 @@
static int ecc_jasper_initialise()
{
#if JASPER_VERSION_MAJOR == 3
#if JASPER_VERSION_MAJOR >= 3
int jaserr = 0;
jas_conf_clear();
jas_conf_set_max_mem_usage(jas_get_total_mem_size());
@ -36,7 +36,7 @@ static int ecc_jasper_initialise()
static jas_image_t* ecc_jasper_decode(jas_stream_t *in)
{
#if JASPER_VERSION_MAJOR == 3
#if JASPER_VERSION_MAJOR >= 3
/* Second argument (=fmt) < 0 means "If possible, try to determine the format of the input data" */
return jas_image_decode(in, -1, 0);
#else
@ -46,7 +46,7 @@ static jas_image_t* ecc_jasper_decode(jas_stream_t *in)
static int ecc_jasper_encode(jas_image_t *image, jas_stream_t *jpcstream, char *optstr)
{
#if JASPER_VERSION_MAJOR == 3
#if JASPER_VERSION_MAJOR >= 3
const int fmt = jas_image_strtofmt("jpc");
return jas_image_encode(image, jpcstream, fmt, optstr);
#else
@ -56,7 +56,7 @@ static int ecc_jasper_encode(jas_image_t *image, jas_stream_t *jpcstream, char *
static void ecc_jasper_cleanup()
{
#if JASPER_VERSION_MAJOR == 3
#if JASPER_VERSION_MAJOR >= 3
jas_cleanup_thread();
jas_cleanup_library();
#endif

View File

@ -29,3 +29,11 @@ for file in $files; do
rm -f $temp $temp1 $temp2
done
# ECC-523
infile=${data_dir}/gfs.complex.mvmu.grib2
grib_check_key_equals $infile 'missingValueManagementUsed,bitmapPresent' '1 0'
grib_check_key_equals $infile 'numberOfMissing,numberOfMissingValues' '556901 556901'
infile=${data_dir}/gfs.c255.grib2
grib_check_key_equals $infile 'missingValueManagementUsed,bitmapPresent' '0 1'
grib_check_key_equals $infile 'numberOfMissing,numberOfMissingValues' '7665 7665'

View File

@ -14,6 +14,7 @@
#define NUMBER(x) (sizeof(x) / sizeof(x[0]))
int assertion_caught = 0;
int logging_caught = 0;
typedef enum
{
@ -1468,6 +1469,28 @@ static void test_assertion_catching()
free(list);
}
static void my_logging_proc(const grib_context* c, int level, const char* mesg)
{
logging_caught = 1;
}
static void test_logging_proc()
{
grib_context* context = grib_context_get_default();
Assert(logging_caught == 0);
/* Override default behaviour */
grib_context_set_logging_proc(context, my_logging_proc);
grib_context_log(context, GRIB_LOG_ERROR, "This error will be handled by me");
Assert(logging_caught == 1);
/* Restore the logging proc */
logging_caught = 0;
grib_context_set_logging_proc(context, NULL);
grib_context_log(context, GRIB_LOG_ERROR, "This will come out as normal");
Assert(logging_caught == 0);
}
static void test_concept_condition_strings()
{
int err = 0;
@ -1631,10 +1654,12 @@ static void test_parse_keyval_string()
free( (void*)values3[0].name );
}
int main(int argc, char** argv)
{
printf("Doing unit tests. ecCodes version = %ld\n", grib_get_api_version());
test_logging_proc();
test_grib_binary_search();
test_parse_keyval_string();

View File

@ -27,7 +27,7 @@ static void usage(const char* prog)
int main(int argc, char* argv[])
{
char* resource_path = NULL;
char* resource_name = NULL;
const char* resource_name = NULL;
ResourceType resource_type = UNKNOWN;
char* full_path = NULL;
char* out_file = NULL;

View File

@ -58,7 +58,7 @@ int grib_tool_init(grib_runtime_options* options)
{
int opt = grib_options_on("C") + grib_options_on("O") + grib_options_on("D");
options->dump_mode = "default";
options->dump_mode = (char*)"default";
if (opt > 1) {
@ -67,12 +67,12 @@ int grib_tool_init(grib_runtime_options* options)
}
if (grib_options_on("O")) {
options->dump_mode = "wmo";
options->dump_mode = (char*)"wmo";
options->dump_flags = GRIB_DUMP_FLAG_CODED | GRIB_DUMP_FLAG_OCTET | GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY;
}
if (grib_options_on("D")) {
options->dump_mode = "debug";
options->dump_mode = (char*)"debug";
options->dump_flags = GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY;
}

View File

@ -56,7 +56,7 @@ int grib_tool_init(grib_runtime_options* options)
{
int opt = grib_options_on("C") + grib_options_on("O") + grib_options_on("D");
options->dump_mode = "default";
options->dump_mode = (char*)"default";
if (opt > 1) {
printf("%s: simultaneous O/D options not allowed\n", tool_name);
@ -64,22 +64,22 @@ int grib_tool_init(grib_runtime_options* options)
}
if (grib_options_on("O")) {
options->dump_mode = "wmo";
options->dump_mode = (char*)"wmo";
options->dump_flags = GRIB_DUMP_FLAG_CODED | GRIB_DUMP_FLAG_OCTET | GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY;
}
if (grib_options_on("D")) {
options->dump_mode = "debug";
options->dump_mode = (char*)"debug";
options->dump_flags = GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY;
}
if (grib_options_on("J")) {
options->dump_mode = "json";
options->dump_mode = (char*)"json";
options->dump_flags = GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY;
}
if (grib_options_on("X")) {
options->dump_mode = "xml";
options->dump_mode = (char*)"xml";
options->dump_flags = GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY;
}