mirror of https://github.com/ecmwf/eccodes.git
Merge branch 'develop' into feature/ECC-1466-GRIB2Defs
This commit is contained in:
commit
a2fed197a2
|
@ -15,12 +15,9 @@ GRIB {
|
|||
# Unknown code table entry (grib1/0.ecmf.table)
|
||||
subCentre = 0;
|
||||
paramId = 130;
|
||||
#-READ ONLY- cfNameECMF = air_temperature;
|
||||
#-READ ONLY- cfName = air_temperature;
|
||||
#-READ ONLY- cfVarNameECMF = t;
|
||||
#-READ ONLY- cfVarName = t;
|
||||
#-READ ONLY- units = K;
|
||||
#-READ ONLY- nameECMF = Temperature;
|
||||
#-READ ONLY- name = Temperature;
|
||||
decimalScaleFactor = 0;
|
||||
dataDate = 20080206;
|
||||
|
|
|
@ -15,12 +15,9 @@ GRIB {
|
|||
# Unknown code table entry (grib1/0.ecmf.table)
|
||||
subCentre = 0;
|
||||
paramId = 130;
|
||||
#-READ ONLY- cfNameECMF = air_temperature;
|
||||
#-READ ONLY- cfName = air_temperature;
|
||||
#-READ ONLY- cfVarNameECMF = t;
|
||||
#-READ ONLY- cfVarName = t;
|
||||
#-READ ONLY- units = K;
|
||||
#-READ ONLY- nameECMF = Temperature;
|
||||
#-READ ONLY- name = Temperature;
|
||||
decimalScaleFactor = 0;
|
||||
dataDate = 20080206;
|
||||
|
|
|
@ -160,16 +160,16 @@ concept paramIdECMF (defaultParameter,"paramId.def",conceptsMasterDir,conceptsLo
|
|||
concept paramId (paramIdECMF,"paramId.def",conceptsDir2,conceptsDir1): long_type,dump;
|
||||
# transient pid = paramId : hidden;
|
||||
|
||||
concept cfNameECMF(defaultName,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only;
|
||||
concept cfNameECMF(defaultName,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only;
|
||||
concept cfName(cfNameECMF,"cfName.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only;
|
||||
|
||||
concept cfVarNameECMF(defaultName,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only;
|
||||
concept cfVarNameECMF(defaultName,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only;
|
||||
concept cfVarName(cfVarNameECMF,"cfVarName.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only;
|
||||
|
||||
concept unitsECMF(defaultName,"units.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only;
|
||||
concept units(unitsECMF,"units.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only;
|
||||
|
||||
concept nameECMF(defaultName,"name.def",conceptsMasterDir,conceptsLocalDirECMF) : dump,no_copy,read_only;
|
||||
concept nameECMF(defaultName,"name.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,read_only;
|
||||
concept name(nameECMF,"name.def",conceptsDir2,conceptsDir1) : dump,no_copy,read_only;
|
||||
|
||||
signed[2] decimalScaleFactor :dump;
|
||||
|
|
|
@ -7,24 +7,24 @@ concept paramIdLegacyECMF(defaultParameter,"paramId.legacy.def",conceptsMasterDi
|
|||
concept paramIdECMF (paramIdLegacyECMF,"paramId.def",conceptsMasterDir,conceptsLocalDirECMF): long_type,no_copy;
|
||||
concept paramId (paramIdECMF,"paramId.def",conceptsDir2,conceptsDir1): long_type;
|
||||
|
||||
concept shortNameLegacyECMF(defaultShortName,"shortName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden;
|
||||
concept shortNameECMF (shortNameLegacyECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump;
|
||||
concept shortNameLegacyECMF(defaultShortName,"shortName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden;
|
||||
concept shortNameECMF (shortNameLegacyECMF,"shortName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy;
|
||||
concept ls.shortName (shortNameECMF,"shortName.def",conceptsDir2,conceptsDir1): no_copy,dump;
|
||||
|
||||
concept unitsLegacyECMF(defaultName,"units.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden,read_only;
|
||||
concept unitsECMF (unitsLegacyECMF,"units.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy, read_only;
|
||||
concept units (unitsECMF,"units.def",conceptsDir2,conceptsDir1): no_copy, read_only;
|
||||
|
||||
concept nameLegacyECMF(defaultName,"name.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden;
|
||||
concept nameECMF(nameLegacyECMF,"name.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump;
|
||||
concept nameLegacyECMF(defaultName,"name.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden;
|
||||
concept nameECMF(nameLegacyECMF,"name.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy;
|
||||
concept name(nameECMF,"name.def",conceptsDir2,conceptsDir1): no_copy,dump;
|
||||
|
||||
concept cfNameLegacyECMF(defaultShortName,"cfName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden;
|
||||
concept cfNameECMF(cfNameLegacyECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy,dump;
|
||||
concept cfNameLegacyECMF(defaultShortName,"cfName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden;
|
||||
concept cfNameECMF(cfNameLegacyECMF,"cfName.def",conceptsMasterDir,conceptsLocalDirECMF) : no_copy;
|
||||
concept cfName(cfNameECMF,"cfName.def",conceptsDir2,conceptsDir1) : no_copy,dump;
|
||||
|
||||
concept cfVarNameLegacyECMF(defaultShortName,"cfVarName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump,hidden;
|
||||
concept cfVarNameECMF (cfVarNameLegacyECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,dump;
|
||||
concept cfVarNameLegacyECMF(defaultShortName,"cfVarName.legacy.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy,hidden;
|
||||
concept cfVarNameECMF (cfVarNameLegacyECMF,"cfVarName.def",conceptsMasterDir,conceptsLocalDirECMF): no_copy;
|
||||
concept cfVarName (cfVarNameECMF,"cfVarName.def",conceptsDir2,conceptsDir1): no_copy,dump;
|
||||
|
||||
# modelName: Contribution from Daniel Lee @ DWD
|
||||
|
|
|
@ -45,3 +45,9 @@ meta values data_apply_bitmap(codedValues,
|
|||
alias data.packedValues = codedValues;
|
||||
|
||||
template statistics "common/statistics_grid.def";
|
||||
if(missingValueManagementUsed) {
|
||||
#remove numberOfMissing;
|
||||
#remove numberOfMissingValues;
|
||||
meta numberOfMissing count_missing(bitmap,unusedBitsInBitmap,numberOfDataPoints,missingValueManagementUsed) : dump;
|
||||
alias numberOfMissingValues=numberOfMissing;
|
||||
}
|
||||
|
|
|
@ -42,3 +42,9 @@ meta values data_apply_bitmap(codedValues,
|
|||
alias data.packedValues=codedValues;
|
||||
|
||||
template statistics "common/statistics_grid.def";
|
||||
if(missingValueManagementUsed) {
|
||||
#remove numberOfMissingValues;
|
||||
#remove numberOfMissing;
|
||||
meta numberOfMissing count_missing(bitmap,unusedBitsInBitmap,numberOfDataPoints,missingValueManagementUsed) : dump;
|
||||
alias numberOfMissingValues=numberOfMissing;
|
||||
}
|
||||
|
|
|
@ -300,7 +300,9 @@ int grib_section_adjust_sizes(grib_section* s, int update, int depth)
|
|||
|
||||
if (offset != a->offset) {
|
||||
grib_context_log(a->context, GRIB_LOG_ERROR,
|
||||
"Offset mismatch %s A->offset %ld offset %ld\n", a->name, (long)a->offset, (long)offset);
|
||||
"Offset mismatch accessor=%s: accessor's offset=%ld, but actual offset=%ld",
|
||||
a->name, (long)a->offset, (long)offset);
|
||||
grib_context_log(a->context, GRIB_LOG_ERROR, "Hint: Check section lengths are in sync with their contents");
|
||||
a->offset = offset;
|
||||
return GRIB_DECODING_ERROR;
|
||||
}
|
||||
|
|
|
@ -21,6 +21,7 @@
|
|||
MEMBERS = const char* bitmap
|
||||
MEMBERS = const char* unusedBitsInBitmap
|
||||
MEMBERS = const char* numberOfDataPoints
|
||||
MEMBERS = const char* missingValueManagementUsed
|
||||
END_CLASS_DEF
|
||||
|
||||
*/
|
||||
|
@ -49,6 +50,7 @@ typedef struct grib_accessor_count_missing
|
|||
const char* bitmap;
|
||||
const char* unusedBitsInBitmap;
|
||||
const char* numberOfDataPoints;
|
||||
const char* missingValueManagementUsed;
|
||||
} grib_accessor_count_missing;
|
||||
|
||||
extern grib_accessor_class* grib_accessor_class_long;
|
||||
|
@ -181,15 +183,41 @@ static void init(grib_accessor* a, const long len, grib_arguments* arg)
|
|||
{
|
||||
int n = 0;
|
||||
grib_accessor_count_missing* self = (grib_accessor_count_missing*)a;
|
||||
grib_handle* h = grib_handle_of_accessor(a);
|
||||
a->length = 0;
|
||||
a->flags |= GRIB_ACCESSOR_FLAG_READ_ONLY;
|
||||
self->bitmap = grib_arguments_get_name(grib_handle_of_accessor(a), arg, n++);
|
||||
self->unusedBitsInBitmap = grib_arguments_get_name(grib_handle_of_accessor(a), arg, n++);
|
||||
self->numberOfDataPoints = grib_arguments_get_name(grib_handle_of_accessor(a), arg, n++);
|
||||
self->bitmap = grib_arguments_get_name(h, arg, n++);
|
||||
self->unusedBitsInBitmap = grib_arguments_get_name(h, arg, n++);
|
||||
self->numberOfDataPoints = grib_arguments_get_name(h, arg, n++);
|
||||
self->missingValueManagementUsed = grib_arguments_get_name(h, arg, n++); /* Can be NULL */
|
||||
}
|
||||
|
||||
static const int used[] = { 0, 1, 3, 7, 15, 31, 63, 127, 255 };
|
||||
|
||||
static int get_count_of_missing_values(grib_handle* h, long* p_count_of_missing)
|
||||
{
|
||||
int err = 0;
|
||||
long count_of_missing = 0;
|
||||
size_t vsize = 0, ii = 0;
|
||||
double* values = NULL;
|
||||
double mv = 0;
|
||||
if ((err = grib_get_double(h, "missingValue", &mv)) != GRIB_SUCCESS)
|
||||
return err;
|
||||
if ((err = grib_get_size(h, "values", &vsize)) != GRIB_SUCCESS)
|
||||
return err;
|
||||
values = (double*)grib_context_malloc(h->context, vsize * sizeof(double));
|
||||
if (!values)
|
||||
return GRIB_OUT_OF_MEMORY;
|
||||
if ((err = grib_get_double_array(h, "values", values, &vsize)) != GRIB_SUCCESS)
|
||||
return err;
|
||||
for (ii = 0; ii < vsize; ii++) {
|
||||
if (values[ii] == mv) ++count_of_missing;
|
||||
}
|
||||
grib_context_free(h->context, values);
|
||||
*p_count_of_missing = count_of_missing;
|
||||
|
||||
return GRIB_SUCCESS;
|
||||
}
|
||||
static int unpack_long(grib_accessor* a, long* val, size_t* len)
|
||||
{
|
||||
grib_accessor_count_missing* self = (grib_accessor_count_missing*)a;
|
||||
|
@ -202,10 +230,23 @@ static int unpack_long(grib_accessor* a, long* val, size_t* len)
|
|||
grib_handle* h = grib_handle_of_accessor(a);
|
||||
grib_accessor* bitmap = grib_find_accessor(h, self->bitmap);
|
||||
|
||||
*val = 0;
|
||||
*val = 0; /* By default assume none are missing */
|
||||
*len = 1;
|
||||
if (!bitmap)
|
||||
if (!bitmap) {
|
||||
long mvmu = 0;
|
||||
if (self->missingValueManagementUsed &&
|
||||
grib_get_long(h, self->missingValueManagementUsed, &mvmu) == GRIB_SUCCESS && mvmu != 0)
|
||||
{
|
||||
/* ECC-523: No bitmap. Missing values are encoded in the Data Section.
|
||||
* So we must decode all the data values and count how many are missing
|
||||
*/
|
||||
long count_of_missing = 0;
|
||||
if (get_count_of_missing_values(h, &count_of_missing) == GRIB_SUCCESS) {
|
||||
*val = count_of_missing;
|
||||
}
|
||||
}
|
||||
return GRIB_SUCCESS;
|
||||
}
|
||||
|
||||
size = grib_byte_count(bitmap);
|
||||
offset = grib_byte_offset(bitmap);
|
||||
|
|
|
@ -271,6 +271,6 @@ static void destroy(grib_context* c, grib_accessor* a)
|
|||
|
||||
static int value_count(grib_accessor* a, long* count)
|
||||
{
|
||||
*count = 16;
|
||||
*count = 1; /* ECC-1475 */
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -242,20 +242,22 @@ static void default_print(const grib_context* c, void* descriptor, const char* m
|
|||
|
||||
void grib_context_set_print_proc(grib_context* c, grib_print_proc p)
|
||||
{
|
||||
c = c ? c : grib_context_get_default();
|
||||
c->print = p;
|
||||
c = c ? c : grib_context_get_default();
|
||||
/* Set logging back to the default if p is NULL */
|
||||
c->print = (p ? p : &default_print);
|
||||
}
|
||||
|
||||
void grib_context_set_debug(grib_context* c, int mode)
|
||||
{
|
||||
c = c ? c : grib_context_get_default();
|
||||
c = c ? c : grib_context_get_default();
|
||||
c->debug = mode;
|
||||
}
|
||||
|
||||
void grib_context_set_logging_proc(grib_context* c, grib_log_proc p)
|
||||
{
|
||||
c = c ? c : grib_context_get_default();
|
||||
c->output_log = p;
|
||||
c = c ? c : grib_context_get_default();
|
||||
/* Set logging back to the default if p is NULL */
|
||||
c->output_log = (p ? p : &default_log);
|
||||
}
|
||||
|
||||
long grib_get_api_version()
|
||||
|
|
|
@ -22,7 +22,7 @@
|
|||
|
||||
static int ecc_jasper_initialise()
|
||||
{
|
||||
#if JASPER_VERSION_MAJOR == 3
|
||||
#if JASPER_VERSION_MAJOR >= 3
|
||||
int jaserr = 0;
|
||||
jas_conf_clear();
|
||||
jas_conf_set_max_mem_usage(jas_get_total_mem_size());
|
||||
|
@ -36,7 +36,7 @@ static int ecc_jasper_initialise()
|
|||
|
||||
static jas_image_t* ecc_jasper_decode(jas_stream_t *in)
|
||||
{
|
||||
#if JASPER_VERSION_MAJOR == 3
|
||||
#if JASPER_VERSION_MAJOR >= 3
|
||||
/* Second argument (=fmt) < 0 means "If possible, try to determine the format of the input data" */
|
||||
return jas_image_decode(in, -1, 0);
|
||||
#else
|
||||
|
@ -46,7 +46,7 @@ static jas_image_t* ecc_jasper_decode(jas_stream_t *in)
|
|||
|
||||
static int ecc_jasper_encode(jas_image_t *image, jas_stream_t *jpcstream, char *optstr)
|
||||
{
|
||||
#if JASPER_VERSION_MAJOR == 3
|
||||
#if JASPER_VERSION_MAJOR >= 3
|
||||
const int fmt = jas_image_strtofmt("jpc");
|
||||
return jas_image_encode(image, jpcstream, fmt, optstr);
|
||||
#else
|
||||
|
@ -56,7 +56,7 @@ static int ecc_jasper_encode(jas_image_t *image, jas_stream_t *jpcstream, char *
|
|||
|
||||
static void ecc_jasper_cleanup()
|
||||
{
|
||||
#if JASPER_VERSION_MAJOR == 3
|
||||
#if JASPER_VERSION_MAJOR >= 3
|
||||
jas_cleanup_thread();
|
||||
jas_cleanup_library();
|
||||
#endif
|
||||
|
|
|
@ -29,3 +29,11 @@ for file in $files; do
|
|||
|
||||
rm -f $temp $temp1 $temp2
|
||||
done
|
||||
|
||||
# ECC-523
|
||||
infile=${data_dir}/gfs.complex.mvmu.grib2
|
||||
grib_check_key_equals $infile 'missingValueManagementUsed,bitmapPresent' '1 0'
|
||||
grib_check_key_equals $infile 'numberOfMissing,numberOfMissingValues' '556901 556901'
|
||||
infile=${data_dir}/gfs.c255.grib2
|
||||
grib_check_key_equals $infile 'missingValueManagementUsed,bitmapPresent' '0 1'
|
||||
grib_check_key_equals $infile 'numberOfMissing,numberOfMissingValues' '7665 7665'
|
||||
|
|
|
@ -14,6 +14,7 @@
|
|||
#define NUMBER(x) (sizeof(x) / sizeof(x[0]))
|
||||
|
||||
int assertion_caught = 0;
|
||||
int logging_caught = 0;
|
||||
|
||||
typedef enum
|
||||
{
|
||||
|
@ -1468,6 +1469,28 @@ static void test_assertion_catching()
|
|||
free(list);
|
||||
}
|
||||
|
||||
|
||||
static void my_logging_proc(const grib_context* c, int level, const char* mesg)
|
||||
{
|
||||
logging_caught = 1;
|
||||
}
|
||||
static void test_logging_proc()
|
||||
{
|
||||
grib_context* context = grib_context_get_default();
|
||||
Assert(logging_caught == 0);
|
||||
|
||||
/* Override default behaviour */
|
||||
grib_context_set_logging_proc(context, my_logging_proc);
|
||||
grib_context_log(context, GRIB_LOG_ERROR, "This error will be handled by me");
|
||||
Assert(logging_caught == 1);
|
||||
|
||||
/* Restore the logging proc */
|
||||
logging_caught = 0;
|
||||
grib_context_set_logging_proc(context, NULL);
|
||||
grib_context_log(context, GRIB_LOG_ERROR, "This will come out as normal");
|
||||
Assert(logging_caught == 0);
|
||||
}
|
||||
|
||||
static void test_concept_condition_strings()
|
||||
{
|
||||
int err = 0;
|
||||
|
@ -1631,10 +1654,12 @@ static void test_parse_keyval_string()
|
|||
free( (void*)values3[0].name );
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
printf("Doing unit tests. ecCodes version = %ld\n", grib_get_api_version());
|
||||
|
||||
test_logging_proc();
|
||||
test_grib_binary_search();
|
||||
test_parse_keyval_string();
|
||||
|
||||
|
|
|
@ -27,7 +27,7 @@ static void usage(const char* prog)
|
|||
int main(int argc, char* argv[])
|
||||
{
|
||||
char* resource_path = NULL;
|
||||
char* resource_name = NULL;
|
||||
const char* resource_name = NULL;
|
||||
ResourceType resource_type = UNKNOWN;
|
||||
char* full_path = NULL;
|
||||
char* out_file = NULL;
|
||||
|
|
|
@ -58,7 +58,7 @@ int grib_tool_init(grib_runtime_options* options)
|
|||
{
|
||||
int opt = grib_options_on("C") + grib_options_on("O") + grib_options_on("D");
|
||||
|
||||
options->dump_mode = "default";
|
||||
options->dump_mode = (char*)"default";
|
||||
|
||||
|
||||
if (opt > 1) {
|
||||
|
@ -67,12 +67,12 @@ int grib_tool_init(grib_runtime_options* options)
|
|||
}
|
||||
|
||||
if (grib_options_on("O")) {
|
||||
options->dump_mode = "wmo";
|
||||
options->dump_mode = (char*)"wmo";
|
||||
options->dump_flags = GRIB_DUMP_FLAG_CODED | GRIB_DUMP_FLAG_OCTET | GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY;
|
||||
}
|
||||
|
||||
if (grib_options_on("D")) {
|
||||
options->dump_mode = "debug";
|
||||
options->dump_mode = (char*)"debug";
|
||||
options->dump_flags = GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY;
|
||||
}
|
||||
|
||||
|
|
|
@ -56,7 +56,7 @@ int grib_tool_init(grib_runtime_options* options)
|
|||
{
|
||||
int opt = grib_options_on("C") + grib_options_on("O") + grib_options_on("D");
|
||||
|
||||
options->dump_mode = "default";
|
||||
options->dump_mode = (char*)"default";
|
||||
|
||||
if (opt > 1) {
|
||||
printf("%s: simultaneous O/D options not allowed\n", tool_name);
|
||||
|
@ -64,22 +64,22 @@ int grib_tool_init(grib_runtime_options* options)
|
|||
}
|
||||
|
||||
if (grib_options_on("O")) {
|
||||
options->dump_mode = "wmo";
|
||||
options->dump_mode = (char*)"wmo";
|
||||
options->dump_flags = GRIB_DUMP_FLAG_CODED | GRIB_DUMP_FLAG_OCTET | GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY;
|
||||
}
|
||||
|
||||
if (grib_options_on("D")) {
|
||||
options->dump_mode = "debug";
|
||||
options->dump_mode = (char*)"debug";
|
||||
options->dump_flags = GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY;
|
||||
}
|
||||
|
||||
if (grib_options_on("J")) {
|
||||
options->dump_mode = "json";
|
||||
options->dump_mode = (char*)"json";
|
||||
options->dump_flags = GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY;
|
||||
}
|
||||
|
||||
if (grib_options_on("X")) {
|
||||
options->dump_mode = "xml";
|
||||
options->dump_mode = (char*)"xml";
|
||||
options->dump_flags = GRIB_DUMP_FLAG_VALUES | GRIB_DUMP_FLAG_READ_ONLY;
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue