core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into bugfix/ECC-989-ValgrindErrorBufr

This commit is contained in:
Shahram Najm 2022-03-07 17:56:10 +00:00
parent b6896e9446 60f072f74d
commit ba0081ec69
4 changed files with 90 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/grib_api_prototypes.h
View File

@ -675,14 +675,12 @@ bufr_descriptors_array* grib_accessor_class_expanded_descriptors_get_expanded(gr
/* grib_accessor_class_md5.c */
/* grib_jasper_encoding.c */
int grib_jasper_decode(grib_context* c, unsigned char* buf, size_t* buflen, double* values, size_t* no_values);
int grib_jasper_encode(grib_context* c, j2k_encode_helper* helper);
int grib_jasper_decode(grib_context* c, unsigned char* buf, size_t* buflen, double* val, size_t* n_vals);
int grib_jasper_decode(grib_context* c, unsigned char* buf, const size_t* buflen, double* values, const size_t* n_vals);
int grib_jasper_encode(grib_context* c, j2k_encode_helper* helper);
/* grib_openjpeg_encoding.c */
int grib_openjpeg_decode(grib_context* c, unsigned char* buf, const size_t* buflen, double* values, const size_t* n_vals);
int grib_openjpeg_encode(grib_context* c, j2k_encode_helper* helper);
int grib_openjpeg_decode(grib_context* c, unsigned char* buf, const size_t* buflen, double* val, const size_t* n_vals);
/* action_class_set_missing.c */
grib_action* grib_action_create_set_missing(grib_context* context, const char* name);
@ -714,7 +712,6 @@ grib_index* grib_index_read(grib_context* c, const char* filename, int* err);
int grib_index_search_same(grib_index* index, grib_handle* h);
int grib_index_add_file(grib_index* index, const char* filename);
int _codes_index_add_file(grib_index* index, const char* filename, int message_type);
int grib_index_add_file(grib_index* index, const char* filename);
grib_index* grib_index_new_from_file(grib_context* c, const char* filename, const char* keys, int* err);
int grib_index_get_size(const grib_index* index, const char* key, size_t* size);
int grib_index_get_string(const grib_index* index, const char* key, char** values, size_t* size);
@ -815,14 +812,6 @@ void grib_timer_print(grib_timer* t);
void grib_timer_partial_rate(grib_timer* t, double start, long total);
void grib_print_all_timers(void);
void grib_reset_all_timers(void);
grib_timer* grib_get_timer(grib_context* c, const char* name, const char* statname, int elapsed);
int grib_timer_start(grib_timer* t);
int grib_timer_stop(grib_timer* t, long total);
double grib_timer_value(grib_timer* t);
void grib_timer_print(grib_timer* t);
void grib_timer_partial_rate(grib_timer* t, double start, long total);
void grib_print_all_timers(void);
void grib_reset_all_timers(void);
/* grib_ibmfloat.c */
unsigned long grib_ibm_to_long(double x);
@ -841,15 +830,10 @@ double grib_ieeefloat_error(double x);
double grib_long_to_ieee(unsigned long x);
unsigned long grib_ieee_nearest_smaller_to_long(double x);
int grib_nearest_smaller_ieee_float(double a, double* ret);
double grib_ieeefloat_error(double x);
double grib_long_to_ieee(unsigned long x);
int grib_nearest_smaller_ieee_float(double a, double* x);
unsigned long grib_ieee_to_long(double x);
unsigned long grib_ieee64_to_long(double x);
double grib_long_to_ieee64(unsigned long x);
int grib_ieee_decode_array(grib_context* c, unsigned char* buf, size_t nvals, int bytes, double* val);
int grib_ieee_decode_array(grib_context* c, unsigned char* buf, size_t nvals, int bytes, double* val);
int grib_ieee_encode_array(grib_context* c, double* val, size_t nvals, int bytes, unsigned char* buf);
int grib_ieee_encode_array(grib_context* c, double* val, size_t nvals, int bytes, unsigned char* buf);
/* grib_accessor_class_reference_value_error.c */

98
src/grib_jasper_encoding.c
View File

@ -18,18 +18,67 @@
#undef PACKAGE_TARNAME
#undef PACKAGE_VERSION
#include "jasper/jasper.h"
#define MAXOPTSSIZE 1024
int grib_jasper_decode(grib_context* c, unsigned char* buf, size_t* buflen, double* values, size_t* no_values)
static int ecc_jasper_initialise()
{
/*jas_setdbglevel(99999);*/
#if JASPER_VERSION_MAJOR == 3
int jaserr = 0;
jas_conf_clear();
jas_conf_set_max_mem_usage(jas_get_total_mem_size());
jaserr = jas_init_library();
if (jaserr) return jaserr;
jaserr = jas_init_thread();
if (jaserr) return jaserr;
#endif
return 0;
}
static jas_image_t* ecc_jasper_decode(jas_stream_t *in)
{
#if JASPER_VERSION_MAJOR == 3
/* Second argument (=fmt) < 0 means "If possible, try to determine the format of the input data" */
return jas_image_decode(in, -1, 0);
#else
return jpc_decode(in, NULL);
#endif
}
static int ecc_jasper_encode(jas_image_t *image, jas_stream_t *jpcstream, char *optstr)
{
#if JASPER_VERSION_MAJOR == 3
const int fmt = jas_image_strtofmt("jpc");
return jas_image_encode(image, jpcstream, fmt, optstr);
#else
return jpc_encode(image, jpcstream, optstr);
#endif
}
static void ecc_jasper_cleanup()
{
#if JASPER_VERSION_MAJOR == 3
jas_cleanup_thread();
jas_cleanup_library();
#endif
}
int grib_jasper_decode(grib_context* c, unsigned char* buf, const size_t* buflen, double* values, const size_t* n_vals)
{
/* jas_setdbglevel(99999); */
jas_image_t* image = NULL;
jas_stream_t* jpeg = NULL;
int code = GRIB_SUCCESS;
jas_matrix_t* matrix = NULL;
jas_image_cmpt_t* p;
int i, j, k;
int jaserr = 0; /* 0 means success */
jaserr = ecc_jasper_initialise();
if (jaserr) {
grib_context_log(c, GRIB_LOG_ERROR, "grib_jasper_decode: Failed to initialize JasPer library. JasPer error %d", jaserr);
code = GRIB_DECODING_ERROR;
goto cleanup;
}
jpeg = jas_stream_memopen((char*)buf, *buflen);
if (!jpeg) {
@ -37,9 +86,9 @@ int grib_jasper_decode(grib_context* c, unsigned char* buf, size_t* buflen, doub
goto cleanup;
}
grib_context_log(c, GRIB_LOG_DEBUG, "grib_jasper_decode: Jasper version %s", jas_getversion());
grib_context_log(c, GRIB_LOG_DEBUG, "grib_jasper_decode: JasPer version %s", jas_getversion());
image = jpc_decode(jpeg, NULL);
image = ecc_jasper_decode(jpeg);
if (!image) {
code = GRIB_DECODING_ERROR;
goto cleanup;
@ -54,15 +103,19 @@ int grib_jasper_decode(grib_context* c, unsigned char* buf, size_t* buflen, doub
}
matrix = jas_matrix_create(jas_image_height(image), jas_image_width(image));
if (!matrix) {
code = GRIB_DECODING_ERROR;
goto cleanup;
}
jas_image_readcmpt(image, 0, 0, 0, jas_image_width(image), jas_image_height(image), matrix);
jaserr = jas_image_readcmpt(image, 0, 0, 0, jas_image_width(image), jas_image_height(image), matrix);
if (jaserr) {
grib_context_log(c, GRIB_LOG_ERROR, "grib_jasper_decode: Failed to read JasPer component data. JasPer error %d", jaserr);
code = GRIB_DECODING_ERROR;
goto cleanup;
}
Assert(p->height_ * p->width_ == *no_values);
Assert(p->height_ * p->width_ == *n_vals);
k = 0;
for (i = 0; i < p->height_; i++)
@ -77,6 +130,7 @@ cleanup:
jas_image_destroy(image);
if (jpeg)
jas_stream_close(jpeg);
ecc_jasper_cleanup();
return code;
}
@ -84,7 +138,7 @@ cleanup:
int grib_jasper_encode(grib_context* c, j2k_encode_helper* helper)
{
int code = GRIB_SUCCESS;
int jaserr;
int jaserr = 0;
char opts[MAXOPTSSIZE];
double reference_value = helper->reference_value;
@ -156,7 +210,7 @@ int grib_jasper_encode(grib_context* c, j2k_encode_helper* helper)
}
}
/*jas_init();*/
ecc_jasper_initialise();
opts[0] = 0;
@ -166,7 +220,7 @@ int grib_jasper_encode(grib_context* c, j2k_encode_helper* helper)
}
Assert(cmpt.width_ * cmpt.height_ * cmpt.cps_ == buflen);
grib_context_log(c, GRIB_LOG_DEBUG, "grib_jasper_encode: Jasper version %s", jas_getversion());
grib_context_log(c, GRIB_LOG_DEBUG, "grib_jasper_encode: JasPer version %s", jas_getversion());
pcmpt = &cmpt;
image.cmpts_ = &pcmpt;
@ -175,12 +229,12 @@ int grib_jasper_encode(grib_context* c, j2k_encode_helper* helper)
cmpt.stream_ = istream;
jpcstream = jas_stream_memopen((char*)helper->jpeg_buffer, helper->buffer_size);
jaserr = jpc_encode(&image, jpcstream, opts);
if (jaserr != 0) {
/* increase the number of guard bits */
jaserr = ecc_jasper_encode(&image, jpcstream, opts);
if (jaserr) {
/* Failed to encode. Increase the number of guard bits */
strcat(opts, "\nnumgbits=4");
grib_context_log(c, GRIB_LOG_ERROR, "JASPER: error %d, increasing the number of guard bits", jaserr);
grib_context_log(c, GRIB_LOG_ERROR, "grib_jasper_encode: JasPer error %d, increasing the number of guard bits", jaserr);
jas_stream_close(istream);
istream = 0;
jas_stream_close(jpcstream);
@ -189,11 +243,11 @@ int grib_jasper_encode(grib_context* c, j2k_encode_helper* helper)
istream = jas_stream_memopen((char*)encoded, buflen);
cmpt.stream_ = istream;
jpcstream = jas_stream_memopen((char*)helper->jpeg_buffer, helper->buffer_size);
jaserr = jpc_encode(&image, jpcstream, opts);
jaserr = ecc_jasper_encode(&image, jpcstream, opts);
}
if (jaserr != 0) {
grib_context_log(c, GRIB_LOG_ERROR, "JASPER: error %d", jaserr);
if (jaserr) {
grib_context_log(c, GRIB_LOG_ERROR, "grib_jasper_encode: Failed to encode. JasPer error %d", jaserr);
code = GRIB_ENCODING_ERROR;
goto cleanup;
}
@ -210,23 +264,23 @@ cleanup:
jas_stream_close(istream);
if (jpcstream)
jas_stream_close(jpcstream);
ecc_jasper_cleanup();
return code;
}
#else
int grib_jasper_decode(grib_context* c, unsigned char* buf, size_t* buflen, double* val, size_t* n_vals)
int grib_jasper_decode(grib_context* c, unsigned char* buf, const size_t* buflen, double* val, const size_t* n_vals)
{
grib_context_log(c, GRIB_LOG_ERROR,
"grib_accessor_data_jpeg2000_packing: Jasper JPEG support not enabled.");
"grib_accessor_data_jpeg2000_packing: JasPer JPEG support not enabled.");
return GRIB_FUNCTIONALITY_NOT_ENABLED;
}
int grib_jasper_encode(grib_context* c, j2k_encode_helper* helper)
{
grib_context_log(c, GRIB_LOG_ERROR,
"grib_accessor_data_jpeg2000_packing: Jasper JPEG support not enabled.");
"grib_accessor_data_jpeg2000_packing: JasPer JPEG support not enabled.");
return GRIB_FUNCTIONALITY_NOT_ENABLED;
}

2
tests/grib_packing_order.c
View File

@ -24849,7 +24849,7 @@ int main(int argc, char** argv)
for (i = 0; i < values_len; i++) {
const double diff = fabs(values[i] - vals[i]);
if (diff > EPSILON) {
fprintf(stderr, "Unpacked value different: i=%lu values[i]=%.7f vals[i]=%.7f\n",
fprintf(stderr, "Unpacked value different at i=%lu: original=%.7f decoded=%.7f\n",
i, values[i], vals[i]);
return 1;
}

19
tests/grib_packing_order.sh
View File

@ -71,17 +71,18 @@ fi
# IEEE
# ------------
#tests/grib_packing_order grid_ieee values_before_packing_type x2 # Does not work
$EXEC ${test_dir}/grib_packing_order grid_ieee packing_type_before_values $temp_ieee1
$EXEC ${test_dir}/grib_packing_order grid_ieee values_before_packing_type $temp_ieee2
if [ $HAVE_EXTRA_TESTS -eq 1 ]; then
#tests/grib_packing_order grid_ieee values_before_packing_type x2 # Does not work
$EXEC ${test_dir}/grib_packing_order grid_ieee packing_type_before_values $temp_ieee1
$EXEC ${test_dir}/grib_packing_order grid_ieee values_before_packing_type $temp_ieee2
${tools_dir}/grib_ls -n statistics $temp_ieee1 $temp_ieee2
# TODO
# ${tools_dir}/grib_compare $temp_ieee1 $temp_ieee2
# No point comparing with grid_simple as grid_ieee will be closer to the actual values
# and less lossy
${tools_dir}/grib_ls -n statistics $temp_ieee1 $temp_ieee2
# TODO
# ${tools_dir}/grib_compare $temp_ieee1 $temp_ieee2
# No point comparing with grid_simple as grid_ieee will be closer to the actual values
# and less lossy
fi
# Second order Packing: TODO