diff --git a/.gitignore b/.gitignore index fc242f646..0f3d497a8 100644 --- a/.gitignore +++ b/.gitignore @@ -113,6 +113,7 @@ tests/read_any tests/read_index tests/so_perf tests/unit_tests +tests/ccsds_perf tigge/.libs/ tigge/tigge_accumulations tigge/tigge_check diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ab551db7..999991bf9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,7 @@ ecbuild_declare_project() option( ENABLE_JPG "try to add support for JPG encoding" ON ) option( ENABLE_PNG "try to add support for PNG encoding" OFF ) option( ENABLE_NETCDF "try to add support for NetCDF" ON ) -option( ENABLE_SZIP "try to add support for SZip encoding" OFF ) +option( ENABLE_AEC "try to add support for Adaptive Entropy Coding" OFF ) option( ENABLE_PYTHON "try to build the GRIB_API Python interface" ON ) option( ENABLE_FORTRAN "try to build the GRIB_API Fortran interface" ON ) @@ -55,15 +55,15 @@ set( GRIB_API_EXTRA_DEFINITIONS "" ) find_package( CMath ) -### SZIP support +### AEC support (Adaptive Entropy Coding) -set( HAVE_LIBSZIP 0 ) -if( ENABLE_SZIP ) +set( HAVE_LIBAEC 0 ) +if( ENABLE_AEC ) - find_package( SZip ) + find_package( AEC ) - if( SZIP_FOUND ) - set( HAVE_LIBSZIP 1 ) + if( AEC_FOUND ) + set( HAVE_LIBAEC 1 ) endif() endif() @@ -198,7 +198,7 @@ endif() ### export package to other ecbuild packages -set( GRIB_API_TPLS SZip PNG NetCDF Jasper OpenJPEG CMath ) +set( GRIB_API_TPLS AEC PNG NetCDF Jasper OpenJPEG CMath ) set( GRIB_API_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}/src ) set( GRIB_API_LIBRARIES grib_api ) diff --git a/NOTICE b/NOTICE index 53607ee3d..b35ccc890 100644 --- a/NOTICE +++ b/NOTICE @@ -1,11 +1,12 @@ ECMWF GRIB API Copyright 2005-2014 ECMWF. -This product includes software developed at -ECMWF (http://www.ecmwf.int). +This product includes software developed at ECMWF (http://www.ecmwf.int). -Parts of the definitions provided by -WMO (http://www.wmo.int/pages/index_en.html) +Parts of the definitions provided by WMO (http://www.wmo.int/pages/index_en.html) + +Adaptive Entropy Coding (CCSDS) provided by +The Max-Planck-Institute for Meteorology and Deutsches Klimarechenzentrum GmbH IBM POWER optimisations were developed at The Max-Planck-Institute for Meteorology. diff --git a/configure.ac b/configure.ac index d8cdd87e9..530661a6b 100755 --- a/configure.ac +++ b/configure.ac @@ -455,6 +455,23 @@ then AC_SUBST(JPEG_TEST) fi +dnl check for libaec +CCSDS_TEST="" +AC_ARG_WITH([aec], + [AS_HELP_STRING([--with-aec=DIR], + [use specified libaec installation directory])], + ,[with_aec=no]) + +if test "x$with_aec" != xno ; then + if test "x$with_aec" != xyes ; then + LDFLAGS="$LDFLAGS -L$with_aec/lib" + CPPFLAGS="$CPPFLAGS -I$with_aec/include" + fi + AC_CHECK_LIB(aec, aec_encode, , [AC_MSG_FAILURE( + [aec test failed (--without-aec to disable)])]) + CCSDS_TEST="ccsds.sh" +fi +AC_SUBST(CCSDS_TEST) dnl Check for png AC_ARG_WITH([png-support], diff --git a/data/CMakeLists.txt b/data/CMakeLists.txt index 067085e0f..427f1989d 100644 --- a/data/CMakeLists.txt +++ b/data/CMakeLists.txt @@ -86,6 +86,7 @@ set(files_to_download timeRangeIndicator_5.grib tp_ecmwf.grib v.grib2 + ccsds.grib2 ) foreach( f ${files_to_download} ) diff --git a/data/download.sh b/data/download.sh index e6e6c39f9..17246c4ee 100755 --- a/data/download.sh +++ b/data/download.sh @@ -123,6 +123,7 @@ files=" timeRangeIndicator_5.grib tp_ecmwf.grib v.grib2 + ccsds.grib2 tigge/tigge_ammc_pl_gh.grib tigge/tigge_ammc_pl_q.grib tigge/tigge_ammc_pl_t.grib diff --git a/definitions/grib1/section.4.def b/definitions/grib1/section.4.def index bcfc92087..1cdf7cbed 100644 --- a/definitions/grib1/section.4.def +++ b/definitions/grib1/section.4.def @@ -126,7 +126,7 @@ concept packingType { plusOneinOrdersOfSPD=1; twoOrdersOfSPD=1; } "grid_jpeg" = { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;} "grid_png" = { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;} - "grid_szip" = { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;} + "grid_ccsds" = { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;} "grid_simple_log_preprocessing"= { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;} } : dump; diff --git a/definitions/grib2/section.5.def b/definitions/grib2/section.5.def index bbe350dfc..02212dd89 100644 --- a/definitions/grib2/section.5.def +++ b/definitions/grib2/section.5.def @@ -45,7 +45,7 @@ concept packingType (unknown) { "grid_jpeg" = { dataRepresentationTemplateNumber = 40; } "grid_png" = { dataRepresentationTemplateNumber = 40010; } "grid_png" = { dataRepresentationTemplateNumber = 41; } - "grid_szip" = { dataRepresentationTemplateNumber = 42; } + "grid_ccsds" = { dataRepresentationTemplateNumber = 42; } "grid_ieee" = { dataRepresentationTemplateNumber = 4; } "grid_second_order" = { dataRepresentationTemplateNumber = 50001; } "grid_second_order" = { dataRepresentationTemplateNumber = 50002; } diff --git a/definitions/grib2/template.5.42.def b/definitions/grib2/template.5.42.def index d013a7128..a18f29172 100644 --- a/definitions/grib2/template.5.42.def +++ b/definitions/grib2/template.5.42.def @@ -7,22 +7,19 @@ # virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. # -# TEMPLATE 5.42, Grid point and spectral data - CCSDS szip +# TEMPLATE 5.42, Grid point and spectral data - CCSDS include "template.5.packing.def"; include "template.5.original_values.def"; -unsigned[1] szipOptionsMask : dump; +unsigned[1] ccsdsFlags : dump; -flagbit SZ_ALLOW_K13_OPTION_MASK(szipOptionsMask,0) = 0; -flagbit SZ_CHIP_OPTION_MASK(szipOptionsMask,1) = 0; -flagbit SZ_EC_OPTION_MASK(szipOptionsMask,2) = 0; -flagbit SZ_LSB_OPTION_MASK(szipOptionsMask,3) = 0; -flagbit SZ_MSB_OPTION_MASK(szipOptionsMask,4) = 1; -flagbit SZ_NN_OPTION_MASK(szipOptionsMask,5) = 1; -flagbit SZ_RAW_OPTION_MASK(szipOptionsMask,7) = 1; - -unsigned[1] szipBitsPerPixel : dump; -unsigned[2] szipPixelPerBlock = 32 : dump; -unsigned[2] szipPixelPerScanLine = 128 * szipPixelPerBlock : dump; +flagbit AEC_DATA_SIGNED_OPTION_MASK(ccsdsFlags,0) = 0; +flagbit AEC_DATA_3BYTE_OPTION_MASK(ccsdsFlags,1) = 0; +flagbit AEC_DATA_MSB_OPTION_MASK(ccsdsFlags,2) = 1; +flagbit AEC_DATA_PREPROCESS_OPTION_MASK(ccsdsFlags,3) = 1; +flagbit AEC_RESTRICTED_OPTION_MASK(ccsdsFlags,4) = 0; +flagbit AEC_PAD_RSI_OPTION_MASK(ccsdsFlags,5) = 0; +unsigned[1] ccsdsBlockSize = 32 : dump; +unsigned[2] ccsdsRsi = 128 : dump; diff --git a/definitions/grib2/template.7.42.def b/definitions/grib2/template.7.42.def index f689f256c..c4882685d 100644 --- a/definitions/grib2/template.7.42.def +++ b/definitions/grib2/template.7.42.def @@ -9,9 +9,9 @@ # START 2/template.7.42 ---------------------------------------------------------------------- -# TEMPLATE 7.42, Grid point data - CCSDS szip +# TEMPLATE 7.42, Grid point data - CCSDS - meta codedValues data_szip_packing( + meta codedValues data_ccsds_packing( section7Length, offsetBeforeData, offsetSection7, @@ -23,10 +23,9 @@ numberOfDataPoints, - szipOptionsMask, - szipBitsPerPixel, - szipPixelPerBlock, - szipPixelPerScanLine + ccsdsFlags, + ccsdsBlockSize, + ccsdsRsi ): read_only; diff --git a/grib_api_config.h.in b/grib_api_config.h.in index a8c88e037..80dea1e73 100644 --- a/grib_api_config.h.in +++ b/grib_api_config.h.in @@ -93,7 +93,7 @@ #define HAVE_LIBPNG @HAVE_LIBPNG@ -#cmakedefine HAVE_LIBSZIP +#cmakedefine HAVE_LIBAEC #cmakedefine HAVE_NETCDF diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1c54fc587..5fa187393 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -203,7 +203,7 @@ list( APPEND grib_api_srcs grib_accessor_class_data_g2secondary_bitmap.c grib_accessor_class_data_jpeg2000_packing.c grib_accessor_class_data_png_packing.c - grib_accessor_class_data_szip_packing.c + grib_accessor_class_data_ccsds_packing.c grib_accessor_class_data_raw_packing.c grib_accessor_class_data_complex_packing.c grib_accessor_class_data_g1complex_packing.c diff --git a/src/Makefile.am b/src/Makefile.am index f13c2d149..c420c5360 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -177,7 +177,7 @@ libgrib_api_la_prototypes= \ grib_accessor_class_values.c \ grib_accessor_class_simple_packing_error.c \ grib_accessor_class_data_simple_packing.c \ - grib_accessor_class_data_szip_packing.c \ + grib_accessor_class_data_ccsds_packing.c \ grib_accessor_class_count_missing.c \ grib_accessor_class_data_sh_packed.c \ grib_accessor_class_data_sh_unpacked.c \ diff --git a/src/grib_accessor_class.h b/src/grib_accessor_class.h index c7020c0ac..abe8420a5 100644 --- a/src/grib_accessor_class.h +++ b/src/grib_accessor_class.h @@ -63,7 +63,7 @@ extern grib_accessor_class* grib_accessor_class_data_sh_packed; extern grib_accessor_class* grib_accessor_class_data_sh_unpacked; extern grib_accessor_class* grib_accessor_class_data_shsimple_packing; extern grib_accessor_class* grib_accessor_class_data_simple_packing; -extern grib_accessor_class* grib_accessor_class_data_szip_packing; +extern grib_accessor_class* grib_accessor_class_data_ccsds_packing; extern grib_accessor_class* grib_accessor_class_decimal_precision; extern grib_accessor_class* grib_accessor_class_dictionary; extern grib_accessor_class* grib_accessor_class_dirty; diff --git a/src/grib_accessor_class_data_szip_packing.c b/src/grib_accessor_class_data_ccsds_packing.c similarity index 75% rename from src/grib_accessor_class_data_szip_packing.c rename to src/grib_accessor_class_data_ccsds_packing.c index e82365f9b..e17522b1b 100644 --- a/src/grib_accessor_class_data_szip_packing.c +++ b/src/grib_accessor_class_data_ccsds_packing.c @@ -28,10 +28,9 @@ MEMBERS=const char* number_of_data_points - MEMBERS=const char* szip_options_mask - MEMBERS=const char* szip_bits_per_pixel - MEMBERS=const char* szip_pixels_per_block - MEMBERS=const char* szip_pixels_per_scanline + MEMBERS=const char* ccsds_flags + MEMBERS=const char* ccsds_block_size + MEMBERS=const char* ccsds_rsi END_CLASS_DEF @@ -49,11 +48,11 @@ or edit "accessor.class" and rerun ./make_class.pl static int pack_double(grib_accessor*, const double* val,size_t *len); static int unpack_double(grib_accessor*, double* val,size_t *len); -static int value_count(grib_accessor*,long*); +static int value_count(grib_accessor*, long*); static void init(grib_accessor*,const long, grib_arguments* ); static void init_class(grib_accessor_class*); -typedef struct grib_accessor_data_szip_packing { +typedef struct grib_accessor_data_ccsds_packing { grib_accessor att; /* Members defined in gen */ /* Members defined in values */ @@ -62,25 +61,24 @@ typedef struct grib_accessor_data_szip_packing { const char* offsetdata; const char* offsetsection; int dirty; -/* Members defined in data_szip_packing */ +/* Members defined in data_ccsds_packing */ const char* number_of_values; const char* reference_value; const char* binary_scale_factor; const char* decimal_scale_factor; const char* bits_per_value; const char* number_of_data_points; - const char* szip_options_mask; - const char* szip_bits_per_pixel; - const char* szip_pixels_per_block; - const char* szip_pixels_per_scanline; -} grib_accessor_data_szip_packing; + const char* ccsds_flags; + const char* ccsds_block_size; + const char* ccsds_rsi; +} grib_accessor_data_ccsds_packing; extern grib_accessor_class* grib_accessor_class_values; -static grib_accessor_class _grib_accessor_class_data_szip_packing = { +static grib_accessor_class _grib_accessor_class_data_ccsds_packing = { &grib_accessor_class_values, /* super */ - "data_szip_packing", /* name */ - sizeof(grib_accessor_data_szip_packing), /* size */ + "data_ccsds_packing", /* name */ + sizeof(grib_accessor_data_ccsds_packing), /* size */ 0, /* inited */ &init_class, /* init_class */ &init, /* init */ @@ -95,7 +93,7 @@ static grib_accessor_class _grib_accessor_class_data_szip_packing = { 0, /* get native type */ 0, /* get sub_section */ 0, /* grib_pack procedures long */ - 0, /* grib_pack procedures long */ + 0, /* grib_pack procedures long */ 0, /* grib_pack procedures long */ 0, /* grib_unpack procedures long */ &pack_double, /* grib_pack procedures double */ @@ -120,7 +118,7 @@ static grib_accessor_class _grib_accessor_class_data_szip_packing = { }; -grib_accessor_class* grib_accessor_class_data_szip_packing = &_grib_accessor_class_data_szip_packing; +grib_accessor_class* grib_accessor_class_data_ccsds_packing = &_grib_accessor_class_data_ccsds_packing; static void init_class(grib_accessor_class* c) @@ -138,8 +136,6 @@ static void init_class(grib_accessor_class* c) c->unpack_long = (*(c->super))->unpack_long; c->pack_string = (*(c->super))->pack_string; c->unpack_string = (*(c->super))->unpack_string; - c->pack_string_array = (*(c->super))->pack_string_array; - c->unpack_string_array = (*(c->super))->unpack_string_array; c->pack_bytes = (*(c->super))->pack_bytes; c->unpack_bytes = (*(c->super))->unpack_bytes; c->pack_expression = (*(c->super))->pack_expression; @@ -159,46 +155,45 @@ static void init_class(grib_accessor_class* c) static void init(grib_accessor* a,const long v, grib_arguments* args) { - grib_accessor_data_szip_packing *self =(grib_accessor_data_szip_packing*)a; + grib_accessor_data_ccsds_packing *self =(grib_accessor_data_ccsds_packing*)a; self->number_of_values = grib_arguments_get_name(a->parent->h,args,self->carg++); self->reference_value = grib_arguments_get_name(a->parent->h,args,self->carg++); self->binary_scale_factor = grib_arguments_get_name(a->parent->h,args,self->carg++); self->decimal_scale_factor = grib_arguments_get_name(a->parent->h,args,self->carg++); - self->bits_per_value = grib_arguments_get_name(a->parent->h,args,self->carg++); + self->bits_per_value = grib_arguments_get_name(a->parent->h,args,self->carg++); self->number_of_data_points = grib_arguments_get_name(a->parent->h,args,self->carg++); - - self->szip_options_mask = grib_arguments_get_name(a->parent->h,args,self->carg++); - self->szip_bits_per_pixel = grib_arguments_get_name(a->parent->h,args,self->carg++); - self->szip_pixels_per_block = grib_arguments_get_name(a->parent->h,args,self->carg++); - self->szip_pixels_per_scanline = grib_arguments_get_name(a->parent->h,args,self->carg++); + self->ccsds_flags = grib_arguments_get_name(a->parent->h,args,self->carg++); + self->ccsds_block_size = grib_arguments_get_name(a->parent->h,args,self->carg++); + self->ccsds_rsi = grib_arguments_get_name(a->parent->h,args,self->carg++); a->flags |= GRIB_ACCESSOR_FLAG_DATA; + } static int value_count(grib_accessor* a, long* count) { - grib_accessor_data_szip_packing *self =(grib_accessor_data_szip_packing*)a; + grib_accessor_data_ccsds_packing *self =(grib_accessor_data_ccsds_packing*)a; *count = 0; return grib_get_long_internal(a->parent->h,self->number_of_values,count); } -#ifdef HAVE_LIBSZIP +#ifdef HAVE_LIBAEC -#include "szlib.h" +#include static int unpack_double(grib_accessor* a, double* val, size_t *len) { - grib_accessor_data_szip_packing *self =(grib_accessor_data_szip_packing*)a; + grib_accessor_data_ccsds_packing *self =(grib_accessor_data_ccsds_packing*)a; int err = GRIB_SUCCESS; - int i,j; + int i; size_t buflen = grib_byte_count(a); - SZ_com_t sz_options; + struct aec_stream strm; double bscale = 0; double dscale = 0; unsigned char* buf = NULL; @@ -214,11 +209,9 @@ static int unpack_double(grib_accessor* a, double* val, size_t *len) long bits_per_value =0; long bits8; - - long szip_options_mask ; - long szip_bits_per_pixel ; - long szip_pixels_per_block ; - long szip_pixels_per_scanline ; + long ccsds_flags; + long ccsds_block_size; + long ccsds_rsi; self->dirty=0; @@ -234,14 +227,11 @@ static int unpack_double(grib_accessor* a, double* val, size_t *len) if((err = grib_get_long_internal(a->parent->h,self->decimal_scale_factor, &decimal_scale_factor)) != GRIB_SUCCESS) return err; - - if((err = grib_get_long_internal(a->parent->h,self->szip_options_mask,&szip_options_mask)) != GRIB_SUCCESS) + if((err = grib_get_long_internal(a->parent->h,self->ccsds_flags,&ccsds_flags)) != GRIB_SUCCESS) return err; - if((err = grib_get_long_internal(a->parent->h,self->szip_bits_per_pixel, &szip_bits_per_pixel)) != GRIB_SUCCESS) + if((err = grib_get_long_internal(a->parent->h,self->ccsds_block_size, &ccsds_block_size)) != GRIB_SUCCESS) return err; - if((err = grib_get_long_internal(a->parent->h,self->szip_pixels_per_block, &szip_pixels_per_block)) != GRIB_SUCCESS) - return err; - if((err = grib_get_long_internal(a->parent->h,self->szip_pixels_per_scanline, &szip_pixels_per_scanline)) != GRIB_SUCCESS) + if((err = grib_get_long_internal(a->parent->h,self->ccsds_rsi, &ccsds_rsi)) != GRIB_SUCCESS) return err; @@ -265,11 +255,13 @@ static int unpack_double(grib_accessor* a, double* val, size_t *len) buf = (unsigned char*)a->parent->h->buffer->data; buf += grib_byte_offset(a); - memset(&sz_options, 0, sizeof(sz_options)); - sz_options.options_mask = szip_options_mask; - sz_options.bits_per_pixel = szip_bits_per_pixel; - sz_options.pixels_per_block = szip_pixels_per_block; - sz_options.pixels_per_scanline = szip_pixels_per_scanline; + strm.flags = ccsds_flags; + strm.bits_per_sample = bits_per_value; + strm.block_size = ccsds_block_size; + strm.rsi = ccsds_rsi; + + strm.next_in = buf; + strm.avail_in = buflen; /* printf("sz_options.options_mask %d\n", sz_options.options_mask); @@ -278,18 +270,19 @@ static int unpack_double(grib_accessor* a, double* val, size_t *len) printf("sz_options.pixels_per_scanline %d\n", sz_options.pixels_per_scanline); */ - bits8 = ((szip_bits_per_pixel + 7)/8)*8; - size = n_vals * (szip_bits_per_pixel + 7)/8; + bits8 = ((bits_per_value + 7)/8)*8; + size = n_vals * (bits_per_value + 7)/8; decoded = grib_context_buffer_malloc_clear(a->parent->h->context,size); if(!decoded) { err = GRIB_OUT_OF_MEMORY; goto cleanup; } + strm.next_out = decoded; + strm.avail_out = size; - - if((err = SZ_BufftoBuffDecompress( decoded,&size,buf, buflen , &sz_options)) != SZ_OK) + if((err = aec_buffer_decode(&strm)) != AEC_OK) { - printf("SZ_BufftoBuffDecompress Error %d\n", err); + printf("aec_buffer_decode Error %d\n", err); err = GRIB_ENCODING_ERROR; goto cleanup; } @@ -315,7 +308,7 @@ cleanup: static int pack_double(grib_accessor* a, const double* val, size_t *len) { - grib_accessor_data_szip_packing *self =(grib_accessor_data_szip_packing*)a; + grib_accessor_data_ccsds_packing *self =(grib_accessor_data_ccsds_packing*)a; int err = GRIB_SUCCESS; int i; @@ -338,19 +331,13 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len) unsigned char *p; double divisor; - long ni,nj; - long scanning_mode; - long list_defining_points; long number_of_data_points; - long szip_options_mask ; - long szip_bits_per_pixel ; - long szip_pixels_per_block ; - long szip_pixels_per_scanline ; + long ccsds_flags; + long ccsds_block_size; + long ccsds_rsi; - size_t width, height; - - SZ_com_t sz_options; + struct aec_stream strm; self->dirty=1; @@ -367,13 +354,11 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len) return err; - if((err = grib_get_long_internal(a->parent->h,self->szip_options_mask,&szip_options_mask)) != GRIB_SUCCESS) + if((err = grib_get_long_internal(a->parent->h,self->ccsds_flags,&ccsds_flags)) != GRIB_SUCCESS) return err; - if((err = grib_get_long_internal(a->parent->h,self->szip_bits_per_pixel, &szip_bits_per_pixel)) != GRIB_SUCCESS) + if((err = grib_get_long_internal(a->parent->h,self->ccsds_block_size, &ccsds_block_size)) != GRIB_SUCCESS) return err; - if((err = grib_get_long_internal(a->parent->h,self->szip_pixels_per_block, &szip_pixels_per_block)) != GRIB_SUCCESS) - return err; - if((err = grib_get_long_internal(a->parent->h,self->szip_pixels_per_scanline, &szip_pixels_per_scanline)) != GRIB_SUCCESS) + if((err = grib_get_long_internal(a->parent->h,self->ccsds_rsi, &ccsds_rsi)) != GRIB_SUCCESS) return err; @@ -470,7 +455,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len) /* buflen = n_vals*(bits_per_value/8);*/ grib_context_log(a->parent->h->context, GRIB_LOG_DEBUG, - "grib_accessor_data_szip_packing : pack_double : packing %s, %d values", a->name, n_vals); + "grib_accessor_data_ccsds_packing : pack_double : packing %s, %d values", a->name, n_vals); buflen += 10240; buf = grib_context_buffer_malloc_clear(a->parent->h->context,buflen); @@ -497,11 +482,15 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len) return err; - memset(&sz_options, 0, sizeof(sz_options)); - sz_options.options_mask = szip_options_mask; - sz_options.bits_per_pixel = bits_per_value; - sz_options.pixels_per_block = szip_pixels_per_block; - sz_options.pixels_per_scanline = szip_pixels_per_scanline; + strm.flags = ccsds_flags; + strm.bits_per_sample = bits_per_value; + strm.block_size = ccsds_block_size; + strm.rsi = ccsds_rsi; + + strm.next_out = buf; + strm.avail_out = buflen; + strm.next_in = encoded; + strm.avail_in = bits8/8*n_vals; /* @@ -516,9 +505,9 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len) printf("sz_options.pixels_per_scanline %d\n", sz_options.pixels_per_scanline); */ - if((err = SZ_BufftoBuffCompress(buf,&buflen,encoded, bits8/8*n_vals, &sz_options)) != SZ_OK) + if((err = aec_buffer_encode(&strm)) != AEC_OK) { - printf("SZ_BufftoBuffCompress Error %d\n", err); + printf("aec_buffer_encode Error %d\n", err); err = GRIB_ENCODING_ERROR; goto cleanup; } @@ -528,7 +517,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len) printf("n_vals = %ld, bits8 = %ld\n", (long)n_vals, (long)bits8); printf("in %ld out => %ld\n", (long)bits8/8*n_vals,(long) buflen); */ - + buflen = strm.total_out; grib_buffer_replace(a, buf, buflen ,1,1); cleanup: @@ -539,7 +528,7 @@ cleanup: err = grib_set_long_internal(a->parent->h,self->number_of_values, *len); if(err == GRIB_SUCCESS) - err = grib_set_long_internal(a->parent->h,self->szip_bits_per_pixel, sz_options.bits_per_pixel); + err = grib_set_long_internal(a->parent->h,self->bits_per_value, strm.bits_per_sample); return err; } @@ -548,14 +537,14 @@ cleanup: static int unpack_double(grib_accessor* a, double* val, size_t *len) { grib_context_log(a->parent->h->context, GRIB_LOG_ERROR, - "grib_accessor_data_szip_packing : szip support no enabled. Please rerun configure with --with-szip-support"); + "grib_accessor_data_ccsds_packing : ccsds support no enabled. Please rerun configure with --with-aec"); return GRIB_NOT_IMPLEMENTED; } static int pack_double(grib_accessor* a, const double* val, size_t *len) { grib_context_log(a->parent->h->context, GRIB_LOG_ERROR, - "grib_accessor_data_szip_packing : szip support no enabled. Please rerun configure with --with-szip-support"); + "grib_accessor_data_ccsds_packing : ccsds support no enabled. Please rerun configure with --with-aec"); return GRIB_NOT_IMPLEMENTED; } diff --git a/src/grib_accessor_factory.h b/src/grib_accessor_factory.h index f948b7b7b..1676c4402 100644 --- a/src/grib_accessor_factory.h +++ b/src/grib_accessor_factory.h @@ -63,7 +63,7 @@ { "data_sh_unpacked", &grib_accessor_class_data_sh_unpacked, }, { "data_shsimple_packing", &grib_accessor_class_data_shsimple_packing, }, { "data_simple_packing", &grib_accessor_class_data_simple_packing, }, -{ "data_szip_packing", &grib_accessor_class_data_szip_packing, }, +{ "data_ccsds_packing", &grib_accessor_class_data_ccsds_packing, }, { "decimal_precision", &grib_accessor_class_decimal_precision, }, { "dictionary", &grib_accessor_class_dictionary, }, { "dirty", &grib_accessor_class_dirty, }, diff --git a/src/grib_accessor_factory_hash_list b/src/grib_accessor_factory_hash_list index be5a4a10e..b96763384 100644 --- a/src/grib_accessor_factory_hash_list +++ b/src/grib_accessor_factory_hash_list @@ -67,7 +67,7 @@ data_sh_packed, &grib_accessor_class_data_sh_packed data_sh_unpacked, &grib_accessor_class_data_sh_unpacked data_shsimple_packing, &grib_accessor_class_data_shsimple_packing data_simple_packing, &grib_accessor_class_data_simple_packing -data_szip_packing, &grib_accessor_class_data_szip_packing +data_ccsds_packing, &grib_accessor_class_data_ccsds_packing decimal_precision, &grib_accessor_class_decimal_precision descriptors, &grib_accessor_class_descriptors dictionary, &grib_accessor_class_dictionary diff --git a/src/grib_api_prototypes.h b/src/grib_api_prototypes.h index 64df80a07..271f3a02d 100644 --- a/src/grib_api_prototypes.h +++ b/src/grib_api_prototypes.h @@ -472,8 +472,6 @@ int grib_get_g1_message_size(grib_handle *h, grib_accessor *tl, grib_accessor *s /* grib_accessor_class_data_simple_packing.c */ -/* grib_accessor_class_data_szip_packing.c */ - /* grib_accessor_class_count_missing.c */ /* grib_accessor_class_data_sh_packed.c */ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 06733bf11..5b960ef9b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -100,3 +100,11 @@ ecbuild_add_test( TARGET t_jpeg COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/jpeg.sh RESOURCES include.sh ) + +ecbuild_add_test( TARGET t_ccsds + TYPE SCRIPT + CONDITION ENABLE_AEC + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/ccsds.sh + RESOURCES include.sh +) + diff --git a/tests/Makefile.am b/tests/Makefile.am index 4c591e218..e1f99a36b 100755 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -10,12 +10,13 @@ TESTS = definitions.sh \ bitmap.sh list.sh second_order.sh \ multi_from_message.sh change_scanning.sh \ julian.sh statistics.sh tigge.sh tigge_conversions.sh \ - read_any.sh padding.sh lamb_az_eq_area.sh grib_to_netcdf.sh \ - debug.sh $(JPEG_TEST) + read_any.sh padding.sh lamb_az_eq_area.sh grib_to_netcdf.sh debug.sh \ + $(JPEG_TEST) $(CCSDS_TEST) noinst_PROGRAMS = packing_check gauss_sub read_any double_cmp packing pack_unpack \ multi_from_message julian read_index index gribex_perf\ - jpeg_perf so_perf png_perf bpv_limit laplacian unit_tests + jpeg_perf ccsds_perf so_perf png_perf bpv_limit laplacian \ + unit_tests multi_from_message_SOURCES = multi_from_message.c laplacian_SOURCES = laplacian.c @@ -31,6 +32,7 @@ read_index_SOURCES = read_index.c jpeg_perf_SOURCES = jpeg_perf.c so_perf_SOURCES = so_perf.c png_perf_SOURCES = png_perf.c +ccsds_perf_SOURCES = ccsds_perf.c gribex_perf_SOURCES = gribex_perf.c gauss_sub_SOURCES = gauss_sub.c diff --git a/tests/ccsds_perf.c b/tests/ccsds_perf.c new file mode 100644 index 000000000..56e06612e --- /dev/null +++ b/tests/ccsds_perf.c @@ -0,0 +1,246 @@ +/* + * Copyright 2005-2014 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * + * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by + * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. + */ + +#include "grib_api_internal.h" + +#if GRIB_TIMER + +void usage(char* prog) { + printf("usage: %s [-a outfile | -w outfile ] grib_file repetitions bitsPerValue\n",prog); + exit(1); +} + +size_t grib_handle_write(grib_handle* h,char* filename) { + FILE* of=NULL; + const void *buffer; size_t size; + + of = fopen(filename,"w"); + if(!of) { + perror(filename); + exit(1); + } + GRIB_CHECK(grib_get_message(h,&buffer,&size),0); + if(fwrite(buffer,1,size,of) != size) { + perror(filename); + exit(1); + } + fclose(of); + + return size; + +} + +static void print_timer(grib_timer* t,int repeat) { + printf("%s : %g cpu\n",t->name_,t->timer_/repeat); +} + +int main(int argc, char* argv[]) { + grib_handle *h=NULL; + grib_context* c=NULL; + FILE* fin=NULL; + FILE* fout=NULL; + char* finname; + char* ofilename; + char defofilename[]="ccsds_perf.out"; + double *values=NULL; + int append=0; + size_t nvalues=0; + int count,e=0; + int repeatccsds=1; + int repeatsimple=1; + grib_timer *tes,*tds,*tej,*tdj; + char grid_ccsds[]="grid_ccsds"; + size_t grid_ccsds_l=strlen(grid_ccsds); + char grid_simple[]="grid_simple"; + size_t grid_simple_l=strlen(grid_simple); + char packingType[50]={0,}; + size_t len=50; + char param[50]={0,}; + char gridType[50]={0,}; + char outfilename[255]={0,}; + size_t filesize_ccsds=0; + size_t filesize_simple=0; + double perc=0; + long bitsPerValue=0; + int iarg=1; + char grid[20]={0,}; + char shortName[20]={0,}; + long level; + char levelType[20]={0,}; + + tes=grib_get_timer(0,"encoding simple", 0, 0); + tds=grib_get_timer(0,"decoding simple", 0, 0); + tej=grib_get_timer(0,"encoding ccsds", 0, 0); + tdj=grib_get_timer(0,"decoding ccsds", 0, 0); + + if (argc != 4 && argc != 6 ) usage(argv[0]); + if (!strcmp(argv[iarg],"-w")) { + append=0; + iarg++; + ofilename=argv[iarg]; + iarg++; + } else if (!strcmp(argv[iarg],"-a")) { + append=1; + iarg++; + ofilename=argv[iarg]; + iarg++; + } else { + append=0; + ofilename=defofilename; + } + finname=argv[iarg++]; + repeatsimple=atoi(argv[iarg++]); + bitsPerValue=atoi(argv[iarg++]); + + fin = fopen(finname,"r"); + if(!fin) {perror(finname);exit(1);} + + if (append) + fout = fopen(ofilename,"a"); + else + fout = fopen(ofilename,"w"); + + if(!fout) {perror(ofilename);exit(1);} + + c=grib_context_get_default(); + e=0; + h=grib_handle_new_from_file(c,fin,&e); + fclose(fin); + + GRIB_CHECK(e,0); + + len=50; + grib_get_string(h,"shortName",param,&len); + + len=20; + grib_get_string(h,"levelType",levelType,&len); + + if (!strcmp(levelType,"pl")) { + GRIB_CHECK(grib_get_long(h,"level",&level),0); + sprintf(shortName,"%s%ld",param,level); + } else { + sprintf(shortName,"%s",param); + } + + + grib_set_long(h,"editionNumber",2); + GRIB_CHECK(grib_get_size(h,"values",&nvalues),0); + values=(double*)grib_context_malloc(c,sizeof(double)*nvalues); + if (!values) { printf("%s: memory allocation error\n",argv[0]); exit(1); } + + len=50; + grib_get_string(h,"gridType",gridType,&len); + + len=50; + grib_get_string(h,"packingType",packingType,&len); + + GRIB_CHECK(grib_get_double_array(h,"values",values,&nvalues),0); + grib_set_long(h,"bitsPerValue",bitsPerValue); + GRIB_CHECK(grib_set_double_array(h,"values",values,nvalues),0); + + printf("--------------------------------\n"); + printf("- %s - gridType=%s packingType=%s numberOfValues=%ld bitsPerValue=%ld\n", + param,gridType,packingType,(long)nvalues,bitsPerValue); + + if (!strcmp(packingType,"spectral_complex") || !strcmp(packingType,"spectral_simple")) { + printf("unable to process spectral data\n"); + exit(1); + } + + if (!strcmp(gridType,"reduced_gg") || !strcmp(gridType,"regular_gg")) { + long N; + grib_get_long(h,"N",&N); + printf(" N=%ld\n",N); + sprintf(grid,"%ld",N); + } + if (!strcmp(gridType,"regular_ll")) { + double Di,Dj; + grib_get_double(h,"DiInDegrees",&Di); + grib_get_double(h,"DjInDegrees",&Dj); + printf(" Di=%g Dj=%g\n",Di,Dj); + sprintf(grid,"%g/%g",Di,Dj); + } + + + if (!append) + fprintf(fout, + "shortName gridType numberOfValues bitsPerValue grid sizeSimple sizeccsds encodeccsds encodeSimple decodeccsds decodeSimple\n"); + + /* decode values grid_simple */ + if (strcmp(packingType,grid_simple)) + grib_set_string(h,"packingType",grid_simple,&grid_simple_l); + /* printf("decoding simple\n"); */ + grib_timer_start(tds); + for (count=0;counttimer_/repeatccsds,tes->timer_/repeatsimple,tdj->timer_/repeatccsds,tds->timer_/repeatsimple); + + return 0; +} +#else + +int main(int argc,char* argv[]) { +return 0; +} + +#endif diff --git a/windows/msvc/grib_api_lib/grib_api_lib.vcproj b/windows/msvc/grib_api_lib/grib_api_lib.vcproj index 23449bf63..78c03f61f 100644 --- a/windows/msvc/grib_api_lib/grib_api_lib.vcproj +++ b/windows/msvc/grib_api_lib/grib_api_lib.vcproj @@ -689,7 +689,7 @@ >