GRIB-179: Data representation template 5.42 (CCSDS)

.gitignore

@@ -113,6 +113,7 @@ tests/read_any
 tests/read_index
 tests/so_perf
 tests/unit_tests
+tests/ccsds_perf
 tigge/.libs/
 tigge/tigge_accumulations
 tigge/tigge_check

CMakeLists.txt

@@ -32,7 +32,7 @@ ecbuild_declare_project()
 option( ENABLE_JPG     "try to add support for JPG encoding"          ON )
 option( ENABLE_PNG     "try to add support for PNG encoding"          OFF )
 option( ENABLE_NETCDF  "try to add support for NetCDF"                ON )
-option( ENABLE_SZIP    "try to add support for SZip encoding"         OFF )
+option( ENABLE_AEC     "try to add support for Adaptive Entropy Coding" OFF )
 option( ENABLE_PYTHON  "try to build the GRIB_API Python interface"   ON )
 option( ENABLE_FORTRAN "try to build the GRIB_API Fortran interface"  ON )
 
@@ -55,15 +55,15 @@ set( GRIB_API_EXTRA_DEFINITIONS  "" )
 
 find_package( CMath )
 
-### SZIP support
+### AEC support (Adaptive Entropy Coding)
 
-set( HAVE_LIBSZIP 0 )
+set( HAVE_LIBAEC 0 )
-if( ENABLE_SZIP )
+if( ENABLE_AEC )
 
-    find_package( SZip )
+    find_package( AEC )
 
-    if( SZIP_FOUND )
+    if( AEC_FOUND )
-        set( HAVE_LIBSZIP 1 )
+        set( HAVE_LIBAEC 1 )
     endif()
 endif()
 
@@ -198,7 +198,7 @@ endif()
 
 ### export package to other ecbuild packages
 
-set( GRIB_API_TPLS            SZip PNG NetCDF Jasper OpenJPEG CMath )
+set( GRIB_API_TPLS            AEC PNG NetCDF Jasper OpenJPEG CMath )
 set( GRIB_API_INCLUDE_DIRS    ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}/src )
 set( GRIB_API_LIBRARIES       grib_api )
 

NOTICE

@@ -1,11 +1,12 @@
 ECMWF GRIB API
 Copyright 2005-2014 ECMWF.
 
-This product includes software developed at
+This product includes software developed at ECMWF (http://www.ecmwf.int).
-ECMWF (http://www.ecmwf.int).
 
-Parts of the definitions provided by
+Parts of the definitions provided by WMO (http://www.wmo.int/pages/index_en.html)
-WMO (http://www.wmo.int/pages/index_en.html)
+
+Adaptive Entropy Coding (CCSDS) provided by
+The Max-Planck-Institute for Meteorology and Deutsches Klimarechenzentrum GmbH
 
 IBM POWER optimisations were developed at
 The Max-Planck-Institute for Meteorology.

configure.ac

@@ -455,6 +455,23 @@ then
   AC_SUBST(JPEG_TEST)
 fi
 
+dnl check for libaec
+CCSDS_TEST=""
+AC_ARG_WITH([aec],
+            [AS_HELP_STRING([--with-aec=DIR],
+                            [use specified libaec installation directory])],
+            ,[with_aec=no])
+
+if test "x$with_aec" != xno ; then
+  if test "x$with_aec" != xyes ; then
+    LDFLAGS="$LDFLAGS -L$with_aec/lib"
+    CPPFLAGS="$CPPFLAGS -I$with_aec/include"
+  fi
+  AC_CHECK_LIB(aec, aec_encode, , [AC_MSG_FAILURE(
+                 [aec test failed (--without-aec to disable)])])
+  CCSDS_TEST="ccsds.sh"
+fi
+AC_SUBST(CCSDS_TEST)
 
 dnl Check for png
 AC_ARG_WITH([png-support],

data/CMakeLists.txt

@@ -86,6 +86,7 @@ set(files_to_download
     timeRangeIndicator_5.grib
     tp_ecmwf.grib
     v.grib2
+    ccsds.grib2
 )
 
 foreach( f ${files_to_download} )

data/download.sh

@@ -123,6 +123,7 @@ files="
     timeRangeIndicator_5.grib
     tp_ecmwf.grib
     v.grib2
+    ccsds.grib2
     tigge/tigge_ammc_pl_gh.grib
     tigge/tigge_ammc_pl_q.grib
     tigge/tigge_ammc_pl_t.grib

definitions/grib1/section.4.def

@@ -126,7 +126,7 @@ concept packingType {
 									plusOneinOrdersOfSPD=1; twoOrdersOfSPD=1; }
   "grid_jpeg"                         = { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;}
   "grid_png"                          = { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;}
-  "grid_szip"                          = { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;}
+  "grid_ccsds"                        = { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;}
   "grid_simple_log_preprocessing"= { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;}
 } : dump;
 

definitions/grib2/section.5.def

@@ -45,7 +45,7 @@ concept packingType (unknown) {
   "grid_jpeg"                             = { dataRepresentationTemplateNumber = 40;    }
   "grid_png"                              = { dataRepresentationTemplateNumber = 40010; }
   "grid_png"                              = { dataRepresentationTemplateNumber = 41;    }
-  "grid_szip"                             = { dataRepresentationTemplateNumber = 42; }
+  "grid_ccsds"                            = { dataRepresentationTemplateNumber = 42; }
   "grid_ieee"                             = { dataRepresentationTemplateNumber = 4;     }
   "grid_second_order"                     = { dataRepresentationTemplateNumber = 50001; }
   "grid_second_order"                     = { dataRepresentationTemplateNumber = 50002; }

definitions/grib2/template.5.42.def

@@ -7,22 +7,19 @@
 # virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
 #
 
-# TEMPLATE 5.42, Grid point and spectral data - CCSDS szip
+# TEMPLATE 5.42, Grid point and spectral data - CCSDS
 
 include "template.5.packing.def";
 include "template.5.original_values.def";
 
-unsigned[1] szipOptionsMask : dump;
+unsigned[1] ccsdsFlags : dump;
 
-flagbit SZ_ALLOW_K13_OPTION_MASK(szipOptionsMask,0) = 0;
+flagbit AEC_DATA_SIGNED_OPTION_MASK(ccsdsFlags,0)     = 0;
-flagbit SZ_CHIP_OPTION_MASK(szipOptionsMask,1)      = 0;
+flagbit AEC_DATA_3BYTE_OPTION_MASK(ccsdsFlags,1)      = 0;
-flagbit SZ_EC_OPTION_MASK(szipOptionsMask,2)        = 0;
+flagbit AEC_DATA_MSB_OPTION_MASK(ccsdsFlags,2)        = 1;
-flagbit SZ_LSB_OPTION_MASK(szipOptionsMask,3)       = 0;
+flagbit AEC_DATA_PREPROCESS_OPTION_MASK(ccsdsFlags,3) = 1;
-flagbit SZ_MSB_OPTION_MASK(szipOptionsMask,4)       = 1;
+flagbit AEC_RESTRICTED_OPTION_MASK(ccsdsFlags,4)      = 0;
-flagbit SZ_NN_OPTION_MASK(szipOptionsMask,5)        = 1;
+flagbit AEC_PAD_RSI_OPTION_MASK(ccsdsFlags,5)         = 0;
-flagbit SZ_RAW_OPTION_MASK(szipOptionsMask,7)       = 1;
-
-unsigned[1] szipBitsPerPixel           : dump;
-unsigned[2] szipPixelPerBlock    = 32  : dump;
-unsigned[2] szipPixelPerScanLine = 128 * szipPixelPerBlock : dump;
 
+unsigned[1] ccsdsBlockSize = 32 : dump;
+unsigned[2] ccsdsRsi = 128      : dump;

definitions/grib2/template.7.42.def

@@ -9,9 +9,9 @@
 
 
 # START 2/template.7.42 ----------------------------------------------------------------------
-# TEMPLATE 7.42, Grid point data - CCSDS szip
+# TEMPLATE 7.42, Grid point data - CCSDS
 
-    meta codedValues data_szip_packing(
+    meta codedValues data_ccsds_packing(
               section7Length,
               offsetBeforeData,
               offsetSection7,
@@ -23,10 +23,9 @@
 
               numberOfDataPoints,
 
-              szipOptionsMask,
+              ccsdsFlags,
-              szipBitsPerPixel,
+              ccsdsBlockSize,
-              szipPixelPerBlock,
+              ccsdsRsi
-              szipPixelPerScanLine
 
             ): read_only;
 

grib_api_config.h.in

@@ -93,7 +93,7 @@
 
 #define HAVE_LIBPNG  @HAVE_LIBPNG@
 
-#cmakedefine HAVE_LIBSZIP
+#cmakedefine HAVE_LIBAEC
 
 #cmakedefine HAVE_NETCDF
 

src/CMakeLists.txt

@@ -203,7 +203,7 @@ list( APPEND grib_api_srcs
     grib_accessor_class_data_g2secondary_bitmap.c
     grib_accessor_class_data_jpeg2000_packing.c
     grib_accessor_class_data_png_packing.c
-    grib_accessor_class_data_szip_packing.c
+    grib_accessor_class_data_ccsds_packing.c
     grib_accessor_class_data_raw_packing.c
     grib_accessor_class_data_complex_packing.c
     grib_accessor_class_data_g1complex_packing.c

src/Makefile.am

@@ -177,7 +177,7 @@ libgrib_api_la_prototypes= \
 	grib_accessor_class_values.c \
 	grib_accessor_class_simple_packing_error.c \
 	grib_accessor_class_data_simple_packing.c \
-	grib_accessor_class_data_szip_packing.c \
+	grib_accessor_class_data_ccsds_packing.c \
 	grib_accessor_class_count_missing.c \
 	grib_accessor_class_data_sh_packed.c \
 	grib_accessor_class_data_sh_unpacked.c \

src/grib_accessor_class.h

@@ -63,7 +63,7 @@ extern grib_accessor_class* grib_accessor_class_data_sh_packed;
 extern grib_accessor_class* grib_accessor_class_data_sh_unpacked;
 extern grib_accessor_class* grib_accessor_class_data_shsimple_packing;
 extern grib_accessor_class* grib_accessor_class_data_simple_packing;
-extern grib_accessor_class* grib_accessor_class_data_szip_packing;
+extern grib_accessor_class* grib_accessor_class_data_ccsds_packing;
 extern grib_accessor_class* grib_accessor_class_decimal_precision;
 extern grib_accessor_class* grib_accessor_class_dictionary;
 extern grib_accessor_class* grib_accessor_class_dirty;

src/grib_accessor_class_data_ccsds_packing.c

@@ -28,10 +28,9 @@
 
    MEMBERS=const char*   number_of_data_points
 
-   MEMBERS=const char*   szip_options_mask
+   MEMBERS=const char*   ccsds_flags
-   MEMBERS=const char*   szip_bits_per_pixel
+   MEMBERS=const char*   ccsds_block_size
-   MEMBERS=const char*   szip_pixels_per_block
+   MEMBERS=const char*   ccsds_rsi
-   MEMBERS=const char*   szip_pixels_per_scanline
 
    END_CLASS_DEF
 
@@ -49,11 +48,11 @@ or edit "accessor.class" and rerun ./make_class.pl
 
 static int pack_double(grib_accessor*, const double* val,size_t *len);
 static int unpack_double(grib_accessor*, double* val,size_t *len);
-static int value_count(grib_accessor*,long*);
+static int value_count(grib_accessor*, long*);
 static void init(grib_accessor*,const long, grib_arguments* );
 static void init_class(grib_accessor_class*);
 
-typedef struct grib_accessor_data_szip_packing {
+typedef struct grib_accessor_data_ccsds_packing {
     grib_accessor          att;
 /* Members defined in gen */
 /* Members defined in values */
@@ -62,25 +61,24 @@ typedef struct grib_accessor_data_szip_packing {
 	const char* offsetdata;
 	const char* offsetsection;
 	int dirty;
-/* Members defined in data_szip_packing */
+/* Members defined in data_ccsds_packing */
 	const char*   number_of_values;
 	const char*   reference_value;
 	const char*   binary_scale_factor;
 	const char*   decimal_scale_factor;
 	const char*   bits_per_value;
 	const char*   number_of_data_points;
-	const char*   szip_options_mask;
+	const char*   ccsds_flags;
-	const char*   szip_bits_per_pixel;
+	const char*   ccsds_block_size;
-	const char*   szip_pixels_per_block;
+	const char*   ccsds_rsi;
-	const char*   szip_pixels_per_scanline;
+} grib_accessor_data_ccsds_packing;
-} grib_accessor_data_szip_packing;
 
 extern grib_accessor_class* grib_accessor_class_values;
 
-static grib_accessor_class _grib_accessor_class_data_szip_packing = {
+static grib_accessor_class _grib_accessor_class_data_ccsds_packing = {
     &grib_accessor_class_values,                      /* super                     */
-    "data_szip_packing",                      /* name                      */
+    "data_ccsds_packing",                      /* name                      */
-    sizeof(grib_accessor_data_szip_packing),  /* size                      */
+    sizeof(grib_accessor_data_ccsds_packing),  /* size                      */
     0,                           /* inited */
     &init_class,                 /* init_class */
     &init,                       /* init                      */
@@ -120,7 +118,7 @@ static grib_accessor_class _grib_accessor_class_data_szip_packing = {
 };
 
 
-grib_accessor_class* grib_accessor_class_data_szip_packing = &_grib_accessor_class_data_szip_packing;
+grib_accessor_class* grib_accessor_class_data_ccsds_packing = &_grib_accessor_class_data_ccsds_packing;
 
 
 static void init_class(grib_accessor_class* c)
@@ -138,8 +136,6 @@ static void init_class(grib_accessor_class* c)
 	c->unpack_long	=	(*(c->super))->unpack_long;
 	c->pack_string	=	(*(c->super))->pack_string;
 	c->unpack_string	=	(*(c->super))->unpack_string;
-	c->pack_string_array	=	(*(c->super))->pack_string_array;
-	c->unpack_string_array	=	(*(c->super))->unpack_string_array;
 	c->pack_bytes	=	(*(c->super))->pack_bytes;
 	c->unpack_bytes	=	(*(c->super))->unpack_bytes;
 	c->pack_expression	=	(*(c->super))->pack_expression;
@@ -159,7 +155,7 @@ static void init_class(grib_accessor_class* c)
 
 static void init(grib_accessor* a,const long v, grib_arguments* args)
 {
-    grib_accessor_data_szip_packing *self =(grib_accessor_data_szip_packing*)a;
+    grib_accessor_data_ccsds_packing *self =(grib_accessor_data_ccsds_packing*)a;
 
     self->number_of_values      = grib_arguments_get_name(a->parent->h,args,self->carg++);
     self->reference_value       = grib_arguments_get_name(a->parent->h,args,self->carg++);
@@ -169,36 +165,35 @@ static void init(grib_accessor* a,const long v, grib_arguments* args)
 
     self->number_of_data_points  = grib_arguments_get_name(a->parent->h,args,self->carg++);
 
-
+    self->ccsds_flags  = grib_arguments_get_name(a->parent->h,args,self->carg++);
-    self->szip_options_mask  = grib_arguments_get_name(a->parent->h,args,self->carg++);
+    self->ccsds_block_size  = grib_arguments_get_name(a->parent->h,args,self->carg++);
-    self->szip_bits_per_pixel  = grib_arguments_get_name(a->parent->h,args,self->carg++);
+    self->ccsds_rsi  = grib_arguments_get_name(a->parent->h,args,self->carg++);
-    self->szip_pixels_per_block  = grib_arguments_get_name(a->parent->h,args,self->carg++);
-    self->szip_pixels_per_scanline  = grib_arguments_get_name(a->parent->h,args,self->carg++);
 
     a->flags |= GRIB_ACCESSOR_FLAG_DATA;
+
 }
 
 static int value_count(grib_accessor* a, long* count)
 {
-    grib_accessor_data_szip_packing *self =(grib_accessor_data_szip_packing*)a;
+    grib_accessor_data_ccsds_packing *self =(grib_accessor_data_ccsds_packing*)a;
     *count = 0;
     return grib_get_long_internal(a->parent->h,self->number_of_values,count);
 }
 
-#ifdef HAVE_LIBSZIP
+#ifdef HAVE_LIBAEC
 
-#include "szlib.h"
+#include <libaec.h>
 
 
 static int  unpack_double(grib_accessor* a, double* val, size_t *len)
 {
 
-    grib_accessor_data_szip_packing *self =(grib_accessor_data_szip_packing*)a;
+    grib_accessor_data_ccsds_packing *self =(grib_accessor_data_ccsds_packing*)a;
 
     int err = GRIB_SUCCESS;
-    int i,j;
+    int i;
     size_t buflen = grib_byte_count(a);
-    SZ_com_t sz_options;
+    struct aec_stream strm;
     double bscale = 0;
     double dscale = 0;
     unsigned char* buf = NULL;
@@ -214,11 +209,9 @@ static int  unpack_double(grib_accessor* a, double* val, size_t *len)
     long bits_per_value =0;
     long bits8;
 
-
+    long ccsds_flags;
-    long szip_options_mask ;
+    long ccsds_block_size;
-    long szip_bits_per_pixel ;
+    long ccsds_rsi;
-    long szip_pixels_per_block  ;
-    long szip_pixels_per_scanline  ;
 
     self->dirty=0;
 
@@ -234,14 +227,11 @@ static int  unpack_double(grib_accessor* a, double* val, size_t *len)
     if((err = grib_get_long_internal(a->parent->h,self->decimal_scale_factor, &decimal_scale_factor)) != GRIB_SUCCESS)
         return err;
 
-
+    if((err = grib_get_long_internal(a->parent->h,self->ccsds_flags,&ccsds_flags)) != GRIB_SUCCESS)
-    if((err = grib_get_long_internal(a->parent->h,self->szip_options_mask,&szip_options_mask)) != GRIB_SUCCESS)
         return err;
-    if((err = grib_get_long_internal(a->parent->h,self->szip_bits_per_pixel, &szip_bits_per_pixel)) != GRIB_SUCCESS)
+    if((err = grib_get_long_internal(a->parent->h,self->ccsds_block_size, &ccsds_block_size)) != GRIB_SUCCESS)
         return err;
-    if((err = grib_get_long_internal(a->parent->h,self->szip_pixels_per_block, &szip_pixels_per_block)) != GRIB_SUCCESS)
+    if((err = grib_get_long_internal(a->parent->h,self->ccsds_rsi, &ccsds_rsi)) != GRIB_SUCCESS)
-        return err;
-    if((err = grib_get_long_internal(a->parent->h,self->szip_pixels_per_scanline, &szip_pixels_per_scanline)) != GRIB_SUCCESS)
         return err;
 
 
@@ -265,11 +255,13 @@ static int  unpack_double(grib_accessor* a, double* val, size_t *len)
     buf = (unsigned char*)a->parent->h->buffer->data;
     buf += grib_byte_offset(a);
 
-    memset(&sz_options, 0, sizeof(sz_options));
+    strm.flags = ccsds_flags;
-    sz_options.options_mask         = szip_options_mask;
+    strm.bits_per_sample = bits_per_value;
-    sz_options.bits_per_pixel       = szip_bits_per_pixel;
+    strm.block_size = ccsds_block_size;
-    sz_options.pixels_per_block     = szip_pixels_per_block;
+    strm.rsi = ccsds_rsi;
-    sz_options.pixels_per_scanline  = szip_pixels_per_scanline;
+
+    strm.next_in = buf;
+    strm.avail_in = buflen;
 
 /*
     printf("sz_options.options_mask %d\n", sz_options.options_mask);
@@ -278,18 +270,19 @@ static int  unpack_double(grib_accessor* a, double* val, size_t *len)
     printf("sz_options.pixels_per_scanline %d\n", sz_options.pixels_per_scanline);
     */
 
-    bits8 = ((szip_bits_per_pixel + 7)/8)*8;
+    bits8 = ((bits_per_value + 7)/8)*8;
-    size = n_vals * (szip_bits_per_pixel + 7)/8;
+    size = n_vals * (bits_per_value + 7)/8;
     decoded = grib_context_buffer_malloc_clear(a->parent->h->context,size);
     if(!decoded) {
         err = GRIB_OUT_OF_MEMORY;
         goto cleanup;
     }
+    strm.next_out = decoded;
+    strm.avail_out = size;
 
-
+    if((err = aec_buffer_decode(&strm)) != AEC_OK)
-    if((err = SZ_BufftoBuffDecompress( decoded,&size,buf, buflen , &sz_options)) != SZ_OK)
     {
-        printf("SZ_BufftoBuffDecompress Error %d\n", err);
+        printf("aec_buffer_decode Error %d\n", err);
         err = GRIB_ENCODING_ERROR;
         goto cleanup;
     }
@@ -315,7 +308,7 @@ cleanup:
 
 static int pack_double(grib_accessor* a, const double* val, size_t *len)
 {
-    grib_accessor_data_szip_packing *self =(grib_accessor_data_szip_packing*)a;
+    grib_accessor_data_ccsds_packing *self =(grib_accessor_data_ccsds_packing*)a;
 
     int err = GRIB_SUCCESS;
     int i;
@@ -338,19 +331,13 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
     unsigned char *p;
     double divisor;
 
-    long ni,nj;
-    long scanning_mode;
-    long list_defining_points;
     long number_of_data_points;
 
-    long szip_options_mask ;
+    long ccsds_flags;
-    long szip_bits_per_pixel ;
+    long ccsds_block_size;
-    long szip_pixels_per_block  ;
+    long ccsds_rsi;
-    long szip_pixels_per_scanline  ;
 
-    size_t width, height;
+    struct aec_stream strm;
-
-    SZ_com_t sz_options;
 
     self->dirty=1;
 
@@ -367,13 +354,11 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
         return err;
 
 
-    if((err = grib_get_long_internal(a->parent->h,self->szip_options_mask,&szip_options_mask)) != GRIB_SUCCESS)
+    if((err = grib_get_long_internal(a->parent->h,self->ccsds_flags,&ccsds_flags)) != GRIB_SUCCESS)
         return err;
-    if((err = grib_get_long_internal(a->parent->h,self->szip_bits_per_pixel, &szip_bits_per_pixel)) != GRIB_SUCCESS)
+    if((err = grib_get_long_internal(a->parent->h,self->ccsds_block_size, &ccsds_block_size)) != GRIB_SUCCESS)
         return err;
-    if((err = grib_get_long_internal(a->parent->h,self->szip_pixels_per_block, &szip_pixels_per_block)) != GRIB_SUCCESS)
+    if((err = grib_get_long_internal(a->parent->h,self->ccsds_rsi, &ccsds_rsi)) != GRIB_SUCCESS)
-        return err;
-    if((err = grib_get_long_internal(a->parent->h,self->szip_pixels_per_scanline, &szip_pixels_per_scanline)) != GRIB_SUCCESS)
         return err;
 
 
@@ -470,7 +455,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
     /*       buflen = n_vals*(bits_per_value/8);*/
 
     grib_context_log(a->parent->h->context, GRIB_LOG_DEBUG,
-                     "grib_accessor_data_szip_packing : pack_double : packing %s, %d values", a->name, n_vals);
+                     "grib_accessor_data_ccsds_packing : pack_double : packing %s, %d values", a->name, n_vals);
 
     buflen += 10240;
     buf = grib_context_buffer_malloc_clear(a->parent->h->context,buflen);
@@ -497,11 +482,15 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
         return err;
 
 
-    memset(&sz_options, 0, sizeof(sz_options));
+    strm.flags = ccsds_flags;
-    sz_options.options_mask         = szip_options_mask;
+    strm.bits_per_sample = bits_per_value;
-    sz_options.bits_per_pixel       = bits_per_value;
+    strm.block_size = ccsds_block_size;
-    sz_options.pixels_per_block     = szip_pixels_per_block;
+    strm.rsi = ccsds_rsi;
-    sz_options.pixels_per_scanline  = szip_pixels_per_scanline;
+
+    strm.next_out = buf;
+    strm.avail_out = buflen;
+    strm.next_in = encoded;
+    strm.avail_in = bits8/8*n_vals;
 
     /*
 
@@ -516,9 +505,9 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
     printf("sz_options.pixels_per_scanline %d\n", sz_options.pixels_per_scanline);
     */
 
-    if((err = SZ_BufftoBuffCompress(buf,&buflen,encoded, bits8/8*n_vals, &sz_options)) != SZ_OK)
+    if((err = aec_buffer_encode(&strm)) != AEC_OK)
     {
-        printf("SZ_BufftoBuffCompress Error %d\n", err);
+        printf("aec_buffer_encode Error %d\n", err);
         err = GRIB_ENCODING_ERROR;
         goto cleanup;
     }
@@ -528,7 +517,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
     printf("n_vals = %ld, bits8 = %ld\n", (long)n_vals, (long)bits8);
     printf("in %ld out => %ld\n", (long)bits8/8*n_vals,(long) buflen);
     */
-
+    buflen = strm.total_out;
     grib_buffer_replace(a, buf, buflen ,1,1);
 
 cleanup:
@@ -539,7 +528,7 @@ cleanup:
         err = grib_set_long_internal(a->parent->h,self->number_of_values, *len);
 
     if(err == GRIB_SUCCESS)
-        err = grib_set_long_internal(a->parent->h,self->szip_bits_per_pixel, sz_options.bits_per_pixel);
+        err = grib_set_long_internal(a->parent->h,self->bits_per_value, strm.bits_per_sample);
 
     return err;
 }
@@ -548,14 +537,14 @@ cleanup:
 static int  unpack_double(grib_accessor* a, double* val, size_t *len)
 {
     grib_context_log(a->parent->h->context, GRIB_LOG_ERROR,
-                     "grib_accessor_data_szip_packing : szip support no enabled. Please rerun configure with --with-szip-support");
+                     "grib_accessor_data_ccsds_packing : ccsds support no enabled. Please rerun configure with --with-aec");
     return GRIB_NOT_IMPLEMENTED;
 }
 
 static int pack_double(grib_accessor* a, const double* val, size_t *len)
 {
     grib_context_log(a->parent->h->context, GRIB_LOG_ERROR,
-                     "grib_accessor_data_szip_packing : szip support no enabled. Please rerun configure with --with-szip-support");
+                     "grib_accessor_data_ccsds_packing : ccsds support no enabled. Please rerun configure with --with-aec");
     return GRIB_NOT_IMPLEMENTED;
 }
 

src/grib_accessor_factory.h

@@ -63,7 +63,7 @@
 { "data_sh_unpacked", &grib_accessor_class_data_sh_unpacked, },
 { "data_shsimple_packing", &grib_accessor_class_data_shsimple_packing, },
 { "data_simple_packing", &grib_accessor_class_data_simple_packing, },
-{ "data_szip_packing", &grib_accessor_class_data_szip_packing, },
+{ "data_ccsds_packing", &grib_accessor_class_data_ccsds_packing, },
 { "decimal_precision", &grib_accessor_class_decimal_precision, },
 { "dictionary", &grib_accessor_class_dictionary, },
 { "dirty", &grib_accessor_class_dirty, },

src/grib_accessor_factory_hash_list

@@ -67,7 +67,7 @@ data_sh_packed, &grib_accessor_class_data_sh_packed
 data_sh_unpacked, &grib_accessor_class_data_sh_unpacked
 data_shsimple_packing, &grib_accessor_class_data_shsimple_packing
 data_simple_packing, &grib_accessor_class_data_simple_packing
-data_szip_packing, &grib_accessor_class_data_szip_packing
+data_ccsds_packing, &grib_accessor_class_data_ccsds_packing
 decimal_precision, &grib_accessor_class_decimal_precision
 descriptors, &grib_accessor_class_descriptors
 dictionary, &grib_accessor_class_dictionary

src/grib_api_prototypes.h

@@ -472,8 +472,6 @@ int grib_get_g1_message_size(grib_handle *h, grib_accessor *tl, grib_accessor *s
 
 /* grib_accessor_class_data_simple_packing.c */
 
-/* grib_accessor_class_data_szip_packing.c */
-
 /* grib_accessor_class_count_missing.c */
 
 /* grib_accessor_class_data_sh_packed.c */

tests/CMakeLists.txt

@@ -100,3 +100,11 @@ ecbuild_add_test( TARGET t_jpeg
                   COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/jpeg.sh
                   RESOURCES include.sh
 )
+
+ecbuild_add_test( TARGET t_ccsds
+                  TYPE SCRIPT
+                  CONDITION ENABLE_AEC
+                  COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/ccsds.sh
+                  RESOURCES include.sh
+)
+

tests/Makefile.am

@@ -10,12 +10,13 @@ TESTS = definitions.sh \
         bitmap.sh list.sh second_order.sh \
         multi_from_message.sh change_scanning.sh \
         julian.sh statistics.sh tigge.sh tigge_conversions.sh \
-        read_any.sh padding.sh lamb_az_eq_area.sh grib_to_netcdf.sh \
+        read_any.sh padding.sh lamb_az_eq_area.sh grib_to_netcdf.sh debug.sh \
-        debug.sh $(JPEG_TEST)
+        $(JPEG_TEST) $(CCSDS_TEST)
 
 noinst_PROGRAMS = packing_check gauss_sub read_any double_cmp packing pack_unpack \
                   multi_from_message julian read_index index gribex_perf\
-                  jpeg_perf so_perf png_perf bpv_limit laplacian unit_tests
+                  jpeg_perf ccsds_perf so_perf png_perf bpv_limit laplacian \
+                  unit_tests
 
 multi_from_message_SOURCES = multi_from_message.c
 laplacian_SOURCES = laplacian.c
@@ -31,6 +32,7 @@ read_index_SOURCES = read_index.c
 jpeg_perf_SOURCES = jpeg_perf.c
 so_perf_SOURCES = so_perf.c
 png_perf_SOURCES = png_perf.c
+ccsds_perf_SOURCES  = ccsds_perf.c
 gribex_perf_SOURCES = gribex_perf.c
 gauss_sub_SOURCES = gauss_sub.c
 

tests/ccsds_perf.c

@@ -0,0 +1,246 @@
+/*
+ * Copyright 2005-2014 ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+ * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+ */
+
+#include "grib_api_internal.h"
+
+#if GRIB_TIMER
+
+void usage(char* prog) {
+  printf("usage: %s [-a outfile | -w outfile ] grib_file repetitions bitsPerValue\n",prog);
+  exit(1);
+}
+
+size_t grib_handle_write(grib_handle* h,char* filename) {
+  FILE* of=NULL;
+  const void *buffer; size_t size;
+
+  of = fopen(filename,"w");
+  if(!of) {
+      perror(filename);
+      exit(1);
+  }
+  GRIB_CHECK(grib_get_message(h,&buffer,&size),0);
+  if(fwrite(buffer,1,size,of) != size) {
+    perror(filename);
+    exit(1);
+  }
+  fclose(of);
+
+  return size;
+
+}
+
+static void print_timer(grib_timer* t,int repeat) {
+   printf("%s : %g cpu\n",t->name_,t->timer_/repeat);
+}
+
+int main(int argc, char* argv[]) {
+  grib_handle *h=NULL;
+  grib_context* c=NULL;
+  FILE* fin=NULL;
+  FILE* fout=NULL;
+  char* finname;
+  char* ofilename;
+  char defofilename[]="ccsds_perf.out";
+  double *values=NULL;
+  int append=0;
+  size_t nvalues=0;
+  int count,e=0;
+  int repeatccsds=1;
+  int repeatsimple=1;
+  grib_timer *tes,*tds,*tej,*tdj;
+  char grid_ccsds[]="grid_ccsds";
+  size_t grid_ccsds_l=strlen(grid_ccsds);
+  char grid_simple[]="grid_simple";
+  size_t grid_simple_l=strlen(grid_simple);
+  char packingType[50]={0,};
+  size_t len=50;
+  char param[50]={0,};
+  char gridType[50]={0,};
+  char outfilename[255]={0,};
+  size_t filesize_ccsds=0;
+  size_t filesize_simple=0;
+  double perc=0;
+  long bitsPerValue=0;
+  int iarg=1;
+  char grid[20]={0,};
+  char shortName[20]={0,};
+  long level;
+  char levelType[20]={0,};
+
+  tes=grib_get_timer(0,"encoding simple", 0, 0);
+  tds=grib_get_timer(0,"decoding simple", 0, 0);
+  tej=grib_get_timer(0,"encoding ccsds", 0, 0);
+  tdj=grib_get_timer(0,"decoding ccsds", 0, 0);
+
+  if (argc != 4 && argc != 6 ) usage(argv[0]);
+  if (!strcmp(argv[iarg],"-w")) {
+    append=0;
+    iarg++;
+    ofilename=argv[iarg];
+    iarg++;
+  } else if (!strcmp(argv[iarg],"-a")) {
+    append=1;
+    iarg++;
+    ofilename=argv[iarg];
+    iarg++;
+  } else {
+    append=0;
+    ofilename=defofilename;
+  }
+  finname=argv[iarg++];
+  repeatsimple=atoi(argv[iarg++]);
+  bitsPerValue=atoi(argv[iarg++]);
+
+  fin = fopen(finname,"r");
+  if(!fin) {perror(finname);exit(1);}
+
+  if (append)
+      fout = fopen(ofilename,"a");
+  else
+      fout = fopen(ofilename,"w");
+
+  if(!fout) {perror(ofilename);exit(1);}
+
+  c=grib_context_get_default();
+  e=0;
+  h=grib_handle_new_from_file(c,fin,&e);
+  fclose(fin);
+
+  GRIB_CHECK(e,0);
+
+  len=50;
+  grib_get_string(h,"shortName",param,&len);
+
+  len=20;
+  grib_get_string(h,"levelType",levelType,&len);
+
+  if (!strcmp(levelType,"pl")) {
+    GRIB_CHECK(grib_get_long(h,"level",&level),0);
+    sprintf(shortName,"%s%ld",param,level);
+  } else {
+    sprintf(shortName,"%s",param);
+  }
+
+
+  grib_set_long(h,"editionNumber",2);
+  GRIB_CHECK(grib_get_size(h,"values",&nvalues),0);
+  values=(double*)grib_context_malloc(c,sizeof(double)*nvalues);
+  if (!values) { printf("%s: memory allocation error\n",argv[0]); exit(1); }
+
+  len=50;
+  grib_get_string(h,"gridType",gridType,&len);
+
+  len=50;
+  grib_get_string(h,"packingType",packingType,&len);
+
+  GRIB_CHECK(grib_get_double_array(h,"values",values,&nvalues),0);
+  grib_set_long(h,"bitsPerValue",bitsPerValue);
+  GRIB_CHECK(grib_set_double_array(h,"values",values,nvalues),0);
+
+  printf("--------------------------------\n");
+  printf("- %s - gridType=%s packingType=%s numberOfValues=%ld bitsPerValue=%ld\n",
+    param,gridType,packingType,(long)nvalues,bitsPerValue);
+
+  if (!strcmp(packingType,"spectral_complex") || !strcmp(packingType,"spectral_simple")) {
+     printf("unable to process spectral data\n");
+     exit(1);
+  }
+
+  if (!strcmp(gridType,"reduced_gg") || !strcmp(gridType,"regular_gg")) {
+     long N;
+     grib_get_long(h,"N",&N);
+     printf("    N=%ld\n",N);
+     sprintf(grid,"%ld",N);
+  }
+  if (!strcmp(gridType,"regular_ll")) {
+     double Di,Dj;
+     grib_get_double(h,"DiInDegrees",&Di);
+     grib_get_double(h,"DjInDegrees",&Dj);
+     printf("    Di=%g Dj=%g\n",Di,Dj);
+     sprintf(grid,"%g/%g",Di,Dj);
+  }
+
+
+  if (!append)
+    fprintf(fout,
+    "shortName gridType numberOfValues bitsPerValue grid sizeSimple sizeccsds encodeccsds encodeSimple decodeccsds decodeSimple\n");
+
+  /* decode values grid_simple */
+  if (strcmp(packingType,grid_simple))
+    grib_set_string(h,"packingType",grid_simple,&grid_simple_l);
+  /* printf("decoding simple\n"); */
+  grib_timer_start(tds);
+  for (count=0;count<repeatsimple;count++)
+    GRIB_CHECK(grib_get_double_array(h,"values",values,&nvalues),0);
+  grib_timer_stop(tds,0);
+  /* printf("%d messages decoded\n\n",count); */
+
+  *outfilename='\0';
+  sprintf(outfilename,"%s_%s_%ld_simple.grib2",param,gridType,bitsPerValue);
+  filesize_simple=grib_handle_write(h,outfilename);
+  printf("file size simple = %ld\n",(long)filesize_simple);
+
+  /* encode values grid_simple*/
+  /* printf("encoding simple\n"); */
+  grib_timer_start(tes);
+  for (count=0;count<repeatsimple;count++)
+    GRIB_CHECK(grib_set_double_array(h,"values",values,nvalues),0);
+  grib_timer_stop(tes,0);
+  /* printf("%d messages encoded \n\n",count); */
+
+  /* decode values grid_ccsds */
+  grib_set_string(h,"packingType",grid_ccsds,&grid_ccsds_l);
+  /* printf("decoding ccsds\n"); */
+  grib_timer_start(tdj);
+  for (count=0;count<repeatccsds;count++)
+    GRIB_CHECK(grib_get_double_array(h,"values",values,&nvalues),0);
+  grib_timer_stop(tdj,0);
+  /* printf("%d messages decoded\n\n",count); */
+
+  *outfilename='\0';
+  sprintf(outfilename,"%s_%s_%ld_ccsds.grib2",param,gridType,bitsPerValue);
+  filesize_ccsds=grib_handle_write(h,outfilename);
+  printf("file size ccsds   = %ld\n",(long)filesize_ccsds);
+
+  perc=(double)filesize_simple/(double)filesize_ccsds;
+
+  printf("compression ratio = %g \n",perc);
+  printf("space savings = %g \n",(1.0-1.0/perc)*100);
+
+  /* encode values grid_ccsds*/
+  /* printf("encoding ccsds\n"); */
+  grib_timer_start(tej);
+  for (count=0;count<repeatccsds;count++)
+    GRIB_CHECK(grib_set_double_array(h,"values",values,nvalues),0);
+  grib_timer_stop(tej,0);
+  /* printf("%d messages encoded \n\n",count); */
+
+  grib_handle_delete(h);
+  grib_context_free(c,values);
+
+  print_timer(tej,repeatccsds);
+  print_timer(tdj,repeatccsds);
+  print_timer(tes,repeatsimple);
+  print_timer(tds,repeatsimple);
+  printf("--------------------------------\n\n");
+  fprintf(fout,"%s %s %ld %ld %s %ld %ld %g %g %g %g\n",
+    shortName,gridType,(long)nvalues,bitsPerValue,
+    grid,(long)filesize_simple,(long)filesize_ccsds,tej->timer_/repeatccsds,tes->timer_/repeatsimple,tdj->timer_/repeatccsds,tds->timer_/repeatsimple);
+
+  return 0;
+}
+#else
+
+int main(int argc,char* argv[]) {
+return 0;
+}
+
+#endif

windows/msvc/grib_api_lib/grib_api_lib.vcproj

@@ -689,7 +689,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\src\grib_accessor_class_data_szip_packing.c"
+				RelativePath="..\..\..\src\grib_accessor_class_data_ccsds_packing.c"
 				>
 			</File>
 			<File