core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GRIB-179: Data representation template 5.42 (CCSDS)

This commit is contained in:
Shahram Najm 2014-08-28 10:11:51 +01:00
parent 73085e73b8
commit 1315edbb48
22 changed files with 386 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -113,6 +113,7 @@ tests/read_any
tests/read_index
tests/so_perf
tests/unit_tests
tests/ccsds_perf
tigge/.libs/
tigge/tigge_accumulations
tigge/tigge_check

16
CMakeLists.txt
View File

@ -32,7 +32,7 @@ ecbuild_declare_project()
option( ENABLE_JPG "try to add support for JPG encoding" ON )
option( ENABLE_PNG "try to add support for PNG encoding" OFF )
option( ENABLE_NETCDF "try to add support for NetCDF" ON )
option( ENABLE_SZIP "try to add support for SZip encoding" OFF )
option( ENABLE_AEC "try to add support for Adaptive Entropy Coding" OFF )
option( ENABLE_PYTHON "try to build the GRIB_API Python interface" ON )
option( ENABLE_FORTRAN "try to build the GRIB_API Fortran interface" ON )
@ -55,15 +55,15 @@ set( GRIB_API_EXTRA_DEFINITIONS "" )
find_package( CMath )
### SZIP support
### AEC support (Adaptive Entropy Coding)
set( HAVE_LIBSZIP 0 )
if( ENABLE_SZIP )
set( HAVE_LIBAEC 0 )
if( ENABLE_AEC )
find_package( SZip )
find_package( AEC )
if( SZIP_FOUND )
set( HAVE_LIBSZIP 1 )
if( AEC_FOUND )
set( HAVE_LIBAEC 1 )
endif()
endif()
@ -198,7 +198,7 @@ endif()
### export package to other ecbuild packages
set( GRIB_API_TPLS SZip PNG NetCDF Jasper OpenJPEG CMath )
set( GRIB_API_TPLS AEC PNG NetCDF Jasper OpenJPEG CMath )
set( GRIB_API_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}/src )
set( GRIB_API_LIBRARIES grib_api )

9
NOTICE
View File

@ -1,11 +1,12 @@
ECMWF GRIB API
Copyright 2005-2014 ECMWF.
This product includes software developed at
ECMWF (http://www.ecmwf.int).
This product includes software developed at ECMWF (http://www.ecmwf.int).
Parts of the definitions provided by
WMO (http://www.wmo.int/pages/index_en.html)
Parts of the definitions provided by WMO (http://www.wmo.int/pages/index_en.html)
Adaptive Entropy Coding (CCSDS) provided by
The Max-Planck-Institute for Meteorology and Deutsches Klimarechenzentrum GmbH
IBM POWER optimisations were developed at
The Max-Planck-Institute for Meteorology.

17
configure.ac
View File

@ -455,6 +455,23 @@ then
AC_SUBST(JPEG_TEST)
fi
dnl check for libaec
CCSDS_TEST=""
AC_ARG_WITH([aec],
[AS_HELP_STRING([--with-aec=DIR],
[use specified libaec installation directory])],
,[with_aec=no])
if test "x$with_aec" != xno ; then
if test "x$with_aec" != xyes ; then
LDFLAGS="$LDFLAGS -L$with_aec/lib"
CPPFLAGS="$CPPFLAGS -I$with_aec/include"
fi
AC_CHECK_LIB(aec, aec_encode, , [AC_MSG_FAILURE(
[aec test failed (--without-aec to disable)])])
CCSDS_TEST="ccsds.sh"
fi
AC_SUBST(CCSDS_TEST)
dnl Check for png
AC_ARG_WITH([png-support],

1
data/CMakeLists.txt
View File

@ -86,6 +86,7 @@ set(files_to_download
timeRangeIndicator_5.grib
tp_ecmwf.grib
v.grib2
ccsds.grib2
)
foreach( f ${files_to_download} )

1
data/download.sh
View File

@ -123,6 +123,7 @@ files="
timeRangeIndicator_5.grib
tp_ecmwf.grib
v.grib2
ccsds.grib2
tigge/tigge_ammc_pl_gh.grib
tigge/tigge_ammc_pl_q.grib
tigge/tigge_ammc_pl_t.grib

2
definitions/grib1/section.4.def
View File

@ -126,7 +126,7 @@ concept packingType {
plusOneinOrdersOfSPD=1; twoOrdersOfSPD=1; }
"grid_jpeg" = { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;}
"grid_png" = { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;}
"grid_szip" = { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;}
"grid_ccsds" = { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;}
"grid_simple_log_preprocessing"= { sphericalHarmonics = 0; complexPacking = 0; additionalFlagPresent = 0;}
} : dump;

2
definitions/grib2/section.5.def
View File

@ -45,7 +45,7 @@ concept packingType (unknown) {
"grid_jpeg" = { dataRepresentationTemplateNumber = 40; }
"grid_png" = { dataRepresentationTemplateNumber = 40010; }
"grid_png" = { dataRepresentationTemplateNumber = 41; }
"grid_szip" = { dataRepresentationTemplateNumber = 42; }
"grid_ccsds" = { dataRepresentationTemplateNumber = 42; }
"grid_ieee" = { dataRepresentationTemplateNumber = 4; }
"grid_second_order" = { dataRepresentationTemplateNumber = 50001; }
"grid_second_order" = { dataRepresentationTemplateNumber = 50002; }

23
definitions/grib2/template.5.42.def
View File

@ -7,22 +7,19 @@
# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
#
# TEMPLATE 5.42, Grid point and spectral data - CCSDS szip
# TEMPLATE 5.42, Grid point and spectral data - CCSDS
include "template.5.packing.def";
include "template.5.original_values.def";
unsigned[1] szipOptionsMask : dump;
unsigned[1] ccsdsFlags : dump;
flagbit SZ_ALLOW_K13_OPTION_MASK(szipOptionsMask,0) = 0;
flagbit SZ_CHIP_OPTION_MASK(szipOptionsMask,1) = 0;
flagbit SZ_EC_OPTION_MASK(szipOptionsMask,2) = 0;
flagbit SZ_LSB_OPTION_MASK(szipOptionsMask,3) = 0;
flagbit SZ_MSB_OPTION_MASK(szipOptionsMask,4) = 1;
flagbit SZ_NN_OPTION_MASK(szipOptionsMask,5) = 1;
flagbit SZ_RAW_OPTION_MASK(szipOptionsMask,7) = 1;
unsigned[1] szipBitsPerPixel : dump;
unsigned[2] szipPixelPerBlock = 32 : dump;
unsigned[2] szipPixelPerScanLine = 128 * szipPixelPerBlock : dump;
flagbit AEC_DATA_SIGNED_OPTION_MASK(ccsdsFlags,0) = 0;
flagbit AEC_DATA_3BYTE_OPTION_MASK(ccsdsFlags,1) = 0;
flagbit AEC_DATA_MSB_OPTION_MASK(ccsdsFlags,2) = 1;
flagbit AEC_DATA_PREPROCESS_OPTION_MASK(ccsdsFlags,3) = 1;
flagbit AEC_RESTRICTED_OPTION_MASK(ccsdsFlags,4) = 0;
flagbit AEC_PAD_RSI_OPTION_MASK(ccsdsFlags,5) = 0;
unsigned[1] ccsdsBlockSize = 32 : dump;
unsigned[2] ccsdsRsi = 128 : dump;

11
definitions/grib2/template.7.42.def
View File

@ -9,9 +9,9 @@
# START 2/template.7.42 ----------------------------------------------------------------------
# TEMPLATE 7.42, Grid point data - CCSDS szip
# TEMPLATE 7.42, Grid point data - CCSDS
meta codedValues data_szip_packing(
meta codedValues data_ccsds_packing(
section7Length,
offsetBeforeData,
offsetSection7,
@ -23,10 +23,9 @@
numberOfDataPoints,
szipOptionsMask,
szipBitsPerPixel,
szipPixelPerBlock,
szipPixelPerScanLine
ccsdsFlags,
ccsdsBlockSize,
ccsdsRsi
): read_only;

2
grib_api_config.h.in
View File

@ -93,7 +93,7 @@
#define HAVE_LIBPNG @HAVE_LIBPNG@
#cmakedefine HAVE_LIBSZIP
#cmakedefine HAVE_LIBAEC
#cmakedefine HAVE_NETCDF

2
src/CMakeLists.txt
View File

@ -203,7 +203,7 @@ list( APPEND grib_api_srcs
grib_accessor_class_data_g2secondary_bitmap.c
grib_accessor_class_data_jpeg2000_packing.c
grib_accessor_class_data_png_packing.c
grib_accessor_class_data_szip_packing.c
grib_accessor_class_data_ccsds_packing.c
grib_accessor_class_data_raw_packing.c
grib_accessor_class_data_complex_packing.c
grib_accessor_class_data_g1complex_packing.c

2
src/Makefile.am
View File

@ -177,7 +177,7 @@ libgrib_api_la_prototypes= \
grib_accessor_class_values.c \
grib_accessor_class_simple_packing_error.c \
grib_accessor_class_data_simple_packing.c \
grib_accessor_class_data_szip_packing.c \
grib_accessor_class_data_ccsds_packing.c \
grib_accessor_class_count_missing.c \
grib_accessor_class_data_sh_packed.c \
grib_accessor_class_data_sh_unpacked.c \

2
src/grib_accessor_class.h
View File

@ -63,7 +63,7 @@ extern grib_accessor_class* grib_accessor_class_data_sh_packed;
extern grib_accessor_class* grib_accessor_class_data_sh_unpacked;
extern grib_accessor_class* grib_accessor_class_data_shsimple_packing;
extern grib_accessor_class* grib_accessor_class_data_simple_packing;
extern grib_accessor_class* grib_accessor_class_data_szip_packing;
extern grib_accessor_class* grib_accessor_class_data_ccsds_packing;
extern grib_accessor_class* grib_accessor_class_decimal_precision;
extern grib_accessor_class* grib_accessor_class_dictionary;
extern grib_accessor_class* grib_accessor_class_dirty;

151
src/grib_accessor_class_data_szip_packing.c → src/grib_accessor_class_data_ccsds_packing.c
View File

@ -28,10 +28,9 @@
MEMBERS=const char* number_of_data_points
MEMBERS=const char* szip_options_mask
MEMBERS=const char* szip_bits_per_pixel
MEMBERS=const char* szip_pixels_per_block
MEMBERS=const char* szip_pixels_per_scanline
MEMBERS=const char* ccsds_flags
MEMBERS=const char* ccsds_block_size
MEMBERS=const char* ccsds_rsi
END_CLASS_DEF
@ -49,11 +48,11 @@ or edit "accessor.class" and rerun ./make_class.pl
static int pack_double(grib_accessor*, const double* val,size_t *len);
static int unpack_double(grib_accessor*, double* val,size_t *len);
static int value_count(grib_accessor*,long*);
static int value_count(grib_accessor*, long*);
static void init(grib_accessor*,const long, grib_arguments* );
static void init_class(grib_accessor_class*);
typedef struct grib_accessor_data_szip_packing {
typedef struct grib_accessor_data_ccsds_packing {
grib_accessor att;
/* Members defined in gen */
/* Members defined in values */
@ -62,25 +61,24 @@ typedef struct grib_accessor_data_szip_packing {
const char* offsetdata;
const char* offsetsection;
int dirty;
/* Members defined in data_szip_packing */
/* Members defined in data_ccsds_packing */
const char* number_of_values;
const char* reference_value;
const char* binary_scale_factor;
const char* decimal_scale_factor;
const char* bits_per_value;
const char* number_of_data_points;
const char* szip_options_mask;
const char* szip_bits_per_pixel;
const char* szip_pixels_per_block;
const char* szip_pixels_per_scanline;
} grib_accessor_data_szip_packing;
const char* ccsds_flags;
const char* ccsds_block_size;
const char* ccsds_rsi;
} grib_accessor_data_ccsds_packing;
extern grib_accessor_class* grib_accessor_class_values;
static grib_accessor_class _grib_accessor_class_data_szip_packing = {
static grib_accessor_class _grib_accessor_class_data_ccsds_packing = {
&grib_accessor_class_values, /* super */
"data_szip_packing", /* name */
sizeof(grib_accessor_data_szip_packing), /* size */
"data_ccsds_packing", /* name */
sizeof(grib_accessor_data_ccsds_packing), /* size */
0, /* inited */
&init_class, /* init_class */
&init, /* init */
@ -95,7 +93,7 @@ static grib_accessor_class _grib_accessor_class_data_szip_packing = {
0, /* get native type */
0, /* get sub_section */
0, /* grib_pack procedures long */
0, /* grib_pack procedures long */
0, /* grib_pack procedures long */
0, /* grib_pack procedures long */
0, /* grib_unpack procedures long */
&pack_double, /* grib_pack procedures double */
@ -120,7 +118,7 @@ static grib_accessor_class _grib_accessor_class_data_szip_packing = {
};
grib_accessor_class* grib_accessor_class_data_szip_packing = &_grib_accessor_class_data_szip_packing;
grib_accessor_class* grib_accessor_class_data_ccsds_packing = &_grib_accessor_class_data_ccsds_packing;
static void init_class(grib_accessor_class* c)
@ -138,8 +136,6 @@ static void init_class(grib_accessor_class* c)
c->unpack_long = (*(c->super))->unpack_long;
c->pack_string = (*(c->super))->pack_string;
c->unpack_string = (*(c->super))->unpack_string;
c->pack_string_array = (*(c->super))->pack_string_array;
c->unpack_string_array = (*(c->super))->unpack_string_array;
c->pack_bytes = (*(c->super))->pack_bytes;
c->unpack_bytes = (*(c->super))->unpack_bytes;
c->pack_expression = (*(c->super))->pack_expression;
@ -159,46 +155,45 @@ static void init_class(grib_accessor_class* c)
static void init(grib_accessor* a,const long v, grib_arguments* args)
{
grib_accessor_data_szip_packing *self =(grib_accessor_data_szip_packing*)a;
grib_accessor_data_ccsds_packing *self =(grib_accessor_data_ccsds_packing*)a;
self->number_of_values = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->reference_value = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->binary_scale_factor = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->decimal_scale_factor = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->bits_per_value = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->bits_per_value = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->number_of_data_points = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->szip_options_mask = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->szip_bits_per_pixel = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->szip_pixels_per_block = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->szip_pixels_per_scanline = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->ccsds_flags = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->ccsds_block_size = grib_arguments_get_name(a->parent->h,args,self->carg++);
self->ccsds_rsi = grib_arguments_get_name(a->parent->h,args,self->carg++);
a->flags |= GRIB_ACCESSOR_FLAG_DATA;
}
static int value_count(grib_accessor* a, long* count)
{
grib_accessor_data_szip_packing *self =(grib_accessor_data_szip_packing*)a;
grib_accessor_data_ccsds_packing *self =(grib_accessor_data_ccsds_packing*)a;
*count = 0;
return grib_get_long_internal(a->parent->h,self->number_of_values,count);
}
#ifdef HAVE_LIBSZIP
#ifdef HAVE_LIBAEC
#include "szlib.h"
#include <libaec.h>
static int unpack_double(grib_accessor* a, double* val, size_t *len)
{
grib_accessor_data_szip_packing *self =(grib_accessor_data_szip_packing*)a;
grib_accessor_data_ccsds_packing *self =(grib_accessor_data_ccsds_packing*)a;
int err = GRIB_SUCCESS;
int i,j;
int i;
size_t buflen = grib_byte_count(a);
SZ_com_t sz_options;
struct aec_stream strm;
double bscale = 0;
double dscale = 0;
unsigned char* buf = NULL;
@ -214,11 +209,9 @@ static int unpack_double(grib_accessor* a, double* val, size_t *len)
long bits_per_value =0;
long bits8;
long szip_options_mask ;
long szip_bits_per_pixel ;
long szip_pixels_per_block ;
long szip_pixels_per_scanline ;
long ccsds_flags;
long ccsds_block_size;
long ccsds_rsi;
self->dirty=0;
@ -234,14 +227,11 @@ static int unpack_double(grib_accessor* a, double* val, size_t *len)
if((err = grib_get_long_internal(a->parent->h,self->decimal_scale_factor, &decimal_scale_factor)) != GRIB_SUCCESS)
return err;
if((err = grib_get_long_internal(a->parent->h,self->szip_options_mask,&szip_options_mask)) != GRIB_SUCCESS)
if((err = grib_get_long_internal(a->parent->h,self->ccsds_flags,&ccsds_flags)) != GRIB_SUCCESS)
return err;
if((err = grib_get_long_internal(a->parent->h,self->szip_bits_per_pixel, &szip_bits_per_pixel)) != GRIB_SUCCESS)
if((err = grib_get_long_internal(a->parent->h,self->ccsds_block_size, &ccsds_block_size)) != GRIB_SUCCESS)
return err;
if((err = grib_get_long_internal(a->parent->h,self->szip_pixels_per_block, &szip_pixels_per_block)) != GRIB_SUCCESS)
return err;
if((err = grib_get_long_internal(a->parent->h,self->szip_pixels_per_scanline, &szip_pixels_per_scanline)) != GRIB_SUCCESS)
if((err = grib_get_long_internal(a->parent->h,self->ccsds_rsi, &ccsds_rsi)) != GRIB_SUCCESS)
return err;
@ -265,11 +255,13 @@ static int unpack_double(grib_accessor* a, double* val, size_t *len)
buf = (unsigned char*)a->parent->h->buffer->data;
buf += grib_byte_offset(a);
memset(&sz_options, 0, sizeof(sz_options));
sz_options.options_mask = szip_options_mask;
sz_options.bits_per_pixel = szip_bits_per_pixel;
sz_options.pixels_per_block = szip_pixels_per_block;
sz_options.pixels_per_scanline = szip_pixels_per_scanline;
strm.flags = ccsds_flags;
strm.bits_per_sample = bits_per_value;
strm.block_size = ccsds_block_size;
strm.rsi = ccsds_rsi;
strm.next_in = buf;
strm.avail_in = buflen;
/*
printf("sz_options.options_mask %d\n", sz_options.options_mask);
@ -278,18 +270,19 @@ static int unpack_double(grib_accessor* a, double* val, size_t *len)
printf("sz_options.pixels_per_scanline %d\n", sz_options.pixels_per_scanline);
*/
bits8 = ((szip_bits_per_pixel + 7)/8)*8;
size = n_vals * (szip_bits_per_pixel + 7)/8;
bits8 = ((bits_per_value + 7)/8)*8;
size = n_vals * (bits_per_value + 7)/8;
decoded = grib_context_buffer_malloc_clear(a->parent->h->context,size);
if(!decoded) {
err = GRIB_OUT_OF_MEMORY;
goto cleanup;
}
strm.next_out = decoded;
strm.avail_out = size;
if((err = SZ_BufftoBuffDecompress( decoded,&size,buf, buflen , &sz_options)) != SZ_OK)
if((err = aec_buffer_decode(&strm)) != AEC_OK)
{
printf("SZ_BufftoBuffDecompress Error %d\n", err);
printf("aec_buffer_decode Error %d\n", err);
err = GRIB_ENCODING_ERROR;
goto cleanup;
}
@ -315,7 +308,7 @@ cleanup:
static int pack_double(grib_accessor* a, const double* val, size_t *len)
{
grib_accessor_data_szip_packing *self =(grib_accessor_data_szip_packing*)a;
grib_accessor_data_ccsds_packing *self =(grib_accessor_data_ccsds_packing*)a;
int err = GRIB_SUCCESS;
int i;
@ -338,19 +331,13 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
unsigned char *p;
double divisor;
long ni,nj;
long scanning_mode;
long list_defining_points;
long number_of_data_points;
long szip_options_mask ;
long szip_bits_per_pixel ;
long szip_pixels_per_block ;
long szip_pixels_per_scanline ;
long ccsds_flags;
long ccsds_block_size;
long ccsds_rsi;
size_t width, height;
SZ_com_t sz_options;
struct aec_stream strm;
self->dirty=1;
@ -367,13 +354,11 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
return err;
if((err = grib_get_long_internal(a->parent->h,self->szip_options_mask,&szip_options_mask)) != GRIB_SUCCESS)
if((err = grib_get_long_internal(a->parent->h,self->ccsds_flags,&ccsds_flags)) != GRIB_SUCCESS)
return err;
if((err = grib_get_long_internal(a->parent->h,self->szip_bits_per_pixel, &szip_bits_per_pixel)) != GRIB_SUCCESS)
if((err = grib_get_long_internal(a->parent->h,self->ccsds_block_size, &ccsds_block_size)) != GRIB_SUCCESS)
return err;
if((err = grib_get_long_internal(a->parent->h,self->szip_pixels_per_block, &szip_pixels_per_block)) != GRIB_SUCCESS)
return err;
if((err = grib_get_long_internal(a->parent->h,self->szip_pixels_per_scanline, &szip_pixels_per_scanline)) != GRIB_SUCCESS)
if((err = grib_get_long_internal(a->parent->h,self->ccsds_rsi, &ccsds_rsi)) != GRIB_SUCCESS)
return err;
@ -470,7 +455,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
/* buflen = n_vals*(bits_per_value/8);*/
grib_context_log(a->parent->h->context, GRIB_LOG_DEBUG,
"grib_accessor_data_szip_packing : pack_double : packing %s, %d values", a->name, n_vals);
"grib_accessor_data_ccsds_packing : pack_double : packing %s, %d values", a->name, n_vals);
buflen += 10240;
buf = grib_context_buffer_malloc_clear(a->parent->h->context,buflen);
@ -497,11 +482,15 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
return err;
memset(&sz_options, 0, sizeof(sz_options));
sz_options.options_mask = szip_options_mask;
sz_options.bits_per_pixel = bits_per_value;
sz_options.pixels_per_block = szip_pixels_per_block;
sz_options.pixels_per_scanline = szip_pixels_per_scanline;
strm.flags = ccsds_flags;
strm.bits_per_sample = bits_per_value;
strm.block_size = ccsds_block_size;
strm.rsi = ccsds_rsi;
strm.next_out = buf;
strm.avail_out = buflen;
strm.next_in = encoded;
strm.avail_in = bits8/8*n_vals;
/*
@ -516,9 +505,9 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
printf("sz_options.pixels_per_scanline %d\n", sz_options.pixels_per_scanline);
*/
if((err = SZ_BufftoBuffCompress(buf,&buflen,encoded, bits8/8*n_vals, &sz_options)) != SZ_OK)
if((err = aec_buffer_encode(&strm)) != AEC_OK)
{
printf("SZ_BufftoBuffCompress Error %d\n", err);
printf("aec_buffer_encode Error %d\n", err);
err = GRIB_ENCODING_ERROR;
goto cleanup;
}
@ -528,7 +517,7 @@ static int pack_double(grib_accessor* a, const double* val, size_t *len)
printf("n_vals = %ld, bits8 = %ld\n", (long)n_vals, (long)bits8);
printf("in %ld out => %ld\n", (long)bits8/8*n_vals,(long) buflen);
*/
buflen = strm.total_out;
grib_buffer_replace(a, buf, buflen ,1,1);
cleanup:
@ -539,7 +528,7 @@ cleanup:
err = grib_set_long_internal(a->parent->h,self->number_of_values, *len);
if(err == GRIB_SUCCESS)
err = grib_set_long_internal(a->parent->h,self->szip_bits_per_pixel, sz_options.bits_per_pixel);
err = grib_set_long_internal(a->parent->h,self->bits_per_value, strm.bits_per_sample);
return err;
}
@ -548,14 +537,14 @@ cleanup:
static int unpack_double(grib_accessor* a, double* val, size_t *len)
{
grib_context_log(a->parent->h->context, GRIB_LOG_ERROR,
"grib_accessor_data_szip_packing : szip support no enabled. Please rerun configure with --with-szip-support");
"grib_accessor_data_ccsds_packing : ccsds support no enabled. Please rerun configure with --with-aec");
return GRIB_NOT_IMPLEMENTED;
}
static int pack_double(grib_accessor* a, const double* val, size_t *len)
{
grib_context_log(a->parent->h->context, GRIB_LOG_ERROR,
"grib_accessor_data_szip_packing : szip support no enabled. Please rerun configure with --with-szip-support");
"grib_accessor_data_ccsds_packing : ccsds support no enabled. Please rerun configure with --with-aec");
return GRIB_NOT_IMPLEMENTED;
}

2
src/grib_accessor_factory.h
View File

@ -63,7 +63,7 @@
{ "data_sh_unpacked", &grib_accessor_class_data_sh_unpacked, },
{ "data_shsimple_packing", &grib_accessor_class_data_shsimple_packing, },
{ "data_simple_packing", &grib_accessor_class_data_simple_packing, },
{ "data_szip_packing", &grib_accessor_class_data_szip_packing, },
{ "data_ccsds_packing", &grib_accessor_class_data_ccsds_packing, },
{ "decimal_precision", &grib_accessor_class_decimal_precision, },
{ "dictionary", &grib_accessor_class_dictionary, },
{ "dirty", &grib_accessor_class_dirty, },

2
src/grib_accessor_factory_hash_list
View File

@ -67,7 +67,7 @@ data_sh_packed, &grib_accessor_class_data_sh_packed
data_sh_unpacked, &grib_accessor_class_data_sh_unpacked
data_shsimple_packing, &grib_accessor_class_data_shsimple_packing
data_simple_packing, &grib_accessor_class_data_simple_packing
data_szip_packing, &grib_accessor_class_data_szip_packing
data_ccsds_packing, &grib_accessor_class_data_ccsds_packing
decimal_precision, &grib_accessor_class_decimal_precision
descriptors, &grib_accessor_class_descriptors
dictionary, &grib_accessor_class_dictionary

2
src/grib_api_prototypes.h
View File

@ -472,8 +472,6 @@ int grib_get_g1_message_size(grib_handle *h, grib_accessor *tl, grib_accessor *s
/* grib_accessor_class_data_simple_packing.c */
/* grib_accessor_class_data_szip_packing.c */
/* grib_accessor_class_count_missing.c */
/* grib_accessor_class_data_sh_packed.c */

8
tests/CMakeLists.txt
View File

@ -100,3 +100,11 @@ ecbuild_add_test( TARGET t_jpeg
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/jpeg.sh
RESOURCES include.sh
)
ecbuild_add_test( TARGET t_ccsds
TYPE SCRIPT
CONDITION ENABLE_AEC
COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/ccsds.sh
RESOURCES include.sh
)

8
tests/Makefile.am
View File

@ -10,12 +10,13 @@ TESTS = definitions.sh \
bitmap.sh list.sh second_order.sh \
multi_from_message.sh change_scanning.sh \
julian.sh statistics.sh tigge.sh tigge_conversions.sh \
read_any.sh padding.sh lamb_az_eq_area.sh grib_to_netcdf.sh \
debug.sh $(JPEG_TEST)
read_any.sh padding.sh lamb_az_eq_area.sh grib_to_netcdf.sh debug.sh \
$(JPEG_TEST) $(CCSDS_TEST)
noinst_PROGRAMS = packing_check gauss_sub read_any double_cmp packing pack_unpack \
multi_from_message julian read_index index gribex_perf\
jpeg_perf so_perf png_perf bpv_limit laplacian unit_tests
jpeg_perf ccsds_perf so_perf png_perf bpv_limit laplacian \
unit_tests
multi_from_message_SOURCES = multi_from_message.c
laplacian_SOURCES = laplacian.c
@ -31,6 +32,7 @@ read_index_SOURCES = read_index.c
jpeg_perf_SOURCES = jpeg_perf.c
so_perf_SOURCES = so_perf.c
png_perf_SOURCES = png_perf.c
ccsds_perf_SOURCES = ccsds_perf.c
gribex_perf_SOURCES = gribex_perf.c
gauss_sub_SOURCES = gauss_sub.c

246
tests/ccsds_perf.c Normal file
View File

@ -0,0 +1,246 @@
/*
* Copyright 2005-2014 ECMWF.
*
* This software is licensed under the terms of the Apache Licence Version 2.0
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
*
* In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
* virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
*/
#include "grib_api_internal.h"
#if GRIB_TIMER
void usage(char* prog) {
printf("usage: %s [-a outfile | -w outfile ] grib_file repetitions bitsPerValue\n",prog);
exit(1);
}
size_t grib_handle_write(grib_handle* h,char* filename) {
FILE* of=NULL;
const void *buffer; size_t size;
of = fopen(filename,"w");
if(!of) {
perror(filename);
exit(1);
}
GRIB_CHECK(grib_get_message(h,&buffer,&size),0);
if(fwrite(buffer,1,size,of) != size) {
perror(filename);
exit(1);
}
fclose(of);
return size;
}
static void print_timer(grib_timer* t,int repeat) {
printf("%s : %g cpu\n",t->name_,t->timer_/repeat);
}
int main(int argc, char* argv[]) {
grib_handle *h=NULL;
grib_context* c=NULL;
FILE* fin=NULL;
FILE* fout=NULL;
char* finname;
char* ofilename;
char defofilename[]="ccsds_perf.out";
double *values=NULL;
int append=0;
size_t nvalues=0;
int count,e=0;
int repeatccsds=1;
int repeatsimple=1;
grib_timer *tes,*tds,*tej,*tdj;
char grid_ccsds[]="grid_ccsds";
size_t grid_ccsds_l=strlen(grid_ccsds);
char grid_simple[]="grid_simple";
size_t grid_simple_l=strlen(grid_simple);
char packingType[50]={0,};
size_t len=50;
char param[50]={0,};
char gridType[50]={0,};
char outfilename[255]={0,};
size_t filesize_ccsds=0;
size_t filesize_simple=0;
double perc=0;
long bitsPerValue=0;
int iarg=1;
char grid[20]={0,};
char shortName[20]={0,};
long level;
char levelType[20]={0,};
tes=grib_get_timer(0,"encoding simple", 0, 0);
tds=grib_get_timer(0,"decoding simple", 0, 0);
tej=grib_get_timer(0,"encoding ccsds", 0, 0);
tdj=grib_get_timer(0,"decoding ccsds", 0, 0);
if (argc != 4 && argc != 6 ) usage(argv[0]);
if (!strcmp(argv[iarg],"-w")) {
append=0;
iarg++;
ofilename=argv[iarg];
iarg++;
} else if (!strcmp(argv[iarg],"-a")) {
append=1;
iarg++;
ofilename=argv[iarg];
iarg++;
} else {
append=0;
ofilename=defofilename;
}
finname=argv[iarg++];
repeatsimple=atoi(argv[iarg++]);
bitsPerValue=atoi(argv[iarg++]);
fin = fopen(finname,"r");
if(!fin) {perror(finname);exit(1);}
if (append)
fout = fopen(ofilename,"a");
else
fout = fopen(ofilename,"w");
if(!fout) {perror(ofilename);exit(1);}
c=grib_context_get_default();
e=0;
h=grib_handle_new_from_file(c,fin,&e);
fclose(fin);
GRIB_CHECK(e,0);
len=50;
grib_get_string(h,"shortName",param,&len);
len=20;
grib_get_string(h,"levelType",levelType,&len);
if (!strcmp(levelType,"pl")) {
GRIB_CHECK(grib_get_long(h,"level",&level),0);
sprintf(shortName,"%s%ld",param,level);
} else {
sprintf(shortName,"%s",param);
}
grib_set_long(h,"editionNumber",2);
GRIB_CHECK(grib_get_size(h,"values",&nvalues),0);
values=(double*)grib_context_malloc(c,sizeof(double)*nvalues);
if (!values) { printf("%s: memory allocation error\n",argv[0]); exit(1); }
len=50;
grib_get_string(h,"gridType",gridType,&len);
len=50;
grib_get_string(h,"packingType",packingType,&len);
GRIB_CHECK(grib_get_double_array(h,"values",values,&nvalues),0);
grib_set_long(h,"bitsPerValue",bitsPerValue);
GRIB_CHECK(grib_set_double_array(h,"values",values,nvalues),0);
printf("--------------------------------\n");
printf("- %s - gridType=%s packingType=%s numberOfValues=%ld bitsPerValue=%ld\n",
param,gridType,packingType,(long)nvalues,bitsPerValue);
if (!strcmp(packingType,"spectral_complex") || !strcmp(packingType,"spectral_simple")) {
printf("unable to process spectral data\n");
exit(1);
}
if (!strcmp(gridType,"reduced_gg") || !strcmp(gridType,"regular_gg")) {
long N;
grib_get_long(h,"N",&N);
printf(" N=%ld\n",N);
sprintf(grid,"%ld",N);
}
if (!strcmp(gridType,"regular_ll")) {
double Di,Dj;
grib_get_double(h,"DiInDegrees",&Di);
grib_get_double(h,"DjInDegrees",&Dj);
printf(" Di=%g Dj=%g\n",Di,Dj);
sprintf(grid,"%g/%g",Di,Dj);
}
if (!append)
fprintf(fout,
"shortName gridType numberOfValues bitsPerValue grid sizeSimple sizeccsds encodeccsds encodeSimple decodeccsds decodeSimple\n");
/* decode values grid_simple */
if (strcmp(packingType,grid_simple))
grib_set_string(h,"packingType",grid_simple,&grid_simple_l);
/* printf("decoding simple\n"); */
grib_timer_start(tds);
for (count=0;count<repeatsimple;count++)
GRIB_CHECK(grib_get_double_array(h,"values",values,&nvalues),0);
grib_timer_stop(tds,0);
/* printf("%d messages decoded\n\n",count); */
*outfilename='\0';
sprintf(outfilename,"%s_%s_%ld_simple.grib2",param,gridType,bitsPerValue);
filesize_simple=grib_handle_write(h,outfilename);
printf("file size simple = %ld\n",(long)filesize_simple);
/* encode values grid_simple*/
/* printf("encoding simple\n"); */
grib_timer_start(tes);
for (count=0;count<repeatsimple;count++)
GRIB_CHECK(grib_set_double_array(h,"values",values,nvalues),0);
grib_timer_stop(tes,0);
/* printf("%d messages encoded \n\n",count); */
/* decode values grid_ccsds */
grib_set_string(h,"packingType",grid_ccsds,&grid_ccsds_l);
/* printf("decoding ccsds\n"); */
grib_timer_start(tdj);
for (count=0;count<repeatccsds;count++)
GRIB_CHECK(grib_get_double_array(h,"values",values,&nvalues),0);
grib_timer_stop(tdj,0);
/* printf("%d messages decoded\n\n",count); */
*outfilename='\0';
sprintf(outfilename,"%s_%s_%ld_ccsds.grib2",param,gridType,bitsPerValue);
filesize_ccsds=grib_handle_write(h,outfilename);
printf("file size ccsds = %ld\n",(long)filesize_ccsds);
perc=(double)filesize_simple/(double)filesize_ccsds;
printf("compression ratio = %g \n",perc);
printf("space savings = %g \n",(1.0-1.0/perc)*100);
/* encode values grid_ccsds*/
/* printf("encoding ccsds\n"); */
grib_timer_start(tej);
for (count=0;count<repeatccsds;count++)
GRIB_CHECK(grib_set_double_array(h,"values",values,nvalues),0);
grib_timer_stop(tej,0);
/* printf("%d messages encoded \n\n",count); */
grib_handle_delete(h);
grib_context_free(c,values);
print_timer(tej,repeatccsds);
print_timer(tdj,repeatccsds);
print_timer(tes,repeatsimple);
print_timer(tds,repeatsimple);
printf("--------------------------------\n\n");
fprintf(fout,"%s %s %ld %ld %s %ld %ld %g %g %g %g\n",
shortName,gridType,(long)nvalues,bitsPerValue,
grid,(long)filesize_simple,(long)filesize_ccsds,tej->timer_/repeatccsds,tes->timer_/repeatsimple,tdj->timer_/repeatccsds,tds->timer_/repeatsimple);
return 0;
}
#else
int main(int argc,char* argv[]) {
return 0;
}
#endif

2
windows/msvc/grib_api_lib/grib_api_lib.vcproj
View File

@ -689,7 +689,7 @@
>
</File>
<File
RelativePath="..\..\..\src\grib_accessor_class_data_szip_packing.c"
RelativePath="..\..\..\src\grib_accessor_class_data_ccsds_packing.c"
>
</File>
<File