core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/LAEA_Oblate

This commit is contained in:
Shahram Najm 2020-06-27 20:14:01 +01:00
parent fbdee2773a 034778d722
commit 6fb11420be
37 changed files with 906 additions and 685 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
definitions/bufr/rdb_key.def
View File

@ -11,18 +11,22 @@ meta localMinute bits(keyData,27,6) : dump,long_type,no_copy;
meta localSecond bits(keyData,33,6) : dump,long_type,no_copy;
meta spare bits(keyData,39,1) : no_copy; # 40 bits = 5 bytes
alias mars.localYear=localYear;
alias mars.localMonth=localMonth;
alias mars.localDay=localDay;
alias mars.localHour=localHour;
alias mars.localMinute=localMinute;
alias mars.localSecond=localSecond;
alias rdb.localYear=localYear;
alias rdb.localMonth=localMonth;
alias rdb.localDay=localDay;
alias rdb.localHour=localHour;
alias rdb.localMinute=localMinute;
alias rdb.localSecond=localSecond;
meta localDate sprintf("%.4d%.2d%.2d",localYear,localMonth,localDay) : no_copy,read_only;
meta localTime sprintf("%.2d%.2d",localHour,localMinute) : no_copy,read_only;
meta localDateTime julian_date(localYear,localMonth,localDay,localHour,localMinute,localSecond) : no_copy;
alias mars.date = localDate;
alias mars.time = localTime;
# isSatellite and other stuff moved to section 3
ascii[8] keyMore : hidden;
byte[9] keySat : hidden;
ascii[8] keyMore : hidden;
byte[9] keySat : hidden;
# 16 bytes
unsigned[2] messageLength;
@ -80,9 +84,10 @@ if (oldSubtype < 255 ) {
alias ls.rdbType=rdbType;
alias ls.rdbSubtype=rdbSubtype;
alias mars.rdbType=rdbType;
alias mars.rdbSubtype=rdbSubtype;
alias rdb.rdbType=rdbType;
alias rdb.rdbSubtype=rdbSubtype;
alias mars.obstype = rdbSubtype;
#Data assimilation loop for purpose of Continous DA
unsigned[1] daLoop;

8
definitions/bufr/section.3.def
View File

@ -75,19 +75,15 @@ if (section2Present && bufrHeaderCentre==98 && section2Length==52) {
oldSubtype==31) {
meta ls.localNumberOfObservations bits(keySat,0,16) : dump,long_type,no_copy;
meta ls.satelliteID bits(keySat,16,16) : dump,long_type,no_copy;
alias mars.satelliteID=satelliteID;
} else {
meta ls.localNumberOfObservations bits(keySat,0,8) : dump,long_type,no_copy;
meta ls.satelliteID bits(keySat,8,16) : dump,long_type,no_copy;
alias mars.satelliteID=satelliteID;
}
} else {
meta ls.localLatitude bits(keyData,72,25,-9000000,100000) : dump,no_copy;
meta ls.localLongitude bits(keyData,40,26,-18000000,100000) : dump,no_copy;
alias ls.ident=keyMore : dump,string_type,no_copy;
alias mars.localLatitude=localLatitude;
alias mars.localLongitude=localLongitude;
meta ident trim(keyMore,1,1): dump, no_copy; # remove whitespaces left and right
alias mars.ident = ident : string_type, no_copy;
}
}

6
examples/F90/bufr_read_tropical_cyclone.f90
View File

@ -11,7 +11,11 @@
!
! Description: how to read data for a tropical cyclone BUFR message.
!
! Please note that tropical cyclone tracks can be encoded in various ways in BUFR.
! Therefore the code below might not work directly for other types of messages
! than the one used in the example. It is advised to use bufr_dump to
! understand the structure of the messages.
!
program bufr_read_tropical_cyclone
use eccodes
implicit none

1
examples/python/bufr_expanded.py
View File

@ -12,7 +12,6 @@
#
# Description: how to read values of different type of keys from BUFR messages.
#
#
from __future__ import absolute_import
from __future__ import print_function

4
examples/python/bufr_keys_iterator.py
View File

@ -11,8 +11,8 @@
# Python implementation: bufr_keys_iterator
#
# Description: Example on how to use keys_iterator functions and the
# codes_bufr_keys_iterator structure to get all the available
# keys in a BUFR message.
# codes_bufr_keys_iterator structure to get all the available
# keys in a BUFR message.
from __future__ import print_function

5
examples/python/bufr_read_tropical_cyclone.py
View File

@ -10,6 +10,11 @@
#
# Description: how to read data of the ECMWF EPS tropical cyclone tracks encoded in BUFR format.
#
# Please note that tropical cyclone tracks can be encoded in various ways in BUFR.
# Therefore the code below might not work directly for other types of messages
# than the one used in the example. It is advised to use bufr_dump to
# understand the structure of the messages.
#
from __future__ import print_function
import traceback

1
src/CMakeLists.txt
View File

@ -184,6 +184,7 @@ list( APPEND grib_api_srcs
grib_accessor_class_spectral_truncation.c
grib_accessor_class_time.c
grib_accessor_class_transient.c
grib_accessor_class_trim.c
grib_accessor_class_transient_darray.c
grib_accessor_class_values.c
grib_accessor_class_simple_packing_error.c

1
src/Makefile.am
View File

@ -201,6 +201,7 @@ libeccodes_la_prototypes= \
grib_accessor_class_spectral_truncation.c \
grib_accessor_class_time.c \
grib_accessor_class_transient.c \
grib_accessor_class_trim.c \
grib_accessor_class_values.c \
grib_accessor_class_simple_packing_error.c \
grib_accessor_class_data_simple_packing.c \

23
src/bufr_keys_iterator.c
View File

@ -89,28 +89,12 @@ int codes_bufr_keys_iterator_rewind(bufr_keys_iterator* ki)
return GRIB_SUCCESS;
}
static int is_ident_key(const bufr_keys_iterator* kiter)
{
if (kiter->current->sub_section)
return 0;
if ((GRIB_ACCESSOR_FLAG_HIDDEN & kiter->current->flags) != 0 &&
strcmp(kiter->current->name, "keyMore") == 0 &&
grib_is_defined(kiter->handle, "ls.ident")) {
return 1;
}
return 0;
}
static int skip(bufr_keys_iterator* kiter)
{
if (kiter->current->sub_section)
return 1;
if (kiter->current->flags & kiter->accessor_flags_skip) {
/* The "ident" key deserves special treatment */
if (is_ident_key(kiter))
return 0;
return 1;
}
@ -233,12 +217,7 @@ char* codes_bufr_keys_iterator_get_name(const bufr_keys_iterator* ckiter)
sprintf(ret, "#%d#%s", *r, kiter->current->name);
}
else {
if (is_ident_key(kiter)) {
strcpy(ret, "ident");
}
else {
strcpy(ret, kiter->current->name);
}
strcpy(ret, kiter->current->name);
}
}

20
src/bufr_util.c
View File

@ -204,6 +204,8 @@ static int bufr_decode_rdb_keys(const void* message, long offset_section2, codes
return GRIB_SUCCESS;
}
#define IDENT_LEN 9 /* 8 chars plus the final 0 terminator */
/* The ECMWF BUFR local use section */
static int bufr_decode_extra_rdb_keys(const void* message, long offset_section2, codes_bufr_header* hdr)
{
@ -261,8 +263,11 @@ static int bufr_decode_extra_rdb_keys(const void* message, long offset_section2,
}
}
else {
size_t i = 0;
long lValue = 0;
size_t i = 0;
long lValue = 0;
char* pTemp = NULL;
char temp[IDENT_LEN] = {0,};
start = 72;
lValue = (long)grib_decode_unsigned_long(pKeyData, &start, 25);
hdr->localLatitude = (lValue - 9000000.0) / 100000.0;
@ -270,11 +275,14 @@ static int bufr_decode_extra_rdb_keys(const void* message, long offset_section2,
lValue = (long)grib_decode_unsigned_long(pKeyData, &start, 26);
hdr->localLongitude = (lValue - 18000000.0) / 100000.0;
/* interpret keyMore as a string */
for (i = 0; i < 8; ++i) {
hdr->ident[i] = *pKeyMore++;
/* interpret keyMore as a string. Copy to a temporary */
for (i = 0; i < IDENT_LEN - 1; ++i) {
temp[i] = *pKeyMore++;
}
hdr->ident[i] = '\0';
temp[i] = '\0';
pTemp = temp;
lrtrim(&pTemp, 1, 1); /* Trim left and right */
strncpy(hdr->ident, pTemp, IDENT_LEN - 1);
}
return GRIB_SUCCESS;

1
src/grib_accessor_class.h
View File

@ -208,6 +208,7 @@ extern grib_accessor_class* grib_accessor_class_to_integer;
extern grib_accessor_class* grib_accessor_class_to_string;
extern grib_accessor_class* grib_accessor_class_transient;
extern grib_accessor_class* grib_accessor_class_transient_darray;
extern grib_accessor_class* grib_accessor_class_trim;
extern grib_accessor_class* grib_accessor_class_uint16;
extern grib_accessor_class* grib_accessor_class_uint16_little_endian;
extern grib_accessor_class* grib_accessor_class_uint32;

200
src/grib_accessor_class_trim.c Normal file
View File

@ -0,0 +1,200 @@
/*
* (C) Copyright 2005- ECMWF.
*
* This software is licensed under the terms of the Apache Licence Version 2.0
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
*
* In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
* virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
*/
#include "grib_api_internal.h"
/*
This is used by make_class.pl
START_CLASS_DEF
CLASS = accessor
SUPER = grib_accessor_class_ascii
IMPLEMENTS = unpack_string;pack_string
IMPLEMENTS = init; string_length
MEMBERS= const char* input
MEMBERS= int trim_left
MEMBERS= int trim_right
END_CLASS_DEF
*/
/* START_CLASS_IMP */
/*
Don't edit anything between START_CLASS_IMP and END_CLASS_IMP
Instead edit values between START_CLASS_DEF and END_CLASS_DEF
or edit "accessor.class" and rerun ./make_class.pl
*/
static int pack_string(grib_accessor*, const char*, size_t* len);
static int unpack_string(grib_accessor*, char*, size_t* len);
static size_t string_length(grib_accessor*);
static void init(grib_accessor*, const long, grib_arguments*);
static void init_class(grib_accessor_class*);
typedef struct grib_accessor_trim
{
grib_accessor att;
/* Members defined in gen */
/* Members defined in ascii */
/* Members defined in trim */
const char* input;
int trim_left;
int trim_right;
} grib_accessor_trim;
extern grib_accessor_class* grib_accessor_class_ascii;
static grib_accessor_class _grib_accessor_class_trim = {
&grib_accessor_class_ascii, /* super */
"trim", /* name */
sizeof(grib_accessor_trim), /* size */
0, /* inited */
&init_class, /* init_class */
&init, /* init */
0, /* post_init */
0, /* free mem */
0, /* describes himself */
0, /* get length of section */
&string_length, /* get length of string */
0, /* get number of values */
0, /* get number of bytes */
0, /* get offset to bytes */
0, /* get native type */
0, /* get sub_section */
0, /* grib_pack procedures long */
0, /* grib_pack procedures long */
0, /* grib_pack procedures long */
0, /* grib_unpack procedures long */
0, /* grib_pack procedures double */
0, /* grib_unpack procedures double */
&pack_string, /* grib_pack procedures string */
&unpack_string, /* grib_unpack procedures string */
0, /* grib_pack array procedures string */
0, /* grib_unpack array procedures string */
0, /* grib_pack procedures bytes */
0, /* grib_unpack procedures bytes */
0, /* pack_expression */
0, /* notify_change */
0, /* update_size */
0, /* preferred_size */
0, /* resize */
0, /* nearest_smaller_value */
0, /* next accessor */
0, /* compare vs. another accessor */
0, /* unpack only ith value */
0, /* unpack a subarray */
0, /* clear */
0, /* clone accessor */
};
grib_accessor_class* grib_accessor_class_trim = &_grib_accessor_class_trim;
static void init_class(grib_accessor_class* c)
{
c->dump = (*(c->super))->dump;
c->next_offset = (*(c->super))->next_offset;
c->value_count = (*(c->super))->value_count;
c->byte_count = (*(c->super))->byte_count;
c->byte_offset = (*(c->super))->byte_offset;
c->get_native_type = (*(c->super))->get_native_type;
c->sub_section = (*(c->super))->sub_section;
c->pack_missing = (*(c->super))->pack_missing;
c->is_missing = (*(c->super))->is_missing;
c->pack_long = (*(c->super))->pack_long;
c->unpack_long = (*(c->super))->unpack_long;
c->pack_double = (*(c->super))->pack_double;
c->unpack_double = (*(c->super))->unpack_double;
c->pack_string_array = (*(c->super))->pack_string_array;
c->unpack_string_array = (*(c->super))->unpack_string_array;
c->pack_bytes = (*(c->super))->pack_bytes;
c->unpack_bytes = (*(c->super))->unpack_bytes;
c->pack_expression = (*(c->super))->pack_expression;
c->notify_change = (*(c->super))->notify_change;
c->update_size = (*(c->super))->update_size;
c->preferred_size = (*(c->super))->preferred_size;
c->resize = (*(c->super))->resize;
c->nearest_smaller_value = (*(c->super))->nearest_smaller_value;
c->next = (*(c->super))->next;
c->compare = (*(c->super))->compare;
c->unpack_double_element = (*(c->super))->unpack_double_element;
c->unpack_double_subarray = (*(c->super))->unpack_double_subarray;
c->clear = (*(c->super))->clear;
c->make_clone = (*(c->super))->make_clone;
}
/* END_CLASS_IMP */
static void init(grib_accessor* a, const long l, grib_arguments* arg)
{
int n = 0;
grib_accessor_trim* self = (grib_accessor_trim*)a;
grib_handle* h = grib_handle_of_accessor(a);
self->input = grib_arguments_get_name(h, arg, n++);
self->trim_left = grib_arguments_get_long(h, arg, n++);
self->trim_right= grib_arguments_get_long(h, arg, n++);
DebugAssert(self->trim_left == 0 || self->trim_left == 1);
DebugAssert(self->trim_right == 0 || self->trim_right == 1);
}
static int unpack_string(grib_accessor* a, char* val, size_t* len)
{
grib_accessor_trim* self = (grib_accessor_trim*)a;
int err = 0;
grib_handle* h = grib_handle_of_accessor(a);
char input[256] = {0,};
size_t size = sizeof(input) / sizeof(*input);
char* pInput = input;
err = grib_get_string(h, self->input, input, &size);
if (err) return err;
lrtrim(&pInput, self->trim_left, self->trim_right);
sprintf(val, "%s", pInput);
size = strlen(val);
*len = size + 1;
return GRIB_SUCCESS;
}
static int pack_string(grib_accessor* a, const char* val, size_t* len)
{
char input[256] = {0,};
size_t inputLen = 256;
char buf[256] = {0,};
char* pBuf = NULL;
int err;
grib_handle* h = grib_handle_of_accessor(a);
grib_accessor_trim* self = (grib_accessor_trim*)a;
grib_accessor* inputAccesstor = grib_find_accessor(h, self->input);
if (!inputAccesstor) {
grib_context_log(a->context, GRIB_LOG_ERROR, "Accessor for %s not found", self->input);
return GRIB_NOT_FOUND;
}
if ((err = grib_get_string(h, self->input, input, &inputLen)) != GRIB_SUCCESS)
return err;
sprintf(buf, "%s", val);
pBuf = buf;
lrtrim(&pBuf, self->trim_left, self->trim_right);
return grib_pack_string(inputAccesstor, pBuf, len);
}
static size_t string_length(grib_accessor* a)
{
return 1024;
}

942
src/grib_accessor_classes_hash.c
View File

File diff suppressed because it is too large Load Diff

1
src/grib_accessor_factory.h
View File

@ -207,6 +207,7 @@
{ "to_string", &grib_accessor_class_to_string, },
{ "transient", &grib_accessor_class_transient, },
{ "transient_darray", &grib_accessor_class_transient_darray, },
{ "trim", &grib_accessor_class_trim, },
{ "uint16", &grib_accessor_class_uint16, },
{ "uint16_little_endian", &grib_accessor_class_uint16_little_endian, },
{ "uint32", &grib_accessor_class_uint32, },

1
src/grib_accessor_factory_hash_list
View File

@ -208,6 +208,7 @@ to_integer, &grib_accessor_class_to_integer
to_string, &grib_accessor_class_to_string
transient, &grib_accessor_class_transient
transient_darray, &grib_accessor_class_transient_darray
trim, &grib_accessor_class_trim
uint16, &grib_accessor_class_uint16
uint16_little_endian, &grib_accessor_class_uint16_little_endian
uint32, &grib_accessor_class_uint32

1
src/grib_api_prototypes.h
View File

@ -1494,6 +1494,7 @@ int codes_bufr_header_get_string(codes_bufr_header* bh, const char* key, char* v
/* string_util.c */
int strcmp_nocase(const char* s1, const char* s2);
void rtrim(char* s);
void lrtrim(char** x, int do_left, int do_right);
const char* extract_filename(const char* filepath);
char** string_split(char* inputString, const char* delimiter);
int string_to_long(const char* input, long* output);

2
src/grib_bits_any_endian.c
View File

@ -420,7 +420,7 @@ int grib_encode_size_tb(unsigned char* p, size_t val, long* bitp, long nb)
#if OMP_PACKING
#include "grib_bits_any_endian_omp.c"
#elif VECTOR
#include "grib_bits_any_endian_vector.c" /* Experimental */
#include "grib_bits_any_endian_vector.c"
#else
#include "grib_bits_any_endian_simple.c"
#endif

61
src/grib_bits_any_endian_vector.c
View File

@ -10,16 +10,14 @@
/***************************************************************************
* Enrico Fucile - 19.06.2007 *
* EXPERIMENTAL CODE - NOT FULLY TESTED *
* Johannes Schick, Deutscher Wetterdienst - 24.06.2020 *
***************************************************************************/
int grib_decode_long_array(const unsigned char* p, long* bitp, long bitsPerValue,
size_t n_vals, long* val)
{
long i = 0;
unsigned long lvalue = 0;
/* SUP-3196: Thanks to Daniel Tameling */
if (bitsPerValue % 8 || (*bitp & 7)) {
int j = 0;
for (i = 0; i < n_vals; i++) {
@ -55,6 +53,7 @@ int grib_decode_long_array(const unsigned char* p, long* bitp, long bitsPerValue
return 0;
}
int grib_decode_double_array(const unsigned char* p, long* bitp, long bitsPerValue,
double reference_value, double s, double d,
size_t n_vals, double* val)
@ -96,14 +95,12 @@ int grib_decode_double_array(const unsigned char* p, long* bitp, long bitsPerVal
return 0;
}
int grib_decode_double_array_complex(const unsigned char* p, long* bitp, long nbits, double reference_value,
double s, double* d, size_t size, double* val)
int grib_decode_double_array_complex(const unsigned char* p, long* bitp, long nbits, double reference_value, double s, double* d, size_t size, double* val)
{
return GRIB_NOT_IMPLEMENTED;
}
int grib_encode_double_array(size_t n_vals, const double* val, long bits_per_value, double reference_value,
double d, double divisor, unsigned char* p, long* off)
int grib_encode_double_array(size_t n_vals, const double* val, long bits_per_value, double reference_value, double d, double divisor, unsigned char* p, long* off)
{
size_t i = 0;
unsigned long unsigned_val = 0;
@ -115,15 +112,47 @@ int grib_encode_double_array(size_t n_vals, const double* val, long bits_per_val
}
}
else {
for (i = 0; i < n_vals; i++) {
int blen = 0;
blen = bits_per_value;
unsigned_val = (unsigned long)((((val[i] * d) - reference_value) * divisor) + 0.5);
while (blen >= 8) {
blen -= 8;
*encoded = (unsigned_val >> blen);
encoded++;
*off += 8;
if (bits_per_value == 16 && !((uint64_t)p & 3)) {
uint32_t* encoded4byte = (uint32_t*)p;
uint32_t tmp;
unsigned long unsigned_val2 = 0;
for (i = 0; i < n_vals - 1; i += 2) {
unsigned_val = (unsigned long)((((val[i] * d) - reference_value) * divisor) + 0.5);
unsigned_val2 = (unsigned long)((((val[i + 1] * d) - reference_value) * divisor) + 0.5);
tmp = (unsigned_val2 & 0xff);
tmp = (tmp << 8) + (unsigned_val2 >> 8 & 0xff);
tmp = (tmp << 8) + (unsigned_val & 0xff);
tmp = (tmp << 8) + ((unsigned_val >> 8) & 0xff);
*encoded4byte = tmp;
encoded4byte++;
*off += 32;
}
/* remainder */
if (n_vals % 2) {
int blen = 0;
i = n_vals - 1;
encoded = (unsigned char*)encoded4byte;
blen = bits_per_value;
unsigned_val = (unsigned long)((((val[i] * d) - reference_value) * divisor) + 0.5);
while (blen >= 8) {
blen -= 8;
*encoded = (unsigned_val >> blen);
encoded++;
*off += 8;
}
}
}
else {
for (i = 0; i < n_vals; i++) {
int blen = 0;
blen = bits_per_value;
unsigned_val = (unsigned long)((((val[i] * d) - reference_value) * divisor) + 0.5);
while (blen >= 8) {
blen -= 8;
*encoded = (unsigned_val >> blen);
encoded++;
*off += 8;
}
}
}
}

17
src/grib_dumper_class_bufr_decode_C.c
View File

@ -547,12 +547,13 @@ static void dump_string_array(grib_dumper* d, grib_accessor* a, const char* comm
(void)err; /* TODO */
}
#define MAX_STRING_SIZE 4096
static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
{
grib_dumper_bufr_decode_C* self = (grib_dumper_bufr_decode_C*)d;
char* value = NULL;
char value[MAX_STRING_SIZE] = {0,}; /* See ECC-710 */
size_t size = MAX_STRING_SIZE;
char* p = NULL;
size_t size = 0;
grib_context* c = a->context;
int r = 0, err = 0;
grib_handle* h = grib_handle_of_accessor(a);
@ -560,23 +561,12 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
if ((a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 || (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0)
return;
_grib_get_string_length(a, &size);
if (size == 0)
return;
value = (char*)grib_context_malloc_clear(c, size);
if (!value) {
grib_context_log(c, GRIB_LOG_FATAL, "unable to allocate %d bytes", (int)size);
return;
}
self->empty = 0;
err = grib_unpack_string(a, value, &size);
p = value;
r = compute_bufr_key_rank(h, self->keys, a->name);
if (grib_is_missing_string(a, (unsigned char*)value, size)) {
grib_context_free(c, value);
return;
}
@ -613,7 +603,6 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
depth -= 2;
}
grib_context_free(c, value);
(void)err; /* TODO */
}

24
src/grib_dumper_class_bufr_decode_filter.c
View File

@ -460,31 +460,21 @@ static void dump_string_array(grib_dumper* d, grib_accessor* a, const char* comm
(void)err; /* TODO */
}
#define MAX_STRING_SIZE 4096
static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
{
grib_dumper_bufr_decode_filter* self = (grib_dumper_bufr_decode_filter*)d;
char* value = NULL;
char* p = NULL;
size_t size = 0;
grib_context* c = a->context;
char value[MAX_STRING_SIZE] = {0,}; /* See ECC-710 */
size_t size = MAX_STRING_SIZE;
char* p = NULL;
grib_context* c = a->context;
int r =0, err = 0;
grib_handle* h = grib_handle_of_accessor(a);
if ((a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 || (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0)
return;
_grib_get_string_length(a, &size);
if (size == 0)
return;
value = (char*)grib_context_malloc_clear(c, size);
if (!value) {
grib_context_log(c, GRIB_LOG_FATAL, "unable to allocate %d bytes", (int)size);
return;
}
else
self->begin = 0;
self->begin = 0;
self->empty = 0;
@ -492,7 +482,6 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
p = value;
r = compute_bufr_key_rank(h, self->keys, a->name);
if (grib_is_missing_string(a, (unsigned char*)value, size)) {
grib_context_free(c, value);
return;
}
@ -528,7 +517,6 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
depth -= 2;
}
grib_context_free(c, value);
(void)err; /* TODO */
}

21
src/grib_dumper_class_bufr_decode_fortran.c
View File

@ -489,36 +489,26 @@ static void dump_string_array(grib_dumper* d, grib_accessor* a, const char* comm
(void)err; /* TODO */
}
#define MAX_STRING_SIZE 4096
static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
{
grib_dumper_bufr_decode_fortran* self = (grib_dumper_bufr_decode_fortran*)d;
char* value = NULL;
char* p = NULL;
size_t size = 0;
grib_context* c = a->context;
char value[MAX_STRING_SIZE] = {0,}; /* See ECC-710 */
size_t size = MAX_STRING_SIZE;
char* p = NULL;
grib_context* c = a->context;
int r = 0, err = 0;
grib_handle* h = grib_handle_of_accessor(a);
if ((a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 || (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0)
return;
_grib_get_string_length(a, &size);
if (size == 0)
return;
value = (char*)grib_context_malloc_clear(c, size);
if (!value) {
grib_context_log(c, GRIB_LOG_FATAL, "unable to allocate %d bytes", (int)size);
return;
}
self->empty = 0;
err = grib_unpack_string(a, value, &size);
p = value;
r = compute_bufr_key_rank(h, self->keys, a->name);
if (grib_is_missing_string(a, (unsigned char*)value, size)) {
grib_context_free(c, value);
return;
}
@ -556,7 +546,6 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
depth -= 2;
}
grib_context_free(c, value);
(void)err; /* TODO */
}

17
src/grib_dumper_class_bufr_decode_python.c
View File

@ -502,12 +502,13 @@ static void dump_string_array(grib_dumper* d, grib_accessor* a, const char* comm
(void)err; /* TODO */
}
#define MAX_STRING_SIZE 4096
static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
{
grib_dumper_bufr_decode_python* self = (grib_dumper_bufr_decode_python*)d;
char* value = NULL;
char value[MAX_STRING_SIZE] = {0,}; /* See ECC-710 */
char* p = NULL;
size_t size = 0;
size_t size = MAX_STRING_SIZE;
grib_context* c = a->context;
int r = 0, err = 0;
grib_handle* h = grib_handle_of_accessor(a);
@ -515,23 +516,12 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
if ((a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 || (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0)
return;
_grib_get_string_length(a, &size);
if (size == 0)
return;
value = (char*)grib_context_malloc_clear(c, size);
if (!value) {
grib_context_log(c, GRIB_LOG_FATAL, "unable to allocate %d bytes", (int)size);
return;
}
self->empty = 0;
err = grib_unpack_string(a, value, &size);
p = value;
r = compute_bufr_key_rank(h, self->keys, a->name);
if (grib_is_missing_string(a, (unsigned char*)value, size)) {
grib_context_free(c, value);
return;
}
@ -567,7 +557,6 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
depth -= 2;
}
grib_context_free(c, value);
(void)err; /* TODO */
}

14
src/grib_dumper_class_bufr_encode_C.c
View File

@ -694,18 +694,8 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
if (size == 0)
return;
if ((a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 || (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0) {
/* ECC-356: Solution for the special local section key 'keyMore' and its alias 'ident' */
int skip = 1;
if ((a->flags & GRIB_ACCESSOR_FLAG_HIDDEN) != 0) {
if (strcmp(a->name, "keyMore") == 0 && grib_is_defined(h, "ls.ident")) {
skip = 0;
acc_name = "ident";
}
}
if (skip)
return;
}
if ((a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 || (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0)
return;
value = (char*)grib_context_malloc_clear(c, size);
if (!value) {

14
src/grib_dumper_class_bufr_encode_fortran.c
View File

@ -748,18 +748,8 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
if (size == 0)
return;
if ((a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 || (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0) {
/* ECC-356: Solution for the special local section key 'keyMore' and its alias 'ident' */
int skip = 1;
if ((a->flags & GRIB_ACCESSOR_FLAG_HIDDEN) != 0) {
if (strcmp(a->name, "keyMore") == 0 && grib_is_defined(h, "ls.ident")) {
skip = 0;
acc_name = "ident";
}
}
if (skip)
return;
}
if ((a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 || (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0)
return;
value = (char*)grib_context_malloc_clear(c, size);
if (!value) {

14
src/grib_dumper_class_bufr_encode_python.c
View File

@ -691,18 +691,8 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
if (size == 0)
return;
if ((a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 || (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0) {
/* ECC-356: Solution for the special local section key 'keyMore' and its alias 'ident' */
int skip = 1;
if ((a->flags & GRIB_ACCESSOR_FLAG_HIDDEN) != 0) {
if (strcmp(acc_name, "keyMore") == 0 && grib_is_defined(h, "ls.ident")) {
skip = 0;
acc_name = "ident";
}
}
if (skip)
return;
}
if ((a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 || (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0)
return;
value = (char*)grib_context_malloc_clear(c, size);
if (!value) {

26
src/grib_dumper_class_bufr_simple.c
View File

@ -628,12 +628,13 @@ static void dump_string_array(grib_dumper* d, grib_accessor* a, const char* comm
(void)err; /* TODO */
}
#define MAX_STRING_SIZE 4096
static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
{
grib_dumper_bufr_simple* self = (grib_dumper_bufr_simple*)d;
char* value = NULL;
char value[MAX_STRING_SIZE] = {0,}; /* See ECC-710 */
char* p = NULL;
size_t size = 0;
size_t size = MAX_STRING_SIZE;
grib_context* c = a->context;
int r = 0;
int is_missing = 0;
@ -641,32 +642,14 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
grib_handle* h = grib_handle_of_accessor(a);
const char* acc_name = a->name;
_grib_get_string_length(a, &size);
if (size == 0)
return;
if ((a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0 || (a->flags & GRIB_ACCESSOR_FLAG_READ_ONLY) != 0) {
/* ECC-356: Solution for the special local section key 'keyMore' and its alias 'ident' */
int skip = 1;
if ((a->flags & GRIB_ACCESSOR_FLAG_HIDDEN) != 0) {
if (strcmp(a->name, "keyMore") == 0 && grib_is_defined(h, "ls.ident")) {
skip = 0;
acc_name = "ident";
}
}
if (skip)
return;
}
value = (char*)grib_context_malloc_clear(c, size);
if (!value) {
grib_context_log(c, GRIB_LOG_FATAL, "unable to allocate %d bytes", (int)size);
return;
}
self->empty = 0;
err = grib_unpack_string(a, value, &size);
Assert(size < MAX_STRING_SIZE);
p = value;
r = compute_bufr_key_rank(h, self->keys, acc_name);
if (grib_is_missing_string(a, (unsigned char*)value, size)) {
@ -707,7 +690,6 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
grib_context_free(c, prefix);
}
grib_context_free(c, value);
(void)err; /* TODO */
}

14
src/grib_dumper_class_json.c
View File

@ -450,24 +450,14 @@ static void dump_string(grib_dumper* d, grib_accessor* a, const char* comment)
{
grib_dumper_json* self = (grib_dumper_json*)d;
char value[MAX_STRING_SIZE] = {0,}; /* See ECC-710 */
size_t size = MAX_STRING_SIZE;
char* p = NULL;
size_t size = MAX_STRING_SIZE;
int is_missing = 0;
int err = 0;
const char* acc_name = a->name;
if ((a->flags & GRIB_ACCESSOR_FLAG_DUMP) == 0) {
/* ECC-356: Solution for the special local section key 'keyMore' and its alias 'ident' */
int skip = 1;
if ((a->flags & GRIB_ACCESSOR_FLAG_HIDDEN) != 0) {
grib_handle* h = grib_handle_of_accessor(a);
if (strcmp(a->name, "keyMore") == 0 && grib_is_defined(h, "ls.ident")) {
skip = 0;
acc_name = "ident";
}
}
if (skip)
return;
return;
}
/* ECC-710: It is MUCH slower determining the string length here

20
src/string_util.c
View File

@ -38,6 +38,26 @@ void rtrim(char* s)
s[len] = '\0';
}
void lrtrim(char** x, int do_left, int do_right)
{
DebugAssert(do_left || do_right);
if (do_left) {
while (isspace(**x) && **x != '\0')
(*x)++;
}
if (**x == '\0')
return;
if (do_right) {
char* p = (*x) + strlen(*x) - 1;
while (isspace(*p)) {
*p = '\0';
p--;
}
if (isspace(*p))
*p = '\0';
}
}
/* Return the component after final slash */
/* "/tmp/x" -> "x" */
/* "/tmp/" -> "" */

10
tests/bufr_dump_encode_fortran.sh
View File

@ -25,7 +25,7 @@ mkdir -p $tempDir
bufr_files=`cat ${data_dir}/bufr/bufr_data_files.txt`
# If FORTRAN is enabled, then the pkgconfig should be one level above the test dir
PKGCONFIG_FILE=../../eccodes_f90.pc
PKGCONFIG_FILE=../../lib/pkgconfig/eccodes_f90.pc
CACHE_FILE=../../CMakeCache.txt
COMPILE_AND_RUN=0
@ -69,16 +69,14 @@ do
# valgrind --error-exitcode=1 ./$tempExe
./$tempExe
# ECC-356: have to blacklist 'ident' because of the spaces
${tools_dir}/bufr_compare -b ident ${data_dir}/bufr/$file $tempBufr
${tools_dir}/bufr_compare ${data_dir}/bufr/$file $tempBufr
TEMP_OUT1=${label}.$file.dump.out
TEMP_OUT2=${label}.$tempBufr.dump.out
${tools_dir}/bufr_dump -p ${data_dir}/bufr/$file > $TEMP_OUT1
${tools_dir}/bufr_dump -p $tempBufr > $TEMP_OUT2
# Using the '-w' (--ignore-all-space option) for diff because of the 'ident' key
# See ECC-356
diff -w $TEMP_OUT1 $TEMP_OUT2
diff $TEMP_OUT1 $TEMP_OUT2
rm -f $TEMP_OUT1 $TEMP_OUT2
fi

5
tests/bufr_filter.sh
View File

@ -636,6 +636,7 @@ EOF
#Clean up
rm -f ${dSplit}/*
rmdir ${dSplit}
rm -f $fLog $fRules
rm -f $fBufrTmp
@ -877,7 +878,7 @@ diff ${fOut}.log.ref ${fOut}.log
${tools_dir}/bufr_compare ${fOut} ${fOut}.ref
rm -f ${fOut}.log
rm -f $fLog $fRules ${fOut}
rm -f $fLog $fRules ${fOut} ${fOut}.log.ref
#-----------------------------------------------------------
# Test: add section 2
#-----------------------------------------------------------
@ -909,7 +910,7 @@ EOF
fi
rm -f ${fOut}.log
rm -f $fLog $fRules ${fOut}
rm -f $fLog $fRules ${fOut} ${fOut}.log.ref
#-----------------------------------------------------------
# Test: extract subsets uncompressed data

1
tests/bufr_filter_extract_area.sh
View File

@ -136,6 +136,7 @@ fieldOfViewNumber=7 8 9 10 11 12 13 8 9 10 11 12 11 12
EOF
diff $outputRef $outputFilt
rm -f $outputBufr
# Uncompressed message
# ---------------------

40
tests/bufr_get.sh
View File

@ -10,22 +10,23 @@
. ./include.sh
#Enter data dir
# Enter data dir
cd ${data_dir}/bufr
#Define a common label for all the tmp files
# Define a common label for all the tmp files
label="bufr_get_test"
#Create log file
# Create log file
fLog=${label}".log"
rm -f $fLog
touch $fLog
#Define tmp file
# Define tmp file
fTmp=${label}".tmp.txt"
rm -f $fTmp
tempRef=${label}".tmp.ref.txt"
rm -f $fTmp $tempRef
#Define another tmp file to store the test results
# Define another tmp file to store the test results
res_get=${label}".get.test"
rm -f $res_get
@ -41,14 +42,14 @@ id=`${tools_dir}/bufr_get -p edition,identifier aaen_55.bufr`
#-------------------------------------------
f="aaen_55.bufr"
#The reference is the same as for ls
# The reference is the same as for ls
ref_get=$f".ls.ref"
echo "Test: -p switch" >> $fLog
echo "file: $f" >> $fLog
${tools_dir}/bufr_get -p totalLength,bufrHeaderCentre,bufrHeaderSubCentre,masterTableNumber,masterTablesVersionNumber,localTablesVersionNumber,numberOfSubsets,localNumberOfObservations $f > $fTmp
#Write the values into a file and compare with ref
# Write the values into a file and compare with ref
cat $fTmp | awk '{split($0,a," "); for (i=1; i<=8; i++) print a[i]}' > $res_get
diff $ref_get $res_get
@ -68,17 +69,30 @@ result=`${tools_dir}/bufr_get -s unpack=1 -p satelliteIdentifier wavb_134.bufr`
# ECC-315: BUFR keys in the MARS namespace
#-------------------------------------------
result=`${tools_dir}/bufr_get -m aaen_55.bufr`
[ "$result" = "2 55 2012 11 2 0 0 8 209" ]
[ "$result" = "55 20121102 0000" ]
result=`${tools_dir}/bufr_get -m syno_1.bufr`
[ "$result" = "1 1 2012 10 30 0 0 0 7.45 151.83" ]
[ "$result" = "1 20121030 0000 91334" ]
${tools_dir}/bufr_ls -j -m syno_1.bufr > $fTmp
cat > $tempRef <<EOF
{ "messages" : [
{
"obstype": 1,
"date": 20121030,
"time": "0000",
"ident": 91334
}
]}
EOF
cat $tempRef
diff $tempRef $fTmp
#-------------------------------------------
# Local ECMWF section: 'ident' key
#-------------------------------------------
result=`${tools_dir}/bufr_get -p isSatellite,ident syno_1.bufr`
[ "$result" = "0 91334 " ]
[ "$result" = "0 91334" ]
result=`${tools_dir}/bufr_get -p isSatellite,ident temp_102.bufr`
[ "$result" = "0 ASDE3 " ]
[ "$result" = "0 ASDE3" ]
result=`${tools_dir}/bufr_get -p isSatellite,ident b004_145.bufr`
[ "$result" = "0 FAVRTLZA" ]
@ -87,4 +101,4 @@ result=`${tools_dir}/bufr_get -f -p isSatellite,ident b003_56.bufr`
# Clean up
rm -f $fLog $fTmp $res_get
rm -f $fLog $fTmp $res_get $tempRef

1
tests/bufr_json_data.sh
View File

@ -55,6 +55,7 @@ for c in 1 3 1/3; do
if test "x$JSON_CHECK" != "x"; then
json_xs < ${file}.json >$REDIRECT 2> $REDIRECT
fi
rm -f ${file}.json
done

5
tests/bufr_set.sh
View File

@ -170,6 +170,11 @@ ${tools_dir}/bufr_set -s keyMore=ABCD $f $fBufrTmp
result=`${tools_dir}/bufr_get -p keyMore,ident $fBufrTmp`
[ "$result" = "ABCD ABCD" ]
${tools_dir}/bufr_set -s ident=' AB CD ' $f $fBufrTmp
result=`${tools_dir}/bufr_get -p ident $fBufrTmp`
[ "$result" = "AB CD" ]
# ${tools_dir}/bufr_compare $f $fBufrTmp
#Clean up

32
tests/unit_tests.c
View File

@ -1485,9 +1485,41 @@ static void test_concept_condition_strings()
grib_handle_delete(h);
}
void test_trimming()
{
char a[] = " Standing ";
char b[] = " Weeping ";
char c[] = " Silhouette ";
char d[] = " The Forest Of October ";
char e[] = "\t\n Apostle In Triumph \r ";
char* pA = a;
char* pB = b;
char* pC = c;
char* pD = d;
char* pE = e;
lrtrim(&pA, 0, 1); /*right only*/
assert( strcmp(pA, " Standing")==0 );
lrtrim(&pB, 1, 0); /*left only*/
assert( strcmp(pB, "Weeping ")==0 );
lrtrim(&pC, 1, 1); /*both ends*/
assert( strcmp(pC, "Silhouette")==0 );
lrtrim(&pD, 1, 1); /*make sure other spaces are not removed*/
assert( strcmp(pD, "The Forest Of October")==0 );
lrtrim(&pE, 1, 1); /* Other chars */
assert( strcmp(pE, "Apostle In Triumph")==0 );
}
int main(int argc, char** argv)
{
/*printf("Doing unit tests. ecCodes version = %ld\n", grib_get_api_version());*/
test_trimming();
test_get_git_sha1();
test_concept_condition_strings();

10
tools/bufr_compare.c
View File

@ -1319,16 +1319,6 @@ static int compare_all_dump_keys(grib_handle* handle1, grib_handle* handle2, gri
grib_keys_iterator_delete(iter);
/* ECC-356: Handling special case of 'ident' key */
name = "ls.ident";
if (!blacklisted("ident") && grib_is_defined(handle1, name) && grib_is_defined(handle2, name)) {
if (compare_values(options, handle1, handle2, "ident", GRIB_TYPE_STRING)) {
(*pErr)++;
write_messages(handle1, handle2);
ret = 1;
}
}
return ret;
}

1
windows/msvc/grib_api_lib/grib_api_lib.vcxproj
View File

@ -408,6 +408,7 @@
<ClCompile Include="..\..\..\src\grib_accessor_class_to_string.c" />
<ClCompile Include="..\..\..\src\grib_accessor_class_transient.c" />
<ClCompile Include="..\..\..\src\grib_accessor_class_transient_darray.c" />
<ClCompile Include="..\..\..\src\grib_accessor_class_trim.c" />
<ClCompile Include="..\..\..\src\grib_accessor_class_uint16.c" />
<ClCompile Include="..\..\..\src\grib_accessor_class_uint16_little_endian.c" />
<ClCompile Include="..\..\..\src\grib_accessor_class_uint32.c" />