core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
eccodes/src/grib_io.c

1707 lines
48 KiB
C
Raw Blame History

/*
* (C) Copyright 2005- ECMWF.
*
* This software is licensed under the terms of the Apache Licence Version 2.0
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
*
* In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
* virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
*/
#include "grib_api_internal.h"
#if GRIB_PTHREADS
static pthread_once_t once = PTHREAD_ONCE_INIT;
static pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t mutex2 = PTHREAD_MUTEX_INITIALIZER;
static void init()
{
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&mutex1, &attr);
pthread_mutex_init(&mutex2, &attr);
pthread_mutexattr_destroy(&attr);
}
#elif GRIB_OMP_THREADS
static int once = 0;
static omp_nest_lock_t mutex1;
static omp_nest_lock_t mutex2;
static void init()
{
GRIB_OMP_CRITICAL(lock_grib_io_c)
{
if (once == 0) {
omp_init_nest_lock(&mutex1);
omp_init_nest_lock(&mutex2);
once = 1;
}
}
}
#endif
#define GRIB 0x47524942
#define BUDG 0x42554447
#define DIAG 0x44494147
#define TIDE 0x54494445
#define BUFR 0x42554652
#define HDF5 0x89484446
#define WRAP 0x57524150
#define GRIB_API_READS_BUFR 1
#define GRIB_API_READS_HDF5 1
#define GRIB_API_READS_WRAP 1
typedef struct alloc_buffer
{
size_t size;
void* buffer;
} alloc_buffer;
typedef size_t (*readproc)(void*, void*, size_t, int*);
typedef int (*seekproc)(void*, off_t);
typedef off_t (*tellproc)(void*);
typedef void* (*allocproc)(void*, size_t*, int*);
typedef struct reader
{
void* read_data;
readproc read;
void* alloc_data;
allocproc alloc;
int headers_only;
seekproc seek;
seekproc seek_from_start;
tellproc tell;
off_t offset;
size_t message_size;
} reader;
static int read_the_rest(reader* r, size_t message_length, unsigned char* tmp, int already_read, int check7777)
{
int err = GRIB_SUCCESS;
size_t buffer_size;
size_t rest;
unsigned char* buffer;
if (message_length == 0)
return GRIB_BUFFER_TOO_SMALL;
buffer_size = message_length;
rest = message_length - already_read;
r->message_size = message_length;
buffer = (unsigned char*)r->alloc(r->alloc_data, &buffer_size, &err);
if (err)
return err;
if (buffer == NULL || (buffer_size < message_length)) {
return GRIB_BUFFER_TOO_SMALL;
}
memcpy(buffer, tmp, already_read);
if ((r->read(r->read_data, buffer + already_read, rest, &err) != rest) || err) {
/*fprintf(stderr, "read_the_rest: r->read failed: %s\n", grib_get_error_message(err));*/
return err;
}
if (check7777 && !r->headers_only && (buffer[message_length - 4] != '7' || buffer[message_length - 3] != '7' || buffer[message_length - 2] != '7' || buffer[message_length - 1] != '7')) {
grib_context* c = grib_context_get_default();
grib_context_log(c, GRIB_LOG_DEBUG, "read_the_rest: No final 7777 at expected location (Coded length=%lu)", message_length);
return GRIB_WRONG_LENGTH;
}
return GRIB_SUCCESS;
}
#define CHECK_TMP_SIZE(a) \
if (sizeof(tmp) < (a)) { \
fprintf(stderr, "%s:%d sizeof(tmp)<%s %d<%d\n", __FILE__, __LINE__, #a, (int)sizeof(tmp), (int)(a)); \
return GRIB_INTERNAL_ARRAY_TOO_SMALL; \
}
#define GROW_BUF_IF_REQUIRED(desired_length) \
if (buf->length < (desired_length)) { \
grib_grow_buffer(c, buf, desired_length); \
tmp = buf->data; \
}
#define UINT3(a, b, c) (size_t)((a << 16) + (b << 8) + c);
static int read_GRIB(reader* r)
{
unsigned char* tmp = NULL;
size_t length = 0;
size_t total_length = 0;
long edition = 0;
int err = 0;
int i = 0, j;
size_t sec1len = 0;
size_t sec2len = 0;
size_t sec3len = 0;
size_t sec4len = 0;
unsigned long flags;
size_t buflen = 32768; /* See ECC-515: was 16368 */
grib_context* c;
grib_buffer* buf;
/*TODO proper context*/
c = grib_context_get_default();
tmp = (unsigned char*)malloc(buflen);
if (!tmp)
return GRIB_OUT_OF_MEMORY;
buf = grib_new_buffer(c, tmp, buflen);
buf->property = GRIB_MY_BUFFER;
tmp[i++] = 'G';
tmp[i++] = 'R';
tmp[i++] = 'I';
tmp[i++] = 'B';
r->offset = r->tell(r->read_data) - 4;
if (r->read(r->read_data, &tmp[i], 3, &err) != 3 || err)
return err;
length = UINT3(tmp[i], tmp[i + 1], tmp[i + 2]);
i += 3;
/* Edition number */
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
edition = tmp[i++];
switch (edition) {
case 1:
if (r->headers_only) {
/* Read section 1 length */
if (r->read(r->read_data, &tmp[i], 3, &err) != 3 || err)
return err;
sec1len = UINT3(tmp[i], tmp[i + 1], tmp[i + 2]);
i += 3;
/* Read section 1. 3 = length */
if ((r->read(r->read_data, tmp + i, sec1len - 3, &err) != sec1len - 3) || err)
return err;
flags = tmp[15];
i += sec1len - 3;
GROW_BUF_IF_REQUIRED(i + 3);
if (flags & (1 << 7)) {
/* Section 2 */
if (r->read(r->read_data, &tmp[i], 3, &err) != 3 || err)
return err;
sec2len = UINT3(tmp[i], tmp[i + 1], tmp[i + 2]);
GROW_BUF_IF_REQUIRED(i + sec2len);
i += 3;
/* Read section 2 */
if ((r->read(r->read_data, tmp + i, sec2len - 3, &err) != sec2len - 3) || err)
return err;
i += sec2len - 3;
}
if (flags & (1 << 6)) {
/* Section 3 */
GROW_BUF_IF_REQUIRED(i + 3);
for (j = 0; j < 3; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
sec3len <<= 8;
sec3len |= tmp[i];
i++;
}
/* Read section 3 */
GROW_BUF_IF_REQUIRED(i + sec3len);
if ((r->read(r->read_data, tmp + i, sec3len - 3, &err) != sec3len - 3) || err)
return err;
i += sec3len - 3;
}
GROW_BUF_IF_REQUIRED(i + 11);
/* Section 4 */
for (j = 0; j < 3; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
sec4len <<= 8;
sec4len |= tmp[i];
i++;
}
/* we don't read the data, only headers */
if ((r->read(r->read_data, tmp + i, 8, &err) != 8) || err)
return err;
i += 8;
total_length = length;
/* length=8+sec1len + sec2len+sec3len+11; */
length = i;
err = r->seek(r->read_data, total_length - length - 1);
}
else if (length & 0x800000) {
/* Large GRIBs */
/* Read section 1 length */
for (j = 0; j < 3; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
sec1len <<= 8;
sec1len |= tmp[i];
i++;
}
/* table version */
if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
return err;
/* center */
if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
return err;
/* process */
if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
return err;
/* grid */
if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
return err;
/* flags */
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
flags = tmp[i++];
/* fprintf(stderr," sec1len=%d i=%d flags=%x\n",sec1len,i,flags); */
GROW_BUF_IF_REQUIRED(8 + sec1len + 4 + 3);
/* Read section 1. 3 = length, 5 = table,center,process,grid,flags */
if ((r->read(r->read_data, tmp + i, sec1len - 3 - 5, &err) != sec1len - 3 - 5) || err)
return err;
i += sec1len - 3 - 5;
if (flags & (1 << 7)) {
/* Section 2 */
for (j = 0; j < 3; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
sec2len <<= 8;
sec2len |= tmp[i];
i++;
}
/* Read section 2 */
GROW_BUF_IF_REQUIRED(i + sec2len);
if ((r->read(r->read_data, tmp + i, sec2len - 3, &err) != sec2len - 3) || err)
return err;
i += sec2len - 3;
}
GROW_BUF_IF_REQUIRED(sec1len + sec2len + 4 + 3);
if (flags & (1 << 6)) {
/* Section 3 */
for (j = 0; j < 3; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
sec3len <<= 8;
sec3len |= tmp[i];
i++;
}
/* Read section 3 */
GROW_BUF_IF_REQUIRED(sec1len + sec2len + sec3len + 4 + 3);
if ((r->read(r->read_data, tmp + i, sec3len - 3, &err) != sec3len - 3) || err)
return err;
i += sec3len - 3;
}
/* fprintf(stderr,"%s sec1len=%d i=%d\n",type,sec1len,i); */
GROW_BUF_IF_REQUIRED(sec1len + sec2len + sec3len + 4 + 3);
for (j = 0; j < 3; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
sec4len <<= 8;
sec4len |= tmp[i];
i++;
}
if (sec4len < 120) {
/* Special coding */
length &= 0x7fffff;
length *= 120;
length -= sec4len;
length += 4;
}
else {
/* length is already set to the right value */
}
}
break;
case 2:
case 3:
length = 0;
if (sizeof(long) >= 8) {
for (j = 0; j < 8; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
length <<= 8;
length |= tmp[i];
i++;
}
}
else {
/* Check if the length fits in a long */
for (j = 0; j < 4; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
length <<= 8;
length |= tmp[i];
i++;
}
if (length)
return GRIB_MESSAGE_TOO_LARGE; /* Message too large */
for (j = 0; j < 4; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
length <<= 8;
length |= tmp[i];
i++;
}
}
break;
default:
r->seek_from_start(r->read_data, r->offset + 4);
return GRIB_UNSUPPORTED_EDITION;
break;
}
/* Assert(i <= buf->length); */
err = read_the_rest(r, length, tmp, i, 1);
if (err)
r->seek_from_start(r->read_data, r->offset + 4);
grib_buffer_delete(c, buf);
return err;
}
static int read_PSEUDO(reader* r, const char* type)
{
unsigned char tmp[32]; /* Should be enough */
size_t sec1len = 0;
size_t sec4len = 0;
int err = 0;
int i = 0, j = 0;
Assert(strlen(type) == 4);
for (j = 0; j < 4; j++) {
tmp[i] = type[i];
i++;
}
for (j = 0; j < 3; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
sec1len <<= 8;
sec1len |= tmp[i];
i++;
}
/* fprintf(stderr,"%s sec1len=%d i=%d\n",type,sec1len,i); */
CHECK_TMP_SIZE(sec1len + 4 + 3);
/* Read sectoin1 */
if ((r->read(r->read_data, tmp + i, sec1len - 3, &err) != sec1len - 3) || err)
return err;
i += sec1len - 3;
for (j = 0; j < 3; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
sec4len <<= 8;
sec4len |= tmp[i];
i++;
}
/* fprintf(stderr,"%s sec4len=%d i=%d l=%d\n",type,sec4len,i,4+sec1len+sec4len+4); */
Assert(i <= sizeof(tmp));
return read_the_rest(r, 4 + sec1len + sec4len + 4, tmp, i, 1);
}
static int read_HDF5_offset(reader* r, int length, unsigned long* v, unsigned char* tmp, int* i)
{
unsigned char buf[8];
int j, k;
int err = 0;
if ((r->read(r->read_data, buf, length, &err) != length) || err) {
return err;
}
k = *i;
for (j = 0; j < length; j++) {
tmp[k++] = buf[j];
}
*i = k;
*v = 0;
for (j = length - 1; j >= 0; j--) {
*v <<= 8;
*v |= buf[j];
}
return 0;
}
static int read_HDF5(reader* r)
{
/*
* See: http://www.hdfgroup.org/HDF5/doc/H5.format.html#Superblock
*/
unsigned char tmp[49]; /* Should be enough */
unsigned char buf[4];
unsigned char version_of_superblock, size_of_offsets, size_of_lengths, consistency_flags;
unsigned long base_address, superblock_extension_address, end_of_file_address;
int i = 0, j;
int err = 0;
grib_context* c = grib_context_get_default();
tmp[i++] = 137;
tmp[i++] = 'H';
tmp[i++] = 'D';
tmp[i++] = 'F';
if ((r->read(r->read_data, buf, 4, &err) != 4) || err) {
return err;
}
if (!(buf[0] == '\r' && buf[1] == '\n' && buf[2] == 26 && buf[3] == '\n')) {
/* Invalid magic, we should not use grib_context_log without a context */
grib_context_log(c, GRIB_LOG_ERROR, "read_HDF5: invalid signature");
return GRIB_INVALID_MESSAGE;
}
for (j = 0; j < 4; j++) {
tmp[i++] = buf[j];
}
if ((r->read(r->read_data, &version_of_superblock, 1, &err) != 1) || err) {
return err;
}
tmp[i++] = version_of_superblock;
if (version_of_superblock == 2 || version_of_superblock == 3) {
if ((r->read(r->read_data, &size_of_offsets, 1, &err) != 1) || err) {
return err;
}
tmp[i++] = size_of_offsets;
if (size_of_offsets > 8) {
grib_context_log(c, GRIB_LOG_ERROR, "read_HDF5: invalid size_of_offsets: %ld, only <= 8 is supported", (long)size_of_offsets);
return GRIB_NOT_IMPLEMENTED;
}
if ((r->read(r->read_data, &size_of_lengths, 1, &err) != 1) || err) {
return err;
}
tmp[i++] = size_of_lengths;
if ((r->read(r->read_data, &consistency_flags, 1, &err) != 1) || err) {
return err;
}
tmp[i++] = consistency_flags;
err = read_HDF5_offset(r, size_of_offsets, &base_address, tmp, &i);
if (err) {
return err;
}
err = read_HDF5_offset(r, size_of_offsets, &superblock_extension_address, tmp, &i);
if (err) {
return err;
}
err = read_HDF5_offset(r, size_of_offsets, &end_of_file_address, tmp, &i);
if (err) {
return err;
}
}
else if (version_of_superblock == 0 || version_of_superblock == 1) {
char skip[4];
unsigned long file_free_space_info;
unsigned char version_of_file_free_space, version_of_root_group_symbol_table, version_number_shared_header, ch;
if ((r->read(r->read_data, &version_of_file_free_space, 1, &err) != 1) || err)
return err;
tmp[i++] = version_of_file_free_space;
if ((r->read(r->read_data, &version_of_root_group_symbol_table, 1, &err) != 1) || err)
return err;
tmp[i++] = version_of_root_group_symbol_table;
if ((r->read(r->read_data, &ch, 1, &err) != 1) || err)
return err; /* reserved */
tmp[i++] = ch;
if ((r->read(r->read_data, &version_number_shared_header, 1, &err) != 1) || err)
return err;
tmp[i++] = version_number_shared_header;
if ((r->read(r->read_data, &size_of_offsets, 1, &err) != 1) || err)
return err;
tmp[i++] = size_of_offsets;
if (size_of_offsets > 8) {
grib_context_log(c, GRIB_LOG_ERROR, "read_HDF5: invalid size_of_offsets: %ld, only <= 8 is supported", (long)size_of_offsets);
return GRIB_NOT_IMPLEMENTED;
}
if ((r->read(r->read_data, &size_of_lengths, 1, &err) != 1) || err)
return err;
tmp[i++] = size_of_lengths;
if ((r->read(r->read_data, &ch, 1, &err) != 1) || err)
return err; /*reserved*/
tmp[i++] = ch;
if ((r->read(r->read_data, &skip, 4, &err) != 4) || err)
return err; /* Group Leaf/Internal Node K: 4 bytes */
tmp[i++] = skip[0];
tmp[i++] = skip[1];
tmp[i++] = skip[2];
tmp[i++] = skip[3];
if ((r->read(r->read_data, &skip, 4, &err) != 4) || err)
return err; /* consistency_flags: 4 bytes */
tmp[i++] = skip[0];
tmp[i++] = skip[1];
tmp[i++] = skip[2];
tmp[i++] = skip[3];
if (version_of_superblock == 1) {
/* Indexed storage internal node K and reserved: only in version 1 of superblock */
if ((r->read(r->read_data, &skip, 4, &err) != 4) || err)
return err;
tmp[i++] = skip[0];
tmp[i++] = skip[1];
tmp[i++] = skip[2];
tmp[i++] = skip[3];
}
err = read_HDF5_offset(r, size_of_offsets, &base_address, tmp, &i);
if (err)
return err;
err = read_HDF5_offset(r, size_of_offsets, &file_free_space_info, tmp, &i);
if (err)
return err;
err = read_HDF5_offset(r, size_of_offsets, &end_of_file_address, tmp, &i);
if (err)
return err;
}
else {
grib_context_log(c, GRIB_LOG_ERROR, "read_HDF5: invalid version of superblock: %ld", (long)version_of_superblock);
return GRIB_NOT_IMPLEMENTED;
}
Assert(i <= sizeof(tmp));
return read_the_rest(r, end_of_file_address, tmp, i, 0);
}
static int read_WRAP(reader* r)
{
/*
* See: http://www.hdfgroup.org/HDF5/doc/H5.format.html#Superblock
*/
unsigned char tmp[36]; /* Should be enough */
unsigned char buf[8];
unsigned long long length = 0;
int i = 0, j;
int err = 0;
tmp[i++] = 'W';
tmp[i++] = 'R';
tmp[i++] = 'A';
tmp[i++] = 'P';
if ((r->read(r->read_data, buf, 8, &err) != 8) || err) {
printf("error\n");
return err;
}
for (j = 0; j < 8; j++) {
length <<= 8;
length |= buf[j];
tmp[i++] = buf[j];
}
Assert(i <= sizeof(tmp));
return read_the_rest(r, length, tmp, i, 1);
}
static int read_BUFR(reader* r)
{
/* unsigned char tmp[65536];*/ /* Should be enough */
size_t length = 0;
long edition = 0;
int err = 0;
int i = 0, j;
size_t buflen = 2048;
unsigned char* tmp = NULL;
grib_context* c = NULL;
grib_buffer* buf = NULL;
/*TODO proper context*/
c = grib_context_get_default();
tmp = (unsigned char*)malloc(buflen);
if (!tmp)
return GRIB_OUT_OF_MEMORY;
buf = grib_new_buffer(c, tmp, buflen);
buf->property = GRIB_MY_BUFFER;
r->offset = r->tell(r->read_data) - 4;
tmp[i++] = 'B';
tmp[i++] = 'U';
tmp[i++] = 'F';
tmp[i++] = 'R';
for (j = 0; j < 3; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
length <<= 8;
length |= tmp[i];
i++;
}
if (length == 0) {
grib_buffer_delete(c, buf);
return GRIB_INVALID_MESSAGE;
}
/* Edition number */
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
edition = tmp[i++];
/* Assert(edition != 1); */
switch (edition) {
case 0:
case 1: {
int n;
size_t sec1len = 0;
size_t sec2len = 0;
size_t sec3len = 0;
size_t sec4len = 0;
unsigned long flags;
sec1len = length;
/* table version */
if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
return err;
/* center */
if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
return err;
/* update */
if (r->read(r->read_data, &tmp[i++], 1, &err) != 1 || err)
return err;
/* flags */
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
flags = tmp[i++];
GROW_BUF_IF_REQUIRED(sec1len + 4 + 3);
/* Read section 1. 3 = length, 5 = table,center,process,flags */
n = sec1len - 8; /* Just a guess */
if ((r->read(r->read_data, tmp + i, n, &err) != n) || err)
return err;
i += n;
if (flags & (1 << 7)) {
/* Section 2 */
for (j = 0; j < 3; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
sec2len <<= 8;
sec2len |= tmp[i];
i++;
}
GROW_BUF_IF_REQUIRED(sec1len + sec2len + 4 + 3);
/* Read section 2 */
if ((r->read(r->read_data, tmp + i, sec2len - 3, &err) != sec2len - 3) || err)
return err;
i += sec2len - 3;
}
/* Section 3 */
for (j = 0; j < 3; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
sec3len <<= 8;
sec3len |= tmp[i];
i++;
}
GROW_BUF_IF_REQUIRED(sec1len + sec2len + sec3len + 4 + 3);
/* Read section 3 */
if ((r->read(r->read_data, tmp + i, sec3len - 3, &err) != sec3len - 3) || err)
return err;
i += sec3len - 3;
for (j = 0; j < 3; j++) {
if (r->read(r->read_data, &tmp[i], 1, &err) != 1 || err)
return err;
sec4len <<= 8;
sec4len |= tmp[i];
i++;
}
/* fprintf(stderr," sec1len=%d sec2len=%d sec3len=%d sec4len=%d\n",sec1len, sec2len,sec3len,sec4len); */
length = 4 + sec1len + sec2len + sec3len + sec4len + 4;
/* fprintf(stderr,"length = %d i = %d\n",length,i); */
} break;
case 2:
case 3:
case 4:
break;
default:
r->seek_from_start(r->read_data, r->offset + 4);
grib_buffer_delete(c, buf);
return GRIB_UNSUPPORTED_EDITION;
}
/* Assert(i <= sizeof(tmp)); */
err = read_the_rest(r, length, tmp, i, 1);
if (err)
r->seek_from_start(r->read_data, r->offset + 4);
grib_buffer_delete(c, buf);
return err;
}
static int _read_any(reader* r, int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok)
{
unsigned char c;
int err = 0;
unsigned long magic = 0;
while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
magic <<= 8;
magic |= c;
switch (magic & 0xffffffff) {
case GRIB:
if (grib_ok) {
err = read_GRIB(r);
return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
}
break;
case BUFR:
if (bufr_ok) {
err = read_BUFR(r);
return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
}
break;
case HDF5:
if (hdf5_ok) {
err = read_HDF5(r);
return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
}
break;
case WRAP:
if (wrap_ok) {
err = read_WRAP(r);
return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
}
break;
case BUDG:
if (grib_ok) {
err = read_PSEUDO(r, "BUDG");
return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
}
break;
case DIAG:
if (grib_ok) {
err = read_PSEUDO(r, "DIAG");
return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
}
break;
case TIDE:
if (grib_ok) {
err = read_PSEUDO(r, "TIDE");
return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
}
break;
}
}
return err;
}
static int read_any(reader* r, int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok)
{
int result = 0;
GRIB_MUTEX_INIT_ONCE(&once, &init);
GRIB_MUTEX_LOCK(&mutex1);
result = _read_any(r, grib_ok, bufr_ok, hdf5_ok, wrap_ok);
GRIB_MUTEX_UNLOCK(&mutex1);
return result;
}
static int read_any_gts(reader* r)
{
unsigned char c;
int err = 0;
unsigned char* buffer = NULL;
unsigned long magic = 0;
unsigned long start = 0x010d0d0a; /* SOH CR CR LF */
unsigned long theEnd = 0x0d0d0a03; /* CR CR LF ETX */
unsigned char tmp[1024] = {0,}; /* See ECC-735 */
size_t message_size = 0;
size_t already_read = 0;
int i = 0;
while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
magic <<= 8;
magic |= c;
magic &= 0xffffffff;
if (magic == start) {
tmp[i++] = 0x01;
tmp[i++] = 0x0d;
tmp[i++] = 0x0d;
tmp[i++] = 0x0a;
r->offset = r->tell(r->read_data) - 4;
if (r->read(r->read_data, &tmp[i], 6, &err) != 6 || err)
return err == GRIB_END_OF_FILE ? GRIB_PREMATURE_END_OF_FILE : err; /* Premature EOF */
if (tmp[7] != 0x0d || tmp[8] != 0x0d || tmp[9] != 0x0a) {
r->seek(r->read_data, -6);
continue;
}
magic = 0;
already_read = 10;
message_size = already_read;
while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
message_size++;
magic <<= 8;
magic |= c;
magic &= 0xffffffff;
if (magic == theEnd) {
r->seek(r->read_data, already_read - message_size);
buffer = (unsigned char*)r->alloc(r->alloc_data, &message_size, &err);
if (!buffer)
return GRIB_OUT_OF_MEMORY;
if (err)
return err;
memcpy(buffer, tmp, already_read);
r->read(r->read_data, buffer + already_read, message_size - already_read, &err);
r->message_size = message_size;
return err;
}
}
}
}
return err;
}
static int read_any_taf(reader* r)
{
unsigned char c;
int err = 0;
unsigned char* buffer = NULL;
unsigned long magic = 0;
unsigned long start = 0x54414620;
unsigned char tmp[1000] = {0,}; /* Should be enough */
size_t message_size = 0;
size_t already_read = 0;
int i = 0;
while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
magic <<= 8;
magic |= c;
magic &= 0xffffffff;
if (magic == start) {
tmp[i++] = 0x54;
tmp[i++] = 0x41;
tmp[i++] = 0x46;
tmp[i++] = 0x20;
r->offset = r->tell(r->read_data) - 4;
already_read = 4;
message_size = already_read;
while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
message_size++;
if (c == '=') {
r->seek(r->read_data, already_read - message_size);
buffer = (unsigned char*)r->alloc(r->alloc_data, &message_size, &err);
if (!buffer)
return GRIB_OUT_OF_MEMORY;
if (err)
return err;
memcpy(buffer, tmp, already_read);
r->read(r->read_data, buffer + already_read, message_size - already_read, &err);
r->message_size = message_size;
return err;
}
}
}
}
return err;
}
static int read_any_metar(reader* r)
{
unsigned char c;
int err = 0;
unsigned char* buffer = NULL;
unsigned long magic = 0;
unsigned long start = 0x4d455441;
unsigned char tmp[32] = {0,}; /* Should be enough */
size_t message_size = 0;
size_t already_read = 0;
int i = 0;
while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
magic <<= 8;
magic |= c;
magic &= 0xffffffff;
if (magic == start) {
if (r->read(r->read_data, &c, 1, &err) != 1 || err != 0)
break;
if (c == 'R') {
tmp[i++] = 0x4d;
tmp[i++] = 0x45;
tmp[i++] = 0x54;
tmp[i++] = 0x41;
tmp[i++] = 'R';
r->offset = r->tell(r->read_data) - 4;
already_read = 5;
message_size = already_read;
while (r->read(r->read_data, &c, 1, &err) == 1 && err == 0) {
message_size++;
if (c == '=') {
r->seek(r->read_data, already_read - message_size);
buffer = (unsigned char*)r->alloc(r->alloc_data, &message_size, &err);
if (!buffer)
return GRIB_OUT_OF_MEMORY;
if (err)
return err;
memcpy(buffer, tmp, already_read);
r->read(r->read_data, buffer + already_read, message_size - already_read, &err);
r->message_size = message_size;
return err;
}
}
}
}
}
return err;
}
off_t stdio_tell(void* data)
{
FILE* f = (FILE*)data;
return ftello(f);
}
int stdio_seek(void* data, off_t len)
{
FILE* f = (FILE*)data;
int err = 0;
if (fseeko(f, len, SEEK_CUR))
err = GRIB_IO_PROBLEM;
return err;
}
int stdio_seek_from_start(void* data, off_t len)
{
FILE* f = (FILE*)data;
int err = 0;
if (fseeko(f, len, SEEK_SET))
err = GRIB_IO_PROBLEM;
return err;
}
size_t stdio_read(void* data, void* buf, size_t len, int* err)
{
FILE* f = (FILE*)data;
size_t n;
/* char iobuf[1024*1024]; */
if (len == 0)
return 0;
/* setvbuf(f,iobuf,_IOFBF,sizeof(iobuf)); */
n = fread(buf, 1, len, f);
/* fprintf(stderr,"read %d = %x %c\n",1,(int)buf[0],buf[0]); */
if (n != len) {
/* fprintf(stderr,"Failed to read %d, only got %d\n",len,n); */
*err = GRIB_IO_PROBLEM;
if (feof(f))
*err = GRIB_END_OF_FILE;
if (ferror(f))
*err = GRIB_IO_PROBLEM;
}
return n;
}
/*================== */
typedef struct user_buffer
{
void* user_buffer;
size_t buffer_size;
} user_buffer;
static void* user_provider_buffer(void* data, size_t* length, int* err)
{
user_buffer* u = (user_buffer*)data;
*length = u->buffer_size;
return u->user_buffer;
}
static int _wmo_read_any_from_file(FILE* f, void* buffer, size_t* len, int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok)
{
int err;
user_buffer u;
reader r;
u.user_buffer = buffer;
u.buffer_size = *len;
r.message_size = 0;
r.read_data = f;
r.read = &stdio_read;
r.seek = &stdio_seek;
r.seek_from_start = &stdio_seek_from_start;
r.tell = &stdio_tell;
r.alloc_data = &u;
r.alloc = &user_provider_buffer;
r.headers_only = 0;
err = read_any(&r, grib_ok, bufr_ok, hdf5_ok, wrap_ok);
*len = r.message_size;
return err;
}
int wmo_read_any_from_file(FILE* f, void* buffer, size_t* len)
{
return _wmo_read_any_from_file(f, buffer, len, 1, 1, 1, 1);
}
int wmo_read_grib_from_file(FILE* f, void* buffer, size_t* len)
{
return _wmo_read_any_from_file(f, buffer, len, 1, 0, 0, 0);
}
int wmo_read_bufr_from_file(FILE* f, void* buffer, size_t* len)
{
return _wmo_read_any_from_file(f, buffer, len, 0, 1, 0, 0);
}
int wmo_read_gts_from_file(FILE* f, void* buffer, size_t* len)
{
int err;
user_buffer u;
reader r;
u.user_buffer = buffer;
u.buffer_size = *len;
r.message_size = 0;
r.read_data = f;
r.read = &stdio_read;
r.seek = &stdio_seek;
r.seek_from_start = &stdio_seek_from_start;
r.tell = &stdio_tell;
r.alloc_data = &u;
r.alloc = &user_provider_buffer;
r.headers_only = 0;
err = read_any_gts(&r);
*len = r.message_size;
return err;
}
int wmo_read_taf_from_file(FILE* f, void* buffer, size_t* len)
{
int err;
user_buffer u;
reader r;
u.user_buffer = buffer;
u.buffer_size = *len;
r.read_data = f;
r.read = &stdio_read;
r.seek = &stdio_seek;
r.seek_from_start = &stdio_seek_from_start;
r.tell = &stdio_tell;
r.alloc_data = &u;
r.alloc = &user_provider_buffer;
r.headers_only = 0;
err = read_any_taf(&r);
*len = r.message_size;
return err;
}
int wmo_read_metar_from_file(FILE* f, void* buffer, size_t* len)
{
int err;
user_buffer u;
reader r;
u.user_buffer = buffer;
u.buffer_size = *len;
r.read_data = f;
r.read = &stdio_read;
r.seek = &stdio_seek;
r.seek_from_start = &stdio_seek_from_start;
r.tell = &stdio_tell;
r.alloc_data = &u;
r.alloc = &user_provider_buffer;
r.headers_only = 0;
err = read_any_metar(&r);
*len = r.message_size;
return err;
}
/*================== */
typedef struct stream_struct
{
void* stream_data;
long (*stream_proc)(void*, void* buffer, long len);
} stream_struct;
static off_t stream_tell(void* data)
{
return 0;
}
static int stream_seek(void* data, off_t len)
{
return 0;
}
static size_t stream_read(void* data, void* buffer, size_t len, int* err)
{
stream_struct* s = (stream_struct*)data;
long n = len;
if (n != len) {
/* size_t cannot be coded into long */
*err = GRIB_INTERNAL_ERROR;
return -1;
}
n = s->stream_proc(s->stream_data, buffer, len);
if (n != len) {
*err = GRIB_IO_PROBLEM;
if (n == -1)
*err = GRIB_END_OF_FILE;
}
return n;
}
/*================== */
static void* allocate_buffer(void* data, size_t* length, int* err)
{
alloc_buffer* u = (alloc_buffer*)data;
u->buffer = malloc(*length);
u->size = *length;
if (u->buffer == NULL)
*err = GRIB_OUT_OF_MEMORY; /* Cannot allocate buffer */
return u->buffer;
}
int wmo_read_any_from_stream(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), void* buffer, size_t* len)
{
int err;
stream_struct s;
user_buffer u;
reader r;
s.stream_data = stream_data;
s.stream_proc = stream_proc;
u.user_buffer = buffer;
u.buffer_size = *len;
r.message_size = 0;
r.offset = 0;
r.read_data = &s;
r.read = &stream_read;
r.seek = &stream_seek;
r.seek_from_start = &stream_seek;
r.tell = &stream_tell;
r.alloc_data = &u;
r.alloc = &user_provider_buffer;
r.headers_only = 0;
err = read_any(&r, 1, 1, 1, 1);
*len = r.message_size;
return err;
}
/* This function allocates memory for the result so the user is responsible for freeing it */
void* wmo_read_any_from_stream_malloc(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), size_t* size, int* err)
{
alloc_buffer u;
stream_struct s;
reader r;
u.buffer = NULL;
s.stream_data = stream_data;
s.stream_proc = stream_proc;
r.message_size = 0;
r.offset = 0;
r.read_data = &s;
r.read = &stream_read;
r.seek = &stream_seek;
r.seek_from_start = &stream_seek;
r.tell = &stream_tell;
r.alloc_data = &u;
r.alloc = &allocate_buffer;
r.headers_only = 0;
*err = read_any(&r, 1, 1, 1, 1);
*size = r.message_size;
return u.buffer;
}
/*================== */
/* This function allocates memory for the result so the user is responsible for freeing it */
void* wmo_read_gts_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err)
{
alloc_buffer u;
reader r;
u.buffer = NULL;
r.offset = 0;
r.message_size = 0;
r.read_data = f;
r.read = &stdio_read;
r.seek = &stdio_seek;
r.seek_from_start = &stdio_seek_from_start;
r.tell = &stdio_tell;
r.alloc_data = &u;
r.alloc = &allocate_buffer;
r.headers_only = headers_only;
*err = read_any_gts(&r);
*size = r.message_size;
*offset = r.offset;
return u.buffer;
}
/* This function allocates memory for the result so the user is responsible for freeing it */
void* wmo_read_taf_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err)
{
alloc_buffer u;
reader r;
u.buffer = NULL;
r.offset = 0;
r.message_size = 0;
r.read_data = f;
r.read = &stdio_read;
r.seek = &stdio_seek;
r.seek_from_start = &stdio_seek_from_start;
r.tell = &stdio_tell;
r.alloc_data = &u;
r.alloc = &allocate_buffer;
r.headers_only = headers_only;
*err = read_any_taf(&r);
*size = r.message_size;
*offset = r.offset;
return u.buffer;
}
/* This function allocates memory for the result so the user is responsible for freeing it */
void* wmo_read_metar_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err)
{
alloc_buffer u;
reader r;
u.buffer = NULL;
r.message_size = 0;
r.read_data = f;
r.offset = 0;
r.read = &stdio_read;
r.seek = &stdio_seek;
r.seek_from_start = &stdio_seek_from_start;
r.tell = &stdio_tell;
r.alloc_data = &u;
r.alloc = &allocate_buffer;
r.headers_only = headers_only;
*err = read_any_metar(&r);
*size = r.message_size;
*offset = r.offset;
return u.buffer;
}
/* This function allocates memory for the result so the user is responsible for freeing it */
static void* _wmo_read_any_from_file_malloc(FILE* f, int* err, size_t* size, off_t* offset,
int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok, int headers_only)
{
alloc_buffer u;
reader r;
u.buffer = NULL;
u.size = 0;
r.message_size = 0;
r.read_data = f;
r.read = &stdio_read;
r.seek = &stdio_seek;
r.seek_from_start = &stdio_seek_from_start;
r.tell = &stdio_tell;
r.alloc_data = &u;
r.alloc = &allocate_buffer;
r.headers_only = headers_only;
r.offset = 0;
*err = read_any(&r, grib_ok, bufr_ok, hdf5_ok, wrap_ok);
*size = r.message_size;
*offset = r.offset;
return u.buffer;
}
/* This function allocates memory for the result so the user is responsible for freeing it */
void* wmo_read_any_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err)
{
return _wmo_read_any_from_file_malloc(f, err, size, offset, 1, 1, 1, 1, headers_only);
}
/* This function allocates memory for the result so the user is responsible for freeing it */
void* wmo_read_grib_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err)
{
return _wmo_read_any_from_file_malloc(f, err, size, offset, 1, 0, 0, 0, headers_only);
}
/* This function allocates memory for the result so the user is responsible for freeing it */
void* wmo_read_bufr_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err)
{
return _wmo_read_any_from_file_malloc(f, err, size, offset, 0, 1, 0, 0, headers_only);
}
/* ======================================= */
typedef struct context_alloc_buffer
{
grib_context* ctx;
void* buffer;
size_t length;
} context_alloc_buffer;
static void* context_allocate_buffer(void* data, size_t* length, int* err)
{
context_alloc_buffer* u = (context_alloc_buffer*)data;
u->buffer = grib_context_malloc(u->ctx, *length);
u->length = *length;
if (u->buffer == NULL)
*err = GRIB_OUT_OF_MEMORY; /* Cannot allocate buffer */
return u->buffer;
}
int grib_read_any_headers_only_from_file(grib_context* ctx, FILE* f, void* buffer, size_t* len)
{
int err;
user_buffer u;
reader r;
u.user_buffer = buffer;
u.buffer_size = *len;
r.message_size = 0;
r.read_data = f;
r.read = &stdio_read;
r.seek = &stdio_seek;
r.seek_from_start = &stdio_seek_from_start;
r.tell = &stdio_tell;
r.alloc_data = &u;
r.alloc = &user_provider_buffer;
r.headers_only = 1;
err = read_any(&r, 1, GRIB_API_READS_BUFR, GRIB_API_READS_HDF5, GRIB_API_READS_WRAP);
*len = r.message_size;
return err;
}
int grib_read_any_from_file(grib_context* ctx, FILE* f, void* buffer, size_t* len)
{
int err;
user_buffer u;
reader r;
off_t offset;
u.user_buffer = buffer;
u.buffer_size = *len;
r.message_size = 0;
r.read_data = f;
r.read = &stdio_read;
r.seek = &stdio_seek;
r.seek_from_start = &stdio_seek_from_start;
r.tell = &stdio_tell;
r.alloc_data = &u;
r.alloc = &user_provider_buffer;
r.headers_only = 0;
offset = ftello(f);
err = read_any(&r, 1, GRIB_API_READS_BUFR, GRIB_API_READS_HDF5, GRIB_API_READS_WRAP);
if (err == GRIB_BUFFER_TOO_SMALL) {
if (fseeko(f, offset, SEEK_SET))
err = GRIB_IO_PROBLEM;
}
*len = r.message_size;
return err;
}
/* ======================================= */
typedef struct memory_read_data
{
unsigned char* data;
size_t data_len;
} memory_read_data;
static off_t memory_tell(void* data)
{
return 0;
}
static int memory_seek(void* data, off_t len)
{
return 0;
}
static size_t memory_read(void* data, void* buf, size_t len, int* err)
{
memory_read_data* m = (memory_read_data*)data;
if (len == 0) {
*err = GRIB_END_OF_FILE;
return 0;
}
else {
size_t l = len > m->data_len ? m->data_len : len;
memcpy(buf, m->data, l);
m->data_len -= l;
m->data += l;
return l;
}
}
int grib_read_any_from_memory_alloc(grib_context* ctx, unsigned char** data, size_t* data_length, void** buffer, size_t* length)
{
int err;
memory_read_data m;
context_alloc_buffer u;
reader r;
m.data = *data;
m.data_len = *data_length;
u.buffer = NULL;
u.length = 0;
u.ctx = ctx ? ctx : grib_context_get_default();
r.message_size = 0;
r.read_data = &m;
r.read = &memory_read;
r.seek = &memory_seek;
r.seek_from_start = &memory_seek;
r.tell = &memory_tell;
r.alloc_data = &u;
r.alloc = &context_allocate_buffer;
r.headers_only = 0;
err = read_any(&r, 1, GRIB_API_READS_BUFR, GRIB_API_READS_HDF5, GRIB_API_READS_WRAP);
*buffer = u.buffer;
*length = u.length;
*data_length = m.data_len;
*data = m.data;
return err;
}
int grib_read_any_from_memory(grib_context* ctx, unsigned char** data, size_t* data_length, void* buffer, size_t* len)
{
int err;
memory_read_data m;
user_buffer u;
reader r;
m.data = *data;
m.data_len = *data_length;
u.user_buffer = buffer;
u.buffer_size = *len;
r.message_size = 0;
r.read_data = &m;
r.read = &memory_read;
r.seek = &memory_seek;
r.seek_from_start = &memory_seek;
r.tell = &memory_tell;
r.alloc_data = &u;
r.alloc = &user_provider_buffer;
r.headers_only = 0;
err = read_any(&r, 1, GRIB_API_READS_BUFR, GRIB_API_READS_HDF5, GRIB_API_READS_WRAP);
*len = r.message_size;
*data_length = m.data_len;
*data = m.data;
return err;
}
int grib_count_in_file(grib_context* c, FILE* f, int* n)
{
int err = 0;
*n = 0;
if (!c)
c = grib_context_get_default();
if (c->multi_support_on) {
/* GRIB-395 */
grib_handle* h = NULL;
while ((h = grib_handle_new_from_file(c, f, &err)) != NULL) {
grib_handle_delete(h);
(*n)++;
}
}
else {
void* mesg = NULL;
size_t size = 0;
off_t offset = 0;
while ((mesg = wmo_read_any_from_file_malloc(f, 0, &size, &offset, &err)) != NULL && err == GRIB_SUCCESS) {
grib_context_free(c, mesg);
(*n)++;
}
}
rewind(f);
return err == GRIB_END_OF_FILE ? 0 : err;
}
int grib_count_in_filename(grib_context* c, const char* filename, int* n)
{
int err = 0;
FILE* fp = NULL;
if (!c)
c = grib_context_get_default();
fp = fopen(filename, "rb");
if (!fp) {
grib_context_log(c, GRIB_LOG_ERROR, "grib_count_in_filename: Unable to read file \"%s\"", filename);
perror(filename);
return GRIB_IO_PROBLEM;
}
err = grib_count_in_file(c, fp, n);
fclose(fp);
return err;
}
Reference in New Issue View Git Blame Copy Permalink