core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fast read: Simplify interface

This commit is contained in:
Shahram Najm 2023-06-19 16:31:14 +00:00
parent bf162920e9
commit 5139e09187
3 changed files with 27 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/eccodes_prototypes.h
View File

@ -1100,10 +1100,10 @@ int wmo_read_bufr_from_file(FILE* f, void* buffer, size_t* len);
int wmo_read_gts_from_file(FILE* f, void* buffer, size_t* len); int wmo_read_gts_from_file(FILE* f, void* buffer, size_t* len);
int wmo_read_taf_from_file(FILE* f, void* buffer, size_t* len); int wmo_read_taf_from_file(FILE* f, void* buffer, size_t* len);
int wmo_read_metar_from_file(FILE* f, void* buffer, size_t* len); int wmo_read_metar_from_file(FILE* f, void* buffer, size_t* len);
int wmo_read_any_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset); int wmo_read_any_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset);
int wmo_read_grib_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset); int wmo_read_grib_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset);
int wmo_read_bufr_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset); int wmo_read_bufr_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset);
int wmo_read_gts_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset); int wmo_read_gts_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset);
int wmo_read_any_from_stream(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), void* buffer, size_t* len); int wmo_read_any_from_stream(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), void* buffer, size_t* len);
void* wmo_read_any_from_stream_malloc(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), size_t* size, int* err); void* wmo_read_any_from_stream_malloc(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), size_t* size, int* err);
void* wmo_read_gts_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err); void* wmo_read_gts_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err);

24
src/grib_io.cc
View File

@ -1209,26 +1209,34 @@ int wmo_read_bufr_from_file(FILE* f, void* buffer, size_t* len)
} }
// Fast versions // Fast versions
int wmo_read_any_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset) int wmo_read_any_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset)
{ {
return ecc_wmo_read_any_from_file(f, buffer, len, offset, /*no_alloc=*/1, 1, 1, 1, 1); unsigned char buffer[64] = {0,};
*msg_len = sizeof(buffer);
return ecc_wmo_read_any_from_file(f, buffer, msg_len, msg_offset, /*no_alloc=*/1, 1, 1, 1, 1);
} }
int wmo_read_grib_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset) int wmo_read_grib_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset)
{ {
return ecc_wmo_read_any_from_file(f, buffer, len, offset, /*no_alloc=*/1, 1, 0, 0, 0); unsigned char buffer[64] = {0,};
*msg_len = sizeof(buffer);
return ecc_wmo_read_any_from_file(f, buffer, msg_len, msg_offset, /*no_alloc=*/1, 1, 0, 0, 0);
} }
int wmo_read_bufr_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset) int wmo_read_bufr_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset)
{ {
return ecc_wmo_read_any_from_file(f, buffer, len, offset, /*no_alloc=*/1, 0, 1, 0, 0); unsigned char buffer[64] = {0,};
*msg_len = sizeof(buffer);
return ecc_wmo_read_any_from_file(f, buffer, msg_len, msg_offset, /*no_alloc=*/1, 0, 1, 0, 0);
} }
int wmo_read_gts_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset) int wmo_read_gts_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset)
{ {
//TODO(masn): Needs proper implementation; no malloc //TODO(masn): Needs proper implementation; no malloc
unsigned char buffer[1024] = {0,};
void* mesg = NULL; void* mesg = NULL;
int err = GRIB_SUCCESS; int err = GRIB_SUCCESS;
grib_context* c = grib_context_get_default(); grib_context* c = grib_context_get_default();
mesg = wmo_read_gts_from_file_malloc(f, 0, len, offset, &err); *msg_len = sizeof(buffer);
mesg = wmo_read_gts_from_file_malloc(f, 0, msg_len, msg_offset, &err);
grib_context_free(c, mesg); grib_context_free(c, mesg);
return err; return err;
} }

15
tools/codes_count.cc
View File

@ -74,12 +74,11 @@ static int count_messages_slow(FILE* in, int message_type, unsigned long* count)
static int count_messages_fast(FILE* in, int message_type, unsigned long* count) static int count_messages_fast(FILE* in, int message_type, unsigned long* count)
{ {
int err = GRIB_SUCCESS; int err = GRIB_SUCCESS;
//Signature: FILE* f, void* buffer, size_t* len, off_t* offset // Signature: FILE* f, size_t* msg_len, off_t* msg_offset
typedef int (*wmo_read_proc)(FILE* , void* , size_t*, off_t*); typedef int (*wmo_read_proc)(FILE* , size_t*, off_t*);
wmo_read_proc wmo_read = NULL; wmo_read_proc wmo_read = NULL;
unsigned char buffer[1000] = {0,}; size_t msg_len = 0;
size_t size = sizeof(buffer); off_t msg_offset = 0;
off_t offset = 0;
if (message_type == CODES_GRIB) if (message_type == CODES_GRIB)
wmo_read = wmo_read_grib_from_file_fast; wmo_read = wmo_read_grib_from_file_fast;
@ -91,15 +90,15 @@ static int count_messages_fast(FILE* in, int message_type, unsigned long* count)
wmo_read = wmo_read_any_from_file_fast; wmo_read = wmo_read_any_from_file_fast;
if (fail_on_error) { if (fail_on_error) {
while ((err = wmo_read(in, buffer, &size, &offset)) == GRIB_SUCCESS) { while ((err = wmo_read(in, &msg_len, &msg_offset)) == GRIB_SUCCESS) {
//printf("%zu %lld\n", size,offset); //printf("%zu %ld\n", msg_len,msg_offset);
(*count)++; (*count)++;
} }
} }
else { else {
int done = 0; int done = 0;
while (!done) { while (!done) {
err = wmo_read(in, buffer, &size, &offset); err = wmo_read(in, &msg_len, &msg_offset);
if (err) { if (err) {
if (err == GRIB_END_OF_FILE || err == GRIB_PREMATURE_END_OF_FILE) { if (err == GRIB_END_OF_FILE || err == GRIB_PREMATURE_END_OF_FILE) {
done = 1; // reached the end done = 1; // reached the end