From 5139e091873651a616c97b5b3b0f18b41f6d9143 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 19 Jun 2023 16:31:14 +0000 Subject: [PATCH] Fast read: Simplify interface --- src/eccodes_prototypes.h | 8 ++++---- src/grib_io.cc | 24 ++++++++++++++++-------- tools/codes_count.cc | 15 +++++++-------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/eccodes_prototypes.h b/src/eccodes_prototypes.h index f994c624f..946343664 100644 --- a/src/eccodes_prototypes.h +++ b/src/eccodes_prototypes.h @@ -1100,10 +1100,10 @@ int wmo_read_bufr_from_file(FILE* f, void* buffer, size_t* len); int wmo_read_gts_from_file(FILE* f, void* buffer, size_t* len); int wmo_read_taf_from_file(FILE* f, void* buffer, size_t* len); int wmo_read_metar_from_file(FILE* f, void* buffer, size_t* len); -int wmo_read_any_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset); -int wmo_read_grib_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset); -int wmo_read_bufr_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset); -int wmo_read_gts_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset); +int wmo_read_any_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset); +int wmo_read_grib_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset); +int wmo_read_bufr_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset); +int wmo_read_gts_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset); int wmo_read_any_from_stream(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), void* buffer, size_t* len); void* wmo_read_any_from_stream_malloc(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), size_t* size, int* err); void* wmo_read_gts_from_file_malloc(FILE* f, int headers_only, size_t* size, off_t* offset, int* err); diff --git a/src/grib_io.cc b/src/grib_io.cc index 41bc117b6..0dee58dd7 100644 --- a/src/grib_io.cc +++ b/src/grib_io.cc @@ -1209,26 +1209,34 @@ int wmo_read_bufr_from_file(FILE* f, void* buffer, size_t* len) } // Fast versions -int wmo_read_any_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset) +int wmo_read_any_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset) { - return ecc_wmo_read_any_from_file(f, buffer, len, offset, /*no_alloc=*/1, 1, 1, 1, 1); + unsigned char buffer[64] = {0,}; + *msg_len = sizeof(buffer); + return ecc_wmo_read_any_from_file(f, buffer, msg_len, msg_offset, /*no_alloc=*/1, 1, 1, 1, 1); } -int wmo_read_grib_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset) +int wmo_read_grib_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset) { - return ecc_wmo_read_any_from_file(f, buffer, len, offset, /*no_alloc=*/1, 1, 0, 0, 0); + unsigned char buffer[64] = {0,}; + *msg_len = sizeof(buffer); + return ecc_wmo_read_any_from_file(f, buffer, msg_len, msg_offset, /*no_alloc=*/1, 1, 0, 0, 0); } -int wmo_read_bufr_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset) +int wmo_read_bufr_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset) { - return ecc_wmo_read_any_from_file(f, buffer, len, offset, /*no_alloc=*/1, 0, 1, 0, 0); + unsigned char buffer[64] = {0,}; + *msg_len = sizeof(buffer); + return ecc_wmo_read_any_from_file(f, buffer, msg_len, msg_offset, /*no_alloc=*/1, 0, 1, 0, 0); } -int wmo_read_gts_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset) +int wmo_read_gts_from_file_fast(FILE* f, size_t* msg_len, off_t* msg_offset) { //TODO(masn): Needs proper implementation; no malloc + unsigned char buffer[1024] = {0,}; void* mesg = NULL; int err = GRIB_SUCCESS; grib_context* c = grib_context_get_default(); - mesg = wmo_read_gts_from_file_malloc(f, 0, len, offset, &err); + *msg_len = sizeof(buffer); + mesg = wmo_read_gts_from_file_malloc(f, 0, msg_len, msg_offset, &err); grib_context_free(c, mesg); return err; } diff --git a/tools/codes_count.cc b/tools/codes_count.cc index cc9aa139f..c96ba374d 100644 --- a/tools/codes_count.cc +++ b/tools/codes_count.cc @@ -74,12 +74,11 @@ static int count_messages_slow(FILE* in, int message_type, unsigned long* count) static int count_messages_fast(FILE* in, int message_type, unsigned long* count) { int err = GRIB_SUCCESS; - //Signature: FILE* f, void* buffer, size_t* len, off_t* offset - typedef int (*wmo_read_proc)(FILE* , void* , size_t*, off_t*); + // Signature: FILE* f, size_t* msg_len, off_t* msg_offset + typedef int (*wmo_read_proc)(FILE* , size_t*, off_t*); wmo_read_proc wmo_read = NULL; - unsigned char buffer[1000] = {0,}; - size_t size = sizeof(buffer); - off_t offset = 0; + size_t msg_len = 0; + off_t msg_offset = 0; if (message_type == CODES_GRIB) wmo_read = wmo_read_grib_from_file_fast; @@ -91,15 +90,15 @@ static int count_messages_fast(FILE* in, int message_type, unsigned long* count) wmo_read = wmo_read_any_from_file_fast; if (fail_on_error) { - while ((err = wmo_read(in, buffer, &size, &offset)) == GRIB_SUCCESS) { - //printf("%zu %lld\n", size,offset); + while ((err = wmo_read(in, &msg_len, &msg_offset)) == GRIB_SUCCESS) { + //printf("%zu %ld\n", msg_len,msg_offset); (*count)++; } } else { int done = 0; while (!done) { - err = wmo_read(in, buffer, &size, &offset); + err = wmo_read(in, &msg_len, &msg_offset); if (err) { if (err == GRIB_END_OF_FILE || err == GRIB_PREMATURE_END_OF_FILE) { done = 1; // reached the end