From cf630743ab56f1c4177e67b00de46d7060500684 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 19 Jun 2023 12:54:43 +0100 Subject: [PATCH] Fast read: provide offset arguments --- src/grib_api.h | 8 ++++---- src/grib_io.cc | 31 +++++++++++++++++-------------- tools/codes_count.cc | 9 ++++++--- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/grib_api.h b/src/grib_api.h index 007485d2f..c347efc31 100644 --- a/src/grib_api.h +++ b/src/grib_api.h @@ -1377,10 +1377,10 @@ int wmo_read_bufr_from_file(FILE* f, void* buffer, size_t* len); int wmo_read_gts_from_file(FILE* f, void* buffer, size_t* len); int wmo_read_any_from_stream(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), void* buffer, size_t* len); -int wmo_read_any_from_file_fast(FILE* f, void* buffer, size_t* len); -int wmo_read_grib_from_file_fast(FILE* f, void* buffer, size_t* len); -int wmo_read_bufr_from_file_fast(FILE* f, void* buffer, size_t* len); -int wmo_read_gts_from_file_fast(FILE* f, void* buffer, size_t* len); +int wmo_read_any_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset); +int wmo_read_grib_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset); +int wmo_read_bufr_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset); +int wmo_read_gts_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset); /* These functions allocate memory for the result so the user is responsible for freeing it */ void* wmo_read_any_from_stream_malloc(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), size_t* size, int* err); diff --git a/src/grib_io.cc b/src/grib_io.cc index 4c01dfa54..67f2b6975 100644 --- a/src/grib_io.cc +++ b/src/grib_io.cc @@ -1164,7 +1164,8 @@ static void* user_provider_buffer(void* data, size_t* length, int* err) return u->user_buffer; } -static int ecc_wmo_read_any_from_file(FILE* f, void* buffer, size_t* len, int no_alloc, int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok) +static int ecc_wmo_read_any_from_file(FILE* f, void* buffer, size_t* len, off_t* offset, + int no_alloc, int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok) { int err; user_buffer_t u; @@ -1186,45 +1187,47 @@ static int ecc_wmo_read_any_from_file(FILE* f, void* buffer, size_t* len, int no err = read_any(&r, no_alloc, grib_ok, bufr_ok, hdf5_ok, wrap_ok); *len = r.message_size; + *offset = r.offset; return err; } int wmo_read_any_from_file(FILE* f, void* buffer, size_t* len) { - return ecc_wmo_read_any_from_file(f, buffer, len, /*no_alloc=*/0, 1, 1, 1, 1); + off_t offset = 0; + return ecc_wmo_read_any_from_file(f, buffer, len, &offset, /*no_alloc=*/0, 1, 1, 1, 1); } int wmo_read_grib_from_file(FILE* f, void* buffer, size_t* len) { - return ecc_wmo_read_any_from_file(f, buffer, len, /*no_alloc=*/0, 1, 0, 0, 0); + off_t offset = 0; + return ecc_wmo_read_any_from_file(f, buffer, len, &offset, /*no_alloc=*/0, 1, 0, 0, 0); } int wmo_read_bufr_from_file(FILE* f, void* buffer, size_t* len) { - return ecc_wmo_read_any_from_file(f, buffer, len, /*no_alloc=*/0, 0, 1, 0, 0); + off_t offset = 0; + return ecc_wmo_read_any_from_file(f, buffer, len, &offset, /*no_alloc=*/0, 0, 1, 0, 0); } -int wmo_read_any_from_file_fast(FILE* f, void* buffer, size_t* len) +int wmo_read_any_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset) { - return ecc_wmo_read_any_from_file(f, buffer, len, /*no_alloc=*/1, 1, 1, 1, 1); + return ecc_wmo_read_any_from_file(f, buffer, len, offset, /*no_alloc=*/1, 1, 1, 1, 1); } -int wmo_read_grib_from_file_fast(FILE* f, void* buffer, size_t* len) +int wmo_read_grib_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset) { - return ecc_wmo_read_any_from_file(f, buffer, len, /*no_alloc=*/1, 1, 0, 0, 0); + return ecc_wmo_read_any_from_file(f, buffer, len, offset, /*no_alloc=*/1, 1, 0, 0, 0); } -int wmo_read_bufr_from_file_fast(FILE* f, void* buffer, size_t* len) +int wmo_read_bufr_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset) { - return ecc_wmo_read_any_from_file(f, buffer, len, /*no_alloc=*/1, 0, 1, 0, 0); + return ecc_wmo_read_any_from_file(f, buffer, len, offset, /*no_alloc=*/1, 0, 1, 0, 0); } -int wmo_read_gts_from_file_fast(FILE* f, void* buffer, size_t* len) +int wmo_read_gts_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset) { //TODO(masn): Needs proper implementation; no malloc void* mesg = NULL; - size_t size = 0; - off_t offset = 0; int err = GRIB_SUCCESS; grib_context* c = grib_context_get_default(); - mesg = wmo_read_gts_from_file_malloc(f, 0, &size, &offset, &err); + mesg = wmo_read_gts_from_file_malloc(f, 0, len, offset, &err); grib_context_free(c, mesg); return err; } diff --git a/tools/codes_count.cc b/tools/codes_count.cc index 237620fdf..cc9aa139f 100644 --- a/tools/codes_count.cc +++ b/tools/codes_count.cc @@ -74,10 +74,12 @@ static int count_messages_slow(FILE* in, int message_type, unsigned long* count) static int count_messages_fast(FILE* in, int message_type, unsigned long* count) { int err = GRIB_SUCCESS; - typedef int (*wmo_read_proc)(FILE* , void* , size_t*); + //Signature: FILE* f, void* buffer, size_t* len, off_t* offset + typedef int (*wmo_read_proc)(FILE* , void* , size_t*, off_t*); wmo_read_proc wmo_read = NULL; unsigned char buffer[1000] = {0,}; size_t size = sizeof(buffer); + off_t offset = 0; if (message_type == CODES_GRIB) wmo_read = wmo_read_grib_from_file_fast; @@ -89,14 +91,15 @@ static int count_messages_fast(FILE* in, int message_type, unsigned long* count) wmo_read = wmo_read_any_from_file_fast; if (fail_on_error) { - while ((err = wmo_read(in, buffer, &size)) == GRIB_SUCCESS) { + while ((err = wmo_read(in, buffer, &size, &offset)) == GRIB_SUCCESS) { + //printf("%zu %lld\n", size,offset); (*count)++; } } else { int done = 0; while (!done) { - err = wmo_read(in, buffer, &size); + err = wmo_read(in, buffer, &size, &offset); if (err) { if (err == GRIB_END_OF_FILE || err == GRIB_PREMATURE_END_OF_FILE) { done = 1; // reached the end