core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fast read: provide offset arguments

This commit is contained in:
Shahram Najm 2023-06-19 12:54:43 +01:00
parent 20379383e4
commit cf630743ab
3 changed files with 27 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/grib_api.h
View File

@ -1377,10 +1377,10 @@ int wmo_read_bufr_from_file(FILE* f, void* buffer, size_t* len);
int wmo_read_gts_from_file(FILE* f, void* buffer, size_t* len);
int wmo_read_any_from_stream(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), void* buffer, size_t* len);
int wmo_read_any_from_file_fast(FILE* f, void* buffer, size_t* len);
int wmo_read_grib_from_file_fast(FILE* f, void* buffer, size_t* len);
int wmo_read_bufr_from_file_fast(FILE* f, void* buffer, size_t* len);
int wmo_read_gts_from_file_fast(FILE* f, void* buffer, size_t* len);
int wmo_read_any_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset);
int wmo_read_grib_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset);
int wmo_read_bufr_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset);
int wmo_read_gts_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset);
/* These functions allocate memory for the result so the user is responsible for freeing it */
void* wmo_read_any_from_stream_malloc(void* stream_data, long (*stream_proc)(void*, void* buffer, long len), size_t* size, int* err);

31
src/grib_io.cc
View File

@ -1164,7 +1164,8 @@ static void* user_provider_buffer(void* data, size_t* length, int* err)
return u->user_buffer;
}
static int ecc_wmo_read_any_from_file(FILE* f, void* buffer, size_t* len, int no_alloc, int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok)
static int ecc_wmo_read_any_from_file(FILE* f, void* buffer, size_t* len, off_t* offset,
int no_alloc, int grib_ok, int bufr_ok, int hdf5_ok, int wrap_ok)
{
int err;
user_buffer_t u;
@ -1186,45 +1187,47 @@ static int ecc_wmo_read_any_from_file(FILE* f, void* buffer, size_t* len, int no
err = read_any(&r, no_alloc, grib_ok, bufr_ok, hdf5_ok, wrap_ok);
*len = r.message_size;
*offset = r.offset;
return err;
}
int wmo_read_any_from_file(FILE* f, void* buffer, size_t* len)
{
return ecc_wmo_read_any_from_file(f, buffer, len, /*no_alloc=*/0, 1, 1, 1, 1);
off_t offset = 0;
return ecc_wmo_read_any_from_file(f, buffer, len, &offset, /*no_alloc=*/0, 1, 1, 1, 1);
}
int wmo_read_grib_from_file(FILE* f, void* buffer, size_t* len)
{
return ecc_wmo_read_any_from_file(f, buffer, len, /*no_alloc=*/0, 1, 0, 0, 0);
off_t offset = 0;
return ecc_wmo_read_any_from_file(f, buffer, len, &offset, /*no_alloc=*/0, 1, 0, 0, 0);
}
int wmo_read_bufr_from_file(FILE* f, void* buffer, size_t* len)
{
return ecc_wmo_read_any_from_file(f, buffer, len, /*no_alloc=*/0, 0, 1, 0, 0);
off_t offset = 0;
return ecc_wmo_read_any_from_file(f, buffer, len, &offset, /*no_alloc=*/0, 0, 1, 0, 0);
}
int wmo_read_any_from_file_fast(FILE* f, void* buffer, size_t* len)
int wmo_read_any_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset)
{
return ecc_wmo_read_any_from_file(f, buffer, len, /*no_alloc=*/1, 1, 1, 1, 1);
return ecc_wmo_read_any_from_file(f, buffer, len, offset, /*no_alloc=*/1, 1, 1, 1, 1);
}
int wmo_read_grib_from_file_fast(FILE* f, void* buffer, size_t* len)
int wmo_read_grib_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset)
{
return ecc_wmo_read_any_from_file(f, buffer, len, /*no_alloc=*/1, 1, 0, 0, 0);
return ecc_wmo_read_any_from_file(f, buffer, len, offset, /*no_alloc=*/1, 1, 0, 0, 0);
}
int wmo_read_bufr_from_file_fast(FILE* f, void* buffer, size_t* len)
int wmo_read_bufr_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset)
{
return ecc_wmo_read_any_from_file(f, buffer, len, /*no_alloc=*/1, 0, 1, 0, 0);
return ecc_wmo_read_any_from_file(f, buffer, len, offset, /*no_alloc=*/1, 0, 1, 0, 0);
}
int wmo_read_gts_from_file_fast(FILE* f, void* buffer, size_t* len)
int wmo_read_gts_from_file_fast(FILE* f, void* buffer, size_t* len, off_t* offset)
{
//TODO(masn): Needs proper implementation; no malloc
void* mesg = NULL;
size_t size = 0;
off_t offset = 0;
int err = GRIB_SUCCESS;
grib_context* c = grib_context_get_default();
mesg = wmo_read_gts_from_file_malloc(f, 0, &size, &offset, &err);
mesg = wmo_read_gts_from_file_malloc(f, 0, len, offset, &err);
grib_context_free(c, mesg);
return err;
}

9
tools/codes_count.cc
View File

@ -74,10 +74,12 @@ static int count_messages_slow(FILE* in, int message_type, unsigned long* count)
static int count_messages_fast(FILE* in, int message_type, unsigned long* count)
{
int err = GRIB_SUCCESS;
typedef int (*wmo_read_proc)(FILE* , void* , size_t*);
//Signature: FILE* f, void* buffer, size_t* len, off_t* offset
typedef int (*wmo_read_proc)(FILE* , void* , size_t*, off_t*);
wmo_read_proc wmo_read = NULL;
unsigned char buffer[1000] = {0,};
size_t size = sizeof(buffer);
off_t offset = 0;
if (message_type == CODES_GRIB)
wmo_read = wmo_read_grib_from_file_fast;
@ -89,14 +91,15 @@ static int count_messages_fast(FILE* in, int message_type, unsigned long* count)
wmo_read = wmo_read_any_from_file_fast;
if (fail_on_error) {
while ((err = wmo_read(in, buffer, &size)) == GRIB_SUCCESS) {
while ((err = wmo_read(in, buffer, &size, &offset)) == GRIB_SUCCESS) {
//printf("%zu %lld\n", size,offset);
(*count)++;
}
}
else {
int done = 0;
while (!done) {
err = wmo_read(in, buffer, &size);
err = wmo_read(in, buffer, &size, &offset);
if (err) {
if (err == GRIB_END_OF_FILE || err == GRIB_PREMATURE_END_OF_FILE) {
done = 1; // reached the end