core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Speedy Geoiterator: Compute lats and lons without decoding. Added test too

This commit is contained in:
Shahram Najm 2023-02-07 17:33:51 +00:00
parent 9b81223df3
commit 1c761ec0df
5 changed files with 99 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/grib_accessor_class_latitudes.cc
View File

@ -188,11 +188,11 @@ static int unpack_double(grib_accessor* a, double* val, size_t* len)
{
grib_context* c = a->context;
grib_accessor_latitudes* self = (grib_accessor_latitudes*)a;
int ret = 0;
double* v = val;
double dummyLon = 0, dummyVal = 0;
size_t size = 0;
long count = 0;
int ret = 0;
double* v = val;
double dummyLon = 0;
size_t size = 0;
long count = 0;
grib_iterator* iter = NULL;
self->save = 1;
@ -221,7 +221,8 @@ static int unpack_double(grib_accessor* a, double* val, size_t* len)
return GRIB_SUCCESS;
}
iter = grib_iterator_new(grib_handle_of_accessor(a), 0, &ret);
// Performance: We do not need the values to be decoded
iter = grib_iterator_new(grib_handle_of_accessor(a), GRIB_GEOITERATOR_NO_VALUES, &ret);
if (ret != GRIB_SUCCESS) {
if (iter)
grib_iterator_delete(iter);
@ -229,7 +230,7 @@ static int unpack_double(grib_accessor* a, double* val, size_t* len)
return ret;
}
while (grib_iterator_next(iter, v++, &dummyLon, &dummyVal)) {}
while (grib_iterator_next(iter, v++, &dummyLon, NULL)) {}
grib_iterator_delete(iter);
*len = size;
@ -275,13 +276,15 @@ static int get_distinct(grib_accessor* a, double** val, long* len)
double prev;
double* v = NULL;
double* v1 = NULL;
double dummyLon = 0, dummyVal = 0;
double dummyLon = 0;
int ret = 0;
int i;
long jScansPositively = 0; /* default: north to south */
size_t size = *len;
grib_context* c = a->context;
grib_iterator* iter = grib_iterator_new(grib_handle_of_accessor(a), 0, &ret);
// Performance: We do not need the values to be decoded
grib_iterator* iter = grib_iterator_new(grib_handle_of_accessor(a), GRIB_GEOITERATOR_NO_VALUES, &ret);
if (ret != GRIB_SUCCESS) {
if (iter)
grib_iterator_delete(iter);
@ -295,7 +298,7 @@ static int get_distinct(grib_accessor* a, double** val, long* len)
}
*val = v;
while (grib_iterator_next(iter, v++, &dummyLon, &dummyVal)) {}
while (grib_iterator_next(iter, v++, &dummyLon, NULL)) {}
grib_iterator_delete(iter);
v = *val;

19
src/grib_accessor_class_longitudes.cc
View File

@ -165,9 +165,9 @@ static int unpack_double(grib_accessor* a, double* val, size_t* len)
grib_accessor_longitudes* self = (grib_accessor_longitudes*)a;
int ret = 0;
double* v = val;
double dummyLat = 0, dummyVal = 0;
size_t size = 0;
long count = 0;
double dummyLat = 0;
size_t size = 0;
long count = 0;
grib_iterator* iter = NULL;
self->save = 1;
@ -197,7 +197,8 @@ static int unpack_double(grib_accessor* a, double* val, size_t* len)
return GRIB_SUCCESS;
}
iter = grib_iterator_new(grib_handle_of_accessor(a), 0, &ret);
// Performance: We do not need the values to be decoded
iter = grib_iterator_new(grib_handle_of_accessor(a), GRIB_GEOITERATOR_NO_VALUES, &ret);
if (ret != GRIB_SUCCESS) {
if (iter)
grib_iterator_delete(iter);
@ -205,7 +206,7 @@ static int unpack_double(grib_accessor* a, double* val, size_t* len)
return ret;
}
while (grib_iterator_next(iter, &dummyLat, v++, &dummyVal)) {}
while (grib_iterator_next(iter, &dummyLat, v++, NULL)) {}
grib_iterator_delete(iter);
*len = size;
@ -250,12 +251,14 @@ static int get_distinct(grib_accessor* a, double** val, long* len)
double prev;
double* v = NULL;
double* v1 = NULL;
double dummyLat = 0, dummyVal = 0;
double dummyLat = 0;
int ret = 0;
int i;
size_t size = *len;
grib_context* c = a->context;
grib_iterator* iter = grib_iterator_new(grib_handle_of_accessor(a), 0, &ret);
// Performance: We do not need the values to be decoded
grib_iterator* iter = grib_iterator_new(grib_handle_of_accessor(a), GRIB_GEOITERATOR_NO_VALUES, &ret);
if (ret != GRIB_SUCCESS) {
if (iter)
grib_iterator_delete(iter);
@ -269,7 +272,7 @@ static int get_distinct(grib_accessor* a, double** val, long* len)
}
*val = v;
while (grib_iterator_next(iter, &dummyLat, v++, &dummyVal)) {}
while (grib_iterator_next(iter, &dummyLat, v++, NULL)) {}
grib_iterator_delete(iter);
v = *val;

5
src/grib_iterator_class_gen.cc
View File

@ -184,6 +184,7 @@ static int init(grib_iterator* iter, grib_handle* h, grib_arguments* args)
self->missingValue = grib_arguments_get_name(h, args, self->carg++);
s_rawData = grib_arguments_get_name(h, args, self->carg++);
iter->data = NULL;
iter->h = h; /* We may not need to keep them */
iter->args = args;
if ((err = grib_get_size(h, s_rawData, &dli)) != GRIB_SUCCESS)
@ -203,7 +204,9 @@ static int init(grib_iterator* iter, grib_handle* h, grib_arguments* args)
return GRIB_WRONG_GRID;
}
if (iter->flags == 0) {
if ((iter->flags & GRIB_GEOITERATOR_NO_VALUES) == 0) {
// When the NO_VALUES flag is unset, decode the values and store them in the iterator.
// By default (and legacy) flags==0, so we decode
iter->data = (double*)grib_context_malloc(h->context, (iter->nv) * sizeof(double));
if ((err = grib_get_double_array_internal(h, s_rawData, iter->data, &(iter->nv))))

51
tests/grib_geo_iter.cc
View File

@ -13,29 +13,60 @@
#include "grib_api_internal.h"
typedef enum IterMode
{
WITH_VALUES,
NO_VALUES
} IterMode;
int main(int argc, char** argv)
{
int err = 0;
double lat, lon, value;
int n = 0;
const int numberOfDataPoints = 13280;
const char* sample_filename = "gg_sfc_grib2";
grib_handle* h = NULL;
grib_iterator* iter = NULL;
double lat = 0, lon = 0, value = 0;
long n = 0, numberOfDataPoints = 0;
char* filename = NULL;
FILE* fin = NULL;
grib_handle* h = NULL;
grib_iterator* iter = NULL;
char* option = NULL;
IterMode mode = WITH_VALUES;
int flags = 0;
double* pValue = NULL;
bool verbose = false;
h = grib_handle_new_from_samples(0, sample_filename);
Assert(argc == 3);
option = argv[1];
filename = argv[2];
if (strcmp(option, "-n")==0) {
mode = NO_VALUES;
}
fin = fopen(filename, "rb");
Assert(fin);
h = grib_handle_new_from_file(0, fin, &err);
Assert(!err);
Assert(h);
iter = grib_iterator_new(h, 0, &err);
GRIB_CHECK(grib_get_long(h,"numberOfDataPoints", &numberOfDataPoints),0);
flags = (mode == NO_VALUES) ? GRIB_GEOITERATOR_NO_VALUES : 0;
iter = grib_iterator_new(h, flags, &err);
Assert(!err);
Assert(iter);
Assert(grib_iterator_has_next(iter));
n = 0;
while (grib_iterator_next(iter, &lat, &lon, &value)) {
if (n < numberOfDataPoints - 1)
pValue = (mode == NO_VALUES) ? NULL : &value;
while (grib_iterator_next(iter, &lat, &lon, pValue)) {
if (n < numberOfDataPoints - 1) {
Assert(grib_iterator_has_next(iter));
}
if (verbose) {
printf("%9.3f %9.3f ",lat, lon);
if (pValue) printf(" %.10e\n", *pValue);
else printf("\n");
}
n++;
}
Assert(n == numberOfDataPoints);

31
tests/grib_geo_iter.sh
View File

@ -12,4 +12,33 @@
label="grib_geo_iter_test"
$EXEC ${test_dir}/grib_geo_iter
infiles="
$ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl
$ECCODES_SAMPLES_PATH/reduced_ll_sfc_grib1.tmpl
$ECCODES_SAMPLES_PATH/regular_gg_ml_grib2.tmpl
$ECCODES_SAMPLES_PATH/polar_stereographic_pl_grib2.tmpl
$data_dir/regular_latlon_surface.grib1
$data_dir/mercator.grib2
"
# Run the iterator in two modes:
# -v decodes the values
# -n does not decode the values
infile=$ECCODES_SAMPLES_PATH/gg_sfc_grib2.tmpl
for infile in $infiles; do
$EXEC ${test_dir}/grib_geo_iter -v $infile
$EXEC ${test_dir}/grib_geo_iter -n $infile
done
# Test a case where decoding is not possible but the iterator would work
if [ $HAVE_JPEG -eq 0 ]; then
# No JPEG, so cannot decode the field but the iterator doesn't need to
infile=$data_dir/jpeg.grib2
set +e
${tools_dir}/grib_get -p min,max $infile
status=$?
set -e
[ $status -ne 0 ] # Make sure it fails to decode
$EXEC ${test_dir}/grib_geo_iter -n $infile
fi