From ae2e7288550df720d248d8c95785c04890922a86 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 19 Oct 2020 13:33:27 +0100 Subject: [PATCH 1/4] Testing: fix awk expression that fails on HPC --- tests/bufr_extract_headers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bufr_extract_headers.sh b/tests/bufr_extract_headers.sh index 80d2d9b6e..908f9ba29 100755 --- a/tests/bufr_extract_headers.sh +++ b/tests/bufr_extract_headers.sh @@ -62,7 +62,7 @@ r=`${test_dir}/bufr_extract_headers centre ${data_dir}/bufr/israel_observations_ # Check all centres with an abbreviation centre_table=${ECCODES_DEFINITION_PATH}/common/c-11.table -centres=`awk 'NR > 1 && $2 ~ /^[A-z]/ {print $2}' < $centre_table` +centres=`awk 'NR > 1 {print $2}' < $centre_table` for c in $centres; do ${tools_dir}/bufr_set -s centre=$c $ECCODES_SAMPLES_PATH/BUFR4.tmpl $temp1 r=`${test_dir}/bufr_extract_headers centre $temp1` From 45975c115bf1dd8812f75f1a1ade4cbb19bf07d3 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 19 Oct 2020 13:46:11 +0100 Subject: [PATCH 2/4] ECC-1158: GRIB encoding: Implement data quality checks (limits) for spectral fields --- src/grib_accessor_class_data_complex_packing.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/grib_accessor_class_data_complex_packing.c b/src/grib_accessor_class_data_complex_packing.c index fade0feb8..d392d4e18 100644 --- a/src/grib_accessor_class_data_complex_packing.c +++ b/src/grib_accessor_class_data_complex_packing.c @@ -672,6 +672,15 @@ static int pack_double(grib_accessor* a, const double* val, size_t* len) return GRIB_INTERNAL_ERROR; } + /* Data Quality checks */ + if (a->context->grib_data_quality_checks) { + /* First value is the field's average */ + double min_val = val[0]; + double max_val = min_val; + if ((ret = grib_util_grib_data_quality_check(gh, min_val, max_val)) != GRIB_SUCCESS) + return ret; + } + if (pen_j == sub_j) { double* values; d = grib_power(decimal_scale_factor, 10); @@ -819,7 +828,6 @@ static int pack_double(grib_accessor* a, const double* val, size_t* len) } else { /* _test(val[i]*d,0); */ - grib_encode_unsigned_long(hres, encode_float(val[i++]), &hpos, 8 * bytes); /* _test(val[i]*d,0); */ grib_encode_unsigned_long(hres, encode_float(val[i++]), &hpos, 8 * bytes); From 44d1f10876de49014e4336eafc87e031ae393d87 Mon Sep 17 00:00:00 2001 From: Shahram Najm Date: Mon, 19 Oct 2020 14:17:35 +0100 Subject: [PATCH 3/4] ECC-1158: Test --- tests/grib_data_quality_checks.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/grib_data_quality_checks.sh b/tests/grib_data_quality_checks.sh index 45c725f7b..65fad1ede 100755 --- a/tests/grib_data_quality_checks.sh +++ b/tests/grib_data_quality_checks.sh @@ -140,24 +140,34 @@ echo "Override the defaults..." tempDir=tempdir.$label rm -rf $tempDir mkdir -p $tempDir -# Set a large limit for temperature +# Change limits for 2m temperature (grid-point) and Temperature (spectral) cat > $tempDir/param_limits.def < Date: Mon, 19 Oct 2020 14:39:11 +0100 Subject: [PATCH 4/4] Black formatting (in preparation for GitHub PR 44) --- memfs.py | 75 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/memfs.py b/memfs.py index 642534d2e..f3864a992 100755 --- a/memfs.py +++ b/memfs.py @@ -9,22 +9,22 @@ assert len(sys.argv) > 2 # Exclude experimental features e.g. GRIB3 and TAF # The BUFR codetables is not used in the engine -EXCLUDED = ['grib3', 'codetables', 'taf', 'stations'] +EXCLUDED = ["grib3", "codetables", "taf", "stations"] pos = 1 -if sys.argv[1] == '-exclude': +if sys.argv[1] == "-exclude": product = sys.argv[2] - if product == 'bufr': + if product == "bufr": EXCLUDED.append(product) - elif product == 'grib': - EXCLUDED.extend(['grib1', 'grib2']) + elif product == "grib": + EXCLUDED.extend(["grib1", "grib2"]) else: - assert False, 'Invalid product %s' % product + assert False, "Invalid product %s" % product pos = 3 dirs = [os.path.realpath(x) for x in sys.argv[pos:-1]] -print('Directories: ', dirs) -print('Excluding: ', EXCLUDED) +print("Directories: ", dirs) +print("Excluding: ", EXCLUDED) FILES = {} SIZES = {} @@ -33,10 +33,10 @@ CHUNK = 5500 * 1000 # chunk size in bytes # Binary to ASCII function. Different in Python 2 and 3 try: - str(b'\x23\x20','ascii') - ascii = lambda x: str(x, 'ascii') # Python 3 + str(b"\x23\x20", "ascii") + ascii = lambda x: str(x, "ascii") # Python 3 except: - ascii = lambda x: str(x) # Python 2 + ascii = lambda x: str(x) # Python 2 def get_outfile_name(base, count): @@ -48,7 +48,7 @@ output_file_base = sys.argv[-1] totsize = 0 # amount written fcount = 0 opath = get_outfile_name(output_file_base, fcount) -print('MEMFS: Generating output: ', opath) +print("MEMFS: Generating output: ", opath) g = open(opath, "w") for directory in dirs: @@ -65,31 +65,31 @@ for directory in dirs: # Prune the walk by modifying the dirnames in-place dirnames[:] = [dirname for dirname in dirnames if dirname not in EXCLUDED] for name in files: - full = '%s/%s' % (dirpath, name) + full = "%s/%s" % (dirpath, name) _, ext = os.path.splitext(full) - if ext not in ['.def', '.table', '.tmpl', '.list', '.txt']: + if ext not in [".def", ".table", ".tmpl", ".list", ".txt"]: continue - if name == 'CMakeLists.txt': + if name == "CMakeLists.txt": continue fsize = os.path.getsize(full) totsize += fsize full = full.replace("\\", "/") - fname = full[full.find("/%s/" % (dname,)):] - #print("MEMFS: Add ", fname) - name = re.sub(r'\W', '_', fname) + fname = full[full.find("/%s/" % (dname,)) :] + # print("MEMFS: Add ", fname) + name = re.sub(r"\W", "_", fname) assert name not in FILES assert name not in SIZES FILES[name] = fname SIZES[name] = fsize - print('const unsigned char %s[] = {' % (name,), file=g) + print("const unsigned char %s[] = {" % (name,), file=g) - with open(full, 'rb') as f: + with open(full, "rb") as f: i = 0 # Python 2 - #contents_hex = f.read().encode("hex") + # contents_hex = f.read().encode("hex") # Python 2 and 3 contents_hex = binascii.hexlify(f.read()) @@ -97,18 +97,18 @@ for directory in dirs: # Read two characters at a time and convert to C hex # e.g. 23 -> 0x23 for n in range(0, len(contents_hex), 2): - twoChars = ascii(contents_hex[n:n + 2]) + twoChars = ascii(contents_hex[n : n + 2]) print("0x%s," % (twoChars,), end="", file=g) i += 1 if (i % 20) == 0: print("", file=g) - print('};', file=g) + print("};", file=g) if totsize >= CHUNK: g.close() fcount += 1 opath = get_outfile_name(output_file_base, fcount) - print('MEMFS: Generating output: ', opath) + print("MEMFS: Generating output: ", opath) g = open(opath, "w") totsize = 0 @@ -117,10 +117,11 @@ g.close() # See memfs/CMakeLists.txt assert fcount == 3, fcount opath = output_file_base + "_final.c" -print('MEMFS: Generating output: ', opath) +print("MEMFS: Generating output: ", opath) g = open(opath, "w") -print(""" +print( + """ #include "eccodes_config.h" #ifdef ECCODES_HAVE_FMEMOPEN #define _GNU_SOURCE @@ -131,18 +132,23 @@ print(""" #include #include #include "eccodes_windef.h" -""", file=g) +""", + file=g, +) # Write extern variables with sizes for k, v in SIZES.items(): - print('extern const unsigned char %s[%d];' % (k, v), file=g) + print("extern const unsigned char %s[%d];" % (k, v), file=g) -print(""" +print( + """ struct entry { const char* path; const unsigned char* content; size_t length; -} entries[] = { """, file=g) +} entries[] = { """, + file=g, +) items = [(v, k) for k, v in FILES.items()] @@ -150,7 +156,8 @@ for k, v in sorted(items): print('{"/MEMFS%s", &%s[0], sizeof(%s) / sizeof(%s[0]) },' % (k, v, v, v), file=g) -print("""}; +print( + """}; #if defined(ECCODES_HAVE_FUNOPEN) && !defined(ECCODES_HAVE_FMEMOPEN) @@ -299,6 +306,8 @@ FILE* codes_memfs_open(const char* path) { return fmemopen((void*)mem, size, "r"); } -""", file=g) +""", + file=g, +) -print('Finished') +print("Finished")