diff --git a/memfs.py b/memfs.py index dc250945a..36a07e0db 100755 --- a/memfs.py +++ b/memfs.py @@ -4,10 +4,13 @@ import os import re import sys import binascii -import StringIO +import time assert len(sys.argv) > 2 +start = time.time() +print("MEMFS: starting") + # Exclude experimental features e.g. GRIB3 and TAF # The BUFR codetables is not used in the engine EXCLUDED = ["grib3", "codetables", "taf", "stations"] @@ -47,21 +50,10 @@ def get_outfile_name(base, count): # The last argument is the base name of the generated C file(s) output_file_base = sys.argv[-1] -buffer = StringIO.StringIO() +buffer = None fcount = -1 -def dump(): - global buffer - global fcount - - fcount += 1 - opath = get_outfile_name(output_file_base, fcount) - print("MEMFS: Generating output: ", opath, "size:", buffer.tell()) - with open(opath, "wb") as f: - f.write(buffer.getvalue()) - - buffer = StringIO.StringIO() for directory in dirs: @@ -71,13 +63,21 @@ for directory in dirs: NAMES.append(dname) for dirpath, dirnames, files in os.walk(directory, followlinks=True): - # for ex in EXCLUDED: - # if ex in dirnames: - # print('Note: eccodes memfs.py script: %s/%s will not be included.' % (dirpath,ex)) + + # Prune the walk by modifying the dirnames in-place dirnames[:] = [dirname for dirname in dirnames if dirname not in EXCLUDED] for name in files: + + + if buffer is None: + fcount += 1 + opath = get_outfile_name(output_file_base, fcount) + print("MEMFS: Generating output:", opath) + buffer = open(opath, 'wb') + + full = "%s/%s" % (dirpath, name) _, ext = os.path.splitext(full) if ext not in [".def", ".table", ".tmpl", ".list", ".txt"]: @@ -116,11 +116,13 @@ for directory in dirs: buffer.write("};\n") if buffer.tell() >= CHUNK: - dump() + buffer.close() + buffer = None -if buffer.tell(): - dump() +if buffer is not None: + buffer.close() + # The number of generated C files is hard coded. # See memfs/CMakeLists.txt assert fcount == 6, fcount @@ -319,3 +321,5 @@ FILE* codes_memfs_open(const char* path) { ) print("Finished") + +print("MEMFS: done", time.time() - start)