Memfs generation

eccodes.code-workspace

@@ -0,0 +1,7 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	]
+}

memfs.py

@@ -4,37 +4,38 @@ import os
 import re
 import sys
 import binascii
+import StringIO
 
 assert len(sys.argv) > 2
 
 # Exclude experimental features e.g. GRIB3 and TAF
 # The BUFR codetables is not used in the engine
-EXCLUDED = ['grib3', 'codetables', 'taf', 'stations']
+EXCLUDED = ["grib3", "codetables", "taf", "stations"]
 
 pos = 1
-if sys.argv[1] == '-exclude':
+if sys.argv[1] == "-exclude":
     product = sys.argv[2]
-    if product == 'bufr':
+    if product == "bufr":
         EXCLUDED.append(product)
-    elif product == 'grib':
-        EXCLUDED.extend(['grib1', 'grib2'])
+    elif product == "grib":
+        EXCLUDED.extend(["grib1", "grib2"])
     else:
-        assert False, 'Invalid product %s' % product
+        assert False, "Invalid product %s" % product
     pos = 3
 
 dirs = [os.path.realpath(x) for x in sys.argv[pos:-1]]
-print('Directories: ', dirs)
-print('Excluding: ', EXCLUDED)
+print("Directories: ", dirs)
+print("Excluding: ", EXCLUDED)
 
 FILES = {}
 SIZES = {}
 NAMES = []
-CHUNK = 5500 * 1000  # chunk size in bytes
+CHUNK = 16 * 1024 * 1024  # chunk size in bytes
 
 # Binary to ASCII function. Different in Python 2 and 3
 try:
-    str(b'\x23\x20','ascii')
-    ascii = lambda x: str(x, 'ascii')  # Python 3
+    str(b"\x23\x20", "ascii")
+    ascii = lambda x: str(x, "ascii")  # Python 3
 except:
     ascii = lambda x: str(x)  # Python 2
 
@@ -45,11 +46,23 @@ def get_outfile_name(base, count):
 
 # The last argument is the base name of the generated C file(s)
 output_file_base = sys.argv[-1]
-totsize = 0  # amount written
-fcount = 0
+
+buffer = StringIO.StringIO()
+fcount = -1
+
+
+def dump():
+    global buffer
+    global fcount
+
+    fcount += 1
     opath = get_outfile_name(output_file_base, fcount)
-print('MEMFS: Generating output: ', opath)
-g = open(opath, "w")
+    print("MEMFS: Generating output: ", opath, "size:", buffer.tell())
+    with open(opath, "wb") as f:
+        f.write(buffer.getvalue())
+
+    buffer = StringIO.StringIO()
+
 
 for directory in dirs:
 
@@ -65,28 +78,26 @@ for directory in dirs:
         # Prune the walk by modifying the dirnames in-place
         dirnames[:] = [dirname for dirname in dirnames if dirname not in EXCLUDED]
         for name in files:
-            full = '%s/%s' % (dirpath, name)
+            full = "%s/%s" % (dirpath, name)
             _, ext = os.path.splitext(full)
-            if ext not in ['.def', '.table', '.tmpl', '.list', '.txt']:
+            if ext not in [".def", ".table", ".tmpl", ".list", ".txt"]:
                 continue
-            if name == 'CMakeLists.txt':
+            if name == "CMakeLists.txt":
                 continue
 
-            fsize = os.path.getsize(full)
-            totsize += fsize
             full = full.replace("\\", "/")
             fname = full[full.find("/%s/" % (dname,)) :]
             # print("MEMFS: Add ", fname)
-            name = re.sub(r'\W', '_', fname)
+            name = re.sub(r"\W", "_", fname)
 
             assert name not in FILES
             assert name not in SIZES
             FILES[name] = fname
-            SIZES[name] = fsize
+            SIZES[name] = os.path.getsize(full)
 
-            print('const unsigned char %s[] = {' % (name,), file=g)
+            buffer.write("const unsigned char %s[] = {" % (name,))
 
-            with open(full, 'rb') as f:
+            with open(full, "rb") as f:
                 i = 0
                 # Python 2
                 # contents_hex = f.read().encode("hex")
@@ -98,29 +109,27 @@ for directory in dirs:
                 # e.g. 23 -> 0x23
                 for n in range(0, len(contents_hex), 2):
                     twoChars = ascii(contents_hex[n : n + 2])
-                    print("0x%s," % (twoChars,), end="", file=g)
+                    buffer.write("0x%s," % (twoChars,))
                     i += 1
                     if (i % 20) == 0:
-                        print("", file=g)
+                        buffer.write("\n")
 
-            print('};', file=g)
-            if totsize >= CHUNK:
-                g.close()
-                fcount += 1
-                opath = get_outfile_name(output_file_base, fcount)
-                print('MEMFS: Generating output: ', opath)
-                g = open(opath, "w")
-                totsize = 0
+            buffer.write("};\n")
+            if buffer.tell() >= CHUNK:
+                dump()
 
-g.close()
+
+if buffer.tell():
+    dump()
 # The number of generated C files is hard coded.
 # See memfs/CMakeLists.txt
-assert fcount == 3, fcount
+assert fcount == 6, fcount
 opath = output_file_base + "_final.c"
-print('MEMFS: Generating output: ', opath)
+print("MEMFS: Generating output: ", opath)
 g = open(opath, "w")
 
-print("""
+print(
+    """
 #include "eccodes_config.h"
 #ifdef ECCODES_HAVE_FMEMOPEN
 #define _GNU_SOURCE
@@ -131,18 +140,23 @@ print("""
 #include <errno.h>
 #include <string.h>
 #include "eccodes_windef.h"
-""", file=g)
+""",
+    file=g,
+)
 
 # Write extern variables with sizes
 for k, v in SIZES.items():
-    print('extern const unsigned char %s[%d];' % (k, v), file=g)
+    print("extern const unsigned char %s[%d];" % (k, v), file=g)
 
-print("""
+print(
+    """
 struct entry {
     const char* path;
     const unsigned char* content;
     size_t length;
-} entries[] = { """, file=g)
+} entries[] = { """,
+    file=g,
+)
 
 items = [(v, k) for k, v in FILES.items()]
 
@@ -150,7 +164,8 @@ for k, v in sorted(items):
     print('{"/MEMFS%s", &%s[0], sizeof(%s) / sizeof(%s[0]) },' % (k, v, v, v), file=g)
 
 
-print("""};
+print(
+    """};
 
 #if defined(ECCODES_HAVE_FUNOPEN) && !defined(ECCODES_HAVE_FMEMOPEN)
 
@@ -299,6 +314,8 @@ FILE* codes_memfs_open(const char* path) {
     return fmemopen((void*)mem, size, "r");
 }
 
-""", file=g)
+""",
+    file=g,
+)
 
-print('Finished')
+print("Finished")

memfs/CMakeLists.txt

@@ -10,6 +10,9 @@ set( generated_c_files
     ${CMAKE_CURRENT_BINARY_DIR}/memfs_gen_001.c
     ${CMAKE_CURRENT_BINARY_DIR}/memfs_gen_002.c
     ${CMAKE_CURRENT_BINARY_DIR}/memfs_gen_003.c
+    ${CMAKE_CURRENT_BINARY_DIR}/memfs_gen_004.c
+    ${CMAKE_CURRENT_BINARY_DIR}/memfs_gen_005.c
+    ${CMAKE_CURRENT_BINARY_DIR}/memfs_gen_006.c
     ${CMAKE_CURRENT_BINARY_DIR}/memfs_gen_final.c)
 
 add_custom_command(