core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improved struct and variable parsing

This commit is contained in:
kevstone 2023-10-29 20:32:43 +00:00
parent 2788e23f46
commit 1b4ac29df7
24 changed files with 2721 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
debug.txt Normal file
View File

8
src/conversion/accessor_data.py
View File

@ -3,14 +3,6 @@ import arg_conv
import func
from jinja2 import Environment, FileSystemLoader, StrictUndefined
common_includes = [
"AccessorFactory.h",
"AccessorUtils/ConversionHelper.h",
"AccessorUtils/GribUtils.h",
"AccessorUtils/GribAccessorFlag.h",
"AccessorUtils/AccessorException.h"
]
# Represents the converted C++ code for an Accessor
class AccessorData:
def __init__(self, name, super, factory_name):

120
src/conversion/arg.py
View File

@ -39,41 +39,28 @@ class Arg:
# Create Arg from an input string
@classmethod
def from_string(cls, input):
arg_type, arg_name = parse_type_and_name_from_string(input)
# Note: "return x;" looks like a variable declaration, so we explicitly exclude this...
# Groups: 1 2 3 4 5 6 7 8
# Groups: const struct unsigned TYPE * const NAME [N]
m = re.match(r"(const)?(struct)?\s*(unsigned)?\s*(\w+)\s*(\*+)?\s*(const)?\s+(\w+)\s*(\[\d*\])?", input)
if m:
for text in ["return", "typedef"]:
if m.group(4).startswith(text):
debug.line("from_string", f"Ignoring invalid arg type [{text}]: {m.group(0)}")
return None
arg_type = ""
if m.group(1):
arg_type += m.group(1) + " "
if m.group(2):
arg_type += m.group(2) + " "
if m.group(3):
arg_type += m.group(3) + " "
arg_type += m.group(4)
if m.group(5):
arg_type += m.group(5) # Add * if present...
if m.group(6):
arg_type += " " + m.group(6)
arg_name = m.group(7)
if m.group(8):
# Handle array declaration e.g. char buf[10]
arg_type += m.group(8)
if arg_type and arg_name:
#debug.line("from_string", f"Creating arg type=[{arg_type}] name=[{arg_name}] from: {input}")
return cls(arg_type, arg_name)
debug.line("from_string", f"Input is not an arg declaration: {input}")
return None
# Create Arg from a function argument input string - this is similar to from_string(), except it will
# accept just the type, so can parse unused function args that have no name (in this case name is None)
@classmethod
def from_func_arg_string(cls, input):
arg_type, arg_name = parse_type_and_name_from_string(input)
if arg_type:
#debug.line("from_string", f"Creating function arg type=[{arg_type}] name=[{arg_name}] from: {input}")
return cls(arg_type, arg_name)
debug.line("from_string", f"Input is not a function arg declaration: {input}")
return None
# Generate a string to represent the Arg's declaration
def as_declaration(self):
arg_type = self.type
@ -116,3 +103,76 @@ def arg_string(arg):
return arg.as_declaration()
return "None"
# Helper to extract the type and name from an input string - used by the Arg class
# Returns arg_type, arg_name as strings, or None if the input could no be parsed
def parse_type_and_name_from_string(input):
# The parsing is split into two phases, type then name
#
# This is to support unused function parameters where no name is supplied
#
# Phase 1 - Type
#
# The regex is quite complicated, using the following groups:
#
# Groups: 1 2 3 4 5 6
# Groups: const struct unsigned TYPE PTR const
#
# Group 5 will match the pointer regardless of whether it is with the type or name, i.e.
# int* var;
# int *var;
# int** var;
# int **var;
#
# However to simplify future parsing, the pointer type will be stored as int*, not int *
#
# Phase 2 - Name
#
# If type is successfully extracted, then the name and index (if present, e.g. [2]) are parsed
#
# Notes:
# 1. "return x;" looks like a variable declaration, so we explicitly exclude this...
#
# 2. We ensure that AT LEAST ONE whitespace character must be matched after group 6 to ensure we
# have two separate identifiers for type and name
#
# 3. The index is actually added to the type to help with future parsing
#
arg_type = arg_name = None
# Phase 1 - type
m = re.match(r"(const)?(struct)?\s*(unsigned)?\s*(\w+)(\s\*+|\*+\s?|\s)(const)?", input)
if m:
arg_type = ""
if m.group(1):
arg_type += m.group(1) + " "
if m.group(2):
arg_type += m.group(2) + " "
if m.group(3):
arg_type += m.group(3) + " "
arg_type += m.group(4)
if m.group(5):
arg_type += m.group(5).strip() # Add * if present...
if m.group(6):
arg_type += " " + m.group(6)
assert arg_type, f"Error extracting arg type from input=[{input}]"
# Phase 2 - name
if arg_type:
m = re.match(r"\s*(\w+)\s*(\[\d*\])?", input[m.end():])
if m:
arg_name = m.group(1)
if arg_name in ["return", "typedef"]:
debug.line("from_string", f"Ignoring invalid arg type [{arg_name}]: {input}")
return None
if m.group(2):
# Handle array declaration e.g. char buf[10]
arg_type += m.group(2)
assert arg_name, f"Error extracting arg name from input=[{input}], type=[{arg_type}]"
return arg_type, arg_name

5
src/conversion/constructor_method_conv.py
View File

@ -12,11 +12,10 @@ class ConstructorMethodConverter(MethodConverter):
def create_cpp_function(self, cppfuncsig):
return constructor_method.ConstructorMethod(cppfuncsig)
def process_variables_initial_pass(self, line):
def update_cfunction_names(self, line):
# Transform the argument getters
line = re.sub(rf"\blen\b", "initData.length", line)
line = re.sub(rf"\bgrib_arguments_get_name\s*\(.*?,\s*\w+\s*,\s*(.*)?\)", rf"AccessorName(std::get<std::string>(initData.args[\1].second))", line)
line = re.sub(rf"\bgrib_arguments_get_(\w+)\(.*?, arg, (\d+)\)", rf"std::get<\1>(initData.args[\2].second)", line)
return super().process_variables_initial_pass(line)
return super().update_cfunction_names(line)

4
src/conversion/constructor_method_funcsig_conv.py
View File

@ -1,13 +1,15 @@
from method_funcsig_conv import *
from funcsig import FuncSig
from arg_indexes import ArgIndexes
from arg import Arg
class ConstructorMethodFuncSigConverter(MethodFuncSigConverter):
constructor_method_conversions = [
# static void init(grib_accessor*, const long, grib_arguments*);
FuncSigMapping(FuncSig("", "init", [Arg("grib_context*", ""), Arg("const long", ""), Arg("grib_arguments*", "")]),
FuncSig("Constructor", None, [None, None, Arg("AccessorInitData const&", "initData")]) ),
FuncSig("Constructor", None, [None, None, Arg("AccessorInitData const&", "initData")]),
ArgIndexes(cbuffer=2, clength=1, cpp_container=2) ),
]
def __init__(self, cfuncsig):

4
src/conversion/convert.py
View File

@ -130,10 +130,10 @@ def parse_file(path):
# Add function name to the global body as a forward declaration, so it is
# declared in the correct place for any other globals!
#
# Note: We just use the placeholder [FORWARD_DECLARATION]function_name here,
# Note: We just use the placeholder @FORWARD_DECLARATION:function_name here,
# when it is processed the placeholder will be used to check that it
# is a valid static function: if not it will be deleted
forward_declaration = f"[FORWARD_DECLARATION]{function_name}"
forward_declaration = f"@FORWARD_DECLARATION:{function_name}"
global_function.add_line(forward_declaration)
cfuncsig.template = template

2
src/conversion/converter_collection.py
View File

@ -51,7 +51,7 @@ default_converters = {
}
def converters_for(accessor_name):
converters = copy.copy(default_converters)
converters = copy.deepcopy(default_converters)
try:
m = importlib.import_module(f"converters.{accessor_name}")

6
src/conversion/debug.py
View File

@ -1,9 +1,9 @@
from datetime import datetime
func_pad = 30
debug_enabled = False
debug_filter_include = []
func_pad = 35
debug_enabled = True
debug_filter_include = ["DEBUG"] #["to_accessor_data"]
debug_filter_exclude = []
show_time = False

610
src/conversion/func_conv.py
View File

@ -6,6 +6,8 @@ import arg_conv
import c_subs
import re
import grib_api_converter
import struct_arg
import variable
# Change C-style snake_case function name to C++-style camelCase name
# Also, remove get_ and set_ prefixes
@ -37,6 +39,8 @@ class FunctionConverter:
# Store the C to C++ function arg transforms
for index, carg in enumerate(self._cfunction.func_sig.args):
assert carg, f"carg none for function name=[{cfunction.name}]"
cpparg = cppfuncsig.args[index]
self._transforms.add_local_args(carg, cpparg)
@ -55,6 +59,11 @@ class FunctionConverter:
debug.line("skip_line", f"[Empty]: {line}")
return True
# Ignore macros (for now)
if line.startswith("#define") or line.endswith("\\"):
debug.line("skip_line", f"[Macro]: {line}")
return True
if re.match(r"^\s*//", line):
debug.line("skip_line", f"[C++ Comment]: {line}")
return True
@ -85,62 +94,83 @@ class FunctionConverter:
return False
# Override this to provide any initial conversions before the main update_cpp_line runs
def process_variables_initial_pass(self, line):
# ======================================== UPDATE FUNCTIONS ========================================
def update_grib_api_cfunctions(self, line):
line = self.convert_grib_utils(line)
line = grib_api_converter.convert_grib_api_functions(line)
return line
# Special-handling for lengths that apply to buffers. The {ptr,length} C args are replaced by
# a single C++ container arg, however we still need to deal with size-related code in the body
# Note: The {ptr, length} and container arg indices are defined in the transforms object
def process_len_args(self, line):
mapping = self._transforms.funcsig_mapping_for(self._cfunction.name)
if not mapping or not mapping.arg_indexes:
def update_cfunction_names(self, line):
line = c_subs.apply_all_substitutions(line)
# Static function substitutions
# Note: (?=\() ensures group(3) is followed by a "(" without capturing it - ensuring it's a function name!
m = re.search(rf"(&)?\b(\w+)(?=\()", line)
if m:
for mapping in self._transforms.static_funcsig_mappings:
if m.group(2) == mapping.cfuncsig.name:
prefix = m.group(1) if m.group(1) is not None else ""
line = re.sub(m.re, rf"{prefix}{mapping.cppfuncsig.name}", line)
debug.line("update_cfunction_names", f"Updating static function {m.group(0)} [after ]: {line}")
return line
# Override this to provide any function updates specific to a class
def custom_cfunction_updates(self, line):
return line
# Special transforms for well-known return types
def transform_return_carg(self, carg):
if carg.name in ["err", "ret"]:
return arg.Arg("GribStatus", arg_conv.transform_variable_name(carg.name))
else:
return None
# Returns the C++ equivalent, or None if var should be deleted
def transform_carg(self, carg):
cpparg = self.transform_return_carg(carg)
if not cpparg:
arg_converter = arg_conv.ArgConverter(carg)
cpparg = arg_converter.to_cpp_arg(self._transforms)
return cpparg
# Find C variable declarations e.g. size_t len; char* buf = "Data"; and then:
# 1. Transform to C++ and store in the arg_map
# 2. Delete the line if no longer valid
# Note: Assignments are not processed at this stage...
def update_cvariable_declarations(self, line):
m = re.match(rf"^(?:static)?\s*([^=;]*)[=;]", line)
if not m:
return line
buffer_arg_index = mapping.arg_indexes.cbuffer
buffer_len_arg_index = mapping.arg_indexes.clength
container_index = mapping.arg_indexes.cpp_container
if not buffer_arg_index and not buffer_len_arg_index and not container_index:
carg = arg.Arg.from_string(m.group(1))
if not carg:
return line
len_carg = mapping.cfuncsig.args[buffer_len_arg_index]
assert len_carg, f"Len arg should not be none for c function: {mapping.cfuncsig.name}"
container_arg = mapping.cppfuncsig.args[container_index]
assert container_arg, f"Container arg should not be none for C++ function: {mapping.cppfuncsig.name}"
cpparg = self.transform_carg(carg)
# Note: Some code uses len[0] instead of *len, so we check for both...
# Store even if C++ is same as C so we can further process it later...
self._transforms.add_local_args(carg, cpparg)
debug.line("update_cvariable_declarations", f"Added local arg: {arg.arg_string(carg)} -> {arg.arg_string(cpparg)}")
# Replace *len = N with CONTAINER.clear() if N=0, or CONTAINER.resize() the line if N is any other value
m = re.search(rf"\*?\b{len_carg.name}\b(\[0\])?\s*=\s*(\w+).*?;", line)
if m:
if container_arg.is_const():
line = re.sub(rf"^(\s*)", rf"// [length assignment removed - var is const] \1", line)
debug.line("process_len_args", f"Removed len assignment for const variable [after]: {line}")
elif m.group(2) == "0":
line = re.sub(rf"{re.escape(m.group(0))}", rf"{container_arg.name}.clear();", line)
debug.line("process_len_args", f"Replaced *len = 0 with .clear() [after]: {line}")
else:
line = re.sub(rf"{re.escape(m.group(0))}", rf"{container_arg.name}.resize({m.group(2)});", line)
debug.line("process_len_args", f"Replaced *len = N with .resize(N) [after]: {line}")
if not cpparg:
debug.line("update_cvariable_declarations", f"--> deleted line: {line}")
return ""
# Replace *len <=> N with CONTAINER.size() <=> N
m = re.search(rf"\*?\b{len_carg.name}\b(\[0\])?\s*([<>!=]=?)\s*(\w+)", line)
if m:
line = re.sub(rf"{re.escape(m.group(0))}", rf"{container_arg.name}.size() {m.group(2)} {m.group(3)}", line)
debug.line("process_len_args", f"Replaced *len <=> N with .size() <=> N [after]: {line}")
# Replace any other *len with CONTAINER.size() <=> N
m = re.search(rf"\*?\b{len_carg.name}\b(\[0\])?", line)
if m:
line = re.sub(rf"{re.escape(m.group(0))}", rf"{container_arg.name}.size()", line)
debug.line("process_len_args", f"Replaced *len <=> N with .size() <=> N [after]: {line}")
if carg != cpparg:
line = re.sub(rf"{re.escape(m.group(1))}", f"{cpparg.as_declaration()}", line)
debug.line("update_cvariable_declarations", f"--> updated line: {line}")
return line
# Find type declarations and store in the transforms
def process_type_declarations(self, line):
def update_cstruct_type_declarations(self, line):
# struct declaration [1]: [typedef] struct S [S]
m = re.match(r"^(typedef)?\s*struct\s+(\w+)\s*(\w*)?(;)?$", line)
@ -154,95 +184,376 @@ class FunctionConverter:
self._transforms.add_to_types(ctype, cpptype)
line = re.sub(m.re, f"struct {cpptype}", line)
debug.line("process_type_declarations", f"Added struct type transform: {ctype} -> {cpptype} [after ]: {line}")
debug.line("update_cstruct_type_declarations", f"Added struct type transform: {ctype} -> {cpptype} [after ]: {line}")
carg = arg.Arg("struct", ctype)
cpparg = arg.Arg("struct", cpptype)
self._transforms.add_local_args(carg, cpparg)
debug.line("process_type_declarations", f"Adding struct to local arg map: {carg.type} {carg.name} -> {cpparg.type} {cpparg.name} [after ]: {line}")
debug.line("update_cstruct_type_declarations", f"Adding struct to local arg map: {carg.type} {carg.name} -> {cpparg.type} {cpparg.name} [after ]: {line}")
# struct declaration [2]: } S
m = re.match(r"^}\s*(\w+)", line)
if m and m.group(1) not in ["else"]:
# We'll assume this definition is covered by [1]
line = re.sub(m.re, "}", line)
debug.line("process_type_declarations", f"Removed extraneous struct definition: {m.group(1)} [after ]: {line}")
debug.line("update_cstruct_type_declarations", f"Removed extraneous struct definition: {m.group(1)} [after ]: {line}")
return line
# Update any well know return values
def process_return_variables(self, line):
ret = "GribStatus"
if self._cppfunction.return_type == ret:
for ret_var in ["err", "ret"]:
# [1] Assigning to function result - we assume the function returns the correct type!
m = re.search(rf"\bint\s+{ret_var}\s*=\s*(.*)\(", line)
if m:
line = re.sub(m.re, rf"{ret} {ret_var} = {m.group(1)}(", line)
debug.line("process_return_variables", f"return value assigned via function [after ]: {line}")
m = re.search(rf"\bint\b(\s+{ret_var}\s+=\s*)(\d+)[,;]", line)
if m:
line = re.sub(m.re, rf"{ret}\1{ret}{{\2}};", line)
debug.line("process_return_variables", f"return value assigned to value [after ]: {line}")
m = re.search(rf"(\(\s*{ret_var}\s*)\)", line)
if m:
line = re.sub(m.re, rf"\1 != {ret}::SUCCESS)", line)
debug.line("process_return_variables", f"return value comparison updated [after ]: {line}")
return line
# Find variable declarations (including with assignments), e.g. size_t len; char* buf = "Data"; and then
# pass on to functions to:
# 1. Update the type if required / delete the line if no longer valid
# 2. Store in the arg_map for future reference
# 3. Ensure any assignments are valid...
def process_variable_declarations(self, line):
m = re.match(rf"^(?:static)?\s*([^=;]*)([=;])(.*)", line)
# Special handling for function pointers: [typedef] RET (*PFUNC)(ARG1, ARG2, ..., ARGN);
def update_cfunction_pointers(self, line):
m = re.match(r"^(?:typedef)\s*(\w+\**)\s*\(\s*\*(\s*\w+)\s*\)\s*\((.*)\)", line)
if m:
carg = arg.Arg.from_string(m.group(1))
if m and carg:
carg = arg.Arg(m.group(1), m.group(2))
arg_converter = arg_conv.ArgConverter(carg)
cpparg = arg_converter.to_cpp_arg(self._transforms)
if not cpparg:
debug.line("process_variable_declarations", f"Found var declaration to delete: {carg.type} {carg.name}")
self._transforms.add_local_args(carg, None)
debug.line("process_variable_declarations", f"--> deleting: {line}")
return ""
else:
debug.line("process_variable_declarations", f"Found var declaration to store: {carg.type} {carg.name} -> {cpparg.type} {cpparg.name}")
self._transforms.add_local_args(carg, cpparg)
if carg.type != cpparg.type or carg.name != cpparg.name:
line = re.sub(rf"{re.escape(m.group(1))}", f"{cpparg.as_declaration()}", line)
debug.line("process_variable_declarations", f"Transformed line: {line}")
# Assume functions returning int will now return GribStatus
if cpparg.type == "int":
cpparg.type = "GribStatus"
debug.line("update_cfunction_pointers", f"Adding var to local arg map: {carg.type} {carg.name} -> {cpparg.type} {cpparg.name} [after ]: {line}")
self._transforms.add_local_args(carg, cpparg)
# Parse the function arg types
cpp_arg_types = []
for arg_type in [a.strip() for a in m.group(3).split(",")]:
arg_converter = arg_conv.ArgConverter(arg.Arg(arg_type, "Dummy"))
cpp_sig_arg = arg_converter.to_cpp_func_sig_arg(self._transforms)
if cpp_sig_arg:
cpp_arg_types.append(cpp_sig_arg.type)
# Apply the transform
line = re.sub(rf"(\w+\**)\s*\(\s*\*(\s*\w+)\s*\)\s*\((.*)\)", f"{cpparg.type}(*{cpparg.name})({','.join([a for a in cpp_arg_types])})", line)
debug.line("update_cfunction_pointers", f"Transformed line: {line}")
return line
# Return None to delete the line
def transform_cstruct_arg_name(self, cstruct_arg):
assert cstruct_arg and cstruct_arg.name, f"Unexpected empty (None) cstruct_arg"
# Remove any variables that have been marked for deletion
def process_deleted_variables(self, line):
# Find any deleted variables that are being assigned to, and delete the line
m = re.match(r"^\s*\**(\w+)\s*=", line)
if m:
debug.line("process_deleted_variables", f"Found var assignment: var={m.group(1)}: {line}")
for carg, cpparg in self._transforms.all_args.items():
if carg.name == m.group(1) and not cpparg:
debug.line("process_deleted_variables", f"Var assignment marked for delete, var={m.group(1)} - deleting: {line}")
line = f"// [{m.group(1)} removed] " + line
return line
# Remove any deleted vars that remain (i.e. as an argument to a function call)
for carg, cpparg in self._transforms.all_args.items():
if not cpparg and re.match(rf"^.*\b{carg.name}\b\s*,*", line):
line = re.sub(rf"[&\*]?\b{carg.name}(->)?\b\s*,*", "", line)
debug.line("process_deleted_variables", f"Removing arg={carg.name} [after ]: {line}")
if carg.name == cstruct_arg.name:
return cpparg.name if cpparg else None
return cstruct_arg.name
# If all else fails, use this!
# Conversion is *foo[1]->bar[3]->baz[7] => foo[1].bar[3].baz[7]
def apply_default_cstruct_arg_transform(self, cstruct_arg):
cppstruct_arg = None
if cstruct_arg:
cppstruct_access = ""
cppstruct_name = self.transform_cstruct_arg_name(cstruct_arg)
cppstruct_index = cstruct_arg.index
cppstruct_arg = struct_arg.StructArg(cppstruct_access, cppstruct_name, cppstruct_index)
cmember = cstruct_arg.member
cppmember = cppstruct_arg
while cmember:
cppstruct_access = "."
cppstruct_name = self.transform_cstruct_arg_name(cmember)
cppstruct_index = cmember.index
cppmember.member = struct_arg.StructArg(cppstruct_access, cppstruct_name, cppstruct_index)
cmember = cmember.member
cppmember = cppmember.member
return cppstruct_arg
# If the type is well-known (e.g. grib_darray) then apply the transform
# using the supplied cppname and the appropriate rules.
# Returns the transformed cppstruct_arg, or None if ctype not recognised
def apply_cstruct_arg_transforms_for_ctype(self, ctype, cstruct_arg, cppname):
# Check if grib_array
cppstruct_arg = grib_api_converter.process_cstruct_arg_for_grib_api_ctype(ctype, cstruct_arg, cppname)
return cppstruct_arg
# Transform cstruct_arg to cppstruct_arg
# This is the main customisation point for sub-classes
def transform_cstruct_arg(self, cstruct_arg):
cppstruct_arg = None
for carg, cpparg in self._transforms.all_args.items():
if cstruct_arg.name == carg.name and cpparg:
cppstruct_arg = self.apply_cstruct_arg_transforms_for_ctype(carg.type, cstruct_arg, cpparg.name)
if not cppstruct_arg:
cppstruct_arg = self.apply_default_cstruct_arg_transform(cstruct_arg)
return cppstruct_arg
# Update the remainder (of the current line) to correctly set the cppstruct_arg - returns the updated remainder
def update_cppstruct_arg_assignment(self, cppstruct_arg, remainder):
# Default - do nothing!
return remainder
def update_cstruct_access(self, line, depth):
assert depth<10, f"Unexpected recursion depth [{depth}]"
cstruct_arg, match_start, match_end = struct_arg.cstruct_arg_from_string(line)
if cstruct_arg:
if depth == 0: debug.line("update_cstruct_access", f"IN : {line}")
if(match_end < len(line)):
# Recurse, transforming the remainder, then apply here...
remainder = self.update_cstruct_access(line[match_end:], depth+1)
cppstruct_arg = self.transform_cstruct_arg(cstruct_arg)
assert cppstruct_arg, f"Could not transform struct {cstruct_arg.name} to C++"
if not cppstruct_arg.name:
# TODO: Only delete line if assignment, else just remove struct...
line = f"// [Deleted struct {cstruct_arg.name}] " + line
debug.line("update_cstruct_access", f"[{depth}] Deleting {cstruct_arg.name} Line: {line}")
return line
# Check if this is a "set" operation (it's match group 3 to avoid false match with ==)
m = re.search(r"([=!\+\-\*/]=)|([,;\)])|(=)", line[match_end:])
if m and m.group(3):
match_end += m.end()
remainder = " = " + self.update_cppstruct_arg_assignment(cppstruct_arg, line[match_end:])
# Finally, update the line
line = line[:match_start] + cppstruct_arg.as_string() + remainder
if depth == 0: debug.line("update_cstruct_access", f"OUT : {line}")
return line
def update_cstruct_variables(self, line):
line = self.update_cstruct_access(line, 0)
return line
# Checks if the supplied cvariable represents a known variable, and returns a
# transformed C++ variable, or None
#
# "Known variables" are determined by comparing the cvariable name to the stored lists.
# If the cvariable name is found, the return value is a cpp variable with the components
# updated (which may be set to None if no transform available). For example:
#
# cvariable cname -> cppname cppvariable
# (*, foo_name, "") foo_name -> fooName ("", fooName, "")
# (*, handle, "") handle -> None (None, None, None)
# (*, bar, "") NOT FOUND None
#
# Note 1: We assume *var and &var should be replaced with var
# Note 2: When a transformed cppvariable is returned, it has its type set as well (in case this is useful!)
#
def transform_if_cvariable(self, cvariable):
for carg, cpparg in self._transforms.all_args.items():
if cvariable.name == carg.name:
if cpparg:
cppvariable = variable.Variable("", cpparg.name, cvariable.index)
cppvariable.type = cpparg.type
return cppvariable
else:
return variable.Variable(None, None, None)
return None
# Update the remainder (of the current line) to correctly set the cppvariable - returns the updated remainder
def update_cppvariable_assignment(self, cppname, remainder):
# Default - do nothing!
return remainder
# Called first from transform_cvariable_access - override to provide specialised transforms
def custom_transform_cvariable_access(self, cvariable, match_token, post_match_string):
return None
# Special transforms for return variables
def transform_return_cvariable_access(self, cvariable, match_token, post_match_string):
ret = "GribStatus"
cppvariable = self.transform_if_cvariable(cvariable)
if cppvariable and cppvariable.type == ret:
# If match is e.g. "err)" then we'll assume it's a boolean test eg if(err) and not the last arg of a function call, so
# we'll update it to a comparison
if match_token.value == ")":
transformed_string = cppvariable.as_string() + f" != {ret}::SUCCESS" + match_token.as_string() + post_match_string
debug.line("transform_return_cvariable_access", f"transformed boolean return value test: {transformed_string}")
return transformed_string
elif match_token.is_separator or match_token.is_terminator:
# Just need to transform the name
transformed_string = cppvariable.as_string() + match_token.as_string() + post_match_string
debug.line("transform_return_cvariable_access", f"return value transformed: {transformed_string}")
return transformed_string
else:
# Assignment/Comparison, so need to ensure the "rvalue" is the correct type
# Check if assigned to a function call - we assume the function returns the correct type!
m = re.match(r"\s*([^\(]*)\(", post_match_string)
if m:
transformed_string = cppvariable.as_string() + match_token.as_string() + post_match_string
debug.line("transform_return_cvariable_access", f"return value via function call transformed: {transformed_string}")
return transformed_string
# Handle everything else: extract the assigned value and cast to the return type
m = re.match(r"\s*([^,\);]*)(?:\s*[,\);])", post_match_string)
assert m, f"Could not extract assigned value from: {post_match_string}"
transformed_string = cppvariable.as_string() + match_token.as_string() + f" {ret}{{{m.group(1)}}}" + post_match_string[m.end(1):]
debug.line("transform_return_cvariable_access", f"assigned return value transformed: {transformed_string}")
return transformed_string
return None
# Special transforms for lengths that apply to buffers. The {ptr,length} C args are replaced by
# a single C++ container arg, however we still need to deal with size-related code in the body
# Note: The {ptr, length} and container arg indices are defined in the transforms object.
#
# For example:
# cvariable match_token post_match_string transformed_string
# *len[0] = 0; container.clear();
# *len[0] = 4; container.resize(4);
# x == *len; x == container.size();
# y > foo(*len); y > foo(container.size());
#
# Note: Some code uses len[0] instead of *len, so we check for both...
#
def transform_len_cvariable_access(self, cvariable, match_token, post_match_string):
mapping = self._transforms.funcsig_mapping_for(self._cfunction.name)
if not mapping or not mapping.arg_indexes:
return None
buffer_arg_index = mapping.arg_indexes.cbuffer
buffer_len_arg_index = mapping.arg_indexes.clength
container_index = mapping.arg_indexes.cpp_container
if not buffer_arg_index and not buffer_len_arg_index and not container_index:
return None
len_carg = mapping.cfuncsig.args[buffer_len_arg_index]
assert len_carg, f"Len arg should not be None for c function: {mapping.cfuncsig.name}"
if len_carg.name != cvariable.name:
return None
container_arg = mapping.cppfuncsig.args[container_index]
assert container_arg, f"Container arg should not be None for C++ function: {mapping.cppfuncsig.name}"
# Replace *len = N with CONTAINER.clear() if N=0, or CONTAINER.resize() the line if N is any other value
if match_token.is_assignment:
if container_arg.is_const():
debug.line("transform_len_cvariable_access", f"Removed len assignment for const variable [{cvariable.as_string()}]")
return f"// [length assignment removed - var is const] " + cvariable.as_string() + match_token.as_string() + post_match_string
# Extract the assigned value
m = re.match(r"\s*([^,\);]+)\s*[,\);]", post_match_string)
assert m, f"Could not extract assigned value from: {post_match_string}"
if m.group(1) == "0":
debug.line("transform_len_cvariable_access", f"Replaced {cvariable.as_string()} = 0 with .clear()")
return f"{container_arg.name}.clear();"
else:
debug.line("transform_len_cvariable_access", f"Replaced {cvariable.as_string()} = {m.group(1)} with .resize({m.group(1)})")
return f"{container_arg.name}.resize({m.group(1)});"
# Replace any other *len with CONTAINER.size()
debug.line("transform_len_cvariable_access", f"Replaced {cvariable.as_string()} with {container_arg.name}.size()")
return f"{container_arg.name}.size()" + match_token.as_string() + post_match_string
# Fallback if custom transforms don't match
def default_transform_cvariable_access(self, cvariable, match_token, post_match_string):
cppvariable = self.transform_if_cvariable(cvariable)
if cppvariable:
if not cppvariable.name: # Marked for delete
if match_token.is_assignment:
debug.line("default_transform_cvariable_access", f"Deleted [{cvariable.name}]")
return f"// [Deleted variable {cvariable.name}] " + cvariable.as_string() + match_token.as_string() + post_match_string
debug.line("default_transform_cvariable_access", f"Removed [{cvariable.name}] for match [{match_token.value}]")
if match_token.is_terminator:
return match_token.as_string() + post_match_string
else:
return post_match_string
return cppvariable.as_string() + match_token.as_string() + post_match_string
return None
# Takes the cvariable, match_value and post_match_string and return an updated string
# representing the transform, or None if no transform available. For example:
#
# cvariable match_token post_match_string transformed_string
# *foo[0] = 4; foo[4] = 4;
# x == value_count; x = valueCount;
# y > bar(*another_foo); y > bar(anotherFoo);
def transform_cvariable_access(self, cvariable, match_token, post_match_string):
debug.line("transform_cvariable_access", f"[IN] cvariable=[{cvariable.as_string()}] match_token=[{match_token.value}] post_match_string=[{post_match_string}]")
for transform_func in [
self.custom_transform_cvariable_access,
self.transform_return_cvariable_access,
self.transform_len_cvariable_access,
self.default_transform_cvariable_access,
]:
transformed_string = transform_func(cvariable, match_token, post_match_string)
if transformed_string:
return transformed_string
return None
def update_cvariable_access(self, line, depth):
assert depth<15, f"Unexpected recursion depth [{depth}]"
# Regex groups:
# 12 3 4
# *my_foo[6] TOKEN
#
# TOKEN matches one of:
# = (group 5 - assignment)
# != == += >= etc (group 6)
# , ) [ ] ; (group 7 - terminator)
#
# Note:
# (?<!%) is to avoid matching e.g %ld in a printf
# (=(?!=)) matches exactly one "=" so we can distinguish "=" (group 5) from "==" (group 6)
# (?![<>\*&]) is added to ensure pointer/reference types and templates are not captured, e.g. in (grib_accessor*)self; or std::vector<double>
# Group 2 matches a character first to avoid matching numbers as variables
m = re.search(r"([&\*])?\b((?<!%)[a-zA-Z][\w\.]*(?![<>\*&]))(\[\d+\])?\s*((=(?!=))|([!=<>&\|\+\-\*/]+)|([,\)\[\];]))", line)
if m:
if m.group(2) in ["vector", "string", "return", "break"]:
debug.line("update_cvariable_access", f"IN False match [{m.group(2)}] : {line}")
remainder = self.update_cvariable_access(line[m.end():], depth+1)
line = line[:m.end()] + remainder
debug.line("update_cvariable_access", f"OUT False match [{m.group(2)}] : {line}")
else:
cvariable = variable.Variable(pointer=m.group(1), name=m.group(2), index=m.group(3))
match_token = variable.MatchToken(m.group(4))
debug.line("update_cvariable_access", f"IN [{depth}][{cvariable.as_string()}][{match_token.value}]: {line}")
# First process the remainder of the string (recursively), updating it along the way, so we can use it later...
remainder = self.update_cvariable_access(line[m.end():], depth+1)
if True: #remainder:
transformed_remainder = self.transform_cvariable_access(cvariable, match_token, remainder)
if transformed_remainder:
line = line[:m.start()] + transformed_remainder
debug.line("update_cvariable_access", f"OUT [{depth}][{cvariable.as_string()}][{match_token.value}]: {line}")
else:
line = line[:m.end()] + remainder
debug.line("update_cvariable_access", f"OUT [{depth}][No transformed_remainder]: {line}")
return line
def update_cvariables(self, line):
line = self.update_cvariable_access(line, 0)
return line
# ======================================== UPDATE FUNCTIONS ========================================
# Transform any variables, definitions, etc with a "grib" type prefix...
def apply_grib_api_transforms(self, line):
line = grib_api_converter.process_grib_api_variables(line, self._transforms.all_args)
@ -268,36 +579,6 @@ class FunctionConverter:
return line
# Special handling for function pointers: [typedef] RET (*PFUNC)(ARG1, ARG2, ..., ARGN);
def process_function_pointers(self, line):
m = re.match(r"^(?:typedef)\s*(\w+\**)\s*\(\s*\*(\s*\w+)\s*\)\s*\((.*)\)", line)
if m:
carg = arg.Arg(m.group(1), m.group(2))
arg_converter = arg_conv.ArgConverter(carg)
cpparg = arg_converter.to_cpp_arg(self._transforms)
# Assume functions returning int will now return GribStatus
if cpparg.type == "int":
cpparg.type = "GribStatus"
debug.line("process_function_pointers", f"Adding var to local arg map: {carg.type} {carg.name} -> {cpparg.type} {cpparg.name} [after ]: {line}")
self._transforms.add_local_args(carg, cpparg)
# Parse the function arg types
cpp_arg_types = []
for arg_type in [a.strip() for a in m.group(3).split(",")]:
arg_converter = arg_conv.ArgConverter(arg.Arg(arg_type, "Dummy"))
cpp_sig_arg = arg_converter.to_cpp_func_sig_arg(self._transforms)
if cpp_sig_arg:
cpp_arg_types.append(cpp_sig_arg.type)
# Apply the transform
line = re.sub(rf"(\w+\**)\s*\(\s*\*(\s*\w+)\s*\)\s*\((.*)\)", f"{cpparg.type}(*{cpparg.name})({','.join([a for a in cpp_arg_types])})", line)
debug.line("process_function_pointers", f"Transformed line: {line}")
return line
def process_remaining_cargs(self, line):
# Update any C arg that remain (i.e. as an argument to a function call)
@ -345,12 +626,6 @@ class FunctionConverter:
return line
def convert_grib_api_functions(self, line):
line = self.convert_grib_utils(line)
line = grib_api_converter.convert_grib_api_functions(line)
return line
def apply_get_set_substitutions(self, line):
# [1] grib_[gs]et_TYPE[_array][_internal](...) -> unpackTYPE(...)
# Note: This regex is complicated (!) by calls like grib_get_double(h, lonstr, &(lon[i]));
@ -377,20 +652,6 @@ class FunctionConverter:
return line
def apply_function_transforms(self, line):
line = c_subs.apply_all_substitutions(line)
# Static function substitutions
m = re.search(rf"(?<!\")(&)?\b(\w+)\b(?!\")", line)
if m:
for mapping in self._transforms.static_funcsig_mappings:
if m.group(2) == mapping.cfuncsig.name:
prefix = m.group(1) if m.group(1) is not None else ""
line = re.sub(m.re, rf"{prefix}{mapping.cppfuncsig.name}", line)
debug.line("apply_function_transforms", f"Updating static function {m.group(0)} [after ]: {line}")
return line
# Make sure all container variables have sensible assignments, comparisons etc after any transformations
def validate_container_variables(self, line):
for k, v in self._transforms.all_args.items():
@ -425,30 +686,33 @@ class FunctionConverter:
return line
# Override this to provide any function transforms specific to a class
def special_function_transforms(self, line):
return line
def update_cpp_line(self, line):
# Note: These apply in order, be careful if re-arranging!
update_functions = [
# [1] function updates that expect the original C variable names
self.convert_grib_api_functions,
self.apply_function_transforms,
self.special_function_transforms,
# [1] Update C functions only
self.update_grib_api_cfunctions,
self.update_cfunction_names,
self.custom_cfunction_updates,
# [2] Update C variable declarations (inc. typedefs and function pointers)
self.update_cvariable_declarations,
self.update_cstruct_type_declarations,
self.update_cfunction_pointers,
# [3] Update C variable access (get/set)
self.update_cstruct_variables,
self.update_cvariables,
# [4] Update C++ variable asignments
# [5] All other transforms...
# [2] The remaining updates must work with C variables that may have been renamed to C++
self.process_variable_declarations,
self.process_variables_initial_pass,
self.process_len_args,
self.process_type_declarations,
self.process_return_variables,
#self.process_variable_declarations,
self.process_deleted_variables,
self.apply_grib_api_transforms,
self.apply_variable_transforms,
self.process_function_pointers,
self.process_remaining_cargs,
self.process_global_cargs,
self.apply_get_set_substitutions,

4
src/conversion/funcsig.py
View File

@ -28,7 +28,7 @@ class FuncSig:
for entry in [a.strip() for a in m.group(4).split(",")]:
if not entry:
continue
args.append(arg.Arg.from_string(entry))
args.append(arg.Arg.from_func_arg_string(entry))
sig = cls(return_type, name, args)
if sig and m.group(1):
@ -65,4 +65,4 @@ class FuncSig:
self._template = value
def as_string(self):
return f"{'static ' if self.static else ''}{self.return_type} {self.name}({', '.join([a.type + ' ' + a.name for a in self.args if a])})"
return f"{'static ' if self.static else ''}{self.return_type} {self.name}({', '.join([a.type + ' ' + a.name if a.name else '' for a in self.args if a])})"

24
src/conversion/global_func_conv.py
View File

@ -15,42 +15,42 @@ class GlobalFunctionConverter(FunctionConverter):
return global_func.GlobalFunction(cppfuncsig)
# Overridden to apply any static_func_name_transforms
def apply_function_transforms(self, line):
def update_cfunction_names(self, line):
m = re.search(rf"(?<!\")(&)?\b(\w+)\b(?!\")", line)
if m:
for cfuncname, cppfuncname in self._static_func_name_transforms.items():
if m.group(2) == cfuncname:
prefix = m.group(1) if m.group(1) is not None else ""
line = re.sub(m.re, rf"{prefix}{cppfuncname}", line)
debug.line("apply_function_transforms", f"[Global Converter] Updating static function {m.group(0)} [after ]: {line}")
debug.line("update_cfunction_names", f"[Global Converter] Updating static function {m.group(0)} [after ]: {line}")
return super().apply_function_transforms(line)
return super().update_cfunction_names(line)
# If the line starts [FORWARD_DECLARATION] then it is a placeholder from the file parser
# If the line starts @FORWARD_DECLARATION: then it is a placeholder from the file parser
# We ignore it here, but will process it later (once everything else has been resolved)
def special_function_transforms(self, line):
m = re.match(rf"^\[FORWARD_DECLARATION\](\w+)", line)
def custom_cfunction_updates(self, line):
m = re.match(rf"^@FORWARD_DECLARATION:(\w+)", line)
if m:
debug.line("special_function_transforms",f"Ignoring (for now) [FORWARD_DECLARATION] name={m.group(1)}")
debug.line("custom_cfunction_updates",f"Ignoring (for now) @FORWARD_DECLARATION: name={m.group(1)}")
return super().special_function_transforms(line)
return super().custom_cfunction_updates(line)
# Replace any line that starts [FORWARD_DECLARATION] with the actual declaration
# Replace any line that starts @FORWARD_DECLARATION: with the actual declaration
# This need to be called once all other functions have been converted to ensure the
# signatures are correct
def process_forward_declarations(self, static_funcsig_mappings):
lines = self._cppfunction.code
for i in range(len(lines)):
m = re.match(rf"^\[FORWARD_DECLARATION\](\w+)", lines[i])
m = re.match(rf"^@FORWARD_DECLARATION:(\w+)", lines[i])
if m:
processed = False
for mapping in static_funcsig_mappings:
if mapping.cfuncsig.name == m.group(1):
lines[i] = f"{mapping.cppfuncsig.as_string()};"
debug.line("process_forward_declarations",f"[FORWARD_DECLARATION] Updated: {lines[i]}")
debug.line("process_forward_declarations",f"@FORWARD_DECLARATION: Updated: {lines[i]}")
processed = True
if not processed:
lines[i] = ""
debug.line("process_forward_declarations",f"[FORWARD_DECLARATION] Removed (not a static function) : {m.group(0)}")
debug.line("process_forward_declarations",f"@FORWARD_DECLARATION: Removed (not a static function) : {m.group(0)}")

5
src/conversion/grib_accessor.py
View File

@ -9,6 +9,7 @@ import inherited_method
import private_method
import os
import re
import arg
# Represents the C code parsed from a grib_accessor_class*cc file
class GribAccessor:
@ -133,8 +134,8 @@ def create_cfunction(func_sig, definitions):
if not cfunction:
# If any arg starts with a "ptr type name", then it's a private method (as we've already extracted inherited functions)
for arg in func_sig.args:
if re.search(r"grib_accessor(\w*)?\*", arg.type):
for arg_entry in func_sig.args:
if re.search(r"grib_accessor(\w*)?\*", arg_entry.type):
cfunction = private_method.PrivateMethod(func_sig)
if not cfunction:

17
src/conversion/grib_accessor_conv.py
View File

@ -7,6 +7,7 @@ from converter_collection import Converter, converters_for
import transforms
from funcsig_mapping import FuncSigMapping
import grib_api_converter
import copy
prefix = "grib_accessor_class_"
rename = {
@ -111,13 +112,13 @@ class GribAccessorConverter:
return result
def create_transforms(self):
funcsig_type_transforms = common_funcsig_type_transforms
funcsig_type_transforms = copy.deepcopy(common_funcsig_type_transforms)
funcsig_type_transforms.update(grib_api_converter.grib_api_funcsig_type_transforms())
for k,v in funcsig_type_transforms.items():
debug.line("create_transforms",f"Funcsig type transform: {k} -> {v}")
type_transforms = common_type_transforms
type_transforms = copy.deepcopy(common_type_transforms)
type_transforms.update(grib_api_converter.grib_api_type_transforms())
for k,v in type_transforms.items():
@ -125,8 +126,16 @@ class GribAccessorConverter:
self._transforms = transforms.Transforms(funcsig_types=funcsig_type_transforms, types=type_transforms)
self._transforms.add_to_class_types("self", self._grib_accessor.name, self._accessor_data.name)
self._transforms.add_to_class_types("super", self._grib_accessor.super, self._accessor_data.super)
self._transforms.add_to_types("self", self._accessor_data.name)
self._transforms.add_to_types("super", self._accessor_data.super)
# There are some instances of accessing super->super, so we'll store this as well!
cpp_super_super_name = None
if self._grib_accessor.super in self.other_grib_accessors.keys():
cpp_super_super_name = self.transform_class_name(self.other_grib_accessors[self._grib_accessor.super].super)
self._transforms.add_to_types("supersuper", cpp_super_super_name)
def add_global_function(self):
global_func_funcsig_converter = self._converters[Converter.GLOBAL_FUNC_FUNCSIG](self._grib_accessor.global_function.func_sig)

17
src/conversion/grib_api/grib_array.py
View File

@ -1,6 +1,7 @@
import re
import debug
from grib_api.grib_type_transforms import grib_array_type_transforms
import struct_arg
grib_xarray_substitutions = {
r"\bgrib_v?[dis]array_push\(\s*(.*)?,\s*(.*)?\s*\)": r"\1.push_back(\2)",
@ -26,4 +27,18 @@ def process_grib_array_variables(line, carg, cpparg):
# remove any ->v entries (keep the [i] as the cpparg should be a container!)
line = line.replace(f"{cpparg.name}->v", f"{cpparg.name}")
return line
return line
# By default, just replace s->v[4]->v[5] with s[4][5]
def process_grib_array_cstruct_arg(cstruct_arg, cppname):
debug.line("process_grib_array_cstruct_arg", f"cstruct_arg={cstruct_arg.as_string()} cppname={cppname}")
cppstruct_arg = struct_arg.StructArg("", cppname, cstruct_arg.index)
cmember = cstruct_arg.member
cppmember = cppstruct_arg
while cmember:
cppmember.member = struct_arg.StructArg("", "", cmember.index)
cmember = cmember.member
cppmember = cppmember.member
return cppstruct_arg

4
src/conversion/grib_api/grib_type_transforms.py
View File

@ -1,7 +1,7 @@
# Note: grib_accessor* different to funcisg!
# Note: grib_accessor* different to funcsig!
common_grib_type_transforms = {
"grib_accessor*" : "AccessorName",
"grib_accessor*" : None,
"grib_handle*" : None,
"grib_context*" : None,
}

12
src/conversion/grib_api_converter.py
View File

@ -26,4 +26,14 @@ def process_grib_api_variables(line, arg_transforms):
if re.match(r"grib_v?[dis]array", carg.underlying_type):
line = grib_array.process_grib_array_variables(line, carg, cpparg)
return line
return line
# If the ctype is a valid grib_api type, transform it using the supplied cppname and return
# the appropriate cpp struct, else None
def process_cstruct_arg_for_grib_api_ctype(ctype, cstruct_arg, cppname):
cppstruct_arg = None
if re.match(r"grib_v?[dis]array", ctype):
cppstruct_arg = grib_array.process_grib_array_cstruct_arg(cstruct_arg, cppname)
return cppstruct_arg

10
src/conversion/inherited_method_funcsig_conv.py
View File

@ -131,3 +131,13 @@ class InheritedMethodFuncSigConverter(MethodFuncSigConverter):
def __init__(self, cfuncsig):
super().__init__(cfuncsig)
self._conversions.extend(self.inherited_method_conversions)
# Helper to get the mapping of an inherited method from the c function name
# Use for transforms when the "virtual" method is referenced, but not implemented, in a Class
# Returns None if not mapping exists...
def cpp_inherited_method_mapping_for(cfunction_name):
for mapping in InheritedMethodFuncSigConverter.inherited_method_conversions:
if mapping.cfuncsig.name == cfunction_name:
return mapping
return None

24
src/conversion/member_conv.py
View File

@ -14,20 +14,18 @@ class MemberConverter(arg_conv.ArgConverter):
def to_cpp_arg(self, transforms):
cppmember = None
cpp_arg = super().to_cpp_arg(transforms)
if cpp_arg:
cppmember = member.Member(cpp_arg.type, cpp_arg.name + "_")
# We'll assume "const char*" means this variable refers to another accessor...
if self._carg.type == "const char*":
cppmember.type = "AccessorName"
if cppmember.type == "AccessorName":
cppmember.default_value = "{\"\"}"
else:
cppmember.default_value = ""
# We'll assume "const char*" and "grib_accessor*" types mean this variable refers to another accessor...
if self._carg.type in ["const char*", "grib_accessor*"]:
cppmember_name = arg_conv.transform_variable_name(self._carg.name) + "_"
cppmember = member.Member("AccessorName", cppmember_name)
cppmember.default_value = "{\"\"}"
cppmember._mutable = False
else:
cpp_arg = super().to_cpp_arg(transforms)
if cpp_arg:
cppmember = member.Member(cpp_arg.type, cpp_arg.name + "_")
cppmember.default_value = ""
cppmember._mutable = False
return cppmember

149
src/conversion/method_conv.py
View File

@ -1,6 +1,7 @@
from func_conv import *
import method
import inherited_method_funcsig_conv
# Define base class member mapping
base_members_map = {
@ -20,72 +21,77 @@ class MethodConverter(FunctionConverter):
def create_cpp_function(self, cppfuncsig):
return method.Method(cppfuncsig)
# Extra processing required for grib_handle members that are referenced
def update_grib_handle_members(self, line):
# Overridden to process self->, super-> etc
def transform_cstruct_arg(self, cstruct_arg):
for k in self._transforms.all_args.keys():
if k.type == "grib_handle*":
m = re.search(rf"\b{k.name}->buffer", line)
if m:
line = re.sub(rf"{m.group(0)}->data", "buffer_.data()", line)
debug.line("update_grib_handle_members", f"Updated buffer ref [after ]: {line}")
if cstruct_arg:
cppstruct_arg = None
cstruct_member = cstruct_arg.member
return line
# Process member access
if cstruct_arg.name in ["super", "self", "a"]:
cppstruct_member = None
def update_class_members(self, line):
line,count = re.subn(r"\bsuper->\b", f"{self._transforms.class_types['super'].cppvalue}::", line)
if count:
debug.line("update_class_members", f"begin [after ]: {line}")
# Find member arg
for member_dict in [base_members_map, self._transforms.members]:
for carg, cpparg in member_dict.items():
if carg.name == cstruct_member.name:
cppstruct_member = self.apply_cstruct_arg_transforms_for_ctype(carg.type, cstruct_member, cpparg.name)
if not cppstruct_member:
cppstruct_member = struct_arg.StructArg("", cpparg.name, cstruct_member.index)
if cstruct_member.member:
# TODO: Additonal members here means that we've not processed something correctly - need to fix!
cppstruct_member_member = self.apply_default_cstruct_arg_transform(cstruct_member.member)
cppstruct_member.member = cppstruct_member_member
debug.line("transform_cstruct_arg", f"WARNING: Unexpected member, so not processed correctly: {cstruct_member.member.as_string()}")
break
# Extra processing for a-> structs where we've failed to match a member
if not cppstruct_member and cstruct_arg.name == "a":
if cstruct_arg.member.name == "name":
# Special handling: Replace name member with a string literal (it's only used in logging)
cppstruct_member = struct_arg.StructArg("",f"\"{self._transforms.types['self']}\"", "")
else:
# Set name to None to mark it for deletion
debug.line("transform_cstruct_arg", f"Marking for deletion: {cstruct_arg.as_string()}")
cppstruct_member = struct_arg.StructArg("", None, "")
accessor_variable_name = arg_conv.transform_variable_name(self._transforms.class_types["self"].cppvalue)
# If super-> then replace with the correct AccessorName:: call, else remove top-level (self->, a-> etc)
if cstruct_arg.name == "super":
if not cppstruct_member and cstruct_arg.name == "super":
# special case super->super
cppstruct_arg = struct_arg.StructArg("", self._transforms.types['supersuper'], "")
else:
cppstruct_arg = struct_arg.StructArg("", self._transforms.types['super'], "")
cppstruct_arg.member = struct_arg.StructArg("::", cppstruct_member.name, cppstruct_member.index, cppstruct_member.member)
else:
cppstruct_arg = cppstruct_member
# Replace name member with a string literal (it's only used in logging)
line,count = re.subn(r"\ba->name", f"\"{self._transforms.class_types['super'].cppvalue}\"", line)
if count:
debug.line("update_class_members", f"Changed a->name to string literal [after ]: {line}")
assert cppstruct_arg, f"Could not transform cstruct_arg: [{cstruct_arg.as_string()}]"
for n in [r"\bself\b", r"\ba\b"]:
line,count = re.subn(n, f"{accessor_variable_name}", line)
if count:
debug.line("update_class_members", f"this [after ]: {line}")
# Extra processing required for grib_handle members that are referenced
elif cstruct_arg.name == "grib_handle_of_accessor(a)":
if cstruct_member.name == "buffer":
cppstruct_arg = struct_arg.StructArg("", transform_function_name(cstruct_member.name), "" )
if cstruct_member.member and cstruct_member.member.name == "data":
cppstruct_arg.member = struct_arg.StructArg(".", "data()", "")
if re.match(rf".*\b{accessor_variable_name}\s+=", line):
debug.line("update_class_members", f"deleting: {line}")
line = ""
if cppstruct_arg:
return cppstruct_arg
if re.match(rf".*\bsuper\s+=\s+\*\({accessor_variable_name}->cclass->super\)", line):
debug.line("update_class_members", f"deleting: {line}")
line = ""
for k,v in base_members_map.items():
line,count = re.subn(rf"\b{accessor_variable_name}->{k.name}\b", rf"{v.name}", line)
if(count):
debug.line("update_class_members", f"base_members_map [after ]: {line}")
# Process all members in the hierarchy
m = re.search(rf"\b{accessor_variable_name}->(\w+)\b", line)
if m:
for k,v in self._transforms.members.items():
assert v is not None, f"v is None for k={arg.arg_string(k)}"
if m.group(1) == k.name:
line = re.sub(m.re, rf"{v.name}", line)
debug.line("update_class_members", f"members_in_hierarchy [after ]: {line}")
# Not accessing a member, delegate to super()
return super().transform_cstruct_arg(cstruct_arg)
# Overridden to update private member calls
def custom_cfunction_updates(self, line):
m = re.search(rf"\b(\w+)\s*\(\s*([^,]+),", line)
if m:
for mapping in self._transforms.private_funcsig_mappings:
if m.group(1) == mapping.cfuncsig.name:
line = re.sub(m.re, f"{accessor_variable_name}.{mapping.cfuncsig.name}(", line)
debug.line("update_class_members", f"private_methods [after ]: {line}")
line = re.sub(m.re, f"{mapping.cppfuncsig.name}(", line)
debug.line("custom_cfunction_updates", f"private_methods [after]: {line}")
return line
# Override this to provide any initial conversions before the main update_line runs
def process_variables_initial_pass(self, line):
line = self.update_class_members(line)
line = self.update_grib_handle_members(line)
return super().process_variables_initial_pass(line)
return super().custom_cfunction_updates(line)
# Convert grib_(un)pack_TYPE calls to the equivalent member function
# Note: We don't convert the vars here, but we do remove those not needed in the C++ function
@ -107,23 +113,36 @@ class MethodConverter(FunctionConverter):
return line
# Overridden to apply member function substitutions
def apply_function_transforms(self, line):
#
# Note: Group 2 matches self-> and super->
def update_cfunction_names(self, line):
line = self.convert_grib_un_pack_functions(line)
m = re.search(rf"(?<!\")(&)?\b(\w+)\b(?!\")", line)
# Note: (?=\() ensures group(3) is followed by a "(" without capturing it - ensuring it's a function name!
m = re.search(rf"(&)?\b(\w+->)?(\w+)(?=\()", line)
if m:
for mapping in self._transforms.inherited_funcsig_mappings:
if m.group(2) == mapping.cfuncsig.name:
prefix = m.group(1) if m.group(1) is not None else ""
line = re.sub(m.re, rf"{prefix}{mapping.cppfuncsig.name}", line)
debug.line("apply_function_transforms", f"Updating inherited method {m.group(0)} [after ]: {line}")
prefix = m.group(1) if m.group(1) is not None else ""
struct_name = m.group(2)
cfunction_name = m.group(3)
for mapping in self._transforms.private_funcsig_mappings:
if m.group(2) == mapping.cfuncsig.name:
prefix = m.group(1) if m.group(1) is not None else ""
line = re.sub(m.re, rf"{prefix}{mapping.cppfuncsig.name}", line)
debug.line("apply_function_transforms", f"Updating private method {m.group(0)} [after ]: {line}")
if struct_name in ["self->", "super->"]:
# Must be calling an inherited function
mapping = inherited_method_funcsig_conv.cpp_inherited_method_mapping_for(cfunction_name)
assert mapping, f"Expected virtual function call, but could not find mapping for function [{cfunction_name}]"
if struct_name == "super->":
struct_name = self._transforms.types['super'] + "::"
else:
struct_name = ""
line = re.sub(m.re, rf"{prefix}{struct_name}{mapping.cppfuncsig.name}", line)
debug.line("update_cfunction_names", f"Updating inherited class method {m.group(0)} [after ]: {line}")
return line
else:
for mapping in self._transforms.private_funcsig_mappings:
if cfunction_name == mapping.cfuncsig.name:
line = re.sub(m.re, rf"{prefix}{struct_name}{mapping.cppfuncsig.name}", line)
debug.line("update_cfunction_names", f"Updating private class method {m.group(0)} [after ]: {line}")
return line
return super().apply_function_transforms(line)
return super().update_cfunction_names(line)

6
src/conversion/static_func_conv.py
View File

@ -10,10 +10,10 @@ class StaticFunctionConverter(FunctionConverter):
def create_cpp_function(self, cppfuncsig):
return static_func.StaticFunction(cppfuncsig)
def special_function_transforms(self, line):
def custom_cfunction_updates(self, line):
for mapping in self._transforms.static_funcsig_mappings:
line,count = re.subn(rf"\b{mapping.cfuncsig.name}\s*\(", f"{mapping.cppfuncsig.name}(", line)
if(count):
debug.line("update_static_function_calls", f"name={mapping.cfuncsig.name} [after ]: {line}")
debug.line("custom_cfunction_updates", f"name={mapping.cfuncsig.name} [after ]: {line}")
return super().special_function_transforms(line)
return super().custom_cfunction_updates(line)

117
src/conversion/struct_arg.py Executable file
View File

@ -0,0 +1,117 @@
import debug
import re
# Represent the argument of a struct as defined in code, in order to manipulate/transform it
#
# It is recursive to allow structs within structs to be represented
#
# For example: *foo->bar->baz[4]
#
# At the top level: StructArg sa:
# sa.access = "*"
# sa.name = "foo"
# sa.index = ""
#
# Then, one level down, is sa.member:
# sa.member.access = "->"
# sa.member.name = "bar"
# sa.member.index = ""
#
# There is one more level in this example: sa.member.member:
# sa.member.member.access = "->"
# sa.member.member.name = "baz"
# sa.member.member.index = "[4]"
#
# Also:
# from_string() will generate this object from an input string
# as_string() will recursively create the string representation for the entire call chain...
#
class StructArg:
def __init__(self, access, name, index, member=None) -> None:
self._access = access
self._name = name
self._index = index
self._member = member
@property
def access(self):
return self._access
@property
def name(self):
return self._name
@property
def index(self):
return self._index
@property
def member(self):
return self._member
@member.setter
def member(self, value):
self._member = value
# Generate a string to represent the assignment text
def as_string(self):
str = self._access if self._access is not None else ""
str += self._name if self._name is not None else ""
str += self._index if self._index is not None else ""
if self.member:
str += self.member.as_string()
return str
# Find the first struct arg in the input string
#
# Returns: StructArg, match_start, match_end if a match is found, where:
# StructArg represents the matched struct argument
# match_start is the index from the input string where the match starts
# match_end is the index from the input string where the match ends
#
# Returns: None, 0, 0 if no match
#
# Note: the first match group, (/\*), is to ensure we don't match inside a comment!
#
def cstruct_arg_from_string(input):
cstruct_arg = None
match_start = match_end = 0
# Removed '.' match as it messes with container.size() etc calls...
#m = re.search(r"(/\*)|(\*)?(\w+)(\.|->)(\w+)(\[[\w\d]*\])?", input)
# Note: (?:\(.+\))? is a non-capturing group that optionally matches (TEXT)
# and therefore allows us to capture function calls that result in
# struct access, for example: grib_handle_of_accessor(a)->buffer->data;
m = re.search(r"(/\*)|(\*)?(\w+(?:\(.+\))?)(->)(\w+)(\[[\w\d]*\])?", input)
if m and m.group(1) != "/*":
access = m.group(2)
name = m.group(3)
index = None
member = StructArg(access=m.group(4), name=m.group(5), index=m.group(6))
match_start = m.start()
match_end = m.end()
cstruct_arg = StructArg(access, name, index, member)
# Loop, adding any extra member sections (->foo[4]) that exist...
next_member = cstruct_arg.member
while m and m.end() < len(input):
m = re.match(r"(->)(\w+)(\[[\w\d]*\])?", input[match_end:])
if not m:
break
member = StructArg(access=m.group(1), name=m.group(2), index=m.group(3))
match_end += m.end()
next_member.member = member
next_member = next_member.member
if not cstruct_arg:
debug.line("from_string", f"Input does not contain a struct member: {input}")
return cstruct_arg, match_start, match_end

27
src/conversion/transforms.py
View File

@ -3,7 +3,6 @@
import re
import debug
import arg
from collections import namedtuple
import copy
from funcsig_mapping import FuncSigMapping
@ -15,13 +14,6 @@ from funcsig_mapping import FuncSigMapping
# inherited_funcsig_mappings : [funcsig_mapping, ...]
# private_funcsig_mappings : [funcsig_mapping, ...]
# static_funcsig_mappings : [funcsig_mapping, ...]
# class_types : {"Entry", {cvalue, cppvalue} } "Entry" is "self", "super", etc
# These entries are expected to exist!
default_class_type_entries = ["self", "super"]
# Allow access of class_types tuples using .cvalue and .cppvalue for convenience!
TypePair = namedtuple('TypePair', ['cvalue', 'cppvalue'])
class Transforms:
def __init__(self, *, funcsig_types={}, types={}) -> None:
@ -34,7 +26,6 @@ class Transforms:
self._private_funcsig_mappings = []
self._static_funcsig_mappings = []
self._other_funcsig_mappings = []
self._class_types = {}
# Note these are only supplied at __init__, any new types are added to the types list instead
@property
@ -55,12 +46,21 @@ class Transforms:
@property
def all_args(self):
return self._all_args
# Helper to return the ctype for the supplied cname, or None
def ctype_of(self, cname):
for carg in self.all_args.keys():
if carg.name == cname:
return carg.type
return None
def add_local_args(self, carg, cpparg):
if carg in self._all_args:
assert self._all_args[carg] == cpparg, f"Updating an existing local arg transform: C Arg = {arg.arg_string(carg)} -> {arg.arg_string(cpparg)} Previous arg = {arg.arg_string(self._all_args[carg])}"
else:
debug.line("Transforms", f"Adding new local arg transform: {arg.arg_string(carg)} -> {arg.arg_string(cpparg)}")
assert carg, f"ADDING carg which is None!"
self._all_args[carg] = cpparg
def clear_local_args(self):
@ -152,12 +152,3 @@ class Transforms:
for entry in self._other_funcsig_mappings:
assert entry.cfuncsig.name != mapping.cfuncsig.name, f"Setting an existing other_funcsig_mappings transform: {mapping.cfuncsig.name} -> {mapping.cppfuncsig.name}"
self._other_funcsig_mappings.append(mapping)
@property
def class_types(self):
return self._class_types
def add_to_class_types(self, entry, cvalue, cppvalue):
assert entry not in self._class_types, f"Updating an existing class type entry[{entry}] with cvalue={cvalue} cppvalue={cppvalue}"
self._class_types[entry] = TypePair(cvalue, cppvalue)

64
src/conversion/variable.py Normal file
View File

@ -0,0 +1,64 @@
# Helpers for matching variables
class Variable:
def __init__(self, pointer, name, index):
self._pointer = pointer
self._name = name
self._index = index
self._type = None
@property
def pointer(self):
return self._pointer
@property
def name(self):
return self._name
@property
def index(self):
return self._index
# Type can be optionally set when useful, but is not a core member
@property
def type(self):
return self._type
@type.setter
def type(self, value):
self._type = value
def as_string(self):
varstring = self._pointer if self._pointer else ""
varstring += self._name if self._name else ""
varstring += self._index if self._index else ""
return varstring
# Represents the match token when evaluating variables, eg = == != , ; etc
class MatchToken:
def __init__(self, value) -> None:
self._value = value
@property
def value(self):
return self._value
@property
def is_assignment(self):
return self._value == "="
@property
def is_separator(self):
return self._value == ","
@property
def is_terminator(self):
return self._value in [")", "[", "]", ";"]
# Generate a string representing the value in a format that can be added to a string,
# i.e. there is a space before "=", ">=" etc but not ",", ";" etc
def as_string(self):
if self.is_separator or self.is_terminator:
return self._value
else:
return " " + self._value

1826
src/debug.txt Normal file
View File

File diff suppressed because it is too large Load Diff