core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Better struct parsing

This commit is contained in:
kevstone 2024-02-15 09:58:57 +00:00
parent 8702f56f22
commit 123a955e79
10 changed files with 124 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/clang_convert/ast_object/ast_parser.py
View File

@ -692,6 +692,10 @@ class AstParser:
return c_unary_op
def parse_BINARY_OPERATOR(self, node):
debug.line("parse_BINARY_OPERATOR", f"DEBUG NODE DUMP:")
ast_utils.dump_node(node)
children = list(node.get_children())
assert len(children) == 2, f"Expected exactly two children for binary operator"
left_operand, right_operand = children
@ -742,16 +746,15 @@ class AstParser:
# The child node defines the first parameter ("self" in this case)
# However, as we're just storing strings, we'll use the tokens directly!
def parse_MEMBER_REF_EXPR(self, node):
debug.line("parse_MEMBER_REF_EXPR", f"Nodes...")
ast_utils.dump_node(node)
tokens = [token.spelling for token in node.get_tokens()]
debug.line("parse_MEMBER_REF_EXPR", f"[IN] node spelling=[{node.spelling}] type=[{node.type.spelling}] tokens=[{tokens}]")
assert len(tokens) >= 3, f"Expected at least 3 tokens for member ref, but got [{len(tokens)}]"
cstruct_member_access = ast_utils.extract_struct_member_access(tokens)
cstruct_member_access = ast_utils.create_code_object_from_tokens(tokens)
debug.line("parse_MEMBER_REF_EXPR", f"[OUT] cstruct_member_access=[{debug.as_debug_string(cstruct_member_access)}]")
return cstruct_member_access
def parse_CALL_EXPR(self, node):

96
src/clang_convert/ast_object/ast_utils.py
View File

@ -8,6 +8,8 @@ import code_object.init_list as init_list
import code_object.declaration_specifier as declaration_specifier
import code_object.struct_arg as struct_arg
import code_object.struct_member_access as struct_member_access
import code_object.literal as literal
import code_object.array_access as array_access
# Utilities for working with C AST Nodes
@ -130,59 +132,61 @@ def create_cinit_list(cnode):
return cinit_list
# Create a StructMemberAccess from the tokens
def create_struct_member_access(tokens):
assert len(tokens) >= 1, f"Expected at least 1 token, got [{len(tokens)}]"
access = name = index = None
symbol_tokens = ["[", "]", "->", "."]
# Iterate over list of tokens and create the appropriate code_objects
def create_code_object_from_tokens(tokens, depth=0):
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}:IN] tokens={tokens}")
if tokens[0] in ["->", "."]:
access = tokens[0]
tokens = tokens[1:]
name = tokens[0]
tokens = tokens[1:]
assert len(tokens) != 0
name = ""
if len(tokens) > 0:
# Check for index token '[', function call token '(' or template token '<'
if tokens[0] == "[":
index = " ".join([t for t in tokens])
elif tokens[0] == "(":
name += " ".join([t for t in tokens])
elif tokens[0] == "<":
name += " ".join([t for t in tokens])
# Build the name from the non-symbol token(s)
for tok in list(tokens):
if tok not in symbol_tokens:
name += tokens.pop(0)
else:
assert False, f"Unexpected token = [{tokens[0]}]"
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}] name = [{debug.as_debug_string(name)}] tokens={tokens}")
break
assert name and name not in symbol_tokens, f"Name token can't be in {symbol_tokens}!"
name_obj = literal.Literal(name)
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}] name_obj=[{debug.as_debug_string(name_obj)}] [{type(name_obj)}]")
return struct_member_access.StructMemberAccess(access, name, index)
# Parse the tokens, extracting the members
# Returns StructMemberAccess or None
def extract_struct_member_access(tokens):
debug.line("extract_struct_member_access", f"[IN] tokens={tokens}")
pointer_indexes = []
for i in range(len(tokens)):
if tokens[i] in ["->", "."]:
pointer_indexes.append(i)
debug.line("extract_struct_member_access", f"pointer_indexes={pointer_indexes}")
if len(pointer_indexes) == 0:
assert False, f"No -> or . found: could not extract members!"
assert pointer_indexes[0] != 0, f"Member access should not start with -> or ."
if len(tokens) == 0:
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}:OUT] No more tokens: name_obj=[{debug.as_debug_string(name_obj)}] [{type(name_obj)}]")
return name_obj
cstruct_member_access = create_struct_member_access(tokens[0:pointer_indexes[0]])
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}.1] tokens={tokens}")
symbol = tokens.pop(0)
# Add end of token string as final index (to ensure all tokens are processed!)
pointer_indexes.append(len(tokens))
if symbol == "[":
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}] Found a [ - creating an array access object, tokens={tokens}")
index_obj = create_code_object_from_tokens(tokens, depth+1)
array_access_obj = array_access.ArrayAccess(name_obj, index_obj)
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}] array_access_obj=[{debug.as_debug_string(array_access_obj)}] [{type(array_access_obj)}] tokens={tokens}")
name_obj = array_access_obj
if len(tokens) == 0:
return name_obj
symbol = tokens.pop(0)
next_cmember = cstruct_member_access
while len(pointer_indexes) >= 2:
next_cmember.member = create_struct_member_access(tokens[pointer_indexes[0]:pointer_indexes[1]])
pointer_indexes = pointer_indexes[1:]
next_cmember = next_cmember.member
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}.2] tokens={tokens}")
return cstruct_member_access
if symbol == "]":
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}:OUT] Found ], name_obj=[{debug.as_debug_string(name_obj)}] tokens={tokens}")
return name_obj
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}.3] tokens={tokens}")
if symbol in ["->", "."]:
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}] [{symbol}] Creating StructMemberAccess from tokens={tokens}")
member_name_obj = create_code_object_from_tokens(tokens, depth+1)
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}] [{symbol}] member_name_obj=[{debug.as_debug_string(member_name_obj)}]")
member_obj = struct_member_access.StructMemberAccess(symbol, member_name_obj)
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}] [{symbol}] member_obj=[{debug.as_debug_string(member_obj)}]")
struct_member_access_obj = struct_member_access.StructMemberAccess("", name_obj, member_obj)
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}:OUT] struct_member_access_obj=[{debug.as_debug_string(struct_member_access_obj)}] [{type(struct_member_access_obj)}]")
return struct_member_access_obj
debug.line("create_code_object_from_tokens", f"{' ' * depth}[{depth}.4] tokens={tokens}")
assert False, "Didn't expect to get here! tokens={tokens}"

4
src/clang_convert/code_object/array_access.py
View File

@ -9,8 +9,8 @@ class ArrayAccess(code_interface.CodeInterface):
def __init__(self, name, index) -> None:
self._name = name
self._index = index
assert isinstance(self._name, code_interface.CodeInterface), f"Name must be a CodeInterface class"
assert isinstance(self._index, code_interface.CodeInterface), f"Index must be a CodeInterface class"
assert isinstance(self._name, code_interface.CodeInterface), f"Name must be a CodeInterface class, not [{type(self._name)}]"
assert isinstance(self._index, code_interface.CodeInterface), f"Index must be a CodeInterface class, not [{type(self._index)}]"
# TODO - Should this be an Arg in order to access the decl_spec interface?
@property

45
src/clang_convert/code_object/struct_member_access.py
View File

@ -10,27 +10,26 @@ from code_object.code_interface import NONE_VALUE
# For example: *foo->bar->baz[4]
#
# At the top level: StructMemberAccess sa:
# sa.access = "*"
# sa.name = "foo"
# sa.index = ""
# sa.access = "*"
# sa.variable = "foo"
#
# Then, one level down, is sa.member:
# sa.member.access = "->"
# sa.member.name = "bar"
# sa.member.index = ""
# sa.member.access = "->"
# sa.member.variable = "bar"
#
# There is one more level in this example: sa.member.member:
# sa.member.member.access = "->"
# sa.member.member.name = "baz"
# sa.member.member.index = "[4]"
# sa.member.member.access = "->"
# sa.member.member.variable = "baz[4]"
#
class StructMemberAccess(code_interface.CodeInterface):
def __init__(self, access, name, index, member=None) -> None:
def __init__(self, access, variable, member=None) -> None:
self._access = access
self._name = name
self._index = index
self._variable = variable
self._member = member
assert isinstance(self._variable, code_interface.CodeInterface), f"Variable must be a CodeInterface class"
assert self._member is None or isinstance(self._member, code_interface.CodeInterface), f"Member must be a CodeInterface class"
@property
def access(self):
return self._access
@ -40,21 +39,13 @@ class StructMemberAccess(code_interface.CodeInterface):
self._access = value
@property
def name(self):
return self._name
def variable(self):
return self._variable
@name.setter
@variable.setter
def name(self, value):
self._name = value
@property
def index(self):
return self._index
@index.setter
def index(self, value):
assert value[0] == "[" and value[-1] == "]", f"Invalid index [{value}]"
self._index = value
assert isinstance(value, code_interface.CodeInterface), f"Variable must be a CodeInterface class"
self._variable = value
@property
def member(self):
@ -62,12 +53,12 @@ class StructMemberAccess(code_interface.CodeInterface):
@member.setter
def member(self, value):
assert isinstance(value, StructMemberAccess), f"Member must be a StructMemberAccess class"
self._member = value
def as_lines(self):
access_str = self._access if self._access is not None else ""
access_str += self._name if self._name is not None else ""
access_str += self._index if self._index is not None else ""
access_str += self._variable.as_string() if self._variable is not None else ""
if self.member and self.member != NONE_VALUE:
access_str += self.member.as_string()

5
src/clang_convert/code_object_converter/array_access_converter.py
View File

@ -4,6 +4,7 @@ import code_object.array_access as array_access
import code_object_converter.code_interface_converter as code_interface_converter
import code_object_converter.conversion_funcs as conversion_funcs
import code_object.struct_member_access as struct_member_access
from code_object.code_interface import NONE_VALUE
class ArrayAccessConverter(code_interface_converter.CodeInterfaceConverter):
def __init__(self, ccode_object) -> None:
@ -14,6 +15,10 @@ class ArrayAccessConverter(code_interface_converter.CodeInterfaceConverter):
cpp_name = conversion_funcs.convert_ccode_object(self._ccode_object.name, conversion_pack)
cpp_index = conversion_funcs.convert_ccode_object(self._ccode_object.index, conversion_pack)
if cpp_name == NONE_VALUE:
debug.line("create_cpp_code_object", f"cpp_name=[{debug.as_debug_string(cpp_name)}] cpp_index=[{debug.as_debug_string(cpp_index)}] for self._ccode_object=[{debug.as_debug_string(self._ccode_object)}] => returning NONE_VALUE")
return NONE_VALUE
# If the name has been converted to a struct_member_access, representing a container size, e.g. foo.size(),
# then we need to deal with the array access part!
if isinstance(cpp_name, struct_member_access.StructMemberAccess):

10
src/clang_convert/code_object_converter/conversion_pack/arg_utils.py
View File

@ -32,9 +32,13 @@ def extract_name(cpp_obj):
elif isinstance(cpp_obj, VariableDeclaration):
cppname = cpp_obj.variable
elif isinstance(cpp_obj, ArrayAccess):
cppname = cpp_obj.name
cppname = extract_name(cpp_obj.name)
elif isinstance(cpp_obj, StructMemberAccess):
cppname = cpp_obj.name
if cpp_obj.name:
cppname = extract_name(cpp_obj.name)
if not cppname and cpp_obj.member.name:
cppname = extract_name(cpp_obj.member.name)
elif isinstance(cpp_obj, ValueDeclarationReference):
cppname = cpp_obj.value
elif isinstance(cpp_obj, UnaryOperation):
@ -71,7 +75,7 @@ def to_cpparg(cpp_obj, conversion_data):
if not cpparg:
cpparg = conversion_data.cpparg_for_cname(name)
assert cpparg is None or isinstance(cpparg, Arg), f"cpparg should be Arg, not [{type(cpparg).__name__}]"
assert cpparg is None or isinstance(cpparg, Arg), f"cpp_obj=[{debug.as_debug_string(cpp_obj)}] : cpparg should be Arg, not [{type(cpparg).__name__}]"
debug.line("to_cpparg", f"cpp_obj=[{debug.as_debug_string(cpp_obj)}] -> cpparg=[{debug.as_debug_string(cpparg)}]")
return cpparg

4
src/clang_convert/code_object_converter/conversion_pack/container_utils.py
View File

@ -9,13 +9,13 @@ from code_object.code_interface import NONE_VALUE
class ContainerUtils:
def create_cpp_container_buffer_arg(self, name):
cpp_member = StructMemberAccess(None, name, None)
cpp_member = StructMemberAccess(None, Literal(name), None)
debug.line("create_cpp_container_buffer_arg", f"cpp_member=[{debug.as_debug_string(cpp_member)}]")
return cpp_member
def create_cpp_container_length_arg(self, name):
cpp_member = self.create_cpp_container_buffer_arg(name)
cpp_member.member = StructMemberAccess(".", "size()", None)
cpp_member.member = StructMemberAccess(".", Literal("size()"), None)
debug.line("create_cpp_container_length_arg", f"cpp_member=[{debug.as_debug_string(cpp_member)}]")
return cpp_member

15
src/clang_convert/code_object_converter/conversion_pack/conversion_data.py
View File

@ -401,6 +401,9 @@ class ConversionData:
# Given the cppname, search all stores to see if an arg exists
def cpparg_for_cppname(self, cppname):
if not isinstance(cppname, str):
cppname = arg_utils.extract_name(cppname)
cpparg = self.funcbody_cpparg_for_cpparg_name(cppname)
if cpparg:
return cpparg
@ -421,6 +424,9 @@ class ConversionData:
# Given the cname, search all stores to see if an arg exists
def cpparg_for_cname(self, cname):
if not isinstance(cname, str):
cname = arg_utils.extract_name(cname)
cpparg = self.funcbody_cpparg_for_carg_name(cname)
if cpparg:
return cpparg
@ -436,6 +442,9 @@ class ConversionData:
return None
def is_cppdata_member(self, cppdata_member_name):
if not isinstance(cppdata_member_name, str):
cppdata_member_name = arg_utils.extract_name(cppdata_member_name)
for mapping in self.all_mappings():
for key, value in mapping.data_member_mappings.items():
if value and value.name == cppdata_member_name:
@ -443,6 +452,9 @@ class ConversionData:
return False
def cppdata_member_for_cppname(self, cppname):
if not isinstance(cppname, str):
cppname = arg_utils.extract_name(cppname)
for mapping in self.all_mappings():
for key, value in mapping.data_member_mappings.items():
if value and value.name == cppname:
@ -504,6 +516,9 @@ class ConversionData:
return False
def is_self_class_pointer_name(self, name):
if not isinstance(name, str):
name = arg_utils.extract_name(name)
debug.line("is_self_class_pointer_name", f"Testing name=[{debug.as_debug_string(name)}]")
for mapping in self.all_mappings():
for entry in mapping.self_class_pointer_names:

36
src/clang_convert/code_object_converter/struct_member_access_converter.py
View File

@ -4,6 +4,8 @@ import code_object.struct_member_access as struct_member_access
import code_object_converter.code_interface_converter as code_interface_converter
import code_object_converter.conversion_funcs as conversion_funcs
from code_object.code_interface import NONE_VALUE
import code_object_converter.conversion_pack.arg_utils as arg_utils
import code_object.literal as literal
class StructMemberAccessConverter(code_interface_converter.CodeInterfaceConverter):
def __init__(self, ccode_object) -> None:
@ -17,30 +19,29 @@ class StructMemberAccessConverter(code_interface_converter.CodeInterfaceConverte
debug.line("create_cpp_code_object",f"StructMemberAccessConverter [IN] cstruct_member_access=[{debug.as_debug_string(cstruct_member_access)}]")
# Check if this is a pointer to a class member
if conversion_pack.conversion_data.is_self_class_pointer_name(cstruct_member_access.name):
debug.line("create_cpp_code_object",f"StructMemberAccessConverter [1] cstruct_member_access.name=[{debug.as_debug_string(cstruct_member_access.name)}] is a self class pointer name")
variable_name = arg_utils.extract_name(cstruct_member_access.variable)
if conversion_pack.conversion_data.is_self_class_pointer_name(variable_name):
debug.line("create_cpp_code_object",f"StructMemberAccessConverter [1] cstruct_member_access.variable=[{debug.as_debug_string(cstruct_member_access.variable)}] is a self class pointer variable")
cppstruct_member_access = self.try_to_convert_data_member_access(cstruct_member_access.member)
if not cppstruct_member_access:
# Check if the primary member is valid...
cpparg = conversion_pack.conversion_data.cpparg_for_cname(cstruct_member_access.name)
cpparg = conversion_pack.conversion_data.cpparg_for_cname(variable_name)
if cpparg==NONE_VALUE:
debug.line("create_cpp_code_object", f"StructMemberAccessConverter [2] cstruct_member_access.name=[{cstruct_member_access.name}] cpparg=[{debug.as_debug_string(cpparg)}] cstruct_member_access.member=[{debug.as_debug_string(cstruct_member_access.member)}]")
debug.line("create_cpp_code_object", f"StructMemberAccessConverter [2] cstruct_member_access.variable=[{cstruct_member_access.variable}] cpparg=[{debug.as_debug_string(cpparg)}] cstruct_member_access.member=[{debug.as_debug_string(cstruct_member_access.member)}]")
if cstruct_member_access.member:
# See if we can convert the member...
debug.line("create_cpp_code_object", f"StructMemberAccessConverter [2.1] Attempting to convert cstruct_member_access.member=[{debug.as_debug_string(cstruct_member_access.member)}]...")
test_member_access = struct_member_access.StructMemberAccess("",
cstruct_member_access.member.name,
cstruct_member_access.member.index,
cstruct_member_access.member.variable,
cstruct_member_access.member.member)
cppstruct_member_access = self.try_to_convert_data_member_access(test_member_access)
else:
cpp_access = conversion_funcs.convert_ccode_object(cstruct_member_access.access, conversion_pack)
cpp_name = conversion_funcs.convert_ccode_object(cstruct_member_access.name, conversion_pack)
cpp_index = conversion_funcs.convert_ccode_object(cstruct_member_access.index, conversion_pack)
cpp_variable = conversion_funcs.convert_ccode_object(cstruct_member_access.variable, conversion_pack)
cpp_member = conversion_funcs.convert_ccode_object(cstruct_member_access.member, conversion_pack)
cppstruct_member_access = struct_member_access.StructMemberAccess(cpp_access, cpp_name, cpp_index, cpp_member)
cppstruct_member_access = struct_member_access.StructMemberAccess(cpp_access, cpp_variable, cpp_member)
if not cppstruct_member_access:
return NONE_VALUE
@ -51,21 +52,20 @@ class StructMemberAccessConverter(code_interface_converter.CodeInterfaceConverte
def try_to_convert_data_member_access(self, candidate_member_access):
debug.line("try_to_convert_data_member_access",f"[IN] candidate_member_access=[{debug.as_debug_string(candidate_member_access)}]")
cppstruct_member_access = None
search_name = candidate_member_access.name
debug.line("try_to_convert_data_member_access", f"Searching [1] for data member for name=[{search_name}]")
cpp_data_member = self._conversion_pack.conversion_data.cppdata_member_for_cdata_member_name(search_name)
search_variable = arg_utils.extract_name(candidate_member_access.variable)
debug.line("try_to_convert_data_member_access", f"Searching [1] for data member for variable=[{search_variable}]")
cpp_data_member = self._conversion_pack.conversion_data.cppdata_member_for_cdata_member_name(search_variable)
if not cpp_data_member:
search_name = candidate_member_access.as_string()
debug.line("try_to_convert_data_member_access", f"Searching [2] for data member for name=[{search_name}]")
cpp_data_member = self._conversion_pack.conversion_data.cppdata_member_for_cdata_member_name(search_name)
search_variable = candidate_member_access.as_string()
debug.line("try_to_convert_data_member_access", f"Searching [2] for data member for variable=[{search_variable}]")
cpp_data_member = self._conversion_pack.conversion_data.cppdata_member_for_cdata_member_name(search_variable)
if cpp_data_member:
cpp_access = ""
cpp_name = cpp_data_member.name
cpp_index = conversion_funcs.convert_ccode_object(candidate_member_access.index, self._conversion_pack)
cpp_variable = literal.Literal(cpp_data_member.name)
cpp_member = conversion_funcs.convert_ccode_object(candidate_member_access.member, self._conversion_pack)
cppstruct_member_access = struct_member_access.StructMemberAccess(cpp_access, cpp_name, cpp_index, cpp_member)
cppstruct_member_access = struct_member_access.StructMemberAccess(cpp_access, cpp_variable, cpp_member)
debug.line("try_to_convert_data_member_access",f"[OUT] cppstruct_member_access=[{debug.as_debug_string(cppstruct_member_access)}]")
return cppstruct_member_access

1
src/clang_convert/code_object_converter/switch_statement_converter.py
View File

@ -13,6 +13,7 @@ class SwitchStatementConverter(code_interface_converter.CodeInterfaceConverter):
cpp_condition = conversion_funcs.convert_ccode_object(self._ccode_object.condition, conversion_pack)
cpp_statement = conversion_funcs.convert_ccode_object(self._ccode_object.statement, conversion_pack)
debug.line("create_cpp_code_object", f"cpp_condition=[{debug.as_debug_string(cpp_condition)}] for self._ccode_object.condition=[{debug.as_debug_string(self._ccode_object.condition)}]")
cpp_while = switch_statement.SwitchStatement(cpp_condition, cpp_statement)
return conversion_pack.conversion_validation.validate_while_statement(self._ccode_object, cpp_while)