Added better multi-line and inherited method parsing

This commit is contained in:
kevstone 2023-11-04 20:13:34 +00:00
parent f33093b41d
commit d2d84c10c5
17 changed files with 319 additions and 162 deletions

View File

@ -59,28 +59,32 @@ def parse_file(path):
# Some function calls are split over multiple lines, which causes issues when
# parsing, so we combine these into a single long line
# Note: Multiline is disabled until we reach the /* END_CLASS_IMP */ line as
# the accessor definitions need to be parsed as individual lines
multiline_enabled = False
multiline = ""
function_start_re = r"\b[^(\s]*\("
function_continues_re = r"[,\(\"]\s*$"
comment_or_space_re = r"((\s*)|(\s*/\*.*\*/\s*)|(\s*//.*))"
function_continues_re = r"[,\(\"]" + comment_or_space_re + "$"
f = open(path, "r")
for line in f:
# Multiline function parsing - start
if multiline_enabled:
if multiline:
if re.search(rf"{function_continues_re}", line):
multiline += line.lstrip()
else:
multiline = multiline.replace("\n", "")
line = multiline + line.lstrip()
multiline = ""
if multiline:
if re.search(rf"{function_continues_re}", line):
multiline += line.lstrip()
else:
multiline = multiline.replace("\n", "")
line = multiline + line.lstrip()
multiline = ""
elif re.search(rf"({function_start_re}$)|({function_start_re}.*{function_continues_re})", line):
multiline = line
elif re.search(rf"{function_start_re}.*{function_continues_re}", line):
multiline = line
if multiline:
continue
if multiline:
continue
# Multiline function parsing - end
@ -103,6 +107,7 @@ def parse_file(path):
if stripped_line.startswith("/* END_CLASS_IMP */"):
in_implementation = False
multiline_enabled = True
continue
if in_implementation:

View File

@ -108,7 +108,115 @@ class FunctionConverter:
debug.line("convert_grib_utils", f"Replaced {util} with {cpp_util} [after ]: {line}")
return line
# Helper for transform_cfunction_call to apply any conversions specified in the
# supplied conversions map
# Returns the transformed cpp function name and C params, or None (for each)
def transform_cfunction_call_from_conversions(self, cfuncname, cparams, funcsig_conversions):
cppfuncsig = None
for mapping in funcsig_conversions:
if mapping.cfuncsig.name == cfuncname:
cppfuncsig = mapping.cppfuncsig
break
if not cppfuncsig:
return None, None
# Map the cparams to their equivalent C++ params
# Note: 1: The params are not converted to C++
# 2: any & prefixes are removed
transformed_cparams = []
for index, func_arg in enumerate(cppfuncsig.args):
if not func_arg:
continue
param = cparams[index]
# C params are converted later, however if one of them is a string representing an accessor name then we'll
# have to convert it now...
if func_arg.type.startswith("AccessorName") and param[0] == "\"":
param = f"AccessorName({param})"
if param[0] == "&":
param = param[1:]
transformed_cparams.append(param)
return cppfuncsig.name, transformed_cparams
# Convert any C function calls to the C++ equivalent
# This will update which arguments are supplied, but does NOT transform the args
# to C++ (this is done later)
# Returns the transformed cpp function name and C params, or None (for each)
# Override to specialise (e.g. to provide a particular mapping to process)
def transform_cfunction_call(self, cfuncname, cparams):
cppfuncname = transformed_cparams = None
# Get list of grib conversions
for grib_conversions in grib_api_converter.grib_funcsig_conversions():
cppfuncname, transformed_cparams = self.transform_cfunction_call_from_conversions(cfuncname, cparams, grib_conversions)
if cppfuncname:
break
return cppfuncname, transformed_cparams
# Takes a C function name and params and transforms the func name to C++
# and updates the arguments list to match the C++ function, but without transforming each
# remaining arg (this is done later)
# The result is returned as a string, or None if no transform exists
#
# Override e.g. to convert foo(a,b) to a->foo(b)
def convert_cfunction_to_cpp_format(self, cfuncname, cparams):
cppfuncname, transformed_cparams = self.transform_cfunction_call(cfuncname, cparams)
if cppfuncname:
return f"{cppfuncname}({','.join([p for p in transformed_cparams])})"
return None
# Find any C function calls in the line and pass to transform_cfunction_call
# If a transformed cppfunction is returned, update the line
def convert_cfunction_calls(self, line):
# Find function calls
m = re.search(r"\b(grib[^(\s]*)\(", line)
if not m:
return line
cfuncname = m.group(1)
match_start = m.start()
match_end = m.end()
# Capture the parameters
cparams = []
param_re = r"\s*([^,]*)\s*"
while match_end < len(line):
m = re.search(rf"{param_re},", line[match_end:])
if m:
match_end += m.end()
cparams.append(m.group(1))
else:
break
# Final param...
param_re = r"\s*([^\)]*)\s*"
m = re.search(rf"{param_re}\)", line[match_end:])
assert m, f"No final param - is it multi-line? Input line [{line}]"
cparams.append(m.group(1))
match_end += m.end()
cppfunction_call = self.convert_cfunction_to_cpp_format(cfuncname, cparams)
if cppfunction_call:
line = line[:match_start] + cppfunction_call + line[match_end:]
debug.line("convert_funcsig",f"Converted function [{cfuncname}] [After]: {line}")
return line
def update_grib_api_cfunctions(self, line):
line = self.convert_grib_utils(line)
line = grib_api_converter.convert_grib_api_functions(line)
@ -837,6 +945,7 @@ class FunctionConverter:
# Note: These apply in order, be careful if re-arranging!
update_functions = [
# [1] Update C functions only
self.convert_cfunction_calls,
self.update_grib_api_cfunctions,
self.update_cfunction_names,
self.custom_cfunction_updates,

View File

@ -66,4 +66,5 @@ class FuncSig:
self._template = value
def as_string(self):
return f"{'static ' if self.static else ''}{self.return_type} {self.name}({', '.join([a.type + ' ' + a.name if a.name else '' for a in self.args if a])})"
return f"{'static ' if self.static else ''}{self.return_type} {self.name}({', '.join([a.type + ' ' + a.name if a.name else '' for a in self.args if a])})"

View File

@ -17,6 +17,7 @@ rename = {
}
common_includes = [
"Accessor.h",
"AccessorFactory.h",
"AccessorUtils/ConversionHelper.h",
"GribCpp/GribQuery.h",
@ -46,9 +47,10 @@ common_funcsig_type_transforms = {
# These will be used if no other supplied...
common_type_transforms = {
"char**" : "std::string&",
"char*" : "std::string",
"char[]" : "std::string",
"unsigned char*" : "DataPointer",
"char**" : "std::string&",
"char*" : "std::string",
"char[]" : "std::string",
}
# Convert GribAccessor to AccessorData

View File

@ -9,71 +9,3 @@ funcsig_conversions = [
grib_value_funcsig_conv.grib_value_funcsig_conversions
]
def convert_funcsig(line):
m = re.search(r"\b(grib[^(\s]*)\(", line)
if m:
func_name = m.group(1)
cppfuncsig = None
for funcsig_conv in funcsig_conversions:
for mapping in funcsig_conv:
if mapping.cfuncsig.name == func_name:
cppfuncsig = mapping.cppfuncsig
break
if not cppfuncsig:
return line
match_start = m.start()
match_end = m.end()
# Capture the parameters
params = []
param_re = r"\s*([^,]*)\s*"
while match_end < len(line):
m = re.search(rf"{param_re},", line[match_end:])
if m:
match_end += m.end()
params.append(m.group(1))
else:
break
# Final param...
param_re = r"\s*([^\)]*)\s*"
m = re.search(rf"{param_re}\)", line[match_end:])
if m:
params.append(m.group(1))
match_end += m.end()
else:
debug.line("convert_funcsig", f"No final param - is it multi-line? Input line [{line}]")
converted_params = convert_params(cppfuncsig, params)
converted_func = f"{cppfuncsig.name}({','.join([p for p in converted_params])})"
line = line[:match_start] + converted_func + line[match_end:]
debug.line("convert_funcsig",f"Converted function [{func_name}] [After]: {line}")
return line
def convert_params(cppfuncsig, params):
converted_params = []
for index, func_arg in enumerate(cppfuncsig.args):
if not func_arg:
continue
param = params[index]
if func_arg.type.startswith("AccessorName"):
if param[0] == "\"":
param = "AccessorName(" + param + ")"
if param[0] == "&":
param = param[1:]
converted_params.append(param)
return converted_params

View File

@ -16,8 +16,11 @@ def grib_api_type_transforms():
type_transforms = grib_type_transforms.all_grib_type_transforms()
return type_transforms
# Return a list of all the grib_*** funcsig conversions
def grib_funcsig_conversions():
return grib_funcsig_conv.funcsig_conversions
def convert_grib_api_functions(line):
line = grib_funcsig_conv.convert_funcsig(line)
line = grib_array.convert_grib_array_functions(line)
return line

View File

@ -1,6 +1,7 @@
from method_conv import *
import inherited_method
import inherited_method_funcsig_conv
# Specialisation of MethodConverter for AccessorData Methods
class InheritedMethodConverter(MethodConverter):
@ -35,3 +36,42 @@ class InheritedMethodConverter(MethodConverter):
return self.create_commented_cpp_body()
return super().create_cpp_body()
# Version of transform_cfunction_call for inherited methods
# Added to avoid traversing the map twice through calls to convert_cfunction_to_cpp_format
# Calls directly to transform_cfunction_call will also come through here...
def transform_cinherited_method_call(self, cfuncname, cparams):
m = re.match(r"grib_(\w*)", cfuncname)
if m:
cmethod_name = m.group(1)
else:
cmethod_name = cfuncname
# Need to use the full list in case the current method's local mappings don't support it!
conversions = inherited_method_funcsig_conv.InheritedMethodFuncSigConverter.inherited_method_conversions
cppfuncname, transformed_cparams = self.transform_cfunction_call_from_conversions(cmethod_name, cparams, conversions)
return cppfuncname, transformed_cparams
# Overridden to handle inherited methods, and also grib_pack_long() etc that
# convert to accessor_ptr->packLong()
def transform_cfunction_call(self, cfuncname, cparams):
cppfuncname, transformed_cparams = self.transform_cinherited_method_call(cfuncname, cparams)
if cppfuncname:
return cppfuncname, transformed_cparams
return super().transform_cfunction_call(cfuncname, cparams)
# Overridden e.g. to convert foo(a,b) to a->foo(b) (as required)
def convert_cfunction_to_cpp_format(self, cfuncname, cparams):
cppfuncname, transformed_cparams = self.transform_cinherited_method_call(cfuncname, cparams)
if cppfuncname:
assert len(cparams) >= 1, f"Expected at least 1 parameter for function [{cfuncname}]"
return f"{cparams[0]}->{cppfuncname}({','.join([p for p in transformed_cparams ])})"
return super().convert_cfunction_to_cpp_format(cfuncname, cparams)

View File

@ -21,63 +21,100 @@ class MethodConverter(FunctionConverter):
def create_cpp_function(self, cppfuncsig):
return method.Method(cppfuncsig)
# transform_cstruct_arg helpers - return cppstruct_arg or None
def transform_cstruct_arg_member(self, cstruct_arg):
cppstruct_arg = None
cstruct_member = cstruct_arg.member
if cstruct_arg.name in ["super", "self", "a"]:
cppstruct_member = None
# Find member arg
for member_dict in [base_members_map, self._transforms.members]:
for carg, cpparg in member_dict.items():
if carg.name == cstruct_member.name:
cppstruct_member = self.apply_cstruct_arg_transforms_for_ctype(carg.type, cstruct_member, cpparg.name)
if not cppstruct_member:
cppstruct_member = struct_arg.StructArg("", cpparg.name, cstruct_member.index)
if cstruct_member.member:
# TODO: Additonal members here means that we've not processed something correctly - need to fix!
cppstruct_member_member = self.apply_default_cstruct_arg_transform(cstruct_member.member)
cppstruct_member.member = cppstruct_member_member
debug.line("transform_cstruct_arg_member", f"WARNING: Unexpected member, so not processed correctly: {cstruct_member.member.as_string()}")
break
# Extra processing for a-> structs where we've failed to match a member
if not cppstruct_member and cstruct_arg.name == "a":
if cstruct_arg.member.name == "name":
# Special handling: Replace name member with a string literal (it's only used in logging)
cppstruct_member = struct_arg.StructArg("",f"\"{self._transforms.types['self']}\"", "")
else:
# Set name to None to mark it for deletion
debug.line("transform_cstruct_arg_member", f"Marking for deletion: {cstruct_arg.as_string()}")
cppstruct_member = struct_arg.StructArg("", None, "")
# If super-> then replace with the correct AccessorName:: call, else remove top-level (self->, a-> etc)
if cstruct_arg.name == "super":
if not cppstruct_member and cstruct_arg.name == "super":
# special case super->super
cppstruct_arg = struct_arg.StructArg("", self._transforms.types['supersuper'], "")
else:
cppstruct_arg = struct_arg.StructArg("", self._transforms.types['super'], "")
cppstruct_arg.member = struct_arg.StructArg("::", cppstruct_member.name, cppstruct_member.index, cppstruct_member.member)
else:
cppstruct_arg = cppstruct_member
assert cppstruct_arg, f"Could not transform cstruct_arg: [{cstruct_arg.as_string()}]"
return cppstruct_arg
def transform_cstruct_arg_grib_handle_member(self, cstruct_arg):
cppstruct_arg = None
cstruct_member = cstruct_arg.member
if self._transforms.ctype_of(cstruct_arg.name) == "grib_handle*" or cstruct_arg.name == "grib_handle_of_accessor(a)":
if cstruct_member.name == "buffer":
debug.line("transform_cstruct_arg_member", f"buffer: cstruct_arg=[{cstruct_arg.as_string()}]")
cppstruct_arg = struct_arg.StructArg("", "buffer_", "" )
if cstruct_member.member and cstruct_member.member.name == "data":
cppstruct_arg.member = struct_arg.StructArg(".", "data()", cstruct_member.member.index)
return cppstruct_arg
def transform_cstruct_arg_accessor_ptr(self, cstruct_arg):
cppstruct_arg = None
# If AccessorPtr, keep access as "->"
cpparg = self._transforms.cpparg_for(cstruct_arg.name)
if cpparg and cpparg.type == "AccessorPtr":
if cstruct_arg.member.name == "name":
# Special handling: Replace name member with a string literal (it's only used in logging)
cppstruct_arg = struct_arg.StructArg("", f"\"{self._transforms.types['self']}\"", "" )
else:
cppstruct_arg = struct_arg.StructArg("", cpparg.name, cstruct_arg.index )
cppstruct_arg.member = cstruct_arg.member
debug.line("transform_cstruct_arg_accessor_ptr", f"[{cstruct_arg.as_string()}] -> [{cppstruct_arg.as_string()}]")
return cppstruct_arg
# Overridden to process self->, super-> etc
def transform_cstruct_arg(self, cstruct_arg):
assert cstruct_arg, f"Unexpected cstruct_arg with None value"
if cstruct_arg:
cppstruct_arg = None
cstruct_member = cstruct_arg.member
transform_funcs = [
self.transform_cstruct_arg_member,
self.transform_cstruct_arg_grib_handle_member,
self.transform_cstruct_arg_accessor_ptr,
]
# Process member access
if cstruct_arg.name in ["super", "self", "a"]:
cppstruct_member = None
cppstruct_arg = None
# Find member arg
for member_dict in [base_members_map, self._transforms.members]:
for carg, cpparg in member_dict.items():
if carg.name == cstruct_member.name:
cppstruct_member = self.apply_cstruct_arg_transforms_for_ctype(carg.type, cstruct_member, cpparg.name)
if not cppstruct_member:
cppstruct_member = struct_arg.StructArg("", cpparg.name, cstruct_member.index)
if cstruct_member.member:
# TODO: Additonal members here means that we've not processed something correctly - need to fix!
cppstruct_member_member = self.apply_default_cstruct_arg_transform(cstruct_member.member)
cppstruct_member.member = cppstruct_member_member
debug.line("transform_cstruct_arg", f"WARNING: Unexpected member, so not processed correctly: {cstruct_member.member.as_string()}")
break
# Extra processing for a-> structs where we've failed to match a member
if not cppstruct_member and cstruct_arg.name == "a":
if cstruct_arg.member.name == "name":
# Special handling: Replace name member with a string literal (it's only used in logging)
cppstruct_member = struct_arg.StructArg("",f"\"{self._transforms.types['self']}\"", "")
else:
# Set name to None to mark it for deletion
debug.line("transform_cstruct_arg", f"Marking for deletion: {cstruct_arg.as_string()}")
cppstruct_member = struct_arg.StructArg("", None, "")
# If super-> then replace with the correct AccessorName:: call, else remove top-level (self->, a-> etc)
if cstruct_arg.name == "super":
if not cppstruct_member and cstruct_arg.name == "super":
# special case super->super
cppstruct_arg = struct_arg.StructArg("", self._transforms.types['supersuper'], "")
else:
cppstruct_arg = struct_arg.StructArg("", self._transforms.types['super'], "")
cppstruct_arg.member = struct_arg.StructArg("::", cppstruct_member.name, cppstruct_member.index, cppstruct_member.member)
else:
cppstruct_arg = cppstruct_member
assert cppstruct_arg, f"Could not transform cstruct_arg: [{cstruct_arg.as_string()}]"
# Extra processing required for grib_handle members that are referenced
if self._transforms.ctype_of(cstruct_arg.name) == "grib_handle*" or cstruct_arg.name == "grib_handle_of_accessor(a)":
if cstruct_member.name == "buffer":
debug.line("transform_cstruct_arg", f"buffer: cstruct_arg=[{cstruct_arg.as_string()}]")
cppstruct_arg = struct_arg.StructArg("", "buffer_", "" )
if cstruct_member.member and cstruct_member.member.name == "data":
cppstruct_arg.member = struct_arg.StructArg(".", "data()", cstruct_member.member.index)
for transform_func in transform_funcs:
cppstruct_arg = transform_func(cstruct_arg)
if cppstruct_arg:
return cppstruct_arg

View File

@ -70,6 +70,14 @@ class Transforms:
return None
# Helper to return the cpparg for the supplied cname, or None
def cpparg_for(self, cname):
for carg, cpparg in self.all_args.items():
if carg.name == cname:
return cpparg
return None
def add_local_args(self, carg, cpparg):
if carg in self._all_args:
assert self._all_args[carg] == cpparg, f"Updating an existing local arg transform: C Arg = {arg.arg_string(carg)} -> {arg.arg_string(cpparg)} Previous arg = {arg.arg_string(self._all_args[carg])}"

View File

@ -50,12 +50,12 @@ int Accessor::isMissing() const
return data_->isMissing();
}
bool Accessor::newBuffer(AccessorBuffer const& accBuffer)
bool Accessor::newBuffer(AccessorDataBuffer const& accBuffer)
{
return data_->newBuffer(accBuffer);
}
AccessorBuffer Accessor::currentBuffer() const
AccessorDataBuffer Accessor::currentBuffer() const
{
return data_->currentBuffer();
}
@ -71,5 +71,16 @@ std::vector<double> Accessor::unpackSubarray(std::size_t start) const
return data_->unpackSubarray(values, start) == GribStatus::SUCCESS ? values : std::vector<double>{};
}
// Conversion support
long Accessor::byteCount() const
{
return data_->byteCount();
}
long Accessor::byteOffset() const
{
return data_->byteOffset();
}
}

View File

@ -19,16 +19,22 @@ class Accessor
public:
Accessor(AccessorName const& name, AccessorNameSpace const& nameSpace, AccessorDataPtr data);
AccessorName name() const;
std::size_t stringLength() const;
long valueCount() const;
GribType nativeType() const;
double nearestSmallerValue(double val) const;
int compare(AccessorPtr const rhs) const;
int isMissing() const;
AccessorName name() const;
std::size_t stringLength() const;
long valueCount() const;
GribType nativeType() const;
double nearestSmallerValue(double val) const;
int compare(AccessorPtr const rhs) const;
int isMissing() const;
bool newBuffer(AccessorBuffer const& accBuffer);
AccessorBuffer currentBuffer() const;
bool newBuffer(AccessorDataBuffer const& accBuffer);
AccessorDataBuffer currentBuffer() const;
// Required to support C -> C++ Conversion - Start
long byteCount() const;
long byteOffset() const;
// Required to support C -> C++ Conversion - End
template<typename T>
GribStatus pack(T const& values);

View File

@ -15,9 +15,10 @@
namespace eccodes::accessor {
template<typename T>
class AccessorBuffer {
public:
using value_type = char;
using value_type = T;
using pointer = value_type*;
using const_pointer = value_type const*;
using reference = value_type&;
@ -30,9 +31,8 @@ public:
constexpr AccessorBuffer(AccessorBuffer const&) noexcept = default;
constexpr AccessorBuffer& operator=(AccessorBuffer const&) noexcept = default;
template<typename T>
constexpr AccessorBuffer(T *const buffer, const size_type num_elements) noexcept
: data_(reinterpret_cast<pointer>(buffer)), size_(sizeof(T) * num_elements) {}
constexpr AccessorBuffer(value_type *const buffer, const size_type num_elements) noexcept
: data_(reinterpret_cast<pointer>(buffer)), size_(sizeof(value_type) * num_elements) {}
[[nodiscard]] constexpr size_type size_bytes() const noexcept {
return size_;
@ -51,4 +51,7 @@ private:
size_type size_;
};
using AccessorDataBuffer = AccessorBuffer<unsigned char>;
using DataPointer = unsigned char*;
}

View File

@ -15,13 +15,13 @@ AccessorData::AccessorData(AccessorInitData const& initData)
AccessorData::~AccessorData() = default;
bool AccessorData::newBuffer(AccessorBuffer const& accBuffer)
bool AccessorData::newBuffer(AccessorDataBuffer const& accBuffer)
{
buffer_ = accBuffer;
return true;
}
AccessorBuffer AccessorData::currentBuffer() const
AccessorDataBuffer AccessorData::currentBuffer() const
{
return buffer_;
}

View File

@ -22,8 +22,8 @@ public:
AccessorData(AccessorInitData const& initData);
virtual ~AccessorData() = 0;
bool newBuffer(AccessorBuffer const& accBuffer);
AccessorBuffer currentBuffer() const;
bool newBuffer(AccessorDataBuffer const& accBuffer);
AccessorDataBuffer currentBuffer() const;
virtual void dump() const;
virtual std::size_t stringLength() const;
@ -76,7 +76,7 @@ public:
// Ideally these would be private, but that makes the conversion much harder so they are protected instead
// This will be revisited later...
protected:
AccessorBuffer buffer_{};
AccessorDataBuffer buffer_{};
long length_{};
long offset_{};
unsigned long flags_{};

View File

@ -75,7 +75,7 @@ GribBuffer parseGribFile(std::filesystem::path gribFile)
GribBuffer buffer(fileSize);
file.read(buffer.data(), fileSize);
file.read(reinterpret_cast<char*>(buffer.data()), fileSize);
return buffer;
}

View File

@ -7,7 +7,7 @@
namespace eccodes::accessor {
using GribBuffer = std::vector<char>;
using GribBuffer = std::vector<unsigned char>;
std::ostream& operator<<(std::ostream& os, GribBuffer const& buffer);

View File

@ -45,7 +45,7 @@ void createAccessors(LayoutEntries const& entries, GribBuffer& buffer)
{
auto accessor = AccessorFactory::instance().build(AccessorType(entry.type_), AccessorName(entry.name_), AccessorNameSpace(""), AccessorInitData{});
AccessorBuffer accBuffer{buffer.data() + offset, static_cast<std::size_t>(entry.byteCount_)};
AccessorDataBuffer accBuffer{buffer.data() + offset, static_cast<std::size_t>(entry.byteCount_)};
offset += entry.byteCount_;
accessor->newBuffer(accBuffer);