grib_unpack processing

This commit is contained in:
kevstone 2024-02-07 10:37:31 +00:00
parent 62f3edcd71
commit 7a575acd5a
7 changed files with 294 additions and 69 deletions

View File

@ -0,0 +1,206 @@
# Start the line with # to exclude the file
grib_accessor_class_ascii.cc
grib_accessor_class_bit.cc
grib_accessor_class_long.cc
grib_accessor_class_proj_string.cc
#grib_accessor_class_abstract_long_vector.cc
#grib_accessor_class_abstract_vector.cc
#grib_accessor_class_ascii.cc
#grib_accessor_class_bit.cc
#grib_accessor_class_bitmap.cc
#grib_accessor_class_bits.cc
#grib_accessor_class_bits_per_value.cc
#grib_accessor_class_blob.cc
#grib_accessor_class_budgdate.cc
#grib_accessor_class_bufr_data_array.cc
#grib_accessor_class_bufr_data_element.cc
#grib_accessor_class_bufr_elements_table.cc
#grib_accessor_class_bufr_extract_area_subsets.cc
#grib_accessor_class_bufr_extract_datetime_subsets.cc
#grib_accessor_class_bufr_extract_subsets.cc
#grib_accessor_class_bufr_group.cc
#grib_accessor_class_bufr_simple_thinning.cc
#grib_accessor_class_bufr_string_values.cc
#grib_accessor_class_bufrdc_expanded_descriptors.cc
#grib_accessor_class_bytes.cc
#grib_accessor_class_change_alternative_row_scanning.cc
#grib_accessor_class_change_scanning_direction.cc
#grib_accessor_class_check_internal_version.cc
#grib_accessor_class_closest_date.cc
#grib_accessor_class_codeflag.cc
#grib_accessor_class_codetable.cc
#grib_accessor_class_codetable_title.cc
#grib_accessor_class_codetable_units.cc
#grib_accessor_class_concept.cc
#grib_accessor_class_constant.cc
#grib_accessor_class_count_file.cc
#grib_accessor_class_count_missing.cc
#grib_accessor_class_count_total.cc
#grib_accessor_class_data_apply_bitmap.cc
#grib_accessor_class_data_apply_boustrophedonic.cc
#grib_accessor_class_data_apply_boustrophedonic_bitmap.cc
#grib_accessor_class_data_ccsds_packing.cc
#grib_accessor_class_data_complex_packing.cc
#grib_accessor_class_data_dummy_field.cc
#grib_accessor_class_data_g1complex_packing.cc
#grib_accessor_class_data_g1second_order_constant_width_packing.cc
#grib_accessor_class_data_g1second_order_general_extended_packing.cc
#grib_accessor_class_data_g1second_order_general_packing.cc
#grib_accessor_class_data_g1second_order_row_by_row_packing.cc
#grib_accessor_class_data_g1secondary_bitmap.cc
#grib_accessor_class_data_g1shsimple_packing.cc
#grib_accessor_class_data_g1simple_packing.cc
#grib_accessor_class_data_g22order_packing.cc
#grib_accessor_class_data_g2bifourier_packing.cc
#grib_accessor_class_data_g2complex_packing.cc
#grib_accessor_class_data_g2secondary_bitmap.cc
#grib_accessor_class_data_g2shsimple_packing.cc
#grib_accessor_class_data_g2simple_packing.cc
#grib_accessor_class_data_g2simple_packing_with_preprocessing.cc
#grib_accessor_class_data_jpeg2000_packing.cc
#grib_accessor_class_data_png_packing.cc
#grib_accessor_class_data_raw_packing.cc
#grib_accessor_class_data_run_length_packing.cc
#grib_accessor_class_data_secondary_bitmap.cc
#grib_accessor_class_data_sh_packed.cc
#grib_accessor_class_data_sh_unpacked.cc
#grib_accessor_class_data_shsimple_packing.cc
#grib_accessor_class_data_simple_packing.cc
#grib_accessor_class_decimal_precision.cc
#grib_accessor_class_dictionary.cc
#grib_accessor_class_dirty.cc
#grib_accessor_class_divdouble.cc
#grib_accessor_class_double.cc
#grib_accessor_class_element.cc
#grib_accessor_class_evaluate.cc
#grib_accessor_class_expanded_descriptors.cc
#grib_accessor_class_from_scale_factor_scaled_value.cc
#grib_accessor_class_g1_half_byte_codeflag.cc
#grib_accessor_class_g1_message_length.cc
#grib_accessor_class_g1_section4_length.cc
#grib_accessor_class_g1bitmap.cc
#grib_accessor_class_g1date.cc
#grib_accessor_class_g1day_of_the_year_date.cc
#grib_accessor_class_g1end_of_interval_monthly.cc
#grib_accessor_class_g1fcperiod.cc
#grib_accessor_class_g1forecastmonth.cc
#grib_accessor_class_g1monthlydate.cc
#grib_accessor_class_g1number_of_coded_values_sh_complex.cc
#grib_accessor_class_g1number_of_coded_values_sh_simple.cc
#grib_accessor_class_g1step_range.cc
#grib_accessor_class_g1verificationdate.cc
#grib_accessor_class_g2_aerosol.cc
#grib_accessor_class_g2_chemical.cc
#grib_accessor_class_g2_eps.cc
#grib_accessor_class_g2_mars_labeling.cc
#grib_accessor_class_g2bitmap.cc
#grib_accessor_class_g2bitmap_present.cc
#grib_accessor_class_g2date.cc
#grib_accessor_class_g2end_step.cc
#grib_accessor_class_g2grid.cc
#grib_accessor_class_g2latlon.cc
#grib_accessor_class_g2level.cc
#grib_accessor_class_g2lon.cc
#grib_accessor_class_g2step_range.cc
#grib_accessor_class_gaussian_grid_name.cc
#grib_accessor_class_gds_is_present.cc
#grib_accessor_class_gds_not_present_bitmap.cc
#grib_accessor_class_getenv.cc
#grib_accessor_class_global_gaussian.cc
#grib_accessor_class_group.cc
#grib_accessor_class_gts_header.cc
#grib_accessor_class_hash_array.cc
#grib_accessor_class_headers_only.cc
#grib_accessor_class_ibmfloat.cc
#grib_accessor_class_ieeefloat.cc
#grib_accessor_class_ifs_param.cc
#grib_accessor_class_iterator.cc
#grib_accessor_class_julian_date.cc
#grib_accessor_class_julian_day.cc
#grib_accessor_class_ksec1expver.cc
#grib_accessor_class_label.cc
#grib_accessor_class_latitudes.cc
#grib_accessor_class_latlon_increment.cc
#grib_accessor_class_latlonvalues.cc
#grib_accessor_class_library_version.cc
#grib_accessor_class_local_definition.cc
#grib_accessor_class_long.cc
#grib_accessor_class_long_vector.cc
#grib_accessor_class_longitudes.cc
#grib_accessor_class_lookup.cc
#grib_accessor_class_mars_param.cc
#grib_accessor_class_mars_step.cc
#grib_accessor_class_md5.cc
#grib_accessor_class_message.cc
#grib_accessor_class_message_copy.cc
#grib_accessor_class_nearest.cc
#grib_accessor_class_non_alpha.cc
#grib_accessor_class_number_of_coded_values.cc
#grib_accessor_class_number_of_points.cc
#grib_accessor_class_number_of_points_gaussian.cc
#grib_accessor_class_number_of_values.cc
#grib_accessor_class_number_of_values_data_raw_packing.cc
#grib_accessor_class_octahedral_gaussian.cc
#grib_accessor_class_octet_number.cc
#grib_accessor_class_offset_file.cc
#grib_accessor_class_offset_values.cc
#grib_accessor_class_pack_bufr_values.cc
#grib_accessor_class_packing_type.cc
#grib_accessor_class_pad.cc
#grib_accessor_class_padding.cc
#grib_accessor_class_padto.cc
#grib_accessor_class_padtoeven.cc
#grib_accessor_class_padtomultiple.cc
#grib_accessor_class_position.cc
#grib_accessor_class_proj_string.cc
#grib_accessor_class_raw.cc
#grib_accessor_class_rdbtime_guess_date.cc
#grib_accessor_class_reference_value_error.cc
#grib_accessor_class_round.cc
#grib_accessor_class_scale.cc
#grib_accessor_class_scale_values.cc
#grib_accessor_class_second_order_bits_per_value.cc
#grib_accessor_class_section.cc
#grib_accessor_class_section_length.cc
#grib_accessor_class_section_padding.cc
#grib_accessor_class_section_pointer.cc
#grib_accessor_class_select_step_template.cc
#grib_accessor_class_sexagesimal2decimal.cc
#grib_accessor_class_signed.cc
#grib_accessor_class_signed_bits.cc
#grib_accessor_class_simple_packing_error.cc
#grib_accessor_class_size.cc
#grib_accessor_class_smart_table.cc
#grib_accessor_class_smart_table_column.cc
#grib_accessor_class_spd.cc
#grib_accessor_class_spectral_truncation.cc
#grib_accessor_class_sprintf.cc
#grib_accessor_class_statistics.cc
#grib_accessor_class_statistics_spectral.cc
#grib_accessor_class_step_human_readable.cc
#grib_accessor_class_step_in_units.cc
#grib_accessor_class_sum.cc
#grib_accessor_class_suppressed.cc
#grib_accessor_class_time.cc
#grib_accessor_class_to_double.cc
#grib_accessor_class_to_integer.cc
#grib_accessor_class_to_string.cc
#grib_accessor_class_transient.cc
#grib_accessor_class_transient_darray.cc
#grib_accessor_class_trim.cc
#grib_accessor_class_uint16.cc
#grib_accessor_class_uint32.cc
#grib_accessor_class_uint32_little_endian.cc
#grib_accessor_class_uint64.cc
#grib_accessor_class_uint64_little_endian.cc
#grib_accessor_class_uint8.cc
#grib_accessor_class_unexpanded_descriptors.cc
#grib_accessor_class_unpack_bufr_values.cc
#grib_accessor_class_unsigned.cc
#grib_accessor_class_unsigned_bits.cc
#grib_accessor_class_validity_date.cc
#grib_accessor_class_validity_time.cc
#grib_accessor_class_values.cc
#grib_accessor_class_variable.cc
#grib_accessor_class_vector.cc
#grib_accessor_class_when.cc

View File

@ -24,6 +24,7 @@ import code_object.value_declaration_reference as value_declaration_reference
import code_object.variable_declaration as variable_declaration
import code_object.while_statement as while_statement
from code_object.code_interface import NONE_VALUE
from code_object_converter.conversion_funcs import as_commented_out_code
# Parse AstCode and create code interface objects: classes that implement the CodeInterface
#
@ -676,7 +677,9 @@ class AstParser:
debug.line("parse_CSTYLE_CAST_EXPR", f"*** IGNORING *** child spelling=[{child.spelling}] type=[{child.type.spelling}] kind=[{child.kind}]")
return None
cast_expression = "".join([t.spelling for t in node.get_tokens()])
debug.line("parse_CSTYLE_CAST_EXPR", f"Commenting out cast expression [{cast_expression}]")
return as_commented_out_code(cast_expression, "Removing unecessary cast")
def parse_ARRAY_SUBSCRIPT_EXPR(self, node):
# We expect two children: the variable name and the index

View File

@ -9,6 +9,27 @@ from code_object.array_access import ArrayAccess
from code_object.struct_member_access import StructMemberAccess
from code_object.value_declaration_reference import ValueDeclarationReference
# Try to extract a name value from the object, else return ""
def extract_name(cpp_obj):
assert isinstance(cpp_obj, CodeInterface), f"Expected CodeInterfacce, got [{type(cpp_obj).__name__}]"
if isinstance(cpp_obj, Literal):
return cpp_obj.value
elif isinstance(cpp_obj, Arg):
return cpp_obj.name
elif isinstance(cpp_obj, StructArg):
return cpp_obj.name
elif isinstance(cpp_obj, VariableDeclaration):
return cpp_obj.variable
elif isinstance(cpp_obj, ArrayAccess):
return cpp_obj.name
elif isinstance(cpp_obj, StructMemberAccess):
return cpp_obj.name
elif isinstance(cpp_obj, ValueDeclarationReference):
return cpp_obj.value
return ""
# If the code_object has an Arg representation, then this will be returned,
# otherwise None
@ -17,20 +38,9 @@ def to_cpparg(cpp_obj, conversion_data):
assert isinstance(cpp_obj, CodeInterface), f"Expected CodeInterfacce, got [{type(cpp_obj).__name__}]"
if isinstance(cpp_obj, Literal):
cpparg = conversion_data.cpparg_for_cppname(cpp_obj.value)
elif isinstance(cpp_obj, Arg):
cpparg = cpp_obj
elif isinstance(cpp_obj, StructArg):
cpparg = conversion_data.cpparg_for_cppname(cpp_obj.name)
elif isinstance(cpp_obj, VariableDeclaration):
cpparg = cpp_obj.variable
elif isinstance(cpp_obj, ArrayAccess):
cpparg = conversion_data.cpparg_for_cppname(cpp_obj.name)
elif isinstance(cpp_obj, StructMemberAccess):
cpparg = conversion_data.cpparg_for_cppname(cpp_obj.name)
elif isinstance(cpp_obj, ValueDeclarationReference):
cpparg = conversion_data.cpparg_for_cppname(cpp_obj.value)
cppname = extract_name(cpp_obj)
if cppname:
cpparg = conversion_data.cpparg_for_cppname(cppname)
assert cpparg is None or isinstance(cpparg, Arg), f"cpparg should be Arg, not [{type(cpparg).__name__}]"

View File

@ -2,7 +2,6 @@
import utils.debug as debug
from enum import Enum, auto
import code_object_converter.conversion_pack.conversion_data as conversion_data
import code_object.declaration_specifier as declaration_specifier
# Functions and data to help the conversion data object

View File

@ -25,14 +25,7 @@ class DefaultConversionValidation(conversion_validation.ConversionValidation):
debug.line("validate_function_call", f"cfunction_call=[{debug.as_debug_string(cfunction_call)}] cppfunction_call=[{debug.as_debug_string(cppfunction_call)}] mapping=[{mapping}]")
if mapping:
debug.line("validate_function_call", f"mapping.cfuncsig=[{debug.as_debug_string(mapping.cfuncsig)}] mapping.cppfuncsig=[{debug.as_debug_string(mapping.cppfuncsig)}]")
cpp_args = []
for arg_entry in mapping.cppfuncsig.args:
if arg_entry != NONE_VALUE:
# The size of cpp_args should be the same as the next valid index :-)
cpp_arg_entry = self.validate_function_call_arg(cppfunction_call.args[len(cpp_args)], arg_entry)
cpp_args.append(cpp_arg_entry)
return function_call.FunctionCall(cppfunction_call.name, cpp_args)
return self.validate_function_call_args(cppfunction_call, mapping.cppfuncsig)
if cfunction_call.name == "strcmp":
return binary_operation.BinaryOperation(cppfunction_call.args[0], "==", cppfunction_call.args[1])
@ -45,10 +38,29 @@ class DefaultConversionValidation(conversion_validation.ConversionValidation):
if cfunction_call.name == "strlen":
# Replace the function call with a StructMemberAccess representing the container.size() call...
return self._container_utils.create_cpp_container_length_arg(cppfunction_call.args[0].name)
cpparg = arg_utils.to_cpparg(cppfunction_call.args[0], self._conversion_data)
assert cpparg
return self._container_utils.create_cpp_container_length_arg(cpparg.name)
return cppfunction_call
# Confirm the args are correct for the target type
# Returns an updated cppfunction_call
#
# NOTE: This function assumes all NONE_VALUE args have already been removed from the C++ call!
def validate_function_call_args(self, cppfunction_call, target_cppfuncsig):
debug.line("validate_function_call_args", f"cppfunction_call=[{debug.as_debug_string(cppfunction_call)}] target_cppfuncsig=[{debug.as_debug_string(target_cppfuncsig)}]")
cpp_args = []
arg_index = 0
for arg_entry in target_cppfuncsig.args:
if arg_entry != NONE_VALUE:
cpp_arg_entry = self.validate_function_call_arg(cppfunction_call.args[arg_index], arg_entry)
cpp_args.append(cpp_arg_entry)
arg_index += 1
return function_call.FunctionCall(cppfunction_call.name, cpp_args)
# Check use of references when calling functions...
def validate_function_call_arg(self, calling_arg_value, target_arg):
@ -136,7 +148,7 @@ class DefaultConversionValidation(conversion_validation.ConversionValidation):
if cpparg and self._conversion_data.is_container_type(cpparg.decl_spec.type):
cppright_value = cppright.as_string()
if is_number(cppright_value) or not self.is_cppfunction_returning_container(cppright.name):
if is_number(cppright_value) or not self.is_cppfunction_returning_container(cppright):
cppleft.index = "[0]"
debug.line("validate_binary_operation", f"Assigning number to container, so updating it to access first element: cppleft=[{debug.as_debug_string(cppleft)}] cppright_value=[{cppright_value}]")
return binary_operation.BinaryOperation(cppleft, cppbinary_op, cppright)
@ -172,9 +184,11 @@ class DefaultConversionValidation(conversion_validation.ConversionValidation):
# Helper to determine how a container should receive the value returned by the function
# Returns True if the function returns e.g. std::vector<int> instead of int
# Override as required...
def is_cppfunction_returning_container(self, cppfuncname):
cppfuncsig = self._conversion_data.cppfuncsig_for_cppfuncname(cppfuncname)
if cppfuncsig and self._conversion_data.is_container_type(cppfuncsig.return_type.type):
return True
def is_cppfunction_returning_container(self, cppfunc_object):
cppname = arg_utils.extract_name(cppfunc_object)
if cppname:
cppfuncsig = self._conversion_data.cppfuncsig_for_cppfuncname(cppname)
if cppfuncsig and self._conversion_data.is_container_type(cppfuncsig.return_type.type):
return True
return False

View File

@ -14,7 +14,7 @@ import code_object.constructor_function as constructor_function
import code_object.function_call as function_call
import code_object_converter.conversion_pack.arg_utils as arg_utils
from grib_accessor.grib_accessor_conversion_pack.grib_accessor_special_function_call_conversion import apply_special_function_call_conversions
from grib_accessor.grib_accessor_conversion_pack.grib_accessor_special_function_call_conversion import special_function_name_mapping
from code_object.code_interface import NONE_VALUE
import grib_accessor.grib_accessor_conversion_pack.grib_accessor_type_info as grib_accessor_type_info
from code_object_converter.conversion_funcs import as_commented_out_code
@ -60,7 +60,7 @@ class GribAccessorConversionValidation(default_conversion_validation.DefaultConv
def validate_function_call(self, cfunction_call, cppfunction_call):
debug.line("validate_function_call", f"cfunction_call=[{debug.as_debug_string(cfunction_call)}] cppfunction_call=[{debug.as_debug_string(cppfunction_call)}]")
special_function_call = apply_special_function_call_conversions(cfunction_call, cppfunction_call)
special_function_call = self.apply_special_function_call_conversions(cfunction_call, cppfunction_call)
if special_function_call:
return special_function_call
@ -76,6 +76,36 @@ class GribAccessorConversionValidation(default_conversion_validation.DefaultConv
return super().validate_function_call_arg(calling_arg_value, target_arg)
def apply_special_function_call_conversions(self, cfunction_call, cppfunction_call):
if cfunction_call.name == "grib_arguments_get_name":
arg_entry = literal.Literal(f"initData.args[{cfunction_call.args[2].as_string()}].second")
return function_call.FunctionCall(f"std::get<std::string>", [arg_entry])
if cfunction_call.name == "grib_arguments_get_long":
arg_entry = literal.Literal(f"initData.args[{cfunction_call.args[2].as_string()}].second")
return function_call.FunctionCall(f"std::get<long>", [arg_entry])
# If we're calling gribPackXXX or gribUnpackXXX and the first argument is "a", then we're actually calling ourself!
if cfunction_call.name.startswith("grib_pack") or cfunction_call.name.startswith("grib_unpack"):
if len(cppfunction_call.args) > 0 and cppfunction_call.args[0].as_string() == "a":
updated_cfuncname = cfunction_call.name[5:]
mapping = self._conversion_data.funcsig_mapping_for_cfuncname(updated_cfuncname)
if mapping:
updated_cppfunction_call = function_call.FunctionCall(mapping.cppfuncsig.name, cppfunction_call.args[1:])
updated_cppfunction_call = self.validate_function_call_args(updated_cppfunction_call, mapping.cppfuncsig)
debug.line("apply_special_function_call_conversions", f"Updated C++ function call=[{debug.as_debug_string(cppfunction_call)}] to [{debug.as_debug_string(updated_cppfunction_call)}]")
return updated_cppfunction_call
for cfuncname, cppfuncname in special_function_name_mapping.items():
if cfunction_call.name == cfuncname:
if cppfuncname:
return function_call.FunctionCall(cppfuncname, cppfunction_call.args)
else:
return as_commented_out_code(cfunction_call, f"Removed call to {cfuncname}")
return None
def validate_variable_declaration(self, cvariable_declaration, cppvariable_declaration):
if "GribStatus" in cppvariable_declaration.variable.as_string() and \
not isinstance(cppvariable_declaration.value, function_call.FunctionCall):

View File

@ -1,10 +1,4 @@
import utils.debug as debug
import code_object.literal as literal
import code_object.function_call as function_call
from code_object_converter.conversion_funcs import as_commented_out_code
from utils.standard_transforms import transform_variable_name
special_function_name_mapping = {
"snprintf" : "fmtString",
"strtol" : "strToLong",
@ -20,34 +14,3 @@ special_function_name_mapping = {
"grib_context_free_persistent" : "",
"grib_context_buffer_free" : "",
}
def apply_special_function_call_conversions(cfunction_call, cppfunction_call):
if cfunction_call.name == "grib_arguments_get_name":
arg_entry = literal.Literal(f"initData.args[{cfunction_call.args[2].as_string()}].second")
return function_call.FunctionCall(f"std::get<std::string>", [arg_entry])
if cfunction_call.name == "grib_arguments_get_long":
arg_entry = literal.Literal(f"initData.args[{cfunction_call.args[2].as_string()}].second")
return function_call.FunctionCall(f"std::get<long>", [arg_entry])
# If we're calling gribPackXXX or gribUnpackXXX and the first argument is "a", then we're actually calling ourself!
cppfuncname = ""
if cppfunction_call.name.startswith("gribPack"):
cppfuncname = "pack"
elif cppfunction_call.name.startswith("gribUnpack"):
cppfuncname = "unpack"
if cppfuncname and len(cppfunction_call.args) > 0 and cppfunction_call.args[0].as_string() == "a":
debug.line("apply_special_function_call_conversions", f"Updated function call=[{cppfunction_call.name}] to [{cppfuncname}]")
return function_call.FunctionCall(cppfuncname, cppfunction_call.args[1:])
for cfuncname, cppfuncname in special_function_name_mapping.items():
if cfunction_call.name == cfuncname:
if cppfuncname:
return function_call.FunctionCall(cppfuncname, cppfunction_call.args)
else:
return as_commented_out_code(cfunction_call, f"Removed call to {cfuncname}")
return None