diff --git a/reccmp/cvdump/types.py b/reccmp/cvdump/types.py index 3a182f3ac..b7d1d5fc3 100644 --- a/reccmp/cvdump/types.py +++ b/reccmp/cvdump/types.py @@ -198,7 +198,7 @@ class CvdumpTypesParser: ) LF_FIELDLIST_ENUMERATE = re.compile( - r"list\[\d+\] = LF_ENUMERATE,.*value = (?P\d+), name = '(?P[^']+)'" + r"list\[\d+\] = LF_ENUMERATE,.*value = (?:\([\w_]*\)\s)?(?P-?\d+)(?:\([\w_]*\))?, name = '(?P[^']+)'" ) LF_ARRAY_RE = re.compile( @@ -356,8 +356,17 @@ def get(self, type_key: CvdumpTypeKey) -> TypeInfo: if obj is None: raise CvdumpKeyError(type_key) - if obj.get("type") == "LF_POINTER": - return self.get(CVInfoTypeEnum.T_32PVOID) + obj_type = obj.get("type") + + if obj_type == "LF_POINTER": + # FIXME: This introduces regressions on datacmp and also fails some unit tests. + # Need to reconsider if there is a better approach for preserving full type information. + # It may make the most sense to adapt our domain types to our needs. + + element_type_key = obj.get("element_type") + assert element_type_key is not None + pointee_type = self.get(element_type_key) + return TypeInfo(key=type_key, size=4, name=f"{pointee_type.name} *") if obj.get("is_forward_ref", False): # Get the forward reference to follow. @@ -370,7 +379,7 @@ def get(self, type_key: CvdumpTypeKey) -> TypeInfo: return self.get(forward_ref) # These type references are just a wrapper around a scalar - if obj.get("type") == "LF_ENUM": + if obj_type == "LF_ENUM": underlying_type = obj.get("underlying_type") if underlying_type is None: raise CvdumpKeyError(f"Missing 'underlying_type' in {obj}") @@ -378,10 +387,13 @@ def get(self, type_key: CvdumpTypeKey) -> TypeInfo: return self.get(underlying_type) # Else it is not a forward reference, so build out the object here. - if obj.get("type") == "LF_ARRAY": + if obj_type == "LF_ARRAY": members = self._mock_array_members(obj) - else: + elif obj_type in ("LF_CLASS", "LF_STRUCTURE", "LF_UNION", "LF_FIELDLIST"): members = self._get_field_list(obj) + else: + # e.g. obj_type == "LF_PROCEDURE" + members = None return TypeInfo( key=type_key, diff --git a/reccmp/ghidra/importer/ghidra_helper.py b/reccmp/ghidra/importer/ghidra_helper.py index c4d4a1144..035d87309 100644 --- a/reccmp/ghidra/importer/ghidra_helper.py +++ b/reccmp/ghidra/importer/ghidra_helper.py @@ -148,7 +148,7 @@ def set_ghidra_label(api: FlatProgramAPI, address: int, label_with_namespace: st and existing_label_name == name ): logger.debug( - "Label '%s' at 0x%s already exists", label_with_namespace, address_hex + "Label '%s' at %s already exists", label_with_namespace, address_hex ) else: logger.debug( diff --git a/reccmp/ghidra/importer/importer.py b/reccmp/ghidra/importer/importer.py index 278e9914d..aed4320c5 100644 --- a/reccmp/ghidra/importer/importer.py +++ b/reccmp/ghidra/importer/importer.py @@ -91,10 +91,6 @@ def _do_execute_import( ): pdb_functions = extraction.get_function_list() - if api is None: - logger.info("Completed the dry run outside Ghidra.") - return - # pylint: disable=possibly-used-before-assignment type_importer = PdbTypeImporter(api, extraction, ignore_types=ignore_types) diff --git a/reccmp/ghidra/importer/pdb_extraction.py b/reccmp/ghidra/importer/pdb_extraction.py index 2e3a91aeb..28c870c84 100644 --- a/reccmp/ghidra/importer/pdb_extraction.py +++ b/reccmp/ghidra/importer/pdb_extraction.py @@ -31,7 +31,6 @@ class CppRegisterSymbol(CppStackOrRegisterSymbol): @dataclass class FunctionSignature: - original_function_symbol: SymbolsEntry call_type: str arglist: list[CvdumpTypeKey] return_type: CvdumpTypeKey @@ -119,7 +118,6 @@ def get_func_signature(self, fn: SymbolsEntry) -> FunctionSignature | None: this_adjust = function_type.get("this_adjust", 0) return FunctionSignature( - original_function_symbol=fn, call_type=call_type, arglist=arg_list_pdb_types, return_type=function_type["return_type"], diff --git a/reccmp/ghidra/importer/type_importer.py b/reccmp/ghidra/importer/type_importer.py index e37323c87..c1d538edb 100644 --- a/reccmp/ghidra/importer/type_importer.py +++ b/reccmp/ghidra/importer/type_importer.py @@ -20,7 +20,6 @@ TypedefDataType, ComponentOffsetSettingsDefinition, ) -from ghidra.util.task import ConsoleTaskMonitor from reccmp.cvdump.types import ( CvdumpParsedType, @@ -85,7 +84,7 @@ def import_pdb_type_into_ghidra( """ Recursively imports a type from the PDB into Ghidra. @param type_index Either a scalar type like `T_INT4(...)` or a PDB reference like `0x10ba` - @param slim_for_vbase If true, the current invocation + @param slim_for_vbase If `True`, the current invocation imports a superclass of some class where virtual inheritance is involved (directly or indirectly). This case requires special handling: Let's say we have `class C: B` and `class B: virtual A`. Then cvdump reports a size for B that includes both B's fields as well as the A contained at an offset within B, @@ -137,7 +136,11 @@ def _import_scalar_type(self, type_key: CvdumpTypeKey) -> DataType: cvtype = CvdumpTypeMap[type_key] if cvtype.pointer is None: - return get_scalar_ghidra_type(type_key) + # Scalars need to be added to the database explicitly since there can be multiple + # non-identical instances of the same scalar. See the failing unit tests if you remove the wrapper. + return add_data_type_or_reuse_existing( + self.api, get_scalar_ghidra_type(type_key) + ) points_to = get_scalar_ghidra_type(cvtype.pointer) return get_or_add_pointer_type(self.api, points_to) @@ -564,9 +567,7 @@ def _delete_and_recreate_struct_data_type( category_path = category_path_of(sanitized_name.namespace_path) assert ( - self.api.getCurrentProgram() - .getDataTypeManager() - .remove(existing_data_type, ConsoleTaskMonitor()) + self.api.getCurrentProgram().getDataTypeManager().remove(existing_data_type) ), f"Failed to delete and re-create data type {sanitized_name}" data_type: DataType = StructureDataType( category_path, str(sanitized_name), class_size diff --git a/tests/conftest.py b/tests/conftest.py index a9805a5a9..5302f2f9e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,11 @@ from reccmp.formats import NEImage, PEImage, detect_image from .binfiles_test_setup import BINFILE_ISLE, BINFILE_LEGO1, BINFILE_SKI, TestBinfile -from .ghidra_integration_test_setup import ghidra_integration_test_program +from .ghidra_integration_test_setup import ( + GhidraFunctionTestHelper, + GhidraTypeTestHelper, + ghidra_integration_test_program, +) # Suppress linter warnings related to the fact that the header support for Ghidra is limited # and that we cannot import Ghidra classes before Ghidra has been loaded @@ -19,6 +23,7 @@ if TYPE_CHECKING: from ghidra.program.flatapi import FlatProgramAPI from ghidra.program.model.listing import Program + from reccmp.ghidra.importer.type_importer import PdbTypeImporter REQUIRE_BINFILES_OPTION = "--require-binfiles" @@ -143,3 +148,17 @@ def fixture_ghidra_program(ghidra_program: "Program") -> "Iterator[FlatProgramAP finally: # Revert all side effects of the test that just ran transaction.abort() + + +@pytest.fixture(name="type_helper", scope="function") +def ghidra_type_helper_fixture( + ghidra: "FlatProgramAPI", +) -> Iterator[GhidraTypeTestHelper]: + yield GhidraTypeTestHelper(ghidra) + + +@pytest.fixture(name="function_helper", scope="function") +def ghidra_function_helper_fixture( + ghidra: "FlatProgramAPI", +) -> Iterator[GhidraFunctionTestHelper]: + yield GhidraFunctionTestHelper(ghidra) diff --git a/tests/ghidra_integration_test_setup.py b/tests/ghidra_integration_test_setup.py index 1c3f3733e..a29f8098a 100644 --- a/tests/ghidra_integration_test_setup.py +++ b/tests/ghidra_integration_test_setup.py @@ -1,11 +1,17 @@ from contextlib import contextmanager from pathlib import Path from typing import TYPE_CHECKING, Iterator, cast - - +from unittest.mock import Mock import pytest + from pyghidra import HeadlessPyGhidraLauncher # type: ignore[import-untyped] +from reccmp.compare.core import Compare +from reccmp.compare.ingest import load_cvdump_types +from reccmp.cvdump.analysis import CvdumpAnalysis +from reccmp.cvdump.parser import CvdumpParser +from .raw_image import RawImage + # Suppress linter warnings related to the fact that the header support for Ghidra is limited # and that we cannot import Ghidra classes before Ghidra has been loaded @@ -14,7 +20,8 @@ if TYPE_CHECKING: from ghidra.program.model.listing import Program - + from ghidra.program.flatapi import FlatProgramAPI + from reccmp.ghidra.importer.type_importer import PdbTypeImporter GHIDRA_PROJECT_NAME = "ghidra-integration-test" GHIDRA_FOLDER_NAME = "/" @@ -41,6 +48,17 @@ def ghidra_integration_test_program( ) -> "Iterator[Program]": assert request.config.cache is not None + try: + # Gets rid of spurious stack traces from a misunderstanding between pytest and jpype. + # See https://jpype.readthedocs.io/en/latest/userguide.html#errors-reported-by-python-fault-handler + import faulthandler + + faulthandler.enable() + faulthandler.disable() + # pylint: disable-next=broad-exception-caught # This is fine to fail, we don't need to handle it + except Exception: + pass + HeadlessPyGhidraLauncher().start() # pylint: disable-next=import-error @@ -126,3 +144,75 @@ def ghidra_integration_test_program( finally: ghidra_project.getProject().close() + + +class GhidraTypeTestHelper: + def __init__(self, ghidra: "FlatProgramAPI"): + from reccmp.ghidra.importer.type_importer import ( + PdbTypeImporter, + PdbFunctionExtractor, + ) + + self.ghidra = ghidra + self.compare = Compare( + RawImage.from_memory(), RawImage.from_memory(), Mock(), "TEST" + ) + self.type_importer = PdbTypeImporter( + ghidra, PdbFunctionExtractor(self.compare), set() + ) + + def set_up_cvdump_types(self, cvdump_types: str): + parser = CvdumpParser() + parser.read_section("TYPES", cvdump_types) + analysis = CvdumpAnalysis(parser) + load_cvdump_types(analysis, self.compare.types) + + +class GhidraFunctionTestHelper: + ORIG_FN_TO_OVERWRITE_PRIMARY = 0x00402880 # readIntFromRegistry() + ORIG_FN_TO_OVERWRITE_SECONDARY = 0x00402C20 # IsleApp::Tick() + ORIG_DATA_TO_OVERWRITE = 0x00410040 # g_windowRect + + def __init__(self, ghidra: "FlatProgramAPI"): + self.ghidra = ghidra + self.orig_address = self.ORIG_FN_TO_OVERWRITE_PRIMARY + self.address_ghidra = ghidra.getAddressFactory().getAddress( + hex(self.orig_address) + ) + self.ghidra_function = self.ghidra.getFunctionContaining(self.address_ghidra) + assert ( + self.ghidra_function is not None + ), f"No Ghidra function at address {self.address_ghidra}" + + def overwrite_example_function(self, data: bytes): + from jpype import JArray, JByte # type: ignore[import-untyped] + + assert len(data) > 0 + + # Clear the existing decompiled code so we can overwrite it + listing = self.ghidra.getCurrentProgram().getListing() + end_addr = self.address_ghidra.add(len(data) - 1) + listing.clearCodeUnits(self.address_ghidra, end_addr, False) + + # Overwrite the memory + self.ghidra.getCurrentProgram().getMemory().setBytes( + self.address_ghidra, JArray.of(data, JByte) + ) + + def assert_c_code(self, code: str): + from ghidra.app.decompiler import DecompInterface + from ghidra.util.task import TaskMonitor + + iface = DecompInterface() + iface.openProgram(self.ghidra.getCurrentProgram()) + + res = iface.decompileFunction(self.ghidra_function, 5, TaskMonitor.DUMMY) + assert res.decompileCompleted(), "Decompilation failed" + + # The line endings returned by the API differ between Windows and Linux + decompiled_c_code = res.getDecompiledFunction().getC().replace("\r\n", "\n") + + # This print statement is suppressed when the test passes, but is helpful if the test fails + print(decompiled_c_code) + + assert decompiled_c_code == code diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 000000000..a1060728c --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,9 @@ +from typing import TypeVar + +T = TypeVar("T") + + +def assert_instance(value: object, expected_class: type[T]) -> T: + """Type narrowing does not work well in the IDE for some reason, this makes it explicit""" + assert isinstance(value, expected_class) + return value diff --git a/tests/test_cvdump_types.py b/tests/test_cvdump_types.py index 652ce1d88..558dd44b4 100644 --- a/tests/test_cvdump_types.py +++ b/tests/test_cvdump_types.py @@ -1018,7 +1018,7 @@ def test_this_adjust_hex(empty_parser: CvdumpTypesParser): # codespell:ignore-begin empty_parser.read_all("""\ 0x657a : Length = 26, Leaf = 0x1009 LF_MFUNCTION - Return type = T_VOID(0003), Class type = 0x15ED, This type = 0x15EE, + Return type = T_VOID(0003), Class type = 0x15ED, This type = 0x15EE, Call type = ThisCall, Func attr = none Parms = 3, Arg list type = 0x6579, This adjust = 24""") # codespell:ignore-end @@ -1028,7 +1028,7 @@ def test_this_adjust_hex(empty_parser: CvdumpTypesParser): BIG_TYPE_KEY_SAMPLE = """ 0x1023 : Length = 70, Leaf = 0x1505 LF_STRUCTURE - # members = 0, field list type 0x0000, FORWARD REF, + # members = 0, field list type 0x0000, FORWARD REF, Derivation list type 0x0000, VT shape type 0x0000 Size = 0, class name = threadlocaleinfostruct, unique name = Uthreadlocaleinfostruct@@, UDT(0x00011738) @@ -1036,14 +1036,14 @@ def test_this_adjust_hex(empty_parser: CvdumpTypesParser): Element type = 0x00011735 Index type = T_ULONG(0022) length = 96 - Name = + Name = 0x00011737 : Length = 418, Leaf = 0x1203 LF_FIELDLIST list[0] = LF_MEMBER, public, type = 0x00011736, offset = 72 member name = 'lc_category' 0x00011738 : Length = 70, Leaf = 0x1505 LF_STRUCTURE - # members = 18, field list type 0x11737, + # members = 18, field list type 0x11737, Derivation list type 0x0000, VT shape type 0x0000 Size = 216, class name = threadlocaleinfostruct, unique name = Uthreadlocaleinfostruct@@, UDT(0x00011738) """ @@ -1074,6 +1074,39 @@ def test_enum_keys_over_ffff(empty_parser: CvdumpTypesParser): assert empty_parser.keys[TK(0x105DA)]["name"] == "e_STATE_t" +ENUM_WITH_DATA_TYPES_SAMPLES = """ +0x4e63 : Length = 102, Leaf = 0x1203 LF_FIELDLIST + list[0] = LF_ENUMERATE, public, value = 1, name = 'c_act1' + list[1] = LF_ENUMERATE, public, value = 2, name = 'c_imain' + list[2] = LF_ENUMERATE, public, value = 16, name = 'c_ielev' + list[3] = LF_ENUMERATE, public, value = 32, name = 'c_iisle' + list[4] = LF_ENUMERATE, public, value = (LF_USHORT) 32768, name = 'c_act2' + list[5] = LF_ENUMERATE, public, value = (LF_ULONG) 65536, name = 'c_act3' + +0x5695 : Length = 50, Leaf = 0x1203 LF_FIELDLIST + list[0] = LF_ENUMERATE, public, value = (LF_CHAR) -1(0xFF), name = 'c_unknownminusone' + list[1] = LF_ENUMERATE, public, value = 8, name = 'c_unknown8' +""" + + +def test_enum_with_data_types(empty_parser: CvdumpTypesParser): + """Make sure we can read an LF_FIELDLIST of an enum with a negative value (GH #380)""" + empty_parser.read_all(ENUM_WITH_DATA_TYPES_SAMPLES) + assert empty_parser.keys[TK(0x4E63)]["variants"] == [ + EnumItem(name="c_act1", value=1), + EnumItem(name="c_imain", value=2), + EnumItem(name="c_ielev", value=16), + EnumItem(name="c_iisle", value=32), + EnumItem(name="c_act2", value=32768), + EnumItem(name="c_act3", value=65536), + ] + + assert empty_parser.keys[TK(0x5695)]["variants"] == [ + EnumItem(name="c_unknownminusone", value=-1), + EnumItem(name="c_unknown8", value=8), + ] + + ARRAY_WITH_UNKNOWN_ELEMENT = """ 0x1000 : Length = 14, Leaf = 0x1503 LF_ARRAY Element type = ???(0555) diff --git a/tests/test_ghidra_integration.py b/tests/test_ghidra_integration.py deleted file mode 100644 index 3f7d03c6a..000000000 --- a/tests/test_ghidra_integration.py +++ /dev/null @@ -1,87 +0,0 @@ -# Suppress linter warnings related to the fact that the header support for Ghidra is limited -# and that we cannot import Ghidra classes before Ghidra has been loaded - -# pylint: disable=import-outside-toplevel -# pyright: reportMissingModuleSource=false - -from typing import TYPE_CHECKING -from unittest.mock import Mock - -import pytest -from reccmp.compare.core import Compare -from reccmp.cvdump.cvinfo import CVInfoTypeEnum, CvdumpTypeKey, CvdumpTypeMap -from reccmp.cvdump.types import CvdumpParsedType, FieldListItem -from reccmp.ghidra.importer.exceptions import TypeNotFoundError -from reccmp.ghidra.importer.pdb_extraction import PdbFunctionExtractor -from tests.test_image_raw import RawImage - -if TYPE_CHECKING: - from ghidra.program.flatapi import FlatProgramAPI - -verified_types = ( - t - for t in CVInfoTypeEnum - if CvdumpTypeMap[t].verified and t != CVInfoTypeEnum.T_NOTYPE -) - - -@pytest.mark.parametrize("scalar_type", verified_types) -def test_ghidra_scalar_types(ghidra, scalar_type): - from reccmp.ghidra.importer.type_importer import PdbTypeImporter - from ghidra.program.model.data import Pointer - - type_importer = PdbTypeImporter(ghidra, Mock(), set()) - - cv_type_info = CvdumpTypeMap[scalar_type] - - ghidra_type = type_importer.import_pdb_type_into_ghidra(scalar_type) - assert ghidra_type.length == cv_type_info.size - - if cv_type_info.pointer is not None: - assert isinstance(ghidra_type, Pointer) - - -def test_ghidra_type_not_found(ghidra: "FlatProgramAPI"): - from reccmp.ghidra.importer.type_importer import PdbTypeImporter - - compare = Compare(RawImage.from_memory(), RawImage.from_memory(), Mock(), "TEST") - type_importer = PdbTypeImporter(ghidra, PdbFunctionExtractor(compare), set()) - - with pytest.raises(TypeNotFoundError, match="Failed to find referenced type"): - type_importer.import_pdb_type_into_ghidra(CvdumpTypeKey.from_str("0x1001")) - - -def test_ghidra_type_class(ghidra: "FlatProgramAPI"): - from reccmp.ghidra.importer.type_importer import PdbTypeImporter - from ghidra.program.model.data import Structure - - field_list_key = CvdumpTypeKey.from_str("0x1001") - class_key = CvdumpTypeKey.from_str("0x1002") - compare = Compare(RawImage.from_memory(), RawImage.from_memory(), Mock(), "TEST") - compare.types.keys[field_list_key] = CvdumpParsedType( - type="LF_FIELDLIST", - members=[ - FieldListItem(offset=0, name="id", type=CVInfoTypeEnum.T_INT4), - FieldListItem(offset=4, name="name", type=CVInfoTypeEnum.T_32PCHAR), - ], - ) - compare.types.keys[class_key] = CvdumpParsedType( - type="LF_CLASS", name="TestClass", field_list_type=field_list_key, size=8 - ) - type_importer = PdbTypeImporter(ghidra, PdbFunctionExtractor(compare), set()) - - x = type_importer.import_pdb_type_into_ghidra(class_key) - - assert isinstance(x, Structure) - assert x.length == 8 - - [id_component, name_component] = list(x.getComponents()) - assert id_component.getOffset() == 0 - assert id_component.getDataType().name == "int" - assert name_component.getOffset() == 4 - assert name_component.getDataType().name == "char *" - - -def test_ghidra_verify_test_isolation(ghidra: "FlatProgramAPI"): - """Make sure that the `TestClass` created above was rolled back.""" - assert not list(ghidra.getDataTypes("TestClass")) diff --git a/tests/test_ghidra_integration_function_imports.py b/tests/test_ghidra_integration_function_imports.py new file mode 100644 index 000000000..a6cb69692 --- /dev/null +++ b/tests/test_ghidra_integration_function_imports.py @@ -0,0 +1,262 @@ +# Suppress linter warnings related to the fact that the header support for Ghidra is limited +# and that we cannot import Ghidra classes before Ghidra has been loaded + +# pylint: disable=import-outside-toplevel +# pyright: reportMissingModuleSource=false + +import json +from typing import TYPE_CHECKING + +from reccmp.cvdump.analysis import CvdumpNode +from reccmp.cvdump.cvinfo import CVInfoTypeEnum, CvdumpTypeKey +from reccmp.compare.db import ReccmpMatch +from reccmp.cvdump.types import TypeInfo +from reccmp.ghidra.importer.pdb_extraction import ( + CppRegisterSymbol, + CppStackSymbol, + FunctionSignature, +) +from .ghidra_integration_test_setup import ( + GhidraFunctionTestHelper, + GhidraTypeTestHelper, +) + +if TYPE_CHECKING: + from ghidra.program.flatapi import FlatProgramAPI + + +def test_import_trivial_function( + ghidra: "FlatProgramAPI", + function_helper: GhidraFunctionTestHelper, + type_helper: GhidraTypeTestHelper, +): + from reccmp.ghidra.importer.function_importer import ( + PdbFunctionImporter, + PdbFunction, + ) + + function_helper.overwrite_example_function(b"\xc3") + + func_signature = FunctionSignature( + call_type="__stdcall", + arglist=[], + return_type=CVInfoTypeEnum.T_VOID, + class_type=None, + stack_symbols=[], + this_adjust=0, + ) + pdb_function = PdbFunction( + ReccmpMatch( + function_helper.orig_address, 1234, json.dumps({"name": "MyTestFn"}) + ), + func_signature, + is_stub=False, + ) + + PdbFunctionImporter.build( + ghidra, pdb_function, type_helper.type_importer, [] + ).overwrite_ghidra_function(function_helper.ghidra_function) + + function_helper.assert_c_code(""" +void MyTestFn(void) + +{ + return; +} + +""") + + +def test_record_array_access( + ghidra: "FlatProgramAPI", + function_helper: GhidraFunctionTestHelper, + type_helper: GhidraTypeTestHelper, +): + from reccmp.ghidra.importer.function_importer import ( + PdbFunctionImporter, + PdbFunction, + ) + + # Shortened version of a BETA10 recompilation + cvdump_types = """ +0x12c8 : Length = 42, Leaf = 0x1505 LF_STRUCTURE + # members = 0, field list type 0x0000, FORWARD REF, + Derivation list type 0x0000, VT shape type 0x0000 + Size = 0, class name = LegoAnimActorEntry, UDT(0x00006081) + +0x12c9 : Length = 10, Leaf = 0x1002 LF_POINTER + Pointer (NEAR32), Size: 0 + Element type : 0x12C8 + +0x12cd : Length = 314, Leaf = 0x1203 LF_FIELDLIST + list[10] = LF_MEMBER, protected, type = T_LONG(0012), offset = 8 + member name = 'm_duration' + list[11] = LF_MEMBER, protected, type = 0x12C9, offset = 12 + member name = 'm_modelList' + list[12] = LF_MEMBER, protected, type = T_ULONG(0022), offset = 16 + member name = 'm_numActors' + +0x12cf : Length = 30, Leaf = 0x1504 LF_CLASS + # members = 15, field list type 0x12cd, CONSTRUCTOR, + Derivation list type 0x0000, VT shape type 0x12ce + Size = 24, class name = LegoAnim, UDT(0x000012cf) + +0x6080 : Length = 62, Leaf = 0x1203 LF_FIELDLIST + list[1] = LF_MEMBER, public, type = T_32PRCHAR(0470), offset = 0 + member name = 'm_name' + list[2] = LF_MEMBER, public, type = T_ULONG(0022), offset = 4 + member name = 'm_type' + +0x6081 : Length = 42, Leaf = 0x1505 LF_STRUCTURE + # members = 3, field list type 0x6080, + Derivation list type 0x0000, VT shape type 0x0000 + Size = 8, class name = LegoAnimActorEntry, UDT(0x00006081) + """ + legoanim_class_key = CvdumpTypeKey(0x12CF) + + type_helper.set_up_cvdump_types(cvdump_types) + + # shortened version of LEGO1 0x100a0f20 + function_helper.overwrite_example_function( + b"\x8b\x54\x24\x04" # MOV EDX, dword ptr [ESP + p_index] + b"\x8b\x41\x0c" # MOV EAX, dword ptr [ECX + this->m_modelList] + b"\x8b\x04\xd0" # MOV EAX, dword ptr [EAX + EDX*0x8] + b"\xc2\x04\x00" # RET 0x4 + ) + + func_signature = FunctionSignature( + call_type="__thiscall", + arglist=[CVInfoTypeEnum.T_INT4], + return_type=CVInfoTypeEnum.T_32PCHAR, + class_type=legoanim_class_key, + stack_symbols=[ + CppRegisterSymbol("this", legoanim_class_key, "ecx"), + CppStackSymbol("p_index", CVInfoTypeEnum.T_INT4, 4), + ], + this_adjust=0, + ) + pdb_function = PdbFunction( + ReccmpMatch( + function_helper.orig_address, + 1234, # arbitrary + json.dumps({"name": "LegoAnim::GetActorName"}), + ), + func_signature, + is_stub=False, + ) + + PdbFunctionImporter.build( + ghidra, pdb_function, type_helper.type_importer, [] + ).overwrite_ghidra_function(function_helper.ghidra_function) + + function_helper.assert_c_code(""" +char * __thiscall LegoAnim::GetActorName(LegoAnim *this,int p_index) + +{ + return this->m_modelList[p_index].m_name; +} + +""") + + +def test_global_array_access( + ghidra: "FlatProgramAPI", + function_helper: GhidraFunctionTestHelper, + type_helper: GhidraTypeTestHelper, +): + from reccmp.ghidra.importer.function_importer import ( + PdbFunctionImporter, + PdbFunction, + ) + from reccmp.ghidra.importer.globals_importer import import_global_into_ghidra + + # based on a BETA10 recompilation + cvdump_types = """ +0x1002 : Length = 34, Leaf = 0x1505 LF_STRUCTURE + # members = 0, field list type 0x0000, FORWARD REF, + Derivation list type 0x0000, VT shape type 0x0000 + Size = 0, class name = LegoActorInfo, UDT(0x000056f5) + +0x1003 : Length = 14, Leaf = 0x1503 LF_ARRAY + Element type = 0x1002 + Index type = T_SHORT(0011) + length = 264 + Name = + +0x56f4 : Length = 154, Leaf = 0x1203 LF_FIELDLIST + list[1] = LF_MEMBER, public, type = T_32PRCHAR(0470), offset = 0 + member name = 'm_name' + +0x56f5 : Length = 34, Leaf = 0x1505 LF_STRUCTURE + # members = 8, field list type 0x56f4, + Derivation list type 0x0000, VT shape type 0x0000 + Size = 264, class name = LegoActorInfo, UDT(0x000056f5) + """ + lego_actor_info_array_key = CvdumpTypeKey(0x1003) + recomp_address_of_global = 5678 # arbitrary, but needs to be consistent + + type_helper.set_up_cvdump_types(cvdump_types) + + type_helper.compare.cvdump_analysis.nodes = [ + CvdumpNode( + addr=recomp_address_of_global, + section=0, + offset=0, + decorated_name="g_actorInfo", + data_type=TypeInfo( + lego_actor_info_array_key, + size=174, + ), + ) + ] + + import_global_into_ghidra( + ghidra, + type_helper.compare, + type_helper.type_importer, + ReccmpMatch(function_helper.ORIG_DATA_TO_OVERWRITE, recomp_address_of_global), + ) + + # based on BETA10 0x100742eb + function_helper.overwrite_example_function( + b"\x8b\xec" # MOV EBP, ESP + b"\x8b\x45\x04" # MOV EAX, dword ptr [EBP + p_index] + b"\x8b\xc8" # MOV this, EAX + b"\xc1\xe0\x05" # SHL EAX, 0x5 + b"\x03\xc1" # ADD EAX, this + b"\x8b\x04\xc5\x40\x00\x41\x00" # MOV EAX, dword ptr [EAX *0x8 + g_actorInfo] + b"\xc2\x04\x00" # RET 0x4 + ) + + func_signature = FunctionSignature( + call_type="__stdcall", + arglist=[CVInfoTypeEnum.T_INT4], + return_type=CVInfoTypeEnum.T_32PCHAR, + class_type=None, + stack_symbols=[ + CppStackSymbol("p_index", CVInfoTypeEnum.T_INT4, 4), + ], + this_adjust=0, + ) + pdb_function = PdbFunction( + ReccmpMatch( + function_helper.orig_address, + 1234, # arbitrary + json.dumps({"name": "LegoCharacterManager::GetActorName"}), + ), + func_signature, + is_stub=False, + ) + + PdbFunctionImporter.build( + ghidra, pdb_function, type_helper.type_importer, [] + ).overwrite_ghidra_function(function_helper.ghidra_function) + + function_helper.assert_c_code(""" +char * LegoCharacterManager::GetActorName(int p_index) + +{ + return g_actorInfo[p_index].m_name; +} + +""") diff --git a/tests/test_ghidra_integration_type_imports.py b/tests/test_ghidra_integration_type_imports.py new file mode 100644 index 000000000..77354dac2 --- /dev/null +++ b/tests/test_ghidra_integration_type_imports.py @@ -0,0 +1,372 @@ +# Suppress linter warnings related to the fact that the header support for Ghidra is limited +# and that we cannot import Ghidra classes before Ghidra has been loaded + +# pylint: disable=import-outside-toplevel +# pyright: reportMissingModuleSource=false + +from typing import TYPE_CHECKING + +import pytest +from reccmp.cvdump.cvinfo import CVInfoTypeEnum, CvdumpTypeKey, CvdumpTypeMap +from reccmp.ghidra.importer.exceptions import TypeNotFoundError, TypeNotImplementedError +from .ghidra_integration_test_setup import GhidraTypeTestHelper +from .helpers import assert_instance + +if TYPE_CHECKING: + from ghidra.program.flatapi import FlatProgramAPI + from ghidra.program.model.data import DataType + + +# Shortened version of a BETA10 recompilation +# codespell:ignore-begin +CVDUMP_TYPES = """ +0x1199 : Length = 10, Leaf = 0x1002 LF_POINTER + const Pointer (NEAR32), Size: 0 + Element type : T_RCHAR(0070) + +0x12c8 : Length = 42, Leaf = 0x1505 LF_STRUCTURE + # members = 0, field list type 0x0000, FORWARD REF, + Derivation list type 0x0000, VT shape type 0x0000 + Size = 0, class name = LegoAnimActorEntry, UDT(0x00006081) + +0x12c9 : Length = 10, Leaf = 0x1002 LF_POINTER + Pointer (NEAR32), Size: 0 + Element type : 0x12C8 + +0x12cd : Length = 314, Leaf = 0x1203 LF_FIELDLIST + list[10] = LF_MEMBER, protected, type = T_LONG(0012), offset = 8 + member name = 'm_duration' + list[11] = LF_MEMBER, protected, type = 0x12C9, offset = 12 + member name = 'm_modelList' + list[12] = LF_MEMBER, protected, type = T_ULONG(0022), offset = 16 + member name = 'm_numActors' + +0x12cf : Length = 30, Leaf = 0x1504 LF_CLASS + # members = 15, field list type 0x12cd, CONSTRUCTOR, + Derivation list type 0x0000, VT shape type 0x12ce + Size = 24, class name = LegoAnim, UDT(0x000012cf) + +0x147f : Length = 74, Leaf = 0x1203 LF_FIELDLIST + list[0] = LF_MEMBER, public, type = T_ULONG(0022), offset = 0 + member name = 'LowPart' + list[1] = LF_MEMBER, public, type = T_LONG(0012), offset = 4 + member name = 'HighPart' + list[2] = LF_MEMBER, public, type = 0x147E, offset = 0 + member name = 'u' + list[3] = LF_MEMBER, public, type = T_QUAD(0013), offset = 0 + member name = 'QuadPart' + +0x1480 : Length = 30, Leaf = 0x1506 LF_UNION + # members = 4, field list type 0x147f, Size = 8 ,class name = _LARGE_INTEGER, UDT(0x00001480) + +0x3159 : Length = 30, Leaf = 0x1505 LF_STRUCTURE + # members = 0, field list type 0x0000, FORWARD REF, + Derivation list type 0x0000, VT shape type 0x0000 + Size = 0, class name = HWND__ + +0x31bb : Length = 14, Leaf = 0x1503 LF_ARRAY + Element type = T_ULONG(0022) + Index type = T_SHORT(0011) + length = 16 + Name = + +0x4ef1 : Length = 18, Leaf = 0x1201 LF_ARGLIST argument count = 3 + list[0] = T_32PRCHAR(0470) + list[1] = T_LONG(0012) + list[2] = T_32PVOID(0403) + +0x4ef2 : Length = 14, Leaf = 0x1008 LF_PROCEDURE + Return type = T_VOID(0003), Call type = C Near + Func attr = none + # Parms = 3, Arg list type = 0x4ef1 + +0x5695 : Length = 50, Leaf = 0x1203 LF_FIELDLIST + list[0] = LF_ENUMERATE, public, value = (LF_CHAR) -1(0xFF), name = 'c_unknownminusone' + list[1] = LF_ENUMERATE, public, value = 8, name = 'c_unknown8' + +0x5696 : Length = 42, Leaf = 0x1507 LF_ENUM + # members = 2, type = T_INT4(0074) field list type 0x5695 +NESTED, enum name = LegoCarBuild::Unknown0xf8, UDT(0x00005696) + +0x6080 : Length = 62, Leaf = 0x1203 LF_FIELDLIST + list[1] = LF_MEMBER, public, type = T_32PRCHAR(0470), offset = 0 + member name = 'm_name' + list[2] = LF_MEMBER, public, type = T_ULONG(0022), offset = 4 + member name = 'm_type' + +0x6081 : Length = 42, Leaf = 0x1505 LF_STRUCTURE + # members = 3, field list type 0x6080, + Derivation list type 0x0000, VT shape type 0x0000 + Size = 8, class name = LegoAnimActorEntry, UDT(0x00006081) + +0x608e : Length = 130, Leaf = 0x1203 LF_FIELDLIST + list[0] = LF_ENUMERATE, public, value = 0, name = 'c_initial' + list[1] = LF_ENUMERATE, public, value = 1, name = 'c_ready' + list[2] = LF_ENUMERATE, public, value = 2, name = 'c_hit' + list[3] = LF_ENUMERATE, public, value = 3, name = 'c_hitAnimation' + list[4] = LF_ENUMERATE, public, value = 4, name = 'c_disabled' + list[5] = LF_ENUMERATE, public, value = 255, name = 'c_maxState' + list[6] = LF_ENUMERATE, public, value = 256, name = 'c_noCollide' + +0x608f : Length = 42, Leaf = 0x1507 LF_ENUM + # members = 7, type = T_INT4(0074) field list type 0x608e +NESTED, enum name = LegoPathActor::ActorState, UDT(0x0000608f) +""" +# codespell:ignore-end + +pointer_to_char_key = CvdumpTypeKey(0x1199) +legoanimactor_forward_ref_key = CvdumpTypeKey(0x12C8) +legoanimactor_pointer_key = CvdumpTypeKey(0x12C9) +union_key = CvdumpTypeKey(0x1480) +hwnd_key = CvdumpTypeKey(0x3159) +array_key = CvdumpTypeKey(0x31BB) +procedure_key = CvdumpTypeKey(0x4EF2) +enum_with_negative_value_key = CvdumpTypeKey(0x5696) +legoanimactor_class_key = CvdumpTypeKey(0x6081) +enum_key = CvdumpTypeKey(0x608F) + + +def _assert_legoanimactorentry(imported_structure: "DataType"): + from ghidra.program.model.data import Structure + + assert isinstance(imported_structure, Structure) + + assert imported_structure.getDisplayName() == "LegoAnimActorEntry" + assert imported_structure.length == 8 + + [name_component, id_component] = list(imported_structure.getComponents()) + assert name_component.getOffset() == 0 + assert name_component.getDataType().name == "char *" + assert id_component.getOffset() == 4 + assert id_component.getDataType().name == "ulong" + + +verified_types = ( + t + for t in CVInfoTypeEnum + if CvdumpTypeMap[t].verified and t != CVInfoTypeEnum.T_NOTYPE +) + + +@pytest.mark.parametrize("scalar_type", verified_types) +def test_ghidra_scalar_types( + type_helper: GhidraTypeTestHelper, scalar_type: CVInfoTypeEnum +): + from ghidra.program.model.data import Pointer + + cv_type_info = CvdumpTypeMap[scalar_type] + + ghidra_type = type_helper.type_importer.import_pdb_type_into_ghidra(scalar_type) + assert ghidra_type.length == cv_type_info.size + + if cv_type_info.pointer is not None: + assert isinstance(ghidra_type, Pointer) + + second_import = type_helper.type_importer.import_pdb_type_into_ghidra(scalar_type) + assert second_import == ghidra_type + + +def test_ghidra_type_not_found(type_helper: GhidraTypeTestHelper): + with pytest.raises(TypeNotFoundError, match="Failed to find referenced type"): + type_helper.type_importer.import_pdb_type_into_ghidra(CvdumpTypeKey(0x1001)) + + +def test_ghidra_type_class(type_helper: GhidraTypeTestHelper): + type_helper.set_up_cvdump_types(CVDUMP_TYPES) + imported_structure = type_helper.type_importer.import_pdb_type_into_ghidra( + legoanimactor_class_key + ) + _assert_legoanimactorentry(imported_structure) + + second_import = type_helper.type_importer.import_pdb_type_into_ghidra( + legoanimactor_class_key + ) + assert second_import == imported_structure + + +def test_ghidra_verify_test_isolation(ghidra: "FlatProgramAPI"): + """Make sure that the `LegoAnimActorEntry` created above was rolled back.""" + assert not list(ghidra.getDataTypes("LegoAnimActorEntry")) + + +def test_ghidra_forward_ref_to_pdb_type(type_helper: GhidraTypeTestHelper): + type_helper.set_up_cvdump_types(CVDUMP_TYPES) + imported_structure = type_helper.type_importer.import_pdb_type_into_ghidra( + legoanimactor_forward_ref_key + ) + _assert_legoanimactorentry(imported_structure) + + second_import = type_helper.type_importer.import_pdb_type_into_ghidra( + legoanimactor_forward_ref_key + ) + assert second_import == imported_structure + + +def test_forward_ref_to_missing_type(type_helper: GhidraTypeTestHelper): + type_helper.set_up_cvdump_types(CVDUMP_TYPES) + + with pytest.raises( + TypeNotImplementedError, + match="forward ref without target, needs to be created manually:", + ): + type_helper.type_importer.import_pdb_type_into_ghidra(hwnd_key) + + +def test_forward_ref_to_pre_existing_type( + ghidra: "FlatProgramAPI", type_helper: GhidraTypeTestHelper +): + data_type_manager = ghidra.getCurrentProgram().getDataTypeManager() + from ghidra.program.model.data import ( + TypedefDataType, + VoidDataType, + DataTypeConflictHandler, + ) + + hwnd = data_type_manager.addDataType( + TypedefDataType("HWND__", VoidDataType()), DataTypeConflictHandler.KEEP_HANDLER + ) + + type_helper.set_up_cvdump_types(CVDUMP_TYPES) + + imported_hwnd = type_helper.type_importer.import_pdb_type_into_ghidra(hwnd_key) + assert imported_hwnd == hwnd + + +def test_ghidra_pointer_to_class(type_helper: GhidraTypeTestHelper): + from ghidra.program.model.data import Pointer + + type_helper.set_up_cvdump_types(CVDUMP_TYPES) + imported_pointer = assert_instance( + type_helper.type_importer.import_pdb_type_into_ghidra( + legoanimactor_pointer_key + ), + Pointer, + ) + _assert_legoanimactorentry(imported_pointer.dataType) + + second_import = type_helper.type_importer.import_pdb_type_into_ghidra( + legoanimactor_pointer_key + ) + assert second_import == imported_pointer + + +def test_pointer_to_scalar(type_helper: GhidraTypeTestHelper): + from ghidra.program.model.data import Pointer + + type_helper.set_up_cvdump_types(CVDUMP_TYPES) + imported_pointer = assert_instance( + type_helper.type_importer.import_pdb_type_into_ghidra(pointer_to_char_key), + Pointer, + ) + assert ( + imported_pointer.dataType + == type_helper.type_importer.import_pdb_type_into_ghidra(CVInfoTypeEnum.T_CHAR) + ) + + second_import = type_helper.type_importer.import_pdb_type_into_ghidra( + pointer_to_char_key + ) + assert second_import == imported_pointer + + +def test_array(type_helper: GhidraTypeTestHelper): + from ghidra.program.model.data import Array + + type_helper.set_up_cvdump_types(CVDUMP_TYPES) + + imported_array = assert_instance( + type_helper.type_importer.import_pdb_type_into_ghidra(array_key), Array + ) + + assert imported_array.getLength() == 16 + assert imported_array.getElementLength() == 4 + assert ( + imported_array.getDataType() + == type_helper.type_importer.import_pdb_type_into_ghidra(CVInfoTypeEnum.T_ULONG) + ) + + second_import = type_helper.type_importer.import_pdb_type_into_ghidra(array_key) + assert second_import == imported_array + + +def test_enum(type_helper: GhidraTypeTestHelper): + from ghidra.program.model.data import Enum + + type_helper.set_up_cvdump_types(CVDUMP_TYPES) + + imported_enum = assert_instance( + type_helper.type_importer.import_pdb_type_into_ghidra(enum_key), Enum + ) + assert imported_enum.getDisplayName() == "ActorState" + assert imported_enum.getCount() == 7 + assert list(imported_enum.getNames()) == [ + "c_initial", + "c_ready", + "c_hit", + "c_hitAnimation", + "c_disabled", + "c_maxState", + "c_noCollide", + ] + assert list(imported_enum.getValues()) == [ + 0, + 1, + 2, + 3, + 4, + 255, + 256, + ] + + second_import = type_helper.type_importer.import_pdb_type_into_ghidra(enum_key) + assert second_import == imported_enum + + +def test_enum_with_negative_value(type_helper: GhidraTypeTestHelper): + from ghidra.program.model.data import Enum + + type_helper.set_up_cvdump_types(CVDUMP_TYPES) + + imported_enum = assert_instance( + type_helper.type_importer.import_pdb_type_into_ghidra( + enum_with_negative_value_key + ), + Enum, + ) + assert imported_enum.getDisplayName() == "Unknown0xf8" + assert imported_enum.getCount() == 2 + assert list(imported_enum.getNames()) == ["c_unknownminusone", "c_unknown8"] + assert list(imported_enum.getValues()) == [-1, 8] + + second_import = type_helper.type_importer.import_pdb_type_into_ghidra( + enum_with_negative_value_key + ) + assert second_import == imported_enum + + +def test_fallback_procedure_import(type_helper: GhidraTypeTestHelper): + """The feature is not fully implemented. This test asserts on the fallback behaviour.""" + + type_helper.set_up_cvdump_types(CVDUMP_TYPES) + + imported_type = type_helper.type_importer.import_pdb_type_into_ghidra(procedure_key) + # Fallback behaviour. This assertion should be changed if proper support is implemented + assert imported_type == type_helper.type_importer.import_pdb_type_into_ghidra( + CVInfoTypeEnum.T_VOID + ) + + second_import = type_helper.type_importer.import_pdb_type_into_ghidra(procedure_key) + assert second_import == imported_type + + +@pytest.mark.xfail(reason="Union import not yet implemented") +def test_union(type_helper: GhidraTypeTestHelper): + from ghidra.program.model.data import Union + + type_helper.set_up_cvdump_types(CVDUMP_TYPES) + + _imported_union = assert_instance( + type_helper.type_importer.import_pdb_type_into_ghidra(union_key), Union + ) + + # More assertions are needed once we have proper support