ihateninjas/OdysseyDecomp
1
0
Fork 0
mirror of https://github.com/MonsterDruide1/OdysseyDecomp synced 2026-04-23 09:04:21 +00:00
Code Issues Actions 17 Packages Projects Releases Wiki Activity

Integrate new toolchain

Browse source
This commit is contained in:
MonsterDruide1 2021-08-01 00:21:42 +02:00
parent 6b69f6e8a6
commit 61d71e83f2
28 changed files with 151404 additions and 74955 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
.gitignore vendored
View file

@ -1,18 +1,26 @@
# Python
__pycache__/
*.pyc
*.egg-info/
*.dist-info/
*.so
*.dll
dist/
build/
bin/
.mypy_cache/
.benchmarks/
# Binaries
*.so
*.dll
*.elf
*.nso
dist/
build/
bin/
# IDEs
.idea/
.vscode/
.DS_Store
# IDA
*.id0
*.id1
*.id2
@ -21,11 +29,15 @@ bin/
*.nam
*.til
main.elf
# Ghidra
*.gpr
*.rep/
*.lock
perf.mData
perf.mData.old
# Debugging
perf.data
perf.data.old
.gdb_history
.DS_Store
tools/aarch64-none-elf-objdump
# Tooling
/toolchain/clang-*

6
.gitmodules vendored
View file

@ -7,3 +7,9 @@
[submodule "include/agl"]
path = include/agl
url = https://github.com/MonsterDruide1/agl
[submodule "toolchain/musl"]
path = toolchain/musl
url = https://github.com/open-ead/botw-lib-musl
[submodule "tools/nx-decomp-tools-binaries"]
path = tools/nx-decomp-tools-binaries
url = https://github.com/open-ead/nx-decomp-tools-binaries

4
CMakeLists.txt
View file

@ -2,7 +2,8 @@ cmake_minimum_required(VERSION 3.13)
project(odyssey CXX)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_NINJA_FORCE_RESPONSE_FILE ON)
# Use response files when linking objects because of the large number of source files
set(CMAKE_CXX_USE_RESPONSE_FILE_FOR_OBJECTS ON)
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
add_compile_options(-fdiagnostics-color=always)
elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
@ -19,6 +20,7 @@ set(CMAKE_CXX_EXTENSIONS OFF)
add_executable(odyssey)
target_include_directories(odyssey PRIVATE src)
target_compile_definitions(odyssey PRIVATE NON_MATCHING)
target_compile_options(odyssey PRIVATE -fno-rtti -fno-exceptions)
target_compile_options(odyssey PRIVATE -Wall -Wextra -Wdeprecated)
target_compile_options(odyssey PRIVATE -Wno-unused-parameter -Wno-unused-private-field)

46
ToolchainNX64.cmake
View file

@ -1,46 +0,0 @@
if (NOT DEFINED ENV{ODYSSEY_CLANG})
message(FATAL_ERROR "Please define the ODYSSEY_CLANG env variable. It should point to a path such that $ODYSSEY_CLANG/bin/clang exists")
endif()
if (NOT DEFINED ENV{DEVKITA64})
message(FATAL_ERROR "Please define the DEVKITA64 env variable.")
endif()
set(ODYSSEY_CLANG "$ENV{ODYSSEY_CLANG}")
set(DEVKITA64 "$ENV{DEVKITA64}")
set(NX64_OPT_FLAGS "-O3 -g")
set(triple aarch64-none-elf)
set(CMAKE_SYSTEM_NAME Generic)
set(CMAKE_SYSTEM_VERSION 1)
set(CMAKE_SYSTEM_PROCESSOR aarch64)
set(CMAKE_SYSROOT ${ODYSSEY_CLANG})
set(CMAKE_C_COMPILER "${ODYSSEY_CLANG}/bin/clang")
set(CMAKE_C_COMPILER_TARGET ${triple})
set(CMAKE_CXX_COMPILER "${ODYSSEY_CLANG}/bin/clang++")
set(CMAKE_CXX_COMPILER_TARGET ${triple})
set(CMAKE_C_FLAGS_RELEASE ${NX64_OPT_FLAGS})
set(CMAKE_CXX_FLAGS_RELEASE ${NX64_OPT_FLAGS})
set(CMAKE_C_FLAGS_RELWITHDEBINFO ${NX64_OPT_FLAGS})
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO ${NX64_OPT_FLAGS})
set(ARCH "-mcpu=cortex-a57+fp+simd+crypto+crc")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ARCH} -isystem ${DEVKITA64}/aarch64-none-elf/include")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${ODYSSEY_CLANG}/include/c++/v1 -D _LIBCPP_HAS_THREAD_API_PTHREAD ${CMAKE_C_FLAGS}")
set(CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -x assembler-with-cpp ${ARCH}")
add_compile_options(-fPIC -stdlib=libc++ -mno-implicit-float)
add_link_options(-B ${DEVKITA64}/bin -fPIC -Wl,-Bsymbolic-functions -shared -nodefaultlibs)
if(EXISTS "${DEVKITA64}/bin/ld.lld")
add_link_options(-fuse-ld=lld -Wl,-z,notext)
endif()
add_definitions(-D SWITCH -D __DEVKITA64__ -D __ELF__)
add_definitions(-D NNSDK)
add_definitions(-D MATCHING_HACK_NX_CLANG)
add_definitions(-D VER_100)
# Helps with matching as this causes Clang to emit debug type info even for dynamic classes
# with undefined vtables.
add_compile_options(-fstandalone-debug)

55
convertFunctions.py Normal file
View file

@ -0,0 +1,55 @@
import csv
prefixes = ("sub_")
good_prefixes = ("eco::", "ui::")
last_addr = 0
with open("data/ukf.csv", "r") as csvfile, open("data/uking_functions.csv", "w") as f:
w = csv.writer(f)
w.writerow(["Address", "Quality", "Size", "Name"])
for row in csv.reader(csvfile):
addr = row[0].strip()
ida_name = row[1].strip()
if not ida_name.startswith(good_prefixes) and "sub" in ida_name.lower() and "nullsub" not in ida_name.lower() and ida_name.lower().endswith(addr[-10:].lower()):
if not ida_name.lower().startswith("sub"):
print(ida_name)
assert False
ida_name = ""
name = ida_name
size = "{:06d}".format(int(row[2].strip(), 0))
assert len(size) == 6
code_name = row[3].strip()
quality = "U"
#if "nullsub" in name:
# print(name, size)
addr_val = int(row[0], 16)
assert addr_val > last_addr
last_addr = addr_val
addr = "0x{0:016x}".format(addr_val)
assert len(addr) == 18
assert not ida_name or (ida_name[-1] != "?" and ida_name[-1] != "!")
assert int(size) > 1
if code_name and code_name != "l":
assert len(code_name) > 1
if code_name[-1] == "?":
quality = "m"
name = code_name[:-1]
elif code_name[-1] == "!":
quality = "M"
name = code_name[:-1]
else:
quality = "O"
name = code_name
elif code_name == "l":
quality = "L"
if set(name) == {"x"}:
print(row)
name = ""
#assert name
w.writerow([addr, quality, size, name])

149567
data/odyssey_functions.csv
View file

File diff suppressed because it is too large Load diff

74783
data/smo_backup.csv Normal file
View file

File diff suppressed because it is too large Load diff

48
toolchain/ToolchainNX64.cmake Normal file
View file

@ -0,0 +1,48 @@
if (DEFINED ENV{ODYSSEY_CLANG})
set(ODYSSEY_CLANG "$ENV{ODYSSEY_CLANG}")
else()
set(ODYSSEY_CLANG "${CMAKE_CURRENT_LIST_DIR}/clang-3.9.1")
endif()
if (DEFINED ENV{ODYSSEY_CLANG_LLD})
set(ODYSSEY_CLANG_LLD "$ENV{ODYSSEY_CLANG_LLD}")
else()
set(ODYSSEY_CLANG_LLD "${CMAKE_CURRENT_LIST_DIR}/clang-4.0.1")
endif()
set(NX64_OPT_FLAGS "-O3 -g")
set(NX64_TRIPLE aarch64-linux-elf)
set(CMAKE_SYSTEM_NAME Generic)
set(CMAKE_SYSTEM_VERSION 1)
set(CMAKE_SYSTEM_PROCESSOR aarch64)
set(CMAKE_SYSROOT ${CMAKE_CURRENT_LIST_DIR}/musl)
set(CMAKE_C_COMPILER "${ODYSSEY_CLANG}/bin/clang")
set(CMAKE_C_COMPILER_TARGET ${NX64_TRIPLE})
set(CMAKE_CXX_COMPILER "${ODYSSEY_CLANG}/bin/clang++")
set(CMAKE_CXX_COMPILER_TARGET ${NX64_TRIPLE})
set(CMAKE_C_FLAGS_RELEASE ${NX64_OPT_FLAGS})
set(CMAKE_CXX_FLAGS_RELEASE ${NX64_OPT_FLAGS})
set(CMAKE_C_FLAGS_RELWITHDEBINFO ${NX64_OPT_FLAGS})
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO ${NX64_OPT_FLAGS})
# Target options
add_compile_options(-mcpu=cortex-a57+fp+simd+crypto+crc)
add_compile_options(-mno-implicit-float)
# Environment
add_compile_options(-stdlib=libc++)
add_compile_options(-fPIC)
# Helps with matching as this causes Clang to emit debug type info even for dynamic classes
# with undefined vtables.
add_compile_options(-fstandalone-debug)
add_definitions(-D SWITCH)
add_definitions(-D NNSDK)
add_definitions(-D MATCHING_HACK_NX_CLANG)
add_link_options(-stdlib=libc++ -nostdlib)
add_link_options(-fPIC -Wl,-Bsymbolic-functions -shared)
# Use lld for performance reasons (and because we don't want a dependency on GNU tools)
add_link_options(-fuse-ld=${ODYSSEY_CLANG_LLD}/bin/ld.lld)

1
toolchain/musl Submodule

@ -0,0 +1 @@
Subproject commit 9a9252a54a67f54c066966f8f0599c450391bd44

68
tools/check.py
View file

@ -1,68 +0,0 @@
#!/usr/bin/env python3
import sys
from typing import Optional
from colorama import Fore
import util.elf
import util.checker
from util import utils
def check_function(checker: util.checker.FunctionChecker, addr: int, size: int, name: str,
base_fn: Optional[util.elf.Function] = None) -> bool:
if base_fn is None:
try:
base_fn = util.elf.get_fn_from_base_elf(addr, size)
except KeyError:
utils.print_error(f"couldn't find base function 0x{addr:016x} for {utils.format_symbol_name_for_msg(name)}")
return False
my_fn = util.elf.get_fn_from_my_elf(name)
return checker.check(base_fn, my_fn)
def main() -> None:
failed = False
nonmatching_fns_with_dump = {p.stem: util.elf.Function(p.read_bytes(), 0) for p in
(utils.get_repo_root() / "expected").glob("*.bin")}
checker = util.checker.FunctionChecker(log_mismatch_cause=True)
for func in utils.get_functions():
if not func.decomp_name:
continue
try:
util.elf.get_fn_from_my_elf(func.decomp_name)
except KeyError:
utils.warn(f"couldn't find {utils.format_symbol_name_for_msg(func.decomp_name)}")
continue
if func.status == utils.FunctionStatus.Matching:
if not check_function(checker, func.addr, func.size, func.decomp_name):
utils.print_error(
f"function {utils.format_symbol_name_for_msg(func.decomp_name)} is marked as matching but does not match")
a1, a2, reason = checker.get_mismatch()
if a1 != -1:
sys.stderr.write(f" at {a1 | 0x7100000000:#x} : {Fore.CYAN}{reason}{Fore.RESET}\n")
failed = True
elif func.status == utils.FunctionStatus.Equivalent or func.status == utils.FunctionStatus.NonMatching:
if check_function(checker, func.addr, func.size, func.decomp_name):
utils.print_note(
f"function {utils.format_symbol_name_for_msg(func.decomp_name)} is marked as non-matching but matches")
fn_dump = nonmatching_fns_with_dump.get(func.decomp_name, None)
if fn_dump is not None and not check_function(checker, func.addr, len(fn_dump), func.decomp_name, fn_dump):
utils.print_error(
f"function {utils.format_symbol_name_for_msg(func.decomp_name)} does not match expected output")
failed = True
if failed:
sys.exit(1)
if __name__ == "__main__":
main()

19
tools/diff_settings.py
View file

@ -1,5 +1,18 @@
#!/usr/bin/env python3
from pathlib import Path
import platform
ROOT = Path(__file__).resolve().parent.parent
def get_tools_bin_dir():
path = ROOT / 'tools' / 'nx-decomp-tools-binaries'
system = platform.system()
if system == "Linux":
return str(path) + "/linux/"
if system == "Darwin":
return str(path) + "/macos/"
return ""
def apply(config, args):
@ -7,7 +20,7 @@ def apply(config, args):
config['baseimg'] = 'data/main.elf'
config['myimg'] = 'build/odyssey'
config['source_directories'] = ['src', 'lib']
config['objdump_executable'] = 'tools/aarch64-none-elf-objdump'
config['objdump_executable'] = get_tools_bin_dir() + 'aarch64-none-elf-objdump'
for dir in ('build', 'build/nx64-release'):
if (Path(dir) / 'build.ninja').is_file():
@ -20,4 +33,4 @@ def map_build_target(make_target: str):
# TODO: When support for directly diffing object files is added, this needs to strip
# the build/ prefix from the object file targets.
return make_target
return make_target

119
tools/ghidra_scripts/RenameFunctionsInGhidra.java Normal file
View file

@ -0,0 +1,119 @@
// Script to load BotW CSV data into Ghidra
//@author AlexApps99
//@category BotW
import ghidra.app.script.GhidraScript;
import ghidra.program.model.symbol.SourceType;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.File;
import ghidra.program.model.address.Address;
import ghidra.program.model.listing.Function;
import ghidra.program.model.listing.FunctionManager;
import ghidra.app.cmd.label.DemanglerCmd;
import ghidra.program.model.address.AddressSet;
import ghidra.util.NumericUtilities;
import ghidra.app.cmd.label.DemanglerCmd;
import ghidra.program.model.listing.FunctionTag;
import ghidra.util.exception.DuplicateNameException;
import ghidra.program.model.listing.FunctionTagManager;
public class RenameFunctionsInGhidra extends GhidraScript {
private FunctionManager func_mgr;
private FunctionTagManager func_tag_mgr;
private String ok;
private String minor;
private String major;
private String wip;
private String undecompiled;
private String lib;
private FunctionTag getOrMake(String name) {
FunctionTag f = func_tag_mgr.getFunctionTag(name);
if (f == null) f = func_tag_mgr.createFunctionTag(name, null);
return f;
}
@Override
public void run() throws Exception {
func_mgr = currentProgram.getFunctionManager();
func_tag_mgr = func_mgr.getFunctionTagManager();
ok = getOrMake("OK").getName();
minor = getOrMake("MINOR").getName();
major = getOrMake("MAJOR").getName();
wip = getOrMake("WIP").getName();
undecompiled = getOrMake("UNDECOMPILED").getName();
lib = getOrMake("LIBRARY").getName();
File input_csv = askFile("odyssey_functions.csv", "Go");
try (BufferedReader br = new BufferedReader(new FileReader(input_csv))) {
// Skip header
String line = br.readLine();
while ((line = br.readLine()) != null) {
String[] pieces = line.split(",", -4); // Don't skip empty last column
if (pieces.length != 4) throw new Exception("Invalid CSV row: " + line);
Address addr = toAddr(pieces[0]);
String status = pieces[1];
long func_size = func_size = NumericUtilities.parseLong(pieces[2].strip());
String name = pieces[3].strip();
Function func = applyFunction(addr, status, name, func_size);
}
}
}
// TODO the j_ prefix probably breaks demangling
private Function applyFunction(Address addr, String status, String name, long func_size) throws Exception {
if (name.isEmpty()) name = null;
Function func = func_mgr.getFunctionAt(addr);
AddressSet body = new AddressSet(addr, addr.addNoWrap(func_size - 1));
if (func != null) {
// Demangling can break this, hence the try-catch
try {
if (func.getName() != name) func.setName(name, SourceType.IMPORTED);
} catch (DuplicateNameException e) {}
if (!func.getBody().hasSameAddresses(body)) {
func.setBody(body);
}
} else {
func = func_mgr.createFunction(name, addr, body, SourceType.IMPORTED);
}
if (name != null) {
DemanglerCmd cmd = new DemanglerCmd(addr, name);
if (!cmd.applyTo(currentProgram, monitor)) {
// Something that isn't mangled
}
}
func.removeTag(ok);
func.removeTag(minor);
func.removeTag(major);
func.removeTag(wip);
func.removeTag(undecompiled);
func.removeTag(lib);
if (status.equals("O")) {
func.addTag(ok);
} else if (status.equals("m")) {
func.addTag(minor);
} else if (status.equals("M")) {
func.addTag(major);
} else if (status.equals("W")) {
func.addTag(wip);
} else if (status.equals("L")) {
func.addTag(lib);
func.addTag(undecompiled);
} else {
func.addTag(undecompiled);
}
return func;
}
}

1
tools/nx-decomp-tools-binaries Submodule

@ -0,0 +1 @@
Subproject commit 011c369bc4bead403d650dd1d1eeb69c04eb19c7

19
tools/progress.py
View file

@ -2,8 +2,6 @@
import argparse
from collections import defaultdict
from colorama import Back, Fore, Style
import enum
from pathlib import Path
from util import utils
from util.utils import FunctionStatus
import typing as tp
@ -17,8 +15,6 @@ parser.add_argument("--print-eq", "-e", action="store_true",
help="Print non-matching functions with minor issues")
parser.add_argument("--print-ok", "-m", action="store_true",
help="Print matching functions")
parser.add_argument("--hide-nonmatchings-with-dumps", "-H", help="Hide non-matching functions that have expected "
"output dumps", action="store_true")
args = parser.parse_args()
code_size_total = 0
@ -26,15 +22,6 @@ num_total = 0
code_size: tp.DefaultDict[FunctionStatus, int] = defaultdict(int)
counts: tp.DefaultDict[FunctionStatus, int] = defaultdict(int)
nonmatching_fns_with_dump = {p.stem for p in (Path(__file__).parent.parent / "expected").glob("*.bin")}
def should_hide_nonmatching(name: str) -> bool:
if not args.hide_nonmatchings_with_dumps:
return False
return name in nonmatching_fns_with_dump
for info in utils.get_functions():
code_size_total += info.size
num_total += 1
@ -47,10 +34,10 @@ for info in utils.get_functions():
if not args.csv:
if info.status == FunctionStatus.NonMatching:
if args.print_nm and not should_hide_nonmatching(info.decomp_name):
if args.print_nm:
print(f"{Fore.RED}NM{Fore.RESET} {utils.format_symbol_name(info.decomp_name)}")
elif info.status == FunctionStatus.Equivalent:
if args.print_eq and not should_hide_nonmatching(info.decomp_name):
if args.print_eq:
print(f"{Fore.YELLOW}EQ{Fore.RESET} {utils.format_symbol_name(info.decomp_name)}")
elif info.status == FunctionStatus.Matching:
if args.print_ok:
@ -111,4 +98,4 @@ else:
print(format_progress_for_status(f"{Fore.GREEN}matching", FunctionStatus.Matching))
print(format_progress_for_status(f"{Fore.YELLOW}non-matching (minor issues)", FunctionStatus.Equivalent))
print(format_progress_for_status(f"{Fore.RED}non-matching (major issues)", FunctionStatus.NonMatching))
print()
print()

16
tools/rename_functions_in_ida.py
View file

@ -7,18 +7,12 @@ import os
csv_path = os.path.join(os.path.dirname(__file__), "../data/odyssey_functions.csv")
MARKERS = ("|", "?", "!")
with open(csv_path, "r") as f:
reader = csv.reader(f)
# Skip headers
next(reader)
for fn in reader:
addr = int(fn[0], 16)
decomp_name = fn[3]
if not decomp_name or decomp_name == "l":
continue
# Get rid of status markers.
if decomp_name[-1] in MARKERS:
decomp_name = decomp_name[:-1]
idc.set_name(addr, decomp_name)
name = fn[3]
if name and fn[1] != "L":
idc.set_name(addr, name)

165
tools/setup.py Normal file
View file

@ -0,0 +1,165 @@
#!/usr/bin/env python3
import argparse
import hashlib
import platform
from pathlib import Path
import subprocess
import sys
import tarfile
import tempfile
import urllib.request
ROOT = Path(__file__).parent.parent
def fail(error: str):
print(">>> " + error)
sys.exit(1)
def _get_tool_binary_path():
base = ROOT / "tools" / "nx-decomp-tools-binaries"
system = platform.system()
if system == "Linux":
return str(base / "linux") + "/"
if system == "Darwin":
return str(base / "macos") + "/"
return ""
def _convert_nso_to_elf(nso_path: Path):
print(">>>> converting NSO to ELF...")
binpath = _get_tool_binary_path()
subprocess.check_call([binpath + "nx2elf", str(nso_path)])
def _decompress_nso(nso_path: Path, dest_path: Path):
print(">>>> decompressing NSO...")
binpath = _get_tool_binary_path()
subprocess.check_call([binpath + "hactool", "-tnso",
"--uncompressed=" + str(dest_path), str(nso_path)])
def prepare_executable(original_nso: Path):
COMPRESSED_V150_HASH = "e21692d90f8fd2def2d2d22d983d62ac81df3b8b3c762d1f2dca9d9ab7b3053a"
UNCOMPRESSED_V150_HASH = "18ece865061704d551fe456e0600c604c26345ecb38dcbe328a24d5734b3b4eb"
# The uncompressed v1.5.0 main NSO.
TARGET_HASH = UNCOMPRESSED_V150_HASH
TARGET_PATH = ROOT / "data" / "main.nso"
TARGET_ELF_PATH = ROOT / "data" / "main.elf"
if TARGET_PATH.is_file() and hashlib.sha256(TARGET_PATH.read_bytes()).hexdigest() == TARGET_HASH and TARGET_ELF_PATH.is_file():
print(">>> NSO is already set up")
return
if not original_nso.is_file():
fail(f"{original_nso} is not a file")
nso_data = original_nso.read_bytes()
nso_hash = hashlib.sha256(nso_data).hexdigest()
if nso_hash == UNCOMPRESSED_V150_HASH:
print(">>> found uncompressed 1.5.0 NSO")
TARGET_PATH.write_bytes(nso_data)
elif nso_hash == COMPRESSED_V150_HASH:
print(">>> found compressed 1.5.0 NSO")
_decompress_nso(original_nso, TARGET_PATH)
else:
fail(f"unknown executable: {nso_hash}")
if not TARGET_PATH.is_file():
fail("internal error while preparing executable (missing NSO); please report")
if hashlib.sha256(TARGET_PATH.read_bytes()).hexdigest() != TARGET_HASH:
fail("internal error while preparing executable (wrong NSO hash); please report")
_convert_nso_to_elf(TARGET_PATH)
if not TARGET_ELF_PATH.is_file():
fail("internal error while preparing executable (missing ELF); please report")
def set_up_compiler(version):
compiler_dir = ROOT / "toolchain" / ("clang-"+version)
if compiler_dir.is_dir():
print(">>> clang is already set up: nothing to do")
return
system = platform.system()
machine = platform.machine()
if version == "3.9.1":
builds = {
# Linux
("Linux", "x86_64"): {
"url": "https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz",
"dir_name": "clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04",
},
("Linux", "aarch64"): {
"url": "https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-aarch64-linux-gnu.tar.xz",
"dir_name": "clang+llvm-3.9.1-aarch64-linux-gnu",
},
}
elif version == "4.0.1":
builds = {
# Linux
("Linux", "x86_64"): {
"url": "https://releases.llvm.org/4.0.1/clang+llvm-4.0.1-x86_64-linux-gnu-Fedora-25.tar.xz",
"dir_name": "clang+llvm-4.0.1-x86_64-linux-gnu-Fedora-25",
},
("Linux", "aarch64"): {
"url": "https://releases.llvm.org/4.0.1/clang+llvm-4.0.1-aarch64-linux-gnu.tar.xz",
"dir_name": "clang+llvm-4.0.1-aarch64-linux-gnu",
},
}
build_info = builds.get((system, machine))
if build_info is None:
fail(
f"unknown platform: {platform.platform()} (please report if you are on Linux)")
url: str = build_info["url"]
dir_name: str = build_info["dir_name"]
print(f">>> downloading Clang {version} from {url}...")
with tempfile.TemporaryDirectory() as tmpdir:
path = tmpdir + "/" + url.split("/")[-1]
urllib.request.urlretrieve(url, path)
print(f">>> extracting Clang {version}...")
with tarfile.open(path) as f:
f.extractall(compiler_dir.parent)
(compiler_dir.parent / dir_name).rename(compiler_dir)
print(">>> successfully set up Clang "+version)
def create_build_dir():
build_dir = ROOT / "build"
if build_dir.is_dir():
print(">>> build directory already exists: nothing to do")
return
subprocess.check_call(
"cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_TOOLCHAIN_FILE=toolchain/ToolchainNX64.cmake -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -B build/".split(" "))
print(">>> created build directory")
def main():
parser = argparse.ArgumentParser(
"setup.py", description="Set up the Super Mario Odyssey decompilation project")
parser.add_argument("original_nso", type=Path,
help="Path to the original NSO (compressed or not)")
args = parser.parse_args()
prepare_executable(args.original_nso)
set_up_compiler("3.9.1")
set_up_compiler("4.0.1")
create_build_dir()
if __name__ == "__main__":
main()

1
tools/smocheck/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
/target

28
tools/smocheck/Cargo.toml Normal file
View file

@ -0,0 +1,28 @@
[package]
name = "smocheck"
version = "1.0.0"
edition = "2018"
[profile.release]
debug = 1
lto = "thin"
[dependencies]
anyhow = "1.0"
capstone = { git = "https://github.com/leoetlino/capstone-rs" }
colored = "2"
cpp_demangle = "0.3.3"
csv = "1.1"
goblin = "0.4"
itertools = "0.10.1"
lazy-init = "0.5.0"
memmap = "0.6.1"
mimalloc = { version = "*", default-features = false }
owning_ref = "0.4.1"
rayon = "1.5.1"
rustc-hash = "1.1.0"
textwrap = "0.14.2"
[[bin]]
name = "smo-check"
path = "src/tools/check.rs"

21
tools/smocheck/LICENSE Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2021 leoetlino
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

89
tools/smocheck/src/capstone_utils.rs Normal file
View file

@ -0,0 +1,89 @@
use anyhow::{bail, Result};
use capstone as cs;
use cs::arch::arm64::{Arm64Insn, Arm64OpMem, Arm64Operand, Arm64OperandType};
use cs::{arch::ArchOperand, RegId};
pub fn translate_cs_error<T>(err: cs::Error) -> Result<T> {
bail!("capstone error: {}", err)
}
#[inline]
pub fn map_two<'a, T, R, F: FnMut(&'a T) -> R>(x: &'a T, y: &'a T, mut f: F) -> (R, R) {
(f(x), f(y))
}
#[inline]
pub fn map_pair<'a, T, R, F: FnMut(&'a T) -> R>(pair: &'a (T, T), f: F) -> (R, R) {
map_two(&pair.0, &pair.1, f)
}
#[inline]
pub fn try_map_two<'a, T, R, F: FnMut(&'a T) -> Result<R, cs::Error>>(
x: &'a T,
y: &'a T,
mut f: F,
) -> Result<(R, R)> {
Ok((
f(x).or_else(translate_cs_error)?,
f(y).or_else(translate_cs_error)?,
))
}
/// Checks if `id` is in [start, end] (inclusive range).
#[inline]
pub fn is_id_in_range(start: Arm64Insn, end: Arm64Insn, id: Arm64Insn) -> bool {
let range = (start as u32)..=(end as u32);
range.contains(&(id as u32))
}
/// Used to make accessing arch-specific data less cumbersome.
pub trait CsArchOperandUtil {
fn arm64(&self) -> &Arm64Operand;
}
impl CsArchOperandUtil for ArchOperand {
fn arm64(&self) -> &Arm64Operand {
match self {
Self::Arm64Operand(x) => x,
_ => unreachable!(),
}
}
}
/// Used to make accessing arch-specific data less cumbersome.
pub trait CsArm64OperandTypeUtil {
fn reg(&self) -> RegId;
fn imm(&self) -> i64;
fn try_mem(&self) -> Option<Arm64OpMem>;
fn mem(&self) -> Arm64OpMem;
}
impl CsArm64OperandTypeUtil for Arm64OperandType {
fn reg(&self) -> RegId {
match self {
Self::Reg(x) => *x,
_ => panic!("expected Reg, got {:#?}", &self),
}
}
fn imm(&self) -> i64 {
match self {
Self::Imm(x) => *x,
_ => panic!("expected Imm, got {:#?}", &self),
}
}
fn try_mem(&self) -> Option<Arm64OpMem> {
match self {
Self::Mem(x) => Some(*x),
_ => None,
}
}
fn mem(&self) -> Arm64OpMem {
match self {
Self::Mem(x) => *x,
_ => panic!("expected Mem, got {:#?}", &self),
}
}
}

518
tools/smocheck/src/checks.rs Normal file
View file

@ -0,0 +1,518 @@
use anyhow::{ensure, Result};
use capstone as cs;
use cs::arch::arm64::{Arm64Insn, Arm64Operand, Arm64OperandType};
use itertools::zip;
use lazy_init::Lazy;
use rustc_hash::FxHashMap;
use std::collections::{HashMap, HashSet};
use std::convert::TryInto;
use std::path::{Path, PathBuf};
use crate::{capstone_utils::*, elf, functions, repo, ui};
struct DataSymbol {
/// Address of the symbol in the original executable.
pub addr: u64,
/// Name of the symbol in our source code.
pub name: String,
/// Size of the symbol in our source code (according to ELF info).
pub size: u64,
}
/// Keeps track of known data symbols so that data loads can be validated.
#[derive(Default)]
struct KnownDataSymbolMap {
/// Symbols. Must be sorted by address.
symbols: Vec<DataSymbol>,
}
impl KnownDataSymbolMap {
fn new() -> Self {
Default::default()
}
fn load(&mut self, csv_path: &Path, decomp_symtab: &elf::SymbolTableByName) -> Result<()> {
let mut reader = csv::ReaderBuilder::new()
.has_headers(false)
.quoting(false)
.from_path(csv_path)?;
for (line, maybe_record) in reader.records().enumerate() {
let record = &maybe_record?;
ensure!(
record.len() == 2,
"invalid number of fields on line {}",
line
);
let addr = functions::parse_address(&record[0])?;
let name = &record[1];
let symbol = decomp_symtab.get(name);
// Ignore missing symbols.
if symbol.is_none() {
continue;
}
let symbol = symbol.unwrap();
self.symbols.push(DataSymbol {
addr,
name: name.to_string(),
size: symbol.st_size,
});
}
self.symbols.sort_by_key(|sym| sym.addr);
Ok(())
}
/// If addr is part of a known data symbol, this function returns the corresponding symbol.
fn get_symbol(&self, addr: u64) -> Option<&DataSymbol> {
// Perform a binary search since `symbols` is sorted.
let mut a: isize = 0;
let mut b: isize = self.symbols.len() as isize - 1;
while a <= b {
let m = a + (b - a) / 2;
let mid_symbol = &self.symbols[m as usize];
let mid_addr_begin = mid_symbol.addr;
let mid_addr_end = mid_addr_begin + mid_symbol.size as u64;
if mid_addr_begin <= addr && addr < mid_addr_end {
return Some(mid_symbol);
}
if addr <= mid_addr_begin {
b = m - 1;
} else if addr >= mid_addr_end {
a = m + 1;
} else {
break;
}
}
None
}
}
fn get_data_symbol_csv_path() -> Result<PathBuf> {
let mut path = repo::get_repo_root()?;
path.push("data");
path.push("data_symbols.csv");
Ok(path)
}
#[derive(Debug)]
pub struct ReferenceDiff {
pub referenced_symbol: u64,
pub expected_ref_in_decomp: u64,
pub actual_ref_in_decomp: u64,
pub expected_symbol_name: String,
pub actual_symbol_name: String,
}
impl std::fmt::Display for ReferenceDiff {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"wrong reference to {ref} {ref_name}\n\
--> decomp source code is referencing {actual} {actual_name}\n\
--> expected to see {expected} to match original code",
ref=ui::format_address(self.referenced_symbol),
ref_name=ui::format_symbol_name(&self.expected_symbol_name),
expected=ui::format_address(self.expected_ref_in_decomp),
actual=ui::format_address(self.actual_ref_in_decomp),
actual_name=ui::format_symbol_name(&self.actual_symbol_name),
)
}
}
#[derive(Debug)]
pub enum MismatchCause {
FunctionSize,
Register,
Mnemonic,
BranchTarget,
FunctionCall(ReferenceDiff),
DataReference(ReferenceDiff),
Immediate,
Unknown,
}
impl std::fmt::Display for MismatchCause {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match &self {
Self::FunctionSize => write!(f, "wrong function size"),
Self::Register => write!(f, "wrong register"),
Self::Mnemonic => write!(f, "wrong mnemonic"),
Self::BranchTarget => write!(f, "wrong branch target"),
Self::FunctionCall(diff) => write!(f, "wrong function call\n{}", diff),
Self::DataReference(diff) => write!(f, "wrong data reference\n{}", diff),
Self::Immediate => write!(f, "wrong immediate"),
Self::Unknown => write!(f, "unknown reason; check diff.py"),
}
}
}
#[derive(Debug)]
pub struct Mismatch {
pub addr_orig: u64,
pub addr_decomp: u64,
pub cause: MismatchCause,
}
impl std::fmt::Display for Mismatch {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"mismatch at {}: {}",
ui::format_address(self.addr_orig),
self.cause,
)
}
}
pub struct FunctionChecker<'a, 'functions, 'orig_elf, 'decomp_elf> {
decomp_elf: &'decomp_elf elf::OwnedElf,
decomp_symtab: &'a elf::SymbolTableByName<'decomp_elf>,
decomp_glob_data_table: elf::GlobDataTable,
// Optional, only initialized when a mismatch is detected.
decomp_addr_to_name_map: Lazy<elf::AddrToNameMap<'decomp_elf>>,
known_data_symbols: KnownDataSymbolMap,
known_functions: FxHashMap<u64, &'functions functions::Info>,
orig_elf: &'orig_elf elf::OwnedElf,
orig_got_section: &'orig_elf goblin::elf::SectionHeader,
}
impl<'a, 'functions, 'orig_elf, 'decomp_elf>
FunctionChecker<'a, 'functions, 'orig_elf, 'decomp_elf>
{
pub fn new(
orig_elf: &'orig_elf elf::OwnedElf,
decomp_elf: &'decomp_elf elf::OwnedElf,
decomp_symtab: &'a elf::SymbolTableByName<'decomp_elf>,
decomp_glob_data_table: elf::GlobDataTable,
functions: &'functions [functions::Info],
) -> Result<Self> {
let mut known_data_symbols = KnownDataSymbolMap::new();
known_data_symbols.load(get_data_symbol_csv_path()?.as_path(), &decomp_symtab)?;
let known_functions = functions::make_known_function_map(functions);
let orig_got_section = elf::find_section(orig_elf, ".got")?;
Ok(FunctionChecker {
decomp_elf,
decomp_symtab,
decomp_glob_data_table,
decomp_addr_to_name_map: Lazy::new(),
known_data_symbols,
known_functions,
orig_elf,
orig_got_section,
})
}
pub fn check(
&self,
cs: &mut cs::Capstone,
orig_fn: &elf::Function,
decomp_fn: &elf::Function,
) -> Result<Option<Mismatch>> {
// Keep track of registers that are used with ADRP so that we can check global data
// references even when data is not placed at the same addresses
// as in the original executable.
#[derive(Default)]
struct State {
gprs1: HashMap<cs::RegId, u64>,
gprs2: HashMap<cs::RegId, u64>,
adrp_pair_registers: HashSet<cs::RegId>,
}
impl State {
fn forget_modified_registers(&mut self, detail: &cs::InsnDetail) {
for reg in detail.regs_write() {
self.adrp_pair_registers.remove(&reg);
}
}
}
let mut state = State::default();
if orig_fn.code.len() != decomp_fn.code.len() {
return Ok(Some(Mismatch {
addr_orig: orig_fn.addr,
addr_decomp: decomp_fn.addr,
cause: MismatchCause::FunctionSize,
}));
}
let mut instructions = try_map_two(&orig_fn, &decomp_fn, |func| {
cs.disasm_iter(func.code, func.addr)
})?;
// Check every pair of instructions.
while let (Some(i1), Some(i2)) = (instructions.0.next(), instructions.1.next()) {
let ids = map_two(&i1, &i2, |i| i.id().0);
let detail = try_map_two(&i1, &i2, |insn| cs.insn_detail(&insn))?;
let arch_detail = map_pair(&detail, |d| d.arch_detail());
let ops = map_pair(&arch_detail, |a| a.arm64().unwrap().operands_ref());
if ids.0 != ids.1 {
return Self::make_mismatch(&i1, &i2, MismatchCause::Mnemonic);
}
let id = ids.0;
match id.into() {
// Branches or function calls.
Arm64Insn::ARM64_INS_B | Arm64Insn::ARM64_INS_BL => {
let target =
map_pair(&ops, |ops| Arm64Operand::from(&ops[0]).op_type.imm() as u64);
// If we are branching outside the function, this is likely a tail call.
// Treat it as a function call.
if !orig_fn.get_addr_range().contains(&target.0) {
if let Some(mismatch_cause) = self.check_function_call(target.0, target.1) {
return Self::make_mismatch(&i1, &i2, mismatch_cause);
}
} else {
// Otherwise, it's a simple branch, and both targets must match.
if i1.bytes() != i2.bytes() {
return Self::make_mismatch(&i1, &i2, MismatchCause::BranchTarget);
}
}
}
// Catch ADRP + (ADD/load/store) instruction pairs.
Arm64Insn::ARM64_INS_ADRP => {
let reg = map_pair(&ops, |ops| Arm64Operand::from(&ops[0]).op_type.reg());
let imm =
map_pair(&ops, |ops| Arm64Operand::from(&ops[1]).op_type.imm() as u64);
if reg.0 != reg.1 {
return Self::make_mismatch(&i1, &i2, MismatchCause::Register);
}
state.gprs1.insert(reg.0, imm.0);
state.gprs2.insert(reg.1, imm.1);
state.adrp_pair_registers.insert(reg.0);
}
// Catch ADRP + ADD instruction pairs.
Arm64Insn::ARM64_INS_ADD => {
let mut diff_ok = false;
if ops.0.len() == 3 {
let dest_reg =
map_pair(&ops, |ops| Arm64Operand::from(&ops[0]).op_type.reg());
let reg = map_pair(&ops, |ops| Arm64Operand::from(&ops[1]).op_type.reg());
if let Arm64OperandType::Imm(_) = Arm64Operand::from(&ops.0[2]).op_type {
let imm =
map_pair(&ops, |ops| Arm64Operand::from(&ops[2]).op_type.imm());
if dest_reg.0 != dest_reg.1 || reg.0 != reg.1 {
return Self::make_mismatch(&i1, &i2, MismatchCause::Register);
}
// Is this an ADRP pair we can check?
if state.adrp_pair_registers.contains(&reg.0) {
let orig_addr = state.gprs1[&reg.0] + imm.0 as u64;
let decomp_addr = state.gprs2[&reg.1] + imm.1 as u64;
if let Some(mismatch_cause) =
self.check_data_symbol(orig_addr, decomp_addr)
{
return Self::make_mismatch(&i1, &i2, mismatch_cause);
}
// If the data symbol reference matches, allow the instructions to be different.
diff_ok = true;
}
}
}
if !diff_ok && i1.bytes() != i2.bytes() {
return Self::make_mismatch(&i1, &i2, MismatchCause::Unknown);
}
state.forget_modified_registers(&detail.0);
}
// Loads and stores (single or paired).
id if is_id_in_range(Arm64Insn::ARM64_INS_LD1, Arm64Insn::ARM64_INS_LDXRH, id)
|| is_id_in_range(Arm64Insn::ARM64_INS_ST1, Arm64Insn::ARM64_INS_STXR, id) =>
{
let mut diff_ok = false;
// Check all operands for mismatches, except the Arm64OpMem which will be checked later.
let mut mem = (None, None);
for (op1, op2) in zip(ops.0, ops.1) {
let op1 = Arm64Operand::from(op1);
let op2 = Arm64Operand::from(op2);
if let Some(mem1) = op1.op_type.try_mem() {
if let Some(mem2) = op2.op_type.try_mem() {
ensure!(
mem.0.is_none() && mem.1.is_none(),
"found more than one OpMem"
);
mem.0 = Some(mem1);
mem.1 = Some(mem2);
continue;
}
}
if op1 != op2 {
return Self::make_mismatch(&i1, &i2, MismatchCause::Unknown);
}
}
ensure!(mem.0.is_some() && mem.1.is_some(), "didn't find an OpMem");
let mem = (mem.0.unwrap(), mem.1.unwrap());
if mem.0.base() != mem.1.base() {
return Self::make_mismatch(&i1, &i2, MismatchCause::Register);
}
let reg = mem.0.base();
// Is this an ADRP pair we can check?
if state.adrp_pair_registers.contains(&reg) {
let orig_addr_ptr = (state.gprs1[&reg] as i64 + mem.0.disp() as i64) as u64;
let decomp_addr_ptr =
(state.gprs2[&reg] as i64 + mem.1.disp() as i64) as u64;
if let Some(mismatch_cause) =
self.check_data_symbol_ptr(orig_addr_ptr, decomp_addr_ptr)
{
return Self::make_mismatch(&i1, &i2, mismatch_cause);
}
// If the data symbol reference matches, allow the instructions to be different.
diff_ok = true;
}
if !diff_ok && i1.bytes() != i2.bytes() {
return Self::make_mismatch(&i1, &i2, MismatchCause::Unknown);
}
state.forget_modified_registers(&detail.0);
}
// Anything else.
_ => {
if i1.bytes() != i2.bytes() {
return Self::make_mismatch(&i1, &i2, MismatchCause::Unknown);
}
state.forget_modified_registers(&detail.0);
}
}
}
Ok(None)
}
/// Returns None on success and a MismatchCause on failure.
fn check_function_call(&self, orig_addr: u64, decomp_addr: u64) -> Option<MismatchCause> {
let info = *self.known_functions.get(&orig_addr)?;
let name = info.name.as_str();
let decomp_symbol = self.decomp_symtab.get(name)?;
let expected = decomp_symbol.st_value;
if decomp_addr == expected {
None
} else {
let actual_symbol_name = self.translate_decomp_addr_to_name(decomp_addr);
Some(MismatchCause::FunctionCall(ReferenceDiff {
referenced_symbol: orig_addr,
expected_ref_in_decomp: expected,
actual_ref_in_decomp: decomp_addr,
expected_symbol_name: name.to_string(),
actual_symbol_name: actual_symbol_name.unwrap_or("unknown").to_string(),
}))
}
}
/// Returns None on success and a MismatchCause on failure.
fn check_data_symbol_ex(
&self,
orig_addr: u64,
decomp_addr: u64,
symbol: &DataSymbol,
) -> Option<MismatchCause> {
let decomp_symbol = self.decomp_symtab.get(symbol.name.as_str())?;
let expected = decomp_symbol.st_value;
if decomp_addr == expected {
None
} else {
let actual_symbol_name = self.translate_decomp_addr_to_name(decomp_addr);
Some(MismatchCause::DataReference(ReferenceDiff {
referenced_symbol: orig_addr,
expected_ref_in_decomp: expected,
actual_ref_in_decomp: decomp_addr,
expected_symbol_name: symbol.name.to_string(),
actual_symbol_name: actual_symbol_name.unwrap_or("unknown").to_string(),
}))
}
}
/// Returns None on success and a MismatchCause on failure.
fn check_data_symbol(&self, orig_addr: u64, decomp_addr: u64) -> Option<MismatchCause> {
let symbol = self.known_data_symbols.get_symbol(orig_addr)?;
self.check_data_symbol_ex(orig_addr, decomp_addr, symbol)
}
/// Returns None on success and a MismatchCause on failure.
/// Unlike check_data_symbol, this function takes the addresses of *pointers to* possible data symbols,
/// not the symbols themselves.
fn check_data_symbol_ptr(
&self,
orig_addr_ptr: u64,
decomp_addr_ptr: u64,
) -> Option<MismatchCause> {
if !elf::is_in_section(&self.orig_got_section, orig_addr_ptr, 8) {
return None;
}
let orig_offset = elf::get_offset_in_file(&self.orig_elf, orig_addr_ptr).ok()? as u64;
let orig_addr = u64::from_le_bytes(
elf::get_elf_bytes(&self.orig_elf, orig_offset, 8)
.ok()?
.try_into()
.ok()?,
);
let data_symbol = self.known_data_symbols.get_symbol(orig_addr)?;
let decomp_addr = *self.decomp_glob_data_table.get(&decomp_addr_ptr)?;
self.check_data_symbol_ex(orig_addr, decomp_addr, &data_symbol)
}
fn make_mismatch(
i1: &cs::Insn,
i2: &cs::Insn,
cause: MismatchCause,
) -> Result<Option<Mismatch>> {
Ok(Some(Mismatch {
addr_orig: i1.address(),
addr_decomp: i2.address(),
cause,
}))
}
#[cold]
#[inline(never)]
fn translate_decomp_addr_to_name(&self, decomp_addr: u64) -> Option<&'decomp_elf str> {
let map = self.decomp_addr_to_name_map.get_or_create(|| {
let map = elf::make_addr_to_name_map(&self.decomp_elf).ok();
map.unwrap_or_default()
});
map.get(&decomp_addr).copied()
}
}

311
tools/smocheck/src/elf.rs Normal file
View file

@ -0,0 +1,311 @@
use std::{collections::HashMap, ffi::CStr, fs::File, ops::Range, path::Path};
use anyhow::{anyhow, bail, Context, Result};
use goblin::{
container,
elf::{
dynamic, reloc, section_header, sym, Dynamic, Elf, ProgramHeader, RelocSection,
SectionHeader, Sym, Symtab,
},
elf64::program_header::PT_LOAD,
strtab::Strtab,
};
use memmap::{Mmap, MmapOptions};
use owning_ref::OwningHandle;
use rustc_hash::FxHashMap;
use crate::repo;
pub type OwnedElf = OwningHandle<Box<(Box<File>, Mmap)>, Box<Elf<'static>>>;
pub type SymbolTableByName<'a> = HashMap<&'a str, goblin::elf::Sym>;
pub type SymbolTableByAddr = FxHashMap<u64, goblin::elf::Sym>;
pub type AddrToNameMap<'a> = FxHashMap<u64, &'a str>;
pub type GlobDataTable = FxHashMap<u64, u64>;
pub struct Function<'a> {
/// The virtual address of the function in its containing executable.
/// *Note*: does not contain the IDA base (0x7100000000).
pub addr: u64,
/// The bytes that make up the code for this function.
pub code: &'a [u8],
}
impl<'a> Function<'a> {
#[inline]
pub fn get_addr_range(&self) -> Range<u64> {
self.addr..(self.addr + self.code.len() as u64)
}
}
#[inline]
fn make_goblin_ctx() -> container::Ctx {
// 64-bit, little endian
container::Ctx::new(container::Container::Big, container::Endian::Little)
}
/// A stripped down version of `goblin::elf::Elf::parse`, parsing only the sections that we need.
///
/// *Warning*: In particular, `strtab`, `dynstrtab`, `soname` and `libraries` are **not** parsed.
fn parse_elf_faster(bytes: &[u8]) -> Result<Elf> {
let header = Elf::parse_header(bytes)?;
let mut elf = Elf::lazy_parse(header)?;
let ctx = make_goblin_ctx();
elf.program_headers =
ProgramHeader::parse(bytes, header.e_phoff as usize, header.e_phnum as usize, ctx)?;
elf.section_headers =
SectionHeader::parse(bytes, header.e_shoff as usize, header.e_shnum as usize, ctx)?;
let get_strtab = |section_headers: &[SectionHeader], section_idx: usize| {
if section_idx >= section_headers.len() {
Ok(Strtab::default())
} else {
let shdr = &section_headers[section_idx];
shdr.check_size(bytes.len())?;
Strtab::parse(bytes, shdr.sh_offset as usize, shdr.sh_size as usize, 0x0)
}
};
let strtab_idx = header.e_shstrndx as usize;
elf.shdr_strtab = get_strtab(&elf.section_headers, strtab_idx)?;
for shdr in &elf.section_headers {
if shdr.sh_type as u32 == section_header::SHT_SYMTAB {
let size = shdr.sh_entsize;
let count = if size == 0 { 0 } else { shdr.sh_size / size };
elf.syms = Symtab::parse(bytes, shdr.sh_offset as usize, count as usize, ctx)?;
}
}
elf.dynamic = Dynamic::parse(bytes, &elf.program_headers, ctx)?;
if let Some(ref dynamic) = elf.dynamic {
let dyn_info = &dynamic.info;
// parse the dynamic relocations
elf.dynrelas = RelocSection::parse(bytes, dyn_info.rela, dyn_info.relasz, true, ctx)?;
elf.dynrels = RelocSection::parse(bytes, dyn_info.rel, dyn_info.relsz, false, ctx)?;
let is_rela = dyn_info.pltrel as u64 == dynamic::DT_RELA;
elf.pltrelocs =
RelocSection::parse(bytes, dyn_info.jmprel, dyn_info.pltrelsz, is_rela, ctx)?;
}
Ok(elf)
}
pub fn load_elf(path: &Path) -> Result<OwnedElf> {
let file = Box::new(File::open(path)?);
let mmap = unsafe { MmapOptions::new().map(&file)? };
OwningHandle::try_new(Box::new((file, mmap)), |pair| unsafe {
let elf = parse_elf_faster(&(*pair).1).with_context(|| "failed to load ELF")?;
Ok(Box::new(elf))
})
}
pub fn load_orig_elf() -> Result<OwnedElf> {
let mut path = repo::get_repo_root()?;
path.push("data");
path.push("main.elf");
load_elf(path.as_path())
}
pub fn load_decomp_elf() -> Result<OwnedElf> {
let mut path = repo::get_repo_root()?;
path.push("build");
path.push("odyssey");
load_elf(path.as_path())
}
struct SymbolStringTable<'elf> {
bytes: &'elf [u8],
}
impl<'elf> SymbolStringTable<'elf> {
pub fn from_elf(elf: &'elf OwnedElf) -> Result<Self> {
let bytes = &*elf.as_owner().1;
for shdr in &elf.section_headers {
if shdr.sh_type as u32 == section_header::SHT_SYMTAB {
let table_hdr = elf
.section_headers
.get(shdr.sh_link as usize)
.ok_or_else(|| anyhow!("symbol string table index out of bounds"))?;
table_hdr.check_size(bytes.len())?;
let start = table_hdr.sh_offset as usize;
let end = start + table_hdr.sh_size as usize;
return Ok(SymbolStringTable {
bytes: &bytes[start..end],
});
}
}
bail!("couldn't find symbol string table")
}
pub fn get_string(&self, offset: usize) -> &'elf str {
unsafe {
std::str::from_utf8_unchecked(
CStr::from_ptr(self.bytes[offset..self.bytes.len()].as_ptr() as *const i8)
.to_bytes(),
)
}
}
}
fn filter_out_useless_syms(sym: &Sym) -> bool {
matches!(
sym.st_type(),
sym::STT_OBJECT | sym::STT_FUNC | sym::STT_COMMON | sym::STT_TLS
)
}
pub fn make_symbol_map_by_name(elf: &OwnedElf) -> Result<SymbolTableByName> {
let mut map = SymbolTableByName::with_capacity_and_hasher(
elf.syms.iter().filter(filter_out_useless_syms).count(),
Default::default(),
);
let strtab = SymbolStringTable::from_elf(&elf)?;
for symbol in elf.syms.iter().filter(filter_out_useless_syms) {
map.entry(strtab.get_string(symbol.st_name))
.or_insert(symbol);
}
Ok(map)
}
pub fn make_symbol_map_by_addr(elf: &OwnedElf) -> SymbolTableByAddr {
let mut map = SymbolTableByAddr::with_capacity_and_hasher(
elf.syms.iter().filter(filter_out_useless_syms).count(),
Default::default(),
);
for symbol in elf.syms.iter().filter(filter_out_useless_syms) {
map.entry(symbol.st_value).or_insert(symbol);
}
map
}
pub fn make_addr_to_name_map(elf: &OwnedElf) -> Result<AddrToNameMap> {
let mut map = AddrToNameMap::with_capacity_and_hasher(
elf.syms.iter().filter(filter_out_useless_syms).count(),
Default::default(),
);
let strtab = SymbolStringTable::from_elf(&elf)?;
for symbol in elf.syms.iter().filter(filter_out_useless_syms) {
map.entry(symbol.st_value)
.or_insert_with(|| strtab.get_string(symbol.st_name));
}
Ok(map)
}
fn parse_symtab<'a>(elf: &'a OwnedElf, shdr: &'a SectionHeader) -> Result<Symtab<'a>> {
let bytes = &elf.as_owner().1;
let size = shdr.sh_entsize;
let count = if size == 0 { 0 } else { shdr.sh_size / size };
let syms = Symtab::parse(
bytes,
shdr.sh_offset as usize,
count as usize,
make_goblin_ctx(),
)?;
Ok(syms)
}
pub fn find_section<'a>(elf: &'a OwnedElf, name: &str) -> Result<&'a SectionHeader> {
elf.section_headers
.iter()
.find(|&header| &elf.shdr_strtab[header.sh_name] == name)
.ok_or_else(|| anyhow!("failed to find {} section", name))
}
pub fn get_linked_section<'a>(
elf: &'a OwnedElf,
shdr: &'a SectionHeader,
) -> Result<&'a SectionHeader> {
elf.section_headers
.get(shdr.sh_link as usize)
.ok_or_else(|| anyhow!("could not get linked section"))
}
#[inline]
pub fn is_in_section(section: &SectionHeader, addr: u64, size: u64) -> bool {
let begin = section.sh_addr;
let end = begin + section.sh_size;
(begin..end).contains(&addr) && (begin..=end).contains(&(addr + size))
}
pub fn build_glob_data_table(elf: &OwnedElf) -> Result<GlobDataTable> {
let section = &elf.dynrelas;
let section_hdr = find_section(elf, ".rela.dyn")?;
// The corresponding symbol table.
let symtab = parse_symtab(elf, get_linked_section(elf, &section_hdr)?)?;
let mut table = GlobDataTable::with_capacity_and_hasher(section.len(), Default::default());
for reloc in section.iter() {
let symbol_value: u64 = symtab
.get(reloc.r_sym)
.ok_or_else(|| anyhow!("invalid symbol index"))?
.st_value;
match reloc.r_type {
reloc::R_AARCH64_GLOB_DAT => {
table.insert(
reloc.r_offset,
(symbol_value as i64 + reloc.r_addend.unwrap()) as u64,
);
}
reloc::R_AARCH64_RELATIVE => {
// FIXME: this should be Delta(S) + A.
table.insert(
reloc.r_offset,
(symbol_value as i64 + reloc.r_addend.unwrap()) as u64,
);
}
_ => (),
}
}
Ok(table)
}
pub fn get_offset_in_file(elf: &OwnedElf, addr: u64) -> Result<usize> {
let addr = addr as usize;
for segment in elf.program_headers.iter() {
if segment.p_type != PT_LOAD {
continue;
}
if segment.vm_range().contains(&addr) {
return Ok(segment.file_range().start + addr - segment.vm_range().start);
}
}
bail!("{:#x} doesn't belong to any segment", addr)
}
pub fn get_elf_bytes(elf: &OwnedElf, addr: u64, size: u64) -> Result<&[u8]> {
let offset = get_offset_in_file(&elf, addr)?;
let size = size as usize;
Ok(&elf.as_owner().1[offset..(offset + size)])
}
pub fn get_function(elf: &OwnedElf, addr: u64, size: u64) -> Result<Function> {
Ok(Function {
addr,
code: get_elf_bytes(&elf, addr, size)?,
})
}
pub fn get_function_by_name<'a>(
elf: &'a OwnedElf,
symbols: &SymbolTableByName,
name: &str,
) -> Result<Function<'a>> {
let symbol = symbols
.get(&name)
.ok_or_else(|| anyhow!("unknown function: {}", name))?;
get_function(&elf, symbol.st_value, symbol.st_size)
}

142
tools/smocheck/src/functions.rs Normal file
View file

@ -0,0 +1,142 @@
use crate::repo;
use anyhow::{bail, ensure, Context, Result};
use rustc_hash::FxHashMap;
use std::path::{Path, PathBuf};
pub enum Status {
Matching,
NonMatchingMinor,
NonMatchingMajor,
NotDecompiled,
Library,
}
pub struct Info {
pub addr: u64,
pub size: u32,
pub name: String,
pub status: Status,
}
impl Info {
pub fn is_decompiled(&self) -> bool {
!matches!(self.status, Status::NotDecompiled | Status::Library)
}
}
pub const ADDRESS_BASE: u64 = 0x71_0000_0000;
fn parse_base_16(value: &str) -> Result<u64> {
if let Some(stripped) = value.strip_prefix("0x") {
Ok(u64::from_str_radix(stripped, 16)?)
} else {
Ok(u64::from_str_radix(value, 16)?)
}
}
pub fn parse_address(value: &str) -> Result<u64> {
Ok(parse_base_16(value)? - ADDRESS_BASE)
}
fn parse_function_csv_entry(record: &csv::StringRecord) -> Result<Info> {
ensure!(record.len() == 4, "invalid record");
let addr = parse_address(&record[0])?;
let status_code = record[1].chars().next();
let size = record[2].parse::<u32>()?;
let decomp_name = record[3].to_string();
let status = match status_code {
Some('m') => Status::NonMatchingMinor,
Some('M') => Status::NonMatchingMajor,
Some('O') => Status::Matching,
Some('U') => Status::NotDecompiled,
Some('L') => Status::Library,
Some(code) => bail!("unexpected status code: {}", code),
None => bail!("missing status code"),
};
Ok(Info {
addr,
size,
name: decomp_name,
status,
})
}
pub fn get_functions_csv_path() -> Result<PathBuf> {
let mut path = repo::get_repo_root()?;
path.push("data");
path.push("odyssey_functions.csv");
Ok(path)
}
/// Returns a Vec of all functions that are listed in the specified CSV.
pub fn get_functions_for_path(csv_path: &Path) -> Result<Vec<Info>> {
let mut reader = csv::ReaderBuilder::new()
.has_headers(false)
.quoting(false)
.from_path(csv_path)?;
// We build the result array manually without using csv iterators for performance reasons.
let mut result = Vec::with_capacity(110_000);
let mut record = csv::StringRecord::new();
let mut line_number = 1;
if reader.read_record(&mut record)? {
// Verify that the CSV has the correct format.
ensure!(record.len() == 4, "invalid record; expected 4 fields");
ensure!(
&record[0] == "Address"
&& &record[1] == "Quality"
&& &record[2] == "Size"
&& &record[3] == "Name",
"wrong CSV format; this program only works with the new function list format (added in commit 1d4c815fbae3)"
);
line_number += 1;
}
while reader.read_record(&mut record)? {
result.push(
parse_function_csv_entry(&record)
.with_context(|| format!("failed to parse CSV record at line {}", line_number))?,
);
line_number += 1;
}
Ok(result)
}
/// Returns a Vec of all known functions in the executable.
pub fn get_functions() -> Result<Vec<Info>> {
get_functions_for_path(get_functions_csv_path()?.as_path())
}
pub fn make_known_function_map(functions: &[Info]) -> FxHashMap<u64, &Info> {
let mut known_functions =
FxHashMap::with_capacity_and_hasher(functions.len(), Default::default());
for function in functions {
if !function.is_decompiled() {
continue;
}
known_functions.insert(function.addr, function);
}
known_functions
}
/// Demangle a C++ symbol.
pub fn demangle_str(name: &str) -> Result<String> {
if !name.starts_with("_Z") {
bail!("not an external mangled name");
}
let symbol = cpp_demangle::Symbol::new(name)?;
let options = cpp_demangle::DemangleOptions::new();
Ok(symbol.demangle(&options)?)
}
pub fn get_expected_dir_path() -> Result<PathBuf> {
let mut path = repo::get_repo_root()?;
path.push("expected");
Ok(path)
}

6
tools/smocheck/src/lib.rs Normal file
View file

@ -0,0 +1,6 @@
pub mod capstone_utils;
pub mod checks;
pub mod elf;
pub mod functions;
pub mod repo;
pub mod ui;

20
tools/smocheck/src/repo.rs Normal file
View file

@ -0,0 +1,20 @@
use anyhow::{bail, Result};
use std::path::PathBuf;
pub fn get_repo_root() -> Result<PathBuf> {
let current_dir = std::env::current_dir()?;
let mut dir = current_dir.as_path();
loop {
if ["data", "src"].iter().all(|name| dir.join(name).is_dir()) {
return Ok(dir.to_path_buf());
}
match dir.parent() {
None => {
bail!("failed to find repo root -- run this program inside the repo");
}
Some(parent) => dir = parent,
};
}
}

180
tools/smocheck/src/tools/check.rs Normal file
View file

@ -0,0 +1,180 @@
use anyhow::bail;
use anyhow::Context;
use anyhow::Result;
use capstone as cs;
use capstone::arch::BuildsCapstone;
use colored::*;
use rayon::prelude::*;
use std::cell::RefCell;
use std::sync::atomic::AtomicBool;
use smocheck::checks::FunctionChecker;
use smocheck::elf;
use smocheck::functions;
use smocheck::functions::Status;
use smocheck::ui;
use mimalloc::MiMalloc;
#[global_allocator]
static GLOBAL: MiMalloc = MiMalloc;
/// Returns false if the program should exit with a failure code at the end.
fn check_function(
checker: &FunctionChecker,
mut cs: &mut capstone::Capstone,
orig_elf: &elf::OwnedElf,
decomp_elf: &elf::OwnedElf,
decomp_symtab: &elf::SymbolTableByName,
function: &functions::Info,
) -> Result<bool> {
if !function.is_decompiled() {
return Ok(true);
}
let name = function.name.as_str();
let decomp_fn = elf::get_function_by_name(&decomp_elf, &decomp_symtab, &name);
if decomp_fn.is_err() {
let error = decomp_fn.err().unwrap();
ui::print_warning(&format!(
"couldn't check {}: {}",
ui::format_symbol_name(name),
error.to_string().dimmed(),
));
return Ok(true);
}
let decomp_fn = decomp_fn.unwrap();
let get_orig_fn = || {
elf::get_function(&orig_elf, function.addr, function.size as u64).with_context(|| {
format!(
"failed to get function {} ({}) from the original executable",
name,
ui::format_address(function.addr),
)
})
};
match function.status {
Status::Matching => {
let orig_fn = get_orig_fn()?;
let result = checker
.check(&mut cs, &orig_fn, &decomp_fn)
.with_context(|| format!("checking {}", name))?;
if let Some(mismatch) = result {
let stderr = std::io::stderr();
let mut lock = stderr.lock();
ui::print_error_ex(
&mut lock,
&format!(
"function {} is marked as matching but does not match",
ui::format_symbol_name(name),
),
);
ui::print_detail_ex(&mut lock, &format!("{}", mismatch));
return Ok(false);
}
}
Status::NonMatchingMinor | Status::NonMatchingMajor => {
let orig_fn = get_orig_fn()?;
let result = checker
.check(&mut cs, &orig_fn, &decomp_fn)
.with_context(|| format!("checking {}", name))?;
if result.is_none() {
ui::print_note(&format!(
"function {} is marked as non-matching but matches",
ui::format_symbol_name(name),
));
}
}
Status::NotDecompiled | Status::Library => unreachable!(),
};
Ok(true)
}
#[cold]
#[inline(never)]
fn make_cs() -> Result<cs::Capstone> {
cs::Capstone::new()
.arm64()
.mode(cs::arch::arm64::ArchMode::Arm)
.detail(true)
.build()
.or_else(smocheck::capstone_utils::translate_cs_error)
}
thread_local! {
static CAPSTONE: RefCell<cs::Capstone> = RefCell::new(make_cs().unwrap());
}
fn main() -> Result<()> {
let orig_elf = elf::load_orig_elf().with_context(|| "failed to load original ELF")?;
let decomp_elf = elf::load_decomp_elf().with_context(|| "failed to load decomp ELF")?;
// Load these in parallel.
let mut decomp_symtab = None;
let mut decomp_glob_data_table = None;
let mut functions = None;
rayon::scope(|s| {
s.spawn(|_| decomp_symtab = Some(elf::make_symbol_map_by_name(&decomp_elf)));
s.spawn(|_| decomp_glob_data_table = Some(elf::build_glob_data_table(&decomp_elf)));
s.spawn(|_| functions = Some(functions::get_functions()));
});
let decomp_symtab = decomp_symtab
.unwrap()
.with_context(|| "failed to make symbol map")?;
let decomp_glob_data_table = decomp_glob_data_table
.unwrap()
.with_context(|| "failed to make global data table")?;
let functions = functions
.unwrap()
.with_context(|| "failed to load function CSV")?;
let checker = FunctionChecker::new(
&orig_elf,
&decomp_elf,
&decomp_symtab,
decomp_glob_data_table,
&functions,
)
.with_context(|| "failed to construct FunctionChecker")?;
let failed = AtomicBool::new(false);
functions.par_iter().try_for_each(|function| {
CAPSTONE.with(|cs| -> Result<()> {
let mut cs = cs.borrow_mut();
let ok = check_function(
&checker,
&mut cs,
&orig_elf,
&decomp_elf,
&decomp_symtab,
function,
)?;
if !ok {
failed.store(true, std::sync::atomic::Ordering::Relaxed);
}
Ok(())
})
})?;
if failed.load(std::sync::atomic::Ordering::Relaxed) {
bail!("found at least one error");
} else {
Ok(())
}
}

59
tools/smocheck/src/ui.rs Normal file
View file

@ -0,0 +1,59 @@
use colored::*;
use std::io::StderrLock;
use std::io::Write;
use textwrap::indent;
use crate::functions;
pub fn print_note(msg: &str) {
eprintln!("{}{}{}", "note".bold().cyan(), ": ".bold(), msg.bold())
}
pub fn print_warning(msg: &str) {
eprintln!("{}{}{}", "warning".bold().yellow(), ": ".bold(), msg.bold())
}
pub fn print_error(msg: &str) {
let stderr = std::io::stderr();
let mut lock = stderr.lock();
print_error_ex(&mut lock, msg);
}
pub fn print_error_ex(lock: &mut StderrLock, msg: &str) {
writeln!(
lock,
"{}{}{}",
"error".bold().red(),
": ".bold(),
msg.bold()
)
.unwrap();
}
pub fn format_symbol_name(name: &str) -> String {
functions::demangle_str(name).map_or(name.blue().to_string(), |demangled| {
format!("{} ({})", demangled.blue(), name.blue().dimmed(),)
})
}
pub fn format_address(addr: u64) -> String {
format!("{:#x}", addr).green().to_string()
}
pub fn print_detail(msg: &str) {
let stderr = std::io::stderr();
let mut lock = stderr.lock();
print_detail_ex(&mut lock, msg);
}
pub fn print_detail_ex(lock: &mut StderrLock, msg: &str) {
writeln!(
lock,
"{}\n",
indent(
&msg.clear().to_string(),
&" │ ".bold().dimmed().to_string()
)
)
.unwrap();
}

35
tools/util/utils.py
View file

@ -28,32 +28,31 @@ class FunctionInfo(tp.NamedTuple):
name: str
size: int
decomp_name: str
library: bool
status: FunctionStatus
raw_row: tp.List[str]
_markers = {
"?": FunctionStatus.Equivalent,
"!": FunctionStatus.NonMatching,
"|": FunctionStatus.Wip,
"O": FunctionStatus.Matching,
"m": FunctionStatus.Equivalent,
"M": FunctionStatus.NonMatching,
"W": FunctionStatus.Wip,
"U": FunctionStatus.NotDecompiled,
"L": FunctionStatus.NotDecompiled,
}
def parse_function_csv_entry(row) -> FunctionInfo:
ea, name, size, decomp_name = row
if decomp_name:
status = FunctionStatus.Matching
ea, stat, size, name = row
status = _markers.get(stat, FunctionStatus.NotDecompiled)
decomp_name = ""
for marker, new_status in _markers.items():
if decomp_name[-1] == marker:
status = new_status
decomp_name = decomp_name[:-1]
break
else:
status = FunctionStatus.NotDecompiled
if status != FunctionStatus.NotDecompiled:
decomp_name = name
addr = int(ea, 16) - 0x7100000000
return FunctionInfo(addr, name, int(size, 0), decomp_name, status, row)
return FunctionInfo(addr, name, int(size), decomp_name, stat == "L", status, row)
def get_functions_csv_path() -> Path:
@ -65,11 +64,13 @@ def get_functions(path: tp.Optional[Path] = None) -> tp.Iterable[FunctionInfo]:
path = get_functions_csv_path()
with path.open() as f:
reader = csv.reader(f)
# Skip headers
next(reader)
for row in reader:
try:
entry = parse_function_csv_entry(row)
# excluded library function
if entry.decomp_name == "l":
if entry.library:
continue
yield entry
except ValueError as e:
@ -82,7 +83,7 @@ def add_decompiled_functions(new_matches: tp.Dict[int, str],
writer = csv.writer(buffer, lineterminator="\n")
for func in get_functions():
if new_orig_names is not None and func.status == FunctionStatus.NotDecompiled and func.addr in new_orig_names:
func.raw_row[1] = new_orig_names[func.addr]
func.raw_row[3] = new_orig_names[func.addr]
if func.status == FunctionStatus.NotDecompiled and func.addr in new_matches:
func.raw_row[3] = new_matches[func.addr]
writer.writerow(func.raw_row)
@ -125,4 +126,4 @@ def fail(msg: str, prefix: str = ""):
def get_repo_root() -> Path:
return Path(__file__).parent.parent.parent
return Path(__file__).parent.parent.parent