mirror of
https://github.com/MonsterDruide1/OdysseyDecomp
synced 2026-04-23 09:04:21 +00:00
workflow: Generate progress report for decomp.dev (#698)
This commit is contained in:
MonsterDruide1
GitHub
parent
165aaade29
commit
b15b715c12
114
.github/files/decomp-dev/report.proto
vendored
Normal file
114
.github/files/decomp-dev/report.proto
vendored
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
// From https://github.com/encounter/objdiff/blob/main/objdiff-core/protos/report.proto
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package objdiff.report;
|
||||
|
||||
// Project progress report
|
||||
message Report {
|
||||
// Overall progress info
|
||||
Measures measures = 1;
|
||||
// Units within this report
|
||||
repeated ReportUnit units = 2;
|
||||
// Report version
|
||||
uint32 version = 3;
|
||||
// Progress categories
|
||||
repeated ReportCategory categories = 4;
|
||||
}
|
||||
|
||||
// Progress info for a report or unit
|
||||
message Measures {
|
||||
// Overall match percent, including partially matched functions and data
|
||||
float fuzzy_match_percent = 1;
|
||||
// Total size of code in bytes
|
||||
uint64 total_code = 2;
|
||||
// Fully matched code size in bytes
|
||||
uint64 matched_code = 3;
|
||||
// Fully matched code percent
|
||||
float matched_code_percent = 4;
|
||||
// Total size of data in bytes
|
||||
uint64 total_data = 5;
|
||||
// Fully matched data size in bytes
|
||||
uint64 matched_data = 6;
|
||||
// Fully matched data percent
|
||||
float matched_data_percent = 7;
|
||||
// Total number of functions
|
||||
uint32 total_functions = 8;
|
||||
// Fully matched functions
|
||||
uint32 matched_functions = 9;
|
||||
// Fully matched functions percent
|
||||
float matched_functions_percent = 10;
|
||||
// Completed (or "linked") code size in bytes
|
||||
uint64 complete_code = 11;
|
||||
// Completed (or "linked") code percent
|
||||
float complete_code_percent = 12;
|
||||
// Completed (or "linked") data size in bytes
|
||||
uint64 complete_data = 13;
|
||||
// Completed (or "linked") data percent
|
||||
float complete_data_percent = 14;
|
||||
// Total number of units
|
||||
uint32 total_units = 15;
|
||||
// Completed (or "linked") units
|
||||
uint32 complete_units = 16;
|
||||
}
|
||||
|
||||
message ReportCategory {
|
||||
// The ID of the category
|
||||
string id = 1;
|
||||
// The name of the category
|
||||
string name = 2;
|
||||
// Progress info for this category
|
||||
Measures measures = 3;
|
||||
}
|
||||
|
||||
// A unit of the report (usually a translation unit)
|
||||
message ReportUnit {
|
||||
// The name of the unit
|
||||
string name = 1;
|
||||
// Progress info for this unit
|
||||
Measures measures = 2;
|
||||
// Sections within this unit
|
||||
repeated ReportItem sections = 3;
|
||||
// Functions within this unit
|
||||
repeated ReportItem functions = 4;
|
||||
// Extra metadata for this unit
|
||||
optional ReportUnitMetadata metadata = 5;
|
||||
}
|
||||
|
||||
// Extra metadata for a unit
|
||||
message ReportUnitMetadata {
|
||||
// Whether this unit is marked as complete (or "linked")
|
||||
optional bool complete = 1;
|
||||
// The name of the module this unit belongs to
|
||||
optional string module_name = 2;
|
||||
// The ID of the module this unit belongs to
|
||||
optional uint32 module_id = 3;
|
||||
// The path to the source file of this unit
|
||||
optional string source_path = 4;
|
||||
// Progress categories for this unit
|
||||
repeated string progress_categories = 5;
|
||||
// Whether this unit is automatically generated (not user-provided)
|
||||
optional bool auto_generated = 6;
|
||||
}
|
||||
|
||||
// A section or function within a unit
|
||||
message ReportItem {
|
||||
// The name of the item
|
||||
string name = 1;
|
||||
// The size of the item in bytes
|
||||
uint64 size = 2;
|
||||
// The overall match percent for this item
|
||||
float fuzzy_match_percent = 3;
|
||||
// Extra metadata for this item
|
||||
optional ReportItemMetadata metadata = 4;
|
||||
// Address of the item (section-relative offset)
|
||||
optional uint64 address = 5;
|
||||
}
|
||||
|
||||
// Extra metadata for an item
|
||||
message ReportItemMetadata {
|
||||
// The demangled name of the function
|
||||
optional string demangled_name = 1;
|
||||
// The virtual address of the function or section
|
||||
optional uint64 virtual_address = 2;
|
||||
}
|
||||
48
.github/files/decomp-dev/report_pb2.py
vendored
Normal file
48
.github/files/decomp-dev/report_pb2.py
vendored
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# NO CHECKED-IN PROTOBUF GENCODE
|
||||
# source: report.proto
|
||||
# Protobuf Python Version: 6.31.1
|
||||
"""Generated protocol buffer code."""
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import descriptor_pool as _descriptor_pool
|
||||
from google.protobuf import runtime_version as _runtime_version
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf.internal import builder as _builder
|
||||
_runtime_version.ValidateProtobufRuntimeVersion(
|
||||
_runtime_version.Domain.PUBLIC,
|
||||
6,
|
||||
31,
|
||||
1,
|
||||
'',
|
||||
'report.proto'
|
||||
)
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0creport.proto\x12\x0eobjdiff.report\"\xa4\x01\n\x06Report\x12*\n\x08measures\x18\x01 \x01(\x0b\x32\x18.objdiff.report.Measures\x12)\n\x05units\x18\x02 \x03(\x0b\x32\x1a.objdiff.report.ReportUnit\x12\x0f\n\x07version\x18\x03 \x01(\r\x12\x32\n\ncategories\x18\x04 \x03(\x0b\x32\x1e.objdiff.report.ReportCategory\"\xa7\x03\n\x08Measures\x12\x1b\n\x13\x66uzzy_match_percent\x18\x01 \x01(\x02\x12\x12\n\ntotal_code\x18\x02 \x01(\x04\x12\x14\n\x0cmatched_code\x18\x03 \x01(\x04\x12\x1c\n\x14matched_code_percent\x18\x04 \x01(\x02\x12\x12\n\ntotal_data\x18\x05 \x01(\x04\x12\x14\n\x0cmatched_data\x18\x06 \x01(\x04\x12\x1c\n\x14matched_data_percent\x18\x07 \x01(\x02\x12\x17\n\x0ftotal_functions\x18\x08 \x01(\r\x12\x19\n\x11matched_functions\x18\t \x01(\r\x12!\n\x19matched_functions_percent\x18\n \x01(\x02\x12\x15\n\rcomplete_code\x18\x0b \x01(\x04\x12\x1d\n\x15\x63omplete_code_percent\x18\x0c \x01(\x02\x12\x15\n\rcomplete_data\x18\r \x01(\x04\x12\x1d\n\x15\x63omplete_data_percent\x18\x0e \x01(\x02\x12\x13\n\x0btotal_units\x18\x0f \x01(\r\x12\x16\n\x0e\x63omplete_units\x18\x10 \x01(\r\"V\n\x0eReportCategory\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12*\n\x08measures\x18\x03 \x01(\x0b\x32\x18.objdiff.report.Measures\"\xeb\x01\n\nReportUnit\x12\x0c\n\x04name\x18\x01 \x01(\t\x12*\n\x08measures\x18\x02 \x01(\x0b\x32\x18.objdiff.report.Measures\x12,\n\x08sections\x18\x03 \x03(\x0b\x32\x1a.objdiff.report.ReportItem\x12-\n\tfunctions\x18\x04 \x03(\x0b\x32\x1a.objdiff.report.ReportItem\x12\x39\n\x08metadata\x18\x05 \x01(\x0b\x32\".objdiff.report.ReportUnitMetadataH\x00\x88\x01\x01\x42\x0b\n\t_metadata\"\xff\x01\n\x12ReportUnitMetadata\x12\x15\n\x08\x63omplete\x18\x01 \x01(\x08H\x00\x88\x01\x01\x12\x18\n\x0bmodule_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tmodule_id\x18\x03 \x01(\rH\x02\x88\x01\x01\x12\x18\n\x0bsource_path\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1b\n\x13progress_categories\x18\x05 \x03(\t\x12\x1b\n\x0e\x61uto_generated\x18\x06 \x01(\x08H\x04\x88\x01\x01\x42\x0b\n\t_completeB\x0e\n\x0c_module_nameB\x0c\n\n_module_idB\x0e\n\x0c_source_pathB\x11\n\x0f_auto_generated\"\xaf\x01\n\nReportItem\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04size\x18\x02 \x01(\x04\x12\x1b\n\x13\x66uzzy_match_percent\x18\x03 \x01(\x02\x12\x39\n\x08metadata\x18\x04 \x01(\x0b\x32\".objdiff.report.ReportItemMetadataH\x00\x88\x01\x01\x12\x14\n\x07\x61\x64\x64ress\x18\x05 \x01(\x04H\x01\x88\x01\x01\x42\x0b\n\t_metadataB\n\n\x08_address\"v\n\x12ReportItemMetadata\x12\x1b\n\x0e\x64\x65mangled_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1c\n\x0fvirtual_address\x18\x02 \x01(\x04H\x01\x88\x01\x01\x42\x11\n\x0f_demangled_nameB\x12\n\x10_virtual_addressb\x06proto3')
|
||||
|
||||
_globals = globals()
|
||||
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
||||
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'report_pb2', _globals)
|
||||
if not _descriptor._USE_C_DESCRIPTORS:
|
||||
DESCRIPTOR._loaded_options = None
|
||||
_globals['_REPORT']._serialized_start=33
|
||||
_globals['_REPORT']._serialized_end=197
|
||||
_globals['_MEASURES']._serialized_start=200
|
||||
_globals['_MEASURES']._serialized_end=623
|
||||
_globals['_REPORTCATEGORY']._serialized_start=625
|
||||
_globals['_REPORTCATEGORY']._serialized_end=711
|
||||
_globals['_REPORTUNIT']._serialized_start=714
|
||||
_globals['_REPORTUNIT']._serialized_end=949
|
||||
_globals['_REPORTUNITMETADATA']._serialized_start=952
|
||||
_globals['_REPORTUNITMETADATA']._serialized_end=1207
|
||||
_globals['_REPORTITEM']._serialized_start=1210
|
||||
_globals['_REPORTITEM']._serialized_end=1385
|
||||
_globals['_REPORTITEMMETADATA']._serialized_start=1387
|
||||
_globals['_REPORTITEMMETADATA']._serialized_end=1505
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
149
.github/scripts/decomp-dev.py
vendored
Normal file
149
.github/scripts/decomp-dev.py
vendored
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import yaml
|
||||
from functools import reduce
|
||||
import cxxfilt
|
||||
|
||||
import sys, os
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "tools"))
|
||||
sys.path.append(os.path.join(os.path.dirname(__file__), "..", "files", "decomp-dev"))
|
||||
|
||||
from report_pb2 import Report, Measures, ReportUnit, ReportUnitMetadata, ReportItem, ReportItemMetadata
|
||||
|
||||
data = yaml.load(open("data/file_list.yml"), Loader=yaml.CLoader)
|
||||
|
||||
def get_measures(unit_name=None):
|
||||
funcs = data[unit_name][".text"] if unit_name else reduce(lambda a,b: a+b, [unit[".text"] for unit in data.values()])
|
||||
|
||||
total_units = len(data)
|
||||
total_code = sum([fun["size"] for fun in funcs])
|
||||
total_functions = len(funcs)
|
||||
|
||||
matched_code = sum([fun["size"] for fun in funcs if fun["status"] == "Matching"])
|
||||
matched_functions = len([fun for fun in funcs if fun["status"] == "Matching"])
|
||||
matched_code_percent = matched_code / total_code * 100 if total_code > 0 else 0.0
|
||||
matched_functions_percent = matched_functions / total_functions * 100 if total_functions > 0 else 0.0
|
||||
|
||||
fuzzy_code = sum([fun["size"] for fun in funcs if fun["status"] != "NotDecompiled"])
|
||||
fuzzy_code_percent = fuzzy_code / total_code * 100 if total_code > 0 else 0.0
|
||||
fuzzy_match_percent = fuzzy_code_percent * 0.5 # TODO: 0.5 for data being missing entirely
|
||||
|
||||
# TODO: we don't handle data or linking yet, so 0 for all of those
|
||||
total_data = 0
|
||||
matched_data = 0
|
||||
matched_data_percent = 0.0
|
||||
complete_code = 0
|
||||
complete_code_percent = 0.0
|
||||
complete_data = 0
|
||||
complete_data_percent = 0.0
|
||||
complete_units = 0
|
||||
|
||||
measures = Measures()
|
||||
measures.fuzzy_match_percent = fuzzy_match_percent
|
||||
measures.total_code = total_code
|
||||
measures.matched_code = matched_code
|
||||
measures.matched_code_percent = matched_code_percent
|
||||
measures.total_data = total_data
|
||||
measures.matched_data = matched_data
|
||||
measures.matched_data_percent = matched_data_percent
|
||||
measures.total_functions = total_functions
|
||||
measures.matched_functions = matched_functions
|
||||
measures.matched_functions_percent = matched_functions_percent
|
||||
measures.complete_code = complete_code
|
||||
measures.complete_code_percent = complete_code_percent
|
||||
measures.complete_data = complete_data
|
||||
measures.complete_data_percent = complete_data_percent
|
||||
measures.total_units = total_units
|
||||
measures.complete_units = complete_units
|
||||
return measures
|
||||
|
||||
def get_functions(unit_name):
|
||||
funcs = data[unit_name][".text"]
|
||||
functions = []
|
||||
|
||||
# TODO: determine actual percentage based on tools/check?
|
||||
def get_function_fuzzy_match_percent(fun):
|
||||
if fun["status"] == "Matching":
|
||||
return 100.0
|
||||
elif fun["status"] == "NonMatchingMinor":
|
||||
return 75.0
|
||||
elif fun["status"] == "NonMatchingMajor":
|
||||
return 25.0
|
||||
else:
|
||||
return 0.0
|
||||
|
||||
def get_function_demangled_name(name):
|
||||
try:
|
||||
return cxxfilt.demangle(name)
|
||||
except cxxfilt.InvalidName:
|
||||
return name
|
||||
|
||||
for fun in funcs:
|
||||
function = ReportItem()
|
||||
function.name = fun["label"] if isinstance(fun["label"], str) else fun["label"][0]
|
||||
function.size = fun["size"]
|
||||
function.fuzzy_match_percent = get_function_fuzzy_match_percent(fun)
|
||||
|
||||
metadata = ReportItemMetadata()
|
||||
metadata.demangled_name = get_function_demangled_name(function.name)
|
||||
metadata.virtual_address = fun["offset"]
|
||||
|
||||
function.metadata.CopyFrom(metadata)
|
||||
function.address = fun["offset"]
|
||||
functions.append(function)
|
||||
|
||||
return functions
|
||||
|
||||
def get_units():
|
||||
units = []
|
||||
for unit_name, unit_data in data.items():
|
||||
unit = ReportUnit()
|
||||
unit.name = unit_name
|
||||
unit.measures.CopyFrom(get_measures(unit_name))
|
||||
unit.sections.extend([]) # TODO: no splitting by sections yet
|
||||
unit.functions.extend(get_functions(unit_name))
|
||||
|
||||
metadata = ReportUnitMetadata()
|
||||
metadata.complete = False # TODO: no linking yet
|
||||
metadata.module_name = "/".join(unit_name.split("/")[:-1])
|
||||
metadata.module_id = hash(metadata.module_name) & 0xffffffff
|
||||
metadata.source_path = unit_name
|
||||
metadata.progress_categories.extend([]) # TODO: no progress categories yet
|
||||
metadata.auto_generated = False
|
||||
|
||||
unit.metadata.CopyFrom(metadata)
|
||||
units.append(unit)
|
||||
|
||||
return units
|
||||
|
||||
def build_report():
|
||||
report = Report()
|
||||
report.measures.CopyFrom(get_measures())
|
||||
report.units.extend(get_units())
|
||||
report.version = 1
|
||||
report.categories.extend([])
|
||||
return report
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--version", help="Specify which game version to generate the report for")
|
||||
parser.add_argument("--output", "-o", help="Output JSON file")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.version != "1.0":
|
||||
print(f"Unsupported version: {args.version}. Only '1.0' is supported.")
|
||||
sys.exit(1)
|
||||
|
||||
report = build_report()
|
||||
|
||||
from google.protobuf.json_format import MessageToJson
|
||||
json_str = MessageToJson(report, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, "w") as f:
|
||||
f.write(json_str)
|
||||
else:
|
||||
print(json_str)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
32
.github/workflows/decomp-dev.yml
vendored
Normal file
32
.github/workflows/decomp-dev.yml
vendored
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
name: decomp-dev
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
publish_progress_decomp_dev:
|
||||
if: github.repository == 'MonsterDruide1/OdysseyDecomp'
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
version: ["1.0"]
|
||||
steps:
|
||||
- name: Check out project
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: Set up python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
cache: 'pip'
|
||||
- name: Set up python package dependencies
|
||||
run: pip install cxxfilt pyyaml protobuf
|
||||
- name: Run report generator for decomp.dev
|
||||
run: python .github/scripts/decomp-dev.py --version ${{ matrix.version }} -o report.json
|
||||
- name: Upload report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ matrix.version }}_report
|
||||
path: report.json
|
||||
Loading…
Reference in a new issue