Setup and building: Use clang, lld and viking from cache repo (#348)

.devcontainer/Dockerfile

@@ -11,12 +11,11 @@ RUN apt install -y tzdata
 RUN apt install -y git
 
 # install dependencies for building and running the project
-RUN apt install -y ccache clang cmake curl less libncurses6 libssl-dev ninja-build pip pkg-config python3-full xdelta3
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+RUN apt install -y ccache cmake curl less libncurses6 libssl-dev ninja-build pip pkg-config python3
 RUN pip install --break-system-packages capstone colorama cxxfilt pyelftools python-Levenshtein toml watchdog
 
 # install dependencies for code environment
-RUN apt install -y clangd clang-format clang-tidy
+RUN apt install -y clangd clang-format
 
 # install (outdated) libtinfo5, required for old clang version
 RUN curl -o libtinfo5_6.3-2_amd64.deb http://archive.ubuntu.com/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2_amd64.deb && dpkg -i libtinfo5_6.3-2_amd64.deb && rm -f libtinfo5_6.3-2_amd64.deb

.github/workflows/compile-check.yml

@@ -41,21 +41,17 @@ jobs:
         cache: 'pip'
     - name: Set up python package dependencies
       run: pip install capstone colorama cxxfilt pyelftools watchdog python-Levenshtein toml
-    - name: Set up rust toolchain
-      uses: actions-rust-lang/setup-rust-toolchain@v1.10.1
-      with:
-        cache: false
-    - name: Set up rust caching
-      uses: Swatinem/rust-cache@v2
-      with:
-        workspaces: "tools/common/viking"
-        cache-directories: |
-          toolchain/clang-3.9.1
-          toolchain/clang-4.0.1
     - name: Download main.nso from secret
       env:
         EXEFS_SHARED_PASS: ${{ secrets.EXEFS_SHARED_PASS }}
       run: curl -u "github-odyssey:$EXEFS_SHARED_PASS" https://monsterdruide.one/secrets/smo-main.nso -O
+    - name: Set up cache for toolchain
+      uses: actions/cache@v4
+      with:
+        key: tools-libs
+        path: |
+          toolchain/bin
+          toolchain/include
     - name: Run setup
       run: tools/setup.py smo-main.nso
     - name: Build project

.github/workflows/test-compile.yml

@@ -20,13 +20,13 @@ jobs:
         cache: 'pip'
     - name: Set up python package dependencies
       run: pip install toml
-    - name: Set up cache for clang
+    - name: Set up cache for toolchain
       uses: actions/cache@v4
       with:
-        key: clang391-401
+        key: tools-libs
         path: |
-          toolchain/clang-3.9.1
-          toolchain/clang-4.0.1
+          toolchain/bin
+          toolchain/include
     - name: Run simplified setup
       run: tools/setup.py --project
     - name: Create testing source files

.gitignore

@@ -50,6 +50,8 @@ perf.data.old
 
 # Tooling
 /toolchain/clang-*
+toolchain/include
+toolchain/cache-version-url.txt
 tools/check
 tools/listsym
 tools/decompme

README.md

@@ -48,13 +48,11 @@ All other systems have to manually install the required packages and programs. W
     * If you are on Ubuntu 18.04, you must
       first [update CMake by using the official CMake APT repository](https://apt.kitware.com/).
 * ccache (to speed up builds)
-* xdelta3
-* clang (not for compiling SMO code, but for compiling Rust tools)
 
 Ubuntu users can install those dependencies by running:
 
 ```shell
-sudo apt install python3 ninja-build cmake ccache xdelta3 clang libssl-dev libncurses5
+sudo apt install python3 ninja-build cmake ccache libssl-dev libncurses5
 ```
 
 If you are running Ubuntu 23.10 or later, the `libncurses5` package won't be available anymore. You can install it from
@@ -90,10 +88,7 @@ Additionally, you'll also need:
     * This will:
         * install tools/check to check for differences in decompiled code
         * convert the executable if necessary
-        * set up [Clang 3.9.1](https://releases.llvm.org/download.html#3.9.1) by downloading it from the official LLVM
-          website
-        * set up [Clang 4.0.1](https://releases.llvm.org/download.html#4.0.1) by downloading it from the official LLVM
-          website
+        * set up clang, ld.lld and other tools by downloading them from the releases of [OdysseyDecompToolsCache](https://github.com/MonsterDruide1/OdysseyDecompToolsCache/)
         * create a build directory in `build/`
     * If something goes wrong, follow the instructions given to you by the script.
     * If you wish to use a CMake generator that isn't Ninja, use `--cmake_backend` to specify it.

flake.lock

@@ -20,11 +20,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1746123514,
-        "narHash": "sha256-UNO+MbVHLl4AkVWYqekk72/gqFNSLYNkBgto7h+7P3U=",
+        "lastModified": 1747467164,
+        "narHash": "sha256-JBXbjJ0t6T6BbVc9iPVquQI9XSXCGQJD8c8SgnUquus=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "b5dd9efc912ecabeafa4f082d31e19cb1c74266c",
+        "rev": "3fcbdcfc707e0aa42c541b7743e05820472bdaec",
         "type": "github"
       },
       "original": {
@@ -49,48 +49,13 @@
         "type": "github"
       }
     },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1744536153,
-        "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "18dd725c29603f582cf1900e0d25f9f1063dbf11",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
         "flake-parts": "flake-parts",
         "nixpkgs": "nixpkgs",
-        "rust-overlay": "rust-overlay",
         "systems": "systems"
       }
     },
-    "rust-overlay": {
-      "inputs": {
-        "nixpkgs": "nixpkgs_2"
-      },
-      "locked": {
-        "lastModified": 1746153335,
-        "narHash": "sha256-vwKelhJJS8haCdH3t8uf96VFao7/YzJahPG5JLTO1PU=",
-        "owner": "oxalica",
-        "repo": "rust-overlay",
-        "rev": "ebc7823c3ffde594c7733113042b72694d996de9",
-        "type": "github"
-      },
-      "original": {
-        "owner": "oxalica",
-        "repo": "rust-overlay",
-        "type": "github"
-      }
-    },
     "systems": {
       "locked": {
         "lastModified": 1681028828,

flake.nix

@@ -3,47 +3,43 @@
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     flake-parts.url = "github:hercules-ci/flake-parts";
     systems.url = "github:nix-systems/default";
-    rust-overlay.url = "github:oxalica/rust-overlay";
   };
 
   outputs = inputs: inputs.flake-parts.lib.mkFlake { inherit inputs; } {
     systems = import inputs.systems;
-    perSystem = { config, self', inputs', pkgs, system, ... }: let
-      overlays = [ (import inputs.rust-overlay) ];
-      pkgs = import inputs.nixpkgs {
-        inherit system overlays;
-      };
-      rustToolchain = pkgs.pkgsBuildHost.rust-bin.stable.latest.default;
-    in {
-      _module.args.pkgs = import inputs.nixpkgs {
-        config.allowUnfree = true;
-      };
-      devShells.default = pkgs.mkShell rec {
-        buildInputs = with pkgs; [
-          cmake
-          ninja
-          llvmPackages_18.clang
-          ccache
-          pkg-config
+    perSystem = { config, self', inputs', pkgs, system, ... }:
+      let
+        pkgs = import inputs.nixpkgs {
+          inherit system;
+        };
+      in
+      {
+        _module.args.pkgs = import inputs.nixpkgs {
+          config.allowUnfree = true;
+        };
+        devShells.default = pkgs.mkShell rec {
+          buildInputs = with pkgs; [
+            cmake
+            ninja
+            llvmPackages_18.clang-tools
+            ccache
+            pkg-config
 
-          rustToolchain
-          (python3.withPackages (python-pkgs: [
-            python-pkgs.capstone
-            python-pkgs.colorama
-            python-pkgs.cxxfilt
-            python-pkgs.pyelftools
-            python-pkgs.watchdog
-            python-pkgs.python-Levenshtein
-            python-pkgs.toml
-          ]))
-          openssl
-          llvmPackages_18.libclang
-          ncurses5
-          ncurses6
-        ];
-        LD_LIBRARY_PATH = "${pkgs.lib.makeLibraryPath buildInputs}";
-        LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
+            (python311.withPackages (python-pkgs: [
+              python-pkgs.capstone
+              python-pkgs.colorama
+              python-pkgs.cxxfilt
+              python-pkgs.pyelftools
+              python-pkgs.watchdog
+              python-pkgs.python-Levenshtein
+              python-pkgs.toml
+            ]))
+            openssl
+            ncurses5
+            ncurses6
+          ];
+          LD_LIBRARY_PATH = "${pkgs.lib.makeLibraryPath buildInputs}";
+        };
       };
-    };
   };
 }

toolchain/ToolchainNX64.cmake

@@ -1,13 +1,23 @@
+if (NOT HAS_WARNED_TOOLCHAIN AND (EXISTS "${CMAKE_CURRENT_LIST_DIR}/clang-3.9.1" OR EXISTS "${CMAKE_CURRENT_LIST_DIR}/clang-4.0.1"))
+    message(WARNING "Full clang toolchains are no longer needed to build the project, please rerun setup.py!")
+    # Only show this warning the first time cmake is reran by build.py
+    set(HAS_WARNED_TOOLCHAIN 1 CACHE STRING "")
+endif()
+
 if (DEFINED ENV{ODYSSEY_CLANG})
     set(ODYSSEY_CLANG "$ENV{ODYSSEY_CLANG}")
+elseif(EXISTS "${CMAKE_CURRENT_LIST_DIR}/bin/clang")
+    set(ODYSSEY_CLANG "${CMAKE_CURRENT_LIST_DIR}/bin/clang")
 else()
-    set(ODYSSEY_CLANG "${CMAKE_CURRENT_LIST_DIR}/clang-3.9.1")
+    set(ODYSSEY_CLANG "${CMAKE_CURRENT_LIST_DIR}/clang-3.9.1/bin/clang")
 endif()
 
 if (DEFINED ENV{ODYSSEY_CLANG_LLD})
     set(ODYSSEY_CLANG_LLD "$ENV{ODYSSEY_CLANG_LLD}")
+elseif(EXISTS "${CMAKE_CURRENT_LIST_DIR}/bin/ld.lld")
+    set(ODYSSEY_CLANG_LLD "${CMAKE_CURRENT_LIST_DIR}/bin/ld.lld")
 else()
-    set(ODYSSEY_CLANG_LLD "${CMAKE_CURRENT_LIST_DIR}/clang-4.0.1")
+    set(ODYSSEY_CLANG_LLD "${CMAKE_CURRENT_LIST_DIR}/clang-4.0.1/bin/ld.lld")
 endif()
 
 set(NX64_OPT_FLAGS "-O3 -g")
@@ -18,9 +28,9 @@ set(CMAKE_SYSTEM_VERSION 1)
 set(CMAKE_SYSTEM_PROCESSOR aarch64)
 
 set(CMAKE_SYSROOT ${CMAKE_CURRENT_LIST_DIR}/musl)
-set(CMAKE_C_COMPILER "${ODYSSEY_CLANG}/bin/clang")
+set(CMAKE_C_COMPILER "${ODYSSEY_CLANG}")
 set(CMAKE_C_COMPILER_TARGET ${NX64_TRIPLE})
-set(CMAKE_CXX_COMPILER "${ODYSSEY_CLANG}/bin/clang++")
+set(CMAKE_CXX_COMPILER "${ODYSSEY_CLANG}")
 set(CMAKE_CXX_COMPILER_TARGET ${NX64_TRIPLE})
 
 set(CMAKE_C_FLAGS_RELEASE ${NX64_OPT_FLAGS})
@@ -45,6 +55,6 @@ add_definitions(-D MATCHING_HACK_NX_CLANG)
 add_link_options(-stdlib=libc++ -nostdlib)
 add_link_options(-fPIC -Wl,-Bsymbolic-functions -shared)
 # Use lld for performance reasons (and because we don't want a dependency on GNU tools)
-add_link_options(-fuse-ld=${ODYSSEY_CLANG_LLD}/bin/ld.lld)
-#add_link_options(-B "/home/monsterdruide1/botw/toolchain/lld-path")
-#add_link_options(-fuse-ld=lld)
+add_link_options(-fuse-ld=${ODYSSEY_CLANG_LLD})
+
+include_directories(SYSTEM ${CMAKE_CURRENT_LIST_DIR}/include)

tools/setup.py

@@ -2,14 +2,25 @@
 
 import argparse
 import hashlib
+import os
+import shutil
 from pathlib import Path
 import subprocess
 from typing import Optional
 from common import setup_common as setup
 from enum import Enum
+import platform
+import tarfile
+import tempfile
+import urllib.request
+import urllib.parse
+import urllib.error
+from common.util.config import get_repo_root
 
 TARGET_PATH = setup.get_target_path()
 TARGET_ELF_PATH = setup.get_target_elf_path()
+CACHE_REPO_RELEASE_URL = "https://github.com/MonsterDruide1/OdysseyDecompToolsCache/releases/download/v1.0"
+LIBCXX_SRC_URL = "https://releases.llvm.org/3.9.1/libcxx-3.9.1.src.tar.xz"
 
 class Version(Enum):
     VER_100 = "1.0"
@@ -59,8 +70,92 @@ def prepare_executable(original_nso: Optional[Path]):
 def get_build_dir():
     return setup.ROOT / "build"
 
+def setup_project_tools(tools_from_source):
+
+    def exists_tool(tool_name):
+        return os.path.isfile(f"{get_repo_root()}/tools/{tool_name}") or os.path.islink(f"{get_repo_root()}/tools/{tool_name}")
+
+    def exists_toolchain_file(file_path_rel):
+        return os.path.isfile(f"{get_repo_root()}/toolchain/{file_path_rel}")
+
+    def build_tools_from_source(tmpdir_path):
+        cwd = os.getcwd()
+        url_parts = CACHE_REPO_RELEASE_URL.split("/")
+        tag_name = url_parts[len(url_parts) - 1]
+        subprocess.check_call(["git", "clone", "--depth=1", "--branch", tag_name, "https://github.com/MonsterDruide1/OdysseyDecompToolsCache.git", f"{tmpdir_path}/OdysseyDecompToolsCache"])
+        os.chdir(f"{tmpdir}/OdysseyDecompToolsCache")
+        subprocess.check_call(["git", "submodule", "update", "--init"])
+        subprocess.check_call(["./generate.sh", "--no-tarball"])
+        os.chdir(cwd)
+        shutil.copytree(f"{tmpdir_path}/OdysseyDecompToolsCache/build/OdysseyDecomp-binaries_{platform.machine()}-{platform.system()}/bin", f"{get_repo_root()}/toolchain/bin")
+
+    def remove_old_toolchain():
+        if exists_toolchain_file("clang-3.9.1/bin/clang"):
+            print("Removing toolchain/clang-3.9.1 since full toolchains are no longer needed")
+            shutil.rmtree(f"{get_repo_root()}/toolchain/clang-3.9.1")
+        if exists_toolchain_file("clang-4.0.1/bin/lld"):
+            print("Removing toolchain/clang-4.0.1")
+            shutil.rmtree(f"{get_repo_root()}/toolchain/clang-4.0.1")
+
+    def check_download_url_updated():
+        if not exists_toolchain_file("cache-version-url.txt"):
+            with open(f"{get_repo_root()}/toolchain/cache-version-url.txt", "w") as f:
+                f.write(CACHE_REPO_RELEASE_URL)
+            return
+        with open(f"{get_repo_root()}/toolchain/cache-version-url.txt", "r+") as f:
+            data = f.read()
+            if data != CACHE_REPO_RELEASE_URL:
+                f.seek(0)
+                f.write(CACHE_REPO_RELEASE_URL)
+                f.truncate()
+                print("Old toolchain files found. Replacing them with ones from the latest release")
+                if exists_toolchain_file("bin/clang"):
+                    shutil.rmtree(f"{get_repo_root()}/toolchain/bin")
+
+    remove_old_toolchain()
+    check_download_url_updated()
+
+    if not exists_tool("check"):
+        os.symlink(f"{get_repo_root()}/toolchain/bin/check", f"{get_repo_root()}/tools/check")
+    if not exists_tool("decompme"):
+        os.symlink(f"{get_repo_root()}/toolchain/bin/decompme", f"{get_repo_root()}/tools/decompme")
+    if not exists_tool("listsym"):
+        os.symlink(f"{get_repo_root()}/toolchain/bin/listsym", f"{get_repo_root()}/tools/listsym")
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        if not exists_toolchain_file("include/__config"):
+            print(">>> Downloading llvm-3.9 libc++ headers...")
+            path = tmpdir + "/libcxx-3.9.1.src.tar.xz"
+            urllib.request.urlretrieve(LIBCXX_SRC_URL, path)
+            print(">>> Extracting libc++ headers...")
+            with tarfile.open(path) as f:
+                f.extractall(tmpdir, filter='tar')
+            shutil.copytree(f"{tmpdir}/libcxx-3.9.1.src/include", f"{get_repo_root()}/toolchain/include", dirs_exist_ok=True)
+
+        if not exists_tool("check") or not exists_tool("decompme") or not exists_tool("listsym") or not exists_toolchain_file("bin/clang") or not exists_toolchain_file("bin/ld.lld"):
+
+            if os.path.isdir(get_build_dir()):
+                shutil.rmtree(get_build_dir())
+
+            if tools_from_source:
+                build_tools_from_source(tmpdir)
+                return
+
+            target = f"{platform.machine()}-{platform.system()}"
+            path = tmpdir + f"/OdysseyDecomp-binaries_{target}"
+            try:
+                print(">>> Downloading clang, lld and viking...")
+                url = CACHE_REPO_RELEASE_URL + urllib.parse.quote(f"/OdysseyDecomp-binaries_{target}.tar.xz")
+                urllib.request.urlretrieve(url, path)
+                print(">>> Extracting tools...")
+                with tarfile.open(path) as f:
+                    f.extractall(f"{get_repo_root()}/toolchain/", filter='tar')
+            except urllib.error.HTTPError:
+                input(f"Prebuilt binaries not found for platform: {target}. Do you want to build llvm, clang, lld and viking from source? (Press enter to accept)")
+                build_tools_from_source(tmpdir)
+
 def create_build_dir(ver, cmake_backend):
-    if(ver != Version.VER_100): return # TODO remove this when multiple versions should be built
+    if(ver != Version.VER_100): return # TODO: remove this when multiple versions should be built
     build_dir = get_build_dir()
     if build_dir.is_dir():
         print(">>> build directory already exists: nothing to do")
@@ -78,14 +173,14 @@ def main():
     parser.add_argument("--cmake_backend", type=str,
                         help="CMake backend to use (Ninja, Unix Makefiles, etc.)", nargs="?", default="Ninja")
     parser.add_argument("--project-only", action="store_true",
-                        help="Disable viking and original NSO setup")
+                    help="Disable original NSO setup")
+    parser.add_argument("--tools-from-src", action="store_true",
+                    help="Build llvm, clang, lld and viking from source instead of using a prebuilt binaries")
     args = parser.parse_args()
-    
+
+    setup_project_tools(args.tools_from_src)
     if not args.project_only:
-        setup.install_viking()
         prepare_executable(args.original_nso)
-    setup.set_up_compiler("3.9.1")
-    setup.set_up_compiler("4.0.1")
     create_build_dir(Version.VER_100, args.cmake_backend)
     create_build_dir(Version.VER_101, args.cmake_backend)
     create_build_dir(Version.VER_110, args.cmake_backend)