backups/4jcraft
1
0
Fork 0
mirror of https://github.com/4jcraft/4jcraft.git synced 2026-04-23 13:13:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: convert DLC string data directly to UTF-8 w/ simdutf

Browse source
This commit is contained in:
Tropical 2026-04-08 01:41:34 -05:00
parent 00fec789b5
commit dcf9e0bcdb
8 changed files with 66 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
meson.build
View file

@ -76,7 +76,10 @@ glm_dep = dependency('glm')
stb = subproject('stb').get_variable('stb_inc')
stb_dep = declare_dependency(include_directories: stb)
simdutf_dep = dependency('simdutf',
fallback: ['simdutf', 'simdutf_dep'],
default_options: ['utf8=true', 'utf16=true', 'utf32=true']
)
miniaudio_dep = dependency('miniaudio')
subdir('targets/util')

2
subprojects/packagefiles/simdutf/meson.build
View file

@ -55,7 +55,7 @@ simdutf_lib = static_library(
)
simdutf_dep = declare_dependency(
sources: simdutf_amalgamate,
include_directories: include_directories('.'),
link_with: simdutf_lib,
)

56
targets/app/common/DLC/DLCAudioFile.cpp
View file

@ -5,13 +5,14 @@
#include <cstring>
#include <unordered_map>
#include "platform/renderer/renderer.h"
#include "platform/storage/storage.h"
#include "DLCManager.h"
#include "app/common/DLC/DLCFile.h"
#include "app/linux/LinuxGame.h"
#include "platform/XboxStubs.h"
#include "platform/renderer/renderer.h"
#include "platform/storage/storage.h"
#include "util/StringHelpers.h"
#if defined(_WINDOWS64)
#include "app/windows/XML/ATGXmlParser.h"
#include "app/windows/XML/xmlFilesCallback.h"
@ -20,26 +21,6 @@
namespace {
constexpr std::size_t AUDIO_DLC_WCHAR_BIN_SIZE = 2;
#if WCHAR_MAX > 0xFFFF
static std::wstring ReadAudioDlcWString(const void* data) {
const std::uint16_t* chars = static_cast<const std::uint16_t*>(data);
const std::uint16_t* end = chars;
while (*end != 0) {
++end;
}
std::wstring out(static_cast<std::size_t>(end - chars), 0);
for (std::size_t i = 0; i < out.size(); ++i) {
out[i] = static_cast<wchar_t>(chars[i]);
}
return out;
}
#else
static std::wstring ReadAudioDlcWString(const void* data) {
return std::wstring(static_cast<const wchar_t*>(data));
}
#endif
template <typename T>
T ReadAudioDlcValue(const std::uint8_t* data, unsigned int offset = 0) {
T value;
@ -59,14 +40,15 @@ inline unsigned int AudioParamAdvance(unsigned int wcharCount) {
}
inline unsigned int AudioDetailAdvance(unsigned int wcharCount) {
return static_cast<unsigned int>(sizeof(IPlatformStorage::DLC_FILE_DETAILS) +
wcharCount * AUDIO_DLC_WCHAR_BIN_SIZE);
return static_cast<unsigned int>(
sizeof(IPlatformStorage::DLC_FILE_DETAILS) +
wcharCount * AUDIO_DLC_WCHAR_BIN_SIZE);
}
inline std::wstring ReadAudioParamString(const std::uint8_t* data,
unsigned int offset) {
return ReadAudioDlcWString(data + offset +
offsetof(IPlatformStorage::DLC_FILE_PARAM, wchData));
inline std::string ReadAudioParamString(const std::uint8_t* data,
unsigned int offset) {
return dlc_read_wstring(
data + offset + offsetof(IPlatformStorage::DLC_FILE_PARAM, wchData));
}
} // namespace
@ -88,12 +70,13 @@ std::uint8_t* DLCAudioFile::getData(std::uint32_t& dataBytes) {
return m_pbData;
}
const wchar_t* DLCAudioFile::wchTypeNamesA[] = {
L"CUENAME",
L"CREDIT",
const char* DLCAudioFile::wchTypeNamesA[] = {
"CUENAME",
"CREDIT",
};
DLCAudioFile::EAudioParameterType DLCAudioFile::getParameterType(const std::wstring& paramName) {
DLCAudioFile::EAudioParameterType DLCAudioFile::getParameterType(
const std::string& paramName) {
EAudioParameterType type = e_AudioParamType_Invalid;
for (int i = 0; i < e_AudioParamType_Max; ++i) {
@ -122,8 +105,8 @@ void DLCAudioFile::addParameter(EAudioType type, EAudioParameterType ptype,
int maximumChars = 55;
bool bIsSDMode =
!PlatformRenderer.IsHiDef() && !PlatformRenderer.IsWidescreen();
bool bIsSDMode = !PlatformRenderer.IsHiDef() &&
!PlatformRenderer.IsWidescreen();
if (bIsSDMode) {
maximumChars = 45;
@ -202,8 +185,7 @@ bool DLCAudioFile::processDLCDataFile(std::uint8_t* pbData,
for (unsigned int i = 0; i < uiParameterTypeCount; i++) {
// Map DLC strings to application strings, then store the DLC index
// mapping to application index
std::wstring parameterName =
ReadAudioParamString(pbData, uiCurrentByte);
std::string parameterName = ReadAudioParamString(pbData, uiCurrentByte);
EAudioParameterType type = getParameterType(parameterName);
if (type != e_AudioParamType_Invalid) {
parameterMapping[paramBuf.dwType] = type;
@ -238,7 +220,7 @@ bool DLCAudioFile::processDLCDataFile(std::uint8_t* pbData,
if (it != parameterMapping.end()) {
addParameter(type, (EAudioParameterType)paramBuf.dwType,
wstringtofilename(ReadAudioParamString(pbTemp, 0)));
ReadAudioParamString(pbTemp, 0));
}
pbTemp += AudioParamAdvance(paramBuf.dwWchCount);
ReadAudioDlcStruct(&paramBuf, pbTemp);

4
targets/app/common/DLC/DLCAudioFile.h
View file

@ -27,7 +27,7 @@ public:
e_AudioParamType_Max,
};
static const wchar_t* wchTypeNamesA[e_AudioParamType_Max];
static const char* wchTypeNamesA[e_AudioParamType_Max];
DLCAudioFile(const std::string& path);
@ -52,5 +52,5 @@ private:
DLCAudioFile::EAudioParameterType ptype,
const std::string& value);
DLCAudioFile::EAudioParameterType getParameterType(
const std::wstring& paramName);
const std::string& paramName);
};

89
targets/app/common/DLC/DLCManager.cpp
View file

@ -13,6 +13,8 @@
#include <unordered_map>
#include <utility>
#include "simdutf.h"
#include "platform/profile/profile.h"
#include "platform/storage/storage.h"
#include "DLCFile.h"
@ -30,44 +32,23 @@
// the DLC was created on windows, with wchar_t beeing 2 bytes and UTF-16
static const std::size_t DLC_WCHAR_BIN_SIZE = 2;
#if WCHAR_MAX > 0xFFFF
// than sizeof(char) != DLC_WCHAR_BIN_SIZE
// e.g. Linux and all Posix/Unix systems with char beeing 4B/32bit
static_assert(sizeof(wchar_t) == 4,
"char is not 4bytes but larger than 2bytes ???");
static inline std::wstring dlc_read_wstring(const void* data) {
const std::uint16_t* p = static_cast<const std::uint16_t*>(data);
// find the end (nullterminated)
const std::uint16_t* end = p;
while (*end) {
std::string dlc_read_wstring(const void* data) {
const char16_t* chars = static_cast<const char16_t*>(data);
const char16_t* end = chars;
while (*end != 0) {
++end;
}
std::size_t len = static_cast<std::size_t>(end - p);
const std::size_t len16 = static_cast<std::size_t>(end - chars);
std::string result(simdutf::utf8_length_from_utf16le(chars, len16), '\0');
auto len = simdutf::convert_utf16le_to_utf8(chars, len16, result.data());
result.resize(len);
// allocate wstring with length len
// it will be nullterminated internally, do not worry.
std::wstring out(len, 0);
// and copy them into thje string
for (std::size_t i = 0; i < len; ++i) {
out[i] = static_cast<wchar_t>(p[i]);
}
return out;
return result;
}
#define DLC_WSTRING(ptr) dlc_read_wstring(ptr)
#else
// just in case.
static_assert(sizeof(wchar_t) == 2,
"How did we get here? wide char smaller than 2 bytes");
// perfectly fine scince char will be 2 bytes (UCS-2/UTF-16)
#define DLC_WSTRING(ptr) std::wstring((wchar_t*)(ptr))
#endif
#define DLC_PARAM_ADV(n) \
(sizeof(IPlatformStorage::DLC_FILE_PARAM) + (n) * DLC_WCHAR_BIN_SIZE)
#define DLC_DETAIL_ADV(n) \
@ -123,20 +104,20 @@ bool readOwnedDlcFile(const std::string& path, std::uint8_t** ppData,
}
} // namespace
const wchar_t* DLCManager::wchTypeNamesA[] = {
L"DISPLAYNAME",
L"THEMENAME",
L"FREE",
L"CREDIT",
L"CAPEPATH",
L"BOX",
L"ANIM",
L"PACKID",
L"NETHERPARTICLECOLOUR",
L"ENCHANTTEXTCOLOUR",
L"ENCHANTTEXTFOCUSCOLOUR",
L"DATAPATH",
L"PACKVERSION",
const char* DLCManager::wchTypeNamesA[] = {
"DISPLAYNAME",
"THEMENAME",
"FREE",
"CREDIT",
"CAPEPATH",
"BOX",
"ANIM",
"PACKID",
"NETHERPARTICLECOLOUR",
"ENCHANTTEXTCOLOUR",
"ENCHANTTEXTFOCUSCOLOUR",
"DATAPATH",
"PACKVERSION",
};
DLCManager::DLCManager() {
@ -152,7 +133,7 @@ DLCManager::~DLCManager() {
}
DLCManager::EDLCParameterType DLCManager::getParameterType(
const std::wstring& paramName) {
const std::string& paramName) {
EDLCParameterType type = e_DLCParamType_Invalid;
for (unsigned int i = 0; i < e_DLCParamType_Max; ++i) {
@ -472,7 +453,7 @@ bool DLCManager::processDLCDataFile(unsigned int& dwFilesProcessed,
for (unsigned int i = 0; i < uiParameterCount; i++) {
// Map DLC strings to application strings, then store the DLC index
// mapping to application index
std::wstring parameterName = DLC_PARAM_WSTR(pbData, uiCurrentByte);
std::string parameterName = DLC_PARAM_WSTR(pbData, uiCurrentByte);
DLCManager::EDLCParameterType type =
DLCManager::getParameterType(parameterName);
if (type != DLCManager::e_DLCParamType_Invalid) {
@ -510,7 +491,7 @@ bool DLCManager::processDLCDataFile(unsigned int& dwFilesProcessed,
new DLCPack(pack->getName(), pack->getLicenseMask());
} else if (type != e_DLCType_PackConfig) {
dlcFile =
pack->addFile(type, wstringtofilename(DLC_DETAIL_WSTR(pbData, uiCurrentByte)));
pack->addFile(type, DLC_DETAIL_WSTR(pbData, uiCurrentByte));
}
// Params
@ -527,14 +508,14 @@ bool DLCManager::processDLCDataFile(unsigned int& dwFilesProcessed,
if (it != parameterMapping.end()) {
if (type == e_DLCType_PackConfig) {
pack->addParameter(it->second, wstringtofilename(DLC_PARAM_WSTR(pbTemp, 0)));
pack->addParameter(it->second, DLC_PARAM_WSTR(pbTemp, 0));
} else {
if (dlcFile != nullptr)
dlcFile->addParameter(it->second,
wstringtofilename(DLC_PARAM_WSTR(pbTemp, 0)));
DLC_PARAM_WSTR(pbTemp, 0));
else if (dlcTexturePack != nullptr)
dlcTexturePack->addParameter(it->second,
wstringtofilename(DLC_PARAM_WSTR(pbTemp, 0)));
DLC_PARAM_WSTR(pbTemp, 0));
}
}
pbTemp += DLC_PARAM_ADV(parBuf.dwWchCount);
@ -571,7 +552,7 @@ bool DLCManager::processDLCDataFile(unsigned int& dwFilesProcessed,
switch (fileBuf.dwType) {
case DLCManager::e_DLCType_Skin:
app.vSkinNames.push_back(
wstringtofilename(DLC_DETAIL_WSTR(pbData, uiCurrentByte)));
DLC_DETAIL_WSTR(pbData, uiCurrentByte));
break;
}
@ -648,7 +629,7 @@ std::uint32_t DLCManager::retrievePackID(std::uint8_t* pbData,
for (unsigned int i = 0; i < uiParameterCount; i++) {
// Map DLC strings to application strings, then store the DLC index
// mapping to application index
std::wstring parameterName = DLC_PARAM_WSTR(pbData, uiCurrentByte);
std::string parameterName = DLC_PARAM_WSTR(pbData, uiCurrentByte);
DLCManager::EDLCParameterType type =
DLCManager::getParameterType(parameterName);
if (type != DLCManager::e_DLCParamType_Invalid) {
@ -685,8 +666,8 @@ std::uint32_t DLCManager::retrievePackID(std::uint8_t* pbData,
if (it != parameterMapping.end()) {
if (type == e_DLCType_PackConfig) {
if (it->second == e_DLCParamType_PackId) {
std::wstring wsTemp = DLC_PARAM_WSTR(pbTemp, 0);
std::wstringstream ss;
std::string wsTemp = DLC_PARAM_WSTR(pbTemp, 0);
std::stringstream ss;
// 4J Stu - numbered using decimal to make it easier for
// artists/people to number manually
ss << std::dec << wsTemp.c_str();

6
targets/app/common/DLC/DLCManager.h
View file

@ -47,7 +47,7 @@ public:
e_DLCParamType_Max,
};
const static wchar_t* wchTypeNamesA[e_DLCParamType_Max];
const static char* wchTypeNamesA[e_DLCParamType_Max];
private:
std::vector<DLCPack*> m_packs;
@ -59,7 +59,7 @@ public:
DLCManager();
~DLCManager();
static EDLCParameterType getParameterType(const std::wstring& paramName);
static EDLCParameterType getParameterType(const std::string& paramName);
unsigned int getPackCount(EDLCType type = e_DLCType_All);
@ -108,3 +108,5 @@ private:
std::uint32_t retrievePackID(std::uint8_t* pbData, unsigned int dwLength,
DLCPack* pack);
};
std::string dlc_read_wstring(const void* data);

1
targets/app/meson.build
View file

@ -40,6 +40,7 @@ client_dependencies = [
dependency('zlib'),
miniaudio_dep,
stb_dep,
simdutf_dep,
util_dep,
]

5
targets/util/meson.build
View file

@ -1,8 +1,3 @@
simdutf_dep = dependency('simdutf',
fallback: ['simdutf', 'simdutf_dep'],
default_options: ['utf8=true', 'utf16=true', 'utf32=true']
)
lib_util = static_library('util',
files('StringHelpers.cpp', 'FrameProfiler.cpp'),
dependencies: [simdutf_dep],