refactor: convert DLC string data directly to UTF-8 w/ simdutf

meson.build

@@ -76,7 +76,10 @@ glm_dep = dependency('glm')
 
 stb = subproject('stb').get_variable('stb_inc')
 stb_dep = declare_dependency(include_directories: stb)
-
+simdutf_dep = dependency('simdutf',
+  fallback: ['simdutf', 'simdutf_dep'],
+  default_options: ['utf8=true', 'utf16=true', 'utf32=true']
+)
 miniaudio_dep = dependency('miniaudio')
 
 subdir('targets/util')

subprojects/packagefiles/simdutf/meson.build

@@ -55,7 +55,7 @@ simdutf_lib = static_library(
 )
 
 simdutf_dep = declare_dependency(
-    sources: simdutf_amalgamate,
+    include_directories: include_directories('.'),
     link_with: simdutf_lib,
 )
 

targets/app/common/DLC/DLCAudioFile.cpp

@@ -5,13 +5,14 @@
 #include <cstring>
 #include <unordered_map>
 
-#include "platform/renderer/renderer.h"
-#include "platform/storage/storage.h"
 #include "DLCManager.h"
 #include "app/common/DLC/DLCFile.h"
 #include "app/linux/LinuxGame.h"
 #include "platform/XboxStubs.h"
+#include "platform/renderer/renderer.h"
+#include "platform/storage/storage.h"
 #include "util/StringHelpers.h"
+
 #if defined(_WINDOWS64)
 #include "app/windows/XML/ATGXmlParser.h"
 #include "app/windows/XML/xmlFilesCallback.h"
@@ -20,26 +21,6 @@
 namespace {
 constexpr std::size_t AUDIO_DLC_WCHAR_BIN_SIZE = 2;
 
-#if WCHAR_MAX > 0xFFFF
-static std::wstring ReadAudioDlcWString(const void* data) {
-    const std::uint16_t* chars = static_cast<const std::uint16_t*>(data);
-    const std::uint16_t* end = chars;
-    while (*end != 0) {
-        ++end;
-    }
-
-    std::wstring out(static_cast<std::size_t>(end - chars), 0);
-    for (std::size_t i = 0; i < out.size(); ++i) {
-        out[i] = static_cast<wchar_t>(chars[i]);
-    }
-    return out;
-}
-#else
-static std::wstring ReadAudioDlcWString(const void* data) {
-    return std::wstring(static_cast<const wchar_t*>(data));
-}
-#endif
-
 template <typename T>
 T ReadAudioDlcValue(const std::uint8_t* data, unsigned int offset = 0) {
     T value;
@@ -59,14 +40,15 @@ inline unsigned int AudioParamAdvance(unsigned int wcharCount) {
 }
 
 inline unsigned int AudioDetailAdvance(unsigned int wcharCount) {
-    return static_cast<unsigned int>(sizeof(IPlatformStorage::DLC_FILE_DETAILS) +
+    return static_cast<unsigned int>(
-                                     wcharCount * AUDIO_DLC_WCHAR_BIN_SIZE);
+        sizeof(IPlatformStorage::DLC_FILE_DETAILS) +
+        wcharCount * AUDIO_DLC_WCHAR_BIN_SIZE);
 }
 
-inline std::wstring ReadAudioParamString(const std::uint8_t* data,
+inline std::string ReadAudioParamString(const std::uint8_t* data,
-                                         unsigned int offset) {
+                                        unsigned int offset) {
-    return ReadAudioDlcWString(data + offset +
+    return dlc_read_wstring(
-                               offsetof(IPlatformStorage::DLC_FILE_PARAM, wchData));
+        data + offset + offsetof(IPlatformStorage::DLC_FILE_PARAM, wchData));
 }
 }  // namespace
 
@@ -88,12 +70,13 @@ std::uint8_t* DLCAudioFile::getData(std::uint32_t& dataBytes) {
     return m_pbData;
 }
 
-const wchar_t* DLCAudioFile::wchTypeNamesA[] = {
+const char* DLCAudioFile::wchTypeNamesA[] = {
-    L"CUENAME",
+    "CUENAME",
-    L"CREDIT",
+    "CREDIT",
 };
 
-DLCAudioFile::EAudioParameterType DLCAudioFile::getParameterType(const std::wstring& paramName) {
+DLCAudioFile::EAudioParameterType DLCAudioFile::getParameterType(
+    const std::string& paramName) {
     EAudioParameterType type = e_AudioParamType_Invalid;
 
     for (int i = 0; i < e_AudioParamType_Max; ++i) {
@@ -122,8 +105,8 @@ void DLCAudioFile::addParameter(EAudioType type, EAudioParameterType ptype,
 
                 int maximumChars = 55;
 
-                bool bIsSDMode =
+                bool bIsSDMode = !PlatformRenderer.IsHiDef() &&
-                    !PlatformRenderer.IsHiDef() && !PlatformRenderer.IsWidescreen();
+                                 !PlatformRenderer.IsWidescreen();
 
                 if (bIsSDMode) {
                     maximumChars = 45;
@@ -202,8 +185,7 @@ bool DLCAudioFile::processDLCDataFile(std::uint8_t* pbData,
     for (unsigned int i = 0; i < uiParameterTypeCount; i++) {
         // Map DLC strings to application strings, then store the DLC index
         // mapping to application index
-        std::wstring parameterName =
+        std::string parameterName = ReadAudioParamString(pbData, uiCurrentByte);
-            ReadAudioParamString(pbData, uiCurrentByte);
         EAudioParameterType type = getParameterType(parameterName);
         if (type != e_AudioParamType_Invalid) {
             parameterMapping[paramBuf.dwType] = type;
@@ -238,7 +220,7 @@ bool DLCAudioFile::processDLCDataFile(std::uint8_t* pbData,
 
             if (it != parameterMapping.end()) {
                 addParameter(type, (EAudioParameterType)paramBuf.dwType,
-                             wstringtofilename(ReadAudioParamString(pbTemp, 0)));
+                             ReadAudioParamString(pbTemp, 0));
             }
             pbTemp += AudioParamAdvance(paramBuf.dwWchCount);
             ReadAudioDlcStruct(&paramBuf, pbTemp);

targets/app/common/DLC/DLCAudioFile.h

@@ -27,7 +27,7 @@ public:
         e_AudioParamType_Max,
 
     };
-    static const wchar_t* wchTypeNamesA[e_AudioParamType_Max];
+    static const char* wchTypeNamesA[e_AudioParamType_Max];
 
     DLCAudioFile(const std::string& path);
 
@@ -52,5 +52,5 @@ private:
                       DLCAudioFile::EAudioParameterType ptype,
                       const std::string& value);
     DLCAudioFile::EAudioParameterType getParameterType(
-        const std::wstring& paramName);
+        const std::string& paramName);
 };

targets/app/common/DLC/DLCManager.cpp

@@ -13,6 +13,8 @@
 #include <unordered_map>
 #include <utility>
 
+#include "simdutf.h"
+
 #include "platform/profile/profile.h"
 #include "platform/storage/storage.h"
 #include "DLCFile.h"
@@ -30,44 +32,23 @@
 // the DLC was created on windows, with wchar_t beeing 2 bytes and UTF-16
 static const std::size_t DLC_WCHAR_BIN_SIZE = 2;
 
-#if WCHAR_MAX > 0xFFFF
+std::string dlc_read_wstring(const void* data) {
-// than sizeof(char) != DLC_WCHAR_BIN_SIZE
+    const char16_t* chars = static_cast<const char16_t*>(data);
-// e.g. Linux and all Posix/Unix systems with char beeing 4B/32bit
+    const char16_t* end = chars;
-static_assert(sizeof(wchar_t) == 4,
+    while (*end != 0) {
-              "char is not 4bytes but larger than 2bytes ???");
-
-static inline std::wstring dlc_read_wstring(const void* data) {
-    const std::uint16_t* p = static_cast<const std::uint16_t*>(data);
-    // find the end (nullterminated)
-    const std::uint16_t* end = p;
-    while (*end) {
         ++end;
     }
 
-    std::size_t len = static_cast<std::size_t>(end - p);
+    const std::size_t len16 = static_cast<std::size_t>(end - chars);
+    std::string result(simdutf::utf8_length_from_utf16le(chars, len16), '\0');
+    auto len = simdutf::convert_utf16le_to_utf8(chars, len16, result.data());
+    result.resize(len);
 
-    // allocate wstring with length len
+    return result;
-    // it will be nullterminated internally, do not worry.
-    std::wstring out(len, 0);
-
-    // and copy them into thje string
-    for (std::size_t i = 0; i < len; ++i) {
-        out[i] = static_cast<wchar_t>(p[i]);
-    }
-
-    return out;
 }
 
 #define DLC_WSTRING(ptr) dlc_read_wstring(ptr)
 
-#else
-// just in case.
-static_assert(sizeof(wchar_t) == 2,
-              "How did we get here? wide char smaller than 2 bytes");
-// perfectly fine scince char will be 2 bytes (UCS-2/UTF-16)
-#define DLC_WSTRING(ptr) std::wstring((wchar_t*)(ptr))
-#endif
-
 #define DLC_PARAM_ADV(n) \
     (sizeof(IPlatformStorage::DLC_FILE_PARAM) + (n) * DLC_WCHAR_BIN_SIZE)
 #define DLC_DETAIL_ADV(n) \
@@ -123,20 +104,20 @@ bool readOwnedDlcFile(const std::string& path, std::uint8_t** ppData,
 }
 }  // namespace
 
-const wchar_t* DLCManager::wchTypeNamesA[] = {
+const char* DLCManager::wchTypeNamesA[] = {
-    L"DISPLAYNAME",
+    "DISPLAYNAME",
-    L"THEMENAME",
+    "THEMENAME",
-    L"FREE",
+    "FREE",
-    L"CREDIT",
+    "CREDIT",
-    L"CAPEPATH",
+    "CAPEPATH",
-    L"BOX",
+    "BOX",
-    L"ANIM",
+    "ANIM",
-    L"PACKID",
+    "PACKID",
-    L"NETHERPARTICLECOLOUR",
+    "NETHERPARTICLECOLOUR",
-    L"ENCHANTTEXTCOLOUR",
+    "ENCHANTTEXTCOLOUR",
-    L"ENCHANTTEXTFOCUSCOLOUR",
+    "ENCHANTTEXTFOCUSCOLOUR",
-    L"DATAPATH",
+    "DATAPATH",
-    L"PACKVERSION",
+    "PACKVERSION",
 };
 
 DLCManager::DLCManager() {
@@ -152,7 +133,7 @@ DLCManager::~DLCManager() {
 }
 
 DLCManager::EDLCParameterType DLCManager::getParameterType(
-    const std::wstring& paramName) {
+    const std::string& paramName) {
     EDLCParameterType type = e_DLCParamType_Invalid;
 
     for (unsigned int i = 0; i < e_DLCParamType_Max; ++i) {
@@ -472,7 +453,7 @@ bool DLCManager::processDLCDataFile(unsigned int& dwFilesProcessed,
     for (unsigned int i = 0; i < uiParameterCount; i++) {
         // Map DLC strings to application strings, then store the DLC index
         // mapping to application index
-        std::wstring parameterName = DLC_PARAM_WSTR(pbData, uiCurrentByte);
+        std::string parameterName = DLC_PARAM_WSTR(pbData, uiCurrentByte);
         DLCManager::EDLCParameterType type =
             DLCManager::getParameterType(parameterName);
         if (type != DLCManager::e_DLCParamType_Invalid) {
@@ -510,7 +491,7 @@ bool DLCManager::processDLCDataFile(unsigned int& dwFilesProcessed,
                 new DLCPack(pack->getName(), pack->getLicenseMask());
         } else if (type != e_DLCType_PackConfig) {
             dlcFile =
-                pack->addFile(type, wstringtofilename(DLC_DETAIL_WSTR(pbData, uiCurrentByte)));
+                pack->addFile(type, DLC_DETAIL_WSTR(pbData, uiCurrentByte));
         }
 
         // Params
@@ -527,14 +508,14 @@ bool DLCManager::processDLCDataFile(unsigned int& dwFilesProcessed,
 
             if (it != parameterMapping.end()) {
                 if (type == e_DLCType_PackConfig) {
-                    pack->addParameter(it->second, wstringtofilename(DLC_PARAM_WSTR(pbTemp, 0)));
+                    pack->addParameter(it->second, DLC_PARAM_WSTR(pbTemp, 0));
                 } else {
                     if (dlcFile != nullptr)
                         dlcFile->addParameter(it->second,
-                                              wstringtofilename(DLC_PARAM_WSTR(pbTemp, 0)));
+                                              DLC_PARAM_WSTR(pbTemp, 0));
                     else if (dlcTexturePack != nullptr)
                         dlcTexturePack->addParameter(it->second,
-                                                     wstringtofilename(DLC_PARAM_WSTR(pbTemp, 0)));
+                                                     DLC_PARAM_WSTR(pbTemp, 0));
                 }
             }
             pbTemp += DLC_PARAM_ADV(parBuf.dwWchCount);
@@ -571,7 +552,7 @@ bool DLCManager::processDLCDataFile(unsigned int& dwFilesProcessed,
             switch (fileBuf.dwType) {
                 case DLCManager::e_DLCType_Skin:
                     app.vSkinNames.push_back(
-                        wstringtofilename(DLC_DETAIL_WSTR(pbData, uiCurrentByte)));
+                        DLC_DETAIL_WSTR(pbData, uiCurrentByte));
                     break;
             }
 
@@ -648,7 +629,7 @@ std::uint32_t DLCManager::retrievePackID(std::uint8_t* pbData,
     for (unsigned int i = 0; i < uiParameterCount; i++) {
         // Map DLC strings to application strings, then store the DLC index
         // mapping to application index
-        std::wstring parameterName = DLC_PARAM_WSTR(pbData, uiCurrentByte);
+        std::string parameterName = DLC_PARAM_WSTR(pbData, uiCurrentByte);
         DLCManager::EDLCParameterType type =
             DLCManager::getParameterType(parameterName);
         if (type != DLCManager::e_DLCParamType_Invalid) {
@@ -685,8 +666,8 @@ std::uint32_t DLCManager::retrievePackID(std::uint8_t* pbData,
             if (it != parameterMapping.end()) {
                 if (type == e_DLCType_PackConfig) {
                     if (it->second == e_DLCParamType_PackId) {
-                        std::wstring wsTemp = DLC_PARAM_WSTR(pbTemp, 0);
+                        std::string wsTemp = DLC_PARAM_WSTR(pbTemp, 0);
-                        std::wstringstream ss;
+                        std::stringstream ss;
                         // 4J Stu - numbered using decimal to make it easier for
                         // artists/people to number manually
                         ss << std::dec << wsTemp.c_str();

targets/app/common/DLC/DLCManager.h

@@ -47,7 +47,7 @@ public:
         e_DLCParamType_Max,
 
     };
-    const static wchar_t* wchTypeNamesA[e_DLCParamType_Max];
+    const static char* wchTypeNamesA[e_DLCParamType_Max];
 
 private:
     std::vector<DLCPack*> m_packs;
@@ -59,7 +59,7 @@ public:
     DLCManager();
     ~DLCManager();
 
-    static EDLCParameterType getParameterType(const std::wstring& paramName);
+    static EDLCParameterType getParameterType(const std::string& paramName);
 
     unsigned int getPackCount(EDLCType type = e_DLCType_All);
 
@@ -108,3 +108,5 @@ private:
     std::uint32_t retrievePackID(std::uint8_t* pbData, unsigned int dwLength,
                                  DLCPack* pack);
 };
+
+std::string dlc_read_wstring(const void* data);

targets/app/meson.build

@@ -40,6 +40,7 @@ client_dependencies = [
   dependency('zlib'),
   miniaudio_dep,
   stb_dep,
+  simdutf_dep,
   util_dep,
 ]
 

targets/util/meson.build

@@ -1,8 +1,3 @@
-simdutf_dep = dependency('simdutf',
-  fallback: ['simdutf', 'simdutf_dep'],
-  default_options: ['utf8=true', 'utf16=true', 'utf32=true']
-)
-
 lib_util = static_library('util',
   files('StringHelpers.cpp', 'FrameProfiler.cpp'),
   dependencies: [simdutf_dep],