Fractured Locales Support (ryubing/ryujinx!238)

See merge request ryubing/ryujinx!238

assets/Languages.json

@@ -0,0 +1,24 @@
+{
+  "Languages": {
+    "ar_SA": "اَلْعَرَبِيَّةُ",
+    "de_DE": "Deutsch",
+    "el_GR": "Ελληνικά",
+    "en_US": "English (US)",
+    "es_ES": "Español (ES)",
+    "fr_FR": "Français (FR)",
+    "he_IL": "עִברִית",
+    "it_IT": "Italiano",
+    "ja_JP": "日本語",
+    "ko_KR": "한국어",
+    "no_NO": "Norsk",
+    "pl_PL": "Polski",
+    "pt_BR": "Português (BR)",
+    "ru_RU": "Русский",
+    "sv_SE": "Svenska",
+    "th_TH": "ภาษาไทย",
+    "tr_TR": "Türkçe",
+    "uk_UA": "Українська",
+    "zh_CN": "简体中文",
+    "zh_TW": "繁體中文 (台灣)"
+  }
+}

assets/Locales.md

@@ -0,0 +1,60 @@
+
+# Ryubing Locales
+
+Ryubing Locales uses a custom format, which uses a file for defining the supported languages and a folder of json files for the locales themselves.
+Each json file holds the locales for a specific part of the emulator, e.g. the Setup Wizard locales are in `SetupWizard.json`, and each locale entry in the file includes all the supported languages in the same place.
+
+## Languages
+in the `/assets/` folder you will find the `Languages.json` file, which defines all the languages supported by the emulator. 
+The file includes a table of the langauge codes and their langauge names.
+
+    #Example of the format for Languages.json	
+    {
+      "Languages": {
+        "ar_SA": "اَلْعَرَبِيَّةُ",
+        "en_US": "English (US)",
+        ...
+        "zh_TW": "繁體中文 (台灣)"
+      }
+    }
+
+## Locales
+in the `/assets/Locales/` folder you will find the json files, which define all the locales supported by the emulator. 
+Each json file holds locales for a specific part of the emulator in a large array of locale objects.
+Each locale is made up an ID used for lookup and a list of the languages and their matching translations.
+Any empty string or null value will automatically use the English translation instead in the emulator.
+
+### Format
+When adding a new locale, you just need to add the ID and the en_US language translation, then the validation system will add default values for the rest of languages automatically, when rebuilding the project.
+If you want to signal that a translation is supposed to match the English translation, you just have to replace the empty string with `null`.
+When you want to check what translations are missing for a language just search for `"<lang_code>": ""`, e.g: `"en_US": ""` (but with any other language, as English will never be missing translations).
+
+### Legacy file (Root.json)
+Currently all older locales are stored in `Root.json`, but they are slowly being moved into newer, more descriptive json files, to make the locale system more accessible.
+Do **not** add new locales to `Root.json`.
+If no json file exists for the specific part of the emulator you're working on, you should instead add a new json file for that part.
+
+    #Example of the format for Root.json	
+    {
+      "Locales": [
+        {
+          "ID": "MenuBarActionsOpenMiiEditor",
+          "Translations": {
+            "ar_SA": "",
+            "en_US": "Mii Editor",
+            ...
+            "zh_TW": "Mii 編輯器"
+          }
+        },
+        {
+          "ID": "KeyNumber9",
+          "Translations": {
+            "ar_SA": "٩",
+            "en_US": "9",
+            ...
+            "zh_TW": null
+          }
+        }
+      ]
+    }
+   

assets/Locales/Root.json

@@ -1,72 +1,5 @@
 {
-  "Info": {
-    "Format1": "The Locale file uses a custom Unified format.",
-    "Format2": "The file starts with a list of all the supported languages.",
-    "Format3": "Each locale is made up an ID used for lookup and a list",
-    "Format4": "of the languages and their matching translations.",
-    "Format5": "When adding a new locale you just need to add the ID and",
-    "Format6": "the en_US language translation, then the validation system",
-    "Format7": "will add the rest of the languages automatically on rebuild.",
-    "Format8": "By default the languages will be added with an empty string.",
-    "Format9": "Any empty string or null value will automatically match the",
-    "Format10": "English translation.",
-    "Format11": "If you want to signal that a translation is supposed to",
-    "Format12": "match the English translation, you just have to replace the",
-    "Format13": "empty string with null.",
-    "Format14": "Translators who want to check what translations are missing",
-    "Format15": "for their language just need to search for:",
-    "Format16": "{'lang_code': ''} with double quotes instead of single",
-    "Format17": "e.g: {'en_US': ''} (but with any other language as English",
-    "Format18": "will never be missing translations)."
-  },
-  "Languages": [
-    "ar_SA",
-    "de_DE",
-    "el_GR",
-    "en_US",
-    "es_ES",
-    "fr_FR",
-    "he_IL",
-    "it_IT",
-    "ja_JP",
-    "ko_KR",
-    "no_NO",
-    "pl_PL",
-    "pt_BR",
-    "ru_RU",
-    "sv_SE",
-    "th_TH",
-    "tr_TR",
-    "uk_UA",
-    "zh_CN",
-    "zh_TW"
-  ],
   "Locales": [
-    {
-      "ID": "Language",
-      "Translations": {
-        "ar_SA": "اَلْعَرَبِيَّةُ",
-        "de_DE": "Deutsch",
-        "el_GR": "Ελληνικά",
-        "en_US": "English (US)",
-        "es_ES": "Español (ES)",
-        "fr_FR": "Français (FR)",
-        "he_IL": "עִברִית",
-        "it_IT": "Italiano",
-        "ja_JP": "日本語",
-        "ko_KR": "한국어",
-        "no_NO": "Norsk",
-        "pl_PL": "Polski",
-        "pt_BR": "Português (BR)",
-        "ru_RU": "Русский",
-        "sv_SE": "Svenska",
-        "th_TH": "ภาษาไทย",
-        "tr_TR": "Türkçe",
-        "uk_UA": "Українська",
-        "zh_CN": "简体中文",
-        "zh_TW": "繁體中文 (台灣)"
-      }
-    },
     {
       "ID": "MenuBarActionsOpenMiiEditor",
       "Translations": {

src/Ryujinx.BuildValidationTasks/LocalesValidationTask.cs

@@ -11,9 +11,7 @@ namespace Ryujinx.BuildValidationTasks
     {
         static readonly JsonSerializerOptions _jsonOptions = new()
         {
-            WriteIndented = true,
-            NewLine = "\n",
-            Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
+            WriteIndented = true, NewLine = "\n", Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
         };
 
         public LocalesValidationTask() { }
@@ -22,77 +20,116 @@ namespace Ryujinx.BuildValidationTasks
         {
             Console.WriteLine("Running Locale Validation Task...");
 
-            string path = projectPath + "assets/locales.json";
+            bool encounteredIssue = false;
+            string langPath = projectPath + "assets/Languages.json";
             string data;
 
-            using (StreamReader sr = new(path))
+            using (StreamReader sr = new(langPath))
             {
                 data = sr.ReadToEnd();
             }
 
-            LocalesJson json;
-
             if (isGitRunner && data.Contains("\r\n"))
-                throw new FormatException("locales.json is using CRLF line endings! It should be using LF line endings, rebuild locally to fix...");
+                throw new FormatException("Languages.json is using CRLF line endings! It should be using LF line endings, rebuild locally to fix...");
+
+            LanguagesJson langJson;
 
             try
             {
-                json = JsonSerializer.Deserialize<LocalesJson>(data);
-
+                langJson = JsonSerializer.Deserialize<LanguagesJson>(data);
             }
             catch (JsonException e)
             {
                 throw new JsonException(e.Message); //shorter and easier stacktrace
             }
 
-            bool encounteredIssue = false;
-
-            for (int i = 0; i < json.Locales.Count; i++)
+            foreach ((string code, string lang) in langJson.Languages)
             {
-                LocalesEntry locale = json.Locales[i];
-
-                foreach (string langCode in json.Languages.Where(lang => !locale.Translations.ContainsKey(lang)))
+                if (string.IsNullOrEmpty(lang))
                 {
-                    encounteredIssue = true;
-
-                    if (!isGitRunner)
-                    {
-                        locale.Translations.Add(langCode, string.Empty);
-                        Console.WriteLine($"Added '{langCode}' to Locale '{locale.ID}'");
-                    }
-                    else
-                    {
-                        Console.WriteLine($"Missing '{langCode}' in Locale '{locale.ID}'!");
-                    }
+                    throw new JsonException($"{code} language name missing!");
                 }
-
-                foreach (string langCode in json.Languages.Where(lang => locale.Translations.ContainsKey(lang) && lang != "en_US" && locale.Translations[lang] == locale.Translations["en_US"]))
-                {
-                    encounteredIssue = true;
-
-                    if (!isGitRunner)
-                    {
-                        locale.Translations[langCode] = string.Empty;
-                        Console.WriteLine($"Lanugage '{langCode}' is a duplicate of en_US in Locale '{locale.ID}'! Resetting it...");
-                    }
-                    else
-                    {
-                        Console.WriteLine($"Lanugage '{langCode}' is a duplicate of en_US in Locale '{locale.ID}'!");
-                    }
-                }
-
-                locale.Translations = locale.Translations.OrderBy(pair => pair.Key).ToDictionary(pair => pair.Key, pair => pair.Value);
-                json.Locales[i] = locale;
             }
 
-            if (isGitRunner && encounteredIssue)
-                throw new JsonException("1 or more locales are invalid! Rebuild locally to fix...");
+            string folderPath = projectPath + "assets/Locales/";
 
-            string jsonString = JsonSerializer.Serialize(json, _jsonOptions);
+            string[] paths = Directory.GetFiles(folderPath, "*.json", SearchOption.AllDirectories);
 
-            using (StreamWriter sw = new(path))
+            foreach (string path in paths)
             {
-                sw.Write(jsonString);
+                using (StreamReader sr = new(path))
+                {
+                    data = sr.ReadToEnd();
+                }
+
+                if (isGitRunner && data.Contains("\r\n"))
+                    throw new FormatException($"{Path.GetFileName(path)} is using CRLF line endings! It should be using LF line endings, rebuild locally to fix...");
+
+                LocalesJson json;
+
+                try
+                {
+                    json = JsonSerializer.Deserialize<LocalesJson>(data);
+                }
+                catch (JsonException e)
+                {
+                    throw new JsonException(e.Message); //shorter and easier stacktrace
+                }
+
+
+                for (int i = 0; i < json.Locales.Count; i++)
+                {
+                    LocalesEntry locale = json.Locales[i];
+
+                    foreach (string langCode in
+                             langJson.Languages.Keys.Where(lang => !locale.Translations.ContainsKey(lang)))
+                    {
+                        encounteredIssue = true;
+
+                        if (!isGitRunner)
+                        {
+                            locale.Translations.Add(langCode, string.Empty);
+                            Console.WriteLine($"Added '{langCode}' to Locale '{locale.ID}'");
+                        }
+                        else
+                        {
+                            Console.WriteLine($"Missing '{langCode}' in Locale '{locale.ID}'!");
+                        }
+                    }
+
+                    foreach (string langCode in langJson.Languages.Keys.Where(lang =>
+                                 locale.Translations.ContainsKey(lang) && lang != "en_US" &&
+                                 locale.Translations[lang] == locale.Translations["en_US"]))
+                    {
+                        encounteredIssue = true;
+
+                        if (!isGitRunner)
+                        {
+                            locale.Translations[langCode] = string.Empty;
+                            Console.WriteLine(
+                                $"Lanugage '{langCode}' is a duplicate of en_US in Locale '{locale.ID}'! Resetting it...");
+                        }
+                        else
+                        {
+                            Console.WriteLine(
+                                $"Lanugage '{langCode}' is a duplicate of en_US in Locale '{locale.ID}'!");
+                        }
+                    }
+
+                    locale.Translations = locale.Translations.OrderBy(pair => pair.Key)
+                        .ToDictionary(pair => pair.Key, pair => pair.Value);
+                    json.Locales[i] = locale;
+                }
+
+                if (isGitRunner && encounteredIssue)
+                    throw new JsonException("1 or more locales are invalid! Rebuild locally to fix...");
+
+                string jsonString = JsonSerializer.Serialize(json, _jsonOptions);
+
+                using (StreamWriter sw = new(path))
+                {
+                    sw.Write(jsonString);
+                }
             }
 
             Console.WriteLine("Finished Locale Validation Task!");
@@ -100,10 +137,13 @@ namespace Ryujinx.BuildValidationTasks
             return true;
         }
 
+        struct LanguagesJson
+        {
+            public Dictionary<string, string> Languages { get; set; }
+        }
+
         struct LocalesJson
         {
-            public Dictionary<string, string> Info { get; set; }
-            public List<string> Languages { get; set; }
             public List<LocalesEntry> Locales { get; set; }
         }
 

src/Ryujinx.Common/Utilities/EmbeddedResources.cs

@@ -127,7 +127,7 @@ namespace Ryujinx.Common
         public static string[] GetAllAvailableResources(string path, string ext = "")
         {
             return ResolveManifestPath(path).Item1.GetManifestResourceNames()
-                .Where(r => r.EndsWith(ext))
+                .Where(r => r.StartsWith(path.Replace('/', '.')) && r.EndsWith(ext))
                 .ToArray();
         }
 

src/Ryujinx.UI.LocaleGenerator/LocaleGenerator.cs

@@ -1,5 +1,7 @@
 using Microsoft.CodeAnalysis;
 using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
 using System.Linq;
 using System.Text;
 
@@ -10,23 +12,34 @@ namespace Ryujinx.UI.LocaleGenerator
     {
         public void Initialize(IncrementalGeneratorInitializationContext context)
         {
-            IncrementalValuesProvider<AdditionalText> localeFile = context.AdditionalTextsProvider.Where(static x => x.Path.EndsWith("locales.json"));
+            IncrementalValuesProvider<AdditionalText> localeFiles = context.AdditionalTextsProvider.Where(static x => Path.GetDirectoryName(x.Path)?.Replace('\\', '/').EndsWith("assets/Locales") ?? false);
 
-            IncrementalValuesProvider<string> contents = localeFile.Select((text, cancellationToken) => text.GetText(cancellationToken)!.ToString());
+            IncrementalValueProvider<ImmutableArray<(string, string)>> collectedContents = localeFiles.Select((text, cancellationToken) => (text.GetText(cancellationToken)!.ToString(), Path.GetFileName(text.Path))).Collect();
 
-            context.RegisterSourceOutput(contents, (spc, content) =>
+            context.RegisterSourceOutput(collectedContents, (spc, contents) =>
             {
-                IEnumerable<string> lines = content.Split('\n').Where(x => x.Trim().StartsWith("\"ID\":")).Select(x => x.Split(':')[1].Trim().Replace("\"", string.Empty).Replace(",", string.Empty));
-
                 StringBuilder enumSourceBuilder = new();
                 enumSourceBuilder.AppendLine("namespace Ryujinx.Ava.Common.Locale;");
                 enumSourceBuilder.AppendLine("public enum LocaleKeys");
                 enumSourceBuilder.AppendLine("{");
-                foreach (string? line in lines)
+                
+                foreach ((string, string) content in contents)
                 {
-                    enumSourceBuilder.AppendLine($"    {line},");
+                    IEnumerable<string> lines = content.Item1.Split('\n').Where(x => x.Trim().StartsWith("\"ID\":")).Select(x => x.Split(':')[1].Trim().Replace("\"", string.Empty).Replace(",", string.Empty));
+                    
+                    foreach (string? line in lines)
+                    {
+                        if (content.Item2 == "Root.json")
+                        {
+                            enumSourceBuilder.AppendLine($"    {line},");
+                        }
+                        else
+                        {
+                            enumSourceBuilder.AppendLine($"    {content.Item2.Split('.')[0]}_{line},");
+                        }
+                    }
                 }
-
+                
                 enumSourceBuilder.AppendLine("}");
 
                 spc.AddSource("LocaleKeys", enumSourceBuilder.ToString());

src/Ryujinx/Common/LocaleManager.cs

@@ -8,6 +8,8 @@ using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Globalization;
+using System.IO;
+using System.Linq;
 using System.Text.Json.Serialization;
 
 namespace Ryujinx.Ava.Common.Locale
@@ -158,52 +160,86 @@ namespace Ryujinx.Ava.Common.Locale
             LocaleChanged?.Invoke();
         }
 
-        private static LocalesJson? _localeData;
+        private static LocalesData? _localeData;
 
         private static Dictionary<LocaleKeys, string> LoadJsonLanguage(string languageCode)
         {
             Dictionary<LocaleKeys, string> localeStrings = new();
 
-            _localeData ??= EmbeddedResources.ReadAllText("Ryujinx/Assets/Locale.json")
-                .Into(it => JsonHelper.Deserialize(it, LocalesJsonContext.Default.LocalesJson));
-
-            foreach (LocalesEntry locale in _localeData.Value.Locales)
+            if (_localeData is null)
             {
-                if (locale.Translations.Count < _localeData.Value.Languages.Count)
+                Dictionary<string, LocalesJson> locales = [];
+
+                foreach (string uri in EmbeddedResources.GetAllAvailableResources("Ryujinx/Assets/Locales", ".json"))
                 {
-                    throw new Exception(
-                        $"Locale key {{{locale.ID}}} is missing languages! Has {locale.Translations.Count} translations, expected {_localeData.Value.Languages.Count}!");
+                    string path = uri[..^".json".Length];
+                    path = path.Replace('.', '/');
+                    path = path.Append(".json");
+                    
+                    locales.TryAdd(Path.GetFileName(path), EmbeddedResources.ReadAllText(path)
+                            .Into(it => JsonHelper.Deserialize(it, LocalesJsonContext.Default.LocalesJson)));
                 }
-
-                if (locale.Translations.Count > _localeData.Value.Languages.Count)
+                
+                _localeData = new LocalesData
                 {
-                    throw new Exception(
-                        $"Locale key {{{locale.ID}}} has too many languages! Has {locale.Translations.Count} translations, expected {_localeData.Value.Languages.Count}!");
-                }
+                    Languages = EmbeddedResources.ReadAllText("Ryujinx/Assets/Languages.json")
+                        .Into(it => JsonHelper.Deserialize(it, LanguagesJsonContext.Default.LanguagesJson)).Languages.Keys.ToList(),
+                    LocalesFiles = locales
+                };
+                
 
-                if (!Enum.TryParse<LocaleKeys>(locale.ID, out LocaleKeys localeKey))
-                    continue;
+            }
 
-                string str = locale.Translations.TryGetValue(languageCode, out string val) && !string.IsNullOrEmpty(val)
-                    ? val
-                    : locale.Translations[DefaultLanguageCode];
-
-                if (string.IsNullOrEmpty(str))
+            foreach (LocalesJson file in _localeData.Value.LocalesFiles.Values)
+            {
+                foreach (LocalesEntry locale in file.Locales)
                 {
-                    throw new Exception(
-                        $"Locale key '{locale.ID}' has no valid translations for desired language {languageCode}! {DefaultLanguageCode} is an empty string or null");
-                }
+                    if (locale.Translations.Count < _localeData.Value.Languages.Count)
+                    {
+                        throw new Exception(
+                            $"Locale key {{{locale.ID}}} is missing languages! Has {locale.Translations.Count} translations, expected {_localeData.Value.Languages.Count}!");
+                    }
 
-                localeStrings[localeKey] = str;
+                    if (locale.Translations.Count > _localeData.Value.Languages.Count)
+                    {
+                        throw new Exception(
+                            $"Locale key {{{locale.ID}}} has too many languages! Has {locale.Translations.Count} translations, expected {_localeData.Value.Languages.Count}!");
+                    }
+
+                    if (!Enum.TryParse<LocaleKeys>(locale.ID, out LocaleKeys localeKey))
+                        continue;
+
+                    string str = locale.Translations.TryGetValue(languageCode, out string val) && !string.IsNullOrEmpty(val)
+                        ? val
+                        : locale.Translations[DefaultLanguageCode];
+
+                    if (string.IsNullOrEmpty(str))
+                    {
+                        throw new Exception(
+                            $"Locale key '{locale.ID}' has no valid translations for desired language {languageCode}! {DefaultLanguageCode} is an empty string or null");
+                    }
+
+                    localeStrings[localeKey] = str;
+                }
             }
 
             return localeStrings;
         }
     }
 
-    public struct LocalesJson
+    public struct LocalesData
     {
         public List<string> Languages { get; set; }
+        public Dictionary<string, LocalesJson> LocalesFiles { get; set; }
+    }
+
+    public struct LanguagesJson
+    {
+        public Dictionary<string, string> Languages { get; set; }
+    }
+
+    public struct LocalesJson
+    {
         public List<LocalesEntry> Locales { get; set; }
     }
 
@@ -216,4 +252,8 @@ namespace Ryujinx.Ava.Common.Locale
     [JsonSourceGenerationOptions(WriteIndented = true)]
     [JsonSerializable(typeof(LocalesJson))]
     internal partial class LocalesJsonContext : JsonSerializerContext;
+    
+    [JsonSourceGenerationOptions(WriteIndented = true)]
+    [JsonSerializable(typeof(LanguagesJson))]
+    internal partial class LanguagesJsonContext : JsonSerializerContext;
 }

src/Ryujinx/Ryujinx.csproj

@@ -134,7 +134,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Remove="Assets\locales.json" />
+    <None Remove="Assets\**\*.json" />
     <None Remove="Assets\Styles\Styles.xaml" />
     <None Remove="Assets\Styles\Themes.xaml" />
     <None Remove="Assets\Icons\Controller_JoyConLeft.svg" />
@@ -156,8 +156,8 @@
     <EmbeddedResource Include="..\..\docs\compatibility.csv" LogicalName="RyujinxGameCompatibilityList">
       <Link>Assets\RyujinxGameCompatibility.csv</Link>
     </EmbeddedResource>
-    <EmbeddedResource Include="..\..\assets\locales.json">
-      <Link>Assets\Locale.json</Link>
+    <EmbeddedResource Include="..\..\assets\**\*.json">
+      <LinkBase>Assets</LinkBase>
     </EmbeddedResource>
     <EmbeddedResource Include="Assets\Styles\Styles.xaml" />
     <EmbeddedResource Include="Assets\Icons\Controller_JoyConLeft.svg" />
@@ -178,6 +178,6 @@
     <EmbeddedResource Include="Assets\UIImages\Logo_Ryujinx_AntiAlias.png" />
   </ItemGroup>
   <ItemGroup>
-    <AdditionalFiles Include="..\..\assets\locales.json" />
+    <AdditionalFiles Include="..\..\assets\Locales\*.json" />
   </ItemGroup>
 </Project>

src/Ryujinx/UI/Views/Main/MainMenuBarView.axaml.cs

@@ -85,29 +85,16 @@ namespace Ryujinx.Ava.UI.Views.Main
 
         private static IEnumerable<MenuItem> GenerateLanguageMenuItems()
         {
-            const string LocalePath = "Ryujinx/Assets/Locale.json";
+            const string LanguagesPath = "Ryujinx/Assets/Languages.json";
 
-            string languageJson = EmbeddedResources.ReadAllText(LocalePath);
+            string languageJson = EmbeddedResources.ReadAllText(LanguagesPath);
             string currentLanguageCode = LocaleManager.Instance.CurrentLanguageCode;
 
-            LocalesJson locales = JsonHelper.Deserialize(languageJson, LocalesJsonContext.Default.LocalesJson);
+            LanguagesJson languages = JsonHelper.Deserialize(languageJson, LanguagesJsonContext.Default.LanguagesJson);
 
-            foreach (string language in locales.Languages)
+            foreach ((string code, string language) in languages.Languages)
             {
-                int index = locales.Locales.FindIndex(x => x.ID == "Language");
-                string languageName;
-
-                if (index == -1)
-                {
-                    languageName = language;
-                }
-                else
-                {
-                    string tr = locales.Locales[index].Translations[language];
-                    languageName = string.IsNullOrEmpty(tr)
-                        ? language
-                        : tr;
-                }
+                string languageName = string.IsNullOrEmpty(language) ? code : language;
 
                 MenuItem menuItem = new()
                 {