Fractured Locales Support (ryubing/ryujinx!238)

See merge request ryubing/ryujinx!238

src/Ryujinx.BuildValidationTasks/LocalesValidationTask.cs

@@ -11,9 +11,7 @@ namespace Ryujinx.BuildValidationTasks
     {
         static readonly JsonSerializerOptions _jsonOptions = new()
         {
-            WriteIndented = true,
-            NewLine = "\n",
-            Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
+            WriteIndented = true, NewLine = "\n", Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
         };
 
         public LocalesValidationTask() { }
@@ -22,77 +20,116 @@ namespace Ryujinx.BuildValidationTasks
         {
             Console.WriteLine("Running Locale Validation Task...");
 
-            string path = projectPath + "assets/locales.json";
+            bool encounteredIssue = false;
+            string langPath = projectPath + "assets/Languages.json";
             string data;
 
-            using (StreamReader sr = new(path))
+            using (StreamReader sr = new(langPath))
             {
                 data = sr.ReadToEnd();
             }
 
-            LocalesJson json;
-
             if (isGitRunner && data.Contains("\r\n"))
-                throw new FormatException("locales.json is using CRLF line endings! It should be using LF line endings, rebuild locally to fix...");
+                throw new FormatException("Languages.json is using CRLF line endings! It should be using LF line endings, rebuild locally to fix...");
+
+            LanguagesJson langJson;
 
             try
             {
-                json = JsonSerializer.Deserialize<LocalesJson>(data);
-
+                langJson = JsonSerializer.Deserialize<LanguagesJson>(data);
             }
             catch (JsonException e)
             {
                 throw new JsonException(e.Message); //shorter and easier stacktrace
             }
 
-            bool encounteredIssue = false;
-
-            for (int i = 0; i < json.Locales.Count; i++)
+            foreach ((string code, string lang) in langJson.Languages)
             {
-                LocalesEntry locale = json.Locales[i];
-
-                foreach (string langCode in json.Languages.Where(lang => !locale.Translations.ContainsKey(lang)))
+                if (string.IsNullOrEmpty(lang))
                 {
-                    encounteredIssue = true;
-
-                    if (!isGitRunner)
-                    {
-                        locale.Translations.Add(langCode, string.Empty);
-                        Console.WriteLine($"Added '{langCode}' to Locale '{locale.ID}'");
-                    }
-                    else
-                    {
-                        Console.WriteLine($"Missing '{langCode}' in Locale '{locale.ID}'!");
-                    }
+                    throw new JsonException($"{code} language name missing!");
                 }
-
-                foreach (string langCode in json.Languages.Where(lang => locale.Translations.ContainsKey(lang) && lang != "en_US" && locale.Translations[lang] == locale.Translations["en_US"]))
-                {
-                    encounteredIssue = true;
-
-                    if (!isGitRunner)
-                    {
-                        locale.Translations[langCode] = string.Empty;
-                        Console.WriteLine($"Lanugage '{langCode}' is a duplicate of en_US in Locale '{locale.ID}'! Resetting it...");
-                    }
-                    else
-                    {
-                        Console.WriteLine($"Lanugage '{langCode}' is a duplicate of en_US in Locale '{locale.ID}'!");
-                    }
-                }
-
-                locale.Translations = locale.Translations.OrderBy(pair => pair.Key).ToDictionary(pair => pair.Key, pair => pair.Value);
-                json.Locales[i] = locale;
             }
 
-            if (isGitRunner && encounteredIssue)
-                throw new JsonException("1 or more locales are invalid! Rebuild locally to fix...");
+            string folderPath = projectPath + "assets/Locales/";
 
-            string jsonString = JsonSerializer.Serialize(json, _jsonOptions);
+            string[] paths = Directory.GetFiles(folderPath, "*.json", SearchOption.AllDirectories);
 
-            using (StreamWriter sw = new(path))
+            foreach (string path in paths)
             {
-                sw.Write(jsonString);
+                using (StreamReader sr = new(path))
+                {
+                    data = sr.ReadToEnd();
+                }
+
+                if (isGitRunner && data.Contains("\r\n"))
+                    throw new FormatException($"{Path.GetFileName(path)} is using CRLF line endings! It should be using LF line endings, rebuild locally to fix...");
+
+                LocalesJson json;
+
+                try
+                {
+                    json = JsonSerializer.Deserialize<LocalesJson>(data);
+                }
+                catch (JsonException e)
+                {
+                    throw new JsonException(e.Message); //shorter and easier stacktrace
+                }
+
+
+                for (int i = 0; i < json.Locales.Count; i++)
+                {
+                    LocalesEntry locale = json.Locales[i];
+
+                    foreach (string langCode in
+                             langJson.Languages.Keys.Where(lang => !locale.Translations.ContainsKey(lang)))
+                    {
+                        encounteredIssue = true;
+
+                        if (!isGitRunner)
+                        {
+                            locale.Translations.Add(langCode, string.Empty);
+                            Console.WriteLine($"Added '{langCode}' to Locale '{locale.ID}'");
+                        }
+                        else
+                        {
+                            Console.WriteLine($"Missing '{langCode}' in Locale '{locale.ID}'!");
+                        }
+                    }
+
+                    foreach (string langCode in langJson.Languages.Keys.Where(lang =>
+                                 locale.Translations.ContainsKey(lang) && lang != "en_US" &&
+                                 locale.Translations[lang] == locale.Translations["en_US"]))
+                    {
+                        encounteredIssue = true;
+
+                        if (!isGitRunner)
+                        {
+                            locale.Translations[langCode] = string.Empty;
+                            Console.WriteLine(
+                                $"Lanugage '{langCode}' is a duplicate of en_US in Locale '{locale.ID}'! Resetting it...");
+                        }
+                        else
+                        {
+                            Console.WriteLine(
+                                $"Lanugage '{langCode}' is a duplicate of en_US in Locale '{locale.ID}'!");
+                        }
+                    }
+
+                    locale.Translations = locale.Translations.OrderBy(pair => pair.Key)
+                        .ToDictionary(pair => pair.Key, pair => pair.Value);
+                    json.Locales[i] = locale;
+                }
+
+                if (isGitRunner && encounteredIssue)
+                    throw new JsonException("1 or more locales are invalid! Rebuild locally to fix...");
+
+                string jsonString = JsonSerializer.Serialize(json, _jsonOptions);
+
+                using (StreamWriter sw = new(path))
+                {
+                    sw.Write(jsonString);
+                }
             }
 
             Console.WriteLine("Finished Locale Validation Task!");
@@ -100,10 +137,13 @@ namespace Ryujinx.BuildValidationTasks
             return true;
         }
 
+        struct LanguagesJson
+        {
+            public Dictionary<string, string> Languages { get; set; }
+        }
+
         struct LocalesJson
         {
-            public Dictionary<string, string> Info { get; set; }
-            public List<string> Languages { get; set; }
             public List<LocalesEntry> Locales { get; set; }
         }
 

src/Ryujinx.Common/Utilities/EmbeddedResources.cs

@@ -127,7 +127,7 @@ namespace Ryujinx.Common
         public static string[] GetAllAvailableResources(string path, string ext = "")
         {
             return ResolveManifestPath(path).Item1.GetManifestResourceNames()
-                .Where(r => r.EndsWith(ext))
+                .Where(r => r.StartsWith(path.Replace('/', '.')) && r.EndsWith(ext))
                 .ToArray();
         }
 

src/Ryujinx.UI.LocaleGenerator/LocaleGenerator.cs

@@ -1,5 +1,7 @@
 using Microsoft.CodeAnalysis;
 using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
 using System.Linq;
 using System.Text;
 
@@ -10,23 +12,34 @@ namespace Ryujinx.UI.LocaleGenerator
     {
         public void Initialize(IncrementalGeneratorInitializationContext context)
         {
-            IncrementalValuesProvider<AdditionalText> localeFile = context.AdditionalTextsProvider.Where(static x => x.Path.EndsWith("locales.json"));
+            IncrementalValuesProvider<AdditionalText> localeFiles = context.AdditionalTextsProvider.Where(static x => Path.GetDirectoryName(x.Path)?.Replace('\\', '/').EndsWith("assets/Locales") ?? false);
 
-            IncrementalValuesProvider<string> contents = localeFile.Select((text, cancellationToken) => text.GetText(cancellationToken)!.ToString());
+            IncrementalValueProvider<ImmutableArray<(string, string)>> collectedContents = localeFiles.Select((text, cancellationToken) => (text.GetText(cancellationToken)!.ToString(), Path.GetFileName(text.Path))).Collect();
 
-            context.RegisterSourceOutput(contents, (spc, content) =>
+            context.RegisterSourceOutput(collectedContents, (spc, contents) =>
             {
-                IEnumerable<string> lines = content.Split('\n').Where(x => x.Trim().StartsWith("\"ID\":")).Select(x => x.Split(':')[1].Trim().Replace("\"", string.Empty).Replace(",", string.Empty));
-
                 StringBuilder enumSourceBuilder = new();
                 enumSourceBuilder.AppendLine("namespace Ryujinx.Ava.Common.Locale;");
                 enumSourceBuilder.AppendLine("public enum LocaleKeys");
                 enumSourceBuilder.AppendLine("{");
-                foreach (string? line in lines)
+                
+                foreach ((string, string) content in contents)
                 {
-                    enumSourceBuilder.AppendLine($"    {line},");
+                    IEnumerable<string> lines = content.Item1.Split('\n').Where(x => x.Trim().StartsWith("\"ID\":")).Select(x => x.Split(':')[1].Trim().Replace("\"", string.Empty).Replace(",", string.Empty));
+                    
+                    foreach (string? line in lines)
+                    {
+                        if (content.Item2 == "Root.json")
+                        {
+                            enumSourceBuilder.AppendLine($"    {line},");
+                        }
+                        else
+                        {
+                            enumSourceBuilder.AppendLine($"    {content.Item2.Split('.')[0]}_{line},");
+                        }
+                    }
                 }
-
+                
                 enumSourceBuilder.AppendLine("}");
 
                 spc.AddSource("LocaleKeys", enumSourceBuilder.ToString());

src/Ryujinx/Common/LocaleManager.cs

@@ -8,6 +8,8 @@ using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Globalization;
+using System.IO;
+using System.Linq;
 using System.Text.Json.Serialization;
 
 namespace Ryujinx.Ava.Common.Locale
@@ -158,52 +160,86 @@ namespace Ryujinx.Ava.Common.Locale
             LocaleChanged?.Invoke();
         }
 
-        private static LocalesJson? _localeData;
+        private static LocalesData? _localeData;
 
         private static Dictionary<LocaleKeys, string> LoadJsonLanguage(string languageCode)
         {
             Dictionary<LocaleKeys, string> localeStrings = new();
 
-            _localeData ??= EmbeddedResources.ReadAllText("Ryujinx/Assets/Locale.json")
-                .Into(it => JsonHelper.Deserialize(it, LocalesJsonContext.Default.LocalesJson));
-
-            foreach (LocalesEntry locale in _localeData.Value.Locales)
+            if (_localeData is null)
             {
-                if (locale.Translations.Count < _localeData.Value.Languages.Count)
+                Dictionary<string, LocalesJson> locales = [];
+
+                foreach (string uri in EmbeddedResources.GetAllAvailableResources("Ryujinx/Assets/Locales", ".json"))
                 {
-                    throw new Exception(
-                        $"Locale key {{{locale.ID}}} is missing languages! Has {locale.Translations.Count} translations, expected {_localeData.Value.Languages.Count}!");
+                    string path = uri[..^".json".Length];
+                    path = path.Replace('.', '/');
+                    path = path.Append(".json");
+                    
+                    locales.TryAdd(Path.GetFileName(path), EmbeddedResources.ReadAllText(path)
+                            .Into(it => JsonHelper.Deserialize(it, LocalesJsonContext.Default.LocalesJson)));
                 }
-
-                if (locale.Translations.Count > _localeData.Value.Languages.Count)
+                
+                _localeData = new LocalesData
                 {
-                    throw new Exception(
-                        $"Locale key {{{locale.ID}}} has too many languages! Has {locale.Translations.Count} translations, expected {_localeData.Value.Languages.Count}!");
-                }
+                    Languages = EmbeddedResources.ReadAllText("Ryujinx/Assets/Languages.json")
+                        .Into(it => JsonHelper.Deserialize(it, LanguagesJsonContext.Default.LanguagesJson)).Languages.Keys.ToList(),
+                    LocalesFiles = locales
+                };
+                
 
-                if (!Enum.TryParse<LocaleKeys>(locale.ID, out LocaleKeys localeKey))
-                    continue;
+            }
 
-                string str = locale.Translations.TryGetValue(languageCode, out string val) && !string.IsNullOrEmpty(val)
-                    ? val
-                    : locale.Translations[DefaultLanguageCode];
-
-                if (string.IsNullOrEmpty(str))
+            foreach (LocalesJson file in _localeData.Value.LocalesFiles.Values)
+            {
+                foreach (LocalesEntry locale in file.Locales)
                 {
-                    throw new Exception(
-                        $"Locale key '{locale.ID}' has no valid translations for desired language {languageCode}! {DefaultLanguageCode} is an empty string or null");
-                }
+                    if (locale.Translations.Count < _localeData.Value.Languages.Count)
+                    {
+                        throw new Exception(
+                            $"Locale key {{{locale.ID}}} is missing languages! Has {locale.Translations.Count} translations, expected {_localeData.Value.Languages.Count}!");
+                    }
 
-                localeStrings[localeKey] = str;
+                    if (locale.Translations.Count > _localeData.Value.Languages.Count)
+                    {
+                        throw new Exception(
+                            $"Locale key {{{locale.ID}}} has too many languages! Has {locale.Translations.Count} translations, expected {_localeData.Value.Languages.Count}!");
+                    }
+
+                    if (!Enum.TryParse<LocaleKeys>(locale.ID, out LocaleKeys localeKey))
+                        continue;
+
+                    string str = locale.Translations.TryGetValue(languageCode, out string val) && !string.IsNullOrEmpty(val)
+                        ? val
+                        : locale.Translations[DefaultLanguageCode];
+
+                    if (string.IsNullOrEmpty(str))
+                    {
+                        throw new Exception(
+                            $"Locale key '{locale.ID}' has no valid translations for desired language {languageCode}! {DefaultLanguageCode} is an empty string or null");
+                    }
+
+                    localeStrings[localeKey] = str;
+                }
             }
 
             return localeStrings;
         }
     }
 
-    public struct LocalesJson
+    public struct LocalesData
     {
         public List<string> Languages { get; set; }
+        public Dictionary<string, LocalesJson> LocalesFiles { get; set; }
+    }
+
+    public struct LanguagesJson
+    {
+        public Dictionary<string, string> Languages { get; set; }
+    }
+
+    public struct LocalesJson
+    {
         public List<LocalesEntry> Locales { get; set; }
     }
 
@@ -216,4 +252,8 @@ namespace Ryujinx.Ava.Common.Locale
     [JsonSourceGenerationOptions(WriteIndented = true)]
     [JsonSerializable(typeof(LocalesJson))]
     internal partial class LocalesJsonContext : JsonSerializerContext;
+    
+    [JsonSourceGenerationOptions(WriteIndented = true)]
+    [JsonSerializable(typeof(LanguagesJson))]
+    internal partial class LanguagesJsonContext : JsonSerializerContext;
 }

src/Ryujinx/Ryujinx.csproj

@@ -134,7 +134,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Remove="Assets\locales.json" />
+    <None Remove="Assets\**\*.json" />
     <None Remove="Assets\Styles\Styles.xaml" />
     <None Remove="Assets\Styles\Themes.xaml" />
     <None Remove="Assets\Icons\Controller_JoyConLeft.svg" />
@@ -156,8 +156,8 @@
     <EmbeddedResource Include="..\..\docs\compatibility.csv" LogicalName="RyujinxGameCompatibilityList">
       <Link>Assets\RyujinxGameCompatibility.csv</Link>
     </EmbeddedResource>
-    <EmbeddedResource Include="..\..\assets\locales.json">
-      <Link>Assets\Locale.json</Link>
+    <EmbeddedResource Include="..\..\assets\**\*.json">
+      <LinkBase>Assets</LinkBase>
     </EmbeddedResource>
     <EmbeddedResource Include="Assets\Styles\Styles.xaml" />
     <EmbeddedResource Include="Assets\Icons\Controller_JoyConLeft.svg" />
@@ -178,6 +178,6 @@
     <EmbeddedResource Include="Assets\UIImages\Logo_Ryujinx_AntiAlias.png" />
   </ItemGroup>
   <ItemGroup>
-    <AdditionalFiles Include="..\..\assets\locales.json" />
+    <AdditionalFiles Include="..\..\assets\Locales\*.json" />
   </ItemGroup>
 </Project>

src/Ryujinx/UI/Views/Main/MainMenuBarView.axaml.cs

@@ -85,29 +85,16 @@ namespace Ryujinx.Ava.UI.Views.Main
 
         private static IEnumerable<MenuItem> GenerateLanguageMenuItems()
         {
-            const string LocalePath = "Ryujinx/Assets/Locale.json";
+            const string LanguagesPath = "Ryujinx/Assets/Languages.json";
 
-            string languageJson = EmbeddedResources.ReadAllText(LocalePath);
+            string languageJson = EmbeddedResources.ReadAllText(LanguagesPath);
             string currentLanguageCode = LocaleManager.Instance.CurrentLanguageCode;
 
-            LocalesJson locales = JsonHelper.Deserialize(languageJson, LocalesJsonContext.Default.LocalesJson);
+            LanguagesJson languages = JsonHelper.Deserialize(languageJson, LanguagesJsonContext.Default.LanguagesJson);
 
-            foreach (string language in locales.Languages)
+            foreach ((string code, string language) in languages.Languages)
             {
-                int index = locales.Locales.FindIndex(x => x.ID == "Language");
-                string languageName;
-
-                if (index == -1)
-                {
-                    languageName = language;
-                }
-                else
-                {
-                    string tr = locales.Locales[index].Translations[language];
-                    languageName = string.IsNullOrEmpty(tr)
-                        ? language
-                        : tr;
-                }
+                string languageName = string.IsNullOrEmpty(language) ? code : language;
 
                 MenuItem menuItem = new()
                 {