Linux: Fix file picker not launching from disabling core dumps (ryubing/ryujinx!249)

See merge request ryubing/ryujinx!249

src/Ryujinx.Common/Utilities/OsUtils.cs

@@ -22,10 +22,11 @@ namespace Ryujinx.Common.Utilities
         }
 
         // "dumpable" attribute of the calling process
+        private const int PR_GET_DUMPABLE = 3;
         private const int PR_SET_DUMPABLE = 4;
 
-        [DllImport("libc", SetLastError = true)]
-        private static extern int prctl(int option, int arg2);
+        [LibraryImport("libc", SetLastError = true)]
+        private static partial int prctl(int option, int arg2);
 
         public static void SetCoreDumpable(bool dumpable)
         {
@@ -36,5 +37,13 @@ namespace Ryujinx.Common.Utilities
                 Debug.Assert(result == 0);
             }
         }
+
+        // Use the below line to display dumpable status in the console:
+        // Console.WriteLine($"{OsUtils.IsCoreDumpable()}");
+        public static bool IsCoreDumpable()
+        {
+            int result = prctl(PR_GET_DUMPABLE, 0);
+            return result == 1;
+        }
     }
 }

src/Ryujinx/Program.cs

@@ -42,6 +42,7 @@ namespace Ryujinx.Ava
         public static bool PreviewerDetached { get; private set; }
         public static bool UseHardwareAcceleration { get; private set; }
         public static string BackendThreadingArg { get; private set; }
+        public static bool CoreDumpArg { get; private set; }
 
         private const uint MbIconwarning = 0x30;
 
@@ -81,6 +82,8 @@ namespace Ryujinx.Ava
             bool noGuiArg = ConsumeCommandLineArgument(ref args, "--no-gui") || ConsumeCommandLineArgument(ref args, "nogui");
             bool coreDumpArg = ConsumeCommandLineArgument(ref args, "--core-dumps");
 
+            CoreDumpArg = coreDumpArg;
+
             // TODO: Ryujinx causes core dumps on Linux when it exits "uncleanly", eg. through an unhandled exception.
             //       This is undesirable and causes very odd behavior during development (the process stops responding, 
             //       the .NET debugger freezes or suddenly detaches, /tmp/ gets filled etc.), unless explicitly requested by the user.

src/Ryujinx/Utilities/StorageProviderExtensions.cs

@@ -1,5 +1,7 @@
 using Avalonia.Platform.Storage;
 using Gommon;
+using Ryujinx.Common.Utilities;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
@@ -11,29 +13,42 @@ namespace Ryujinx.Ava.Utilities
         extension(IStorageProvider storageProvider)
         {
             public Task<Optional<IStorageFolder>> OpenSingleFolderPickerAsync(FolderPickerOpenOptions openOptions = null) =>
-                storageProvider.OpenFolderPickerAsync(FixOpenOptions(openOptions, false))
+                CoreDumpable(() => storageProvider.OpenFolderPickerAsync(FixOpenOptions(openOptions, false)))
                     .Then(folders => folders.FindFirst());
-            
+
             public Task<Optional<IStorageFile>> OpenSingleFilePickerAsync(FilePickerOpenOptions openOptions = null) =>
-                storageProvider.OpenFilePickerAsync(FixOpenOptions(openOptions, false))
+                CoreDumpable(() => storageProvider.OpenFilePickerAsync(FixOpenOptions(openOptions, false)))
                     .Then(files => files.FindFirst());
-            
+
             public Task<Optional<IReadOnlyList<IStorageFolder>>> OpenMultiFolderPickerAsync(FolderPickerOpenOptions openOptions = null) =>
-                storageProvider.OpenFolderPickerAsync(FixOpenOptions(openOptions, true))
+                CoreDumpable(() => storageProvider.OpenFolderPickerAsync(FixOpenOptions(openOptions, true)))
                     .Then(folders => folders.Count > 0 ? Optional.Of(folders) : default);
-            
+
             public Task<Optional<IReadOnlyList<IStorageFile>>> OpenMultiFilePickerAsync(FilePickerOpenOptions openOptions = null) =>
-                storageProvider.OpenFilePickerAsync(FixOpenOptions(openOptions, true))
+                CoreDumpable(() => storageProvider.OpenFilePickerAsync(FixOpenOptions(openOptions, true)))
                     .Then(files => files.Count > 0 ? Optional.Of(files) : default);
         }
 
+        private static async Task<T> CoreDumpable<T>(Func<Task<T>> picker)
+        {
+            OsUtils.SetCoreDumpable(true);
+            try
+            {
+                return await picker();
+            }
+            finally
+            {
+                if (!Program.CoreDumpArg)
+                    OsUtils.SetCoreDumpable(false);
+            }
+        }
+
         private static FilePickerOpenOptions FixOpenOptions(this FilePickerOpenOptions openOptions, bool allowMultiple)
         {
             if (openOptions is null)
                 return new FilePickerOpenOptions { AllowMultiple = allowMultiple };
 
             openOptions.AllowMultiple = allowMultiple;
-
             return openOptions;
         }
 
@@ -43,7 +58,6 @@ namespace Ryujinx.Ava.Utilities
                 return new FolderPickerOpenOptions { AllowMultiple = allowMultiple };
 
             openOptions.AllowMultiple = allowMultiple;
-
             return openOptions;
         }
     }