JJBanana13/ryujinx
1
0
Fork 0
mirror of https://git.ryujinx.app/ryubing/ryujinx.git synced 2026-07-02 09:09:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

instanced jit cache (#152)

Browse Source 
- jit caches are no longer static.
- JitUnwindWindows now supports multiple jit caches.
- Jit caches should no longer cause crashes due to entry offset collisions.
- cpu tests now run concurrently.

Co-authored-by: LotP1 <68976644+LotP1@users.noreply.github.com>
Reviewed-on: https://git.ryujinx.app/projects/Ryubing/pulls/152
This commit is contained in:
LotP
2026-06-30 13:40:18 +00:00
committed by LotP
parent 59a6a3a5f7
commit 6fb71b08ae
26 changed files with 345 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/ARMeilleure/CodeGen/CompiledFunction.cs
View File

@@ -8,7 +8,7 @@ namespace ARMeilleure.CodeGen
/// <summary>
/// Represents a compiled function.
/// </summary>
readonly struct CompiledFunction
public readonly struct CompiledFunction
{
/// <summary>
/// Gets the machine code of the <see cref="CompiledFunction"/>.
@@ -44,10 +44,11 @@ namespace ARMeilleure.CodeGen
/// <typeparamref name="T"/> pointing to the mapped function.
/// </summary>
/// <typeparam name="T">Type of delegate</typeparam>
/// <param name="jitCache">The jit cache to map the function into</param>
/// <returns>A delegate of type <typeparamref name="T"/> pointing to the mapped function</returns>
public T Map<T>()
public T Map<T>(JitCache jitCache)
{
return MapWithPointer<T>(out _);
return MapWithPointer<T>(jitCache, out _);
}
/// <summary>
@@ -55,11 +56,12 @@ namespace ARMeilleure.CodeGen
/// <typeparamref name="T"/> pointing to the mapped function.
/// </summary>
/// <typeparam name="T">Type of delegate</typeparam>
/// <param name="jitCache">The jit cache to map the function into</param>
/// <param name="codePointer">Pointer to the function code in memory</param>
/// <returns>A delegate of type <typeparamref name="T"/> pointing to the mapped function</returns>
public T MapWithPointer<T>(out nint codePointer)
public T MapWithPointer<T>(JitCache jitCache, out nint codePointer)
{
codePointer = JitCache.Map(this);
codePointer = jitCache.Map(this);
return Marshal.GetDelegateForFunctionPointer<T>(codePointer);
}

2
src/ARMeilleure/CodeGen/Linking/RelocEntry.cs
View File

@@ -3,7 +3,7 @@ namespace ARMeilleure.CodeGen.Linking
/// <summary>
/// Represents a relocation.
/// </summary>
readonly struct RelocEntry
public readonly struct RelocEntry
{
public const int Stride = 13; // Bytes.

2
src/ARMeilleure/CodeGen/Linking/RelocInfo.cs
View File

@@ -5,7 +5,7 @@ namespace ARMeilleure.CodeGen.Linking
/// <summary>
/// Represents relocation information about a <see cref="CompiledFunction"/>.
/// </summary>
readonly struct RelocInfo
public readonly struct RelocInfo
{
/// <summary>
/// Gets an empty <see cref="RelocInfo"/>.

2
src/ARMeilleure/CodeGen/Linking/Symbol.cs
View File

@@ -5,7 +5,7 @@ namespace ARMeilleure.CodeGen.Linking
/// <summary>
/// Represents a symbol.
/// </summary>
readonly struct Symbol
public readonly struct Symbol
{
private readonly ulong _value;

2
src/ARMeilleure/CodeGen/Linking/SymbolType.cs
View File

@@ -3,7 +3,7 @@ namespace ARMeilleure.CodeGen.Linking
/// <summary>
/// Types of <see cref="Symbol"/>.
/// </summary>
enum SymbolType : byte
public enum SymbolType : byte
{
/// <summary>
/// Refers to nothing, i.e no symbol.

2
src/ARMeilleure/CodeGen/Unwinding/UnwindInfo.cs
View File

@@ -1,6 +1,6 @@
namespace ARMeilleure.CodeGen.Unwinding
{
struct UnwindInfo
public struct UnwindInfo
{
public const int Stride = 4; // Bytes.

2
src/ARMeilleure/CodeGen/Unwinding/UnwindPseudoOp.cs
View File

@@ -1,6 +1,6 @@
namespace ARMeilleure.CodeGen.Unwinding
{
enum UnwindPseudoOp
public enum UnwindPseudoOp
{
PushReg = 0,
SetFrame = 1,

2
src/ARMeilleure/CodeGen/Unwinding/UnwindPushEntry.cs
View File

@@ -1,6 +1,6 @@
namespace ARMeilleure.CodeGen.Unwinding
{
struct UnwindPushEntry
public struct UnwindPushEntry
{
public const int Stride = 16; // Bytes.

2
src/ARMeilleure/Memory/ReservedRegion.cs
View File

@@ -2,7 +2,7 @@ using System;
namespace ARMeilleure.Memory
{
public class ReservedRegion
public class ReservedRegion : IDisposable
{
public const int DefaultGranularity = 65536; // Mapping granularity in Windows.

13
src/ARMeilleure/Signal/TestMethods.cs
View File

@@ -1,5 +1,6 @@
using ARMeilleure.IntermediateRepresentation;
using ARMeilleure.Translation;
using ARMeilleure.Translation.Cache;
using System.Runtime.InteropServices;
using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
@@ -17,7 +18,7 @@ namespace ARMeilleure.Signal
public delegate int DebugThreadLocalMapGetOrReserve(int threadId, int initialState);
public delegate void DebugNativeWriteLoop(nint nativeWriteLoopPtr, nint writePtr);
public static DebugPartialUnmap GenerateDebugPartialUnmap()
public static DebugPartialUnmap GenerateDebugPartialUnmap(JitCache jitCache)
{
EmitterContext context = new();
@@ -31,10 +32,10 @@ namespace ARMeilleure.Signal
OperandType[] argTypes = [OperandType.I64];
return Compiler.Compile(cfg, argTypes, OperandType.I32, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<DebugPartialUnmap>();
return Compiler.Compile(cfg, argTypes, OperandType.I32, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<DebugPartialUnmap>(jitCache);
}
public static DebugThreadLocalMapGetOrReserve GenerateDebugThreadLocalMapGetOrReserve(nint structPtr)
public static DebugThreadLocalMapGetOrReserve GenerateDebugThreadLocalMapGetOrReserve(JitCache jitCache, nint structPtr)
{
EmitterContext context = new();
@@ -48,10 +49,10 @@ namespace ARMeilleure.Signal
OperandType[] argTypes = [OperandType.I64];
return Compiler.Compile(cfg, argTypes, OperandType.I32, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<DebugThreadLocalMapGetOrReserve>();
return Compiler.Compile(cfg, argTypes, OperandType.I32, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<DebugThreadLocalMapGetOrReserve>(jitCache);
}
public static DebugNativeWriteLoop GenerateDebugNativeWriteLoop()
public static DebugNativeWriteLoop GenerateDebugNativeWriteLoop(JitCache jitCache)
{
EmitterContext context = new();
@@ -77,7 +78,7 @@ namespace ARMeilleure.Signal
OperandType[] argTypes = [OperandType.I64];
return Compiler.Compile(cfg, argTypes, OperandType.None, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<DebugNativeWriteLoop>();
return Compiler.Compile(cfg, argTypes, OperandType.None, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<DebugNativeWriteLoop>(jitCache);
}
}
}

2
src/ARMeilleure/Translation/Cache/CacheEntry.cs
View File

@@ -4,7 +4,7 @@ using System.Diagnostics.CodeAnalysis;
namespace ARMeilleure.Translation.Cache
{
readonly struct CacheEntry : IComparable<CacheEntry>
public readonly struct CacheEntry : IComparable<CacheEntry>
{
public int Offset { get; }
public int Size { get; }

33
src/ARMeilleure/Translation/Cache/CacheMemoryAllocator.cs
View File

@@ -23,11 +23,30 @@ namespace ARMeilleure.Translation.Cache
}
}
private readonly int _regionSize;
private int _regionCount;
private readonly List<MemoryBlock> _blocks = [];
public CacheMemoryAllocator(int capacity)
public CacheMemoryAllocator(int regionSize, int initialRegionCount = 1)
{
_blocks.Add(new MemoryBlock(0, capacity));
_regionCount = 0;
_regionSize = regionSize;
for (; initialRegionCount > 0; initialRegionCount--)
{
_blocks.Add(new MemoryBlock(_regionSize * _regionCount, _regionSize));
_regionCount++;
}
}
public void AddNewBlocks(int count)
{
for (; count > 0; count--)
{
_blocks.Add(new MemoryBlock(_regionSize * _regionCount, _regionSize));
_regionCount++;
}
}
public int Allocate(int size)
@@ -66,12 +85,13 @@ namespace ARMeilleure.Translation.Cache
index = ~index;
}
if (index < _blocks.Count)
int endOffs = block.Offset + block.Size;
// Don't merge blocks from different allocations
if (index < _blocks.Count && endOffs % _regionSize != 0)
{
MemoryBlock next = _blocks[index];
int endOffs = block.Offset + block.Size;
if (next.Offset == endOffs)
{
block = new MemoryBlock(block.Offset, block.Size + next.Size);
@@ -79,7 +99,8 @@ namespace ARMeilleure.Translation.Cache
}
}
if (index > 0)
// Don't merge blocks from different allocations
if (index > 0 && block.Offset % _regionSize != 0)
{
MemoryBlock prev = _blocks[index - 1];

169
src/ARMeilleure/Translation/Cache/JitCache.cs
View File

@@ -14,62 +14,35 @@ using System.Threading;
namespace ARMeilleure.Translation.Cache
{
static partial class JitCache
public partial class JitCache : IDisposable
{
private static readonly int _pageSize = (int)MemoryBlock.GetPageSize();
private static readonly int _pageMask = _pageSize - 1;
private const int CodeAlignment = 4; // Bytes.
private const int CacheSize = 256 * 1024 * 1024;
private const uint CacheSize = 256 * 1024 * 1024;
private static JitCacheInvalidation _jitCacheInvalidator;
private readonly JitCacheInvalidation _jitCacheInvalidator;
private static readonly List<CacheMemoryAllocator> _cacheAllocators = [];
private readonly CacheMemoryAllocator _cacheAllocator;
private static readonly List<CacheEntry> _cacheEntries = [];
private readonly List<CacheEntry> _cacheEntries = [];
private static readonly Lock _lock = new();
private static bool _initialized;
private readonly Lock _lock = new();
private static readonly List<ReservedRegion> _jitRegions = [];
private static int _activeRegionIndex = 0;
private readonly List<ReservedRegion> _jitRegions = [];
[SupportedOSPlatform("windows")]
[LibraryImport("kernel32.dll", SetLastError = true)]
public static partial nint FlushInstructionCache(nint hProcess, nint lpAddress, nuint dwSize);
private static partial nint FlushInstructionCache(nint hProcess, nint lpAddress, nuint dwSize);
public static void Initialize(IJitMemoryAllocator allocator)
public JitCache(IJitMemoryAllocator allocator)
{
lock (_lock)
{
if (_initialized)
{
if (OperatingSystem.IsWindows())
{
JitUnwindWindows.RemoveFunctionTableHandler(
_jitRegions[0].Pointer);
}
_jitRegions.Add(new(allocator, CacheSize));
for (int i = 0; i < _jitRegions.Count; i++)
{
_jitRegions[i].Dispose();
}
_jitRegions.Clear();
_cacheAllocators.Clear();
}
else
{
_initialized = true;
}
_activeRegionIndex = 0;
ReservedRegion firstRegion = new(allocator, CacheSize);
_jitRegions.Add(firstRegion);
CacheMemoryAllocator firstCacheAllocator = new(CacheSize);
_cacheAllocators.Add(firstCacheAllocator);
_cacheAllocator = new((int)CacheSize);
if (!OperatingSystem.IsWindows() && !OperatingSystem.IsMacOS())
{
@@ -79,23 +52,20 @@ namespace ARMeilleure.Translation.Cache
if (OperatingSystem.IsWindows())
{
JitUnwindWindows.InstallFunctionTableHandler(
firstRegion.Pointer, CacheSize, firstRegion.Pointer + Allocate(_pageSize)
this, _jitRegions[0].Pointer, CacheSize, _jitRegions[0].Pointer + Allocate(_pageSize)
);
}
}
}
public static nint Map(CompiledFunction func)
public nint Map(CompiledFunction func)
{
byte[] code = func.Code;
lock (_lock)
{
Debug.Assert(_initialized);
int funcOffset = Allocate(code.Length);
ReservedRegion targetRegion = _jitRegions[_activeRegionIndex];
nint funcPtr = targetRegion.Pointer + funcOffset;
nint funcPtr = GetFunctionPtr(funcOffset);
if (OperatingSystem.IsMacOS() && RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
{
@@ -109,9 +79,9 @@ namespace ARMeilleure.Translation.Cache
}
else
{
ReprotectAsWritable(targetRegion, funcOffset, code.Length);
ReprotectAsWritable(funcOffset, code.Length);
Marshal.Copy(code, 0, funcPtr, code.Length);
ReprotectAsExecutable(targetRegion, funcOffset, code.Length);
ReprotectAsExecutable(funcOffset, code.Length);
if (OperatingSystem.IsWindows() && RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
{
@@ -129,25 +99,24 @@ namespace ARMeilleure.Translation.Cache
}
}
public static void Unmap(nint pointer)
public void Unmap(nint pointer)
{
lock (_lock)
{
Debug.Assert(_initialized);
foreach (ReservedRegion region in _jitRegions)
for (int i = 0; i < _jitRegions.Count; i++)
{
if (pointer.ToInt64() < region.Pointer.ToInt64() ||
pointer.ToInt64() >= (region.Pointer + CacheSize).ToInt64())
ReservedRegion jitRegion = _jitRegions[i];
if (pointer.ToInt64() < jitRegion.Pointer.ToInt64() ||
pointer.ToInt64() >= (jitRegion.Pointer + (nint)CacheSize).ToInt64())
{
continue;
}
int funcOffset = (int)(pointer.ToInt64() - region.Pointer.ToInt64());
int funcOffset = (int)(pointer.ToInt64() - jitRegion.Pointer.ToInt64() + i * CacheSize);
if (TryFind(funcOffset, out CacheEntry entry, out int entryIndex) && entry.Offset == funcOffset)
{
_cacheAllocators[_activeRegionIndex].Free(funcOffset, AlignCodeSize(entry.Size));
_cacheAllocator.Free(funcOffset, AlignCodeSize(entry.Size));
_cacheEntries.RemoveAt(entryIndex);
}
@@ -156,53 +125,63 @@ namespace ARMeilleure.Translation.Cache
}
}
private static void ReprotectAsWritable(ReservedRegion region, int offset, int size)
private void ReprotectAsWritable(int offset, int size)
{
int endOffs = offset + size;
int regionStart = offset & ~_pageMask;
int regionEnd = (endOffs + _pageMask) & ~_pageMask;
int regionStart = (offset % (int)CacheSize) & ~_pageMask;
int regionEnd = ((endOffs % (int)CacheSize) + _pageMask) & ~_pageMask;
region.Block.MapAsRwx((ulong)regionStart, (ulong)(regionEnd - regionStart));
GetRegion(offset).Block.MapAsRwx((ulong)regionStart, (ulong)(regionEnd - regionStart));
}
private static void ReprotectAsExecutable(ReservedRegion region, int offset, int size)
private void ReprotectAsExecutable(int offset, int size)
{
int endOffs = offset + size;
int regionStart = offset & ~_pageMask;
int regionEnd = (endOffs + _pageMask) & ~_pageMask;
int regionStart = (offset % (int)CacheSize) & ~_pageMask;
int regionEnd = ((endOffs % (int)CacheSize) + _pageMask) & ~_pageMask;
region.Block.MapAsRx((ulong)regionStart, (ulong)(regionEnd - regionStart));
GetRegion(offset).Block.MapAsRx((ulong)regionStart, (ulong)(regionEnd - regionStart));
}
private static int Allocate(int codeSize)
private int Allocate(int codeSize)
{
codeSize = AlignCodeSize(codeSize);
int allocOffset = _cacheAllocators[_activeRegionIndex].Allocate(codeSize);
int allocOffset = _cacheAllocator.Allocate(codeSize);
if (allocOffset >= 0)
{
_jitRegions[_activeRegionIndex].ExpandIfNeeded((ulong)allocOffset + (ulong)codeSize);
GetRegion(allocOffset).ExpandIfNeeded((ulong)(allocOffset % (int)CacheSize) + (ulong)codeSize);
return allocOffset;
}
int exhaustedRegion = _activeRegionIndex;
_cacheAllocator.AddNewBlocks(1);
ReservedRegion newRegion = new(_jitRegions[0].Allocator, CacheSize);
Logger.Warning?.Print(LogClass.Cpu, $"JIT Cache of size {(_jitRegions.Count * CacheSize).Bytes()} exhausted, creating new Cache Region ({((_jitRegions.Count + 1) * CacheSize).Bytes()} Total Allocation).");
_jitRegions.Add(newRegion);
_activeRegionIndex = _jitRegions.Count - 1;
Logger.Warning?.Print(LogClass.Cpu, $"JIT Cache Region {exhaustedRegion} exhausted, creating new Cache Region {_activeRegionIndex} ({((long)(_activeRegionIndex + 1) * CacheSize).Bytes()} Total Allocation).");
_cacheAllocators.Add(new CacheMemoryAllocator(CacheSize));
int allocOffsetNew = _cacheAllocators[_activeRegionIndex].Allocate(codeSize);
if (allocOffsetNew < 0)
allocOffset = _cacheAllocator.Allocate(codeSize);
if (allocOffset < 0)
{
throw new OutOfMemoryException("Failed to allocate in new Cache Region!");
}
newRegion.ExpandIfNeeded((ulong)allocOffsetNew + (ulong)codeSize);
return allocOffsetNew;
GetRegion(allocOffset).ExpandIfNeeded((ulong)(allocOffset % (int)CacheSize) + (ulong)codeSize);
return allocOffset;
}
private nint GetFunctionPtr(int offset)
{
return GetRegion(offset).Pointer + (offset % (int)CacheSize);
}
private ReservedRegion GetRegion(int offset)
{
int index = offset / (int)CacheSize;
return _jitRegions[index];
}
private static int AlignCodeSize(int codeSize)
@@ -210,7 +189,7 @@ namespace ARMeilleure.Translation.Cache
return checked(codeSize + (CodeAlignment - 1)) & ~(CodeAlignment - 1);
}
private static void Add(int offset, int size, UnwindInfo unwindInfo)
private void Add(int offset, int size, UnwindInfo unwindInfo)
{
CacheEntry entry = new(offset, size, unwindInfo);
@@ -224,25 +203,22 @@ namespace ARMeilleure.Translation.Cache
_cacheEntries.Insert(index, entry);
}
public static bool TryFind(int offset, out CacheEntry entry, out int entryIndex)
public bool TryFind(int offset, out CacheEntry entry, out int entryIndex)
{
lock (_lock)
{
foreach (ReservedRegion _ in _jitRegions)
int index = _cacheEntries.BinarySearch(new CacheEntry(offset, 0, default));
if (index < 0)
{
int index = _cacheEntries.BinarySearch(new CacheEntry(offset, 0, default));
index = ~index - 1;
}
if (index < 0)
{
index = ~index - 1;
}
if (index >= 0)
{
entry = _cacheEntries[index];
entryIndex = index;
return true;
}
if (index >= 0)
{
entry = _cacheEntries[index];
entryIndex = index;
return true;
}
}
@@ -250,5 +226,18 @@ namespace ARMeilleure.Translation.Cache
entryIndex = 0;
return false;
}
public void Dispose()
{
if (OperatingSystem.IsWindows())
{
JitUnwindWindows.RemoveFunctionTableHandler(_jitRegions[0].Pointer);
}
foreach (ReservedRegion jitRegion in _jitRegions)
{
jitRegion.Dispose();
}
}
}
}

100
src/ARMeilleure/Translation/Cache/JitUnwindWindows.cs
View File

@@ -2,6 +2,8 @@
using ARMeilleure.CodeGen.Unwinding;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.InteropServices;
@@ -27,6 +29,29 @@ namespace ARMeilleure.Translation.Cache
public unsafe fixed ushort UnwindCodes[MaxUnwindCodesArraySize];
}
private unsafe struct InternalFunctionHandler
{
public InternalFunctionHandler(JitCache jitCache, nint workBufferPtr)
{
_jitCache = jitCache;
_runtimeFunction = (RuntimeFunction*)workBufferPtr;
_unwindInfo = (UnwindInfo*)(workBufferPtr + _sizeOfRuntimeFunction);
}
readonly JitCache _jitCache;
readonly RuntimeFunction* _runtimeFunction;
readonly UnwindInfo* _unwindInfo;
public RuntimeFunction* FunctionTableHandler(ulong controlPc, nint context)
{
return JitUnwindWindows.FunctionTableHandler(_jitCache, _runtimeFunction, _unwindInfo, controlPc, context);
}
}
private enum UnwindOp
{
PushNonvol = 0,
@@ -59,27 +84,28 @@ namespace ARMeilleure.Translation.Cache
private static GetRuntimeFunctionCallback _getRuntimeFunctionCallback;
private static int _sizeOfRuntimeFunction;
private static readonly int _sizeOfRuntimeFunction;
private unsafe static RuntimeFunction* _runtimeFunction;
private static readonly ConcurrentDictionary<ulong, InternalFunctionHandler> _functionTableHandlers = new();
private unsafe static UnwindInfo* _unwindInfo;
static JitUnwindWindows()
{
_sizeOfRuntimeFunction = Marshal.SizeOf<RuntimeFunction>();
}
public static void InstallFunctionTableHandler(nint codeCachePointer, uint codeCacheLength, nint workBufferPtr)
public static void InstallFunctionTableHandler(JitCache jitCache, nint codeCachePointer, uint codeCacheLength, nint workBufferPtr)
{
ulong codeCachePtr = (ulong)codeCachePointer.ToInt64();
_sizeOfRuntimeFunction = Marshal.SizeOf<RuntimeFunction>();
bool result;
InternalFunctionHandler handler;
unsafe
{
_runtimeFunction = (RuntimeFunction*)workBufferPtr;
handler = new InternalFunctionHandler(jitCache, workBufferPtr);
_unwindInfo = (UnwindInfo*)(workBufferPtr + _sizeOfRuntimeFunction);
_getRuntimeFunctionCallback = new GetRuntimeFunctionCallback(FunctionTableHandler);
_getRuntimeFunctionCallback = handler.FunctionTableHandler;
result = RtlInstallFunctionTableCallback(
codeCachePtr | 3,
@@ -94,6 +120,8 @@ namespace ARMeilleure.Translation.Cache
{
throw new InvalidOperationException("Failure installing function table callback.");
}
_functionTableHandlers.TryAdd(codeCachePtr, handler);
}
public static void RemoveFunctionTableHandler(nint codeCachePointer)
@@ -111,24 +139,26 @@ namespace ARMeilleure.Translation.Cache
{
throw new InvalidOperationException("Failure removing function table callback.");
}
_functionTableHandlers.Remove(codeCachePtr, out _);
}
private static unsafe RuntimeFunction* FunctionTableHandler(ulong controlPc, nint context)
private static unsafe RuntimeFunction* FunctionTableHandler(JitCache jitCache, RuntimeFunction* runtimeFunction, UnwindInfo* unwindInfo, ulong controlPc, nint context)
{
int offset = (int)((long)controlPc - context.ToInt64());
if (!JitCache.TryFind(offset, out CacheEntry funcEntry, out _))
if (!jitCache.TryFind(offset, out CacheEntry funcEntry, out _))
{
return null; // Not found.
}
CodeGen.Unwinding.UnwindInfo unwindInfo = funcEntry.UnwindInfo;
CodeGen.Unwinding.UnwindInfo funcUnwindInfo = funcEntry.UnwindInfo;
int codeIndex = 0;
for (int index = unwindInfo.PushEntries.Length - 1; index >= 0; index--)
for (int index = funcUnwindInfo.PushEntries.Length - 1; index >= 0; index--)
{
UnwindPushEntry entry = unwindInfo.PushEntries[index];
UnwindPushEntry entry = funcUnwindInfo.PushEntries[index];
switch (entry.PseudoOp)
{
@@ -140,14 +170,14 @@ namespace ARMeilleure.Translation.Cache
if (stackOffset <= 0xFFFF0)
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.SaveXmm128, entry.PrologOffset, entry.RegIndex);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset / 16);
unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.SaveXmm128, entry.PrologOffset, entry.RegIndex);
unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset / 16);
}
else
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.SaveXmm128Far, entry.PrologOffset, entry.RegIndex);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset >> 0);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset >> 16);
unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.SaveXmm128Far, entry.PrologOffset, entry.RegIndex);
unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset >> 0);
unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset >> 16);
}
break;
@@ -161,18 +191,18 @@ namespace ARMeilleure.Translation.Cache
if (allocSize <= 128)
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocSmall, entry.PrologOffset, (allocSize / 8) - 1);
unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocSmall, entry.PrologOffset, (allocSize / 8) - 1);
}
else if (allocSize <= 0x7FFF8)
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocLarge, entry.PrologOffset, 0);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize / 8);
unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocLarge, entry.PrologOffset, 0);
unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize / 8);
}
else
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocLarge, entry.PrologOffset, 1);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize >> 0);
_unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize >> 16);
unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocLarge, entry.PrologOffset, 1);
unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize >> 0);
unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize >> 16);
}
break;
@@ -180,7 +210,7 @@ namespace ARMeilleure.Translation.Cache
case UnwindPseudoOp.PushReg:
{
_unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.PushNonvol, entry.PrologOffset, entry.RegIndex);
unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.PushNonvol, entry.PrologOffset, entry.RegIndex);
break;
}
@@ -192,16 +222,16 @@ namespace ARMeilleure.Translation.Cache
Debug.Assert(codeIndex <= MaxUnwindCodesArraySize);
_unwindInfo->VersionAndFlags = 1; // Flags: The function has no handler.
_unwindInfo->SizeOfProlog = (byte)unwindInfo.PrologSize;
_unwindInfo->CountOfUnwindCodes = (byte)codeIndex;
_unwindInfo->FrameRegister = 0;
unwindInfo->VersionAndFlags = 1; // Flags: The function has no handler.
unwindInfo->SizeOfProlog = (byte)funcUnwindInfo.PrologSize;
unwindInfo->CountOfUnwindCodes = (byte)codeIndex;
unwindInfo->FrameRegister = 0;
_runtimeFunction->BeginAddress = (uint)funcEntry.Offset;
_runtimeFunction->EndAddress = (uint)(funcEntry.Offset + funcEntry.Size);
_runtimeFunction->UnwindData = (uint)_sizeOfRuntimeFunction;
runtimeFunction->BeginAddress = (uint)funcEntry.Offset;
runtimeFunction->EndAddress = (uint)(funcEntry.Offset + funcEntry.Size);
runtimeFunction->UnwindData = (uint)_sizeOfRuntimeFunction;
return _runtimeFunction;
return runtimeFunction;
}
private static ushort PackUnwindOp(UnwindOp op, int prologOffset, int opInfo)

13
src/ARMeilleure/Translation/PTC/Ptc.cs
View File

@@ -4,6 +4,7 @@ using ARMeilleure.CodeGen.Unwinding;
using ARMeilleure.Common;
using ARMeilleure.Memory;
using ARMeilleure.State;
using ARMeilleure.Translation.Cache;
using Humanizer;
using Ryujinx.Common;
using Ryujinx.Common.Configuration;
@@ -33,7 +34,7 @@ namespace ARMeilleure.Translation.PTC
private const string OuterHeaderMagicString = "PTCohd\0\0";
private const string InnerHeaderMagicString = "PTCihd\0\0";
private const uint InternalVersion = 7010; //! To be incremented manually for each change to the ARMeilleure project.
private const uint InternalVersion = 7016; //! To be incremented manually for each change to the ARMeilleure project.
private const string ActualDir = "0";
private const string BackupDir = "1";
@@ -51,6 +52,8 @@ namespace ARMeilleure.Translation.PTC
public PtcProfiler Profiler { get; }
private readonly JitCache _jitCache;
// Carriers.
private MemoryStream _infosStream;
private List<byte[]> _codesList;
@@ -81,7 +84,7 @@ namespace ARMeilleure.Translation.PTC
private volatile int _translateTotalCount;
public event Action<PtcLoadingState, int, int> PtcStateChanged;
public Ptc()
public Ptc(JitCache jitCache)
{
Profiler = new PtcProfiler(this);
@@ -92,6 +95,8 @@ namespace ARMeilleure.Translation.PTC
_waitEvent = new ManualResetEvent(true);
_jitCache = jitCache;
_disposed = false;
TitleIdText = TitleIdTextDefault;
@@ -782,7 +787,7 @@ namespace ARMeilleure.Translation.PTC
return new UnwindInfo(pushEntries, prologueSize);
}
private static TranslatedFunction FastTranslate(
private TranslatedFunction FastTranslate(
byte[] code,
Counter<uint> callCounter,
ulong guestSize,
@@ -790,7 +795,7 @@ namespace ARMeilleure.Translation.PTC
bool highCq)
{
CompiledFunction cFunc = new(code, unwindInfo, RelocInfo.Empty);
GuestFunction gFunc = cFunc.MapWithPointer<GuestFunction>(out nint gFuncPointer);
GuestFunction gFunc = cFunc.MapWithPointer<GuestFunction>(_jitCache, out nint gFuncPointer);
return new TranslatedFunction(gFunc, gFuncPointer, callCounter, guestSize, highCq);
}

11
src/ARMeilleure/Translation/Translator.cs
View File

@@ -24,6 +24,7 @@ namespace ARMeilleure.Translation
private readonly IJitMemoryAllocator _allocator;
private readonly ConcurrentQueue<KeyValuePair<ulong, TranslatedFunction>> _oldFuncs;
public readonly JitCache JitCache;
private readonly Ptc _ptc;
internal TranslatorCache<TranslatedFunction> Functions { get; }
@@ -43,16 +44,17 @@ namespace ARMeilleure.Translation
_oldFuncs = new ConcurrentQueue<KeyValuePair<ulong, TranslatedFunction>>();
_ptc = new Ptc();
JitCache = new JitCache(allocator);
_ptc = new Ptc(JitCache);
Queue = new TranslatorQueue();
JitCache.Initialize(allocator);
CountTable = new EntryTable<uint>();
Functions = new TranslatorCache<TranslatedFunction>();
FunctionTable = functionTable;
Stubs = new TranslatorStubs(FunctionTable);
Stubs = new TranslatorStubs(JitCache, FunctionTable);
FunctionTable.Fill = (ulong)Stubs.SlowDispatchStub;
}
@@ -167,6 +169,7 @@ namespace ARMeilleure.Translation
}
ClearJitCache();
JitCache.Dispose();
Stubs.Dispose();
FunctionTable.Dispose();
@@ -305,7 +308,7 @@ namespace ARMeilleure.Translation
_ptc.WriteCompiledFunction(address, funcSize, hash, highCq, compiledFunc);
}
GuestFunction func = compiledFunc.MapWithPointer<GuestFunction>(out nint funcPointer);
GuestFunction func = compiledFunc.MapWithPointer<GuestFunction>(JitCache, out nint funcPointer);
Allocators.ResetAll();

19
src/ARMeilleure/Translation/TranslatorStubs.cs
View File

@@ -14,6 +14,8 @@ namespace ARMeilleure.Translation
/// </summary>
class TranslatorStubs : IDisposable
{
private readonly JitCache _jitCache;
private readonly Lazy<nint> _slowDispatchStub;
private bool _disposed;
@@ -83,12 +85,15 @@ namespace ARMeilleure.Translation
/// Initializes a new instance of the <see cref="TranslatorStubs"/> class with the specified
/// <see cref="Translator"/> instance.
/// </summary>
/// <param name="jitCache">Jit cache to map functions in</param>
/// <param name="functionTable">Function table used to store pointers to the functions that the guest code will call</param>
/// <exception cref="ArgumentNullException"><paramref name="translator"/> is null</exception>
public TranslatorStubs(IAddressTable<ulong> functionTable)
public TranslatorStubs(JitCache jitCache, IAddressTable<ulong> functionTable)
{
ArgumentNullException.ThrowIfNull(functionTable);
_jitCache = jitCache;
_functionTable = functionTable;
_slowDispatchStub = new(GenerateSlowDispatchStub, isThreadSafe: true);
_dispatchStub = new(GenerateDispatchStub, isThreadSafe: true);
@@ -115,12 +120,12 @@ namespace ARMeilleure.Translation
{
if (_dispatchStub.IsValueCreated)
{
JitCache.Unmap(_dispatchStub.Value);
_jitCache.Unmap(_dispatchStub.Value);
}
if (_dispatchLoop.IsValueCreated)
{
JitCache.Unmap(Marshal.GetFunctionPointerForDelegate(_dispatchLoop.Value));
_jitCache.Unmap(Marshal.GetFunctionPointerForDelegate(_dispatchLoop.Value));
}
_disposed = true;
@@ -188,7 +193,7 @@ namespace ARMeilleure.Translation
OperandType retType = OperandType.I64;
OperandType[] argTypes = [OperandType.I64];
GuestFunction func = Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<GuestFunction>();
GuestFunction func = Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<GuestFunction>(_jitCache);
return Marshal.GetFunctionPointerForDelegate(func);
}
@@ -213,7 +218,7 @@ namespace ARMeilleure.Translation
OperandType retType = OperandType.I64;
OperandType[] argTypes = [OperandType.I64];
GuestFunction func = Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<GuestFunction>();
GuestFunction func = Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<GuestFunction>(_jitCache);
return Marshal.GetFunctionPointerForDelegate(func);
}
@@ -290,7 +295,7 @@ namespace ARMeilleure.Translation
OperandType retType = OperandType.None;
OperandType[] argTypes = [OperandType.I64, OperandType.I64];
return Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<DispatcherFunction>();
return Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<DispatcherFunction>(_jitCache);
}
/// <summary>
@@ -314,7 +319,7 @@ namespace ARMeilleure.Translation
OperandType retType = OperandType.I64;
OperandType[] argTypes = [OperandType.I64, OperandType.I64];
return Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<WrapperFunction>();
return Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<WrapperFunction>(_jitCache);
}
}
}

5
src/ARMeilleure/Translation/TranslatorTestMethods.cs
View File

@@ -1,6 +1,7 @@
using ARMeilleure.CodeGen.X86;
using ARMeilleure.IntermediateRepresentation;
using ARMeilleure.State;
using ARMeilleure.Translation.Cache;
using System;
using System.Runtime.InteropServices;
using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
@@ -59,7 +60,7 @@ namespace ARMeilleure.Translation
return context.VectorExtract(OperandType.I32, vec, 0);
}
public static FpFlagsPInvokeTest GenerateFpFlagsPInvokeTest()
public static FpFlagsPInvokeTest GenerateFpFlagsPInvokeTest(JitCache jitCache)
{
EmitterContext context = new();
@@ -141,7 +142,7 @@ namespace ARMeilleure.Translation
OperandType[] argTypes = [OperandType.I64];
return Compiler.Compile(cfg, argTypes, OperandType.I32, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<FpFlagsPInvokeTest>();
return Compiler.Compile(cfg, argTypes, OperandType.I32, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<FpFlagsPInvokeTest>(jitCache);
}
}
}

33
src/Ryujinx.Cpu/LightningJit/Cache/CacheMemoryAllocator.cs
View File

@@ -24,11 +24,30 @@ namespace Ryujinx.Cpu.LightningJit.Cache
}
}
private readonly int _regionSize;
private int _regionCount;
private readonly List<MemoryBlock> _blocks = [];
public CacheMemoryAllocator(int capacity)
public CacheMemoryAllocator(int regionSize, int initialRegionCount = 1)
{
_blocks.Add(new MemoryBlock(0, capacity));
_regionCount = 0;
_regionSize = regionSize;
for (; initialRegionCount > 0; initialRegionCount--)
{
_blocks.Add(new MemoryBlock(_regionSize * _regionCount, _regionSize));
_regionCount++;
}
}
public void AddNewBlocks(int count)
{
for (; count > 0; count--)
{
_blocks.Add(new MemoryBlock(_regionSize * _regionCount, _regionSize));
_regionCount++;
}
}
public int Allocate(int size)
@@ -101,12 +120,13 @@ namespace Ryujinx.Cpu.LightningJit.Cache
index = ~index;
}
if (index < _blocks.Count)
int endOffs = block.Offset + block.Size;
// Don't merge blocks from different allocations
if (index < _blocks.Count && endOffs % _regionSize != 0)
{
MemoryBlock next = _blocks[index];
int endOffs = block.Offset + block.Size;
if (next.Offset == endOffs)
{
block = new MemoryBlock(block.Offset, block.Size + next.Size);
@@ -114,7 +134,8 @@ namespace Ryujinx.Cpu.LightningJit.Cache
}
}
if (index > 0)
// Don't merge blocks from different allocations
if (index > 0 && block.Offset % _regionSize != 0)
{
MemoryBlock prev = _blocks[index - 1];

146
src/Ryujinx.Cpu/LightningJit/Cache/JitCache.cs
View File

@@ -11,76 +11,45 @@ using System.Threading;
namespace Ryujinx.Cpu.LightningJit.Cache
{
static partial class JitCache
partial class JitCache : IDisposable
{
private static readonly int _pageSize = (int)MemoryBlock.GetPageSize();
private static readonly int _pageMask = _pageSize - 1;
private const int CodeAlignment = 4; // Bytes.
// TODO: JIT Cache size should be application dependent, not global.
private const int CacheSize = 1024 * (1024 * 1024); // Megabytes * Size of Megabytes (since its in bytes).
private const uint CacheSize = 256 * 1024 * 1024; // Megabytes * Size of Megabytes (since its in bytes).
private static JitCacheInvalidation _jitCacheInvalidator;
private readonly JitCacheInvalidation _jitCacheInvalidator;
private static CacheMemoryAllocator _cacheAllocator;
private readonly CacheMemoryAllocator _cacheAllocator;
private static readonly List<CacheEntry> _cacheEntries = [];
private readonly List<CacheEntry> _cacheEntries = [];
private static readonly Lock _lock = new();
private static bool _initialized;
private static readonly List<ReservedRegion> _jitRegions = [];
private static int _activeRegionIndex = 0;
private readonly Lock _lock = new();
[SupportedOSPlatform("windows")]
[LibraryImport("kernel32.dll", SetLastError = true)]
public static partial nint FlushInstructionCache(nint hProcess, nint lpAddress, nuint dwSize);
private readonly List<ReservedRegion> _jitRegions = [];
[SupportedOSPlatform("macos")]
[LibraryImport("libSystem.dylib", EntryPoint = "sys_icache_invalidate")]
internal static partial void SysICacheInvalidate(nint start, nuint len);
[SupportedOSPlatform("linux")]
[LibraryImport("libgcc_s.so.1", EntryPoint = "__clear_cache")]
internal static partial void ClearCache(nint begin, nint end);
public static void Initialize(IJitMemoryAllocator allocator)
public JitCache(IJitMemoryAllocator allocator)
{
if (_initialized)
{
return;
}
lock (_lock)
{
if (_initialized)
{
return;
}
_jitRegions.Add(new(allocator, CacheSize));
ReservedRegion firstRegion = new(allocator, CacheSize);
_jitRegions.Add(firstRegion);
_activeRegionIndex = 0;
_cacheAllocator = new CacheMemoryAllocator((int)CacheSize);
if (!OperatingSystem.IsWindows() && !OperatingSystem.IsMacOS())
{
_jitCacheInvalidator = new JitCacheInvalidation(allocator);
}
_cacheAllocator = new CacheMemoryAllocator(CacheSize);
_initialized = true;
}
}
public unsafe static nint Map(ReadOnlySpan<byte> code)
public nint Map(ReadOnlySpan<byte> code)
{
lock (_lock)
{
Debug.Assert(_initialized);
int funcOffset = Allocate(code.Length);
ReservedRegion targetRegion = _jitRegions[_activeRegionIndex];
nint funcPtr = targetRegion.Pointer + funcOffset;
nint funcPtr = GetFunctionPtr(funcOffset);
if (OperatingSystem.IsMacOS() && RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
{
@@ -94,9 +63,9 @@ namespace Ryujinx.Cpu.LightningJit.Cache
}
else
{
ReprotectAsWritable(targetRegion, funcOffset, code.Length);
ReprotectAsWritable(funcOffset, code.Length);
Marshal.Copy(code.ToArray(), 0, funcPtr, code.Length);
ReprotectAsExecutable(targetRegion, funcOffset, code.Length);
ReprotectAsExecutable(funcOffset, code.Length);
_jitCacheInvalidator?.Invalidate(funcPtr, (ulong)code.Length);
}
@@ -107,21 +76,20 @@ namespace Ryujinx.Cpu.LightningJit.Cache
}
}
public static void Unmap(nint pointer)
public void Unmap(nint pointer)
{
lock (_lock)
{
Debug.Assert(_initialized);
foreach (ReservedRegion region in _jitRegions)
for (int i = 0; i < _jitRegions.Count; i++)
{
ReservedRegion region = _jitRegions[i];
if (pointer.ToInt64() < region.Pointer.ToInt64() ||
pointer.ToInt64() >= (region.Pointer + CacheSize).ToInt64())
pointer.ToInt64() >= (region.Pointer + (nint)CacheSize).ToInt64())
{
continue;
}
int funcOffset = (int)(pointer.ToInt64() - region.Pointer.ToInt64());
int funcOffset = (int)(pointer.ToInt64() - region.Pointer.ToInt64() + i * CacheSize);
if (TryFind(funcOffset, out CacheEntry entry, out int entryIndex) && entry.Offset == funcOffset)
{
@@ -134,59 +102,63 @@ namespace Ryujinx.Cpu.LightningJit.Cache
}
}
private static void ReprotectAsWritable(ReservedRegion region, int offset, int size)
private void ReprotectAsWritable(int offset, int size)
{
int endOffs = offset + size;
int regionStart = offset & ~_pageMask;
int regionEnd = (endOffs + _pageMask) & ~_pageMask;
int regionStart = (offset % (int)CacheSize) & ~_pageMask;
int regionEnd = ((endOffs % (int)CacheSize) + _pageMask) & ~_pageMask;
region.Block.MapAsRwx((ulong)regionStart, (ulong)(regionEnd - regionStart));
GetRegion(offset).Block.MapAsRwx((ulong)regionStart, (ulong)(regionEnd - regionStart));
}
private static void ReprotectAsExecutable(ReservedRegion region, int offset, int size)
private void ReprotectAsExecutable(int offset, int size)
{
int endOffs = offset + size;
int regionStart = offset & ~_pageMask;
int regionEnd = (endOffs + _pageMask) & ~_pageMask;
int regionStart = (offset % (int)CacheSize) & ~_pageMask;
int regionEnd = ((endOffs % (int)CacheSize) + _pageMask) & ~_pageMask;
region.Block.MapAsRx((ulong)regionStart, (ulong)(regionEnd - regionStart));
GetRegion(offset).Block.MapAsRx((ulong)regionStart, (ulong)(regionEnd - regionStart));
}
private static int Allocate(int codeSize)
private int Allocate(int codeSize)
{
codeSize = AlignCodeSize(codeSize);
for (int i = _activeRegionIndex; i < _jitRegions.Count; i++)
{
int allocOffset = _cacheAllocator.Allocate(codeSize);
int allocOffset = _cacheAllocator.Allocate(codeSize);
if (allocOffset >= 0)
{
_jitRegions[i].ExpandIfNeeded((ulong)allocOffset + (ulong)codeSize);
_activeRegionIndex = i;
return allocOffset;
}
if (allocOffset >= 0)
{
GetRegion(allocOffset).ExpandIfNeeded((ulong)(allocOffset % (int)CacheSize) + (ulong)codeSize);
return allocOffset;
}
int exhaustedRegion = _activeRegionIndex;
_cacheAllocator.AddNewBlocks(1);
ReservedRegion newRegion = new(_jitRegions[0].Allocator, CacheSize);
Logger.Warning?.Print(LogClass.Cpu, $"JIT Cache of size {(_jitRegions.Count * CacheSize).Bytes()} exhausted, creating new Cache Region ({((_jitRegions.Count + 1) * CacheSize).Bytes()} Total Allocation).");
_jitRegions.Add(newRegion);
_activeRegionIndex = _jitRegions.Count - 1;
int newRegionNumber = _activeRegionIndex;
Logger.Warning?.Print(LogClass.Cpu, $"JIT Cache Region {exhaustedRegion} exhausted, creating new Cache Region {newRegionNumber} ({((long)(newRegionNumber + 1) * CacheSize).Bytes()} Total Allocation).");
_cacheAllocator = new CacheMemoryAllocator(CacheSize);
int allocOffsetNew = _cacheAllocator.Allocate(codeSize);
if (allocOffsetNew < 0)
allocOffset = _cacheAllocator.Allocate(codeSize);
if (allocOffset < 0)
{
throw new OutOfMemoryException("Failed to allocate in new Cache Region!");
}
newRegion.ExpandIfNeeded((ulong)allocOffsetNew + (ulong)codeSize);
return allocOffsetNew;
GetRegion(allocOffset).ExpandIfNeeded((ulong)(allocOffset % (int)CacheSize) + (ulong)codeSize);
return allocOffset;
}
private nint GetFunctionPtr(int offset)
{
return GetRegion(offset).Pointer + (offset % (int)CacheSize);
}
private ReservedRegion GetRegion(int offset)
{
int index = offset / (int)CacheSize;
return _jitRegions[index];
}
private static int AlignCodeSize(int codeSize)
@@ -194,7 +166,7 @@ namespace Ryujinx.Cpu.LightningJit.Cache
return checked(codeSize + (CodeAlignment - 1)) & ~(CodeAlignment - 1);
}
private static void Add(int offset, int size)
private void Add(int offset, int size)
{
CacheEntry entry = new(offset, size);
@@ -208,7 +180,7 @@ namespace Ryujinx.Cpu.LightningJit.Cache
_cacheEntries.Insert(index, entry);
}
public static bool TryFind(int offset, out CacheEntry entry, out int entryIndex)
public bool TryFind(int offset, out CacheEntry entry, out int entryIndex)
{
lock (_lock)
{
@@ -231,5 +203,13 @@ namespace Ryujinx.Cpu.LightningJit.Cache
entryIndex = 0;
return false;
}
public void Dispose()
{
foreach (ReservedRegion jitRegion in _jitRegions)
{
jitRegion.Dispose();
}
}
}
}

14
src/Ryujinx.Cpu/LightningJit/Translator.cs
View File

@@ -19,6 +19,7 @@ namespace Ryujinx.Cpu.LightningJit
private static bool IsNoWxPlatform => false;
private readonly ConcurrentQueue<KeyValuePair<ulong, TranslatedFunction>> _oldFuncs;
private readonly JitCache _jitCache;
private readonly NoWxCache _noWxCache;
private bool _disposed;
@@ -39,12 +40,12 @@ namespace Ryujinx.Cpu.LightningJit
}
else
{
JitCache.Initialize(new JitMemoryAllocator(forJit: true));
_jitCache = new(new JitMemoryAllocator(forJit: true));
}
Functions = new TranslatorCache<TranslatedFunction>();
FunctionTable = functionTable;
Stubs = new TranslatorStubs(FunctionTable, _noWxCache);
Stubs = new TranslatorStubs(_jitCache, FunctionTable, _noWxCache);
FunctionTable.Fill = (ulong)Stubs.SlowDispatchStub;
@@ -100,7 +101,7 @@ namespace Ryujinx.Cpu.LightningJit
if (oldFunc != func)
{
JitCache.Unmap(func.FuncPointer);
_jitCache.Unmap(func.FuncPointer);
func = oldFunc;
}
@@ -121,7 +122,7 @@ namespace Ryujinx.Cpu.LightningJit
private TranslatedFunction Translate(ulong address, ExecutionMode mode)
{
CompiledFunction func = Compile(address, mode);
nint funcPointer = JitCache.Map(func.Code);
nint funcPointer = _jitCache.Map(func.Code);
return new TranslatedFunction(funcPointer, (ulong)func.GuestCodeLength);
}
@@ -164,14 +165,14 @@ namespace Ryujinx.Cpu.LightningJit
foreach (TranslatedFunction func in functions)
{
JitCache.Unmap(func.FuncPointer);
_jitCache.Unmap(func.FuncPointer);
}
Functions.Clear();
while (_oldFuncs.TryDequeue(out KeyValuePair<ulong, TranslatedFunction> kv))
{
JitCache.Unmap(kv.Value.FuncPointer);
_jitCache.Unmap(kv.Value.FuncPointer);
}
}
@@ -188,6 +189,7 @@ namespace Ryujinx.Cpu.LightningJit
else
{
ClearJitCache();
_jitCache.Dispose();
}
Stubs.Dispose();

13
src/Ryujinx.Cpu/LightningJit/TranslatorStubs.cs
View File

@@ -19,6 +19,8 @@ namespace Ryujinx.Cpu.LightningJit
{
private delegate ulong GetFunctionAddressDelegate(nint framePointer, ulong address);
private readonly JitCache _jitCache;
private readonly Lazy<nint> _slowDispatchStub;
private bool _disposed;
@@ -76,13 +78,16 @@ namespace Ryujinx.Cpu.LightningJit
/// Initializes a new instance of the <see cref="TranslatorStubs"/> class with the specified
/// <see cref="Translator"/> instance.
/// </summary>
/// <param name="jitCache">Jit cache to map functions in</param>
/// <param name="functionTable">Function table used to store pointers to the functions that the guest code will call</param>
/// <param name="noWxCache">Cache used on platforms that enforce W^X, otherwise should be null</param>
/// <exception cref="ArgumentNullException"><paramref name="translator"/> is null</exception>
public TranslatorStubs(IAddressTable<ulong> functionTable, NoWxCache noWxCache)
public TranslatorStubs(JitCache jitCache, IAddressTable<ulong> functionTable, NoWxCache noWxCache)
{
ArgumentNullException.ThrowIfNull(functionTable);
_jitCache = jitCache;
_functionTable = functionTable;
_noWxCache = noWxCache;
_getFunctionAddressRef = NativeInterface.GetFunctionAddress;
@@ -113,12 +118,12 @@ namespace Ryujinx.Cpu.LightningJit
{
if (_dispatchStub.IsValueCreated)
{
JitCache.Unmap(_dispatchStub.Value);
_jitCache.Unmap(_dispatchStub.Value);
}
if (_dispatchLoop.IsValueCreated)
{
JitCache.Unmap(Marshal.GetFunctionPointerForDelegate(_dispatchLoop.Value));
_jitCache.Unmap(Marshal.GetFunctionPointerForDelegate(_dispatchLoop.Value));
}
}
@@ -363,7 +368,7 @@ namespace Ryujinx.Cpu.LightningJit
}
else
{
return JitCache.Map(code);
return _jitCache.Map(code);
}
}

2
src/Ryujinx.Tests/Cpu/CpuTest.cs
View File

@@ -10,6 +10,8 @@ using MemoryPermission = Ryujinx.Tests.Unicorn.MemoryPermission;
namespace Ryujinx.Tests.Cpu
{
[TestFixture]
[Parallelizable(ParallelScope.All)]
[FixtureLifeCycle(LifeCycle.InstancePerTestCase)]
public class CpuTest
{
protected static readonly ulong Size = MemoryBlock.GetPageSize();

2
src/Ryujinx.Tests/Cpu/CpuTest32.cs
View File

@@ -10,6 +10,8 @@ using MemoryPermission = Ryujinx.Tests.Unicorn.MemoryPermission;
namespace Ryujinx.Tests.Cpu
{
[TestFixture]
[Parallelizable(ParallelScope.All)]
[FixtureLifeCycle(LifeCycle.InstancePerTestCase)]
public class CpuTest32
{
protected static readonly uint Size = (uint)MemoryBlock.GetPageSize();

2
src/Ryujinx.Tests/Cpu/EnvironmentTests.cs
View File

@@ -51,7 +51,7 @@ namespace Ryujinx.Tests.Cpu
bool methodCalled = false;
bool isFz = false;
TranslatorTestMethods.FpFlagsPInvokeTest method = TranslatorTestMethods.GenerateFpFlagsPInvokeTest();
TranslatorTestMethods.FpFlagsPInvokeTest method = TranslatorTestMethods.GenerateFpFlagsPInvokeTest(_translator.JitCache);
// This method sets flush-to-zero and then calls the managed method.
// Before and after setting the flags, it ensures subnormal addition works as expected.

4
src/Ryujinx.Tests/Memory/PartialUnmaps.cs
View File

@@ -245,7 +245,7 @@ namespace Ryujinx.Tests.Memory
// Create a large mapping.
mainMemory.MapView(backing, 0, 0, vaSize);
TestMethods.DebugNativeWriteLoop writeFunc = TestMethods.GenerateDebugNativeWriteLoop();
TestMethods.DebugNativeWriteLoop writeFunc = TestMethods.GenerateDebugNativeWriteLoop(_translator.JitCache);
nint writePtr = mainMemory.GetPointer(vaSize - 0x1000, 4);
Thread testThread = new(() =>
@@ -339,7 +339,7 @@ namespace Ryujinx.Tests.Memory
fixed (void* localMap = &state.LocalCounts)
{
TestMethods.DebugThreadLocalMapGetOrReserve getOrReserve = TestMethods.GenerateDebugThreadLocalMapGetOrReserve((nint)localMap);
TestMethods.DebugThreadLocalMapGetOrReserve getOrReserve = TestMethods.GenerateDebugThreadLocalMapGetOrReserve(_translator.JitCache, (nint)localMap);
for (int i = 0; i < ThreadLocalMap<int>.MapSize; i++)
{