diff --git a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs index 24a523d516..7d3b41fb78 100644 --- a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs @@ -15,4 +15,5 @@ [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotMemory,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] +[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.RuntimeAsync,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.TestAdapter,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/AsyncCoreEmitter.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/AsyncCoreEmitter.cs index 4aa179be8b..904dd09db0 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/AsyncCoreEmitter.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/AsyncCoreEmitter.cs @@ -1,5 +1,7 @@ +using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Helpers.Reflection.Emit; using BenchmarkDotNet.Running; using Perfolizer.Horology; @@ -12,11 +14,30 @@ namespace BenchmarkDotNet.Toolchains.InProcess.Emit.Implementation; partial class RunnableEmitter { - // TODO: update this to support runtime-async. private sealed class AsyncCoreEmitter(BuildPartition buildPartition, ModuleBuilder moduleBuilder, BenchmarkBuildInfo benchmark, AwaitableInfo awaitableInfo) : AsyncCoreEmitterBase(buildPartition, moduleBuilder, benchmark) { + private static readonly MethodInfo KeepAliveWithoutBoxingInMethod = typeof(DeadCodeEliminationHelper) + .GetMethods(BindingFlags.Public | BindingFlags.Static) + .First(m => m.Name == nameof(DeadCodeEliminationHelper.KeepAliveWithoutBoxing) + && m.GetParameters().Length == 1 + && m.GetParameters()[0].ParameterType.IsByRef); + protected override void EmitWorkloadCore() { + if (IsRuntimeAsync(Descriptor.WorkloadMethod)) + { + // A runtime-async workload returns a built-in task type, which WorkloadCore returns too unless the caller type is overridden. + var workloadCoreReturnType = Descriptor.WorkloadMethod.ResolveAttribute()?.AsyncCallerType + ?? Descriptor.WorkloadMethod.ReturnType; + // Roslyn only compiles the built-in task types as runtime-async, any other task-like keeps its state machine. + if (AwaitHelper.IsBuiltInTaskType(workloadCoreReturnType)) + { + var runtimeAsyncWorkloadCoreMethod = EmitRuntimeAsyncWorkloadCore(workloadCoreReturnType); + startWorkloadMethod = EmitAsyncSingleCall(StartWorkloadMethodName, typeof(AsyncVoidMethodBuilder), runtimeAsyncWorkloadCoreMethod, SetupCleanupKind.Other); + return; + } + } + var asyncMethodBuilderType = GetWorkloadCoreAsyncMethodBuilderType(Descriptor.WorkloadMethod.ReturnType); var builderInfo = BeginAsyncStateMachineTypeBuilder(WorkloadCoreMethodName, asyncMethodBuilderType, runnableBuilder); var (asyncStateMachineTypeBuilder, publicFields, (ilBuilder, endTryLabel, returnLabel, stateLocal, thisLocal, returnDefaultLocal)) = builderInfo; @@ -242,13 +263,7 @@ void EmitMoveNextImpl() // JIT can't elide whatever produced it. ilBuilder.EmitStloc(resultLocal); ilBuilder.EmitLdloca(resultLocal); - var keepAliveInMethod = typeof(DeadCodeEliminationHelper) - .GetMethods(BindingFlags.Public | BindingFlags.Static) - .First(m => m.Name == nameof(DeadCodeEliminationHelper.KeepAliveWithoutBoxing) - && m.GetParameters().Length == 1 - && m.GetParameters()[0].ParameterType.IsByRef) - .MakeGenericMethod(resultType); - ilBuilder.Emit(OpCodes.Call, keepAliveInMethod); + ilBuilder.Emit(OpCodes.Call, KeepAliveWithoutBoxingInMethod.MakeGenericMethod(resultType)); } // --- Benchmark loop: if (--invokeCount >= 0) goto callBenchmarkLabel --- @@ -358,5 +373,158 @@ void EmitMoveNextImpl() } // end handler } } + + /* + private async Task WorkloadCore() + { + try + { + if (await this.fieldsContainer.workloadValueTaskSource.GetIsComplete()) + { + return; + } + while (true) + { + StartedClock startedClock = ClockExtensions.Start(this.fieldsContainer.clock); + while (--this.fieldsContainer.invokeCount >= 0) + { + await base.Workload(); + } + if (await this.fieldsContainer.workloadValueTaskSource.SetResultAndGetIsComplete(startedClock.GetElapsed())) + { + return; + } + } + } + catch (Exception e) + { + fieldsContainer.workloadValueTaskSource.SetException(e); + return; + } + } + */ + private MethodInfo EmitRuntimeAsyncWorkloadCore(Type workloadCoreReturnType) + { + var methodBuilder = DefineRuntimeAsyncMethod(WorkloadCoreMethodName, workloadCoreReturnType); + // The generated code copies the workload's builder override, which Roslyn keeps as an attribute without using the builder. + if (!Descriptor.WorkloadMethod.HasAttribute() + && Descriptor.WorkloadMethod.GetAsyncMethodBuilderAttribute() is { } builderAttribute + && builderAttribute.GetType().GetProperty(nameof(AsyncMethodBuilderAttribute.BuilderType), BindingFlags.Public | BindingFlags.Instance)?.GetValue(builderAttribute) is Type builderType) + { + var attributeCtor = typeof(AsyncMethodBuilderAttribute).GetConstructor([typeof(Type)]) + ?? throw new MissingMemberException(nameof(AsyncMethodBuilderAttribute)); + methodBuilder.SetCustomAttribute(new CustomAttributeBuilder(attributeCtor, [builderType])); + } + + var ilBuilder = methodBuilder.GetILGenerator(); + + var returnLocal = workloadCoreReturnType.IsGenericType + ? ilBuilder.DeclareLocal(workloadCoreReturnType.GetGenericArguments()[0]) + : null; + var startedClockLocal = ilBuilder.DeclareLocal(typeof(StartedClock)); + var resultLocal = awaitableInfo.ResultType == typeof(void) + ? null + : ilBuilder.DeclareLocal(awaitableInfo.ResultType); + var invokeCountLocal = ilBuilder.DeclareLocal(typeof(long)); + var exceptionLocal = ilBuilder.DeclareLocal(typeof(Exception)); + + var returnLabel = ilBuilder.DefineLabel(); + var startClockLabel = ilBuilder.DefineLabel(); + var callBenchmarkLabel = ilBuilder.DefineLabel(); + var callBenchmarkLoopLabel = ilBuilder.DefineLabel(); + + ilBuilder.BeginExceptionBlock(); + { + // if (await workloadValueTaskSource.GetIsComplete()) return default; + EmitLoadWorkloadValueTaskSource(); + ilBuilder.Emit(OpCodes.Callvirt, typeof(WorkloadValueTaskSource).GetMethod(nameof(WorkloadValueTaskSource.GetIsComplete), BindingFlags.Public | BindingFlags.Instance)!); + EmitRuntimeAsyncAwait(ilBuilder, typeof(ValueTask)); + ilBuilder.Emit(OpCodes.Brfalse, startClockLabel); + ilBuilder.MaybeEmitSetLocalToDefault(returnLocal); + ilBuilder.Emit(OpCodes.Leave, returnLabel); + + // startedClock = ClockExtensions.Start(clock); + ilBuilder.MarkLabel(startClockLabel); + ilBuilder.Emit(OpCodes.Ldarg_0); + ilBuilder.Emit(OpCodes.Ldflda, fieldsContainerField); + ilBuilder.Emit(OpCodes.Ldfld, clockField); + ilBuilder.Emit(OpCodes.Call, GetStartClockMethod()); + ilBuilder.EmitStloc(startedClockLocal); + ilBuilder.Emit(OpCodes.Br, callBenchmarkLoopLabel); + + // T result = await base.Workload(); DeadCodeEliminationHelper.KeepAliveWithoutBoxing(in result); + ilBuilder.MarkLabel(callBenchmarkLabel); + if (!Descriptor.WorkloadMethod.IsStatic) + { + ilBuilder.Emit(OpCodes.Ldarg_0); + } + EmitLoadArgFieldsForCall(ilBuilder, null); + ilBuilder.Emit(OpCodes.Call, Descriptor.WorkloadMethod); + EmitRuntimeAsyncAwait(ilBuilder, Descriptor.WorkloadMethod.ReturnType); + if (resultLocal is not null) + { + ilBuilder.EmitStloc(resultLocal); + ilBuilder.EmitLdloca(resultLocal); + ilBuilder.Emit(OpCodes.Call, KeepAliveWithoutBoxingInMethod.MakeGenericMethod(resultLocal.LocalType)); + } + + // if (--invokeCount >= 0) goto callBenchmarkLabel; + ilBuilder.MarkLabel(callBenchmarkLoopLabel); + ilBuilder.Emit(OpCodes.Ldarg_0); + ilBuilder.Emit(OpCodes.Ldflda, fieldsContainerField); + ilBuilder.Emit(OpCodes.Ldflda, invokeCountField); + ilBuilder.Emit(OpCodes.Dup); + ilBuilder.Emit(OpCodes.Ldind_I8); + ilBuilder.Emit(OpCodes.Ldc_I4_1); + ilBuilder.Emit(OpCodes.Conv_I8); + ilBuilder.Emit(OpCodes.Sub); + ilBuilder.EmitStloc(invokeCountLocal); + ilBuilder.EmitLdloc(invokeCountLocal); + ilBuilder.Emit(OpCodes.Stind_I8); + ilBuilder.EmitLdloc(invokeCountLocal); + ilBuilder.Emit(OpCodes.Ldc_I4_0); + ilBuilder.Emit(OpCodes.Conv_I8); + ilBuilder.Emit(OpCodes.Bge, callBenchmarkLabel); + + // if (!await workloadValueTaskSource.SetResultAndGetIsComplete(startedClock.GetElapsed())) goto startClockLabel; + EmitLoadWorkloadValueTaskSource(); + ilBuilder.EmitLdloca(startedClockLocal); + ilBuilder.Emit(OpCodes.Call, typeof(StartedClock).GetMethod(nameof(StartedClock.GetElapsed), BindingFlags.Public | BindingFlags.Instance)!); + ilBuilder.Emit(OpCodes.Callvirt, typeof(WorkloadValueTaskSource).GetMethod(nameof(WorkloadValueTaskSource.SetResultAndGetIsComplete), BindingFlags.Public | BindingFlags.Instance)!); + EmitRuntimeAsyncAwait(ilBuilder, typeof(ValueTask)); + ilBuilder.Emit(OpCodes.Brfalse, startClockLabel); + // return default; + ilBuilder.MaybeEmitSetLocalToDefault(returnLocal); + ilBuilder.Emit(OpCodes.Leave, returnLabel); + } + ilBuilder.BeginCatchBlock(typeof(Exception)); + { + // workloadValueTaskSource.SetException(exception); + ilBuilder.EmitStloc(exceptionLocal); + EmitLoadWorkloadValueTaskSource(); + ilBuilder.EmitLdloc(exceptionLocal); + ilBuilder.Emit(OpCodes.Callvirt, typeof(WorkloadValueTaskSource).GetMethod(nameof(WorkloadValueTaskSource.SetException), [typeof(Exception)])!); + // return default; + ilBuilder.MaybeEmitSetLocalToDefault(returnLocal); + ilBuilder.Emit(OpCodes.Leave, returnLabel); + } + ilBuilder.EndExceptionBlock(); + + ilBuilder.MarkLabel(returnLabel); + if (returnLocal is not null) + { + ilBuilder.EmitLdloc(returnLocal); + } + ilBuilder.Emit(OpCodes.Ret); + + return methodBuilder; + + void EmitLoadWorkloadValueTaskSource() + { + ilBuilder.Emit(OpCodes.Ldarg_0); + ilBuilder.Emit(OpCodes.Ldflda, fieldsContainerField); + ilBuilder.Emit(OpCodes.Ldfld, workloadValueTaskSourceField); + } + } } } diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs index 0f876942b7..903d9f50dc 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RunnableEmitter.cs @@ -12,6 +12,9 @@ using System.Reflection; using System.Reflection.Emit; using System.Runtime.CompilerServices; +#if NET9_0_OR_GREATER +using System.Runtime.Loader; +#endif using System.Security; using static BenchmarkDotNet.Code.RunnableConstants; using static BenchmarkDotNet.Toolchains.InProcess.Emit.Implementation.RunnableReflectionHelpers; @@ -91,13 +94,24 @@ public static Assembly EmitPartitionAssembly(ArtifactsPaths artifactsPaths, Buil runnableEmitter.EmitRunnableCore(); } - if (saveToDisk) - { - assemblyBuilder.Save(assemblyFileName); - logger.WriteLineInfo($"{assemblyFileName} assembly saved to {assemblyResultPath}"); - } - + if (!saveToDisk) + return assemblyBuilder; + +#if NET9_0_OR_GREATER + // A persisted assembly can't run, so the saved image is loaded back next to the benchmarks. + using var stream = new MemoryStream(); + ((PersistedAssemblyBuilder)assemblyBuilder).Save(stream); + File.WriteAllBytes(assemblyResultPath, stream.ToArray()); + logger.WriteLineInfo($"{assemblyFileName} assembly saved to {assemblyResultPath}"); + + stream.Position = 0; + var benchmarksAssembly = buildPartition.RepresentativeBenchmarkCase.Descriptor.Type.Assembly; + return (AssemblyLoadContext.GetLoadContext(benchmarksAssembly) ?? AssemblyLoadContext.Default).LoadFromStream(stream); +#else + assemblyBuilder.Save(assemblyFileName); + logger.WriteLineInfo($"{assemblyFileName} assembly saved to {assemblyResultPath}"); return assemblyBuilder; +#endif } private static bool ShouldSaveToDisk(IConfig config) @@ -105,7 +119,7 @@ private static bool ShouldSaveToDisk(IConfig config) if (!BenchmarkDotNetInfo.Instance.IsRelease) { // we never want to do that in our official NuGet.org package, it's a hack - return config.Options.IsSet(ConfigOptions.KeepBenchmarkFiles) && Portability.RuntimeInformation.IsFullFramework; + return config.Options.IsSet(ConfigOptions.KeepBenchmarkFiles); } return false; @@ -120,6 +134,15 @@ private static AssemblyBuilder DefineAssemblyBuilder(string assemblyResultPath, { var assemblyName = new AssemblyName { Name = Path.GetFileNameWithoutExtension(assemblyResultPath) }; +#if NET9_0_OR_GREATER + if (saveToDisk) + { + var persistedAssemblyBuilder = new PersistedAssemblyBuilder(assemblyName, typeof(object).Assembly); + DefineAssemblyAttributes(persistedAssemblyBuilder); + return persistedAssemblyBuilder; + } +#endif + var assemblyMode = saveToDisk ? (AssemblyBuilderAccess)3 // https://apisof.net/catalog/System.Reflection.Emit.AssemblyBuilderAccess.RunAndSave : AssemblyBuilderAccess.RunAndCollect; @@ -174,9 +197,13 @@ private static ModuleBuilder DefineModuleBuilder(AssemblyBuilder assemblyBuilder var moduleName = Path.GetFileNameWithoutExtension(moduleFileName) ?? throw new ArgumentNullException(nameof(moduleFileName)); +#if NET9_0_OR_GREATER + var moduleBuilder = assemblyBuilder.DefineDynamicModule(moduleName); +#else var moduleBuilder = saveToDisk ? assemblyBuilder.DefineDynamicModule(moduleName, moduleFileName) : assemblyBuilder.DefineDynamicModule(moduleName); +#endif // [module:UnverifiableCodeAttribute()] var attributeCtor = typeof(UnverifiableCodeAttribute) diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RuntimeAsyncEmitter.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RuntimeAsyncEmitter.cs new file mode 100644 index 0000000000..8be9348304 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/RuntimeAsyncEmitter.cs @@ -0,0 +1,117 @@ +using BenchmarkDotNet.Helpers.Reflection.Emit; +using System.Reflection; +using System.Reflection.Emit; + +namespace BenchmarkDotNet.Toolchains.InProcess.Emit.Implementation; + +partial class RunnableEmitter +{ + // MethodImplAttributes.Async was added in .Net 10. + private const MethodImplAttributes AsyncMethodImplAttribute = (MethodImplAttributes) 8192; + + // Looked up at runtime because the frameworks BenchmarkDotNet targets either lack it or mark it experimental. + private static readonly Lazy AsyncHelpersType = new(() => + typeof(object).Assembly.GetType("System.Runtime.CompilerServices.AsyncHelpers", throwOnError: true)!); + + /// + /// The csproj toolchains compile the runnable with the benchmark project's compiler features, + /// so Roslyn makes a method awaiting a runtime-async method runtime-async too. + /// + private static bool IsRuntimeAsync(MethodInfo method) => (method.MethodImplementationFlags & AsyncMethodImplAttribute) != 0; + + private MethodBuilder DefineRuntimeAsyncMethod(string methodName, Type returnType) + { + var methodBuilder = runnableBuilder + .DefineNonVirtualInstanceMethod( + methodName, + MethodAttributes.Private, + EmitParameterInfo.CreateReturnParameter(returnType) + ) + .SetAggressiveOptimizationImplementationFlag(); + methodBuilder.SetImplementationFlags(methodBuilder.GetMethodImplementationFlags() | AsyncMethodImplAttribute); + return methodBuilder; + } + + /// + /// Awaits the value on the stack returned by a runtime-async method, leaving its result (if any) on the stack. + /// + /// The result type, when nothing is left on the stack. + private static Type EmitRuntimeAsyncAwait(ILGenerator ilBuilder, Type awaitableType) + { + /* + // await task; + IL_0006: call void [System.Runtime]System.Runtime.CompilerServices.AsyncHelpers::Await(class [System.Runtime]System.Threading.Tasks.Task) + */ + var awaitMethod = GetAsyncHelpersAwaitMethod(awaitableType); + ilBuilder.Emit(OpCodes.Call, awaitMethod); + return awaitMethod.ReturnType; + } + + // AsyncHelpers.Await has an overload for each type a runtime-async method can return, which Roslyn calls directly. + private static MethodInfo GetAsyncHelpersAwaitMethod(Type awaitableType) + { + foreach (var method in AsyncHelpersType.Value.GetMethods(BindingFlags.Public | BindingFlags.Static)) + { + if (method.Name != "Await" || method.GetParameters() is not [var parameter]) + continue; + + var parameterType = parameter.ParameterType; + if (!method.IsGenericMethodDefinition) + { + if (parameterType == awaitableType) + return method; + } + else if (awaitableType.IsGenericType + && parameterType.IsGenericType + && parameterType.GetGenericTypeDefinition() == awaitableType.GetGenericTypeDefinition()) + { + return method.MakeGenericMethod(awaitableType.GetGenericArguments()); + } + } + throw new NotSupportedException($"AsyncHelpers has no Await overload for {awaitableType}."); + } + + /* + private async ValueTask GlobalSetup() + { + await base.GlobalSetup(); + } + */ + private void EmitRuntimeAsyncSetupCleanup(string methodName, MethodInfo methodToCall, SetupCleanupKind kind) + { + /* + .method private hidebysig + instance valuetype [System.Runtime]System.Threading.Tasks.ValueTask GlobalSetup () cil managed flags(2200) + */ + var methodBuilder = DefineRuntimeAsyncMethod(methodName, typeof(ValueTask)); + var ilBuilder = methodBuilder.GetILGenerator(); + + if (kind == SetupCleanupKind.GlobalCleanup) + { + EmitExtraGlobalCleanup(ilBuilder, null); + } + /* + // await base.GlobalSetup(); + IL_0000: ldarg.0 + IL_0001: call instance valuetype [System.Runtime]System.Threading.Tasks.ValueTask [BenchmarkDotNet.IntegrationTests]BenchmarkDotNet.IntegrationTests.RuntimeAsync.RuntimeAsyncTaskCaseBenchmark::GlobalSetup() + IL_0006: call void [System.Runtime]System.Runtime.CompilerServices.AsyncHelpers::Await(valuetype [System.Runtime]System.Threading.Tasks.ValueTask) + */ + if (!methodToCall.IsStatic) + { + ilBuilder.Emit(OpCodes.Ldarg_0); + } + ilBuilder.Emit(OpCodes.Call, methodToCall); + if (EmitRuntimeAsyncAwait(ilBuilder, methodToCall.ReturnType) != typeof(void)) + { + ilBuilder.Emit(OpCodes.Pop); + } + if (kind == SetupCleanupKind.GlobalSetup) + { + EmitExtraGlobalSetup(ilBuilder, null); + } + /* + IL_0021: ret + */ + ilBuilder.Emit(OpCodes.Ret); + } +} diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/SetupCleanupEmitter.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/SetupCleanupEmitter.cs index eeeb352bc2..164b76e232 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/SetupCleanupEmitter.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Emitters/SetupCleanupEmitter.cs @@ -69,7 +69,16 @@ [0] valuetype [System.Runtime]System.Threading.Tasks.ValueTask private void EmitAsyncSetupCleanup(string methodName, MethodInfo methodToCall, SetupCleanupKind kind) - => EmitAsyncSingleCall(methodName, typeof(AsyncValueTaskMethodBuilder), methodToCall, kind); + { + if (IsRuntimeAsync(methodToCall)) + { + EmitRuntimeAsyncSetupCleanup(methodName, methodToCall, kind); + } + else + { + EmitAsyncSingleCall(methodName, typeof(AsyncValueTaskMethodBuilder), methodToCall, kind); + } + } protected virtual void EmitExtraGlobalCleanup(ILGenerator ilBuilder, LocalBuilder? thisLocal) { } protected virtual void EmitExtraGlobalSetup(ILGenerator ilBuilder, LocalBuilder? thisLocal) { } diff --git a/tests/BenchmarkDotNet.IntegrationTests.RuntimeAsync/BenchmarkDotNet.IntegrationTests.RuntimeAsync.csproj b/tests/BenchmarkDotNet.IntegrationTests.RuntimeAsync/BenchmarkDotNet.IntegrationTests.RuntimeAsync.csproj new file mode 100644 index 0000000000..9a0aac8c80 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.RuntimeAsync/BenchmarkDotNet.IntegrationTests.RuntimeAsync.csproj @@ -0,0 +1,45 @@ + + + + BenchmarkDotNet.IntegrationTests.RuntimeAsync + net11.0 + + $(Features);runtime-async=on + true + BenchmarkDotNet.IntegrationTests.RuntimeAsync + BenchmarkDotNet.IntegrationTests.RuntimeAsync + true + false + AnyCPU + true + $(NoWarn);CA2007 + + + + + + + + + + + + + + Always + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/tests/BenchmarkDotNet.IntegrationTests.RuntimeAsync/RuntimeAsyncBenchmarks.cs b/tests/BenchmarkDotNet.IntegrationTests.RuntimeAsync/RuntimeAsyncBenchmarks.cs new file mode 100644 index 0000000000..4af1d46e55 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.RuntimeAsync/RuntimeAsyncBenchmarks.cs @@ -0,0 +1,74 @@ +using BenchmarkDotNet.Attributes; +using System.Runtime.CompilerServices; + +namespace BenchmarkDotNet.IntegrationTests.RuntimeAsync; + +public class RuntimeAsyncTaskCaseBenchmark +{ + [GlobalSetup] + public async ValueTask GlobalSetup() => await Task.Yield(); + + [GlobalCleanup] + public async Task GlobalCleanup() => await Task.Yield(); + + [IterationSetup] + public async ValueTask IterationSetup() => await Task.Yield(); + + [IterationCleanup] + public async Task IterationCleanup() => await Task.Yield(); + + [Benchmark] + public async Task TaskCase1() => await Task.CompletedTask; + + [Benchmark, Arguments(1, "1", 0.1)] + public async Task TaskCase1(int x, string y, double? z) => await Task.CompletedTask; + + [Benchmark] + public async Task TaskCase2() => await Task.FromResult("123"); + + [Benchmark, Arguments(2, "2", 0.2)] + public async Task TaskCase2(int x, string y, double? z) => await Task.FromResult("123"); + + [Benchmark] + public async ValueTask TaskCase3() => await Task.CompletedTask; + + [Benchmark, Arguments(3, "3", 0.3)] + public async ValueTask TaskCase3(int x, string y, double? z) => await Task.CompletedTask; + + [Benchmark] + public async ValueTask TaskCase4() => await new ValueTask(true); + + [Benchmark, Arguments(4, "4", 0.4)] + public async ValueTask TaskCase4(int x, string y, double? z) => await new ValueTask(true); +} + +// The generated wrappers follow each setup and cleanup method, whatever the workload is. +public class RuntimeAsyncSetupCleanupBenchmark +{ + [GlobalSetup] + public async Task GlobalSetup() => await Task.Yield(); + + [GlobalCleanup] + public static async ValueTask GlobalCleanup() => await Task.Yield(); + + [IterationSetup] + public async Task IterationSetup() => await Task.FromResult(1); + + [IterationCleanup] + public async ValueTask IterationCleanup() => await new ValueTask(1); + + [Benchmark] + public void Workload() { } +} + +// Roslyn ignores a builder override on a runtime-async method but keeps the attribute, which the generated WorkloadCore copies. +public class RuntimeAsyncBuilderOverrideBenchmark +{ + [Benchmark] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] + public async ValueTask Pooled() => await Task.CompletedTask; + + [Benchmark] + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder))] + public async ValueTask PooledResult() => await new ValueTask(1); +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.RuntimeAsync/RuntimeAsyncEmitTest.cs b/tests/BenchmarkDotNet.IntegrationTests.RuntimeAsync/RuntimeAsyncEmitTest.cs new file mode 100644 index 0000000000..d0414c1ce7 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.RuntimeAsync/RuntimeAsyncEmitTest.cs @@ -0,0 +1,62 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.IntegrationTests.InProcess.EmitTests; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using System.Reflection; + +namespace BenchmarkDotNet.IntegrationTests.RuntimeAsync; + +public class RuntimeAsyncEmitTest(ITestOutputHelper output) : BenchmarkTestExecutor(output) +{ + private const MethodImplAttributes AsyncMethodImpl = (MethodImplAttributes)0x2000; + + [Theory] + [InlineData(typeof(RuntimeAsyncTaskCaseBenchmark))] + [InlineData(typeof(RuntimeAsyncSetupCleanupBenchmark))] + [InlineData(typeof(RuntimeAsyncBuilderOverrideBenchmark))] + public void InProcessBenchmarkEmitsSameIL(Type benchmarkType) + { + // Without runtime-async compiled methods, both sides would be state machines and the diff would prove nothing. + var asyncMethods = benchmarkType + .GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static | BindingFlags.DeclaredOnly) + .Where(m => m.ReturnType != typeof(void)) + .ToArray(); + Assert.NotEmpty(asyncMethods); + Assert.All(asyncMethods, m => Assert.True(m.MethodImplementationFlags.HasFlag(AsyncMethodImpl), $"{m.Name} is not runtime-async")); + + var logger = new OutputLogger(Output); + var config = new ManualConfig() + .AddJob(Job.Dry.WithToolchain(InProcessEmitToolchain.Default).WithInvocationCount(4).WithUnrollFactor(4)) + .AddJob(Job.Dry.WithInvocationCount(4).WithUnrollFactor(4)) + .WithOptions(ConfigOptions.KeepBenchmarkFiles) + .AddLogger(logger); + + var summary = CanExecute(benchmarkType, config); + + DiffEmit(summary); + } + + // Without KeepBenchmarkFiles the runnables are emitted into a collectible assembly instead of a persisted one. + [Fact] + public void InProcessBenchmarkRunsWithoutSaving() + { + var config = new ManualConfig() + .AddJob(Job.Dry.WithToolchain(InProcessEmitToolchain.Default).WithInvocationCount(4).WithUnrollFactor(4)) + .AddLogger(new OutputLogger(Output)); + + CanExecute(config); + } + + private static void DiffEmit(Summary summary) + { + var emittedReport = summary.Reports.First(r => r.BenchmarkCase.GetToolchain() is InProcessEmitToolchain); + var compiledReport = summary.Reports.First(r => r.BenchmarkCase.GetToolchain() is not InProcessEmitToolchain); + NaiveRunnableEmitDiff.RunDiff( + compiledReport.BuildResult.ArtifactsPaths.ExecutablePath, + emittedReport.BuildResult.ArtifactsPaths.ExecutablePath, + ConsoleLogger.Default); + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.RuntimeAsync/xunit.runner.json b/tests/BenchmarkDotNet.IntegrationTests.RuntimeAsync/xunit.runner.json new file mode 100644 index 0000000000..53b385245e --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.RuntimeAsync/xunit.runner.json @@ -0,0 +1,8 @@ +{ + "shadowCopy": false, + "methodDisplay": "method", + "diagnosticMessages": true, + "longRunningTestSeconds": 60, + "parallelizeAssembly": false, + "parallelizeTestCollections": false +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/NaiveRunnableEmitDiff.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/NaiveRunnableEmitDiff.cs index 24f4d2a0f2..af2018e052 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/NaiveRunnableEmitDiff.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcess.EmitTests/NaiveRunnableEmitDiff.cs @@ -15,12 +15,34 @@ public class NaiveRunnableEmitDiff "BenchmarkDotNet.Autogenerated.UniqueProgramName", "BenchmarkDotNet.Autogenerated.DirtyAssemblyResolveHelper", // not required to be used in the InProcess toolchains (it's already used in the host process) // Poly-filled types added in old runtimes. - "System.Runtime.CompilerServices.CompilerFeatureRequiredAttribute" + "System.Runtime.CompilerServices.CompilerFeatureRequiredAttribute", + // Synthesized by the compiler for the csproj-built program, which inherits the benchmark project's + // language version and nullable context. + "Microsoft.CodeAnalysis.EmbeddedAttribute", + "System.Runtime.CompilerServices.RefSafetyRulesAttribute", + "System.Runtime.CompilerServices.NullableAttribute", + "System.Runtime.CompilerServices.NullableContextAttribute", + "System.Runtime.CompilerServices.NullablePublicOnlyAttribute", ]; private static readonly HashSet IgnoredAttributeTypeNames = [ - typeof(AggressivelyOptimizeMethodsAttribute).FullName! + typeof(AggressivelyOptimizeMethodsAttribute).FullName!, + // Assembly info the SDK generates for the csproj-built program. + typeof(System.Runtime.Versioning.TargetFrameworkAttribute).FullName!, + typeof(System.Reflection.AssemblyCompanyAttribute).FullName!, + typeof(System.Reflection.AssemblyConfigurationAttribute).FullName!, + typeof(System.Reflection.AssemblyFileVersionAttribute).FullName!, + typeof(System.Reflection.AssemblyInformationalVersionAttribute).FullName!, + typeof(System.Reflection.AssemblyProductAttribute).FullName!, + typeof(System.Reflection.AssemblyTitleAttribute).FullName!, + typeof(System.Reflection.AssemblyVersionAttribute).FullName!, + // Synthesized by the compiler for the csproj-built program, which inherits the benchmark project's + // language version and nullable context. + "System.Runtime.CompilerServices.RefSafetyRulesAttribute", + "System.Runtime.CompilerServices.NullableAttribute", + "System.Runtime.CompilerServices.NullableContextAttribute", + "System.Runtime.CompilerServices.NullablePublicOnlyAttribute", ]; private static readonly HashSet IgnoredRunnableMethodNames = @@ -49,12 +71,15 @@ public class NaiveRunnableEmitDiff { OpCodes.Leave_S, OpCodes.Leave }, }; - public static void RunDiff(string roslynAssemblyPath, string emittedAssemblyPath, ILogger logger) + /// + /// Set when the compiled assembly comes from the Roslyn toolchain, which doesn't weave AggressiveOptimization in. + /// + public static void RunDiff(string roslynAssemblyPath, string emittedAssemblyPath, ILogger logger, bool ignoreAggressiveOptimization = false) { using (var roslynAssemblyDefinition = AssemblyDefinition.ReadAssembly(roslynAssemblyPath)) using (var emittedAssemblyDefinition = AssemblyDefinition.ReadAssembly(emittedAssemblyPath)) { - Diff(roslynAssemblyDefinition, emittedAssemblyDefinition, logger); + Diff(roslynAssemblyDefinition, emittedAssemblyDefinition, logger, ignoreAggressiveOptimization); } // all checks have passed, so we can remove the files to avoid "file in use" problems @@ -287,12 +312,11 @@ private static void Diff(CustomAttribute left, CustomAttribute? right, ICustomAt object leftValue = attArg1.Value; object rightValue = attArg2.Value; - if (left.AttributeType.FullName == typeof(AsyncStateMachineAttribute).FullName) + if (leftValue is TypeReference leftType && rightValue is TypeReference rightType) { - // We can't compare type arguments of AsyncStateMachineAttribute because the type is generated, - // we just have to compare their ToString. - leftValue = attArg1.Value.ToString()!; - rightValue = attArg2.Value.ToString()!; + // Each assembly has its own references to a type argument, so we compare their names. + leftValue = leftType.FullName; + rightValue = rightType.FullName; } if (!Equals(leftValue, rightValue)) throw new InvalidOperationException($"No matching attribute for {left.AttributeType} ({owner})"); @@ -302,7 +326,8 @@ private static void Diff(CustomAttribute left, CustomAttribute? right, ICustomAt private static void Diff( AssemblyDefinition roslynAssemblyDefinition, AssemblyDefinition emittedAssemblyDefinition, - ILogger logger) + ILogger logger, + bool ignoreAggressiveOptimization) { Diff(roslynAssemblyDefinition.CustomAttributes, emittedAssemblyDefinition.CustomAttributes, roslynAssemblyDefinition); @@ -313,11 +338,11 @@ private static void Diff( if (module2 == null && module1.IsMain) module2 = emittedAssemblyDefinition.MainModule; - Diff(module1, module2!, logger); + Diff(module1, module2!, logger, ignoreAggressiveOptimization); } } - private static void Diff(ModuleDefinition module1, ModuleDefinition module2, ILogger logger) + private static void Diff(ModuleDefinition module1, ModuleDefinition module2, ILogger logger, bool ignoreAggressiveOptimization) { Diff(module1.CustomAttributes, module2.CustomAttributes, module1); @@ -325,11 +350,11 @@ private static void Diff(ModuleDefinition module1, ModuleDefinition module2, ILo { var type2 = module2.Types.SingleOrDefault(t => AreSameTypeIgnoreNested(type1, t)); - Diff(type1, type2, logger); + Diff(type1, type2, logger, ignoreAggressiveOptimization); } } - private static void Diff(TypeDefinition type1, TypeDefinition? type2, ILogger logger) + private static void Diff(TypeDefinition type1, TypeDefinition? type2, ILogger logger, bool ignoreAggressiveOptimization) { try { @@ -350,7 +375,7 @@ private static void Diff(TypeDefinition type1, TypeDefinition? type2, ILogger lo DiffDefinition(type1, type2); - DiffMembers(type1, type2!, logger); + DiffMembers(type1, type2!, logger, ignoreAggressiveOptimization); } catch (Exception ex) { @@ -375,7 +400,7 @@ private static void DiffDefinition(TypeDefinition type1, TypeDefinition? type2) Diff(type1.CustomAttributes, type2.CustomAttributes, type1); } - private static void DiffMembers(TypeDefinition type1, TypeDefinition type2, ILogger logger) + private static void DiffMembers(TypeDefinition type1, TypeDefinition type2, ILogger logger, bool ignoreAggressiveOptimization) { var fields2ByName = type2.Fields.ToLookup(f => f.Name); foreach (var field1 in type1.Fields) @@ -399,7 +424,7 @@ private static void DiffMembers(TypeDefinition type1, TypeDefinition type2, ILog if (method2 == null) method2 = methods2ByName[method1.Name].SingleOrDefault(); - if (Diff(method1, method2!)) + if (Diff(method1, method2!, ignoreAggressiveOptimization)) logger.WriteLineHelp(" OK."); else logger.WriteLineInfo(" SKIPPED."); @@ -419,7 +444,7 @@ private static void DiffMembers(TypeDefinition type1, TypeDefinition type2, ILog continue; var nested2 = nested2ByName[nested1.Name].SingleOrDefault() ?? type2.NestedTypes.SingleOrDefault(t => AreSameTypeIgnoreNested(nested1, t)); - Diff(nested1, nested2, logger); + Diff(nested1, nested2, logger, ignoreAggressiveOptimization); } } @@ -459,14 +484,14 @@ private static void Diff(FieldDefinition field1, FieldDefinition? field2) Diff(field1.CustomAttributes, field2.CustomAttributes, field1); } - private static bool Diff(MethodDefinition method1, MethodDefinition method2) + private static bool Diff(MethodDefinition method1, MethodDefinition method2, bool ignoreAggressiveOptimization) { if (IsRunnable(method1.DeclaringType) && IgnoredRunnableMethodNames.Contains(method1.Name)) { return false; } - DiffDefinition(method1, method2); + DiffDefinition(method1, method2, ignoreAggressiveOptimization); DiffVariables(method1, method2); @@ -475,16 +500,16 @@ private static bool Diff(MethodDefinition method1, MethodDefinition method2) return true; } - private static void DiffDefinition(MethodDefinition method1, MethodDefinition method2) + private static void DiffDefinition(MethodDefinition method1, MethodDefinition method2, bool ignoreAggressiveOptimization) { DiffSignature(method1, method2); if (method1.Attributes != method2.Attributes) throw new InvalidOperationException($"No matching method for {method1}"); - // Roslyn toolchain doesn't apply AggressiveOptimization, and doesn't need to because .Net Framework doesn't have tiered JIT. - // TODO: Don't exclude AggressiveOptimization if/when we compare IL in .Net Core. - if ((method1.ImplAttributes & ~MethodImplAttributes.AggressiveOptimization) != (method2.ImplAttributes & ~MethodImplAttributes.AggressiveOptimization)) + // The Roslyn toolchain doesn't apply AggressiveOptimization, and doesn't need to because .Net Framework doesn't have tiered JIT. + var ignoredImplAttributes = ignoreAggressiveOptimization ? MethodImplAttributes.AggressiveOptimization : 0; + if ((method1.ImplAttributes & ~ignoredImplAttributes) != (method2.ImplAttributes & ~ignoredImplAttributes)) throw new InvalidOperationException($"No matching method for {method1}"); if (method1.Parameters.Count != method2.Parameters.Count) diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs index 672188434e..62f89644d0 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs @@ -11,6 +11,7 @@ using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; using BenchmarkDotNet.Toolchains.Framework; +using BenchmarkDotNet.Toolchains.Roslyn; using JetBrains.Annotations; namespace BenchmarkDotNet.IntegrationTests @@ -32,8 +33,16 @@ private IConfig CreateInProcessConfig(OutputLogger? logger) .AddColumnProvider(DefaultColumnProviders.Instance); } - private IConfig CreateInProcessAndRoslynConfig(OutputLogger logger, bool consumeTasksSynchronously = false) + private IConfig CreateInProcessAndCompiledConfig(OutputLogger logger, bool consumeTasksSynchronously = false) { + // The compiled job builds the same generated code the emitter mirrors, for DiffEmit to compare against. + var compiledJob = Job.Dry + .WithInvocationCount(4) + .WithUnrollFactor(4) + .WithConsumeTasksSynchronously(consumeTasksSynchronously); + if (Portability.RuntimeInformation.IsFullFramework) + compiledJob = compiledJob.WithToolchain(RoslynFrameworkToolchain.Default); + var config = new ManualConfig() .AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()) .AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray()) @@ -46,33 +55,22 @@ private IConfig CreateInProcessAndRoslynConfig(OutputLogger logger, bool consume .WithInvocationCount(4) .WithUnrollFactor(4) .WithConsumeTasksSynchronously(consumeTasksSynchronously)) - .AddJob( - Job.Dry - .WithToolchain(RoslynFrameworkToolchain.Default) - .WithInvocationCount(4) - .WithUnrollFactor(4) - .WithConsumeTasksSynchronously(consumeTasksSynchronously)) + .AddJob(compiledJob) .WithOptions(ConfigOptions.KeepBenchmarkFiles) .AddLogger(logger ?? (Output != null ? new OutputLogger(Output) : ConsoleLogger.Default)); return config; } - private void DiffEmit(Summary summary) + private static void DiffEmit(Summary summary) { - // .Net Core does not support assembly saving so far - // SEE https://github.com/dotnet/corefx/issues/4491 - // TODO: Use new PersistedAssemblyBuilder when BDN and tests are updated to net9.0 or newer. - if (!Portability.RuntimeInformation.IsFullFramework) - return; - - var benchmarkCase = summary.BenchmarksCases.First(); - // The benchmark config built jobs with 2 toolchains, 1 InProcessEmit and 1 Roslyn, - // so we need to subtract 1 from the partition counter to obtain the emit output. + var emittedReport = summary.Reports.First(r => r.BenchmarkCase.GetToolchain() is InProcessEmitToolchain); + var compiledReport = summary.Reports.First(r => r.BenchmarkCase.GetToolchain() is not InProcessEmitToolchain); NaiveRunnableEmitDiff.RunDiff( - $@"{BuildPartition.GetProgramName(benchmarkCase, BuildPartition.s_partitionCounter)}.exe", - $@"{BuildPartition.GetProgramName(benchmarkCase, BuildPartition.s_partitionCounter - 1)}Emitted.dll", - ConsoleLogger.Default); + compiledReport.BuildResult.ArtifactsPaths.ExecutablePath, + emittedReport.BuildResult.ArtifactsPaths.ExecutablePath, + ConsoleLogger.Default, + ignoreAggressiveOptimization: compiledReport.BenchmarkCase.GetToolchain() is RoslynToolchain); } [Fact] @@ -104,7 +102,7 @@ public void InProcessBenchmarkSimpleCasesReflectionEmitSupported() } } - [TheoryEnvSpecific("We can't use Roslyn toolchain for .NET Core because we don't know which assemblies to reference and .NET Core does not support dynamic assembly saving", EnvRequirement.FullFrameworkOnly)] + [Theory] [InlineData(typeof(SampleBenchmark), false)] [InlineData(typeof(RunnableVoidCaseBenchmark), false)] [InlineData(typeof(RunnableRefStructCaseBenchmark), false)] @@ -128,7 +126,7 @@ public void InProcessBenchmarkSimpleCasesReflectionEmitSupported() public void InProcessBenchmarkEmitsSameIL(Type benchmarkType, bool consumeTasksSynchronously) { var logger = new OutputLogger(Output); - var config = CreateInProcessAndRoslynConfig(logger, consumeTasksSynchronously); + var config = CreateInProcessAndCompiledConfig(logger, consumeTasksSynchronously); var summary = CanExecute(benchmarkType, config);