Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions build/crossgen2-scenarios.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,9 @@ parameters:
- displayName: Default
arguments: --property profile=default
condition: 'true'
# Disabled: fails with ILCompiler.CodeGenerationFailedException (IndexOutOfRangeException
# in CorInfoImpl.HandleToModuleToken) during the crossgen2 composite publish, and on
# Windows runs causes crank to hang, blocking the rest of the jobs in the pipeline.
# Root cause fixed by dotnet/runtime#129053 (composite R2R token corruption for
# devirtualized async-variant callees, merged 2026-06-06). Re-enable once that fix
# flows into the .NET 11 SDK consumed by the benchmark agents.
- displayName: CG2 composite # since 6.0-preview4 CG2 is the default when PublishReadyToRun=true is set
arguments: --property profile=composite --application.buildArguments \"/p:PublishReadyToRun=true /p:PublishReadyToRunComposite=true /p:PublishReadyToRunUseCrossgen2=true \"
condition: 'false'
condition: 'true'
- displayName: CG2 non-composite # /p:PublishReadyToRunComposite=false is the default
arguments: --property profile=non-composite --application.buildArguments \"/p:PublishReadyToRun=true /p:PublishReadyToRunUseCrossgen2=true /p:PublishReadyToRunComposite=false\"
condition: 'true'
Expand Down
2 changes: 1 addition & 1 deletion build/frameworks-database-scenarios.yml
Original file line number Diff line number Diff line change
Expand Up @@ -108,5 +108,5 @@ steps:
"name": "crank",
"condition": "(${{ parameters.condition }})",
"retries": 1,
"args": [ "${{ s.arguments }} ${{ parameters.arguments }} ${{ parameters.config }} --application.noClean true --no-metadata --no-measurements --load.options.reuseBuild true --session $(session) --description \"${{ s.displayName }} $(System.JobDisplayName)\" --property buildId=\"$(buildId)\" --property buildNumber=\"$(buildNumber)\" --command-line-property --table TrendBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ]
"args": [ "${{ s.arguments }} ${{ parameters.arguments }} ${{ parameters.config }} --no-metadata --no-measurements --load.options.reuseBuild true --session $(session) --description \"${{ s.displayName }} $(System.JobDisplayName)\" --property buildId=\"$(buildId)\" --property buildNumber=\"$(buildNumber)\" --command-line-property --table TrendBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ]
}
2 changes: 1 addition & 1 deletion build/frameworks-scenarios.yml
Original file line number Diff line number Diff line change
Expand Up @@ -98,5 +98,5 @@ steps:
"name": "crank",
"condition": "(${{ parameters.condition }})",
"retries": 1,
"args": [ "${{ s.arguments }} ${{ parameters.arguments }} ${{ parameters.config }} --application.noClean true --no-metadata --no-measurements --load.options.reuseBuild true --session $(session) --description \"${{ s.displayName }} $(System.JobDisplayName)\" --property buildId=\"$(buildId)\" --property buildNumber=\"$(buildNumber)\" --command-line-property --table TrendBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ]
"args": [ "${{ s.arguments }} ${{ parameters.arguments }} ${{ parameters.config }} --no-metadata --no-measurements --load.options.reuseBuild true --session $(session) --description \"${{ s.displayName }} $(System.JobDisplayName)\" --property buildId=\"$(buildId)\" --property buildNumber=\"$(buildNumber)\" --command-line-property --table TrendBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ]
}
13 changes: 3 additions & 10 deletions build/nativeaot-scenarios.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,7 @@ parameters:

- displayName: Goldilocks Stage 1 (CoreCLR - Trim R2R SingleFile)
arguments: --scenario basicminimalapipublishtrimr2rsinglefile $(goldilocksJobs) --property scenario=Stage1TrimR2RSingleFile --property publish=coreclr --application.environmentVariables DOTNET_GCDynamicAdaptationMode=1
# Disabled: hits dotnet/runtime crossgen2 composite R2R bug on async-variant callees
# (Kestrel Http1ChunkedEncodingMessageBody.PumpAsync). Re-enable once
# https://github.com/dotnet/runtime/pull/129053 flows into the .NET 11 SDK.
condition: 'false'
condition: 'true'

- displayName: Goldilocks Stage 1 (NativeAOT)
# workaround https://github.com/dotnet/runtime/issues/81382 by explicitly referencing a Microsoft.DotNet.ILCompiler version
Expand Down Expand Up @@ -73,9 +70,7 @@ parameters:

- displayName: Goldilocks Stage 2 (CoreCLR - Trim R2R SingleFile)
arguments: --scenario todosapipublishtrimr2rsinglefile $(goldilocksJobs) --property scenario=Stage2TrimR2RSingleFile --property publish=coreclr --application.environmentVariables DOTNET_GCDynamicAdaptationMode=1
# Disabled: see note on "Goldilocks Stage 1 (CoreCLR - Trim R2R SingleFile)" above.
# Re-enable once https://github.com/dotnet/runtime/pull/129053 flows into the .NET 11 SDK.
condition: 'false'
condition: 'true'

- displayName: Goldilocks Stage 2 (NativeAOT)
# workaround https://github.com/dotnet/runtime/issues/81382 by explicitly referencing a Microsoft.DotNet.ILCompiler version
Expand Down Expand Up @@ -111,9 +106,7 @@ parameters:

- displayName: Goldilocks gRPC Stage 1 (CoreCLR - Trim R2R SingleFile)
arguments: --scenario basicgrpcpublishtrimr2rsinglefile $(goldilocksJobs) --property scenario=Stage1GrpcTrimR2RSingleFile --property publish=coreclr --application.environmentVariables DOTNET_GCDynamicAdaptationMode=1
# Disabled: see note on "Goldilocks Stage 1 (CoreCLR - Trim R2R SingleFile)" above.
# Re-enable once https://github.com/dotnet/runtime/pull/129053 flows into the .NET 11 SDK.
condition: 'false'
condition: 'true'

- displayName: Goldilocks gRPC Stage 1 (NativeAOT)
# workaround https://github.com/dotnet/runtime/issues/81382 by explicitly referencing a Microsoft.DotNet.ILCompiler version
Expand Down
2 changes: 1 addition & 1 deletion build/regressions.config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ sources:
- area-perf
- exclude: drogon|fasthttp|actix|gin|express|node|netty|spring|vertx|ntex|Siryn
- exclude: ConnectionClose
- exclude: '[aA]zure|Cloud|Idna'
- exclude: '[aA]zure|[cC]obalt|Cloud|Idna'
- exclude: Arm\s80
daysToLoad: 60
stdevCount: 7
Expand Down
15 changes: 4 additions & 11 deletions build/singlefile-scenarios.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,29 +33,22 @@ parameters:
- displayName: Self-contained
arguments: --application.buildArguments \"/p:PublishReadyToRun=true /p:PublishSingleFile=false\" --property mode=SelfContained
condition: 'true'
# Disabled: hangs / fails with ILCompiler.CodeGenerationFailedException
# (IndexOutOfRangeException in CorInfoImpl.HandleToModuleToken) during the
# crossgen2 composite publish on .NET 11 preview SDKs predating
# dotnet/runtime#129053 ("Fix composite R2R token corruption for devirtualized
# async-variant callees", merged 2026-06-06). Re-enable once that fix flows
# into the SDK consumed by the benchmark agents.
- displayName: Single-file
arguments: --application.buildArguments \"/p:PublishReadyToRun=true /p:PublishSingleFile=true\" --property mode=SingleFile
condition: 'false'
condition: 'true'
- displayName: Trimmed
arguments: --application.buildArguments \"/p:PublishReadyToRun=true /p:PublishSingleFile=true /p:PublishTrimmed=true\" --property mode=Trimmed
condition: 'false'
condition: 'true'
- displayName: AOT
# workaround https://github.com/dotnet/runtime/issues/81382 by explicitly referencing a Microsoft.DotNet.ILCompiler version
arguments: --application.buildArguments \"/p:PublishAot=true /p:StripSymbols=true\" --property mode=Aot --application.packageReferences \"Microsoft.DotNet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion)\"
condition: 'true'
# Disabled: see note on "Single-file" above (dotnet/runtime#129053).
- displayName: R2RComposite-Windows
arguments: --application.buildArguments \"/p:PublishReadyToRun=true /p:PublishReadyToRunComposite=true /p:PublishSingleFile=true /p:RuntimeIdentifier=win-x64\" --property mode=R2RComposite --property rid=win-x64 --application.options.requiredOperatingSystem windows
condition: 'false'
condition: 'true'
- displayName: R2RComposite-Linux
arguments: --application.buildArguments \"/p:PublishReadyToRun=true /p:PublishReadyToRunComposite=true /p:PublishSingleFile=true /p:RuntimeIdentifier=linux-x64\" --property mode=R2RComposite --property rid=linux-x64 --application.options.requiredOperatingSystem linux
condition: 'false'
condition: 'true'

steps:
- ${{ each s in parameters.scenarios }}:
Expand Down
3 changes: 2 additions & 1 deletion docker/mssql-techempower/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ ADD import-data.sh .
USER root

# Grant permissions for the import-data script to be executable
RUN chmod +x import-data.sh
RUN chmod +x import-data.sh entrypoint.sh && \
test -x /opt/mssql-tools18/bin/sqlcmd

USER mssql

Expand Down
26 changes: 13 additions & 13 deletions docker/mssql-techempower/create.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ DECLARE @max INT = 10000;

WHILE @cnt < @max
BEGIN
INSERT INTO [world] ([randomNumber]) VALUES ( ABS(CHECKSUM(NewId())) % 10000 );
INSERT INTO [world] ([randomNumber]) VALUES ( (ABS(CHECKSUM(NewId())) % 10000) + 1 );
SET @cnt = @cnt + 1;
END;
GO
Expand All @@ -32,18 +32,18 @@ CREATE TABLE [fortune] (
)
GO

INSERT INTO [fortune] ([message]) VALUES ('fortune: No such file or directory');
INSERT INTO [fortune] ([message]) VALUES ('A computer scientist is someone who fixes things that aren''t broken.');
INSERT INTO [fortune] ([message]) VALUES ('After enough decimal places, nobody gives a damn.');
INSERT INTO [fortune] ([message]) VALUES ('A bad random number generator: 1, 1, 1, 1, 1, 4.33e+67, 1, 1, 1');
INSERT INTO [fortune] ([message]) VALUES ('A computer program does what you tell it to do, not what you want it to do.');
INSERT INTO [fortune] ([message]) VALUES ('Emacs is a nice operating system, but I prefer UNIX. — Tom Christaensen');
INSERT INTO [fortune] ([message]) VALUES ('Any program that runs right is obsolete.');
INSERT INTO [fortune] ([message]) VALUES ('A list is only as strong as its weakest link. — Donald Knuth');
INSERT INTO [fortune] ([message]) VALUES ('Feature: A bug with seniority.');
INSERT INTO [fortune] ([message]) VALUES ('Computers make very fast, very accurate mistakes.');
INSERT INTO [fortune] ([message]) VALUES ('<script>alert("This should not be displayed in a browser alert box.");</script>');
INSERT INTO [fortune] ([message]) VALUES ('フレームワークのベンチマーク');
INSERT INTO [fortune] ([message]) VALUES (N'fortune: No such file or directory');
INSERT INTO [fortune] ([message]) VALUES (N'A computer scientist is someone who fixes things that aren''t broken.');
INSERT INTO [fortune] ([message]) VALUES (N'After enough decimal places, nobody gives a damn.');
INSERT INTO [fortune] ([message]) VALUES (N'A bad random number generator: 1, 1, 1, 1, 1, 4.33e+67, 1, 1, 1');
INSERT INTO [fortune] ([message]) VALUES (N'A computer program does what you tell it to do, not what you want it to do.');
INSERT INTO [fortune] ([message]) VALUES (N'Emacs is a nice operating system, but I prefer UNIX. — Tom Christaensen');
INSERT INTO [fortune] ([message]) VALUES (N'Any program that runs right is obsolete.');
INSERT INTO [fortune] ([message]) VALUES (N'A list is only as strong as its weakest link. — Donald Knuth');
INSERT INTO [fortune] ([message]) VALUES (N'Feature: A bug with seniority.');
INSERT INTO [fortune] ([message]) VALUES (N'Computers make very fast, very accurate mistakes.');
INSERT INTO [fortune] ([message]) VALUES (N'<script>alert("This should not be displayed in a browser alert box.");</script>');
INSERT INTO [fortune] ([message]) VALUES (N'フレームワークのベンチマーク');
GO

PRINT "Data imported"
19 changes: 15 additions & 4 deletions docker/mssql-techempower/entrypoint.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,15 @@
/opt/mssql/bin/sqlservr & ./import-data.sh
while :; do
sleep 300
done
#!/usr/bin/env bash
set -euo pipefail

/opt/mssql/bin/sqlservr &
sqlservr_pid=$!

cleanup() {
kill "$sqlservr_pid" > /dev/null 2>&1 || true
}

trap cleanup EXIT
./import-data.sh
trap - EXIT

wait "$sqlservr_pid"
31 changes: 27 additions & 4 deletions docker/mssql-techempower/import-data.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,28 @@
#wait for the SQL Server to come up
sleep 20s
#!/usr/bin/env bash
set -euo pipefail

#run the setup script to create the DB and the schema in the DB
/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $SA_PASSWORD -d master -i create.sql
sqlcmd_path="/opt/mssql-tools18/bin/sqlcmd"
sqlcmd=("$sqlcmd_path" -S localhost -U sa -P "$SA_PASSWORD" -C)

if [ ! -x "$sqlcmd_path" ]; then
echo "sqlcmd not found at expected path '$sqlcmd_path'" >&2
exit 1
fi

max_attempts=60
attempt=1
while [ "$attempt" -le "$max_attempts" ]; do
if "${sqlcmd[@]}" -l 1 -Q "SELECT 1" > /dev/null 2>&1; then
break
fi

if [ "$attempt" -eq "$max_attempts" ]; then
echo "Timed out waiting for SQL Server to accept connections after ${max_attempts} attempts." >&2
exit 1
fi

sleep 1
attempt=$((attempt + 1))
done

"${sqlcmd[@]}" -b -d master -i create.sql
3 changes: 2 additions & 1 deletion scenarios/containers.benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ jobs:
environmentVariables:
database: "PostgreSQL"
connectionString: '"Server={{databaseServer}};Database=hello_world;User Id=benchmarkdbuser;Password=benchmarkdbpass;Maximum Pool Size=256;NoResetOnClose=true;Enlist=false;Max Auto Prepare=4"'
noClean: true
options:
reuseBuild: true

scenarios:
json_aspnet:
Expand Down
6 changes: 4 additions & 2 deletions scenarios/database.benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ jobs:
dockerImageName: postgres_te
dockerContextDirectory: toolset/databases/postgres
readyStateText: ready to accept connections
noClean: true
options:
reuseBuild: true

mssql:
source:
Expand All @@ -43,7 +44,8 @@ jobs:
dockerImageName: mssql_te
dockerContextDirectory: docker/mssql-techempower
readyStateText: Data imported
noClean: true
options:
reuseBuild: true

scenarios:
fortunes:
Expand Down
3 changes: 2 additions & 1 deletion scenarios/goldilocks.benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ jobs:
dockerImageName: postgres_te
dockerContextDirectory: toolset/databases/postgres
readyStateText: ready to accept connections
noClean: true
options:
reuseBuild: true


scenarios:
Expand Down
3 changes: 2 additions & 1 deletion scenarios/orchard.benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ jobs:
dockerImageName: postgres_te
dockerContextDirectory: toolset/databases/postgres
readyStateText: ready to accept connections
noClean: true
options:
reuseBuild: true

scenarios:
about-sqlite:
Expand Down
3 changes: 2 additions & 1 deletion scenarios/platform.benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ jobs:
dockerImageName: postgres_te
dockerContextDirectory: toolset/databases/postgres
readyStateText: ready to accept connections
noClean: true
options:
reuseBuild: true

scenarios:
plaintext:
Expand Down
Loading
Loading