diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index e2f24dc2..781fe3cf 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,32 +3,46 @@ "isRoot": true, "tools": { "powershell": { - "version": "7.5.3", + "version": "7.6.1", "commands": [ "pwsh" ], "rollForward": false }, "dotnet-coverage": { - "version": "17.14.2", + "version": "18.6.2", "commands": [ "dotnet-coverage" ], "rollForward": false }, "nbgv": { - "version": "3.7.115", + "version": "3.9.50", "commands": [ "nbgv" ], "rollForward": false }, "docfx": { - "version": "2.78.3", + "version": "2.78.5", "commands": [ "docfx" ], "rollForward": false + }, + "nerdbank.dotnetrepotools": { + "version": "1.3.22", + "commands": [ + "repo" + ], + "rollForward": false + }, + "dotnet-symbol": { + "version": "9.0.661903", + "commands": [ + "dotnet-symbol" + ], + "rollForward": false } } } diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index 4a440efc..00000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -# Refer to https://hub.docker.com/_/microsoft-dotnet-sdk for available versions -FROM mcr.microsoft.com/dotnet/sdk:9.0.305-noble@sha256:604ef064c6d91068eeb9d946036d8ffadbe25589c4cd77a230fc96e0f6d01d72 - -# Installing mono makes `dotnet test` work without errors even for net472. -# But installing it takes a long time, so it's excluded by default. -#RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -#RUN echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic main" | tee /etc/apt/sources.list.d/mono-official-stable.list -#RUN apt-get update -#RUN DEBIAN_FRONTEND=noninteractive apt-get install -y mono-devel - -# Clear the NUGET_XMLDOC_MODE env var so xml api doc files get unpacked, allowing a rich experience in Intellisense. -# See https://github.com/dotnet/dotnet-docker/issues/2790 for a discussion on this, where the prioritized use case -# was *not* devcontainers, sadly. -ENV NUGET_XMLDOC_MODE= diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 1a3a0086..00000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "Dev space", - "dockerFile": "Dockerfile", - "customizations": { - "vscode": { - "settings": { - "terminal.integrated.shell.linux": "/usr/bin/pwsh" - }, - "extensions": [ - "ms-azure-devops.azure-pipelines", - "ms-dotnettools.csharp", - "k--kato.docomment", - "editorconfig.editorconfig", - "esbenp.prettier-vscode", - "pflannery.vscode-versionlens", - "davidanson.vscode-markdownlint", - "dotjoshjohnson.xml", - "ms-vscode-remote.remote-containers", - "ms-azuretools.vscode-docker", - "tintoy.msbuild-project-tools" - ] - } - }, - "postCreateCommand": "./init.ps1 -InstallLocality machine" -} diff --git a/.github/Prime-ForCopilot.ps1 b/.github/Prime-ForCopilot.ps1 new file mode 100644 index 00000000..e0b1fb79 --- /dev/null +++ b/.github/Prime-ForCopilot.ps1 @@ -0,0 +1,5 @@ +if ((git -C $PSScriptRoot rev-parse --is-shallow-repository) -eq 'true') +{ + Write-Host "Shallow clone detected, disabling NBGV Git engine so the build can succeed." + $env:NBGV_GitEngine='Disabled' +} diff --git a/.github/actions/publish-artifacts/action.yaml b/.github/actions/publish-artifacts/action.yaml index 1f345fe6..87b2f9c0 100644 --- a/.github/actions/publish-artifacts/action.yaml +++ b/.github/actions/publish-artifacts/action.yaml @@ -14,46 +14,46 @@ runs: - name: ๐Ÿ“ข Upload project.assets.json files if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: projectAssetsJson-${{ runner.os }} path: ${{ runner.temp }}/_artifacts/projectAssetsJson continue-on-error: true - name: ๐Ÿ“ข Upload variables - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: variables-${{ runner.os }} path: ${{ runner.temp }}/_artifacts/Variables continue-on-error: true - name: ๐Ÿ“ข Upload build_logs if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: build_logs-${{ runner.os }} path: ${{ runner.temp }}/_artifacts/build_logs continue-on-error: true - name: ๐Ÿ“ข Upload testResults if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: testResults-${{ runner.os }} path: ${{ runner.temp }}/_artifacts/testResults continue-on-error: true - name: ๐Ÿ“ข Upload coverageResults if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverageResults-${{ runner.os }} path: ${{ runner.temp }}/_artifacts/coverageResults continue-on-error: true - name: ๐Ÿ“ข Upload symbols - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: symbols-${{ runner.os }} path: ${{ runner.temp }}/_artifacts/symbols continue-on-error: true - name: ๐Ÿ“ข Upload deployables - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: deployables-${{ runner.os }} path: ${{ runner.temp }}/_artifacts/deployables diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 08593502..0187843a 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -3,7 +3,8 @@ ## High level guidance * Review the `CONTRIBUTING.md` file for instructions to build and test the software. -* Set the `NBGV_GitEngine` environment variable to `Disabled` before running any `dotnet` or `msbuild` commands. +* Run the `.github/Prime-ForCopilot.ps1` script (once) before running any `dotnet` or `msbuild` commands. + If you see any build errors about not finding git objects or a shallow clone, it may be time to run this script again. ## Software Design @@ -12,9 +13,68 @@ ## Testing -* There should generally be one test project (under the `test` directory) per shipping project (under the `src` directory). Test projects are named after the project being tested with a `.Test` suffix. -* Tests should use the Xunit testing framework. -* Some tests are known to be unstable. When running tests, you should skip the unstable ones by running `dotnet test --filter "TestCategory!=FailsInCloudTest"`. +**IMPORTANT**: This repository uses Microsoft.Testing.Platform (MTP v2) with xunit v3. Traditional `--filter` syntax does NOT work. Use the options below instead. + +* There should generally be one test project (under the `test` directory) per shipping project (under the `src` directory). Test projects are named after the project being tested with a `.Tests` suffix. +* Tests use xunit v3 with Microsoft.Testing.Platform (MTP v2). Traditional VSTest `--filter` syntax does NOT work. +* Some tests are known to be unstable. When running tests, you should skip the unstable ones by using `-- --filter-not-trait "FailsInCloudTest=true"`. + +### Running Tests + +**Run all tests**: +```bash +dotnet test --no-build -c Release +``` + +**Run tests for a specific test project**: +```bash +dotnet test --project test/Library.Tests/Library.Tests.csproj --no-build -c Release +``` + +**Run a single test method**: +```bash +dotnet test --project test/Library.Tests/Library.Tests.csproj --no-build -c Release -- --filter-method ClassName.MethodName +``` + +**Run all tests in a test class**: +```bash +dotnet test --project test/Library.Tests/Library.Tests.csproj --no-build -c Release -- --filter-class ClassName +``` + +**Run tests with wildcard matching** (supports wildcards at beginning and/or end): +```bash +dotnet test --project test/Library.Tests/Library.Tests.csproj --no-build -c Release -- --filter-method "*Pattern*" +``` + +**Run tests with a specific trait** (equivalent to category filtering): +```bash +dotnet test --project test/Library.Tests/Library.Tests.csproj --no-build -c Release -- --filter-trait "TraitName=value" +``` + +**Exclude tests with a specific trait** (skip unstable tests): +```bash +dotnet test --project test/Library.Tests/Library.Tests.csproj --no-build -c Release -- --filter-not-trait "TestCategory=FailsInCloudTest" +``` + +**Run tests for a specific framework only**: +```bash +dotnet test --project test/Library.Tests/Library.Tests.csproj --no-build -c Release --framework net9.0 +``` + +**List all available tests without running them**: +```bash +cd test/Library.Tests +dotnet run --no-build -c Release --framework net9.0 -- --list-tests +``` + +**Key points about test filtering with MTP v2 / xunit v3**: +- Options after `--` are passed to the test runner, not to `dotnet test` +- Use `--filter-method`, `--filter-class`, `--filter-namespace` for simple filtering +- Use `--filter-trait` and `--filter-not-trait` for trait-based filtering (replaces `--filter "TestCategory=..."`) +- Traditional VSTest `--filter` expressions do NOT work +- Wildcards `*` are supported at the beginning and/or end of filter values +- Multiple simple filters of the same type use OR logic, different types combine with AND +- See `--help` for query filter language for advanced scenarios ## Coding style diff --git a/.github/skills/update-library-template/SKILL.md b/.github/skills/update-library-template/SKILL.md new file mode 100644 index 00000000..941df2f2 --- /dev/null +++ b/.github/skills/update-library-template/SKILL.md @@ -0,0 +1,73 @@ +--- +name: update-library-template +description: Merges the latest Library.Template into this repo (at position of HEAD) and resolves conflicts. +disable-model-invocation: true +--- + +# Instructions + +1. Run `./tools/MergeFrom-Template.ps1` from the repo root. +2. Resolve merge conflicts, taking into account conflict resolution policy below. +3. Validate the changes, as described in the validation section below. +4. Committing your changes (if applicable). + +## Conflict resolution policy + +There may be [special notes](template-release-notes.md) that describe special considerations for certain files or scenarios to help you resolve conflicts appropriately. +Always refer to that file before proceeding. +In particular, focus on the *incoming* part of the file, since it represents the changes from the Library.Template that you are merging into your repo. + +Also consider that some repos choose to reject certain Library.Template patterns. +For example the template uses MTPv2 for test projects, but a repo might have chosen not to adopt that. +When resolving merge conflicts, consider whether it looks like the relevant code file is older than it should be given the changes the template is bringing in. +Ask the user when in doubt as to whether the conflict should be resolved in favor of 'catching up' with the template or keeping the current changes. + +Use #runSubagent to analyze and resolve merge conflicts across files in parallel. + +### Keep Current files + +Conflicts in the following files should always be resolved by keeping the current version (i.e. discard incoming changes): + +* README.md + +### Deleted files + +Very typically, when the incoming change is to a file that was deleted locally, the correct resolution is to re-delete the file. + +In some cases however, the deleted file may have incoming changes that should be applied to other files. +The `test/Library.Tests/Library.Tests.csproj` file is very typical of this. +Changes to this file should very typically be applied to any and all test projects in the repo. +You are responsible for doing this in addition to re-deleting this template file. + +## Updating package and SDK versions + +After the merge, always check global.json for MSBuild Sdks with names starting with `Microsoft.VisualStudio.Internal.MicroBuild`. +These SDK versions should match the value of the `MicroBuildVersion` property found in `Directory.Packages.props`. +Always take the latest of the versions you see among these SDKs and the `MicroBuildVersion` property. + +## Validation + +Validate the merge result (after resolving any conflicts, if applicable). +Use #runSubagent for each step. + +1. Verify that `dotnet restore` succeeds. Fix any issues that come up. +2. Verify that `dotnet build` succeeds. +3. Verify that tests succeed by running `tools/dotnet-test-cloud.ps1`. + +While these validations are described using `dotnet` CLI commands, some repos require using full msbuild.exe. +You can detect this by checking the `azure-pipelines/dotnet.yml` or `.github/workflows/build.yml` files for use of one or the other tool. + +You are *not* responsible for fixing issues that the merge did not cause. +If validation fails for reasons that seem unrelated to the changes brought in by the merge, advise the user and ask how they'd like you to proceed. +That said, sometimes merges will bring in SDK or dependency updates that can cause breaks in seemingly unrelated areas. +In such cases, you should investigate and solve the issues as needed. + +## Committing your changes + +If you have to make any changes for validations to pass, consider whether they qualify as a bad merge conflict resolution or more of a novel change that you're making to work with the Library.Template update. +Merge conflict resolution fixes ideally get amended into the merge commit, while novel changes would go into a novel commit after the merge commit. + +Always author your commits using `git commit --author "๐Ÿค– Copilot "` (and possibly other parameters). +Describe the nature of the merge conflicts you encountered and how you resolved them in your commit message. + +Later, if asked to review pull request validation breaks, always author a fresh commit with each fix that you push, unless the user directs you to do otherwise. diff --git a/.github/skills/update-library-template/template-release-notes.md b/.github/skills/update-library-template/template-release-notes.md new file mode 100644 index 00000000..975da315 --- /dev/null +++ b/.github/skills/update-library-template/template-release-notes.md @@ -0,0 +1,20 @@ +# Template release notes + +This file will describe significant changes in Library.Template as they are introduced, especially if they require special consideration when merging updates into existing repos. +This file is referenced by update-library-template.prompt.md and should remain in place to facilitate future merges, whether done manually or by AI. + +## Solution rename + +Never leave a Library.slnx file in the repository. +You might even see one there even though this particular merge didn't bring it in. +This can be an artifact of having renamed Library.sln to Library.slnx in the template repo, but ultimately the receiving repo should have only one .sln or .slnx file, with a better name than `Library`. +Delete any `Library.slnx` that you see. +Migrate an `.sln` in the repo root to `.slnx` using this command: + +```ps1 +dotnet solution EXISTING.sln migrate +``` + +This will create an EXISTING.slnx file. `git add` that file, then `git rm` the old `.sln` file. +Sometimes a repo will reference the sln filename in a script or doc somewhere. +Search the repo for such references and update them to the slnx file. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index f4f832e5..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: CI - -on: - push: - branches: - - main - - microbuild - - validate/* - pull_request: - -env: - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - BUILDCONFIGURATION: Release - # codecov_token: 4dc9e7e2-6b01-4932-a180-847b52b43d35 # Get a new one from https://codecov.io/ - NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages/ - -jobs: - build: - - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: - - windows-2022 - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # avoid shallow clone so nbgv can do its work. - - name: โš™ Install prerequisites - run: | - ./init.ps1 -UpgradePrerequisites - dotnet --info - - # Print mono version if it is present. - if (Get-Command mono -ErrorAction SilentlyContinue) { - mono --version - } - shell: pwsh - - name: โš™๏ธ Set pipeline variables based on source - run: tools/variables/_define.ps1 - shell: pwsh - - name: ๐Ÿ›  build - run: dotnet build src -t:build,pack --no-restore -c ${{ env.BUILDCONFIGURATION }} /bl:"${{ runner.temp }}/_artifacts/build_logs/build.binlog" - - name: ๐Ÿงช test - run: tools/dotnet-test-cloud.ps1 -Configuration ${{ env.BUILDCONFIGURATION }} -Agent ${{ runner.os }} - shell: pwsh - - name: ๐Ÿ’…๐Ÿป Verify formatted code - run: dotnet format --verify-no-changes --no-restore - shell: pwsh - if: runner.os == 'Linux' - - name: โš™ Update pipeline variables based on build outputs - run: tools/variables/_define.ps1 - shell: pwsh - - name: ๐Ÿ“ฅ Collect artifacts - run: tools/artifacts/_stage_all.ps1 - shell: pwsh - if: always() - - name: ๐Ÿ“ข Upload project.assets.json files - if: always() - uses: actions/upload-artifact@v4 - with: - name: projectAssetsJson-${{ runner.os }} - path: ${{ runner.temp }}/_artifacts/projectAssetsJson - continue-on-error: true - - name: ๐Ÿ“ข Upload variables - uses: actions/upload-artifact@v4 - with: - name: variables-${{ runner.os }} - path: ${{ runner.temp }}/_artifacts/Variables - continue-on-error: true - - name: ๐Ÿ“ข Upload build_logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: build_logs-${{ runner.os }} - path: ${{ runner.temp }}/_artifacts/build_logs - continue-on-error: true - - name: ๐Ÿ“ข Upload test_logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: test_logs-${{ runner.os }} - path: ${{ runner.temp }}/_artifacts/test_logs - continue-on-error: true - - name: ๐Ÿ“ข Upload testResults - if: always() - uses: actions/upload-artifact@v4 - with: - name: testResults-${{ runner.os }} - path: ${{ runner.temp }}/_artifacts/testResults - continue-on-error: true - - name: ๐Ÿ“ข Upload coverageResults - if: always() - uses: actions/upload-artifact@v4 - with: - name: coverageResults-${{ runner.os }} - path: ${{ runner.temp }}/_artifacts/coverageResults - continue-on-error: true - - name: ๐Ÿ“ข Upload symbols - uses: actions/upload-artifact@v4 - with: - name: symbols-${{ runner.os }} - path: ${{ runner.temp }}/_artifacts/symbols - continue-on-error: true - - name: ๐Ÿ“ข Upload deployables - uses: actions/upload-artifact@v4 - with: - name: deployables-${{ runner.os }} - path: ${{ runner.temp }}/_artifacts/deployables - if: always() - - name: ๐Ÿ“ข Publish code coverage results to codecov.io - run: ./azure-pipelines/publish-CodeCov.ps1 -CodeCovToken "${{ env.codecov_token }}" -PathToCodeCoverage "${{ runner.temp }}/_artifacts/coverageResults" -Name "${{ runner.os }} Coverage Results" -Flags "${{ runner.os }}" - shell: pwsh - timeout-minutes: 3 - continue-on-error: true - if: env.codecov_token != '' diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index eb69d92e..3c032fae 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -26,7 +26,7 @@ jobs: # You can define any steps you want, and they will run before the agent starts. # If you do not check out your code, Copilot will do this for you. steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # avoid shallow clone so nbgv can do its work. - name: โš™ Install prerequisites diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a7155c4c..fbfcd144 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,7 +24,7 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # avoid shallow clone so nbgv can do its work. - name: โš™ Install prerequisites @@ -34,10 +34,10 @@ jobs: name: ๐Ÿ“š Generate documentation - name: Upload artifact - uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: docfx/_site - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 diff --git a/.github/workflows/docs_validate.yml b/.github/workflows/docs_validate.yml index 336ac16e..9c6b1be0 100644 --- a/.github/workflows/docs_validate.yml +++ b/.github/workflows/docs_validate.yml @@ -12,11 +12,11 @@ jobs: name: ๐Ÿ“š Doc validation runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # avoid shallow clone so nbgv can do its work. - name: ๐Ÿ”— Markup Link Checker (mlc) - uses: becheran/mlc@18a06b3aa2901ca197de59c8b0b1f54fdba6b3fa # v1.0.0 + uses: becheran/mlc@7ec24825cefe0c9c8c6bac48430e1f69e3ec356e # v1.2.0 with: args: --do-not-warn-for-redirect-to https://learn.microsoft.com*,https://dotnet.microsoft.com/*,https://dev.azure.com/*,https://app.codecov.io/* -p docfx -i https://aka.ms/onboardsupport,https://aka.ms/spot,https://msrc.microsoft.com/*,https://www.microsoft.com/msrc*,https://microsoft.com/msrc*,https://www.npmjs.com/package/*,https://get.dot.net/ - name: โš™ Install prerequisites @@ -26,4 +26,3 @@ jobs: shell: pwsh - name: ๐Ÿ“š Verify docfx build run: dotnet docfx docfx/docfx.json --warningsAsErrors --disableGitFeatures - if: runner.os == 'Linux' diff --git a/.github/workflows/libtemplate-update.yml b/.github/workflows/libtemplate-update.yml index f5cf8666..62a6417d 100644 --- a/.github/workflows/libtemplate-update.yml +++ b/.github/workflows/libtemplate-update.yml @@ -17,7 +17,7 @@ jobs: contents: write pull-requests: write steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # avoid shallow clone so nbgv can do its work. diff --git a/.gitignore b/.gitignore index 1b779930..83b1d55e 100644 --- a/.gitignore +++ b/.gitignore @@ -358,3 +358,6 @@ MigrationBackup/ # Analysis results *.sarif + +# C# Dev Kit cache files +*.lscache diff --git a/Directory.Build.props b/Directory.Build.props index 08f6c7b4..04ffb164 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -15,6 +15,7 @@ true true true + true true @@ -43,7 +44,7 @@ - 13 + 14 16.9 diff --git a/Directory.Packages.props b/Directory.Packages.props index 5eb1dead..54736269 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,11 +4,12 @@ true true - 2.0.201 + 2.2.2 + + 2.0.226 - @@ -16,10 +17,12 @@ - + + + + - - + @@ -29,7 +32,7 @@ - + diff --git a/azure-pipelines/BuildStageVariables.yml b/azure-pipelines/BuildStageVariables.yml index 2a683569..7c61f8fe 100644 --- a/azure-pipelines/BuildStageVariables.yml +++ b/azure-pipelines/BuildStageVariables.yml @@ -2,4 +2,3 @@ variables: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ - # codecov_token: 4dc9e7e2-6b01-4932-a180-847b52b43d35 # Get a new one from https://codecov.io/ diff --git a/azure-pipelines/Merge-CodeCoverage.ps1 b/azure-pipelines/Merge-CodeCoverage.ps1 index 32438237..c552d25f 100644 --- a/azure-pipelines/Merge-CodeCoverage.ps1 +++ b/azure-pipelines/Merge-CodeCoverage.ps1 @@ -28,6 +28,7 @@ try { if ($reports) { $reports |% { $_.FullName } |% { # In addition to replacing {reporoot}, we also normalize on one kind of slash so that the report aggregates data for a file whether data was collected on Windows or not. + Write-Verbose "Processing $_" $xml = [xml](Get-Content -LiteralPath $_) $xml.coverage.packages.package.classes.class |? { $_.filename} |% { $_.filename = $_.filename.Replace('{reporoot}', $RepoRoot).Replace([IO.Path]::AltDirectorySeparatorChar, [IO.Path]::DirectorySeparatorChar) diff --git a/azure-pipelines/OptProf.yml b/azure-pipelines/OptProf.yml deleted file mode 100644 index a70b9362..00000000 --- a/azure-pipelines/OptProf.yml +++ /dev/null @@ -1,121 +0,0 @@ -trigger: none -pr: none -schedules: -- cron: "0 3 * * Fri" # Thu @ 8 or 9 PM Mountain Time (depending on DST) - displayName: Weekly OptProf run - branches: - include: - - 'v*.*' - - main - always: true # we must keep data fresh since optimizationdata drops are purged after 30 days - -# Avoid errant CI builds: https://developercommunity.visualstudio.com/content/problem/1154409/azure-pipeline-is-triggering-due-to-events-that-ne.html -#resources: -# repositories: -# - repository: scripts -# type: git -# name: DeploymentScripts -# ref: refs/heads/test - -parameters: - - name: ShouldSkipOptimize - displayName: Skip OptProf optimization - type: boolean - default: false # Should usually be false so that optprof LKG can apply when tests fail, but may need to be set to true in a manually queued pipeline run if all drops have expired. - -variables: -- template: GlobalVariables.yml -- name: PublicRelease - value: false # avoid using nice version since we're building a preliminary/unoptimized package -- name: IsOptProf - value: true -- name: MicroBuild_NuPkgSigningEnabled - value: false # test-signed nuget packages fail to restore in the VS insertion PR validations. Just don't sign them *at all*. - -stages: -- stage: Library - variables: - - name: OptProf - value: true - - template: BuildStageVariables.yml - jobs: - - template: build.yml - parameters: - Is1ESPT: false - RealSign: false - windowsPool: VSEngSS-MicroBuild2022-1ES - EnableMacOSBuild: false - ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} - IsOptProf: true - RunTests: false - SkipCodesignVerify: true -- stage: QueueVSBuild - jobs: - - job: QueueOptProf - pool: VSEngSS-MicroBuild2022-1ES - variables: - InsertPayloadName: LibraryName - InsertTopicBranch: team/VS-IDE/LibraryName-OptProf-run-$(Build.BuildId) - steps: - - checkout: none # We don't need source from our own repo - clean: true - - # Pipeline YAML does not yet support checking out other repos. So we'll do it by hand. -# - checkout: scripts # We DO need source from the DeploymentScripts repo -# clean: true -# path: $(Agent.TempDirectory)/DeploymentScripts -# fetchDepth: 1 - - script: 'git -c http.extraheader="AUTHORIZATION: bearer $(System.AccessToken)" clone https://devdiv.visualstudio.com/DevDiv/_git/DeploymentScripts --depth 1 --branch test "$(Agent.TempDirectory)/DeploymentScripts"' - displayName: Download DeploymentScripts repo - - - task: DownloadBuildArtifacts@0 - displayName: Download insertion artifacts - inputs: - artifactName: VSInsertion-Windows - downloadPath: $(Agent.TempDirectory) - - task: DownloadBuildArtifacts@0 - displayName: Download variables artifacts - inputs: - artifactName: Variables-Windows - downloadPath: $(Agent.TempDirectory) - - task: PowerShell@2 - displayName: Set pipeline variables based on artifacts - inputs: - targetType: filePath - filePath: $(Agent.TempDirectory)/Variables-Windows/_define.ps1 - - task: NuGetCommand@2 - displayName: Push VS-repo packages to VS feed - inputs: - command: push - packagesToPush: $(Agent.TempDirectory)/VSInsertion-Windows/*.nupkg - publishVstsFeed: 97a41293-2972-4f48-8c0e-05493ae82010 # VS feed - allowPackageConflicts: true - - task: MicroBuildInsertVsPayload@5 - displayName: Insert VS Payload - inputs: - TeamName: $(TeamName) - TeamEmail: $(TeamEmail) - SkipCreatePR: true - CustomScriptExecutionCommand: src\VSSDK\NuGet\AllowUnstablePackages.ps1 - - task: benjhuser.tfs-extensions-build-tasks.trigger-build-task.TriggerBuild@3 - displayName: Trigger a new build of DD-CB-TestSignVS-devCI - inputs: - buildDefinition: DD-CB-TestSignVS-devCI - useSameBranch: false - branchToUse: $(InsertTopicBranch) - storeInEnvironmentVariable: true - queueBuildForUserThatTriggeredBuild: false - authenticationMethod: OAuth Token - password: $(System.AccessToken) - - task: PowerShell@2 - displayName: Associate InsertionOutputs artifacts with CloudBuild - inputs: - targetType: filePath - filePath: $(Agent.TempDirectory)/DeploymentScripts/Scripts/Insertion/WriteArtifact.ps1 - arguments: '-oldBuildID $(Build.BuildId) -newBuildID $(TriggeredBuildIds) -artifactName "InsertionOutputs" -accessToken $(System.AccessToken)' - - task: PowerShell@2 - displayName: Tag the build with LibraryName-insertion - inputs: - targetType: filePath - filePath: $(Agent.TempDirectory)/DeploymentScripts/Scripts/Insertion/TagBuild.ps1 - arguments: '-buildID $(TriggeredBuildIds) -tagName "LibraryName-insertion" -accessToken $(System.AccessToken)' diff --git a/azure-pipelines/OptProf_part2.yml b/azure-pipelines/OptProf_part2.yml deleted file mode 100644 index c59d6999..00000000 --- a/azure-pipelines/OptProf_part2.yml +++ /dev/null @@ -1,91 +0,0 @@ -trigger: none -pr: none - -resources: - pipelines: - - pipeline: VisualStudioBuildUnderTest - source: DD-CB-TestSignVS-devCI - trigger: - tags: - - LibraryName-insertion - - pipeline: DartLab - source: DartLab - branch: main - - pipeline: DartLab.OptProf - source: DartLab.OptProf - branch: main - tags: - - production - repositories: - - repository: DartLabTemplates - type: git - name: DartLab.Templates - ref: refs/heads/main - - repository: DartLabOptProfTemplates - type: git - name: DartLab.OptProf - ref: refs/tags/Production - -parameters: - -# The prefix naming of the OptimizationInputs drop -- name: optimizationDropPrefix - type: string - default: OptimizationInputs/$(System.TeamProject)/$(Build.Repository.Name) - -stages: -- template: \templates\stages\visual-studio\single-runsettings.yml@DartLabOptProfTemplates - parameters: - ##### Required ##### - runSettingsURI: $(Pipeline.Workspace)\VisualStudioBuildUnderTest\BuildArtifacts\runsettings\LibraryName.OptProf.runsettings - visualStudioBootstrapperURI: https://vsdrop.corp.microsoft.com/file/v1/$(VisualStudio.BuildUnderTest.ProductsDropName);bootstrappers/Enterprise/vs_enterprise.exe - ##### Optional ##### - name: OptProfProfilingWorkflow - displayName: OptProf Profiling Workflow - optOptimizationInputsDropName: $(OptimizationInputsDropName) - previousOptimizationInputsDropName: $(PreviousOptimizationInputsDropName) - testLabPoolName: VS-Platform - ##### Step Hooks ##### - preTestMachineConfigurationStepList: - - download: VisualStudioBuildUnderTest - - task: PowerShell@2 - name: SetProductsDropName - displayName: Set 'VisualStudio.BuildUnderTest.ProductsDropName' - inputs: - filePath: $(DartLab.Path)\Scripts\VisualStudio\Build\Get-VisualStudioDropName.ps1 - arguments: -DropNamePrefix 'Products' -VstsDropUrlsJson '$(Pipeline.Workspace)\VisualStudioBuildUnderTest\BuildArtifacts\VstsDropUrls.json' -OutVariableName 'VisualStudio.BuildUnderTest.ProductsDropName' - preDeployAndRunTestsStepList: - - download: VisualStudioBuildUnderTest - prePublishOptimizationInputsDropStepList: - # Set parameter for PreviousOptimizationInputsDropName, MicroBuildCommitID, and OptimizationInputsDropName - - powershell: | - try { - $artifactName = 'InsertionOutputs' - $BuildID = $(resources.pipeline.VisualStudioBuildUnderTest.runID) - $artifact = Get-BuildArtifact -InstanceURL 'https://dev.azure.com/devdiv' -ProjectName 'DevDiv' -BuildID $BuildID -ArtifactName $artifactName -OAuthAccessToken (ConvertTo-SecureString '$(System.AccessToken)' -AsPlainText -Force) - $containerName = $artifact.Resource.Data -Split '/' | Select-Object -Last 1 - $fileName = Join-Path $containerName 'Metadata.json' - $jsonString = Read-BuildArtifactFile -InstanceURL 'https://dev.azure.com/devdiv' -ProjectName 'DevDiv' -BuildID $BuildID -ArtifactName $artifactName -FileName $fileName -OAuthAccessToken (ConvertTo-SecureString '$(System.AccessToken)' -AsPlainText -Force) - $json = $jsonString | ConvertFrom-Json - - Write-Host "The content of the metadata.json file was $json" - - $dropname = $json.OptimizationData - $commitID = $json.CommitID - $OptimizationInputsDropName = "${{parameters.optimizationDropPrefix}}/$($commitID)/$(Build.BuildId)/$(System.StageId)/$(System.StageAttempt)" - - Write-Host "PreviousOptimizationInputsDropName: $dropname" - Set-AzurePipelinesVariable 'PreviousOptimizationInputsDropName' $dropname - - Write-Host "MicroBuildCommitID: $commitID" - Set-AzurePipelinesVariable 'MicroBuildCommitID' $commitID - - Write-Host "OptimizationInputsDropName: $OptimizationInputsDropName" - Set-AzurePipelinesVariable 'OptimizationInputsDropName' $OptimizationInputsDropName - } - catch { - Write-Host $_ - Write-Error "Failed to set OptimizationInputsDropName pipeline variable" - throw - } - displayName: Set MicroBuildCommitID, PreviousOptimizationInputsDropName, and OptimizationInputsDropName diff --git a/azure-pipelines/archive-sourcecode.yml b/azure-pipelines/archive-sourcecode.yml index 77f243bc..211881fa 100644 --- a/azure-pipelines/archive-sourcecode.yml +++ b/azure-pipelines/archive-sourcecode.yml @@ -36,7 +36,7 @@ extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: VSEng-MicroBuildVSStable stages: - stage: archive diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 7869d260..6ad583b8 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -29,14 +29,17 @@ parameters: - name: artifact_names type: object default: - - build_logs - - coverageResults - - deployables - - projectAssetsJson - - symbols - - testResults - - test_symbols - - Variables + - name: build_logs + - name: coverageResults + - name: deployables + sbomEnabled: true + - name: projectAssetsJson + - name: symbols + - name: testResults + testOnly: true + - name: test_symbols + testOnly: true + - name: Variables # The Enable*Build parameters turn non-Windows agents on or off. # Their default value should be based on whether the build and tests are expected/required to pass on that platform. # Callers (e.g. Official.yml) *may* expose these parameters at queue-time in order to turn OFF optional agents. @@ -57,13 +60,12 @@ parameters: - name: Is1ESPT type: boolean -- name: RealSign +# Indicates whether the 'official' 1ES PT templates are being used (as opposed to the unofficial ones). +- name: Is1ESPTOfficial type: boolean default: false -# Whether this particular run is an OptProf profiling run. -# This is used to skip unit tests and other non-essential work to improve reliability of the OptProf pipeline. -- name: IsOptProf +- name: RealSign type: boolean default: false @@ -96,15 +98,15 @@ parameters: - name: windowsPool type: object default: - vmImage: windows-2022 + vmImage: windows-2025 - name: linuxPool type: object default: - vmImage: ubuntu-22.04 + vmImage: ubuntu-24.04 - name: macOSPool type: object default: - vmImage: macOS-14 + vmImage: macOS-15 jobs: - job: Windows @@ -131,8 +133,11 @@ jobs: optprof: enabled: ${{ parameters.EnableOptProf }} ProfilingInputsDropName: $(ProfilingInputsDropName) - OptimizationInputsLookupMethod: DropPrefix - DropNamePrefix: OptimizationInputs/$(System.TeamProject)/$(Build.Repository.Name) + GeneratePropsFile: true + PropsPath: $(Build.ArtifactStagingDirectory)/InsertionOutputs/$(ProfilingInputsPropsName) + OptimizationInputsLookupMethod: GitTagRepo + GitTagProject: DevDiv + GitTagRepo: VS ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} AccessToken: $(System.AccessToken) mbpresteps: @@ -140,21 +145,31 @@ jobs: fetchDepth: 0 # avoid shallow clone so nbgv can do its work. clean: true - ${{ if parameters.EnableOptProf }}: - - powershell: Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSDROPNAME]$(tools/variables/ProfilingInputsDropName.ps1)" - displayName: โš™ Set ProfilingInputsDropName for optprof + - powershell: | + Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSDROPNAME]$(tools/variables/ProfilingInputsDropName.ps1)" + Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSPROPSNAME]$(tools/variables/ProfilingInputsPropsName.ps1)" + displayName: โš™ Setting variables for optprof sdl: binskim: analyzeTargetGlob: $(Build.ArtifactStagingDirectory)\symbols-Windows\** outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - - ${{ each artifact_name in parameters.artifact_names }}: - - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: + - ${{ each artifact in parameters.artifact_names }}: + - ${{ if or(ne(artifact.testOnly, 'true'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact.name }}-Windows + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact.name }}-Windows + artifactName: ${{ artifact.name }}-Windows + ${{ if and(parameters.Is1ESPTOfficial, eq(artifact.sbomEnabled, 'true')) }}: + sbomEnabled: true - output: pipelineArtifact - displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Windows - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Windows - artifactName: ${{ artifact_name }}-Windows - condition: succeededOrFailed() + displayName: ๐Ÿ“ข Publish ${{ artifact.name }}-Windows (for failed attempts) + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact.name }}-Windows + artifactName: ${{ artifact.name }}-Windows-$(System.PhaseAttempt) + ${{ if and(parameters.Is1ESPTOfficial, eq(artifact.sbomEnabled, 'true')) }}: + sbomEnabled: true + condition: failed() - output: pipelineArtifact displayName: ๐Ÿ“ข Publish VSInsertion-Windows targetPath: $(Build.ArtifactStagingDirectory)/VSInsertion-Windows @@ -169,6 +184,15 @@ jobs: displayName: ๐Ÿ“ข Publish APIScanInputs targetPath: $(Build.ArtifactStagingDirectory)/APIScanInputs-Windows artifactName: APIScanInputs + - ${{ if parameters.EnableOptProf }}: + - output: artifactsDrop + displayName: ๐Ÿ“ข Publish to Artifact Services - ProfilingInputs + dropServiceURI: https://devdiv.artifacts.visualstudio.com + buildNumber: $(ProfilingInputsDropName) + sourcePath: $(Build.ArtifactStagingDirectory)\OptProf\ProfilingInputs + toLowerCase: false + retentionDays: 500 + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) steps: - ${{ if not(parameters.Is1ESPT) }}: - checkout: self @@ -178,9 +202,6 @@ jobs: - powershell: Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSDROPNAME]$(tools/variables/ProfilingInputsDropName.ps1)" displayName: โš™ Set ProfilingInputsDropName for optprof - - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: - - template: schedule-only-steps.yml - - template: install-dependencies.yml - script: dotnet nbgv cloud -ca @@ -191,8 +212,6 @@ jobs: - template: microbuild.before.yml parameters: EnableLocalization: ${{ parameters.EnableLocalization }} - EnableOptProf: ${{ parameters.EnableOptProf }} - IsOptProf: ${{ parameters.IsOptProf }} ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} RealSign: ${{ parameters.RealSign }} @@ -200,7 +219,6 @@ jobs: parameters: Is1ESPT: ${{ parameters.Is1ESPT }} RunTests: ${{ parameters.RunTests }} - IsOptProf: ${{ parameters.IsOptProf }} - ${{ if and(parameters.EnableDotNetFormatCheck, not(parameters.EnableLinuxBuild)) }}: - script: dotnet format --verify-no-changes --no-restore src @@ -211,123 +229,138 @@ jobs: - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - template: microbuild.after.yml parameters: - EnableOptProf: ${{ parameters.EnableOptProf }} - IsOptProf: ${{ parameters.IsOptProf }} SkipCodesignVerify: ${{ parameters.SkipCodesignVerify }} -- ${{ if not(parameters.IsOptProf) }}: - - ${{ if parameters.EnableLinuxBuild }}: - - job: Linux - pool: ${{ parameters.linuxPool }} - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - templateContext: - mb: - ${{ if parameters.RealSign }}: - signing: - enabled: false # enable when building unique artifacts on this agent that must be signed - signType: real - signWithProd: true - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - ${{ each artifact_name in parameters.artifact_names }}: - - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: - - output: pipelineArtifact - displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Linux - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Linux - artifactName: ${{ artifact_name }}-Linux - condition: succeededOrFailed() - steps: - - checkout: self - fetchDepth: 0 # avoid shallow clone so nbgv can do its work. - clean: true - - template: install-dependencies.yml - - template: dotnet.yml - parameters: - Is1ESPT: ${{ parameters.Is1ESPT }} - RunTests: ${{ parameters.RunTests }} - BuildRequiresAccessToken: ${{ parameters.RealSign }} # Real signing on non-Windows machines requires passing through access token to build steps that sign - - ${{ if parameters.EnableDotNetFormatCheck }}: - - script: dotnet format --verify-no-changes --no-restore src - displayName: ๐Ÿ’… Verify formatted code - env: - dotnetformat: true # part of a workaround for https://github.com/dotnet/sdk/issues/44951 - - - ${{ if parameters.EnableMacOSBuild }}: - - job: macOS - pool: ${{ parameters.macOSPool }} - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - templateContext: - mb: - ${{ if parameters.RealSign }}: - signing: - enabled: false # enable when building unique artifacts on this agent that must be signed - signType: real - signWithProd: true - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - ${{ each artifact_name in parameters.artifact_names }}: - - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: - - output: pipelineArtifact - displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-macOS - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-macOS - artifactName: ${{ artifact_name }}-macOS - condition: succeededOrFailed() - steps: - - checkout: self - fetchDepth: 0 # avoid shallow clone so nbgv can do its work. - clean: true - - template: install-dependencies.yml - - template: dotnet.yml - parameters: - Is1ESPT: ${{ parameters.Is1ESPT }} - RunTests: ${{ parameters.RunTests }} - BuildRequiresAccessToken: ${{ parameters.RealSign }} # Real signing on non-Windows machines requires passing through access token to build steps that sign +- ${{ if parameters.EnableLinuxBuild }}: + - job: Linux + pool: ${{ parameters.linuxPool }} + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + templateContext: + mb: + ${{ if parameters.RealSign }}: + signing: + enabled: false # enable when building unique artifacts on this agent that must be signed + signType: real + signWithProd: true + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ each artifact in parameters.artifact_names }}: + - ${{ if or(ne(artifact.testOnly, 'true'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact.name }}-Linux + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact.name }}-Linux + artifactName: ${{ artifact.name }}-Linux + ${{ if and(parameters.Is1ESPTOfficial, eq(artifact.sbomEnabled, 'true')) }}: + sbomEnabled: true + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact.name }}-Linux (for failed attempts) + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact.name }}-Linux + artifactName: ${{ artifact.name }}-Linux-$(System.PhaseAttempt) + ${{ if and(parameters.Is1ESPTOfficial, eq(artifact.sbomEnabled, 'true')) }}: + sbomEnabled: true + condition: failed() + steps: + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true + - template: install-dependencies.yml + - template: dotnet.yml + parameters: + Is1ESPT: ${{ parameters.Is1ESPT }} + RunTests: ${{ parameters.RunTests }} + BuildRequiresAccessToken: ${{ parameters.RealSign }} # Real signing on non-Windows machines requires passing through access token to build steps that sign + - ${{ if parameters.EnableDotNetFormatCheck }}: + - script: dotnet format --verify-no-changes + displayName: ๐Ÿ’… Verify formatted code + env: + dotnetformat: true # part of a workaround for https://github.com/dotnet/sdk/issues/44951 + - template: expand-template.yml - - job: WrapUp - dependsOn: - - Windows - - ${{ if parameters.EnableLinuxBuild }}: - - Linux - - ${{ if parameters.EnableMacOSBuild }}: - - macOS - pool: ${{ parameters.windowsPool }} # Use Windows agent because PublishSymbols task requires it (https://github.com/microsoft/azure-pipelines-tasks/issues/13821). - condition: succeededOrFailed() - variables: - ONEES_ENFORCED_CODEQL_ENABLED: false # CodeQL runs on build jobs, we don't need it here +- ${{ if parameters.EnableMacOSBuild }}: + - job: macOS + pool: ${{ parameters.macOSPool }} ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: templateContext: - ${{ if not(parameters.RealSign) }}: - mb: - signing: # if the build is test-signed, install the signing plugin so that CSVTestSignPolicy.xml is available - enabled: true - zipSources: false - signType: test + mb: + ${{ if parameters.RealSign }}: + signing: + enabled: false # enable when building unique artifacts on this agent that must be signed + signType: real + signWithProd: true outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - - output: pipelineArtifact - displayName: ๐Ÿ“ข Publish symbols-legacy - targetPath: $(Build.ArtifactStagingDirectory)/symbols-legacy - artifactName: symbols-legacy - condition: succeededOrFailed() + - ${{ each artifact in parameters.artifact_names }}: + - ${{ if or(ne(artifact.testOnly, 'true'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact.name }}-macOS + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact.name }}-macOS + artifactName: ${{ artifact.name }}-macOS + ${{ if and(parameters.Is1ESPTOfficial, eq(artifact.sbomEnabled, 'true')) }}: + sbomEnabled: true + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact.name }}-macOS (for failed attempts) + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact.name }}-macOS + artifactName: ${{ artifact.name }}-macOS-$(System.PhaseAttempt) + ${{ if and(parameters.Is1ESPTOfficial, eq(artifact.sbomEnabled, 'true')) }}: + sbomEnabled: true + condition: failed() steps: - checkout: self fetchDepth: 0 # avoid shallow clone so nbgv can do its work. clean: true - template: install-dependencies.yml + - template: dotnet.yml parameters: - initArgs: -NoRestore - - template: publish-symbols.yml + Is1ESPT: ${{ parameters.Is1ESPT }} + RunTests: ${{ parameters.RunTests }} + BuildRequiresAccessToken: ${{ parameters.RealSign }} # Real signing on non-Windows machines requires passing through access token to build steps that sign + - template: expand-template.yml + +- job: WrapUp + dependsOn: + - Windows + - ${{ if parameters.EnableLinuxBuild }}: + - Linux + - ${{ if parameters.EnableMacOSBuild }}: + - macOS + pool: ${{ parameters.windowsPool }} # Use Windows agent because PublishSymbols task requires it (https://github.com/microsoft/azure-pipelines-tasks/issues/13821). + condition: succeededOrFailed() + variables: + ONEES_ENFORCED_CODEQL_ENABLED: false # CodeQL runs on build jobs, we don't need it here + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + templateContext: + ${{ if not(parameters.RealSign) }}: + mb: + signing: # if the build is test-signed, install the signing plugin so that CSVTestSignPolicy.xml is available + enabled: true + zipSources: false + signType: test + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish symbols-legacy + targetPath: $(Build.ArtifactStagingDirectory)/symbols-legacy + artifactName: symbols-legacy + condition: succeededOrFailed() + steps: + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true + - template: install-dependencies.yml + parameters: + initArgs: -NoRestore + - template: publish-symbols.yml + parameters: + EnableLinuxBuild: ${{ parameters.EnableLinuxBuild }} + EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} + - ${{ if and(parameters.RunTests, parameters.PublishCodeCoverage) }}: + - template: publish-codecoverage.yml parameters: EnableLinuxBuild: ${{ parameters.EnableLinuxBuild }} EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} - - ${{ if and(parameters.RunTests, parameters.PublishCodeCoverage) }}: - - template: publish-codecoverage.yml - parameters: - EnableLinuxBuild: ${{ parameters.EnableLinuxBuild }} - EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} - - ${{ if parameters.EnableAPIScan }}: - - template: apiscan.yml - parameters: - windowsPool: ${{ parameters.windowsPool }} - RealSign: ${{ parameters.RealSign }} +- ${{ if parameters.EnableAPIScan }}: + - template: apiscan.yml + parameters: + windowsPool: ${{ parameters.windowsPool }} + RealSign: ${{ parameters.RealSign }} diff --git a/azure-pipelines/dotnet.yml b/azure-pipelines/dotnet.yml index 2ca56b4d..eb13352d 100644 --- a/azure-pipelines/dotnet.yml +++ b/azure-pipelines/dotnet.yml @@ -1,8 +1,5 @@ parameters: - name: RunTests -- name: IsOptProf - type: boolean - default: false - name: Is1ESPT type: boolean - name: BuildRequiresAccessToken @@ -17,14 +14,9 @@ steps: env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) -- ${{ if not(parameters.IsOptProf) }}: - - powershell: tools/dotnet-test-cloud.ps1 -Configuration $(BuildConfiguration) -Agent $(Agent.JobName) -PublishResults - displayName: ๐Ÿงช dotnet test - condition: and(succeeded(), ${{ parameters.RunTests }}) - -- ${{ if parameters.IsOptProf }}: - - script: dotnet pack src\VSInsertionMetadata -c $(BuildConfiguration) -warnaserror /bl:"$(Build.ArtifactStagingDirectory)/build_logs/VSInsertion-Pack.binlog" - displayName: ๐Ÿ”ง dotnet pack VSInsertionMetadata +- powershell: tools/dotnet-test-cloud.ps1 -Configuration $(BuildConfiguration) -Agent $(Agent.JobName) -PublishResults + displayName: ๐Ÿงช dotnet test + condition: and(succeeded(), ${{ parameters.RunTests }}) - powershell: tools/variables/_define.ps1 failOnStderr: true @@ -42,11 +34,14 @@ steps: displayName: ๐Ÿ“ข Publish artifacts condition: succeededOrFailed() -- ${{ if and(ne(variables['codecov_token'], ''), parameters.RunTests) }}: +- ${{ if parameters.RunTests }}: - powershell: | $ArtifactStagingFolder = & "tools/Get-ArtifactsStagingDirectory.ps1" $CoverageResultsFolder = Join-Path $ArtifactStagingFolder "coverageResults-$(Agent.JobName)" - tools/publish-CodeCov.ps1 -CodeCovToken "$(codecov_token)" -PathToCodeCoverage "$CoverageResultsFolder" -Name "$(Agent.JobName) Coverage Results" -Flags "$(Agent.JobName)" + tools/publish-CodeCov.ps1 -CodeCovToken "$(CODECOV_TOKEN)" -PathToCodeCoverage "$CoverageResultsFolder" -Name "$(Agent.JobName) Coverage Results" -Flags "$(Agent.JobName)" displayName: ๐Ÿ“ข Publish code coverage results to codecov.io timeoutInMinutes: 3 continueOnError: true + # Set the CODECOV_TOKEN variable in your Azure Pipeline to enable code coverage reporting + # Get a token from https://codecov.io/ + condition: and(succeeded(), ne(variables['CODECOV_TOKEN'], '')) diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 025de4f5..c0e1e7bd 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -1,10 +1,4 @@ parameters: -- name: EnableOptProf - type: boolean - default: false -- name: IsOptProf - type: boolean - default: false - name: SkipCodesignVerify type: boolean @@ -18,21 +12,3 @@ steps: TargetFolders: | $(Build.SourcesDirectory)/bin/Packages/$(BuildConfiguration) condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -- ${{ if parameters.IsOptProf }}: - - task: ms-vscs-artifact.build-tasks.artifactDropTask-1.artifactDropTask@0 - inputs: - dropServiceURI: https://devdiv.artifacts.visualstudio.com - buildNumber: $(ProfilingInputsDropName) - sourcePath: $(Build.ArtifactStagingDirectory)\OptProf\ProfilingInputs - toLowerCase: false - usePat: true - displayName: ๐Ÿ“ข Publish to Artifact Services - ProfilingInputs - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - - - task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: $(Build.ArtifactStagingDirectory)/InsertionOutputs - ArtifactName: InsertionOutputs - ArtifactType: Container - displayName: ๐Ÿ“ข Publish InsertionOutputs as Azure DevOps artifacts diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index d09310b1..29823532 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -2,12 +2,6 @@ parameters: - name: EnableLocalization type: boolean default: false -- name: EnableOptProf - type: boolean - default: false -- name: IsOptProf - type: boolean - default: false - name: ShouldSkipOptimize type: boolean default: false @@ -15,7 +9,7 @@ parameters: type: boolean steps: -- ${{ if and(not(parameters.IsOptProf), ne(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: # notice@0 requires CG detection to run first, and non-default branches don't inject it automatically. # default branch injection (main) is happening too late for notice@0 to run successfully. Adding this as a workaround. - task: ComponentGovernanceComponentDetection@0 @@ -26,28 +20,5 @@ steps: inputs: outputfile: $(System.DefaultWorkingDirectory)/obj/NOTICE outputformat: text - retryCountOnTaskFailure: 3 # fails when the cloud service is overloaded + retryCountOnTaskFailure: 10 # fails when the cloud service is overloaded continueOnError: ${{ not(parameters.RealSign) }} # Tolerate failures when we're not building something that may ship. - -- ${{ if parameters.IsOptProf }}: - # We have to install these plugins ourselves for Optprof runs because those pipelines haven't migrated to 1ES PT yet. - - task: MicroBuildOptProfPlugin@6 - inputs: - ProfilingInputsDropName: $(ProfilingInputsDropName) - OptimizationInputsLookupMethod: DropPrefix - DropNamePrefix: OptimizationInputs/$(System.TeamProject)/$(Build.Repository.Name) - ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} - AccessToken: $(System.AccessToken) - displayName: ๐Ÿ”ง Install OptProf Plugin - - - task: MicroBuildSigningPlugin@4 - inputs: - signType: Test - zipSources: false - displayName: ๐Ÿ”ง Install MicroBuild Signing Plugin - - - ${{ if parameters.EnableLocalization }}: - - task: MicroBuildLocalizationPlugin@4 - inputs: - languages: $(LocLanguages) - displayName: ๐Ÿ”ง Install MicroBuild Localization Plugin diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 939da7fc..cad57075 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -3,12 +3,17 @@ trigger: include: - main pr: none -#schedules: -#- cron: "0 3 * * *" # Daily @ 8 PM PST -# displayName: Daily vs-insertion -# branches: -# include: -# - microbuild + +schedules: +# Monthly compliance sweep: runs at 03:00 UTC (8 PM PST previous day) on the 1st of every month. +# `always: true` ensures it runs even if there have been no code changes since the last successful build, +# so we can detect regressions in compliance tooling (APIScan, PoliCheck, CodeSignValidation, etc.). +- cron: "0 3 1 * *" + displayName: Monthly compliance sweep + branches: + include: + - main + always: true parameters: # As an entrypoint pipeline yml file, all parameters here show up in the Queue Run dialog. @@ -49,11 +54,11 @@ extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: VSEng-MicroBuildVSStable codeSignValidation: enabled: true break: true - additionalTargetsGlobPattern: -|Variables-*\*.ps1;-|LocBin-*\**;-|APIScanInputs-*\**;-|test_symbols-*\**;-|MicroBuild\** + additionalTargetsGlobPattern: -|Variables-*\*.ps1;-|LocBin-*\**;-|APIScanInputs-*\**;-|test_symbols-*\**;-|MicroBuild\**;-|$(Build.ArtifactStagingDirectory)\VSInsertion-Windows\vs-insertion-script.ps1 policheck: enabled: true exclusionsFile: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml @@ -69,10 +74,11 @@ extends: - template: /azure-pipelines/build.yml@self parameters: Is1ESPT: true + Is1ESPTOfficial: true RealSign: true # ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} EnableAPIScan: ${{ parameters.EnableAPIScan }} - windowsPool: VSEngSS-MicroBuild2022-1ES + windowsPool: VSEng-MicroBuildVSStable linuxPool: name: AzurePipelines-EO demands: @@ -80,7 +86,7 @@ extends: os: Linux macOSPool: name: Azure Pipelines - vmImage: macOS-14 + vmImage: macOS-15 os: macOS EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} RunTests: ${{ parameters.RunTests }} diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index fdce906a..9f934f92 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -17,7 +17,7 @@ stages: - ${{ if parameters.ArchiveSymbols }}: - job: symbol_archive displayName: Archive symbols - pool: VSEngSS-MicroBuild2022-1ES + pool: VSEng-MicroBuildVSStable variables: ONEES_ENFORCED_CODEQL_ENABLED: false # CodeQL runs on build stages, we don't need it here steps: diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index 673934c8..bb0220e6 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -21,12 +21,13 @@ extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: VSEng-MicroBuildVSStable stages: - stage: release jobs: - - job: release + - job: nuget + displayName: ๐Ÿ“ฆ Push nuget.org packages pool: name: AzurePipelines-EO demands: @@ -43,6 +44,28 @@ extends: publishFeedCredentials: VisualStudioExtensibility (nuget.org) steps: - checkout: none + - download: CI + artifact: deployables-Windows + displayName: ๐Ÿ”ป Download deployables-Windows artifact + patterns: 'NuGet/*' + - job: github + displayName: ๐Ÿ“ข GitHub release + dependsOn: nuget + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals 1ESPT-Ubuntu22.04 + os: Linux + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + pipeline: CI + artifactName: deployables-Windows + targetPath: $(Pipeline.Workspace)/CI/deployables-Windows + steps: + - checkout: none - powershell: | Write-Host "##vso[build.updatebuildnumber]$(resources.pipeline.CI.runName)" if ('$(resources.pipeline.CI.runName)'.Contains('-')) { @@ -51,10 +74,6 @@ extends: Write-Host "##vso[task.setvariable variable=IsPrerelease]false" } displayName: โš™ Set up pipeline - - download: CI - artifact: deployables-Windows - displayName: ๐Ÿ”ป Download deployables-Windows artifact - patterns: 'NuGet/*' - task: GitHubRelease@1 displayName: ๐Ÿ“ข GitHub release (create) inputs: diff --git a/azure-pipelines/schedule-only-steps.yml b/azure-pipelines/schedule-only-steps.yml deleted file mode 100644 index ad07a341..00000000 --- a/azure-pipelines/schedule-only-steps.yml +++ /dev/null @@ -1,3 +0,0 @@ -steps: -- powershell: echo "##vso[build.addbuildtag]auto-insertion" - displayName: Tag for auto-insertion diff --git a/azure-pipelines/unofficial.yml b/azure-pipelines/unofficial.yml index d7232735..36aaf6e5 100644 --- a/azure-pipelines/unofficial.yml +++ b/azure-pipelines/unofficial.yml @@ -56,7 +56,7 @@ extends: template: azure-pipelines/MicroBuild.1ES.Unofficial.yml@MicroBuildTemplate parameters: sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: VSEng-MicroBuildVSStable credscan: enabled: false suppression: @@ -65,7 +65,7 @@ extends: codeSignValidation: enabled: ${{ parameters.EnableProductionSDL }} break: true - additionalTargetsGlobPattern: -|Variables-*\*.ps1;-|APIScanInputs-*\**;-|test_symbols-*\**;-|MicroBuild\** + additionalTargetsGlobPattern: -|Variables-*\*.ps1;-|APIScanInputs-*\**;-|test_symbols-*\**;-|MicroBuild\**;-|$(Build.ArtifactStagingDirectory)\VSInsertion-Windows\vs-insertion-script.ps1 policyFile: $(MBSIGN_APPFOLDER)\CSVTestSignPolicy.xml policheck: enabled: ${{ parameters.EnableProductionSDL }} @@ -83,7 +83,7 @@ extends: RealSign: false # ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} EnableAPIScan: ${{ parameters.EnableAPIScan }} - windowsPool: VSEngSS-MicroBuild2022-1ES + windowsPool: VSEng-MicroBuildVSStable linuxPool: name: AzurePipelines-EO demands: @@ -91,7 +91,7 @@ extends: os: Linux macOSPool: name: Azure Pipelines - vmImage: macOS-14 + vmImage: macOS-15 os: macOS EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} RunTests: ${{ parameters.RunTests }} diff --git a/azure-pipelines/vs-insertion-script.ps1 b/azure-pipelines/vs-insertion-script.ps1 new file mode 100644 index 00000000..53076254 --- /dev/null +++ b/azure-pipelines/vs-insertion-script.ps1 @@ -0,0 +1,17 @@ +# List of build artifact files [Source => Destination] to be committed into the VS repo. +$FilesToCommit = @{ + "$env:PROFILINGINPUTSPROPSNAME" = "src/Tests/config/runsettings/Official/OptProf/External/$env:PROFILINGINPUTSPROPSNAME"; +} + +foreach ($File in $FilesToCommit.GetEnumerator()) { + $SourcePath = Join-Path $PSScriptRoot $File.Key + if (Test-Path $SourcePath) { + $DestinationPath = Join-Path (Get-Location) $File.Value + Write-Host "Copying $SourcePath to $DestinationPath" + Copy-Item -Path $SourcePath -Destination $DestinationPath + git add $DestinationPath + } + else { + Write-Host "$SourcePath is not present, skipping" + } +} diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index c19331d4..f1a25768 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -15,7 +15,6 @@ resources: trigger: tags: - Real signed - - auto-insertion variables: - template: GlobalVariables.yml @@ -24,14 +23,16 @@ extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: VSEng-MicroBuildVSStable + sbom: + enabled: false stages: - stage: insertion jobs: - job: insertion displayName: VS insertion - pool: VSEngSS-MicroBuild2022-1ES + pool: VSEng-MicroBuildVSStable templateContext: outputParentDirectory: $(Pipeline.Workspace)/CI steps: @@ -64,9 +65,15 @@ extends: InsertionPayloadName: $(Build.Repository.Name) $(Build.BuildNumber) InsertionBuildPolicies: Request Perf DDRITs InsertionReviewers: $(Build.RequestedFor) # Append `,Your team name` (without quotes) + CustomScriptExecutionCommand: $(Pipeline.Workspace)\CI\VSInsertion-Windows\vs-insertion-script.ps1 AutoCompletePR: true AutoCompleteMergeStrategy: Squash ShallowClone: true + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + ConnectedVSDropServiceName: 'VSEng-VSDrop-MI' + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - powershell: | $contentType = 'application/json'; $headers = @{ Authorization = 'Bearer $(System.AccessToken)' }; diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index b9d46b7c..5e543863 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -27,7 +27,7 @@ extends: template: azure-pipelines/MicroBuild.1ES.Unofficial.yml@MicroBuildTemplate parameters: sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: VSEng-MicroBuildVSStable credscan: enabled: false @@ -44,7 +44,7 @@ extends: Is1ESPT: true RealSign: false # ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} - windowsPool: VSEngSS-MicroBuild2022-1ES + windowsPool: VSEng-MicroBuildVSStable linuxPool: name: AzurePipelines-EO demands: @@ -52,7 +52,7 @@ extends: os: Linux macOSPool: name: Azure Pipelines - vmImage: macOS-14 + vmImage: macOS-15 os: macOS EnableMacOSBuild: false RunTests: false @@ -68,7 +68,7 @@ extends: jobs: - job: insertion displayName: VS insertion - pool: VSEngSS-MicroBuild2022-1ES + pool: VSEng-MicroBuildVSStable steps: - checkout: self clean: true @@ -103,12 +103,16 @@ extends: InsertionPayloadName: $(Build.Repository.Name) VALIDATION BUILD $(Build.BuildNumber) ($(Build.SourceBranch)) [Skip-SymbolCheck] [Skip-HashCheck] [Skip-SignCheck] InsertionDescription: | This PR is for **validation purposes only** for !$(System.PullRequest.PullRequestId). **Do not complete**. - CustomScriptExecutionCommand: src/VSSDK/NuGet/AllowUnstablePackages.ps1 + CustomScriptExecutionCommand: $(Pipeline.Workspace)\VSInsertion-Windows\vs-insertion-script.ps1; src\VSSDK\NuGet\AllowUnstablePackages.ps1 InsertionBuildPolicies: Request Perf DDRITs InsertionReviewers: $(Build.RequestedFor) DraftPR: false # set to true and update InsertionBuildPolicy when we can specify all the validations we want to run (https://dev.azure.com/devdiv/DevDiv/_workitems/edit/2224288) AutoCompletePR: false ShallowClone: true + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + ConnectedVSDropServiceName: 'VSEng-VSDrop-MI' + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) - powershell: | $insertionPRId = azure-pipelines/Get-InsertionPRId.ps1 $Markdown = @" diff --git a/global.json b/global.json index 02b36f22..00e706b7 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "9.0.312", + "version": "10.0.204", "rollForward": "patch", "allowPrerelease": false }, diff --git a/src/Microsoft.VisualStudio.Jdt/JsonTransformation.cs b/src/Microsoft.VisualStudio.Jdt/JsonTransformation.cs index 3339a34c..2928e725 100644 --- a/src/Microsoft.VisualStudio.Jdt/JsonTransformation.cs +++ b/src/Microsoft.VisualStudio.Jdt/JsonTransformation.cs @@ -41,7 +41,7 @@ public JsonTransformation(string transformFile, IJsonTransformationLogger logger this.logger = new JsonTransformationContextLogger(transformFile, logger); - using (FileStream transformStream = File.Open(transformFile, FileMode.Open, FileAccess.Read)) + using (FileStream transformStream = File.Open(transformFile, FileMode.Open, FileAccess.Read, FileShare.Read)) { this.SetTransform(transformStream); } @@ -86,7 +86,7 @@ public Stream Apply(string sourceFile) } // Open the file as streams and apply the transforms - using (Stream sourceStream = File.Open(sourceFile, FileMode.Open, FileAccess.Read)) + using (Stream sourceStream = File.Open(sourceFile, FileMode.Open, FileAccess.Read, FileShare.Read)) { return this.ApplyWithSourceName(sourceStream, sourceFile); } diff --git a/src/OptProf.targets b/src/OptProf.targets index d0167d7c..0b6cc92c 100644 --- a/src/OptProf.targets +++ b/src/OptProf.targets @@ -5,11 +5,8 @@ Common7\IDE\PrivateAssemblies\$(TargetFileName) /ExeConfig:"%VisualStudio.InstallationUnderTest.Path%\Common7\IDE\vsn.exe" - - - + - diff --git a/src/VSInsertionMetadata/Library.VSInsertionMetadata.proj b/src/VSInsertionMetadata/Library.VSInsertionMetadata.proj deleted file mode 100644 index 0caaedb9..00000000 --- a/src/VSInsertionMetadata/Library.VSInsertionMetadata.proj +++ /dev/null @@ -1,11 +0,0 @@ - - - netstandard2.0 - true - $(RepoRootPath)bin\Packages\$(Configuration)\VSRepo\ - false - false - Contains metadata for insertion into VS. - - - diff --git a/src/VSInsertionMetadata/ProfilingInputs.props b/src/VSInsertionMetadata/ProfilingInputs.props deleted file mode 100644 index fb19d604..00000000 --- a/src/VSInsertionMetadata/ProfilingInputs.props +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/src/VSInsertionMetadata/VSInsertionMetadata.targets b/src/VSInsertionMetadata/VSInsertionMetadata.targets deleted file mode 100644 index 84ebb5a8..00000000 --- a/src/VSInsertionMetadata/VSInsertionMetadata.targets +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - $(TargetsForTfmSpecificContentInPackage); - SubstituteProfilingInputsMacro; - - - - - - - - - - - - - - - - - - - - - - $(PackageVersion).$(Build_BuildId) - - - - - - diff --git a/test/Microsoft.VisualStudio.Jdt.Tests/Microsoft.VisualStudio.Jdt.Tests.csproj b/test/Microsoft.VisualStudio.Jdt.Tests/Microsoft.VisualStudio.Jdt.Tests.csproj index b511263d..1a0431e0 100644 --- a/test/Microsoft.VisualStudio.Jdt.Tests/Microsoft.VisualStudio.Jdt.Tests.csproj +++ b/test/Microsoft.VisualStudio.Jdt.Tests/Microsoft.VisualStudio.Jdt.Tests.csproj @@ -1,7 +1,7 @@ ๏ปฟ - net472;net5.0 + net472;net8.0 false $(NoWarn);CS1591 diff --git a/tools/Convert-PDB.ps1 b/tools/Convert-PDB.ps1 index 7e1303a2..27346d68 100644 --- a/tools/Convert-PDB.ps1 +++ b/tools/Convert-PDB.ps1 @@ -10,11 +10,11 @@ #> [CmdletBinding()] Param( - [Parameter(Mandatory=$true,Position=0)] + [Parameter(Mandatory = $true, Position = 0)] [string]$DllPath, [Parameter()] [string]$PdbPath, - [Parameter(Mandatory=$true,Position=1)] + [Parameter(Mandatory = $true, Position = 1)] [string]$OutputPath ) @@ -23,26 +23,28 @@ if ($IsMacOS -or $IsLinux) { return } -$version = '1.1.0-beta2-21101-01' -$baseDir = "$PSScriptRoot/../obj/tools" -$pdb2pdbpath = "$baseDir/Microsoft.DiaSymReader.Pdb2Pdb.$version/tools/Pdb2Pdb.exe" -if (-not (Test-Path $pdb2pdbpath)) { - if (-not (Test-Path $baseDir)) { New-Item -Type Directory -Path $baseDir | Out-Null } - $baseDir = (Resolve-Path $baseDir).Path # Normalize it - Write-Verbose "& (& $PSScriptRoot/Get-NuGetTool.ps1) install Microsoft.DiaSymReader.Pdb2Pdb -version $version -PackageSaveMode nuspec -OutputDirectory $baseDir -Source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json | Out-Null" - # This package originally comes from the https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json feed. - # Add this feed as an upstream to whatever feed is in nuget.config if this step fails. - & (& $PSScriptRoot/Get-NuGetTool.ps1) install Microsoft.DiaSymReader.Pdb2Pdb -version $version -PackageSaveMode nuspec -OutputDirectory $baseDir | Out-Null - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to install Microsoft.DiaSymReader.Pdb2Pdb. Consider adding https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json as an upstream to your nuget.config feed." - return - } +# This package originally comes from the https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json feed. +# Add this feed as an upstream to whatever feed is in nuget.config if this step fails. +$packageID = 'Microsoft.DiaSymReader.Pdb2Pdb' +$packageVersion = '1.1.0-beta2-21101-01' +try { + $pdb2pdbpath = & "$PSScriptRoot/Download-NuGetPackage.ps1" -PackageId $packageID -Version $packageVersion -Source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json +} +catch { + Write-Error "Failed to install $packageID. Consider adding https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json as an upstream to your nuget.config feed." + return +} + +$outputDirectory = Split-Path $OutputPath -Parent +if ($outputDirectory) { + New-Item -ItemType Directory -Force -Path $outputDirectory | Out-Null } -$args = $DllPath,'/out',$OutputPath,'/nowarn','0021' +$toolpath = Join-Path $pdb2pdbpath 'tools\Pdb2Pdb.exe' +$arguments = $DllPath, '/out', $OutputPath, '/nowarn', '0021' if ($PdbPath) { - $args += '/pdb',$PdbPath + $arguments += '/pdb', $PdbPath } -Write-Verbose "$pdb2pdbpath $args" -& $pdb2pdbpath $args +Write-Verbose "$toolpath $arguments" +& $toolpath $arguments diff --git a/tools/Download-NuGetPackage.ps1 b/tools/Download-NuGetPackage.ps1 new file mode 100644 index 00000000..5864c689 --- /dev/null +++ b/tools/Download-NuGetPackage.ps1 @@ -0,0 +1,87 @@ +<# +.SYNOPSIS + Downloads a NuGet package to a local folder using dotnet package download. +.PARAMETER PackageId + The Package ID to download. +.PARAMETER Version + The version of the package to download. If unspecified, the latest version is downloaded. +.PARAMETER Source + An additional package source to search. Used as a fallback alongside the configured feeds. +.PARAMETER OutputDirectory + The directory to download the package to. By default, it uses the obj\tools folder at the root of the repo. +.PARAMETER ConfigFile + The nuget.config file to use. By default, it uses the repo root nuget.config. +.PARAMETER Verbosity + The verbosity level for the download. Defaults to quiet. +.OUTPUTS + System.String. The path to the downloaded package directory. +#> +[CmdletBinding()] +Param( + [Parameter(Position=1,Mandatory=$true)] + [string]$PackageId, + [Parameter()] + [string]$Version, + [Parameter()] + [string]$Source, + [Parameter()] + [string]$OutputDirectory="$PSScriptRoot\..\obj\tools", + [Parameter()] + [string]$ConfigFile="$PSScriptRoot\..\nuget.config", + [Parameter()] + [ValidateSet('quiet','minimal','normal','detailed','diagnostic')] + [string]$Verbosity='quiet' +) + +if (!(Test-Path $OutputDirectory)) { New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null } +$OutputDirectory = (Resolve-Path $OutputDirectory).Path +$ConfigFile = (Resolve-Path $ConfigFile).Path + +$packageIdLower = $PackageId.ToLowerInvariant() +$packageRoot = Join-Path $OutputDirectory $packageIdLower + +if ($Version) { + $predictedPackageDir = Join-Path $packageRoot $Version + if (Test-Path -Path $predictedPackageDir -PathType Container) { + Write-Output (Resolve-Path $predictedPackageDir).Path + return + } +} + +$packageArg = $PackageId +if ($Version) { $packageArg = "$PackageId@$Version" } + +$extraArgs = @() +if ($Source) { $extraArgs += '--source', $Source } +if ($Version -and $Version -match '-') { $extraArgs += '--prerelease' } + +$prevErrorActionPreference = $ErrorActionPreference +$ErrorActionPreference = 'Continue' +$downloadOutput = & dotnet package download $packageArg --configfile $ConfigFile --output $OutputDirectory --verbosity $Verbosity @extraArgs 2>&1 +$downloadExitCode = $LASTEXITCODE +$ErrorActionPreference = $prevErrorActionPreference + +if ($downloadExitCode -ne 0) { + $downloadOutput | Write-Host + throw "Failed to download package $packageArg (exit code $downloadExitCode)." +} + +# Return the path to the downloaded package directory (dotnet package download uses lowercase id) +if ($Version) { + $packageDir = Get-ChildItem -Path $packageRoot -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -ieq $Version } | + Select-Object -First 1 + if ($packageDir) { $packageDir = $packageDir.FullName } +} else { + # When no version is specified, pick the most recently written version directory. + $packageDir = Get-ChildItem -Path $packageRoot -Directory -ErrorAction SilentlyContinue | + Sort-Object -Property LastWriteTimeUtc -Descending | + Select-Object -First 1 + if ($packageDir) { $packageDir = $packageDir.FullName } +} + +if ($packageDir -and (Test-Path $packageDir)) { + Write-Output $packageDir +} else { + throw "Package directory not found after download. PackageId='$PackageId'; Version='$Version'; OutputDirectory='$OutputDirectory'; PackageRoot='$packageRoot'." +} diff --git a/tools/Get-3rdPartySymbolFiles.ps1 b/tools/Get-3rdPartySymbolFiles.ps1 deleted file mode 100644 index ef6bbef2..00000000 --- a/tools/Get-3rdPartySymbolFiles.ps1 +++ /dev/null @@ -1,91 +0,0 @@ -Function Get-FileFromWeb([Uri]$Uri, $OutFile) { - $OutDir = Split-Path $OutFile - if (!(Test-Path $OutFile)) { - Write-Verbose "Downloading $Uri..." - if (!(Test-Path $OutDir)) { New-Item -ItemType Directory -Path $OutDir | Out-Null } - try { - (New-Object System.Net.WebClient).DownloadFile($Uri, $OutFile) - } - finally { - # This try/finally causes the script to abort - } - } -} - -Function Unzip($Path, $OutDir) { - $OutDir = (New-Item -ItemType Directory -Path $OutDir -Force).FullName - Add-Type -AssemblyName System.IO.Compression.FileSystem - - # Start by extracting to a temporary directory so that there are no file conflicts. - [System.IO.Compression.ZipFile]::ExtractToDirectory($Path, "$OutDir.out") - - # Now move all files from the temp directory to $OutDir, overwriting any files. - Get-ChildItem -Path "$OutDir.out" -Recurse -File | ForEach-Object { - $destinationPath = Join-Path -Path $OutDir -ChildPath $_.FullName.Substring("$OutDir.out".Length).TrimStart([io.path]::DirectorySeparatorChar, [io.path]::AltDirectorySeparatorChar) - if (!(Test-Path -Path (Split-Path -Path $destinationPath -Parent))) { - New-Item -ItemType Directory -Path (Split-Path -Path $destinationPath -Parent) | Out-Null - } - Move-Item -Path $_.FullName -Destination $destinationPath -Force - } - Remove-Item -Path "$OutDir.out" -Recurse -Force -} - -Function Get-SymbolsFromPackage($id, $version) { - $symbolPackagesPath = "$PSScriptRoot/../obj/SymbolsPackages" - New-Item -ItemType Directory -Path $symbolPackagesPath -Force | Out-Null - $nupkgPath = Join-Path $symbolPackagesPath "$id.$version.nupkg" - $snupkgPath = Join-Path $symbolPackagesPath "$id.$version.snupkg" - $unzippedPkgPath = Join-Path $symbolPackagesPath "$id.$version" - Get-FileFromWeb -Uri "https://www.nuget.org/api/v2/package/$id/$version" -OutFile $nupkgPath - Get-FileFromWeb -Uri "https://www.nuget.org/api/v2/symbolpackage/$id/$version" -OutFile $snupkgPath - - Unzip -Path $nupkgPath -OutDir $unzippedPkgPath - Unzip -Path $snupkgPath -OutDir $unzippedPkgPath - - Get-ChildItem -Recurse -LiteralPath $unzippedPkgPath -Filter *.pdb | % { - # Collect the DLLs/EXEs as well. - $rootName = Join-Path $_.Directory $_.BaseName - if ($rootName.EndsWith('.ni')) { - $rootName = $rootName.Substring(0, $rootName.Length - 3) - } - - $dllPath = "$rootName.dll" - $exePath = "$rootName.exe" - if (Test-Path $dllPath) { - $BinaryImagePath = $dllPath - } - elseif (Test-Path $exePath) { - $BinaryImagePath = $exePath - } - else { - Write-Warning "`"$_`" found with no matching binary file." - $BinaryImagePath = $null - } - - if ($BinaryImagePath) { - Write-Output $BinaryImagePath - Write-Output $_.FullName - } - } -} - -Function Get-PackageVersion($id) { - $versionProps = [xml](Get-Content -LiteralPath $PSScriptRoot\..\Directory.Packages.props) - $version = $versionProps.Project.ItemGroup.PackageVersion | ? { $_.Include -eq $id } | % { $_.Version } - if (!$version) { - Write-Error "No package version found in Directory.Packages.props for the package '$id'" - } - - $version -} - -# All 3rd party packages for which symbols packages are expected should be listed here. -# These must all be sourced from nuget.org, as it is the only feed that supports symbol packages. -$3rdPartyPackageIds = @() - -$3rdPartyPackageIds | % { - $version = Get-PackageVersion $_ - if ($version) { - Get-SymbolsFromPackage -id $_ -version $version - } -} diff --git a/tools/Get-ExternalSymbolFiles.ps1 b/tools/Get-ExternalSymbolFiles.ps1 new file mode 100644 index 00000000..c5dbfd3c --- /dev/null +++ b/tools/Get-ExternalSymbolFiles.ps1 @@ -0,0 +1,118 @@ +[CmdletBinding()] +Param ( +) + +# Symbol servers to search for PDBs, in order of priority. +$SymbolServers = @( + 'https://msdl.microsoft.com/download/symbols' + 'https://symbols.nuget.org/download/symbols' +) + +Function Get-SymbolsFromPackage($id, $version) { + $symbolPackagesPath = "$PSScriptRoot/../obj/SymbolsPackages" + New-Item -ItemType Directory -Path $symbolPackagesPath -Force | Out-Null + $packagePath = $null + + # Download the package from configured feeds (failures are non-fatal for symbol collection) + $previousLastExitCode = $global:LASTEXITCODE + try { + $packagePath = & "$PSScriptRoot\Download-NuGetPackage.ps1" -PackageId $id -Version $version -OutputDirectory $symbolPackagesPath -ErrorAction SilentlyContinue + } + catch { + Write-Warning "Failed to download package $id $version from configured feeds. Skipping if not found locally. $($_.Exception.Message)" + } + $global:LASTEXITCODE = $previousLastExitCode + if (!$packagePath -or !(Test-Path -LiteralPath $packagePath)) { + Write-Warning "Package $id $version not found in configured feeds. Skipping." + return + } + + # Download symbols for each binary using dotnet-symbol + $serverArgs = $SymbolServers | ForEach-Object { '--server-path'; $_ } + $binaries = @(Get-ChildItem -Recurse -LiteralPath $packagePath -Include *.dll, *.exe) + if ($binaries) { + $prevErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + & dotnet symbol --symbols @serverArgs @($binaries.FullName) 2>&1 | Out-Null + $ErrorActionPreference = $prevErrorActionPreference + } + + # Output pairs of binary + PDB paths for archival + Get-ChildItem -Recurse -LiteralPath $packagePath -Filter *.pdb | % { + $rootName = Join-Path $_.Directory $_.BaseName + if ($rootName.EndsWith('.ni')) { + $rootName = $rootName.Substring(0, $rootName.Length - 3) + } + + $dllPath = "$rootName.dll" + $exePath = "$rootName.exe" + if (Test-Path $dllPath) { + $BinaryImagePath = $dllPath + } + elseif (Test-Path $exePath) { + $BinaryImagePath = $exePath + } + else { + Write-Warning "`"$_`" found with no matching binary file." + $BinaryImagePath = $null + } + + if ($BinaryImagePath) { + Write-Output $BinaryImagePath + Write-Output $_.FullName + } + } +} + +Function Get-PackageVersions() { + if ($script:PackageVersions) { + return $script:PackageVersions + } + + $propsPath = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..\Directory.Packages.props')).Path + $output = & dotnet msbuild $propsPath -nologo -verbosity:quiet -getItem:PackageVersion 2>&1 + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to evaluate package versions from Directory.Packages.props.`n$($output | Out-String)" + return @{} + } + + $jsonText = ($output | Out-String).Trim() + $jsonStart = $jsonText.IndexOf('{') + if ($jsonStart -lt 0) { + Write-Error 'Failed to locate JSON output from `dotnet msbuild -getItem:PackageVersion`.' + return @{} + } + + $packageVersions = @{} + foreach ($item in @((ConvertFrom-Json $jsonText.Substring($jsonStart)).Items.PackageVersion)) { + $packageVersions[$item.Identity] = $item.Version + } + + $script:PackageVersions = $packageVersions + $packageVersions +} + +Function Get-PackageVersion($id) { + $version = (Get-PackageVersions)[$id] + if (!$version) { + Write-Error "No package version found in Directory.Packages.props for the package '$id'" + } + + $version +} + +# All 1st party packages for which symbols packages are expected should be listed here. +# These must all be sourced from nuget.org, as it is the only feed that supports symbol packages. +# We should NOT add 3rd party packages to this list because PDBs may be unsafe for our debuggers to load, +# so we should only archive 1st party symbols. +$1stPartyPackageIds = @() + +$1stPartyPackageIds | % { + $version = Get-PackageVersion $_ + if ($version) { + Write-Verbose "Downloading symbols for package '$_' version '$version'." + Get-SymbolsFromPackage -id $_ -version $version + } else { + Write-Warning "No version found for package '$_'. Skipping symbol download." + } +} diff --git a/tools/Get-NuGetTool.ps1 b/tools/Get-NuGetTool.ps1 index bed0cba1..088c5f30 100644 --- a/tools/Get-NuGetTool.ps1 +++ b/tools/Get-NuGetTool.ps1 @@ -6,17 +6,44 @@ #> Param( [Parameter()] - [string]$NuGetVersion='6.14.0' + [string]$NuGetVersion='7.3.1' ) +function Test-NuGetExecutableSignature { + Param( + [Parameter(Mandatory=$true)] + [string]$Path + ) + + if (!(Test-Path -LiteralPath $Path)) { + return $false + } + + $signature = Get-AuthenticodeSignature -FilePath $Path + if ($signature.Status -eq [System.Management.Automation.SignatureStatus]::Valid -and + $null -ne $signature.SignerCertificate -and + $signature.SignerCertificate.Subject -like '*CN=Microsoft Corporation*') { + Write-Verbose "NuGet executable signature is valid." + return $true + } + + Write-Verbose "NuGet executable signature is invalid." + return $false +} + $toolsPath = & "$PSScriptRoot\Get-TempToolsPath.ps1" $binaryToolsPath = Join-Path $toolsPath $NuGetVersion if (!(Test-Path $binaryToolsPath)) { $null = mkdir $binaryToolsPath } $nugetPath = Join-Path $binaryToolsPath nuget.exe -if (!(Test-Path $nugetPath)) { +if (!(Test-Path $nugetPath) -or -not (Test-NuGetExecutableSignature -Path $nugetPath)) { Write-Host "Downloading nuget.exe $NuGetVersion..." -ForegroundColor Yellow (New-Object System.Net.WebClient).DownloadFile("https://dist.nuget.org/win-x86-commandline/v$NuGetVersion/NuGet.exe", $nugetPath) + + if (!(Test-NuGetExecutableSignature -Path $nugetPath)) { + Remove-Item $nugetPath -Force -ErrorAction SilentlyContinue + throw "Downloaded nuget.exe $NuGetVersion failed Authenticode signature validation." + } } return (Resolve-Path $nugetPath).Path diff --git a/tools/Get-ProcDump.ps1 b/tools/Get-ProcDump.ps1 index 6fba954d..9d9bcd8d 100644 --- a/tools/Get-ProcDump.ps1 +++ b/tools/Get-ProcDump.ps1 @@ -2,13 +2,4 @@ .SYNOPSIS Downloads 32-bit and 64-bit procdump executables and returns the path to where they were installed. #> -$version = '0.0.1' -$baseDir = "$PSScriptRoot\..\obj\tools" -$procDumpToolPath = "$baseDir\procdump.$version\bin" -if (-not (Test-Path $procDumpToolPath)) { - if (-not (Test-Path $baseDir)) { New-Item -Type Directory -Path $baseDir | Out-Null } - $baseDir = (Resolve-Path $baseDir).Path # Normalize it - & (& $PSScriptRoot\Get-NuGetTool.ps1) install procdump -version $version -PackageSaveMode nuspec -OutputDirectory $baseDir | Out-Null -} - -(Resolve-Path $procDumpToolPath).Path +Join-Path (& "$PSScriptRoot\Download-NuGetPackage.ps1" -PackageId procdump -Version 0.0.1) 'bin' diff --git a/tools/Install-DotNetSdk.ps1 b/tools/Install-DotNetSdk.ps1 index 402b4307..590581e2 100644 --- a/tools/Install-DotNetSdk.ps1 +++ b/tools/Install-DotNetSdk.ps1 @@ -125,14 +125,14 @@ Function Get-InstallerExe( } if ($TypedVersion.Build -eq -1) { - $versionInfo = -Split (Invoke-WebRequest -Uri "https://dotnetcli.blob.core.windows.net/dotnet/$sku/$Version/latest.version" -UseBasicParsing) + $versionInfo = -Split (Invoke-WebRequest -Uri "https://builds.dotnet.microsoft.com/dotnet/$sku/$Version/latest.version" -UseBasicParsing) $Version = $versionInfo[-1] } $majorMinor = "$($TypedVersion.Major).$($TypedVersion.Minor)" $ReleasesFile = Join-Path $DotNetInstallScriptRoot "$majorMinor\releases.json" if (!(Test-Path $ReleasesFile)) { - Get-FileFromWeb -Uri "https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/$majorMinor/releases.json" -OutDir (Split-Path $ReleasesFile) | Out-Null + Get-FileFromWeb -Uri "https://builds.dotnet.microsoft.com/dotnet/release-metadata/$majorMinor/releases.json" -OutDir (Split-Path $ReleasesFile) | Out-Null } $releases = Get-Content $ReleasesFile | ConvertFrom-Json @@ -197,7 +197,7 @@ if ($InstallLocality -eq 'machine') { $restartRequired = $false $sdks |% { if ($_.Version) { $version = $_.Version } else { $version = $_.Channel } - if ($PSCmdlet.ShouldProcess(".NET SDK $_", "Install")) { + if ($PSCmdlet.ShouldProcess(".NET SDK $version ($arch)", "Install")) { Install-DotNet -Version $version -Architecture $arch $restartRequired = $restartRequired -or ($LASTEXITCODE -eq 3010) @@ -281,10 +281,10 @@ if ($IncludeX86) { } if ($IsMacOS -or $IsLinux) { - $DownloadUri = "https://raw.githubusercontent.com/dotnet/install-scripts/0b09de9bc136cacb5f849a6957ebd4062173c148/src/dotnet-install.sh" + $DownloadUri = "https://raw.githubusercontent.com/dotnet/install-scripts/a3fbd0fd625032bac207f1f590e5353fe26faa59/src/dotnet-install.sh" $DotNetInstallScriptPath = "$DotNetInstallScriptRoot/dotnet-install.sh" } else { - $DownloadUri = "https://raw.githubusercontent.com/dotnet/install-scripts/0b09de9bc136cacb5f849a6957ebd4062173c148/src/dotnet-install.ps1" + $DownloadUri = "https://raw.githubusercontent.com/dotnet/install-scripts/a3fbd0fd625032bac207f1f590e5353fe26faa59/src/dotnet-install.ps1" $DotNetInstallScriptPath = "$DotNetInstallScriptRoot/dotnet-install.ps1" } @@ -306,7 +306,7 @@ $global:LASTEXITCODE = 0 $sdks |% { if ($_.Version) { $parameters = '-Version', $_.Version } else { $parameters = '-Channel', $_.Channel } - if ($PSCmdlet.ShouldProcess(".NET SDK $_", "Install")) { + if ($PSCmdlet.ShouldProcess(".NET SDK $_ ($arch)", "Install")) { $anythingInstalled = $true Invoke-Expression -Command "$DotNetInstallScriptPathExpression $parameters -Architecture $arch -InstallDir $DotNetInstallDir $switches" diff --git a/tools/Install-NuGetPackage.ps1 b/tools/Install-NuGetPackage.ps1 index 3c11b0f6..30e9c4fa 100644 --- a/tools/Install-NuGetPackage.ps1 +++ b/tools/Install-NuGetPackage.ps1 @@ -56,8 +56,30 @@ $nugetArgs += '-Verbosity',$Verbosity if ($PSCmdlet.ShouldProcess($PackageId, 'nuget install')) { $p = Start-Process $nugetPath $nugetArgs -NoNewWindow -Wait -PassThru - if ($null -ne $p.ExitCode -and $p.ExitCode -ne 0) { throw } + if ($null -ne $p.ExitCode -and $p.ExitCode -ne 0) { + throw "NuGet install failed for package '$PackageId' (version '$Version') with exit code $($p.ExitCode)." + } } # Provide the path to the installed package directory to our caller. -Write-Output (Get-ChildItem "$PackagesDir\$PackageId.*")[0].FullName +if ($ExcludeVersion) { + $packageDir = Get-ChildItem -Path $PackagesDir -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -ieq $PackageId } | + Select-Object -First 1 +} elseif ($Version) { + $expectedDirectoryName = "$PackageId.$Version" + $packageDir = Get-ChildItem -Path $PackagesDir -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -ieq $expectedDirectoryName } | + Select-Object -First 1 +} else { + $packageDir = Get-ChildItem -Path $PackagesDir -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -like "$PackageId.*" } | + Sort-Object -Property LastWriteTimeUtc -Descending | + Select-Object -First 1 +} + +if ($packageDir -and (Test-Path -Path $packageDir.FullName -PathType Container)) { + Write-Output $packageDir.FullName +} else { + throw "Installed package directory not found. PackageId='$PackageId'; Version='$Version'; ExcludeVersion='$ExcludeVersion'; PackagesDir='$PackagesDir'." +} diff --git a/tools/Prepare-Legacy-Symbols.ps1 b/tools/Prepare-Legacy-Symbols.ps1 index ae0bc40c..8a007c2a 100644 --- a/tools/Prepare-Legacy-Symbols.ps1 +++ b/tools/Prepare-Legacy-Symbols.ps1 @@ -20,6 +20,20 @@ Get-ChildItem "$ArtifactStagingFolder\*.pdb" -Recurse |% { } if ($BinaryImagePath) { + # Native binaries can't have their PDBs converted to legacy (Windows) format so just skip them + try { + [System.Reflection.AssemblyName]::GetAssemblyName($BinaryImagePath) | Out-Null + $isManaged = $true + } + catch { + $isManaged = $false + } + + if (-not $isManaged) { + Write-Host "Skipping native binary PDB: $_" -ForegroundColor DarkYellow + continue + } + # Convert the PDB to legacy Windows PDBs Write-Host "Converting PDB for $_" -ForegroundColor DarkGray $WindowsPdbDir = "$($_.Directory.FullName)\$WindowsPdbSubDirName" diff --git a/tools/artifacts/VSInsertion.ps1 b/tools/artifacts/VSInsertion.ps1 index a5b940b5..32d35dff 100644 --- a/tools/artifacts/VSInsertion.ps1 +++ b/tools/artifacts/VSInsertion.ps1 @@ -19,6 +19,10 @@ if (!$BuildConfiguration) { $PackagesRoot = "$RepoRoot/bin/Packages/$BuildConfiguration" $NuGetPackages = "$PackagesRoot/NuGet" $VsixPackages = "$PackagesRoot/Vsix" +$AzurePipelinesPath = "$RepoRoot/azure-pipelines" +if ($env:BUILD_ARTIFACTSTAGINGDIRECTORY) { + $InsertionOutputs = Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY 'InsertionOutputs' +} if (!(Test-Path $NuGetPackages) -and !(Test-Path $VsixPackages)) { Write-Warning "Skipping because NuGet and VSIX packages haven't been built yet." @@ -26,16 +30,19 @@ if (!(Test-Path $NuGetPackages) -and !(Test-Path $VsixPackages)) { } $result = @{ - "$NuGetPackages" = (Get-ChildItem $NuGetPackages -Recurse) + "$AzurePipelinesPath" = (Get-ChildItem "$AzurePipelinesPath/vs-insertion-script.ps1"); + "$NuGetPackages" = (Get-ChildItem $NuGetPackages -Recurse); } if (Test-Path $VsixPackages) { $result["$PackagesRoot"] += Get-ChildItem $VsixPackages -Recurse } -if ($env:IsOptProf) { - $VSRepoPackages = "$PackagesRoot/VSRepo" - $result["$VSRepoPackages"] = (Get-ChildItem "$VSRepoPackages\*.VSInsertionMetadata.*.nupkg"); +if ($InsertionOutputs -and $env:PROFILINGINPUTSPROPSNAME) { + $InsertionOutputsProfilingInputs = Join-Path $InsertionOutputs $env:PROFILINGINPUTSPROPSNAME + if (Test-Path -LiteralPath $InsertionOutputsProfilingInputs) { + $result[$InsertionOutputs] = Get-ChildItem -LiteralPath $InsertionOutputsProfilingInputs # OptProf ProfilingInputs + } } $result diff --git a/tools/artifacts/Variables.ps1 b/tools/artifacts/Variables.ps1 index 7a320c7e..c4d97665 100644 --- a/tools/artifacts/Variables.ps1 +++ b/tools/artifacts/Variables.ps1 @@ -26,7 +26,7 @@ Get-ChildItem "$PSScriptRoot/../variables" |% { if ($value) { # We got something, so wrap it with quotes so it's treated like a literal value. - $value = "'$value'" + $value = "'" + $value.Replace("'", "''") + "'" } } diff --git a/tools/artifacts/coverageResults.ps1 b/tools/artifacts/coverageResults.ps1 index 8c68216e..1aadbb74 100644 --- a/tools/artifacts/coverageResults.ps1 +++ b/tools/artifacts/coverageResults.ps1 @@ -1,25 +1,26 @@ -$RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..") +$RepoRoot = Resolve-Path "$PSScriptRoot\..\.." -$coverageFiles = @(Get-ChildItem "$RepoRoot/test/*.cobertura.xml" -Recurse | Where {$_.FullName -notlike "*/In/*" -and $_.FullName -notlike "*\In\*" }) +$coverageFilesUnderRoot = @(Get-ChildItem "$RepoRoot/*.cobertura.xml" -Recurse | Where-Object {$_.FullName -notlike "*/In/*" -and $_.FullName -notlike "*\In\*" }) + +# Under MTP, coverage files are written directly to the artifacts output directory, +# so we need to look there too. +$ArtifactStagingFolder = & "$PSScriptRoot/../Get-ArtifactsStagingDirectory.ps1" +$directTestLogs = Join-Path $ArtifactStagingFolder test_logs +$coverageFilesUnderArtifacts = if (Test-Path $directTestLogs) { @(Get-ChildItem "$directTestLogs/*.cobertura.xml" -Recurse) } else { @() } # Prepare code coverage reports for merging on another machine -$repoRoot = $env:SYSTEM_DEFAULTWORKINGDIRECTORY -if (!$repoRoot) { $repoRoot = $env:GITHUB_WORKSPACE } -if ($repoRoot) { - Write-Host "Substituting $repoRoot with `"{reporoot}`"" - $coverageFiles |% { - $content = Get-Content -LiteralPath $_ |% { $_ -Replace [regex]::Escape($repoRoot), "{reporoot}" } - Set-Content -LiteralPath $_ -Value $content -Encoding UTF8 - } -} else { - Write-Warning "coverageResults: Cloud build not detected. Machine-neutral token replacement skipped." +Write-Host "Substituting $repoRoot with `"{reporoot}`"" +@($coverageFilesUnderRoot + $coverageFilesUnderArtifacts) |? { $_ }|% { + $content = Get-Content -LiteralPath $_ |% { $_ -Replace [regex]::Escape($repoRoot), "{reporoot}" } + Set-Content -LiteralPath $_ -Value $content -Encoding UTF8 } if (!((Test-Path $RepoRoot\bin) -and (Test-Path $RepoRoot\obj))) { return } @{ + $directTestLogs = $coverageFilesUnderArtifacts; $RepoRoot = ( - $coverageFiles + + $coverageFilesUnderRoot + (Get-ChildItem "$RepoRoot\obj\*.cs" -Recurse) ); } diff --git a/tools/artifacts/symbols.ps1 b/tools/artifacts/symbols.ps1 index b5882678..91f83f0d 100644 --- a/tools/artifacts/symbols.ps1 +++ b/tools/artifacts/symbols.ps1 @@ -1,10 +1,10 @@ $BinPath = [System.IO.Path]::GetFullPath("$PSScriptRoot/../../bin") -$3rdPartyPath = [System.IO.Path]::GetFullPath("$PSScriptRoot/../../obj/SymbolsPackages") +$ExternalPath = [System.IO.Path]::GetFullPath("$PSScriptRoot/../../obj/SymbolsPackages") if (!(Test-Path $BinPath)) { return } $symbolfiles = & "$PSScriptRoot/../Get-SymbolFiles.ps1" -Path $BinPath | Get-Unique -$3rdPartyFiles = & "$PSScriptRoot/../Get-3rdPartySymbolFiles.ps1" +$ExternalFiles = & "$PSScriptRoot/../Get-ExternalSymbolFiles.ps1" @{ "$BinPath" = $SymbolFiles; - "$3rdPartyPath" = $3rdPartyFiles; + "$ExternalPath" = $ExternalFiles; } diff --git a/tools/artifacts/testResults.ps1 b/tools/artifacts/testResults.ps1 index 5310fb52..a841967e 100644 --- a/tools/artifacts/testResults.ps1 +++ b/tools/artifacts/testResults.ps1 @@ -4,7 +4,8 @@ Param( $result = @{} -$testRoot = Resolve-Path "$PSScriptRoot\..\..\test" +$RepoRoot = Resolve-Path "$PSScriptRoot\..\.." +$testRoot = Join-Path $RepoRoot test $result[$testRoot] = (Get-ChildItem "$testRoot\TestResults" -Recurse -Directory | Get-ChildItem -Recurse -File) $artifactStaging = & "$PSScriptRoot/../Get-ArtifactsStagingDirectory.ps1" diff --git a/tools/dotnet-test-cloud.ps1 b/tools/dotnet-test-cloud.ps1 index 74e2ea1d..85e371d1 100644 --- a/tools/dotnet-test-cloud.ps1 +++ b/tools/dotnet-test-cloud.ps1 @@ -25,6 +25,7 @@ Param( $RepoRoot = (Resolve-Path "$PSScriptRoot/..").Path $ArtifactStagingFolder = & "$PSScriptRoot/Get-ArtifactsStagingDirectory.ps1" +$OnCI = ($env:CI -or $env:TF_BUILD) $dotnet = 'dotnet' if ($x86) { @@ -47,23 +48,68 @@ if ($x86) { } $testBinLog = Join-Path $ArtifactStagingFolder (Join-Path build_logs test.binlog) -$testDiagLog = Join-Path $ArtifactStagingFolder (Join-Path test_logs diag.log) +$testLogs = Join-Path $ArtifactStagingFolder test_logs -& $dotnet test "$RepoRoot\test\Microsoft.VisualStudio.Jdt.Tests"` - --no-build ` - -c $Configuration ` - --filter "TestCategory!=FailsInCloudTest" ` - --collect "Code Coverage;Format=cobertura" ` - --settings "$PSScriptRoot/test.runsettings" ` - --blame-hang-timeout 60s ` - --blame-crash ` - -bl:"$testBinLog" ` - --diag "$testDiagLog;TraceLevel=info" ` - --logger trx ` +$globalJson = Get-Content $PSScriptRoot/../global.json | ConvertFrom-Json +$isMTP = $globalJson.test.runner -eq 'Microsoft.Testing.Platform' +$extraArgs = @() +$failedTests = 0 + +if ($isMTP) { + if ($OnCI) { $extraArgs += '--no-progress' } + + $dumpSwitches = @( + ,'--hangdump' + ,'--hangdump-timeout','120s' + ,'--crashdump' + ) + $mtpArgs = @( + ,'--coverage' + ,'--coverage-output-format','cobertura' + ,'--diagnostic' + ,'--diagnostic-output-directory',$testLogs + ,'--diagnostic-verbosity','Information' + ,'--results-directory',$testLogs + ,'--report-trx' + ) + + & $dotnet test "$RepoRoot\test\Microsoft.VisualStudio.Jdt.Tests"` + --no-build ` + -c $Configuration ` + -bl:"$testBinLog" ` + --filter-not-trait 'TestCategory=FailsInCloudTest' ` + --coverage-settings "$PSScriptRoot/test.runsettings" ` + @mtpArgs ` + @dumpSwitches ` + @extraArgs + if ($LASTEXITCODE -ne 0) { $failedTests += 1 } + + $trxFiles = Get-ChildItem -Recurse -Path $testLogs\*.trx +} else { + $testDiagLog = Join-Path $ArtifactStagingFolder (Join-Path test_logs diag.log) + & $dotnet test "$RepoRoot\test\Microsoft.VisualStudio.Jdt.Tests"` + --no-build ` + -c $Configuration ` + --filter "TestCategory!=FailsInCloudTest" ` + --collect "Code Coverage;Format=cobertura" ` + --settings "$PSScriptRoot/test.runsettings" ` + --blame-hang-timeout 60s ` + --blame-crash ` + -bl:"$testBinLog" ` + --diag "$testDiagLog;TraceLevel=info" ` + --logger trx ` + @extraArgs + if ($LASTEXITCODE -ne 0) { $failedTests += 1 } + + $trxFiles = Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx +} $unknownCounter = 0 -Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx | % { - Copy-Item $_ -Destination $ArtifactStagingFolder/test_logs/ +$trxFiles |% { + New-Item $testLogs -ItemType Directory -Force | Out-Null + if (!($_.FullName.StartsWith($testLogs, [StringComparison]::OrdinalIgnoreCase))) { + Copy-Item $_ -Destination $testLogs + } if ($PublishResults) { $x = [xml](Get-Content -LiteralPath $_) @@ -87,3 +133,7 @@ Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx | % { Write-Host "##vso[results.publish type=VSTest;runTitle=$runTitle;publishRunAttachments=true;resultFiles=$_;failTaskOnFailedTests=true;testRunSystem=VSTS - PTR;]" } } + +if ($failedTests -ne 0) { + exit $failedTests +} diff --git a/tools/variables/InsertJsonValues.ps1 b/tools/variables/InsertJsonValues.ps1 index 7aa8d30c..fb9390c5 100644 --- a/tools/variables/InsertJsonValues.ps1 +++ b/tools/variables/InsertJsonValues.ps1 @@ -8,11 +8,25 @@ $BasePath = "$PSScriptRoot\..\..\bin\Packages\$BuildConfiguration\Vsix" if (Test-Path $BasePath) { $vsmanFiles = @() - Get-ChildItem $BasePath *.vsman -Recurse -File |% { + Get-ChildItem $BasePath *.vsman -Recurse -File | % { $version = (Get-Content $_.FullName | ConvertFrom-Json).info.buildVersion + $fullPath = (Resolve-Path $_.FullName).Path + $basePath = (Resolve-Path $BasePath).Path + # Cannot use RelativePath or GetRelativePath due to Powershell Core v2.0 limitation + if ($fullPath.StartsWith($basePath, [StringComparison]::OrdinalIgnoreCase)) { + # Get the relative paths then make sure the directory separators match URL format. + $rfn = $fullPath.Substring($basePath.Length).TrimStart('\', '/').Replace('\', '/') + } + else { + $rfn = $fullPath # fallback to full path if it doesn't start with base path + } + $fn = $_.Name - $vsmanFiles += "$fn{$version}=https://vsdrop.corp.microsoft.com/file/v1/$vstsDropNames;$fn" + + # The left side is filename followed by the version and the right side is the drop url and the relative filename + $thisVsManFile = "$fn{$version}=https://vsdrop.corp.microsoft.com/file/v1/$vstsDropNames;$rfn" + $vsmanFiles += $thisVsManFile } - [string]::join(',',$vsmanFiles) + [string]::join(',', $vsmanFiles) } diff --git a/tools/variables/ProfilingInputsPropsName.ps1 b/tools/variables/ProfilingInputsPropsName.ps1 new file mode 100644 index 00000000..9e6bc4a1 --- /dev/null +++ b/tools/variables/ProfilingInputsPropsName.ps1 @@ -0,0 +1,6 @@ +if ($env:SYSTEM_TEAMPROJECT) { + $repoName = $env:BUILD_REPOSITORY_NAME.Replace('/', '.') + "$env:SYSTEM_TEAMPROJECT.$repoName.props" +} else { + Write-Warning "No Azure Pipelines build detected. No profiling inputs filename will be computed." +}