MegaBites-AI/Windows-powershell
0372
1# Copyright (c) Microsoft Corporation.2# Licensed under the MIT License.3 4[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '')]5param()6 7Set-StrictMode -Version 3.08 9$ErrorActionPreference = 'continue'10$repoRoot = Join-Path $PSScriptRoot '..'11$script:administratorsGroupSID = "S-1-5-32-544"12$script:usersGroupSID = "S-1-5-32-545"13 14# set .NET path15$dotNetPath = "$env:USERPROFILE\Appdata\Local\Microsoft\dotnet"16if(Test-Path $dotNetPath)17{18 $env:PATH = $dotNetPath + ';' + $env:PATH19}20 21# import build into the global scope so it can be used by packaging22# argumentList $true says ignore tha we may not be able to build23Write-Verbose "Importing build.psm1" -Verbose24Import-Module (Join-Path $repoRoot 'build.psm1') -Scope Global -ArgumentList $true25$buildCommands = Get-Command -Module build26Write-Verbose "Imported build.psm1 commands: $($buildCommands.Count)" -Verbose27 28Write-Verbose "Importing packaging.psm1" -Verbose29Import-Module (Join-Path $repoRoot 'tools\packaging') -Scope Global30$packagingCommands = Get-Command -Module packaging31Write-Verbose "Imported packaging.psm1 commands: $($packagingCommands.Count)" -Verbose32 33# import the windows specific functcion only in Windows PowerShell or on Windows34if($PSVersionTable.PSEdition -eq 'Desktop' -or $IsWindows)35{36 Import-Module (Join-Path $PSScriptRoot 'WindowsCI.psm1') -Scope Global37}38 39# tests if we should run a daily build40# returns true if the build is scheduled41# or is a pushed tag42Function Test-DailyBuild43{44 $trueString = 'True'45 if(($env:PS_DAILY_BUILD -eq $trueString) -or $env:BUILD_REASON -eq 'Schedule')46 {47 return $true48 }49 return $false50}51 52# Sets a build variable53Function Set-BuildVariable54{55 param(56 [Parameter(Mandatory=$true)]57 [string]58 $Name,59 60 [Parameter(Mandatory=$true)]61 [string]62 $Value,63 64 [switch]65 $IsOutput66 )67 68 $IsOutputString = if ($IsOutput) { 'true' } else { 'false' }69 $command = "vso[task.setvariable variable=$Name;isOutput=$IsOutputString]$Value"70 71 # always log command to make local debugging easier72 Write-Verbose -Message "sending command: $command" -Verbose73 74 if ($env:TF_BUILD) {75 # In VSTS76 Write-Host "##$command"77 # The variable will not show up until the next task.78 }79 80 # Setting in the current session for the same behavior as the CI and to make it show up in the same task81 Set-Item env:/$name -Value $Value82}83 84# Emulates running all of CI but locally85function Invoke-CIFull86{87 param(88 [switch] $CleanRepo89 )90 if($CleanRepo)91 {92 Clear-PSRepo93 }94 95 Invoke-CIInstall96 Invoke-CIBuild97 Invoke-CITest -ErrorAction Continue98 Invoke-CIFinish99}100 101# Implements the CI 'build_script' step102function Invoke-CIBuild103{104 param(105 [ValidateSet('Debug', 'Release', 'CodeCoverage', 'StaticAnalysis')]106 [string]$Configuration = 'Release'107 )108 109 $releaseTag = Get-ReleaseTag110 # check to be sure our test tags are correct111 $result = Get-PesterTag112 if ( $result.Result -ne "Pass" )113 {114 $result.Warnings115 throw "Tags must be CI, Feature, Scenario, or Slow"116 }117 118 if(Test-DailyBuild)119 {120 Start-PSBuild -Configuration 'CodeCoverage' -PSModuleRestore -CI -ReleaseTag $releaseTag121 }122 123 Start-PSBuild -PSModuleRestore -Configuration $Configuration -CI -ReleaseTag $releaseTag -UseNuGetOrg124 Save-PSOptions125 126 $options = (Get-PSOptions)127 128 $path = Split-Path -Path $options.Output129 130 $psOptionsPath = (Join-Path -Path $PSScriptRoot -ChildPath '../psoptions.json')131 $buildZipPath = (Join-Path -Path $PSScriptRoot -ChildPath '../build.zip')132 133 Compress-Archive -Path $path -DestinationPath $buildZipPath134 135 Push-Artifact -Path $psOptionsPath -Name 'build'136 Push-Artifact -Path $buildZipPath -Name 'build'137}138 139# Implements the CI 'install' step140function Invoke-CIInstall141{142 param(143 [switch]144 $SkipUser145 )146 147 # Switch to public sources in CI148 Switch-PSNugetConfig -Source Public149 150 # Make sure we have all the tags151 Sync-PSTags -AddRemoteIfMissing152 153 if(Test-DailyBuild)154 {155 if ($env:BUILD_REASON -eq 'Schedule')156 {157 Write-Host "##vso[build.updatebuildnumber]Daily-$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((Get-Date).ToString("yyyyMMddhhss"))"158 }159 }160 161 if ($env:TF_BUILD -and !$SkipUser.IsPresent)162 {163 # Generate new credential for CI (only) remoting tests.164 Write-Verbose "Creating account for remoting tests in CI." -Verbose165 166 # Password167 $randomObj = [System.Random]::new()168 $password = ""169 1..(Get-Random -Minimum 15 -Maximum 126) | ForEach-Object { $password = $password + [char]$randomObj.next(45,126) }170 171 # Account172 $userName = 'ciRemote'173 New-LocalUser -username $userName -Password $password174 Add-UserToGroup -username $userName -groupSid $script:administratorsGroupSID175 176 # Provide credentials globally for remote tests.177 $ss = ConvertTo-SecureString -String $password -AsPlainText -Force178 $ciRemoteCredential = [PSCredential]::new("$env:COMPUTERNAME\$userName", $ss)179 $ciRemoteCredential | Export-Clixml -Path "$env:TEMP\CIRemoteCred.xml" -Force180 181 # Check that LocalAccountTokenFilterPolicy policy is set, since it is needed for remoting182 # using above local admin account.183 Write-Log -Message "Checking for LocalAccountTokenFilterPolicy in the CI."184 $haveLocalAccountTokenFilterPolicy = $false185 try186 {187 $haveLocalAccountTokenFilterPolicy = ((Get-ItemPropertyValue -Path HKLM:SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System -Name LocalAccountTokenFilterPolicy) -eq 1)188 }189 # ignore if anything is caught:190 catch {}191 if (!$haveLocalAccountTokenFilterPolicy)192 {193 Write-Verbose "Setting the LocalAccountTokenFilterPolicy for remoting tests"194 Set-ItemProperty -Path HKLM:SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System -Name LocalAccountTokenFilterPolicy -Value 1195 }196 }197 198 Set-BuildVariable -Name TestPassed -Value False199}200 201function Invoke-CIxUnit202{203 param(204 [switch]205 $SkipFailing206 )207 $env:CoreOutput = Split-Path -Parent (Get-PSOutput -Options (Get-PSOptions))208 $path = "$env:CoreOutput\pwsh.exe"209 if($IsMacOS -or $IsLinux)210 {211 $path = "$env:CoreOutput\pwsh"212 }213 214 if(!(Test-Path $path))215 {216 throw "CoreCLR pwsh.exe was not built"217 }218 219 $xUnitTestResultsFile = Join-Path -Path $PWD -ChildPath "xUnitTestResults.xml"220 221 Start-PSxUnit -xUnitTestResultsFile $xUnitTestResultsFile222 Push-Artifact -Path $xUnitTestResultsFile -name xunit223 224 if(!$SkipFailing.IsPresent)225 {226 # Fail the build, if tests failed227 Test-XUnitTestResults -TestResultsFile $xUnitTestResultsFile228 }229}230 231# Install Pester module if not already installed with a compatible version232function Install-CIPester233{234 [CmdletBinding()]235 param(236 [string]$MinimumVersion = '5.0.0',237 [string]$MaximumVersion = '5.99.99',238 [switch]$Force239 )240 241 Write-Verbose "Checking for Pester module (required: $MinimumVersion - $MaximumVersion)" -Verbose242 243 # Check if a compatible version of Pester is already installed244 $installedPester = Get-Module -Name Pester -ListAvailable |245 Where-Object { $_.Version -ge $MinimumVersion -and $_.Version -le $MaximumVersion } |246 Sort-Object -Property Version -Descending |247 Select-Object -First 1248 249 if ($installedPester -and -not $Force) {250 Write-Host "Pester version $($installedPester.Version) is already installed and meets requirements" -ForegroundColor Green251 return252 }253 254 if ($Force) {255 Write-Host "Installing Pester module (forced)" -ForegroundColor Yellow256 } else {257 Write-Host "Installing Pester module" -ForegroundColor Yellow258 }259 260 try {261 Install-Module -Name Pester -Force -SkipPublisherCheck -MaximumVersion $MaximumVersion -ErrorAction Stop262 Write-Host "Successfully installed Pester module" -ForegroundColor Green263 }264 catch {265 Write-Error "Failed to install Pester module: $_"266 throw267 }268}269 270# Implement CI 'Test_script'271function Invoke-CITest272{273 [CmdletBinding()]274 param(275 [ValidateSet('UnelevatedPesterTests', 'ElevatedPesterTests')]276 [string] $Purpose,277 [ValidateSet('CI', 'Others')]278 [string] $TagSet,279 [string] $TitlePrefix,280 [string] $OutputFormat = "NUnitXml"281 )282 283 Write-Verbose -Verbose "CI test: OutputFormat: $OutputFormat"284 285 # Set locale correctly for Linux CIs286 Set-CorrectLocale287 288 # Pester doesn't allow Invoke-Pester -TagAll@('CI', 'RequireAdminOnWindows') currently289 # https://github.com/pester/Pester/issues/608290 # To work-around it, we exlude all categories, but 'CI' from the list291 switch ($TagSet) {292 'CI' {293 Write-Host -Foreground Green 'Running "CI" CoreCLR tests..'294 $ExcludeTag = @('Slow', 'Feature', 'Scenario')295 }296 'Others' {297 Write-Host -Foreground Green 'Running non-CI CoreCLR tests..'298 $ExcludeTag = @('CI')299 }300 Default {301 throw "Unknown TagSet: '$TagSet'"302 }303 }304 305 if($IsLinux -or $IsMacOS)306 {307 return Invoke-LinuxTestsCore -Purpose $Purpose -ExcludeTag $ExcludeTag -TagSet $TagSet -TitlePrefix $TitlePrefix -OutputFormat $OutputFormat308 }309 310 # CoreCLR311 312 $env:CoreOutput = Split-Path -Parent (Get-PSOutput -Options (Get-PSOptions))313 Write-Host -Foreground Green 'Run CoreCLR tests'314 $testResultsNonAdminFile = "$PWD\TestsResultsNonAdmin-$TagSet.xml"315 $testResultsAdminFile = "$PWD\TestsResultsAdmin-$TagSet.xml"316 if(!(Test-Path "$env:CoreOutput\pwsh.exe"))317 {318 throw "CoreCLR pwsh.exe was not built"319 }320 321 # Get the experimental feature names and the tests associated with them322 $ExperimentalFeatureTests = Get-ExperimentalFeatureTests323 324 if ($Purpose -eq 'UnelevatedPesterTests') {325 $unelevate = $true326 $environment = Get-EnvironmentInformation327 if ($environment.OSArchitecture -eq 'arm64') {328 Write-Verbose -Verbose "running on arm64, running unelevated tests as elevated"329 $unelevate = $false330 }331 332 $arguments = @{333 Bindir = $env:CoreOutput334 OutputFile = $testResultsNonAdminFile335 Unelevate = $unelevate336 Terse = $true337 Tag = @()338 ExcludeTag = $ExcludeTag + 'RequireAdminOnWindows'339 OutputFormat = $OutputFormat340 }341 342 $title = "Pester Unelevated - $TagSet"343 if ($TitlePrefix) {344 $title = "$TitlePrefix - $title"345 }346 Write-Verbose -Verbose "Starting Pester with output format $($arguments.OutputFormat)"347 Start-PSPester @arguments -Title $title348 349 # Fail the build, if tests failed350 Test-PSPesterResults -TestResultsFile $testResultsNonAdminFile351 352 # Run tests with specified experimental features enabled353 foreach ($entry in $ExperimentalFeatureTests.GetEnumerator())354 {355 $featureName = $entry.Key356 $testFiles = $entry.Value357 358 $expFeatureTestResultFile = "$PWD\TestsResultsNonAdmin.$featureName.xml"359 $arguments['OutputFile'] = $expFeatureTestResultFile360 $arguments['ExperimentalFeatureName'] = $featureName361 if ($testFiles.Count -eq 0) {362 # If an empty array is specified for the feature name, we run all tests with the feature enabled.363 # This allows us to prevent regressions to a critical engine experimental feature.364 $arguments.Remove('Path')365 } else {366 # If a non-empty string or array is specified for the feature name, we only run those test files.367 $arguments['Path'] = $testFiles368 }369 370 $title = "Pester Experimental Unelevated - $featureName"371 if ($TitlePrefix) {372 $title = "$TitlePrefix - $title"373 }374 375 # We just built the test tools, we don't need to rebuild them376 Write-Verbose -Verbose "Starting Pester with output format $($arguments.OutputFormat)"377 Start-PSPester @arguments -Title $title -SkipTestToolBuild378 379 # Fail the build, if tests failed380 Test-PSPesterResults -TestResultsFile $expFeatureTestResultFile381 }382 }383 384 if ($Purpose -eq 'ElevatedPesterTests') {385 $arguments = @{386 Terse = $true387 Bindir = $env:CoreOutput388 OutputFile = $testResultsAdminFile389 Tag = @('RequireAdminOnWindows')390 ExcludeTag = $ExcludeTag391 OutputFormat = $OutputFormat392 }393 394 $title = "Pester Elevated - $TagSet"395 if ($TitlePrefix) {396 $title = "$TitlePrefix - $title"397 }398 399 Write-Verbose -Verbose "Starting Pester with output format $($arguments.OutputFormat)"400 Start-PSPester @arguments -Title $title401 402 # Fail the build, if tests failed403 Test-PSPesterResults -TestResultsFile $testResultsAdminFile404 405 # Run tests with specified experimental features enabled406 foreach ($entry in $ExperimentalFeatureTests.GetEnumerator())407 {408 $featureName = $entry.Key409 $testFiles = $entry.Value410 411 $expFeatureTestResultFile = "$PWD\TestsResultsAdmin.$featureName.xml"412 $arguments['OutputFile'] = $expFeatureTestResultFile413 $arguments['ExperimentalFeatureName'] = $featureName414 if ($testFiles.Count -eq 0)415 {416 # If an empty array is specified for the feature name, we run all tests with the feature enabled.417 # This allows us to prevent regressions to a critical engine experimental feature.418 $arguments.Remove('Path')419 }420 else421 {422 # If a non-empty string or array is specified for the feature name, we only run those test files.423 $arguments['Path'] = $testFiles424 }425 426 $title = "Pester Experimental >levated - $featureName"427 if ($TitlePrefix) {428 $title = "$TitlePrefix - $title"429 }430 431 Write-Verbose -Verbose "Starting Pester with output format $($arguments.OutputFormat)"432 # We just built the test tools, we don't need to rebuild them433 Start-PSPester @arguments -Title $title -SkipTestToolBuild434 435 # Fail the build, if tests failed436 Test-PSPesterResults -TestResultsFile $expFeatureTestResultFile437 }438 }439 440 Set-BuildVariable -Name TestPassed -Value True441}442 443function New-CodeCoverageAndTestPackage444{445 [CmdletBinding()]446 param()447 448 if (Test-DailyBuild)449 {450 Start-PSBuild -Configuration 'CodeCoverage' -Clean451 452 $codeCoverageOutput = Split-Path -Parent (Get-PSOutput)453 $codeCoverageArtifacts = Compress-CoverageArtifacts -CodeCoverageOutput $codeCoverageOutput454 455 $destBasePath = if ($env:TF_BUILD) {456 $env:BUILD_ARTIFACTSTAGINGDIRECTORY457 } else {458 Join-Path (Get-Location).Path "out"459 }460 461 if (-not (Test-Path $destBasePath))462 {463 $null = New-Item -ItemType Directory -Path $destBasePath464 }465 466 Write-Host -ForegroundColor Green 'Upload CodeCoverage artifacts'467 $codeCoverageArtifacts | ForEach-Object {468 Copy-Item -Path $_ -Destination $destBasePath469 $newPath = Join-Path $destBasePath (Split-Path $_ -Leaf)470 Push-Artifact -Path $newPath -Name 'CodeCoverage'471 }472 473 New-TestPackage -Destination $destBasePath474 $testPackageFullName = Join-Path $destBasePath 'TestPackage.zip'475 Write-Verbose "Created TestPackage.zip" -Verbose476 Write-Host -ForegroundColor Green 'Upload test package'477 Push-Artifact $testPackageFullName -Name 'CodeCoverage'478 }479}480 481# Wrapper to push artifact482function Push-Artifact483{484 param(485 [Parameter(Mandatory)]486 [ValidateScript({Test-Path -Path $_})]487 $Path,488 [string]489 $Name490 )491 492 if(!$Name)493 {494 $artifactName = [system.io.path]::GetFileName($Path)495 }496 else497 {498 $artifactName = $Name499 }500 501 if ($env:TF_BUILD) {502 # In Azure DevOps503 Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$Path"504 } elseif ($env:GITHUB_WORKFLOW -and $env:RUNNER_WORKSPACE) {505 # In GitHub Actions506 $destinationPath = Join-Path -Path $env:RUNNER_WORKSPACE -ChildPath $artifactName507 508 # Create the folder if it does not exist509 if (!(Test-Path -Path $destinationPath)) {510 $null = New-Item -ItemType Directory -Path $destinationPath -Force511 }512 513 Copy-Item -Path $Path -Destination $destinationPath -Force -Verbose514 } else {515 Write-Warning "Push-Artifact is not supported in this environment."516 }517}518 519function Compress-CoverageArtifacts520{521 param([string] $CodeCoverageOutput)522 523 # Create archive for test content, OpenCover module and CodeCoverage build524 $artifacts = New-Object System.Collections.ArrayList525 526 Add-Type -AssemblyName System.IO.Compression.FileSystem527 $resolvedPath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath((Join-Path $PSScriptRoot '..\test\tools\OpenCover'))528 $zipOpenCoverPath = Join-Path $PWD 'OpenCover.zip'529 [System.IO.Compression.ZipFile]::CreateFromDirectory($resolvedPath, $zipOpenCoverPath)530 $null = $artifacts.Add($zipOpenCoverPath)531 532 $zipCodeCoveragePath = Join-Path $PWD "CodeCoverage.zip"533 Write-Verbose "Zipping ${CodeCoverageOutput} into $zipCodeCoveragePath" -Verbose534 [System.IO.Compression.ZipFile]::CreateFromDirectory($CodeCoverageOutput, $zipCodeCoveragePath)535 $null = $artifacts.Add($zipCodeCoveragePath)536 537 return $artifacts538}539 540function Get-ReleaseTag541{542 $metaDataPath = Join-Path -Path $PSScriptRoot -ChildPath 'metadata.json'543 $metaData = Get-Content $metaDataPath | ConvertFrom-Json544 $releaseTag = $metadata.NextReleaseTag545 if($env:BUILD_BUILID)546 {547 $releaseTag = $releaseTag.split('.')[0..2] -join '.'548 $releaseTag = $releaseTag + '.' + $env:BUILD_BUILID549 }550 return $releaseTag551}552 553# Implements CI 'on_finish' step554function Invoke-CIFinish555{556 param(557 [string] $Runtime = 'win7-x64',558 [string] $Channel = 'preview',559 [Validateset('Build','Package')]560 [string[]] $Stage = ('Build','Package')561 )562 563 # Switch to public sources in CI564 Switch-PSNugetConfig -Source Public565 566 if ($PSEdition -eq 'Core' -and ($IsLinux -or $IsMacOS) -and $Stage -contains 'Build') {567 return New-LinuxPackage568 }569 570 $artifacts = New-Object System.Collections.ArrayList571 try {572 $buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/mainBuild"573 574 if ($Stage -contains "Build") {575 if ($Channel -eq 'preview') {576 $releaseTag = Get-ReleaseTag577 578 $previewVersion = $releaseTag.Split('-')579 $previewPrefix = $previewVersion[0]580 $previewLabel = $previewVersion[1].replace('.','')581 582 if (Test-DailyBuild) {583 $previewLabel = "daily{0}" -f $previewLabel584 }585 586 $prereleaseIteration = (get-date).Day587 $preReleaseVersion = "$previewPrefix-$previewLabel.$prereleaseIteration"588 # Build clean before backing to remove files from testing589 Start-PSBuild -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean -Runtime $Runtime -output $buildFolder -PSOptionsPath "${buildFolder}/psoptions.json"590 $options = Get-PSOptions591 # Remove symbol files.592 $filter = Join-Path -Path (Split-Path $options.Output) -ChildPath '*.pdb'593 Write-Verbose "Removing symbol files from $filter" -Verbose594 Remove-Item $filter -Force -Recurse595 } else {596 $releaseTag = Get-ReleaseTag597 $releaseTagParts = $releaseTag.split('.')598 $preReleaseVersion = $releaseTagParts[0]+ ".9.9"599 Write-Verbose "newPSReleaseTag: $preReleaseVersion" -Verbose600 Start-PSBuild -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean -Runtime $Runtime -output $buildFolder -PSOptionsPath "${buildFolder}/psoptions.json"601 $options = Get-PSOptions602 # Remove symbol files.603 $filter = Join-Path -Path (Split-Path $options.Output) -ChildPath '*.pdb'604 Write-Verbose "Removing symbol files from $filter" -Verbose605 Remove-Item $filter -Force -Recurse606 }607 608 # Set a variable, both in the current process and in AzDevOps for the packaging stage to get the release tag609 $env:CI_FINISH_RELASETAG=$preReleaseVersion610 $vstsCommandString = "vso[task.setvariable variable=CI_FINISH_RELASETAG]$preReleaseVersion"611 Write-Verbose -Message "$vstsCommandString" -Verbose612 Write-Host -Object "##$vstsCommandString"613 }614 615 if ($Stage -contains "Package") {616 Restore-PSOptions -PSOptionsPath "${buildFolder}/psoptions.json"617 $preReleaseVersion = $env:CI_FINISH_RELASETAG618 619 # Build packages620 $packageTypes = 'zip', 'zip-pdb', 'msix'621 622 $packages = Start-PSPackage -Type $packageTypes -ReleaseTag $preReleaseVersion -SkipReleaseChecks -WindowsRuntime $Runtime623 624 foreach ($package in $packages) {625 if (Test-Path $package -ErrorAction Ignore) {626 Write-Log "Package found: $package"627 } else {628 Write-Warning -Message "Package NOT found: $package"629 }630 631 if ($package -is [string]) {632 $null = $artifacts.Add($package)633 }634 }635 636 }637 } catch {638 Get-Error -InputObject $_639 throw640 } finally {641 $pushedAllArtifacts = $true642 643 $artifacts | ForEach-Object {644 Write-Log -Message "Pushing $_ as CI artifact"645 if (Test-Path $_) {646 Push-Artifact -Path $_ -Name 'artifacts'647 } else {648 $pushedAllArtifacts = $false649 Write-Warning "Artifact $_ does not exist."650 }651 }652 653 if (!$pushedAllArtifacts) {654 throw "Some artifacts did not exist!"655 }656 }657}658 659function Set-Path660{661 param662 (663 [Parameter(Mandatory)]664 [string]665 $Path,666 667 [Parameter(Mandatory)]668 [switch]669 $Append670 )671 672 $machinePathString = [System.Environment]::GetEnvironmentVariable('path',[System.EnvironmentVariableTarget]::Machine)673 $machinePath = $machinePathString -split ';'674 675 if($machinePath -inotcontains $path)676 {677 $newPath = "$machinePathString;$path"678 Write-Verbose "Adding $path to path..." -Verbose679 [System.Environment]::SetEnvironmentVariable('path',$newPath,[System.EnvironmentVariableTarget]::Machine)680 Write-Verbose "Added $path to path." -Verbose681 }682 else683 {684 Write-Verbose "$path already in path." -Verbose685 }686}687 688# Display environment variables in a log group for GitHub Actions689function Show-Environment690{691 Write-LogGroupStart -Title 'Environment'692 Get-ChildItem -Path env: | Out-String -width 9999 -Stream | Write-Verbose -Verbose693 Write-LogGroupEnd -Title 'Environment'694}695 696# Bootstrap script for Linux and macOS697function Invoke-BootstrapStage698{699 $createPackages = Test-DailyBuild700 Write-Log -Message "Executing ci.psm1 Bootstrap Stage"701 # Make sure we have all the tags702 Sync-PSTags -AddRemoteIfMissing703 Start-PSBootstrap -Scenario Package:$createPackages704}705 706# Run pester tests for Linux and macOS707function Invoke-LinuxTestsCore708{709 [CmdletBinding()]710 param(711 [ValidateSet('UnelevatedPesterTests', 'ElevatedPesterTests', 'All')]712 [string] $Purpose = 'All',713 [string[]] $ExcludeTag = @('Slow', 'Feature', 'Scenario'),714 [string] $TagSet = 'CI',715 [string] $TitlePrefix,716 [string] $OutputFormat = "NUnitXml"717 )718 719 $output = Split-Path -Parent (Get-PSOutput -Options (Get-PSOptions))720 $testResultsNoSudo = "$PWD/TestResultsNoSudo.xml"721 $testResultsSudo = "$PWD/TestResultsSudo.xml"722 $testExcludeTag = $ExcludeTag + 'RequireSudoOnUnix'723 $pesterPassThruNoSudoObject = $null724 $pesterPassThruSudoObject = $null725 $noSudoResultsWithExpFeatures = $null726 $sudoResultsWithExpFeatures = $null727 728 $noSudoPesterParam = @{729 'BinDir' = $output730 'PassThru' = $true731 'Terse' = $true732 'Tag' = @()733 'ExcludeTag' = $testExcludeTag734 'OutputFile' = $testResultsNoSudo735 'OutputFormat' = $OutputFormat736 }737 738 # Get the experimental feature names and the tests associated with them739 $ExperimentalFeatureTests = Get-ExperimentalFeatureTests740 741 # Running tests which do not require sudo.742 if($Purpose -eq 'UnelevatedPesterTests' -or $Purpose -eq 'All')743 {744 $title = "Pester No Sudo - $TagSet"745 if ($TitlePrefix) {746 $title = "$TitlePrefix - $title"747 }748 $pesterPassThruNoSudoObject = Start-PSPester @noSudoPesterParam -Title $title749 750 # Running tests that do not require sudo, with specified experimental features enabled751 $noSudoResultsWithExpFeatures = @()752 foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) {753 $featureName = $entry.Key754 $testFiles = $entry.Value755 756 $expFeatureTestResultFile = "$PWD\TestResultsNoSudo.$featureName.xml"757 $noSudoPesterParam['OutputFile'] = $expFeatureTestResultFile758 $noSudoPesterParam['ExperimentalFeatureName'] = $featureName759 if ($testFiles.Count -eq 0) {760 # If an empty array is specified for the feature name, we run all tests with the feature enabled.761 # This allows us to prevent regressions to a critical engine experimental feature.762 $noSudoPesterParam.Remove('Path')763 }764 else765 {766 # If a non-empty string or array is specified for the feature name, we only run those test files.767 $noSudoPesterParam['Path'] = $testFiles768 }769 $title = "Pester Experimental No Sudo - $featureName - $TagSet"770 if ($TitlePrefix) {771 $title = "$TitlePrefix - $title"772 }773 $passThruResult = Start-PSPester @noSudoPesterParam -Title $title -SkipTestToolBuild774 775 $noSudoResultsWithExpFeatures += $passThruResult776 }777 }778 779 # Running tests, which require sudo.780 if($Purpose -eq 'ElevatedPesterTests' -or $Purpose -eq 'All')781 {782 $sudoPesterParam = $noSudoPesterParam.Clone()783 $sudoPesterParam.Remove('Path')784 $sudoPesterParam['Tag'] = @('RequireSudoOnUnix')785 $sudoPesterParam['ExcludeTag'] = $ExcludeTag786 $sudoPesterParam['Sudo'] = $true787 $sudoPesterParam['OutputFile'] = $testResultsSudo788 $sudoPesterParam['OutputFormat'] = $OutputFormat789 790 $title = "Pester Sudo - $TagSet"791 if ($TitlePrefix) {792 $title = "$TitlePrefix - $title"793 }794 $pesterPassThruSudoObject = Start-PSPester @sudoPesterParam -Title $title795 796 # Running tests that require sudo, with specified experimental features enabled797 $sudoResultsWithExpFeatures = @()798 foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) {799 $featureName = $entry.Key800 $testFiles = $entry.Value801 802 $expFeatureTestResultFile = "$PWD\TestResultsSudo.$featureName.xml"803 $sudoPesterParam['OutputFile'] = $expFeatureTestResultFile804 $sudoPesterParam['ExperimentalFeatureName'] = $featureName805 if ($testFiles.Count -eq 0)806 {807 # If an empty array is specified for the feature name, we run all tests with the feature enabled.808 # This allows us to prevent regressions to a critical engine experimental feature.809 $sudoPesterParam.Remove('Path')810 }811 else812 {813 # If a non-empty string or array is specified for the feature name, we only run those test files.814 $sudoPesterParam['Path'] = $testFiles815 }816 817 $title = "Pester Experimental Sudo - $featureName - $TagSet"818 if ($TitlePrefix) {819 $title = "$TitlePrefix - $title"820 }821 822 # We just built the test tools for the main test run, we don't need to rebuild them823 $passThruResult = Start-PSPester @sudoPesterParam -Title $title -SkipTestToolBuild824 825 $sudoResultsWithExpFeatures += $passThruResult826 }827 }828 829 # Determine whether the build passed830 try {831 $allTestResultsWithNoExpFeature = @($pesterPassThruNoSudoObject, $pesterPassThruSudoObject)832 $allTestResultsWithExpFeatures = @($noSudoResultsWithExpFeatures, $sudoResultsWithExpFeatures)833 # This throws if there was an error:834 $allTestResultsWithNoExpFeature | Where-Object {$null -ne $_} | ForEach-Object { Test-PSPesterResults -ResultObject $_ }835 $allTestResultsWithExpFeatures | Where-Object {$null -ne $_} | ForEach-Object { Test-PSPesterResults -ResultObject $_ -CanHaveNoResult }836 $result = "PASS"837 } catch {838 # The build failed, set the result:839 $resultError = $_840 $result = "FAIL"841 }842 843 # If the tests did not pass, throw the reason why844 if ( $result -eq "FAIL" )845 {846 Write-Warning "Tests failed. See the issue below."847 Throw $resultError848 }849 else850 {851 Write-Verbose "Tests did not fail! Nice job!"852 }853}854 855function New-LinuxPackage856{857 858 $isFullBuild = Test-DailyBuild859 $releaseTag = Get-ReleaseTag860 $packageParams = @{}861 $packageParams += @{ReleaseTag=$releaseTag}862 863 # Only build packages for PowerShell/PowerShell repository864 # branches, not pull requests865 $packages = @(Start-PSPackage @packageParams -SkipReleaseChecks -Type deb, rpm, rpm-fxdependent-arm64, tar)866 foreach($package in $packages)867 {868 if (Test-Path $package)869 {870 Write-Log "Package found: $package"871 }872 else873 {874 Write-Error -Message "Package NOT found: $package"875 }876 877 if ($package -isnot [System.IO.FileInfo])878 {879 $packageObj = Get-Item $package880 Write-Error -Message "The PACKAGE is not a FileInfo object"881 }882 else883 {884 $packageObj = $package885 }886 887 # Determine artifacts directory (GitHub Actions or Azure DevOps)888 $artifactsDir = if ($env:GITHUB_ACTIONS -eq 'true') {889 "${env:GITHUB_WORKSPACE}/../packages"890 } else {891 "${env:BUILD_ARTIFACTSTAGINGDIRECTORY}"892 }893 894 # Ensure artifacts directory exists895 if (-not (Test-Path $artifactsDir)) {896 New-Item -ItemType Directory -Path $artifactsDir -Force | Out-Null897 }898 899 Write-Log -message "Artifacts directory: $artifactsDir"900 Copy-Item $packageObj.FullName -Destination $artifactsDir -Force901 }902 903 if ($IsLinux)904 {905 # Determine artifacts directory (GitHub Actions or Azure DevOps)906 $artifactsDir = if ($env:GITHUB_ACTIONS -eq 'true') {907 "${env:GITHUB_WORKSPACE}/../packages"908 } else {909 "${env:BUILD_ARTIFACTSTAGINGDIRECTORY}"910 }911 912 # Create and package Raspbian .tgz913 # Build must be clean for Raspbian914 Start-PSBuild -PSModuleRestore -Clean -Runtime linux-arm -Configuration 'Release'915 $armPackage = Start-PSPackage @packageParams -Type tar-arm -SkipReleaseChecks916 Copy-Item $armPackage -Destination $artifactsDir -Force917 }918}919 920function Invoke-InitializeContainerStage {921 param(922 [string]923 $ContainerPattern = '.'924 )925 926 Write-Verbose "Invoking InitializeContainerStage with ContainerPattern: ${ContainerPattern}" -Verbose927 928 $fallbackSeed = (get-date).DayOfYear929 Write-Verbose "Fall back seed: $fallbackSeed" -Verbose930 931 # For PRs set the seed to the PR number so that the image is always the same932 $seed = $env:SYSTEM_PULLREQUEST_PULLREQUESTID933 934 if(!$seed) {935 # for non-PRs use the integer identifier of the build as the seed.936 $seed = $fallbackSeed937 }938 939 # cut down to 32 bits and keep the most varying parts, which are lower bits940 if ($seed -ge [Int32]::MaxValue) {941 $seed = [int]($seed -band [int]::MaxValue)942 }943 944 Write-Verbose "Seed: $seed" -Verbose945 946 # Get the latest image matrix JSON for preview947 $matrix = ./PowerShell-Docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json948 949 # Chose images that are validated or validating, Linux and can be used in CI.950 $linuxImages = $matrix.preview |951 Where-Object {$_.IsLinux -and $_.UseInCi -and $_.DistributionState -match 'Validat.*' -and $_.JobName -match $ContainerPattern -and $_.JobName -notlike "*arm*"} |952 Select-Object JobName, Taglist |953 Sort-Object -property JobName954 955 # Use the selected seed to pick a container956 $index = Get-Random -Minimum 0 -Maximum $linuxImages.Count -SetSeed $seed957 $selectedImage = $linuxImages[$index]958 959 # Filter to the first test-deps compatible tag960 $tag = $selectedImage.Taglist -split ';' | Where-Object {$_ -match 'preview-\D+'} | Select-Object -First 1961 962 # Calculate the container name963 $containerName = "mcr.microsoft.com/powershell/test-deps:$tag"964 965 Set-BuildVariable -Name containerName -Value $containerName -IsOutput966 Set-BuildVariable -Name containerBuildName -Value $selectedImage.JobName -IsOutput967 968 if($env:BUILD_REASON -eq 'PullRequest') {969 Write-Host "##vso[build.updatebuildnumber]PR-${env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER}-$($selectedImage.JobName)-$((get-date).ToString("yyyyMMddhhmmss"))"970 } else {971 Write-Host "##vso[build.updatebuildnumber]${env:BUILD_SOURCEBRANCHNAME}-$($selectedImage.JobName)-${env:BUILD_SOURCEVERSION}-$((get-date).ToString("yyyyMMddhhmmss"))"972 973 # Cannot do this for a PR974 Write-Host "##vso[build.addbuildtag]$($selectedImage.JobName)"975 }976}977 978Function Test-MergeConflictMarker979{980 <#981 .SYNOPSIS982 Checks files for Git merge conflict markers and outputs results for GitHub Actions.983 .DESCRIPTION984 Scans the specified files for Git merge conflict markers (<<<<<<<, =======, >>>>>>>)985 and generates console output, GitHub Actions outputs, and job summary.986 Designed for use in GitHub Actions workflows.987 .PARAMETER File988 Array of file paths (relative or absolute) to check for merge conflict markers.989 .PARAMETER WorkspacePath990 Base workspace path for resolving relative paths. Defaults to current directory.991 .PARAMETER OutputPath992 Path to write GitHub Actions outputs. Defaults to $env:GITHUB_OUTPUT.993 .PARAMETER SummaryPath994 Path to write GitHub Actions job summary. Defaults to $env:GITHUB_STEP_SUMMARY.995 .EXAMPLE996 Test-MergeConflictMarker -File @('file1.txt', 'file2.cs') -WorkspacePath $env:GITHUB_WORKSPACE997 #>998 [CmdletBinding()]999 param(1000 [Parameter()]1001 [AllowEmptyCollection()]1002 [string[]] $File = @(),1003 1004 [Parameter()]1005 [string] $WorkspacePath = $PWD,1006 1007 [Parameter()]1008 [string] $OutputPath = $env:GITHUB_OUTPUT,1009 1010 [Parameter()]1011 [string] $SummaryPath = $env:GITHUB_STEP_SUMMARY1012 )1013 1014 Write-Host "Starting merge conflict marker check..." -ForegroundColor Cyan1015 1016 # Helper function to write outputs when no files to check1017 function Write-NoFilesOutput {1018 param(1019 [string]$Message,1020 [string]$OutputPath,1021 [string]$SummaryPath1022 )1023 1024 # Output results to GitHub Actions1025 if ($OutputPath) {1026 "files-checked=0" | Out-File -FilePath $OutputPath -Append -Encoding utf81027 "conflicts-found=0" | Out-File -FilePath $OutputPath -Append -Encoding utf81028 }1029 1030 # Create GitHub Actions job summary1031 if ($SummaryPath) {1032 $summaryContent = @"1033# Merge Conflict Marker Check Results1034 1035## Summary1036- **Files Checked:** 01037- **Files with Conflicts:** 01038 1039## ℹ️ No Files to Check1040 1041$Message1042 1043"@1044 $summaryContent | Out-File -FilePath $SummaryPath -Encoding utf81045 }1046 }1047 1048 # Handle empty file list (e.g., when PR only deletes files)1049 if ($File.Count -eq 0) {1050 Write-Host "No files to check (empty file list)" -ForegroundColor Yellow1051 Write-NoFilesOutput -Message "No files were provided for checking (this can happen when a PR only deletes files)." -OutputPath $OutputPath -SummaryPath $SummaryPath1052 return1053 }1054 1055 # Filter out *.cs files from merge conflict checking1056 $filesToCheck = @($File | Where-Object { $_ -notlike "*.cs" })1057 $filteredCount = $File.Count - $filesToCheck.Count1058 1059 if ($filteredCount -gt 0) {1060 Write-Host "Filtered out $filteredCount *.cs file(s) from merge conflict checking" -ForegroundColor Yellow1061 }1062 1063 if ($filesToCheck.Count -eq 0) {1064 Write-Host "No files to check after filtering (all files were *.cs)" -ForegroundColor Yellow1065 Write-NoFilesOutput -Message "All $filteredCount file(s) were filtered out (*.cs files are excluded from merge conflict checking)." -OutputPath $OutputPath -SummaryPath $SummaryPath1066 return1067 }1068 1069 Write-Host "Checking $($filesToCheck.Count) changed files for merge conflict markers" -ForegroundColor Cyan1070 1071 # Convert relative paths to absolute paths for processing1072 $absolutePaths = $filesToCheck | ForEach-Object {1073 if ([System.IO.Path]::IsPathRooted($_)) {1074 $_1075 } else {1076 Join-Path $WorkspacePath $_1077 }1078 }1079 1080 $filesWithConflicts = @()1081 $filesChecked = 01082 1083 foreach ($filePath in $absolutePaths) {1084 # Check if file exists (might be deleted)1085 if (-not (Test-Path $filePath)) {1086 Write-Verbose " Skipping deleted file: $filePath"1087 continue1088 }1089 1090 # Skip binary files and directories1091 if ((Get-Item $filePath) -is [System.IO.DirectoryInfo]) {1092 continue1093 }1094 1095 $filesChecked++1096 1097 # Get relative path for display1098 $relativePath = if ($WorkspacePath -and $filePath.StartsWith($WorkspacePath)) {1099 $filePath.Substring($WorkspacePath.Length).TrimStart([System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar)1100 } else {1101 $filePath1102 }1103 1104 Write-Host " Checking: $relativePath" -ForegroundColor Gray1105 1106 # Search for conflict markers using Select-String1107 try {1108 # Git conflict markers are 7 characters followed by a space or end of line1109 # Regex pattern breakdown:1110 # ^ - Matches the start of a line1111 # (<{7}|={7}|>{7}) - Matches exactly 7 consecutive '<', '=', or '>' characters (Git conflict markers)1112 # (\s|$) - Ensures the marker is followed by whitespace or end of line1113 $pattern = '^(<{7}|={7}|>{7})(\s|$)'1114 $matchedLines = Select-String -Path $filePath -Pattern $pattern -AllMatches -ErrorAction Stop1115 1116 if ($matchedLines) {1117 # Collect marker details with line numbers (Select-String provides LineNumber automatically)1118 $markerDetails = @()1119 1120 foreach ($match in $matchedLines) {1121 $markerDetails += [PSCustomObject]@{1122 Marker = $match.Matches[0].Groups[1].Value1123 Line = $match.LineNumber1124 }1125 }1126 1127 $filesWithConflicts += [PSCustomObject]@{1128 File = $relativePath1129 MarkerDetails = $markerDetails1130 }1131 1132 Write-Host " ❌ CONFLICT MARKERS FOUND in $relativePath" -ForegroundColor Red1133 foreach ($detail in $markerDetails) {1134 Write-Host " Line $($detail.Line): $($detail.Marker)" -ForegroundColor Red1135 }1136 }1137 }1138 catch {1139 # Skip files that can't be read (likely binary)1140 Write-Verbose " Skipping unreadable file: $relativePath"1141 }1142 }1143 1144 # Output results to GitHub Actions1145 if ($OutputPath) {1146 "files-checked=$filesChecked" | Out-File -FilePath $OutputPath -Append -Encoding utf81147 "conflicts-found=$($filesWithConflicts.Count)" | Out-File -FilePath $OutputPath -Append -Encoding utf81148 }1149 1150 Write-Host "`nSummary:" -ForegroundColor Cyan1151 Write-Host " Files checked: $filesChecked" -ForegroundColor Cyan1152 Write-Host " Files with conflicts: $($filesWithConflicts.Count)" -ForegroundColor Cyan1153 1154 # Create GitHub Actions job summary1155 if ($SummaryPath) {1156 $summaryContent = @"1157# Merge Conflict Marker Check Results1158 1159## Summary1160- **Files Checked:** $filesChecked1161- **Files with Conflicts:** $($filesWithConflicts.Count)1162 1163"@1164 1165 if ($filesWithConflicts.Count -gt 0) {1166 Write-Host "`n❌ Merge conflict markers detected in the following files:" -ForegroundColor Red1167 1168 $summaryContent += "`n## ❌ Conflicts Detected`n`n"1169 $summaryContent += "The following files contain merge conflict markers:`n`n"1170 1171 foreach ($fileInfo in $filesWithConflicts) {1172 Write-Host " - $($fileInfo.File)" -ForegroundColor Red1173 1174 $summaryContent += "### 📄 ``$($fileInfo.File)```n`n"1175 $summaryContent += "| Line | Marker |`n"1176 $summaryContent += "|------|--------|`n"1177 1178 foreach ($detail in $fileInfo.MarkerDetails) {1179 Write-Host " Line $($detail.Line): $($detail.Marker)" -ForegroundColor Red1180 $summaryContent += "| $($detail.Line) | ``$($detail.Marker)`` |`n"1181 }1182 $summaryContent += "`n"1183 }1184 1185 $summaryContent += "`n**Action Required:** Please resolve these conflicts before merging.`n"1186 Write-Host "`nPlease resolve these conflicts before merging." -ForegroundColor Red1187 } else {1188 Write-Host "`n✅ No merge conflict markers found" -ForegroundColor Green1189 $summaryContent += "`n## ✅ No Conflicts Found`n`nAll checked files are free of merge conflict markers.`n"1190 }1191 1192 $summaryContent | Out-File -FilePath $SummaryPath -Encoding utf81193 }1194 1195 # Exit with error if conflicts found1196 if ($filesWithConflicts.Count -gt 0) {1197 throw "Merge conflict markers detected in $($filesWithConflicts.Count) file(s)"1198 }1199}1200