MegaBites-AI/Windows-powershell

1# Copyright (c) Microsoft Corporation.2# Licensed under the MIT License.3 4$repoRoot = git rev-parse --show-toplevel5Import-Module "$repoRoot/build.psm1"6 7function Start-Benchmarking8{9    <#10    .SYNOPSIS11    Start a benchmark run.12 13    .PARAMETER TargetPSVersion14    The version of 'Microsoft.PowerShell.SDK' package that we want the benchmark to target.15    The supported versions are 7.0.x and above, including preview versions.16 17    .PARAMETER TargetFramework18    The target framework to run benchmarks against.19 20    .PARAMETER List21    List the available benchmarks, in either 'flat' or 'tree' views.22 23    .PARAMETER Runtime24    Run benchmarks against multiple .NET runtimes.25 26    .PARAMETER Filter27    One or more wildcard patterns to filter the benchmarks to be executed or to be listed.28 29    .PARAMETER Artifacts30    Path to the folder where you want to store the artifacts produced from running benchmarks.31 32    .PARAMETER KeepFiles33    Indicates to keep all temporary files produced for running benchmarks.34    #>35    [CmdletBinding(DefaultParameterSetName = 'TargetFramework')]36    param(37        [Parameter(ParameterSetName = 'TargetPSVersion')]38        [ValidatePattern(39            '^7\.(0|1|2)\.\d+(-preview\.\d{1,2})?$',40            ErrorMessage = 'The package version is invalid or not supported')]41        [string] $TargetPSVersion,42 43        [Parameter(ParameterSetName = 'TargetFramework')]44        [ValidateSet('netcoreapp3.1', 'net5.0', 'net6.0')]45        [string] $TargetFramework = 'net6.0',46 47        [Parameter(ParameterSetName = 'TargetFramework')]48        [ValidateSet('flat', 'tree')]49        [string] $List,50 51        [Parameter(Mandatory, ParameterSetName = 'Runtimes')]52        [ValidateSet('netcoreapp3.1', 'net5.0', 'net6.0')]53        [string[]] $Runtime,54 55        [string[]] $Filter = '*',56        [string] $Artifacts,57        [switch] $KeepFiles58    )59 60    Begin {61        Find-Dotnet62 63        if ($Artifacts) {64            $Artifacts = $PSCmdlet.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Artifacts)65        } else {66            $Artifacts = Join-Path $PSScriptRoot 'BenchmarkDotNet.Artifacts'67        }68 69        if (Test-Path -Path $Artifacts) {70            Remove-Item -Path $Artifacts -Recurse -Force -ErrorAction Stop71        }72 73        if ($Runtime) {74            ## Remove duplicate values.75            $hash = [ordered]@{}76            foreach ($item in $Runtime) {77                if (-not $hash.Contains($item)) {78                    $hash.Add($item, $null)79                }80            }81            $Runtime = $hash.Keys82        }83    }84 85    End {86        try {87            Push-Location -Path "$PSScriptRoot/benchmarks"88            $savedOFS = $OFS; $OFS = $null89 90            ## Aggregate BDN arguments.91            $runArgs = @('--filter')92            foreach ($entry in $Filter) { $runArgs += $entry }93            $runArgs += '--artifacts', $Artifacts94            $runArgs += '--envVars', 'POWERSHELL_TELEMETRY_OPTOUT:1'95 96            if ($List) { $runArgs += '--list', $List }97            if ($KeepFiles) { $runArgs += "--keepFiles" }98 99            switch ($PSCmdlet.ParameterSetName) {100                'TargetPSVersion' {101                    Write-Log -message "Run benchmarks targeting '$TargetFramework' and the 'Microsoft.PowerShell.SDK' version '$TargetPSVersion' ..."102                    $env:PERF_TARGET_VERSION = $TargetPSVersion103 104                    ## Use 'Release' instead of 'release' (note the capital case) because BDN uses 'Release' when building the auto-generated105                    ## project, and MSBuild somehow recognizes 'release' and 'Release' as two different configurations and thus will rebuild106                    ## all dependencies unnecessarily.107                    dotnet run -c Release -f $TargetFramework $runArgs108                }109 110                'TargetFramework' {111                    $message = if ($TargetFramework -eq 'net6.0') { 'the current PowerShell code base ...' } else { "the corresponding latest version of 'Microsoft.PowerShell.SDK' ..." }112                    Write-Log -message "Run benchmarks targeting '$TargetFramework' and $message"113 114                    ## Use 'Release' instead of 'release' (note the capital case) because BDN uses 'Release' when building the auto-generated115                    ## project, and MSBuild somehow recognizes 'release' and 'Release' as two different configurations and thus will rebuild116                    ## all dependencies unnecessarily.117                    dotnet run -c Release -f $TargetFramework $runArgs118                }119 120                'Runtimes' {121                    Write-Log -message "Run benchmarks targeting multiple .NET runtimes: $Runtime ..."122 123                    ## Use 'Release' instead of 'release' (note the capital case) because BDN uses 'Release' when building the auto-generated124                    ## project, and MSBuild somehow recognizes 'release' and 'Release' as two different configurations and thus will rebuild125                    ## all dependencies unnecessarily.126                    dotnet run -c Release -f net6.0 --runtimes $Runtime $runArgs127                }128            }129 130            if (Test-Path $Artifacts) {131                Write-Log -message "`nBenchmark artifacts can be found at $Artifacts"132            }133        }134        finally {135            $OFS = $savedOFS136            $env:PERF_TARGET_VERSION = $null137            Pop-Location138        }139    }140}141 142function Compare-BenchmarkResult143{144    <#145    .SYNOPSIS146    Compare two benchmark run results to find possible regressions.147 148    When running benchmarks with 'Start-Benchmarking', you can define the result folder149    where to save the artifacts by specifying '-Artifacts'.150 151    To compare two benchmark run results, you need to specify the result folder paths152    for both runs, one as the base and one as the diff.153 154    .PARAMETER BaseResultPath155    Path to the benchmark result used as baseline.156 157    .PARAMETER DiffResultPath158    Path to the benchmark result to be compared with the baseline.159 160    .PARAMETER Threshold161    Threshold for Statistical Test. Examples: 5%, 10ms, 100ns, 1s162 163    .PARAMETER Noise164    Noise threshold for Statistical Test.165    The difference for 1.0ns and 1.1ns is 10%, but it's really just noise. Examples: 0.5ns 1ns.166    The default value is 0.3ns.167 168    .PARAMETER Top169    Filter the diff to top `N` results170    #>171    param(172        [Parameter(Mandatory)]173        [string] $BaseResultPath,174 175        [Parameter(Mandatory)]176        [string] $DiffResultPath,177 178        [Parameter(Mandatory)]179        [ValidatePattern('^\d{1,2}%$|^\d+(ms|ns|s)$')]180        [string] $Threshold,181 182        [ValidatePattern('^(\d\.)?\d+(ms|ns|s)$')]183        [string] $Noise,184 185        [ValidateRange(1, 100)]186        [int] $Top187    )188 189    Find-Dotnet190 191    try {192        Push-Location -Path "$PSScriptRoot/dotnet-tools/ResultsComparer"193        $savedOFS = $OFS; $OFS = $null194 195        $runArgs = @()196        if ($Noise) { $runArgs += "--noise $Noise" }197        if ($Top -gt 0) { $runArgs += "--top $Top" }198 199        dotnet run -c Release --base $BaseResultPath --diff $DiffResultPath --threshold $Threshold "$runArgs"200    }201    finally {202        $OFS = $savedOFS203        Pop-Location204    }205}206 207Export-ModuleMember -Function 'Start-Benchmarking', 'Compare-BenchmarkResult'208