MegaBites-AI/Windows-powershell

1// Copyright (c) Microsoft Corporation.2// Licensed under the MIT License.3 4using System.Buffers;5using System.Collections;6using System.Collections.Concurrent;7using System.Collections.Generic;8using System.Diagnostics;9using System.Globalization;10using System.IO;11using System.Management.Automation.Internal;12using System.Management.Automation.Language;13using System.Management.Automation.Runspaces;14using System.Text;15using System.Threading;16using System.Threading.Tasks;17 18using Microsoft.PowerShell.Commands;19 20namespace System.Management.Automation21{22    /// <summary>23    /// Class to manage the caching of analysis data.24    /// For performance, module command caching is flattened after discovery. Many modules have nested25    /// modules that can only be resolved at runtime - for example,26    /// script modules that declare: $env:PATH += "; $psScriptRoot". When27    /// doing initial analysis, we include these in 'ExportedCommands'.28    /// Changes to these type of modules will not be re-analyzed, unless the user re-imports the module,29    /// or runs Get-Module -List.30    /// </summary>31    internal static class AnalysisCache32    {33        private static readonly AnalysisCacheData s_cacheData = AnalysisCacheData.Get();34 35        // This dictionary shouldn't see much use, so low concurrency and capacity36        private static readonly ConcurrentDictionary<string, string> s_modulesBeingAnalyzed =37            new(concurrencyLevel: 1, capacity: 2, StringComparer.OrdinalIgnoreCase);38 39        internal static readonly SearchValues<char> InvalidCommandNameCharacters = SearchValues.Create("#,(){}[]&/\\$^;:\"'<>|?@`*%+=~");40 41        internal static bool ContainsInvalidCommandNameCharacters(ReadOnlySpan<char> text)42            => text.ContainsAny(InvalidCommandNameCharacters);43 44        internal static ConcurrentDictionary<string, CommandTypes> GetExportedCommands(string modulePath, bool testOnly, ExecutionContext context)45        {46            bool etwEnabled = CommandDiscoveryEventSource.Log.IsEnabled();47            if (etwEnabled) CommandDiscoveryEventSource.Log.GetModuleExportedCommandsStart(modulePath);48 49            DateTime lastWriteTime;50            ModuleCacheEntry moduleCacheEntry;51            if (GetModuleEntryFromCache(modulePath, out lastWriteTime, out moduleCacheEntry))52            {53                if (etwEnabled) CommandDiscoveryEventSource.Log.GetModuleExportedCommandsStop(modulePath);54                return moduleCacheEntry.Commands;55            }56 57            ConcurrentDictionary<string, CommandTypes> result = null;58 59            if (!testOnly)60            {61                var extension = Path.GetExtension(modulePath);62                if (extension.Equals(StringLiterals.PowerShellDataFileExtension, StringComparison.OrdinalIgnoreCase))63                {64                    result = AnalyzeManifestModule(modulePath, context, lastWriteTime, etwEnabled);65                }66                else if (extension.Equals(StringLiterals.PowerShellModuleFileExtension, StringComparison.OrdinalIgnoreCase))67                {68                    result = AnalyzeScriptModule(modulePath, context, lastWriteTime);69                }70                else if (extension.Equals(StringLiterals.PowerShellCmdletizationFileExtension, StringComparison.OrdinalIgnoreCase))71                {72                    result = AnalyzeCdxmlModule(modulePath, context, lastWriteTime);73                }74                else if (extension.Equals(StringLiterals.PowerShellILAssemblyExtension, StringComparison.OrdinalIgnoreCase))75                {76                    result = AnalyzeDllModule(modulePath, context, lastWriteTime);77                }78                else if (extension.Equals(StringLiterals.PowerShellILExecutableExtension, StringComparison.OrdinalIgnoreCase))79                {80                    result = AnalyzeDllModule(modulePath, context, lastWriteTime);81                }82            }83 84            if (result != null)85            {86                s_cacheData.QueueSerialization();87                ModuleIntrinsics.Tracer.WriteLine("Returning {0} exported commands.", result.Count);88            }89            else90            {91                ModuleIntrinsics.Tracer.WriteLine("Returning NULL for exported commands.");92            }93 94            if (etwEnabled) CommandDiscoveryEventSource.Log.GetModuleExportedCommandsStop(modulePath);95            return result;96        }97 98        private static ConcurrentDictionary<string, CommandTypes> AnalyzeManifestModule(string modulePath, ExecutionContext context, DateTime lastWriteTime, bool etwEnabled)99        {100            ConcurrentDictionary<string, CommandTypes> result = null;101            try102            {103                var moduleManifestProperties = PsUtils.GetModuleManifestProperties(modulePath, PsUtils.FastModuleManifestAnalysisPropertyNames);104                if (moduleManifestProperties != null)105                {106                    if (!Configuration.PowerShellConfig.Instance.IsImplicitWinCompatEnabled() && ModuleIsEditionIncompatible(modulePath, moduleManifestProperties))107                    {108                        ModuleIntrinsics.Tracer.WriteLine($"Module lies on the Windows System32 legacy module path and is incompatible with current PowerShell edition, skipping module: {modulePath}");109                        return null;110                    }111 112                    Version version;113                    if (ModuleUtils.IsModuleInVersionSubdirectory(modulePath, out version))114                    {115                        var versionInManifest = LanguagePrimitives.ConvertTo<Version>(moduleManifestProperties["ModuleVersion"]);116                        if (version != versionInManifest)117                        {118                            ModuleIntrinsics.Tracer.WriteLine("ModuleVersion in manifest does not match versioned module directory, skipping module: {0}", modulePath);119                            return null;120                        }121                    }122 123                    result = new ConcurrentDictionary<string, CommandTypes>(3, moduleManifestProperties.Count, StringComparer.OrdinalIgnoreCase);124 125                    var sawWildcard = false;126                    var hadCmdlets = AddPsd1EntryToResult(result, moduleManifestProperties["CmdletsToExport"], CommandTypes.Cmdlet, ref sawWildcard);127                    var hadFunctions = AddPsd1EntryToResult(result, moduleManifestProperties["FunctionsToExport"], CommandTypes.Function, ref sawWildcard);128                    var hadAliases = AddPsd1EntryToResult(result, moduleManifestProperties["AliasesToExport"], CommandTypes.Alias, ref sawWildcard);129 130                    var analysisSucceeded = hadCmdlets && hadFunctions && hadAliases;131 132                    if (!analysisSucceeded && !sawWildcard && (hadCmdlets || hadFunctions))133                    {134                        // If we're missing CmdletsToExport, that might still be OK, but only if we have a script module.135                        // Likewise, if we're missing FunctionsToExport, that might be OK, but only if we have a binary module.136 137                        analysisSucceeded = !CheckModulesTypesInManifestAgainstExportedCommands(moduleManifestProperties, hadCmdlets, hadFunctions, hadAliases);138                    }139 140                    if (analysisSucceeded)141                    {142                        var moduleCacheEntry = new ModuleCacheEntry143                        {144                            ModulePath = modulePath,145                            LastWriteTime = lastWriteTime,146                            Commands = result,147                            TypesAnalyzed = false,148                            Types = new ConcurrentDictionary<string, TypeAttributes>(1, 8, StringComparer.OrdinalIgnoreCase)149                        };150                        s_cacheData.Entries[modulePath] = moduleCacheEntry;151                    }152                    else153                    {154                        result = null;155                    }156                }157            }158            catch (Exception e)159            {160                if (etwEnabled) CommandDiscoveryEventSource.Log.ModuleManifestAnalysisException(modulePath, e.Message);161                // Ignore the errors, proceed with the usual module analysis162                ModuleIntrinsics.Tracer.WriteLine("Exception on fast-path analysis of module {0}", modulePath);163            }164 165            if (etwEnabled) CommandDiscoveryEventSource.Log.ModuleManifestAnalysisResult(modulePath, result != null);166 167            return result ?? AnalyzeTheOldWay(modulePath, context, lastWriteTime);168        }169 170        /// <summary>171        /// Check if a module is compatible with the current PSEdition given its path and its manifest properties.172        /// </summary>173        /// <param name="modulePath">The path to the module.</param>174        /// <param name="moduleManifestProperties">The properties of the module's manifest.</param>175        /// <returns></returns>176        internal static bool ModuleIsEditionIncompatible(string modulePath, Hashtable moduleManifestProperties)177        {178#if UNIX179            return false;180#else181            if (!ModuleUtils.IsOnSystem32ModulePath(modulePath))182            {183                return false;184            }185 186            if (!moduleManifestProperties.ContainsKey("CompatiblePSEditions"))187            {188                return true;189            }190 191            return !Utils.IsPSEditionSupported(LanguagePrimitives.ConvertTo<string[]>(moduleManifestProperties["CompatiblePSEditions"]));192#endif193        }194 195        internal static bool ModuleAnalysisViaGetModuleRequired(object modulePathObj, bool hadCmdlets, bool hadFunctions, bool hadAliases)196        {197            if (modulePathObj is not string modulePath)198                return true;199 200            if (modulePath.EndsWith(StringLiterals.PowerShellModuleFileExtension, StringComparison.OrdinalIgnoreCase))201            {202                // A script module can't exactly define cmdlets, but it can import a binary module (as nested), so203                // it can indirectly define cmdlets.  And obviously a script module can define functions and aliases.204                // If we got here, one of those is missing, so analysis is required.205                return true;206            }207 208            if (modulePath.EndsWith(StringLiterals.PowerShellCmdletizationFileExtension, StringComparison.OrdinalIgnoreCase))209            {210                // A cdxml module can only define functions and aliases, so if we have both, no more analysis is required.211                return !hadFunctions || !hadAliases;212            }213 214            if (modulePath.EndsWith(StringLiterals.PowerShellILAssemblyExtension, StringComparison.OrdinalIgnoreCase))215            {216                // A dll just exports cmdlets, so if the manifest doesn't explicitly export any cmdlets,217                // more analysis is required. If the module exports aliases, we can't discover that analyzing218                // the binary, so aliases are always required to be explicit (no wildcards) in the manifest.219                return !hadCmdlets;220            }221 222            if (modulePath.EndsWith(StringLiterals.PowerShellILExecutableExtension, StringComparison.OrdinalIgnoreCase))223            {224                // A dll just exports cmdlets, so if the manifest doesn't explicitly export any cmdlets,225                // more analysis is required. If the module exports aliases, we can't discover that analyzing226                // the binary, so aliases are always required to be explicit (no wildcards) in the manifest.227                return !hadCmdlets;228            }229 230            // Any other extension (or no extension), just assume the worst and analyze the module231            return true;232        }233 234        // Returns true if we need to analyze the manifest module in Get-Module because235        // our quick and dirty module manifest analysis is missing something not easily236        // discovered.237        //238        // TODO - psm1 modules are actually easily handled, so if we only saw a psm1 here,239        // we should just analyze it and not fall back on Get-Module -List.240        private static bool CheckModulesTypesInManifestAgainstExportedCommands(Hashtable moduleManifestProperties, bool hadCmdlets, bool hadFunctions, bool hadAliases)241        {242            var rootModule = moduleManifestProperties["RootModule"];243            if (rootModule != null && ModuleAnalysisViaGetModuleRequired(rootModule, hadCmdlets, hadFunctions, hadAliases))244                return true;245 246            var moduleToProcess = moduleManifestProperties["ModuleToProcess"];247            if (moduleToProcess != null && ModuleAnalysisViaGetModuleRequired(moduleToProcess, hadCmdlets, hadFunctions, hadAliases))248                return true;249 250            var nestedModules = moduleManifestProperties["NestedModules"];251            if (nestedModules != null)252            {253                var nestedModule = nestedModules as string;254                if (nestedModule != null)255                {256                    return ModuleAnalysisViaGetModuleRequired(nestedModule, hadCmdlets, hadFunctions, hadAliases);257                }258 259                if (nestedModules is not object[] nestedModuleArray)260                    return true;261 262                foreach (var element in nestedModuleArray)263                {264                    if (ModuleAnalysisViaGetModuleRequired(element, hadCmdlets, hadFunctions, hadAliases))265                        return true;266                }267            }268 269            return false;270        }271 272        private static bool AddPsd1EntryToResult(ConcurrentDictionary<string, CommandTypes> result, string command, CommandTypes commandTypeToAdd, ref bool sawWildcard)273        {274            if (WildcardPattern.ContainsWildcardCharacters(command))275            {276                sawWildcard = true;277                return false;278            }279 280            // An empty string is one way of saying "no exported commands".281            if (command.Length != 0)282            {283                CommandTypes commandTypes;284                if (result.TryGetValue(command, out commandTypes))285                {286                    commandTypes |= commandTypeToAdd;287                }288                else289                {290                    commandTypes = commandTypeToAdd;291                }292 293                result[command] = commandTypes;294            }295 296            return true;297        }298 299        private static bool AddPsd1EntryToResult(ConcurrentDictionary<string, CommandTypes> result, object value, CommandTypes commandTypeToAdd, ref bool sawWildcard)300        {301            string command = value as string;302            if (command != null)303            {304                return AddPsd1EntryToResult(result, command, commandTypeToAdd, ref sawWildcard);305            }306 307            object[] commands = value as object[];308            if (commands != null)309            {310                foreach (var o in commands)311                {312                    if (!AddPsd1EntryToResult(result, o, commandTypeToAdd, ref sawWildcard))313                        return false;314                }315 316                // An empty array is still success, that's how a manifest declares that317                // no entries are exported (unlike the lack of an entry, or $null).318                return true;319            }320 321            // Unknown type, let Get-Module -List deal with this manifest322            return false;323        }324 325        private static ConcurrentDictionary<string, CommandTypes> AnalyzeScriptModule(string modulePath, ExecutionContext context, DateTime lastWriteTime)326        {327            var scriptAnalysis = ScriptAnalysis.Analyze(modulePath, context);328            if (scriptAnalysis == null)329            {330                return null;331            }332 333            List<WildcardPattern> scriptAnalysisPatterns = new List<WildcardPattern>();334            foreach (string discoveredCommandFilter in scriptAnalysis.DiscoveredCommandFilters)335            {336                scriptAnalysisPatterns.Add(new WildcardPattern(discoveredCommandFilter));337            }338 339            var result = new ConcurrentDictionary<string, CommandTypes>(3,340                scriptAnalysis.DiscoveredExports.Count + scriptAnalysis.DiscoveredAliases.Count,341                StringComparer.OrdinalIgnoreCase);342 343            // Add any directly discovered exports344            foreach (var command in scriptAnalysis.DiscoveredExports)345            {346                if (SessionStateUtilities.MatchesAnyWildcardPattern(command, scriptAnalysisPatterns, true))347                {348                    if (!ContainsInvalidCommandNameCharacters(command))349                    {350                        result[command] = CommandTypes.Function;351                    }352                }353            }354 355            // Add the discovered aliases356            foreach (var pair in scriptAnalysis.DiscoveredAliases)357            {358                var commandName = pair.Key;359                // These are already filtered360                if (!ContainsInvalidCommandNameCharacters(commandName))361                {362                    result.AddOrUpdate(commandName, CommandTypes.Alias,363                        static (_, existingCommandType) => existingCommandType | CommandTypes.Alias);364                }365            }366 367            // Add any files in PsScriptRoot if it added itself to the path368            if (scriptAnalysis.AddsSelfToPath)369            {370                string baseDirectory = Path.GetDirectoryName(modulePath);371 372                try373                {374                    foreach (string item in Directory.EnumerateFiles(baseDirectory, "*.ps1"))375                    {376                        var command = Path.GetFileNameWithoutExtension(item);377                        result.AddOrUpdate(command, CommandTypes.ExternalScript,378                            static (_, existingCommandType) => existingCommandType | CommandTypes.ExternalScript);379                    }380                }381                catch (UnauthorizedAccessException)382                {383                    // Consume this exception here384                }385            }386 387            ConcurrentDictionary<string, TypeAttributes> exportedClasses = new(388                concurrencyLevel: 1,389                capacity: scriptAnalysis.DiscoveredClasses.Count,390                StringComparer.OrdinalIgnoreCase);391            foreach (var exportedClass in scriptAnalysis.DiscoveredClasses)392            {393                exportedClasses[exportedClass.Name] = exportedClass.TypeAttributes;394            }395 396            var moduleCacheEntry = new ModuleCacheEntry397            {398                ModulePath = modulePath,399                LastWriteTime = lastWriteTime,400                Commands = result,401                TypesAnalyzed = true,402                Types = exportedClasses403            };404            s_cacheData.Entries[modulePath] = moduleCacheEntry;405 406            return result;407        }408 409        private static ConcurrentDictionary<string, CommandTypes> AnalyzeCdxmlModule(string modulePath, ExecutionContext context, DateTime lastWriteTime)410        {411            return AnalyzeTheOldWay(modulePath, context, lastWriteTime);412        }413 414        private static ConcurrentDictionary<string, CommandTypes> AnalyzeDllModule(string modulePath, ExecutionContext context, DateTime lastWriteTime)415        {416            return AnalyzeTheOldWay(modulePath, context, lastWriteTime);417        }418 419        private static ConcurrentDictionary<string, CommandTypes> AnalyzeTheOldWay(string modulePath, ExecutionContext context, DateTime lastWriteTime)420        {421            try422            {423                // If we're already analyzing this module, let the recursion bottom out.424                if (!s_modulesBeingAnalyzed.TryAdd(modulePath, modulePath))425                {426                    ModuleIntrinsics.Tracer.WriteLine("{0} is already being analyzed. Exiting.", modulePath);427                    return null;428                }429 430                // Record that we're analyzing this specific module so that we don't get stuck in recursion431                ModuleIntrinsics.Tracer.WriteLine("Started analysis: {0}", modulePath);432                CallGetModuleDashList(context, modulePath);433 434                ModuleCacheEntry moduleCacheEntry;435                if (GetModuleEntryFromCache(modulePath, out lastWriteTime, out moduleCacheEntry))436                {437                    return moduleCacheEntry.Commands;438                }439            }440            catch (Exception e)441            {442                ModuleIntrinsics.Tracer.WriteLine("Module analysis generated an exception: {0}", e);443 444                // Catch-all OK, third-party call-out.445            }446            finally447            {448                ModuleIntrinsics.Tracer.WriteLine("Finished analysis: {0}", modulePath);449                s_modulesBeingAnalyzed.TryRemove(modulePath, out modulePath);450            }451 452            return null;453        }454 455        /// <summary>456        /// Return the exported types for a specific module.457        /// If the module is already cache, return from cache, else cache the module.458        /// Also re-cache the module if the cached item is stale.459        /// </summary>460        /// <param name="modulePath">Path to the module to get exported types from.</param>461        /// <param name="context">Current Context.</param>462        /// <returns></returns>463        internal static ConcurrentDictionary<string, TypeAttributes> GetExportedClasses(string modulePath, ExecutionContext context)464        {465            DateTime lastWriteTime;466            ModuleCacheEntry moduleCacheEntry;467            if (GetModuleEntryFromCache(modulePath, out lastWriteTime, out moduleCacheEntry) && moduleCacheEntry.TypesAnalyzed)468            {469                return moduleCacheEntry.Types;470            }471 472            try473            {474                CallGetModuleDashList(context, modulePath);475                if (GetModuleEntryFromCache(modulePath, out lastWriteTime, out moduleCacheEntry))476                {477                    return moduleCacheEntry.Types;478                }479            }480            catch (Exception e)481            {482                ModuleIntrinsics.Tracer.WriteLine("Module analysis generated an exception: {0}", e);483 484                // Catch-all OK, third-party call-out.485            }486 487            return null;488        }489 490        internal static void CacheModuleExports(PSModuleInfo module, ExecutionContext context)491        {492            ModuleIntrinsics.Tracer.WriteLine("Requested caching for {0}", module.Name);493 494            // Don't cache incompatible modules on the system32 module path even if loaded with495            // -SkipEditionCheck, since it will break subsequent sessions496            if (!Configuration.PowerShellConfig.Instance.IsImplicitWinCompatEnabled() && !module.IsConsideredEditionCompatible)497            {498                ModuleIntrinsics.Tracer.WriteLine($"Module '{module.Name}' not edition compatible and not cached.");499                return;500            }501 502            DateTime lastWriteTime;503            ModuleCacheEntry moduleCacheEntry;504            GetModuleEntryFromCache(module.Path, out lastWriteTime, out moduleCacheEntry);505 506            var realExportedCommands = module.ExportedCommands;507            var realExportedClasses = module.GetExportedTypeDefinitions();508            ConcurrentDictionary<string, CommandTypes> exportedCommands;509            ConcurrentDictionary<string, TypeAttributes> exportedClasses;510 511            // First see if the existing module info is sufficient. GetModuleEntryFromCache does LastWriteTime512            // verification, so this will also return nothing if the cache is out of date or corrupt.513            if (moduleCacheEntry != null)514            {515                bool needToUpdate = false;516 517                // We need to iterate and check as exportedCommands will have more item as it can have aliases as well.518                exportedCommands = moduleCacheEntry.Commands;519                foreach (var pair in realExportedCommands)520                {521                    var commandName = pair.Key;522                    var realCommandType = pair.Value.CommandType;523                    CommandTypes commandType;524                    if (!exportedCommands.TryGetValue(commandName, out commandType) || commandType != realCommandType)525                    {526                        needToUpdate = true;527                        break;528                    }529                }530 531                exportedClasses = moduleCacheEntry.Types;532                foreach (var pair in realExportedClasses)533                {534                    var className = pair.Key;535                    var realTypeAttributes = pair.Value.TypeAttributes;536                    TypeAttributes typeAttributes;537                    if (!exportedClasses.TryGetValue(className, out typeAttributes) ||538                        typeAttributes != realTypeAttributes)539                    {540                        needToUpdate = true;541                        break;542                    }543                }544 545                // Update or not, we've analyzed commands and types now.546                moduleCacheEntry.TypesAnalyzed = true;547 548                if (!needToUpdate)549                {550                    ModuleIntrinsics.Tracer.WriteLine("Existing cached info up-to-date. Skipping.");551                    return;552                }553 554                exportedCommands.Clear();555                exportedClasses.Clear();556            }557            else558            {559                exportedCommands = new ConcurrentDictionary<string, CommandTypes>(3, realExportedCommands.Count, StringComparer.OrdinalIgnoreCase);560                exportedClasses = new ConcurrentDictionary<string, TypeAttributes>(1, realExportedClasses.Count, StringComparer.OrdinalIgnoreCase);561                moduleCacheEntry = new ModuleCacheEntry562                {563                    ModulePath = module.Path,564                    LastWriteTime = lastWriteTime,565                    Commands = exportedCommands,566                    TypesAnalyzed = true,567                    Types = exportedClasses568                };569                moduleCacheEntry = s_cacheData.Entries.GetOrAdd(module.Path, moduleCacheEntry);570            }571 572            // We need to update the cache573            foreach (var exportedCommand in realExportedCommands.Values)574            {575                ModuleIntrinsics.Tracer.WriteLine("Caching command: {0}", exportedCommand.Name);576                exportedCommands.GetOrAdd(exportedCommand.Name, exportedCommand.CommandType);577            }578 579            foreach (var pair in realExportedClasses)580            {581                var className = pair.Key;582                ModuleIntrinsics.Tracer.WriteLine("Caching command: {0}", className);583                moduleCacheEntry.Types.AddOrUpdate(className, pair.Value.TypeAttributes, (k, t) => t);584            }585 586            s_cacheData.QueueSerialization();587        }588 589        private static void CallGetModuleDashList(ExecutionContext context, string modulePath)590        {591            CommandInfo commandInfo = new CmdletInfo("Get-Module", typeof(GetModuleCommand), null, null, context);592            Command getModuleCommand = new Command(commandInfo);593 594            try595            {596                PowerShell.Create(RunspaceMode.CurrentRunspace)597                    .AddCommand(getModuleCommand)598                        .AddParameter("List", true)599                        .AddParameter("ErrorAction", ActionPreference.Ignore)600                        .AddParameter("WarningAction", ActionPreference.Ignore)601                        .AddParameter("InformationAction", ActionPreference.Ignore)602                        .AddParameter("Verbose", false)603                        .AddParameter("Debug", false)604                        .AddParameter("Name", modulePath)605                    .Invoke();606            }607            catch (Exception e)608            {609                ModuleIntrinsics.Tracer.WriteLine("Module analysis generated an exception: {0}", e);610 611                // Catch-all OK, third-party call-out.612            }613        }614 615        private static bool GetModuleEntryFromCache(string modulePath, out DateTime lastWriteTime, out ModuleCacheEntry moduleCacheEntry)616        {617            try618            {619                lastWriteTime = new FileInfo(modulePath).LastWriteTime;620            }621            catch (Exception e)622            {623                ModuleIntrinsics.Tracer.WriteLine("Exception checking LastWriteTime on module {0}: {1}", modulePath, e.Message);624                lastWriteTime = DateTime.MinValue;625            }626 627            if (s_cacheData.Entries.TryGetValue(modulePath, out moduleCacheEntry))628            {629                if (lastWriteTime == moduleCacheEntry.LastWriteTime)630                {631                    return true;632                }633 634                ModuleIntrinsics.Tracer.WriteLine("{0}: cache entry out of date, cached on {1}, last updated on {2}",635                    modulePath, moduleCacheEntry.LastWriteTime, lastWriteTime);636 637                s_cacheData.Entries.TryRemove(modulePath, out moduleCacheEntry);638            }639 640            moduleCacheEntry = null;641            return false;642        }643    }644 645    internal sealed class AnalysisCacheData646    {647        private static byte[] GetHeader()648        {649            return new byte[]650            {651                0x50, 0x53, 0x4d, 0x4f, 0x44, 0x55, 0x4c, 0x45, 0x43, 0x41, 0x43, 0x48, 0x45, // PSMODULECACHE652                0x01 // version #653            };654        }655 656        // The last time the index was maintained.657        public DateTime LastReadTime { get; set; }658 659        public ConcurrentDictionary<string, ModuleCacheEntry> Entries { get; set; }660 661        private int _saveCacheToDiskQueued;662 663        private bool _saveCacheToDisk = true;664 665        public void QueueSerialization()666        {667            if (string.IsNullOrEmpty(s_cacheStoreLocation))668            {669                return;670            }671 672            // We expect many modules to rapidly call for serialization.673            // Instead of doing it right away, we'll queue a task that starts writing674            // after it seems like we've stopped adding stuff to write out.  This is675            // avoids blocking the pipeline thread waiting for the write to finish.676            // We want to make sure we only queue one task.677            if (_saveCacheToDisk && Interlocked.Increment(ref _saveCacheToDiskQueued) == 1)678            {679                Task.Run(async delegate680                {681                    // Wait a while before assuming we've finished the updates,682                    // writing the cache out in a timely matter isn't too important683                    // now anyway.684                    await Task.Delay(10000).ConfigureAwait(false);685                    int counter1, counter2;686                    do687                    {688                        // Check the counter a couple times with a delay,689                        // if it's stable, then proceed with writing.690                        counter1 = _saveCacheToDiskQueued;691                        await Task.Delay(3000).ConfigureAwait(false);692                        counter2 = _saveCacheToDiskQueued;693                    } while (counter1 != counter2);694                    Serialize(s_cacheStoreLocation);695                });696            }697        }698 699        // Remove entries that are not needed anymore, e.g. if a module was removed.700        // If anything is removed, save the cache.701        private void Cleanup()702        {703            Diagnostics.Assert(Environment.GetEnvironmentVariable("PSDisableModuleAnalysisCacheCleanup") == null,704                "Caller to check environment variable before calling");705 706            bool removedSomething = false;707            var keys = Entries.Keys;708            foreach (var key in keys)709            {710                if (!File.Exists(key))711                {712                    removedSomething |= Entries.TryRemove(key, out ModuleCacheEntry _);713                }714            }715 716            if (removedSomething)717            {718                QueueSerialization();719            }720        }721 722        private static unsafe void Write(int val, byte[] bytes, FileStream stream)723        {724            Diagnostics.Assert(bytes.Length >= 4, "Must pass a large enough byte array");725            fixed (byte* b = bytes) *((int*)b) = val;726            stream.Write(bytes, 0, 4);727        }728 729        private static unsafe void Write(long val, byte[] bytes, FileStream stream)730        {731            Diagnostics.Assert(bytes.Length >= 8, "Must pass a large enough byte array");732            fixed (byte* b = bytes) *((long*)b) = val;733            stream.Write(bytes, 0, 8);734        }735 736        private static void Write(string val, byte[] bytes, FileStream stream)737        {738            Write(val.Length, bytes, stream);739            bytes = Encoding.UTF8.GetBytes(val);740            stream.Write(bytes, 0, bytes.Length);741        }742 743        private void Serialize(string filename)744        {745            AnalysisCacheData fromOtherProcess = null;746            Diagnostics.Assert(_saveCacheToDisk, "Serialize should never be called without going through QueueSerialization which has a check");747 748            try749            {750                if (File.Exists(filename))751                {752                    var fileLastWriteTime = new FileInfo(filename).LastWriteTime;753                    if (fileLastWriteTime > this.LastReadTime)754                    {755                        fromOtherProcess = Deserialize(filename);756                    }757                }758                else759                {760                    // Make sure the folder exists761                    var folder = Path.GetDirectoryName(filename);762                    if (!Directory.Exists(folder))763                    {764                        try765                        {766                            Directory.CreateDirectory(folder);767                        }768                        catch (UnauthorizedAccessException)769                        {770                            // service accounts won't be able to create directory771                            _saveCacheToDisk = false;772                            return;773                        }774                    }775                }776            }777            catch (Exception e)778            {779                ModuleIntrinsics.Tracer.WriteLine("Exception checking module analysis cache {0}: {1} ", filename, e.Message);780            }781 782            if (fromOtherProcess != null)783            {784                // We should merge with what another process wrote so we don't clobber useful analysis785                foreach (var otherEntryPair in fromOtherProcess.Entries)786                {787                    var otherModuleName = otherEntryPair.Key;788                    var otherEntry = otherEntryPair.Value;789                    ModuleCacheEntry thisEntry;790                    if (Entries.TryGetValue(otherModuleName, out thisEntry))791                    {792                        if (otherEntry.LastWriteTime > thisEntry.LastWriteTime)793                        {794                            // The other entry is newer, take it over ours795                            Entries[otherModuleName] = otherEntry;796                        }797                    }798                    else799                    {800                        Entries[otherModuleName] = otherEntry;801                    }802                }803            }804 805            // "PSMODULECACHE"     -> 13 bytes806            // byte     ( 1 byte)  -> version807            // int      ( 4 bytes) -> count of entries808            // entries  (?? bytes) -> all entries809            //810            // each entry is811            //   DateTime ( 8 bytes) -> last write time for module file812            //   int      ( 4 bytes) -> path length813            //   string   (?? bytes) -> utf8 encoded path814            //   int      ( 4 bytes) -> count of commands815            //   commands (?? bytes) -> all commands816            //   int      ( 4 bytes) -> count of types, -1 means unanalyzed (and 0 items serialized)817            //   types    (?? bytes) -> all types818            //819            // each command is820            //   int      ( 4 bytes) -> command name length821            //   string   (?? bytes) -> utf8 encoded command name822            //   int      ( 4 bytes) -> CommandTypes enum823            //824            // each type is825            //   int     ( 4 bytes) -> type name length826            //   string  (?? bytes) -> utf8 encoded type name827            //   int     ( 4 bytes) -> type attributes828            try829            {830                var bytes = new byte[8];831 832                using (var stream = File.Create(filename))833                {834                    var headerBytes = GetHeader();835                    stream.Write(headerBytes, 0, headerBytes.Length);836 837                    // Count of entries838                    Write(Entries.Count, bytes, stream);839 840                    foreach (var pair in Entries.ToArray())841                    {842                        var path = pair.Key;843                        var entry = pair.Value;844 845                        // Module last write time846                        Write(entry.LastWriteTime.Ticks, bytes, stream);847 848                        // Module path849                        Write(path, bytes, stream);850 851                        // Commands852                        var commandPairs = entry.Commands.ToArray();853                        Write(commandPairs.Length, bytes, stream);854 855                        foreach (var command in commandPairs)856                        {857                            Write(command.Key, bytes, stream);858                            Write((int)command.Value, bytes, stream);859                        }860 861                        // Types862                        var typePairs = entry.Types.ToArray();863                        Write(entry.TypesAnalyzed ? typePairs.Length : -1, bytes, stream);864 865                        foreach (var type in typePairs)866                        {867                            Write(type.Key, bytes, stream);868                            Write((int)type.Value, bytes, stream);869                        }870                    }871                }872                // We just wrote the file, note this so we can detect writes from another process873                LastReadTime = new FileInfo(filename).LastWriteTime;874            }875            catch (Exception e)876            {877                ModuleIntrinsics.Tracer.WriteLine("Exception writing module analysis cache {0}: {1} ", filename, e.Message);878            }879 880            // Reset our counter so we can write again if asked.881            Interlocked.Exchange(ref _saveCacheToDiskQueued, 0);882        }883 884        private const string TruncatedErrorMessage = "module cache file appears truncated";885        private const string InvalidSignatureErrorMessage = "module cache signature not valid";886        private const string PossibleCorruptionErrorMessage = "possible corruption in module cache";887 888        private static unsafe long ReadLong(FileStream stream, byte[] bytes)889        {890            Diagnostics.Assert(bytes.Length >= 8, "Must pass a large enough byte array");891            if (stream.Read(bytes, 0, 8) != 8)892                throw new Exception(TruncatedErrorMessage);893            fixed (byte* b = bytes)894                return *(long*)b;895        }896 897        private static unsafe int ReadInt(FileStream stream, byte[] bytes)898        {899            Diagnostics.Assert(bytes.Length >= 4, "Must pass a large enough byte array");900            if (stream.Read(bytes, 0, 4) != 4)901                throw new Exception(TruncatedErrorMessage);902            fixed (byte* b = bytes)903                return *(int*)b;904        }905 906        private static string ReadString(FileStream stream, ref byte[] bytes)907        {908            int length = ReadInt(stream, bytes);909            if (length > 10 * 1024)910                throw new Exception(PossibleCorruptionErrorMessage);911            if (length > bytes.Length)912                bytes = new byte[length];913            if (stream.Read(bytes, 0, length) != length)914                throw new Exception(TruncatedErrorMessage);915            return Encoding.UTF8.GetString(bytes, 0, length);916        }917 918        private static void ReadHeader(FileStream stream, byte[] bytes)919        {920            var headerBytes = GetHeader();921            var length = headerBytes.Length;922            Diagnostics.Assert(bytes.Length >= length, "must pass a large enough byte array");923            if (stream.Read(bytes, 0, length) != length)924                throw new Exception(TruncatedErrorMessage);925 926            for (int i = 0; i < length; i++)927            {928                if (bytes[i] != headerBytes[i])929                {930                    throw new Exception(InvalidSignatureErrorMessage);931                }932            }933            // No need to return - we don't use it other than to detect the correct file format934        }935 936        public static AnalysisCacheData Deserialize(string filename)937        {938            using (var stream = File.OpenRead(filename))939            {940                var result = new AnalysisCacheData { LastReadTime = DateTime.Now };941 942                var bytes = new byte[1024];943 944                // Header945                // "PSMODULECACHE"     -> 13 bytes946                // byte     ( 1 byte)  -> version947                ReadHeader(stream, bytes);948 949                // int      ( 4 bytes) -> count of entries950                int entries = ReadInt(stream, bytes);951                if (entries > 20 * 1024)952                    throw new Exception(PossibleCorruptionErrorMessage);953 954                result.Entries = new ConcurrentDictionary<string, ModuleCacheEntry>(/*concurrency*/3, entries, StringComparer.OrdinalIgnoreCase);955 956                // entries  (?? bytes) -> all entries957                while (entries > 0)958                {959                    //   DateTime ( 8 bytes) -> last write time for module file960                    var lastWriteTime = new DateTime(ReadLong(stream, bytes));961 962                    //   int      ( 4 bytes) -> path length963                    //   string   (?? bytes) -> utf8 encoded path964                    var path = ReadString(stream, ref bytes);965 966                    //   int      ( 4 bytes) -> count of commands967                    var countItems = ReadInt(stream, bytes);968                    if (countItems > 20 * 1024)969                        throw new Exception(PossibleCorruptionErrorMessage);970 971                    var commands = new ConcurrentDictionary<string, CommandTypes>(/*concurrency*/3, countItems, StringComparer.OrdinalIgnoreCase);972 973                    //   commands (?? bytes) -> all commands974                    while (countItems > 0)975                    {976                        //   int      ( 4 bytes) -> command name length977                        //   string   (?? bytes) -> utf8 encoded command name978                        var commandName = ReadString(stream, ref bytes);979 980                        //   int      ( 4 bytes) -> CommandTypes enum981                        var commandTypes = (CommandTypes)ReadInt(stream, bytes);982 983                        // Ignore empty entries (possible corruption in the cache or bug?)984                        if (!string.IsNullOrWhiteSpace(commandName))985                            commands[commandName] = commandTypes;986 987                        countItems -= 1;988                    }989 990                    //   int      ( 4 bytes) -> count of types991                    countItems = ReadInt(stream, bytes);992 993                    bool typesAnalyzed = countItems != -1;994                    if (!typesAnalyzed)995                        countItems = 0;996                    if (countItems > 20 * 1024)997                        throw new Exception(PossibleCorruptionErrorMessage);998 999                    var types = new ConcurrentDictionary<string, TypeAttributes>(1, countItems, StringComparer.OrdinalIgnoreCase);1000 1001                    //   types    (?? bytes) -> all types1002                    while (countItems > 0)1003                    {1004                        //   int     ( 4 bytes) -> type name length1005                        //   string  (?? bytes) -> utf8 encoded type name1006                        var typeName = ReadString(stream, ref bytes);1007 1008                        //   int     ( 4 bytes) -> type attributes1009                        var typeAttributes = (TypeAttributes)ReadInt(stream, bytes);1010 1011                        // Ignore empty entries (possible corruption in the cache or bug?)1012                        if (!string.IsNullOrWhiteSpace(typeName))1013                            types[typeName] = typeAttributes;1014 1015                        countItems -= 1;1016                    }1017 1018                    var entry = new ModuleCacheEntry1019                    {1020                        ModulePath = path,1021                        LastWriteTime = lastWriteTime,1022                        Commands = commands,1023                        TypesAnalyzed = typesAnalyzed,1024                        Types = types1025                    };1026                    result.Entries[path] = entry;1027 1028                    entries -= 1;1029                }1030 1031                if (Environment.GetEnvironmentVariable("PSDisableModuleAnalysisCacheCleanup") == null)1032                {1033                    Task.Delay(10000).ContinueWith(_ => result.Cleanup());1034                }1035 1036                return result;1037            }1038        }1039 1040        internal static AnalysisCacheData Get()1041        {1042            int retryCount = 3;1043 1044            do1045            {1046                try1047                {1048                    if (File.Exists(s_cacheStoreLocation))1049                    {1050                        return Deserialize(s_cacheStoreLocation);1051                    }1052                }1053                catch (Exception e)1054                {1055                    ModuleIntrinsics.Tracer.WriteLine("Exception checking module analysis cache: " + e.Message);1056                    if ((object)e.Message == (object)TruncatedErrorMessage1057                        || (object)e.Message == (object)InvalidSignatureErrorMessage1058                        || (object)e.Message == (object)PossibleCorruptionErrorMessage)1059                    {1060                        // Don't retry if we detected something is wrong with the file1061                        // (as opposed to the file being locked or something else)1062                        break;1063                    }1064                }1065 1066                retryCount -= 1;1067                Thread.Sleep(25); // Sleep a bit to give time for another process to finish writing the cache1068            } while (retryCount > 0);1069 1070            return new AnalysisCacheData1071            {1072                LastReadTime = DateTime.Now,1073                // Capacity set to 100 - a bit bigger than the # of modules on a default Win10 client machine1074                // Concurrency=3 to not create too many locks, contention is unclear, but the old code had a single lock1075                Entries = new ConcurrentDictionary<string, ModuleCacheEntry>(/*concurrency*/3, /*capacity*/100, StringComparer.OrdinalIgnoreCase)1076            };1077        }1078 1079        private AnalysisCacheData()1080        {1081        }1082 1083        private static readonly string s_cacheStoreLocation;1084 1085        static AnalysisCacheData()1086        {1087            // If user defines a custom cache path, then use that.1088            string userDefinedCachePath = Environment.GetEnvironmentVariable("PSModuleAnalysisCachePath");1089            if (!string.IsNullOrEmpty(userDefinedCachePath))1090            {1091                s_cacheStoreLocation = userDefinedCachePath;1092                return;1093            }1094 1095            string cacheFileName = "ModuleAnalysisCache";1096 1097            // When multiple copies of pwsh are on the system, they should use their own copy of the cache.1098            // Append hash of `$PSHOME` to cacheFileName.1099            string hashString = CRC32Hash.ComputeHash(Utils.DefaultPowerShellAppBase);1100            cacheFileName = string.Create(CultureInfo.InvariantCulture, $"{cacheFileName}-{hashString}");1101 1102            if (ExperimentalFeature.EnabledExperimentalFeatureNames.Count > 0)1103            {1104                // If any experimental features are enabled, we cannot use the default cache file because those1105                // features may expose commands that are not available in a regular powershell session, and we1106                // should not cache those commands in the default cache file because that will result in wrong1107                // auto-completion suggestions when the default cache file is used in another powershell session.1108                //1109                // Here we will generate a cache file name that represent the combination of enabled feature names.1110                // We first convert enabled feature names to lower case, then we sort the feature names, and then1111                // compute an CRC32 hash from the sorted feature names. We will use the CRC32 hash to generate the1112                // cache file name.1113                int index = 0;1114                string[] featureNames = new string[ExperimentalFeature.EnabledExperimentalFeatureNames.Count];1115                foreach (string featureName in ExperimentalFeature.EnabledExperimentalFeatureNames)1116                {1117                    featureNames[index++] = featureName.ToLowerInvariant();1118                }1119 1120                Array.Sort(featureNames);1121                string allNames = string.Join(Environment.NewLine, featureNames);1122 1123                // Use CRC32 because it's faster.1124                // It's very unlikely to get collision from hashing the combinations of enabled features names.1125                hashString = CRC32Hash.ComputeHash(allNames);1126                cacheFileName = string.Create(CultureInfo.InvariantCulture, $"{cacheFileName}-{hashString}");1127            }1128 1129            Platform.TryDeriveFromCache(cacheFileName, out s_cacheStoreLocation);1130        }1131    }1132 1133    [DebuggerDisplay("ModulePath = {ModulePath}")]1134    internal class ModuleCacheEntry1135    {1136        public DateTime LastWriteTime;1137        public string ModulePath;1138        public bool TypesAnalyzed;1139        public ConcurrentDictionary<string, CommandTypes> Commands;1140        public ConcurrentDictionary<string, TypeAttributes> Types;1141    }1142}1143