MegaBites-AI/Windows-powershell
0372
1// Copyright (c) Microsoft Corporation.2// Licensed under the MIT License.3 4using System.Collections;5using System.Collections.Generic;6using System.Collections.ObjectModel;7using System.Diagnostics;8using System.Globalization;9using System.Linq;10using System.Numerics;11using System.Runtime.CompilerServices;12using System.Text;13 14using Microsoft.PowerShell.Commands;15using System.Management.Automation.Subsystem;16using System.Management.Automation.Subsystem.DSC;17using Microsoft.PowerShell.DesiredStateConfiguration.Internal;18 19namespace System.Management.Automation.Language20{21 /// <summary>22 /// Defines the name modes for a dynamic keyword. A name expression may be required, optional or not permitted.23 /// </summary>24 public enum DynamicKeywordNameMode25 {26 /// <summary>27 /// This keyword does not take a name value.28 /// </summary>29 NoName = 0,30 /// <summary>31 /// Name must be present and simple non-empty bare word.32 /// </summary>33 SimpleNameRequired = 1,34 /// <summary>35 /// Name must be present but can also be an expression.36 /// </summary>37 NameRequired = 2,38 /// <summary>39 /// Name may be optionally present, but if it is present, it must be a non-empty bare word.40 /// </summary>41 SimpleOptionalName = 3,42 /// <summary>43 /// Name may be optionally present, expression or bare word.44 /// </summary>45 OptionalName = 4,46 }47 48 /// <summary>49 /// Defines the body mode for a dynamic keyword. It can be a scriptblock, hashtable or command which means no body.50 /// </summary>51 public enum DynamicKeywordBodyMode52 {53 /// <summary>54 /// The keyword act like a command.55 /// </summary>56 Command = 0,57 /// <summary>58 /// The keyword has a scriptblock body.59 /// </summary>60 ScriptBlock = 1,61 /// <summary>62 /// The keyword has hashtable body.63 /// </summary>64 Hashtable = 2,65 }66 67 /// <summary>68 /// Defines the schema/behaviour for a dynamic keyword.69 /// a constrained.70 /// </summary>71 public class DynamicKeyword72 {73 #region static properties/functions74 75 /// <summary>76 /// Defines a dictionary of dynamic keywords, stored in thread-local storage.77 /// </summary>78 private static Dictionary<string, DynamicKeyword> DynamicKeywords79 {80 get81 {82 return t_dynamicKeywords ??= new Dictionary<string, DynamicKeyword>(StringComparer.OrdinalIgnoreCase);83 }84 }85 86 [ThreadStatic]87 private static Dictionary<string, DynamicKeyword> t_dynamicKeywords;88 89 /// <summary>90 /// Stack of DynamicKeywords Cache.91 /// </summary>92 private static Stack<Dictionary<string, DynamicKeyword>> DynamicKeywordsStack93 {94 get95 {96 return t_dynamicKeywordsStack ??= new Stack<Dictionary<string, DynamicKeyword>>();97 }98 }99 100 [ThreadStatic]101 private static Stack<Dictionary<string, DynamicKeyword>> t_dynamicKeywordsStack;102 103 /// <summary>104 /// Reset the keyword table to a new empty collection.105 /// </summary>106 public static void Reset()107 {108 t_dynamicKeywords = new Dictionary<string, DynamicKeyword>(StringComparer.OrdinalIgnoreCase);109 }110 111 /// <summary>112 /// Push current dynamicKeywords cache into stack.113 /// </summary>114 public static void Push()115 {116 DynamicKeywordsStack.Push(t_dynamicKeywords);117 Reset();118 }119 120 /// <summary>121 /// Pop up previous dynamicKeywords cache.122 /// </summary>123 public static void Pop()124 {125 t_dynamicKeywords = DynamicKeywordsStack.Pop();126 }127 128 /// <summary>129 /// </summary>130 /// <param name="name"></param>131 /// <returns></returns>132 public static DynamicKeyword GetKeyword(string name)133 {134 DynamicKeyword keywordToReturn;135 DynamicKeyword.DynamicKeywords.TryGetValue(name, out keywordToReturn);136 return keywordToReturn;137 }138 139 /// <summary>140 /// Returns a copied list of all of the existing dynamic keyword definitions.141 /// </summary>142 /// <returns></returns>143 public static List<DynamicKeyword> GetKeyword()144 {145 return new List<DynamicKeyword>(DynamicKeyword.DynamicKeywords.Values);146 }147 148 /// <summary>149 /// </summary>150 /// <param name="name"></param>151 /// <returns></returns>152 public static bool ContainsKeyword(string name)153 {154 if (string.IsNullOrEmpty(name))155 {156 PSArgumentNullException e = PSTraceSource.NewArgumentNullException(nameof(name));157 throw e;158 }159 160 return DynamicKeyword.DynamicKeywords.ContainsKey(name);161 }162 163 /// <summary>164 /// </summary>165 /// <param name="keywordToAdd"></param>166 public static void AddKeyword(DynamicKeyword keywordToAdd)167 {168 if (keywordToAdd == null)169 {170 PSArgumentNullException e = PSTraceSource.NewArgumentNullException(nameof(keywordToAdd));171 throw e;172 }173 174 // Allow overwriting of the existing entries175 string name = keywordToAdd.Keyword;176 if (string.IsNullOrEmpty(name))177 {178 throw PSTraceSource.NewArgumentNullException("keywordToAdd.Keyword");179 }180 181 DynamicKeyword.DynamicKeywords.Remove(name);182 DynamicKeyword.DynamicKeywords.Add(name, keywordToAdd);183 }184 185 /// <summary>186 /// Remove a single entry from the dynamic keyword collection187 /// and clean up any associated data.188 /// </summary>189 /// <param name="name"></param>190 public static void RemoveKeyword(string name)191 {192 if (string.IsNullOrEmpty(name))193 {194 PSArgumentNullException e = PSTraceSource.NewArgumentNullException(nameof(name));195 throw e;196 }197 198 DynamicKeyword.DynamicKeywords.Remove(name);199 }200 201 /// <summary>202 /// Check if it is a hidden keyword.203 /// </summary>204 /// <param name="name"></param>205 /// <returns></returns>206 internal static bool IsHiddenKeyword(string name)207 {208 if (string.IsNullOrEmpty(name))209 {210 PSArgumentNullException e = PSTraceSource.NewArgumentNullException(nameof(name));211 throw e;212 }213 214 return s_hiddenDynamicKeywords.Contains(name);215 }216 217 /// <summary>218 /// A set of dynamic keywords that are not supposed to be used in script directly.219 /// They are for internal use only.220 /// </summary>221 private static readonly HashSet<string> s_hiddenDynamicKeywords =222 new HashSet<string>(StringComparer.OrdinalIgnoreCase) { "MSFT_Credential" };223 224 #endregion225 226 /// <summary>227 /// Duplicates the DynamicKeyword.228 /// </summary>229 /// <returns>A copy of the DynamicKeyword.</returns>230 public DynamicKeyword Copy()231 {232 DynamicKeyword keyword = new DynamicKeyword()233 {234 ImplementingModule = this.ImplementingModule,235 ImplementingModuleVersion = this.ImplementingModuleVersion,236 Keyword = this.Keyword,237 ResourceName = this.ResourceName,238 BodyMode = this.BodyMode,239 DirectCall = this.DirectCall,240 NameMode = this.NameMode,241 MetaStatement = this.MetaStatement,242 IsReservedKeyword = this.IsReservedKeyword,243 HasReservedProperties = this.HasReservedProperties,244 PreParse = this.PreParse,245 PostParse = this.PostParse,246 SemanticCheck = this.SemanticCheck247 };248 foreach (KeyValuePair<string, DynamicKeywordProperty> entry in this.Properties)249 {250 keyword.Properties.Add(entry.Key, entry.Value);251 }252 253 foreach (KeyValuePair<string, DynamicKeywordParameter> entry in this.Parameters)254 {255 keyword.Parameters.Add(entry.Key, entry.Value);256 }257 258 return keyword;259 }260 261 /// <summary>262 /// The name of the module that implements the function corresponding to this keyword.263 /// </summary>264 public string ImplementingModule { get; set; }265 266 /// <summary>267 /// The version of the module that implements the function corresponding to this keyword.268 /// </summary>269 public Version ImplementingModuleVersion { get; set; }270 271 /// <summary>272 /// The keyword string273 /// If an alias qualifier exist, use alias.274 /// </summary>275 public string Keyword { get; set; }276 277 /// <summary>278 /// The keyword resource name string.279 /// </summary>280 public string ResourceName { get; set; }281 282 /// <summary>283 /// Set to true if we should be looking for a scriptblock instead of a hashtable.284 /// </summary>285 public DynamicKeywordBodyMode BodyMode { get; set; }286 287 /// <summary>288 /// If true, then don't use the marshalled call. Just289 /// rewrite the node as a simple direct function call.290 /// If NameMode is other than NoName, then the name of the instance291 /// will be passed as the parameter -InstanceName.292 /// </summary>293 public bool DirectCall { get; set; }294 295 /// <summary>296 /// This allows you to specify if the keyword takes a name argument and if so, what form that takes.297 /// </summary>298 public DynamicKeywordNameMode NameMode { get; set; }299 300 /// <summary>301 /// Indicate that the nothing should be added to the AST for this302 /// dynamic keyword.303 /// </summary>304 public bool MetaStatement { get; set; }305 306 /// <summary>307 /// Indicate that the keyword is reserved for future use by powershell.308 /// </summary>309 public bool IsReservedKeyword { get; set; }310 311 /// <summary>312 /// Contains the list of properties that are reserved for future use.313 /// </summary>314 public bool HasReservedProperties { get; set; }315 316 /// <summary>317 /// A list of the properties allowed for this constructor.318 /// </summary>319 public Dictionary<string, DynamicKeywordProperty> Properties320 {321 get322 {323 return _properties ??= new Dictionary<string, DynamicKeywordProperty>(StringComparer.OrdinalIgnoreCase);324 }325 }326 327 private Dictionary<string, DynamicKeywordProperty> _properties;328 329 /// <summary>330 /// A list of the parameters allowed for this constructor.331 /// </summary>332 public Dictionary<string, DynamicKeywordParameter> Parameters333 {334 get335 {336 return _parameters ??= new Dictionary<string, DynamicKeywordParameter>(StringComparer.OrdinalIgnoreCase);337 }338 }339 340 private Dictionary<string, DynamicKeywordParameter> _parameters;341 342 /// <summary>343 /// A custom function that gets executed at parsing time before parsing dynamickeyword block344 /// The delegate has one parameter: DynamicKeyword.345 /// </summary>346 public Func<DynamicKeyword, ParseError[]> PreParse { get; set; }347 348 /// <summary>349 /// A custom function that gets executed at parsing time after parsing dynamickeyword block.350 /// </summary>351 public Func<DynamicKeywordStatementAst, ParseError[]> PostParse { get; set; }352 353 /// <summary>354 /// A custom function that checks semantic for the given <see cref="DynamicKeywordStatementAst"/>355 /// </summary>356 public Func<DynamicKeywordStatementAst, ParseError[]> SemanticCheck { get; set; }357 }358 359 internal static class DynamicKeywordExtension360 {361 internal static bool IsMetaDSCResource(this DynamicKeyword keyword)362 {363 string implementingModule = keyword.ImplementingModule;364 if (implementingModule != null)365 {366 ICrossPlatformDsc dscSubsystem = SubsystemManager.GetSubsystem<ICrossPlatformDsc>();367 if (dscSubsystem != null)368 {369 dscSubsystem.IsDefaultModuleNameForMetaConfigResource(implementingModule);370 }371 else372 {373 return implementingModule.Equals(DscClassCache.DefaultModuleInfoForMetaConfigResource.Item1, StringComparison.OrdinalIgnoreCase);374 }375 }376 377 return false;378 }379 380 internal static bool IsCompatibleWithConfigurationType(this DynamicKeyword keyword, ConfigurationType ConfigurationType)381 {382 return ((ConfigurationType == ConfigurationType.Meta && keyword.IsMetaDSCResource()) ||383 (ConfigurationType != ConfigurationType.Meta && !keyword.IsMetaDSCResource()));384 }385 386 private static readonly Dictionary<string, List<string>> s_excludeKeywords = new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase)387 {388 {@"Node", new List<string> {@"Node"}},389 };390 391 /// <summary>392 /// Get allowed keyword list for a given keyword.393 /// </summary>394 /// <param name="keyword"></param>395 /// <param name="allowedKeywords"></param>396 /// <returns>NULL if no keyword allowed for a given <see cref="DynamicKeyword"/></returns>397 internal static IEnumerable<DynamicKeyword> GetAllowedKeywords(this DynamicKeyword keyword, IEnumerable<DynamicKeyword> allowedKeywords)398 {399 string keywordName = keyword.Keyword;400 if (string.Equals(keywordName, @"Node", StringComparison.OrdinalIgnoreCase))401 {402 List<string> excludeKeywords;403 if (s_excludeKeywords.TryGetValue(keywordName, out excludeKeywords))404 {405 return allowedKeywords.Where(k => !excludeKeywords.Contains(k.Keyword));406 }407 else408 return allowedKeywords;409 }410 411 return null;412 }413 }414 415 /// <summary>416 /// Metadata about a member property for a dynamic keyword.417 /// </summary>418 public class DynamicKeywordProperty419 {420 /// <summary>421 /// The name of the property.422 /// </summary>423 public string Name { get; set; }424 425 /// <summary>426 /// The required type of the property.427 /// </summary>428 public string TypeConstraint { get; set; }429 430 /// <summary>431 /// Any attributes that the property has.432 /// </summary>433 public List<string> Attributes434 {435 get { return _attributes ??= new List<string>(); }436 }437 438 private List<string> _attributes;439 440 /// <summary>441 /// List of strings that may be used as values for this property.442 /// </summary>443 public List<string> Values444 {445 get { return _values ??= new List<string>(); }446 }447 448 private List<string> _values;449 450 /// <summary>451 /// Mapping the descriptive values to the actual values.452 /// </summary>453 public Dictionary<string, string> ValueMap454 {455 get { return _valueMap ??= new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); }456 }457 458 private Dictionary<string, string> _valueMap;459 460 /// <summary>461 /// Indicates that this property is mandatory and must be present.462 /// </summary>463 public bool Mandatory { get; set; }464 465 /// <summary>466 /// Indicates that this property is a key.467 /// </summary>468 public bool IsKey { get; set; }469 470 /// <summary>471 /// Indicates a range constraint on the property value.472 /// </summary>473 public Tuple<int, int> Range { get; set; }474 }475 476 /// <summary>477 /// Metadata about a parameter for a dynamic keyword. Adds one478 /// new property to the base classL Switch for switch parameters479 /// (THere is no such thing as a switch property...)480 /// </summary>481 public class DynamicKeywordParameter : DynamicKeywordProperty482 {483 /// <summary>484 /// Type if this is a switch parameter and takes no argument.485 /// </summary>486 public bool Switch { get; set; }487 }488 489 internal enum TokenizerMode490 {491 Command,492 Expression,493 TypeName,494 Signature, // i.e. class or method declaration495 }496 497 /// <summary>498 /// Indicates which suffix character(s) are present in the numeric literal being parsed by TryGetNumberValue.499 /// </summary>500 [Flags]501 internal enum NumberSuffixFlags502 {503 /// <summary>504 /// Indicates no suffix, a raw numeric literal. May be parsed as Int32, Int64, or Double.505 /// </summary>506 None = 0x0,507 508 /// <summary>509 /// Indicates 'u' suffix for unsigned integers. May be parsed as UInt32 or UInt64, depending on the value.510 /// </summary>511 Unsigned = 0x1,512 513 /// <summary>514 /// Indicates 'y' suffix for signed byte (sbyte) values.515 /// </summary>516 SignedByte = 0x2,517 518 /// <summary>519 /// Indicates 'uy' suffix for unsigned byte values.520 /// This is a compound value, representing both SignedByte and Unsigned flags being set.521 /// </summary>522 UnsignedByte = 0x3,523 524 /// <summary>525 /// Indicates 's' suffix for short (Int16) integers.526 /// </summary>527 Short = 0x4,528 529 /// <summary>530 /// Indicates 'us' suffix for ushort (UInt16) integers.531 /// This is a compound flag value, representing both Unsigned and Short flags being set.532 /// </summary>533 UnsignedShort = 0x5,534 535 /// <summary>536 /// Indicates 'l' suffix for long (Int64) integers.537 /// </summary>538 Long = 0x8,539 540 /// <summary>541 /// Indicates 'ul' suffix for ulong (UInt64) integers.542 /// This is a compound flag value, representing both Unsigned and Long flags being set.543 /// </summary>544 UnsignedLong = 0x9,545 546 /// <summary>547 /// Indicates 'd' suffix for decimal (128-bit) real numbers.548 /// </summary>549 Decimal = 0x10,550 551 /// <summary>552 /// Indicates 'N' suffix for BigInteger (arbitrarily large integer) numerals.553 /// </summary>554 BigInteger = 0x20555 }556 557 /// <summary>558 /// Indicates the format of a numeric literal.559 /// </summary>560 internal enum NumberFormat561 {562 /// <summary>563 /// Indicates standard decimal literal, no necessary prefix.564 /// </summary>565 Decimal = 0x0,566 567 /// <summary>568 /// Indicates hexadecimal literal, with '0x' prefix.569 /// </summary>570 Hex = 0x1,571 572 /// <summary>573 /// Indicates binary literal, with '0b' prefix.574 /// </summary>575 Binary = 0x2576 }577 578 //579 // Class used to do a partial snapshot of the state of the tokenizer.580 // This is used for nested scans on the same string.581 //582 internal class TokenizerState583 {584 internal int NestedTokensAdjustment;585 internal string Script;586 internal int TokenStart;587 internal int CurrentIndex;588 internal Token FirstToken;589 internal Token LastToken;590 internal BitArray SkippedCharOffsets;591 internal List<Token> TokenList;592 }593 594 [DebuggerDisplay("Mode = {Mode}; Script = {_script}")]595 internal class Tokenizer596 {597 private static readonly Dictionary<string, TokenKind> s_keywordTable598 = new Dictionary<string, TokenKind>(StringComparer.OrdinalIgnoreCase);599 600 private static readonly Dictionary<string, TokenKind> s_operatorTable601 = new Dictionary<string, TokenKind>(StringComparer.OrdinalIgnoreCase);602 603 private static readonly char s_invalidChar = char.MaxValue;604 private static readonly int s_maxNumberOfUnicodeHexDigits = 6;605 606 private readonly Parser _parser;607 private PositionHelper _positionHelper;608 private int _nestedTokensAdjustment;609 610 // This BitArray is used to help ensure we get the correct extent in a corner case that looks something like:611 // $(""abc"")612 // In the above, we scan the characters between the parens once and create a token that is scanned later. This613 // first pass scan replaces doubled quotes with single quotes so that a subsequent scan will see: "abc" instead614 // of: ""abc"". (This isn't really necessary, but is done for backwards compatibility.) If we didn't track615 // the skipped quotes, we'd generate incorrect extent of the string token on subsequent rescans.616 private BitArray _skippedCharOffsets;617 618 private string _script;619 private int _tokenStart;620 private int _currentIndex;621 private InternalScriptExtent _beginSignatureExtent;622 623 #region Tables for initialization624 625 private static readonly string[] s_keywordText = new string[] {626 /*1*/ "elseif", "if", "else", "switch", /*1*/627 /*2*/ "foreach", "from", "in", "for", /*2*/628 /*3*/ "while", "until", "do", "try", /*3*/629 /*4*/ "catch", "finally", "trap", "data", /*4*/630 /*5*/ "return", "continue", "break", "exit", /*5*/631 /*6*/ "throw", "begin", "process", "end", /*6*/632 /*7*/ "dynamicparam", "function", "filter", "param", /*7*/633 /*8*/ "class", "define", "var", "using", /*8*/634 /*9*/ "workflow", "parallel", "sequence", "inlinescript", /*9*/635 /*A*/ "configuration", "public", "private", "static", /*A*/636 /*B*/ "interface", "enum", "namespace", "module", /*B*/637 /*C*/ "type", "assembly", "command", "hidden", /*C*/638 /*D*/ "base", "default", "clean", /*D*/639 };640 641 private static readonly TokenKind[] s_keywordTokenKind = new TokenKind[] {642 /*1*/ TokenKind.ElseIf, TokenKind.If, TokenKind.Else, TokenKind.Switch, /*1*/643 /*2*/ TokenKind.Foreach, TokenKind.From, TokenKind.In, TokenKind.For, /*2*/644 /*3*/ TokenKind.While, TokenKind.Until, TokenKind.Do, TokenKind.Try, /*3*/645 /*4*/ TokenKind.Catch, TokenKind.Finally, TokenKind.Trap, TokenKind.Data, /*4*/646 /*5*/ TokenKind.Return, TokenKind.Continue, TokenKind.Break, TokenKind.Exit, /*5*/647 /*6*/ TokenKind.Throw, TokenKind.Begin, TokenKind.Process, TokenKind.End, /*6*/648 /*7*/ TokenKind.Dynamicparam, TokenKind.Function, TokenKind.Filter, TokenKind.Param, /*7*/649 /*8*/ TokenKind.Class, TokenKind.Define, TokenKind.Var, TokenKind.Using, /*8*/650 /*9*/ TokenKind.Workflow, TokenKind.Parallel, TokenKind.Sequence, TokenKind.InlineScript, /*9*/651 /*A*/ TokenKind.Configuration, TokenKind.Public, TokenKind.Private, TokenKind.Static, /*A*/652 /*B*/ TokenKind.Interface, TokenKind.Enum, TokenKind.Namespace, TokenKind.Module, /*B*/653 /*C*/ TokenKind.Type, TokenKind.Assembly, TokenKind.Command, TokenKind.Hidden, /*C*/654 /*D*/ TokenKind.Base, TokenKind.Default, TokenKind.Clean, /*D*/655 };656 657 internal static readonly string[] _operatorText = new string[] {658 /*1*/ "bnot", "not", "eq", "ieq", /*1*/659 /*2*/ "ceq", "ne", "ine", "cne", /*2*/660 /*3*/ "ge", "ige", "cge", "gt", /*3*/661 /*4*/ "igt", "cgt", "lt", "ilt", /*4*/662 /*5*/ "clt", "le", "ile", "cle", /*5*/663 /*6*/ "like", "ilike", "clike", "notlike", /*6*/664 /*7*/ "inotlike", "cnotlike", "match", "imatch", /*7*/665 /*8*/ "cmatch", "notmatch", "inotmatch", "cnotmatch", /*8*/666 /*9*/ "replace", "ireplace", "creplace", "contains", /*9*/667 /*10*/ "icontains", "ccontains", "notcontains", "inotcontains", /*10*/668 /*11*/ "cnotcontains", "in", "iin", "cin", /*11*/669 /*12*/ "notin", "inotin", "cnotin", "split", /*12*/670 /*13*/ "isplit", "csplit", "isnot", "is", /*13*/671 /*14*/ "as", "f", "and", "band", /*14*/672 /*15*/ "or", "bor", "xor", "bxor", /*15*/673 /*16*/ "join", "shl", "shr", /*16*/674 };675 676 private static readonly TokenKind[] s_operatorTokenKind = new TokenKind[] {677 /*1*/ TokenKind.Bnot, TokenKind.Not, TokenKind.Ieq, TokenKind.Ieq, /*1*/678 /*2*/ TokenKind.Ceq, TokenKind.Ine, TokenKind.Ine, TokenKind.Cne, /*2*/679 /*3*/ TokenKind.Ige, TokenKind.Ige, TokenKind.Cge, TokenKind.Igt, /*3*/680 /*4*/ TokenKind.Igt, TokenKind.Cgt, TokenKind.Ilt, TokenKind.Ilt, /*4*/681 /*5*/ TokenKind.Clt, TokenKind.Ile, TokenKind.Ile, TokenKind.Cle, /*5*/682 /*6*/ TokenKind.Ilike, TokenKind.Ilike, TokenKind.Clike, TokenKind.Inotlike, /*6*/683 /*7*/ TokenKind.Inotlike, TokenKind.Cnotlike, TokenKind.Imatch, TokenKind.Imatch, /*7*/684 /*8*/ TokenKind.Cmatch, TokenKind.Inotmatch, TokenKind.Inotmatch, TokenKind.Cnotmatch, /*8*/685 /*9*/ TokenKind.Ireplace, TokenKind.Ireplace, TokenKind.Creplace, TokenKind.Icontains, /*9*/686 /*10*/ TokenKind.Icontains, TokenKind.Ccontains, TokenKind.Inotcontains, TokenKind.Inotcontains, /*10*/687 /*11*/ TokenKind.Cnotcontains, TokenKind.Iin, TokenKind.Iin, TokenKind.Cin, /*11*/688 /*12*/ TokenKind.Inotin, TokenKind.Inotin, TokenKind.Cnotin, TokenKind.Isplit, /*12*/689 /*13*/ TokenKind.Isplit, TokenKind.Csplit, TokenKind.IsNot, TokenKind.Is, /*13*/690 /*14*/ TokenKind.As, TokenKind.Format, TokenKind.And, TokenKind.Band, /*14*/691 /*15*/ TokenKind.Or, TokenKind.Bor, TokenKind.Xor, TokenKind.Bxor, /*15*/692 /*16*/ TokenKind.Join, TokenKind.Shl, TokenKind.Shr, /*16*/693 };694 695 #endregion Tables for initialization696 697 static Tokenizer()698 {699 Diagnostics.Assert(s_keywordText.Length == s_keywordTokenKind.Length, "Keyword table sizes must match");700 Diagnostics.Assert(_operatorText.Length == s_operatorTokenKind.Length, "Operator table sizes must match");701 702 for (int i = 0; i < s_keywordText.Length; ++i)703 {704 s_keywordTable.Add(s_keywordText[i], s_keywordTokenKind[i]);705 }706 707 for (int i = 0; i < _operatorText.Length; ++i)708 {709 s_operatorTable.Add(_operatorText[i], s_operatorTokenKind[i]);710 }711 712 // The real signature (in mshsip.cpp) has spaces, but we ignore whitespace when looking713 // for signatures because we only care about things that look like a signature.714 // The hash we compute is intentionally dumb, we want collisions to catch similar strings,715 // so we just sum up the characters.716 const string beginSig = "sig#beginsignatureblock";717 beginSig.Aggregate(0, static (current, t) => current + t);718 719 // Spot check to help make sure the arrays are in sync720 Diagnostics.Assert(s_keywordTable["using"] == TokenKind.Using, "Keyword table out of sync w/ enum");721 Diagnostics.Assert(s_operatorTable["join"] == TokenKind.Join, "Operator table out of sync w/ enum");722 }723 724 internal Tokenizer(Parser parser)725 {726 _parser = parser;727 }728 729 internal TokenizerMode Mode { get; set; }730 731 internal bool AllowSignedNumbers { get; set; }732 733 // TODO: use auto-properties when making 'ternary operator' an official feature.734 private bool _forceEndNumberOnTernaryOpChars;735 736 internal bool ForceEndNumberOnTernaryOpChars737 {738 get { return _forceEndNumberOnTernaryOpChars; }739 set { _forceEndNumberOnTernaryOpChars = value; }740 }741 742 internal bool WantSimpleName { get; set; }743 744 internal bool InWorkflowContext { get; set; }745 746 internal List<Token> TokenList { get; set; }747 748 internal Token FirstToken { get; private set; }749 750 internal Token LastToken { get; private set; }751 752 private List<Token> RequiresTokens { get; set; }753 754 private bool InCommandMode() { return Mode == TokenizerMode.Command; }755 756 private bool InExpressionMode() { return Mode == TokenizerMode.Expression; }757 758 private bool InTypeNameMode() { return Mode == TokenizerMode.TypeName; }759 760 private bool InSignatureMode() { return Mode == TokenizerMode.Signature; }761 762 internal void Initialize(string fileName, string input, List<Token> tokenList)763 {764 _positionHelper = new PositionHelper(fileName, input);765 _script = input;766 this.TokenList = tokenList;767 this.FirstToken = null;768 this.LastToken = null;769 this.RequiresTokens = null;770 _beginSignatureExtent = null;771 772 List<int> lineStartMap = new List<int>(100) { 0 };773 for (int i = 0; i < input.Length; ++i)774 {775 char c = input[i];776 777 if (c == '\r')778 {779 if ((i + 1) < input.Length && input[i + 1] == '\n')780 {781 i += 1;782 }783 784 lineStartMap.Add(i + 1);785 }786 787 if (c == '\n')788 {789 lineStartMap.Add(i + 1);790 }791 }792 793 _currentIndex = 0;794 Mode = TokenizerMode.Command;795 796 _positionHelper.LineStartMap = lineStartMap.ToArray();797 }798 799 internal TokenizerState StartNestedScan(UnscannedSubExprToken nestedText)800 {801 TokenizerState ts = new TokenizerState802 {803 CurrentIndex = _currentIndex,804 NestedTokensAdjustment = _nestedTokensAdjustment,805 Script = _script,806 TokenStart = _tokenStart,807 FirstToken = FirstToken,808 LastToken = LastToken,809 SkippedCharOffsets = _skippedCharOffsets,810 TokenList = TokenList,811 };812 813 _currentIndex = 0;814 _nestedTokensAdjustment = ((InternalScriptExtent)nestedText.Extent).StartOffset;815 _script = nestedText.Value;816 _tokenStart = 0;817 _skippedCharOffsets = nestedText.SkippedCharOffsets;818 TokenList = (TokenList != null) ? new List<Token>() : null;819 820 return ts;821 }822 823 internal void FinishNestedScan(TokenizerState ts)824 {825 _currentIndex = ts.CurrentIndex;826 _nestedTokensAdjustment = ts.NestedTokensAdjustment;827 _script = ts.Script;828 _tokenStart = ts.TokenStart;829 FirstToken = ts.FirstToken;830 LastToken = ts.LastToken;831 _skippedCharOffsets = ts.SkippedCharOffsets;832 TokenList = ts.TokenList;833 }834 835 #region Utilities836 837 private char GetChar()838 {839 Diagnostics.Assert(_currentIndex >= 0, "GetChar reading before start of input.");840 Diagnostics.Assert(_currentIndex <= _script.Length + 1, "GetChar reading after end of input.");841 842 // Increment _currentIndex, even if it goes over the Length so callers can call UngetChar to unget EOF.843 int current = _currentIndex++;844 if (current >= _script.Length)845 {846 return '\0';847 }848 849 return _script[current];850 }851 852 private void UngetChar()853 {854 Diagnostics.Assert(_currentIndex > 0, "UngetChar ungetting before start of input.");855 856 _currentIndex -= 1;857 }858 859 private char PeekChar()860 {861 Diagnostics.Assert(_currentIndex >= 0 && _currentIndex <= _script.Length, "PeekChar out of range.");862 863 if (_currentIndex == _script.Length)864 {865 return '\0';866 }867 868 return _script[_currentIndex];869 }870 871 private void SkipChar()872 {873 Diagnostics.Assert((_currentIndex + 1) <= _script.Length, "SkipChar can't skip past EOF");874 875 _currentIndex += 1;876 }877 878 private bool AtEof()879 {880 return _currentIndex > _script.Length;881 }882 883 internal static bool IsKeyword(string str)884 {885 if (s_keywordTable.ContainsKey(str))886 {887 return true;888 }889 890 if (DynamicKeyword.ContainsKeyword(str) && !DynamicKeyword.IsHiddenKeyword(str))891 {892 return true;893 }894 895 return false;896 }897 898 internal void SkipNewlines(bool skipSemis)899 {900 // We normally don't create any tokens in a Skip method, but the901 // V2 tokenizer api returns newline, semi-colon, and line902 // continuation tokens so we create them as they are encountered.903 again:904 char c = GetChar();905 switch (c)906 {907 case ' ':908 case '\t':909 case '\f':910 case '\v':911 case SpecialChars.NoBreakSpace:912 case SpecialChars.NextLine:913 SkipWhiteSpace();914 goto again;915 916 case '\r':917 case '\n':918 ScanNewline(c);919 goto again;920 921 case ';':922 if (skipSemis)923 {924 ScanSemicolon();925 goto again;926 }927 928 break;929 930 case '#':931 _tokenStart = _currentIndex - 1;932 ScanLineComment();933 goto again;934 935 case '<':936 if (PeekChar() == '#')937 {938 _tokenStart = _currentIndex - 1;939 SkipChar();940 ScanBlockComment();941 goto again;942 }943 944 break;945 946 case '`':947 char c1 = GetChar();948 if (c1 == '\n' || c1 == '\r')949 {950 ScanLineContinuation(c1);951 goto again;952 }953 954 if (char.IsWhiteSpace(c1))955 {956 SkipWhiteSpace();957 goto again;958 }959 960 UngetChar();961 break;962 963 default:964 if (c.IsWhitespace())965 {966 SkipWhiteSpace();967 goto again;968 }969 970 break;971 }972 973 UngetChar();974 }975 976 private void SkipWhiteSpace()977 {978 while (true)979 {980 char c = PeekChar();981 if (!c.IsWhitespace())982 {983 break;984 }985 986 SkipChar();987 }988 }989 990 private void ScanNewline(char c)991 {992 _tokenStart = _currentIndex - 1;993 NormalizeCRLF(c);994 995 // Memory optimization: only create the token if it will be stored996 if (TokenList != null)997 {998 NewToken(TokenKind.NewLine);999 }1000 }1001 1002 private void ScanSemicolon()1003 {1004 _tokenStart = _currentIndex - 1;1005 1006 // Memory optimization: only create the token if it will be stored1007 if (TokenList != null)1008 {1009 NewToken(TokenKind.Semi);1010 }1011 }1012 1013 private void ScanLineContinuation(char c)1014 {1015 _tokenStart = _currentIndex - 2;1016 NormalizeCRLF(c);1017 1018 // Memory optimization: only create the token if it will be stored1019 if (TokenList != null)1020 {1021 NewToken(TokenKind.LineContinuation);1022 }1023 }1024 1025 internal int GetRestorePoint()1026 {1027 _tokenStart = _currentIndex;1028 return CurrentExtent().StartOffset;1029 }1030 1031 internal void Resync(Token token)1032 {1033 // The parser has decided to backtrack and the tokenizer needs to pretend that it's1034 // starting over from the beginning of token.1035 1036 Resync(((InternalScriptExtent)token.Extent).StartOffset);1037 }1038 1039 internal void Resync(int start)1040 {1041 int adjustment = _nestedTokensAdjustment;1042 if (_skippedCharOffsets != null)1043 {1044 for (int i = _nestedTokensAdjustment; i < start - 1 && i < _skippedCharOffsets.Length; ++i)1045 {1046 if (_skippedCharOffsets[i])1047 {1048 adjustment += 1;1049 }1050 }1051 }1052 1053 _currentIndex = start - adjustment;1054 if (_currentIndex > _script.Length + 1)1055 {1056 _currentIndex = _script.Length + 1;1057 }1058 else if (_currentIndex < 0)1059 {1060 _currentIndex = 0;1061 }1062 1063 if (FirstToken != null && _currentIndex <= ((InternalScriptExtent)FirstToken.Extent).StartOffset)1064 {1065 FirstToken = null;1066 }1067 1068 if (TokenList != null && TokenList.Count > 0)1069 {1070 // If we were saving tokens, remove all tokens from token to the end of the saved tokens.1071 RemoveTokensFromListDuringResync(TokenList, start);1072 }1073 1074 if (RequiresTokens != null && RequiresTokens.Count > 0)1075 {1076 RemoveTokensFromListDuringResync(RequiresTokens, start);1077 }1078 }1079 1080 internal void RemoveTokensFromListDuringResync(List<Token> tokenList, int start)1081 {1082 int removeFrom = 0;1083 // If we were saving tokens, remove all tokens from token to the end of the saved tokens.1084 int i = tokenList.Count - 1;1085 if (i >= 0 && tokenList[i].Kind == TokenKind.EndOfInput)1086 {1087 i -= 1;1088 }1089 1090 for (; i >= 0; i--)1091 {1092 if (((InternalScriptExtent)tokenList[i].Extent).EndOffset <= start)1093 {1094 removeFrom = i + 1;1095 break;1096 }1097 }1098 1099 tokenList.RemoveRange(removeFrom, tokenList.Count - removeFrom);1100 }1101 1102 internal void ReplaceSavedTokens(Token firstOldToken, Token lastOldToken, Token newToken)1103 {1104 int startOffset = ((InternalScriptExtent)firstOldToken.Extent).StartOffset;1105 int endOffset = ((InternalScriptExtent)lastOldToken.Extent).EndOffset;1106 int lastTokenToReplace = -1;1107 for (int i = TokenList.Count - 1; i >= 0; i--)1108 {1109 if (((InternalScriptExtent)TokenList[i].Extent).EndOffset == endOffset)1110 {1111 lastTokenToReplace = i;1112 continue;1113 }1114 1115 if (((InternalScriptExtent)TokenList[i].Extent).StartOffset == startOffset)1116 {1117 TokenList.RemoveRange(i, lastTokenToReplace - i + 1);1118 TokenList.Insert(i, newToken);1119 break;1120 }1121 }1122 }1123 1124 private void NormalizeCRLF(char c)1125 {1126 // CRs in Windows line endings are ignored1127 if (c == '\r' && PeekChar() == '\n')1128 {1129 SkipChar();1130 }1131 }1132 1133 internal void CheckAstIsBeforeSignature(Ast ast)1134 {1135 if (_beginSignatureExtent == null)1136 return;1137 1138 if (_beginSignatureExtent.StartOffset < ast.Extent.StartOffset)1139 {1140 ReportError(ast.Extent,1141 nameof(ParserStrings.TokenAfterEndOfValidScriptText),1142 ParserStrings.TokenAfterEndOfValidScriptText);1143 }1144 }1145 1146 private void ReportError(int errorOffset, string errorId, string errorMsg, params object[] args)1147 {1148 _parser.ReportError(NewScriptExtent(errorOffset, errorOffset + 1), errorId, errorMsg, args);1149 }1150 1151 private void ReportError(IScriptExtent extent, string errorId, string errorMsg)1152 {1153 _parser.ReportError(extent, errorId, errorMsg);1154 }1155 1156 private void ReportError(IScriptExtent extent, string errorId, string errorMsg, object arg)1157 {1158 _parser.ReportError(extent, errorId, errorMsg, arg);1159 }1160 1161 private void ReportError(IScriptExtent extent, string errorId, string errorMsg, object arg1, object arg2)1162 {1163 _parser.ReportError(extent, errorId, errorMsg, arg1, arg2);1164 }1165 1166 private void ReportIncompleteInput(int errorOffset, string errorId, string errorMsg)1167 {1168 _parser.ReportIncompleteInput(NewScriptExtent(errorOffset, _currentIndex), errorId, errorMsg);1169 }1170 1171 private void ReportIncompleteInput(int errorOffset, string errorId, string errorMsg, object arg)1172 {1173 _parser.ReportIncompleteInput(NewScriptExtent(errorOffset, _currentIndex), errorId, errorMsg, arg);1174 }1175 1176 private InternalScriptExtent NewScriptExtent(int start, int end)1177 {1178 return new InternalScriptExtent(_positionHelper, start + _nestedTokensAdjustment, end + _nestedTokensAdjustment);1179 }1180 1181 internal InternalScriptExtent CurrentExtent()1182 {1183 int start = _tokenStart + _nestedTokensAdjustment;1184 int end = _currentIndex + _nestedTokensAdjustment;1185 if (_skippedCharOffsets != null)1186 {1187 int i = _nestedTokensAdjustment;1188 for (; i < start && i < _skippedCharOffsets.Length; ++i)1189 {1190 if (_skippedCharOffsets[i])1191 {1192 start += 1;1193 end += 1;1194 }1195 }1196 1197 for (; i < end && i < _skippedCharOffsets.Length; ++i)1198 {1199 if (_skippedCharOffsets[i])1200 {