aravagarwal/CodeCloakPII
0
1/**2 * @file Python grammar for tree-sitter3 * @author Max Brunsfeld <maxbrunsfeld@gmail.com>4 * @license MIT5 * @see {@link https://docs.python.org/2/reference/grammar.html|Python 2 grammar}6 * @see {@link https://docs.python.org/3/reference/grammar.html|Python 3 grammar}7 */8 9/* eslint-disable arrow-parens */10/* eslint-disable camelcase */11/* eslint-disable-next-line spaced-comment */12/// <reference types="tree-sitter-cli/dsl" />13// @ts-check14 15const PREC = {16 // this resolves a conflict between the usage of ':' in a lambda vs in a17 // typed parameter. In the case of a lambda, we don't allow typed parameters.18 lambda: -2,19 typed_parameter: -1,20 conditional: -1,21 22 parenthesized_expression: 1,23 parenthesized_list_splat: 1,24 or: 10,25 and: 11,26 not: 12,27 compare: 13,28 bitwise_or: 14,29 bitwise_and: 15,30 xor: 16,31 shift: 17,32 plus: 18,33 times: 19,34 unary: 20,35 power: 21,36 call: 22,37};38 39const SEMICOLON = ';';40 41module.exports = grammar({42 name: 'python',43 44 extras: $ => [45 $.comment,46 /[\s\f\uFEFF\u2060\u200B]|\r?\n/,47 $.line_continuation,48 ],49 50 conflicts: $ => [51 [$.primary_expression, $.pattern],52 [$.primary_expression, $.list_splat_pattern],53 [$.tuple, $.tuple_pattern],54 [$.list, $.list_pattern],55 [$.with_item, $._collection_elements],56 [$.named_expression, $.as_pattern],57 [$.print_statement, $.primary_expression],58 [$.type_alias_statement, $.primary_expression],59 ],60 61 supertypes: $ => [62 $._simple_statement,63 $._compound_statement,64 $.expression,65 $.primary_expression,66 $.pattern,67 $.parameter,68 ],69 70 externals: $ => [71 $._newline,72 $._indent,73 $._dedent,74 $.string_start,75 $._string_content,76 $.escape_interpolation,77 $.string_end,78 79 // Mark comments as external tokens so that the external scanner is always80 // invoked, even if no external token is expected. This allows for better81 // error recovery, because the external scanner can maintain the overall82 // structure by returning dedent tokens whenever a dedent occurs, even83 // if no dedent is expected.84 $.comment,85 86 // Allow the external scanner to check for the validity of closing brackets87 // so that it can avoid returning dedent tokens between brackets.88 ']',89 ')',90 '}',91 'except',92 ],93 94 inline: $ => [95 $._simple_statement,96 $._compound_statement,97 $._suite,98 $._expressions,99 $._left_hand_side,100 $.keyword_identifier,101 ],102 103 word: $ => $.identifier,104 105 rules: {106 module: $ => repeat($._statement),107 108 _statement: $ => choice(109 $._simple_statements,110 $._compound_statement,111 ),112 113 // Simple statements114 115 _simple_statements: $ => seq(116 sep1($._simple_statement, SEMICOLON),117 optional(SEMICOLON),118 $._newline,119 ),120 121 _simple_statement: $ => choice(122 $.future_import_statement,123 $.import_statement,124 $.import_from_statement,125 $.print_statement,126 $.assert_statement,127 $.expression_statement,128 $.return_statement,129 $.delete_statement,130 $.raise_statement,131 $.pass_statement,132 $.break_statement,133 $.continue_statement,134 $.global_statement,135 $.nonlocal_statement,136 $.exec_statement,137 $.type_alias_statement,138 ),139 140 import_statement: $ => seq(141 'import',142 $._import_list,143 ),144 145 import_prefix: _ => repeat1('.'),146 147 relative_import: $ => seq(148 $.import_prefix,149 optional($.dotted_name),150 ),151 152 future_import_statement: $ => seq(153 'from',154 '__future__',155 'import',156 choice(157 $._import_list,158 seq('(', $._import_list, ')'),159 ),160 ),161 162 import_from_statement: $ => seq(163 'from',164 field('module_name', choice(165 $.relative_import,166 $.dotted_name,167 )),168 'import',169 choice(170 $.wildcard_import,171 $._import_list,172 seq('(', $._import_list, ')'),173 ),174 ),175 176 _import_list: $ => seq(177 commaSep1(field('name', choice(178 $.dotted_name,179 $.aliased_import,180 ))),181 optional(','),182 ),183 184 aliased_import: $ => seq(185 field('name', $.dotted_name),186 'as',187 field('alias', $.identifier),188 ),189 190 wildcard_import: _ => '*',191 192 print_statement: $ => choice(193 prec(1, seq(194 'print',195 $.chevron,196 repeat(seq(',', field('argument', $.expression))),197 optional(',')),198 ),199 prec(-3, prec.dynamic(-1, seq(200 'print',201 commaSep1(field('argument', $.expression)),202 optional(','),203 ))),204 ),205 206 chevron: $ => seq(207 '>>',208 $.expression,209 ),210 211 assert_statement: $ => seq(212 'assert',213 commaSep1($.expression),214 ),215 216 expression_statement: $ => choice(217 $.expression,218 seq(commaSep1($.expression), optional(',')),219 $.assignment,220 $.augmented_assignment,221 $.yield,222 ),223 224 named_expression: $ => seq(225 field('name', $._named_expression_lhs),226 ':=',227 field('value', $.expression),228 ),229 230 _named_expression_lhs: $ => choice(231 $.identifier,232 $.keyword_identifier,233 ),234 235 return_statement: $ => seq(236 'return',237 optional($._expressions),238 ),239 240 delete_statement: $ => seq(241 'del',242 $._expressions,243 ),244 245 _expressions: $ => choice(246 $.expression,247 $.expression_list,248 ),249 250 raise_statement: $ => seq(251 'raise',252 optional($._expressions),253 optional(seq('from', field('cause', $.expression))),254 ),255 256 pass_statement: _ => prec.left('pass'),257 break_statement: _ => prec.left('break'),258 continue_statement: _ => prec.left('continue'),259 260 // Compound statements261 262 _compound_statement: $ => choice(263 $.if_statement,264 $.for_statement,265 $.while_statement,266 $.try_statement,267 $.with_statement,268 $.function_definition,269 $.class_definition,270 $.decorated_definition,271 $.match_statement,272 ),273 274 if_statement: $ => seq(275 'if',276 field('condition', $.expression),277 ':',278 field('consequence', $._suite),279 repeat(field('alternative', $.elif_clause)),280 optional(field('alternative', $.else_clause)),281 ),282 283 elif_clause: $ => seq(284 'elif',285 field('condition', $.expression),286 ':',287 field('consequence', $._suite),288 ),289 290 else_clause: $ => seq(291 'else',292 ':',293 field('body', $._suite),294 ),295 296 match_statement: $ => seq(297 'match',298 commaSep1(field('subject', $.expression)),299 optional(','),300 ':',301 field('body', alias($._match_block, $.block)),302 ),303 304 _match_block: $ => choice(305 seq(306 $._indent,307 repeat(field('alternative', $.case_clause)),308 $._dedent,309 ),310 $._newline,311 ),312 313 case_clause: $ => seq(314 'case',315 commaSep1($.case_pattern),316 optional(','),317 optional(field('guard', $.if_clause)),318 ':',319 field('consequence', $._suite),320 ),321 322 for_statement: $ => seq(323 optional('async'),324 'for',325 field('left', $._left_hand_side),326 'in',327 field('right', $._expressions),328 ':',329 field('body', $._suite),330 field('alternative', optional($.else_clause)),331 ),332 333 while_statement: $ => seq(334 'while',335 field('condition', $.expression),336 ':',337 field('body', $._suite),338 optional(field('alternative', $.else_clause)),339 ),340 341 try_statement: $ => seq(342 'try',343 ':',344 field('body', $._suite),345 choice(346 seq(347 repeat1($.except_clause),348 optional($.else_clause),349 optional($.finally_clause),350 ),351 seq(352 repeat1($.except_group_clause),353 optional($.else_clause),354 optional($.finally_clause),355 ),356 $.finally_clause,357 ),358 ),359 360 except_clause: $ => seq(361 'except',362 optional(seq(363 $.expression,364 optional(seq(365 choice('as', ','),366 $.expression,367 )),368 )),369 ':',370 $._suite,371 ),372 373 except_group_clause: $ => seq(374 'except*',375 seq(376 $.expression,377 optional(seq(378 'as',379 $.expression,380 )),381 ),382 ':',383 $._suite,384 ),385 386 finally_clause: $ => seq(387 'finally',388 ':',389 $._suite,390 ),391 392 with_statement: $ => seq(393 optional('async'),394 'with',395 $.with_clause,396 ':',397 field('body', $._suite),398 ),399 400 with_clause: $ => choice(401 seq(commaSep1($.with_item), optional(',')),402 seq('(', commaSep1($.with_item), optional(','), ')'),403 ),404 405 with_item: $ => prec.dynamic(1, seq(406 field('value', $.expression),407 )),408 409 function_definition: $ => seq(410 optional('async'),411 'def',412 field('name', $.identifier),413 field('type_parameters', optional($.type_parameter)),414 field('parameters', $.parameters),415 optional(416 seq(417 '->',418 field('return_type', $.type),419 ),420 ),421 ':',422 field('body', $._suite),423 ),424 425 parameters: $ => seq(426 '(',427 optional($._parameters),428 ')',429 ),430 431 lambda_parameters: $ => $._parameters,432 433 list_splat: $ => seq(434 '*',435 $.expression,436 ),437 438 dictionary_splat: $ => seq(439 '**',440 $.expression,441 ),442 443 global_statement: $ => seq(444 'global',445 commaSep1($.identifier),446 ),447 448 nonlocal_statement: $ => seq(449 'nonlocal',450 commaSep1($.identifier),451 ),452 453 exec_statement: $ => seq(454 'exec',455 field('code', choice($.string, $.identifier)),456 optional(457 seq(458 'in',459 commaSep1($.expression),460 ),461 ),462 ),463 464 type_alias_statement: $ => prec.dynamic(1, seq(465 'type',466 $.type,467 '=',468 $.type,469 )),470 471 class_definition: $ => seq(472 'class',473 field('name', $.identifier),474 field('type_parameters', optional($.type_parameter)),475 field('superclasses', optional($.argument_list)),476 ':',477 field('body', $._suite),478 ),479 type_parameter: $ => seq(480 '[',481 commaSep1($.type),482 ']',483 ),484 485 parenthesized_list_splat: $ => prec(PREC.parenthesized_list_splat, seq(486 '(',487 choice(488 alias($.parenthesized_list_splat, $.parenthesized_expression),489 $.list_splat,490 ),491 ')',492 )),493 494 argument_list: $ => seq(495 '(',496 optional(commaSep1(497 choice(498 $.expression,499 $.list_splat,500 $.dictionary_splat,501 alias($.parenthesized_list_splat, $.parenthesized_expression),502 $.keyword_argument,503 ),504 )),505 optional(','),506 ')',507 ),508 509 decorated_definition: $ => seq(510 repeat1($.decorator),511 field('definition', choice(512 $.class_definition,513 $.function_definition,514 )),515 ),516 517 decorator: $ => seq(518 '@',519 $.expression,520 $._newline,521 ),522 523 _suite: $ => choice(524 alias($._simple_statements, $.block),525 seq($._indent, $.block),526 alias($._newline, $.block),527 ),528 529 block: $ => seq(530 repeat($._statement),531 $._dedent,532 ),533 534 expression_list: $ => prec.right(seq(535 $.expression,536 choice(537 ',',538 seq(539 repeat1(seq(540 ',',541 $.expression,542 )),543 optional(','),544 ),545 ),546 )),547 548 dotted_name: $ => prec(1, sep1($.identifier, '.')),549 550 // Match cases551 552 case_pattern: $ => prec(1, choice(553 alias($._as_pattern, $.as_pattern),554 $.keyword_pattern,555 $._simple_pattern,556 )),557 558 _simple_pattern: $ => prec(1, choice(559 $.class_pattern,560 $.splat_pattern,561 $.union_pattern,562 alias($._list_pattern, $.list_pattern),563 alias($._tuple_pattern, $.tuple_pattern),564 $.dict_pattern,565 $.string,566 $.concatenated_string,567 $.true,568 $.false,569 $.none,570 seq(optional('-'), choice($.integer, $.float)),571 $.complex_pattern,572 $.dotted_name,573 '_',574 )),575 576 _as_pattern: $ => seq($.case_pattern, 'as', $.identifier),577 578 union_pattern: $ => prec.right(seq($._simple_pattern, repeat1(prec.left(seq('|', $._simple_pattern))))),579 580 _list_pattern: $ => seq(581 '[',582 optional(seq(583 commaSep1($.case_pattern),584 optional(','),585 )),586 ']',587 ),588 589 _tuple_pattern: $ => seq(590 '(',591 optional(seq(592 commaSep1($.case_pattern),593 optional(','),594 )),595 ')',596 ),597 598 dict_pattern: $ => seq(599 '{',600 optional(seq(601 commaSep1(choice($._key_value_pattern, $.splat_pattern)),602 optional(','),603 )),604 '}',605 ),606 607 _key_value_pattern: $ => seq(608 field('key', $._simple_pattern),609 ':',610 field('value', $.case_pattern),611 ),612 613 keyword_pattern: $ => seq($.identifier, '=', $._simple_pattern),614 615 splat_pattern: $ => prec(1, seq(choice('*', '**'), choice($.identifier, '_'))),616 617 class_pattern: $ => seq(618 $.dotted_name,619 '(',620 optional(seq(621 commaSep1($.case_pattern),622 optional(','),623 )),624 ')',625 ),626 627 complex_pattern: $ => prec(1, seq(628 optional('-'),629 choice($.integer, $.float),630 choice('+', '-'),631 choice($.integer, $.float),632 )),633 634 // Patterns635 636 _parameters: $ => seq(637 commaSep1($.parameter),638 optional(','),639 ),640 641 _patterns: $ => seq(642 commaSep1($.pattern),643 optional(','),644 ),645 646 parameter: $ => choice(647 $.identifier,648 $.typed_parameter,649 $.default_parameter,650 $.typed_default_parameter,651 $.list_splat_pattern,652 $.tuple_pattern,653 $.keyword_separator,654 $.positional_separator,655 $.dictionary_splat_pattern,656 ),657 658 pattern: $ => choice(659 $.identifier,660 $.keyword_identifier,661 $.subscript,662 $.attribute,663 $.list_splat_pattern,664 $.tuple_pattern,665 $.list_pattern,666 ),667 668 tuple_pattern: $ => seq(669 '(',670 optional($._patterns),671 ')',672 ),673 674 list_pattern: $ => seq(675 '[',676 optional($._patterns),677 ']',678 ),679 680 default_parameter: $ => seq(681 field('name', choice($.identifier, $.tuple_pattern)),682 '=',683 field('value', $.expression),684 ),685 686 typed_default_parameter: $ => prec(PREC.typed_parameter, seq(687 field('name', $.identifier),688 ':',689 field('type', $.type),690 '=',691 field('value', $.expression),692 )),693 694 list_splat_pattern: $ => seq(695 '*',696 choice($.identifier, $.keyword_identifier, $.subscript, $.attribute),697 ),698 699 dictionary_splat_pattern: $ => seq(700 '**',701 choice($.identifier, $.keyword_identifier, $.subscript, $.attribute),702 ),703 704 // Extended patterns (patterns allowed in match statement are far more flexible than simple patterns though still a subset of "expression")705 706 as_pattern: $ => prec.left(seq(707 $.expression,708 'as',709 field('alias', alias($.expression, $.as_pattern_target)),710 )),711 712 // Expressions713 714 _expression_within_for_in_clause: $ => choice(715 $.expression,716 alias($.lambda_within_for_in_clause, $.lambda),717 ),718 719 expression: $ => choice(720 $.comparison_operator,721 $.not_operator,722 $.boolean_operator,723 $.lambda,724 $.primary_expression,725 $.conditional_expression,726 $.named_expression,727 $.as_pattern,728 ),729 730 primary_expression: $ => choice(731 $.await,732 $.binary_operator,733 $.identifier,734 $.keyword_identifier,735 $.string,736 $.concatenated_string,737 $.integer,738 $.float,739 $.true,740 $.false,741 $.none,742 $.unary_operator,743 $.attribute,744 $.subscript,745 $.call,746 $.list,747 $.list_comprehension,748 $.dictionary,749 $.dictionary_comprehension,750 $.set,751 $.set_comprehension,752 $.tuple,753 $.parenthesized_expression,754 $.generator_expression,755 $.ellipsis,756 alias($.list_splat_pattern, $.list_splat),757 ),758 759 not_operator: $ => prec(PREC.not, seq(760 'not',761 field('argument', $.expression),762 )),763 764 boolean_operator: $ => choice(765 prec.left(PREC.and, seq(766 field('left', $.expression),767 field('operator', 'and'),768 field('right', $.expression),769 )),770 prec.left(PREC.or, seq(771 field('left', $.expression),772 field('operator', 'or'),773 field('right', $.expression),774 )),775 ),776 777 binary_operator: $ => {778 const table = [779 [prec.left, '+', PREC.plus],780 [prec.left, '-', PREC.plus],781 [prec.left, '*', PREC.times],782 [prec.left, '@', PREC.times],783 [prec.left, '/', PREC.times],784 [prec.left, '%', PREC.times],785 [prec.left, '//', PREC.times],786 [prec.right, '**', PREC.power],787 [prec.left, '|', PREC.bitwise_or],788 [prec.left, '&', PREC.bitwise_and],789 [prec.left, '^', PREC.xor],790 [prec.left, '<<', PREC.shift],791 [prec.left, '>>', PREC.shift],792 ];793 794 // @ts-ignore795 return choice(...table.map(([fn, operator, precedence]) => fn(precedence, seq(796 field('left', $.primary_expression),797 // @ts-ignore798 field('operator', operator),799 field('right', $.primary_expression),800 ))));801 },802 803 unary_operator: $ => prec(PREC.unary, seq(804 field('operator', choice('+', '-', '~')),805 field('argument', $.primary_expression),806 )),807 808 comparison_operator: $ => prec.left(PREC.compare, seq(809 $.primary_expression,810 repeat1(seq(811 field('operators',812 choice(813 '<',814 '<=',815 '==',816 '!=',817 '>=',818 '>',819 '<>',820 'in',821 alias(seq('not', 'in'), 'not in'),822 'is',823 alias(seq('is', 'not'), 'is not'),824 )),825 $.primary_expression,826 )),827 )),828 829 lambda: $ => prec(PREC.lambda, seq(830 'lambda',831 field('parameters', optional($.lambda_parameters)),832 ':',833 field('body', $.expression),834 )),835 836 lambda_within_for_in_clause: $ => seq(837 'lambda',838 field('parameters', optional($.lambda_parameters)),839 ':',840 field('body', $._expression_within_for_in_clause),841 ),842 843 assignment: $ => seq(844 field('left', $._left_hand_side),845 choice(846 seq('=', field('right', $._right_hand_side)),847 seq(':', field('type', $.type)),848 seq(':', field('type', $.type), '=', field('right', $._right_hand_side)),849 ),850 ),851 852 augmented_assignment: $ => seq(853 field('left', $._left_hand_side),854 field('operator', choice(855 '+=', '-=', '*=', '/=', '@=', '//=', '%=', '**=',856 '>>=', '<<=', '&=', '^=', '|=',857 )),858 field('right', $._right_hand_side),859 ),860 861 _left_hand_side: $ => choice(862 $.pattern,863 $.pattern_list,864 ),865 866 pattern_list: $ => seq(867 $.pattern,868 choice(869 ',',870 seq(871 repeat1(seq(872 ',',873 $.pattern,874 )),875 optional(','),876 ),877 ),878 ),879 880 _right_hand_side: $ => choice(881 $.expression,882 $.expression_list,883 $.assignment,884 $.augmented_assignment,885 $.pattern_list,886 $.yield,887 ),888 889 yield: $ => prec.right(seq(890 'yield',891 choice(892 seq(893 'from',894 $.expression,895 ),896 optional($._expressions),897 ),898 )),899 900 attribute: $ => prec(PREC.call, seq(901 field('object', $.primary_expression),902 '.',903 field('attribute', $.identifier),904 )),905 906 subscript: $ => prec(PREC.call, seq(907 field('value', $.primary_expression),908 '[',909 commaSep1(field('subscript', choice($.expression, $.slice))),910 optional(','),911 ']',912 )),913 914 slice: $ => seq(915 optional($.expression),916 ':',917 optional($.expression),918 optional(seq(':', optional($.expression))),919 ),920 921 ellipsis: _ => '...',922 923 call: $ => prec(PREC.call, seq(924 field('function', $.primary_expression),925 field('arguments', choice(926 $.generator_expression,927 $.argument_list,928 )),929 )),930 931 typed_parameter: $ => prec(PREC.typed_parameter, seq(932 choice(933 $.identifier,934 $.list_splat_pattern,935 $.dictionary_splat_pattern,936 ),937 ':',938 field('type', $.type),939 )),940 941 type: $ => choice(942 $.expression,943 $.splat_type,944 $.generic_type,945 $.union_type,946 $.constrained_type,947 $.member_type,948 ),949 splat_type: $ => prec(1, seq(choice('*', '**'), $.identifier)),950 generic_type: $ => prec(1, seq($.identifier, $.type_parameter)),951 union_type: $ => prec.left(seq($.type, '|', $.type)),952 constrained_type: $ => prec.right(seq($.type, ':', $.type)),953 member_type: $ => seq($.type, '.', $.identifier),954 955 keyword_argument: $ => seq(956 field('name', choice($.identifier, $.keyword_identifier)),957 '=',958 field('value', $.expression),959 ),960 961 // Literals962 963 list: $ => seq(964 '[',965 optional($._collection_elements),966 ']',967 ),968 969 set: $ => seq(970 '{',971 $._collection_elements,972 '}',973 ),974 975 tuple: $ => seq(976 '(',977 optional($._collection_elements),978 ')',979 ),980 981 dictionary: $ => seq(982 '{',983 optional(commaSep1(choice($.pair, $.dictionary_splat))),984 optional(','),985 '}',986 ),987 988 pair: $ => seq(989 field('key', $.expression),990 ':',991 field('value', $.expression),992 ),993 994 list_comprehension: $ => seq(995 '[',996 field('body', $.expression),997 $._comprehension_clauses,998 ']',999 ),1000 1001 dictionary_comprehension: $ => seq(1002 '{',1003 field('body', $.pair),1004 $._comprehension_clauses,1005 '}',1006 ),1007 1008 set_comprehension: $ => seq(1009 '{',1010 field('body', $.expression),1011 $._comprehension_clauses,1012 '}',1013 ),1014 1015 generator_expression: $ => seq(1016 '(',1017 field('body', $.expression),1018 $._comprehension_clauses,1019 ')',1020 ),1021 1022 _comprehension_clauses: $ => seq(1023 $.for_in_clause,1024 repeat(choice(1025 $.for_in_clause,1026 $.if_clause,1027 )),1028 ),1029 1030 parenthesized_expression: $ => prec(PREC.parenthesized_expression, seq(1031 '(',1032 choice($.expression, $.yield),1033 ')',1034 )),1035 1036 _collection_elements: $ => seq(1037 commaSep1(choice(1038 $.expression, $.yield, $.list_splat, $.parenthesized_list_splat,1039 )),1040 optional(','),1041 ),1042 1043 for_in_clause: $ => prec.left(seq(1044 optional('async'),1045 'for',1046 field('left', $._left_hand_side),1047 'in',1048 field('right', commaSep1($._expression_within_for_in_clause)),1049 optional(','),1050 )),1051 1052 if_clause: $ => seq(1053 'if',1054 $.expression,1055 ),1056 1057 conditional_expression: $ => prec.right(PREC.conditional, seq(1058 $.expression,1059 'if',1060 $.expression,1061 'else',1062 $.expression,1063 )),1064 1065 concatenated_string: $ => seq(1066 $.string,1067 repeat1($.string),1068 ),1069 1070 string: $ => seq(1071 $.string_start,1072 repeat(choice($.interpolation, $.string_content)),1073 $.string_end,1074 ),1075 1076 string_content: $ => prec.right(repeat1(1077 choice(1078 $.escape_interpolation,1079 $.escape_sequence,1080 $._not_escape_sequence,1081 $._string_content,1082 ))),1083 1084 interpolation: $ => seq(1085 '{',1086 field('expression', $._f_expression),1087 optional('='),1088 optional(field('type_conversion', $.type_conversion)),1089 optional(field('format_specifier', $.format_specifier)),1090 '}',1091 ),1092 1093 _f_expression: $ => choice(1094 $.expression,1095 $.expression_list,1096 $.pattern_list,1097 $.yield,1098 ),1099 1100 escape_sequence: _ => token.immediate(prec(1, seq(1101 '\\',1102 choice(1103 /u[a-fA-F\d]{4}/,1104 /U[a-fA-F\d]{8}/,1105 /x[a-fA-F\d]{2}/,1106 /\d{3}/,1107 /\r?\n/,1108 /['"abfrntv\\]/,1109 /N\{[^}]+\}/,1110 ),1111 ))),1112 1113 _not_escape_sequence: _ => token.immediate('\\'),1114 1115 format_specifier: $ => seq(1116 ':',1117 repeat(choice(1118 token(prec(1, /[^{}\n]+/)),1119 alias($.interpolation, $.format_expression),1120 )),1121 ),1122 1123 type_conversion: _ => /![a-z]/,1124 1125 integer: _ => token(choice(1126 seq(1127 choice('0x', '0X'),1128 repeat1(/_?[A-Fa-f0-9]+/),1129 optional(/[Ll]/),1130 ),1131 seq(1132 choice('0o', '0O'),1133 repeat1(/_?[0-7]+/),1134 optional(/[Ll]/),1135 ),1136 seq(1137 choice('0b', '0B'),1138 repeat1(/_?[0-1]+/),1139 optional(/[Ll]/),1140 ),1141 seq(1142 repeat1(/[0-9]+_?/),1143 choice(1144 optional(/[Ll]/), // long numbers1145 optional(/[jJ]/), // complex numbers1146 ),1147 ),1148 )),1149 1150 float: _ => {1151 const digits = repeat1(/[0-9]+_?/);1152 const exponent = seq(/[eE][\+-]?/, digits);1153 1154 return token(seq(1155 choice(1156 seq(digits, '.', optional(digits), optional(exponent)),1157 seq(optional(digits), '.', digits, optional(exponent)),1158 seq(digits, exponent),1159 ),1160 optional(choice(/[Ll]/, /[jJ]/)),1161 ));1162 },1163 1164 identifier: _ => /[_\p{XID_Start}][_\p{XID_Continue}]*/,1165 1166 keyword_identifier: $ => choice(1167 prec(-3, alias(1168 choice(1169 'print',1170 'exec',1171 'async',1172 'await',1173 'match',1174 ),1175 $.identifier,1176 )),1177 alias('type', $.identifier),1178 ),1179 1180 true: _ => 'True',1181 false: _ => 'False',1182 none: _ => 'None',1183 1184 await: $ => prec(PREC.unary, seq(1185 'await',1186 $.primary_expression,1187 )),1188 1189 comment: _ => token(seq('#', /.*/)),1190 1191 line_continuation: _ => token(seq('\\', choice(seq(optional('\r'), '\n'), '\0'))),1192 1193 positional_separator: _ => '/',1194 keyword_separator: _ => '*',1195 },1196});1197 1198module.exports.PREC = PREC;1199 1200/**