parthtamu/rag-code-assistant
0
1<!DOCTYPE html>2 3<html lang="en" data-content_root="../">4 <head>5 <meta charset="utf-8" />6 <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />7<meta property="og:title" content="token — Constants used with Python parse trees" />8<meta property="og:type" content="website" />9<meta property="og:url" content="https://docs.python.org/3/library/token.html" />10<meta property="og:site_name" content="Python documentation" />11<meta property="og:description" content="Source code: Lib/token.py This module provides constants which represent the numeric values of leaf nodes of the parse tree (terminal tokens). Refer to the file Grammar/Tokens in the Python distrib..." />12<meta property="og:image:width" content="1146" />13<meta property="og:image:height" content="600" />14<meta property="og:image" content="https://docs.python.org/3.15/_images/social_previews/summary_library_token_dfc39315.png" />15<meta property="og:image:alt" content="Source code: Lib/token.py This module provides constants which represent the numeric values of leaf nodes of the parse tree (terminal tokens). Refer to the file Grammar/Tokens in the Python distrib..." />16<meta name="description" content="Source code: Lib/token.py This module provides constants which represent the numeric values of leaf nodes of the parse tree (terminal tokens). Refer to the file Grammar/Tokens in the Python distrib..." />17<meta name="twitter:card" content="summary_large_image" />18<meta name="theme-color" content="#3776ab">19 20 <title>token — Constants used with Python parse trees — Python 3.15.0a6 documentation</title><meta name="viewport" content="width=device-width, initial-scale=1.0">21 22 <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />23 <link rel="stylesheet" type="text/css" href="../_static/classic.css?v=234b1a7c" />24 <link rel="stylesheet" type="text/css" href="../_static/pydoctheme.css?v=89a2f22a" />25 <link rel="stylesheet" type="text/css" href="../_static/profiling-sampling-visualization.css?v=0c2600ae" />26 <link id="pygments_dark_css" media="(prefers-color-scheme: dark)" rel="stylesheet" type="text/css" href="../_static/pygments_dark.css?v=5349f25f" />27 28 <script src="../_static/documentation_options.js?v=6b7c9ff5"></script>29 <script src="../_static/doctools.js?v=9bcbadda"></script>30 <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>31 <script src="../_static/profiling-sampling-visualization.js?v=9811ed04"></script>32 33 <script src="../_static/sidebar.js"></script>34 35 <link rel="search" type="application/opensearchdescription+xml"36 title="Search within Python 3.15.0a6 documentation"37 href="../_static/opensearch.xml"/>38 <link rel="author" title="About these documents" href="../about.html" />39 <link rel="index" title="Index" href="../genindex.html" />40 <link rel="search" title="Search" href="../search.html" />41 <link rel="copyright" title="Copyright" href="../copyright.html" />42 <link rel="next" title="keyword — Testing for Python keywords" href="keyword.html" />43 <link rel="prev" title="symtable — Access to the compiler’s symbol tables" href="symtable.html" />44 45 46 <script defer file-types="bz2,epub,zip" data-domain="docs.python.org" src="https://analytics.python.org/js/script.file-downloads.outbound-links.js"></script>47 48 <link rel="canonical" href="https://docs.python.org/3/library/token.html">49 50 51 52 53 <style>54 @media only screen {55 table.full-width-table {56 width: 100%;57 }58 }59 </style>60<link rel="stylesheet" href="../_static/pydoctheme_dark.css" media="(prefers-color-scheme: dark)" id="pydoctheme_dark_css">61 <link rel="shortcut icon" type="image/png" href="../_static/py.svg">62 <script type="text/javascript" src="../_static/copybutton.js"></script>63 <script type="text/javascript" src="../_static/menu.js"></script>64 <script type="text/javascript" src="../_static/search-focus.js"></script>65 <script type="text/javascript" src="../_static/themetoggle.js"></script> 66 <script type="text/javascript" src="../_static/rtd_switcher.js"></script>67 <meta name="readthedocs-addons-api-version" content="1">68 69 </head>70<body>71<div class="mobile-nav">72 <input type="checkbox" id="menuToggler" class="toggler__input" aria-controls="navigation"73 aria-pressed="false" aria-expanded="false" role="button" aria-label="Menu">74 <nav class="nav-content" role="navigation">75 <label for="menuToggler" class="toggler__label">76 <span></span>77 </label>78 <span class="nav-items-wrapper">79 <a href="https://www.python.org/" class="nav-logo">80 <img src="../_static/py.svg" alt="Python logo">81 </a>82 <span class="version_switcher_placeholder"></span>83 <form role="search" class="search" action="../search.html" method="get">84 <svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" class="search-icon">85 <path fill-rule="nonzero" fill="currentColor" d="M15.5 14h-.79l-.28-.27a6.5 6.5 0 001.48-5.34c-.47-2.78-2.79-5-5.59-5.34a6.505 6.505 0 00-7.27 7.27c.34 2.8 2.56 5.12 5.34 5.59a6.5 6.5 0 005.34-1.48l.27.28v.79l4.25 4.25c.41.41 1.08.41 1.49 0 .41-.41.41-1.08 0-1.49L15.5 14zm-6 0C7.01 14 5 11.99 5 9.5S7.01 5 9.5 5 14 7.01 14 9.5 11.99 14 9.5 14z"></path>86 </svg>87 <input placeholder="Quick search" aria-label="Quick search" type="search" name="q">88 <input type="submit" value="Go">89 </form>90 </span>91 </nav>92 <div class="menu-wrapper">93 <nav class="menu" role="navigation" aria-label="main navigation">94 <div class="language_switcher_placeholder"></div>95 96<label class="theme-selector-label">97 Theme98 <select class="theme-selector" oninput="activateTheme(this.value)">99 <option value="auto" selected>Auto</option>100 <option value="light">Light</option>101 <option value="dark">Dark</option>102 </select>103</label>104 <div>105 <h4>Previous topic</h4>106 <p class="topless"><a href="symtable.html"107 title="previous chapter"><code class="xref py py-mod docutils literal notranslate"><span class="pre">symtable</span></code> — Access to the compiler’s symbol tables</a></p>108 </div>109 <div>110 <h4>Next topic</h4>111 <p class="topless"><a href="keyword.html"112 title="next chapter"><code class="xref py py-mod docutils literal notranslate"><span class="pre">keyword</span></code> — Testing for Python keywords</a></p>113 </div>114 <script>115 document.addEventListener('DOMContentLoaded', () => {116 const title = document.querySelector('meta[property="og:title"]').content;117 const elements = document.querySelectorAll('.improvepage');118 const pageurl = window.location.href.split('?')[0];119 elements.forEach(element => {120 const url = new URL(element.href.split('?')[0].replace("-nojs", ""));121 url.searchParams.set('pagetitle', title);122 url.searchParams.set('pageurl', pageurl);123 url.searchParams.set('pagesource', "library/token.rst");124 element.href = url.toString();125 });126 });127 </script>128 <div role="note" aria-label="source link">129 <h3>This page</h3>130 <ul class="this-page-menu">131 <li><a href="../bugs.html">Report a bug</a></li>132 <li><a class="improvepage" href="../improve-page-nojs.html">Improve this page</a></li>133 <li>134 <a href="https://github.com/python/cpython/blob/main/Doc/library/token.rst?plain=1"135 rel="nofollow">Show source136 </a>137 </li>138 139 </ul>140 </div>141 </nav>142 </div>143</div>144 145 146 <div class="related" role="navigation" aria-label="Related">147 <h3>Navigation</h3>148 <ul>149 <li class="right" style="margin-right: 10px">150 <a href="../genindex.html" title="General Index"151 accesskey="I">index</a></li>152 <li class="right" >153 <a href="../py-modindex.html" title="Python Module Index"154 >modules</a> |</li>155 <li class="right" >156 <a href="keyword.html" title="keyword — Testing for Python keywords"157 accesskey="N">next</a> |</li>158 <li class="right" >159 <a href="symtable.html" title="symtable — Access to the compiler’s symbol tables"160 accesskey="P">previous</a> |</li>161 162 <li><img src="../_static/py.svg" alt="Python logo" style="vertical-align: middle; margin-top: -1px"></li>163 <li><a href="https://www.python.org/">Python</a> »</li>164 <li class="switchers">165 <div class="language_switcher_placeholder"></div>166 <div class="version_switcher_placeholder"></div>167 </li>168 <li>169 170 </li>171 <li id="cpython-language-and-version">172 <a href="../index.html">3.15.0a6 Documentation</a> »173 </li>174 175 <li class="nav-item nav-item-1"><a href="index.html" >The Python Standard Library</a> »</li>176 <li class="nav-item nav-item-2"><a href="language.html" accesskey="U">Python Language Services</a> »</li>177 <li class="nav-item nav-item-this"><a href=""><code class="xref py py-mod docutils literal notranslate"><span class="pre">token</span></code> — Constants used with Python parse trees</a></li>178 <li class="right">179 180 181 <div class="inline-search" role="search">182 <form class="inline-search" action="../search.html" method="get">183 <input placeholder="Quick search" aria-label="Quick search" type="search" name="q" id="search-box">184 <input type="submit" value="Go">185 </form>186 </div>187 |188 </li>189 <li class="right">190<label class="theme-selector-label">191 Theme192 <select class="theme-selector" oninput="activateTheme(this.value)">193 <option value="auto" selected>Auto</option>194 <option value="light">Light</option>195 <option value="dark">Dark</option>196 </select>197</label> |</li>198 199 </ul>200 </div> 201 202 <div class="document">203 <div class="documentwrapper">204 <div class="bodywrapper">205 <div class="body" role="main">206 207 <section id="module-token">208<span id="token-constants-used-with-python-parse-trees"></span><h1><code class="xref py py-mod docutils literal notranslate"><span class="pre">token</span></code> — Constants used with Python parse trees<a class="headerlink" href="#module-token" title="Link to this heading">¶</a></h1>209<p><strong>Source code:</strong> <a class="extlink-source reference external" href="https://github.com/python/cpython/tree/main/Lib/token.py">Lib/token.py</a></p>210<hr class="docutils" />211<p>This module provides constants which represent the numeric values of leaf nodes212of the parse tree (terminal tokens). Refer to the file <code class="file docutils literal notranslate"><span class="pre">Grammar/Tokens</span></code>213in the Python distribution for the definitions of the names in the context of214the language grammar. The specific numeric values which the names map to may215change between Python versions.</p>216<p>The module also provides a mapping from numeric codes to names and some217functions. The functions mirror definitions in the Python C header files.</p>218<p>Note that a token’s value may depend on tokenizer options. For example, a219<code class="docutils literal notranslate"><span class="pre">"+"</span></code> token may be reported as either <a class="reference internal" href="#token.PLUS" title="token.PLUS"><code class="xref py py-data docutils literal notranslate"><span class="pre">PLUS</span></code></a> or <a class="reference internal" href="#token.OP" title="token.OP"><code class="xref py py-data docutils literal notranslate"><span class="pre">OP</span></code></a>, or220a <code class="docutils literal notranslate"><span class="pre">"match"</span></code> token may be either <a class="reference internal" href="#token.NAME" title="token.NAME"><code class="xref py py-data docutils literal notranslate"><span class="pre">NAME</span></code></a> or <a class="reference internal" href="#token.SOFT_KEYWORD" title="token.SOFT_KEYWORD"><code class="xref py py-data docutils literal notranslate"><span class="pre">SOFT_KEYWORD</span></code></a>.</p>221<dl class="py data">222<dt class="sig sig-object py" id="token.tok_name">223<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">tok_name</span></span><a class="headerlink" href="#token.tok_name" title="Link to this definition">¶</a></dt>224<dd><p>Dictionary mapping the numeric values of the constants defined in this module225back to name strings, allowing more human-readable representation of parse trees226to be generated.</p>227</dd></dl>228 229<dl class="py function">230<dt class="sig sig-object py" id="token.ISTERMINAL">231<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ISTERMINAL</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">x</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#token.ISTERMINAL" title="Link to this definition">¶</a></dt>232<dd><p>Return <code class="docutils literal notranslate"><span class="pre">True</span></code> for terminal token values.</p>233</dd></dl>234 235<dl class="py function">236<dt class="sig sig-object py" id="token.ISNONTERMINAL">237<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ISNONTERMINAL</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">x</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#token.ISNONTERMINAL" title="Link to this definition">¶</a></dt>238<dd><p>Return <code class="docutils literal notranslate"><span class="pre">True</span></code> for non-terminal token values.</p>239</dd></dl>240 241<dl class="py function">242<dt class="sig sig-object py" id="token.ISEOF">243<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ISEOF</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">x</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#token.ISEOF" title="Link to this definition">¶</a></dt>244<dd><p>Return <code class="docutils literal notranslate"><span class="pre">True</span></code> if <em>x</em> is the marker indicating the end of input.</p>245</dd></dl>246 247<p>The token constants are:</p>248<dl class="py data">249<dt class="sig sig-object py" id="token.NAME">250<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">NAME</span></span><a class="headerlink" href="#token.NAME" title="Link to this definition">¶</a></dt>251<dd><p>Token value that indicates an <a class="reference internal" href="../reference/lexical_analysis.html#identifiers"><span class="std std-ref">identifier or keyword</span></a>.</p>252</dd></dl>253 254<dl class="py data">255<dt class="sig sig-object py" id="token.NUMBER">256<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">NUMBER</span></span><a class="headerlink" href="#token.NUMBER" title="Link to this definition">¶</a></dt>257<dd><p>Token value that indicates a <a class="reference internal" href="../reference/lexical_analysis.html#numbers"><span class="std std-ref">numeric literal</span></a></p>258</dd></dl>259 260<dl class="py data">261<dt class="sig sig-object py" id="token.STRING">262<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">STRING</span></span><a class="headerlink" href="#token.STRING" title="Link to this definition">¶</a></dt>263<dd><p>Token value that indicates a <a class="reference internal" href="../reference/lexical_analysis.html#strings"><span class="std std-ref">string or byte literal</span></a>,264excluding <a class="reference internal" href="../reference/lexical_analysis.html#f-strings"><span class="std std-ref">formatted string literals</span></a>.265The token string is not interpreted:266it includes the surrounding quotation marks and the prefix (if given);267backslashes are included literally, without processing escape sequences.</p>268</dd></dl>269 270<dl class="py data">271<dt class="sig sig-object py" id="token.OP">272<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">OP</span></span><a class="headerlink" href="#token.OP" title="Link to this definition">¶</a></dt>273<dd><p>A generic token value that indicates an274<a class="reference internal" href="../reference/lexical_analysis.html#operators"><span class="std std-ref">operator</span></a> or <a class="reference internal" href="../reference/lexical_analysis.html#delimiters"><span class="std std-ref">delimiter</span></a>.</p>275<div class="impl-detail compound">276<p><strong>CPython implementation detail:</strong> This value is only reported by the <a class="reference internal" href="tokenize.html#module-tokenize" title="tokenize: Lexical scanner for Python source code."><code class="xref py py-mod docutils literal notranslate"><span class="pre">tokenize</span></code></a> module.277Internally, the tokenizer uses278<a class="reference internal" href="#token-operators-delimiters"><span class="std std-ref">exact token types</span></a> instead.</p>279</div>280</dd></dl>281 282<dl class="py data">283<dt class="sig sig-object py" id="token.COMMENT">284<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">COMMENT</span></span><a class="headerlink" href="#token.COMMENT" title="Link to this definition">¶</a></dt>285<dd><p>Token value used to indicate a comment.286The parser ignores <code class="xref py py-data docutils literal notranslate"><span class="pre">COMMENT</span></code> tokens.</p>287</dd></dl>288 289<dl class="py data">290<dt class="sig sig-object py" id="token.NEWLINE">291<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">NEWLINE</span></span><a class="headerlink" href="#token.NEWLINE" title="Link to this definition">¶</a></dt>292<dd><p>Token value that indicates the end of a <a class="reference internal" href="../reference/lexical_analysis.html#logical-lines"><span class="std std-ref">logical line</span></a>.</p>293</dd></dl>294 295<dl class="py data">296<dt class="sig sig-object py" id="token.NL">297<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">NL</span></span><a class="headerlink" href="#token.NL" title="Link to this definition">¶</a></dt>298<dd><p>Token value used to indicate a non-terminating newline.299<code class="xref py py-data docutils literal notranslate"><span class="pre">NL</span></code> tokens are generated when a logical line of code is continued300over multiple physical lines. The parser ignores <code class="xref py py-data docutils literal notranslate"><span class="pre">NL</span></code> tokens.</p>301</dd></dl>302 303<dl class="py data">304<dt class="sig sig-object py" id="token.INDENT">305<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">INDENT</span></span><a class="headerlink" href="#token.INDENT" title="Link to this definition">¶</a></dt>306<dd><p>Token value used at the beginning of a <a class="reference internal" href="../reference/lexical_analysis.html#logical-lines"><span class="std std-ref">logical line</span></a>307to indicate the start of an <a class="reference internal" href="../reference/lexical_analysis.html#indentation"><span class="std std-ref">indented block</span></a>.</p>308</dd></dl>309 310<dl class="py data">311<dt class="sig sig-object py" id="token.DEDENT">312<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">DEDENT</span></span><a class="headerlink" href="#token.DEDENT" title="Link to this definition">¶</a></dt>313<dd><p>Token value used at the beginning of a <a class="reference internal" href="../reference/lexical_analysis.html#logical-lines"><span class="std std-ref">logical line</span></a>314to indicate the end of an <a class="reference internal" href="../reference/lexical_analysis.html#indentation"><span class="std std-ref">indented block</span></a>.</p>315</dd></dl>316 317<dl class="py data">318<dt class="sig sig-object py" id="token.FSTRING_START">319<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">FSTRING_START</span></span><a class="headerlink" href="#token.FSTRING_START" title="Link to this definition">¶</a></dt>320<dd><p>Token value used to indicate the beginning of an321<a class="reference internal" href="../reference/lexical_analysis.html#f-strings"><span class="std std-ref">f-string literal</span></a>.</p>322<div class="impl-detail compound">323<p><strong>CPython implementation detail:</strong> The token string includes the prefix and the opening quote(s), but none324of the contents of the literal.</p>325</div>326</dd></dl>327 328<dl class="py data">329<dt class="sig sig-object py" id="token.FSTRING_MIDDLE">330<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">FSTRING_MIDDLE</span></span><a class="headerlink" href="#token.FSTRING_MIDDLE" title="Link to this definition">¶</a></dt>331<dd><p>Token value used for literal text inside an <a class="reference internal" href="../reference/lexical_analysis.html#f-strings"><span class="std std-ref">f-string literal</span></a>,332including format specifications.</p>333<div class="impl-detail compound">334<p><strong>CPython implementation detail:</strong> Replacement fields (that is, the non-literal parts of f-strings) use335the same tokens as other expressions, and are delimited by336<a class="reference internal" href="#token.LBRACE" title="token.LBRACE"><code class="xref py py-data docutils literal notranslate"><span class="pre">LBRACE</span></code></a>, <a class="reference internal" href="#token.RBRACE" title="token.RBRACE"><code class="xref py py-data docutils literal notranslate"><span class="pre">RBRACE</span></code></a>, <a class="reference internal" href="#token.EXCLAMATION" title="token.EXCLAMATION"><code class="xref py py-data docutils literal notranslate"><span class="pre">EXCLAMATION</span></code></a> and <a class="reference internal" href="#token.COLON" title="token.COLON"><code class="xref py py-data docutils literal notranslate"><span class="pre">COLON</span></code></a>337tokens.</p>338</div>339</dd></dl>340 341<dl class="py data">342<dt class="sig sig-object py" id="token.FSTRING_END">343<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">FSTRING_END</span></span><a class="headerlink" href="#token.FSTRING_END" title="Link to this definition">¶</a></dt>344<dd><p>Token value used to indicate the end of a <a class="reference internal" href="../reference/lexical_analysis.html#f-strings"><span class="std std-ref">f-string</span></a>.</p>345<div class="impl-detail compound">346<p><strong>CPython implementation detail:</strong> The token string contains the closing quote(s).</p>347</div>348</dd></dl>349 350<dl class="py data">351<dt class="sig sig-object py" id="token.TSTRING_START">352<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">TSTRING_START</span></span><a class="headerlink" href="#token.TSTRING_START" title="Link to this definition">¶</a></dt>353<dd><p>Token value used to indicate the beginning of a template string literal.</p>354<div class="impl-detail compound">355<p><strong>CPython implementation detail:</strong> The token string includes the prefix and the opening quote(s), but none356of the contents of the literal.</p>357</div>358<div class="versionadded">359<p><span class="versionmodified added">Added in version 3.14.</span></p>360</div>361</dd></dl>362 363<dl class="py data">364<dt class="sig sig-object py" id="token.TSTRING_MIDDLE">365<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">TSTRING_MIDDLE</span></span><a class="headerlink" href="#token.TSTRING_MIDDLE" title="Link to this definition">¶</a></dt>366<dd><p>Token value used for literal text inside a template string literal367including format specifications.</p>368<div class="impl-detail compound">369<p><strong>CPython implementation detail:</strong> Replacement fields (that is, the non-literal parts of t-strings) use370the same tokens as other expressions, and are delimited by371<a class="reference internal" href="#token.LBRACE" title="token.LBRACE"><code class="xref py py-data docutils literal notranslate"><span class="pre">LBRACE</span></code></a>, <a class="reference internal" href="#token.RBRACE" title="token.RBRACE"><code class="xref py py-data docutils literal notranslate"><span class="pre">RBRACE</span></code></a>, <a class="reference internal" href="#token.EXCLAMATION" title="token.EXCLAMATION"><code class="xref py py-data docutils literal notranslate"><span class="pre">EXCLAMATION</span></code></a> and <a class="reference internal" href="#token.COLON" title="token.COLON"><code class="xref py py-data docutils literal notranslate"><span class="pre">COLON</span></code></a>372tokens.</p>373</div>374<div class="versionadded">375<p><span class="versionmodified added">Added in version 3.14.</span></p>376</div>377</dd></dl>378 379<dl class="py data">380<dt class="sig sig-object py" id="token.TSTRING_END">381<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">TSTRING_END</span></span><a class="headerlink" href="#token.TSTRING_END" title="Link to this definition">¶</a></dt>382<dd><p>Token value used to indicate the end of a template string literal.</p>383<div class="impl-detail compound">384<p><strong>CPython implementation detail:</strong> The token string contains the closing quote(s).</p>385</div>386<div class="versionadded">387<p><span class="versionmodified added">Added in version 3.14.</span></p>388</div>389</dd></dl>390 391<dl class="py data">392<dt class="sig sig-object py" id="token.ENDMARKER">393<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ENDMARKER</span></span><a class="headerlink" href="#token.ENDMARKER" title="Link to this definition">¶</a></dt>394<dd><p>Token value that indicates the end of input.395Used in <a class="reference internal" href="../reference/toplevel_components.html#top-level"><span class="std std-ref">top-level grammar rules</span></a>.</p>396</dd></dl>397 398<dl class="py data">399<dt class="sig sig-object py" id="token.ENCODING">400<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ENCODING</span></span><a class="headerlink" href="#token.ENCODING" title="Link to this definition">¶</a></dt>401<dd><p>Token value that indicates the encoding used to decode the source bytes402into text. The first token returned by <a class="reference internal" href="tokenize.html#tokenize.tokenize" title="tokenize.tokenize"><code class="xref py py-func docutils literal notranslate"><span class="pre">tokenize.tokenize()</span></code></a> will403always be an <code class="docutils literal notranslate"><span class="pre">ENCODING</span></code> token.</p>404<div class="impl-detail compound">405<p><strong>CPython implementation detail:</strong> This token type isn’t used by the C tokenizer but is needed for406the <a class="reference internal" href="tokenize.html#module-tokenize" title="tokenize: Lexical scanner for Python source code."><code class="xref py py-mod docutils literal notranslate"><span class="pre">tokenize</span></code></a> module.</p>407</div>408</dd></dl>409 410<p>The following token types are not produced by the <a class="reference internal" href="tokenize.html#module-tokenize" title="tokenize: Lexical scanner for Python source code."><code class="xref py py-mod docutils literal notranslate"><span class="pre">tokenize</span></code></a> module,411and are defined for special uses in the tokenizer or parser:</p>412<dl class="py data">413<dt class="sig sig-object py" id="token.TYPE_IGNORE">414<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">TYPE_IGNORE</span></span><a class="headerlink" href="#token.TYPE_IGNORE" title="Link to this definition">¶</a></dt>415<dd><p>Token value indicating that a <code class="docutils literal notranslate"><span class="pre">type:</span> <span class="pre">ignore</span></code> comment was recognized.416Such tokens are produced instead of regular <a class="reference internal" href="#token.COMMENT" title="token.COMMENT"><code class="xref py py-data docutils literal notranslate"><span class="pre">COMMENT</span></code></a> tokens only417with the <a class="reference internal" href="ast.html#ast.PyCF_TYPE_COMMENTS" title="ast.PyCF_TYPE_COMMENTS"><code class="xref py py-data docutils literal notranslate"><span class="pre">PyCF_TYPE_COMMENTS</span></code></a> flag.</p>418</dd></dl>419 420<dl class="py data">421<dt class="sig sig-object py" id="token.TYPE_COMMENT">422<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">TYPE_COMMENT</span></span><a class="headerlink" href="#token.TYPE_COMMENT" title="Link to this definition">¶</a></dt>423<dd><p>Token value indicating that a type comment was recognized.424Such tokens are produced instead of regular <a class="reference internal" href="#token.COMMENT" title="token.COMMENT"><code class="xref py py-data docutils literal notranslate"><span class="pre">COMMENT</span></code></a> tokens only425with the <a class="reference internal" href="ast.html#ast.PyCF_TYPE_COMMENTS" title="ast.PyCF_TYPE_COMMENTS"><code class="xref py py-data docutils literal notranslate"><span class="pre">PyCF_TYPE_COMMENTS</span></code></a> flag.</p>426</dd></dl>427 428<dl class="py data">429<dt class="sig sig-object py" id="token.SOFT_KEYWORD">430<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">SOFT_KEYWORD</span></span><a class="headerlink" href="#token.SOFT_KEYWORD" title="Link to this definition">¶</a></dt>431<dd><p>Token value indicating a <a class="reference internal" href="../reference/lexical_analysis.html#soft-keywords"><span class="std std-ref">soft keyword</span></a>.</p>432<p>The tokenizer never produces this value.433To check for a soft keyword, pass a <a class="reference internal" href="#token.NAME" title="token.NAME"><code class="xref py py-data docutils literal notranslate"><span class="pre">NAME</span></code></a> token’s string to434<a class="reference internal" href="keyword.html#keyword.issoftkeyword" title="keyword.issoftkeyword"><code class="xref py py-func docutils literal notranslate"><span class="pre">keyword.issoftkeyword()</span></code></a>.</p>435</dd></dl>436 437<dl class="py data">438<dt class="sig sig-object py" id="token.ERRORTOKEN">439<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ERRORTOKEN</span></span><a class="headerlink" href="#token.ERRORTOKEN" title="Link to this definition">¶</a></dt>440<dd><p>Token value used to indicate wrong input.</p>441<p>The <a class="reference internal" href="tokenize.html#module-tokenize" title="tokenize: Lexical scanner for Python source code."><code class="xref py py-mod docutils literal notranslate"><span class="pre">tokenize</span></code></a> module generally indicates errors by442raising exceptions instead of emitting this token.443It can also emit tokens such as <a class="reference internal" href="#token.OP" title="token.OP"><code class="xref py py-data docutils literal notranslate"><span class="pre">OP</span></code></a> or <a class="reference internal" href="#token.NAME" title="token.NAME"><code class="xref py py-data docutils literal notranslate"><span class="pre">NAME</span></code></a> with strings that444are later rejected by the parser.</p>445</dd></dl>446 447<p id="token-operators-delimiters">The remaining tokens represent specific <a class="reference internal" href="../reference/lexical_analysis.html#operators"><span class="std std-ref">operators</span></a> and448<a class="reference internal" href="../reference/lexical_analysis.html#delimiters"><span class="std std-ref">delimiters</span></a>.449(The <a class="reference internal" href="tokenize.html#module-tokenize" title="tokenize: Lexical scanner for Python source code."><code class="xref py py-mod docutils literal notranslate"><span class="pre">tokenize</span></code></a> module reports these as <a class="reference internal" href="#token.OP" title="token.OP"><code class="xref py py-data docutils literal notranslate"><span class="pre">OP</span></code></a>; see <code class="docutils literal notranslate"><span class="pre">exact_type</span></code>450in the <code class="xref py py-mod docutils literal notranslate"><span class="pre">tokenize</span></code> documentation for details.)</p>451<table class="docutils align-left">452<thead>453<tr class="row-odd"><th class="head"><p>Token</p></th>454<th class="head"><p>Value</p></th>455</tr>456</thead>457<tbody>458<tr class="row-even"><td><dl class="py data">459<dt class="sig sig-object py" id="token.LPAR">460<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LPAR</span></span><a class="headerlink" href="#token.LPAR" title="Link to this definition">¶</a></dt>461<dd></dd></dl>462 463</td>464<td><p><code class="docutils literal notranslate"><span class="pre">"("</span></code></p></td>465</tr>466<tr class="row-odd"><td><dl class="py data">467<dt class="sig sig-object py" id="token.RPAR">468<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">RPAR</span></span><a class="headerlink" href="#token.RPAR" title="Link to this definition">¶</a></dt>469<dd></dd></dl>470 471</td>472<td><p><code class="docutils literal notranslate"><span class="pre">")"</span></code></p></td>473</tr>474<tr class="row-even"><td><dl class="py data">475<dt class="sig sig-object py" id="token.LSQB">476<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LSQB</span></span><a class="headerlink" href="#token.LSQB" title="Link to this definition">¶</a></dt>477<dd></dd></dl>478 479</td>480<td><p><code class="docutils literal notranslate"><span class="pre">"["</span></code></p></td>481</tr>482<tr class="row-odd"><td><dl class="py data">483<dt class="sig sig-object py" id="token.RSQB">484<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">RSQB</span></span><a class="headerlink" href="#token.RSQB" title="Link to this definition">¶</a></dt>485<dd></dd></dl>486 487</td>488<td><p><code class="docutils literal notranslate"><span class="pre">"]"</span></code></p></td>489</tr>490<tr class="row-even"><td><dl class="py data">491<dt class="sig sig-object py" id="token.COLON">492<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">COLON</span></span><a class="headerlink" href="#token.COLON" title="Link to this definition">¶</a></dt>493<dd></dd></dl>494 495</td>496<td><p><code class="docutils literal notranslate"><span class="pre">":"</span></code></p></td>497</tr>498<tr class="row-odd"><td><dl class="py data">499<dt class="sig sig-object py" id="token.COMMA">500<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">COMMA</span></span><a class="headerlink" href="#token.COMMA" title="Link to this definition">¶</a></dt>501<dd></dd></dl>502 503</td>504<td><p><code class="docutils literal notranslate"><span class="pre">","</span></code></p></td>505</tr>506<tr class="row-even"><td><dl class="py data">507<dt class="sig sig-object py" id="token.SEMI">508<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">SEMI</span></span><a class="headerlink" href="#token.SEMI" title="Link to this definition">¶</a></dt>509<dd></dd></dl>510 511</td>512<td><p><code class="docutils literal notranslate"><span class="pre">";"</span></code></p></td>513</tr>514<tr class="row-odd"><td><dl class="py data">515<dt class="sig sig-object py" id="token.PLUS">516<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">PLUS</span></span><a class="headerlink" href="#token.PLUS" title="Link to this definition">¶</a></dt>517<dd></dd></dl>518 519</td>520<td><p><code class="docutils literal notranslate"><span class="pre">"+"</span></code></p></td>521</tr>522<tr class="row-even"><td><dl class="py data">523<dt class="sig sig-object py" id="token.MINUS">524<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">MINUS</span></span><a class="headerlink" href="#token.MINUS" title="Link to this definition">¶</a></dt>525<dd></dd></dl>526 527</td>528<td><p><code class="docutils literal notranslate"><span class="pre">"-"</span></code></p></td>529</tr>530<tr class="row-odd"><td><dl class="py data">531<dt class="sig sig-object py" id="token.STAR">532<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">STAR</span></span><a class="headerlink" href="#token.STAR" title="Link to this definition">¶</a></dt>533<dd></dd></dl>534 535</td>536<td><p><code class="docutils literal notranslate"><span class="pre">"*"</span></code></p></td>537</tr>538<tr class="row-even"><td><dl class="py data">539<dt class="sig sig-object py" id="token.SLASH">540<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">SLASH</span></span><a class="headerlink" href="#token.SLASH" title="Link to this definition">¶</a></dt>541<dd></dd></dl>542 543</td>544<td><p><code class="docutils literal notranslate"><span class="pre">"/"</span></code></p></td>545</tr>546<tr class="row-odd"><td><dl class="py data">547<dt class="sig sig-object py" id="token.VBAR">548<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">VBAR</span></span><a class="headerlink" href="#token.VBAR" title="Link to this definition">¶</a></dt>549<dd></dd></dl>550 551</td>552<td><p><code class="docutils literal notranslate"><span class="pre">"|"</span></code></p></td>553</tr>554<tr class="row-even"><td><dl class="py data">555<dt class="sig sig-object py" id="token.AMPER">556<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">AMPER</span></span><a class="headerlink" href="#token.AMPER" title="Link to this definition">¶</a></dt>557<dd></dd></dl>558 559</td>560<td><p><code class="docutils literal notranslate"><span class="pre">"&"</span></code></p></td>561</tr>562<tr class="row-odd"><td><dl class="py data">563<dt class="sig sig-object py" id="token.LESS">564<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LESS</span></span><a class="headerlink" href="#token.LESS" title="Link to this definition">¶</a></dt>565<dd></dd></dl>566 567</td>568<td><p><code class="docutils literal notranslate"><span class="pre">"<"</span></code></p></td>569</tr>570<tr class="row-even"><td><dl class="py data">571<dt class="sig sig-object py" id="token.GREATER">572<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">GREATER</span></span><a class="headerlink" href="#token.GREATER" title="Link to this definition">¶</a></dt>573<dd></dd></dl>574 575</td>576<td><p><code class="docutils literal notranslate"><span class="pre">">"</span></code></p></td>577</tr>578<tr class="row-odd"><td><dl class="py data">579<dt class="sig sig-object py" id="token.EQUAL">580<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">EQUAL</span></span><a class="headerlink" href="#token.EQUAL" title="Link to this definition">¶</a></dt>581<dd></dd></dl>582 583</td>584<td><p><code class="docutils literal notranslate"><span class="pre">"="</span></code></p></td>585</tr>586<tr class="row-even"><td><dl class="py data">587<dt class="sig sig-object py" id="token.DOT">588<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">DOT</span></span><a class="headerlink" href="#token.DOT" title="Link to this definition">¶</a></dt>589<dd></dd></dl>590 591</td>592<td><p><code class="docutils literal notranslate"><span class="pre">"."</span></code></p></td>593</tr>594<tr class="row-odd"><td><dl class="py data">595<dt class="sig sig-object py" id="token.PERCENT">596<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">PERCENT</span></span><a class="headerlink" href="#token.PERCENT" title="Link to this definition">¶</a></dt>597<dd></dd></dl>598 599</td>600<td><p><code class="docutils literal notranslate"><span class="pre">"%"</span></code></p></td>601</tr>602<tr class="row-even"><td><dl class="py data">603<dt class="sig sig-object py" id="token.LBRACE">604<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LBRACE</span></span><a class="headerlink" href="#token.LBRACE" title="Link to this definition">¶</a></dt>605<dd></dd></dl>606 607</td>608<td><p><code class="docutils literal notranslate"><span class="pre">"{"</span></code></p></td>609</tr>610<tr class="row-odd"><td><dl class="py data">611<dt class="sig sig-object py" id="token.RBRACE">612<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">RBRACE</span></span><a class="headerlink" href="#token.RBRACE" title="Link to this definition">¶</a></dt>613<dd></dd></dl>614 615</td>616<td><p><code class="docutils literal notranslate"><span class="pre">"}"</span></code></p></td>617</tr>618<tr class="row-even"><td><dl class="py data">619<dt class="sig sig-object py" id="token.EQEQUAL">620<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">EQEQUAL</span></span><a class="headerlink" href="#token.EQEQUAL" title="Link to this definition">¶</a></dt>621<dd></dd></dl>622 623</td>624<td><p><code class="docutils literal notranslate"><span class="pre">"=="</span></code></p></td>625</tr>626<tr class="row-odd"><td><dl class="py data">627<dt class="sig sig-object py" id="token.NOTEQUAL">628<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">NOTEQUAL</span></span><a class="headerlink" href="#token.NOTEQUAL" title="Link to this definition">¶</a></dt>629<dd></dd></dl>630 631</td>632<td><p><code class="docutils literal notranslate"><span class="pre">"!="</span></code></p></td>633</tr>634<tr class="row-even"><td><dl class="py data">635<dt class="sig sig-object py" id="token.LESSEQUAL">636<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LESSEQUAL</span></span><a class="headerlink" href="#token.LESSEQUAL" title="Link to this definition">¶</a></dt>637<dd></dd></dl>638 639</td>640<td><p><code class="docutils literal notranslate"><span class="pre">"<="</span></code></p></td>641</tr>642<tr class="row-odd"><td><dl class="py data">643<dt class="sig sig-object py" id="token.GREATEREQUAL">644<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">GREATEREQUAL</span></span><a class="headerlink" href="#token.GREATEREQUAL" title="Link to this definition">¶</a></dt>645<dd></dd></dl>646 647</td>648<td><p><code class="docutils literal notranslate"><span class="pre">">="</span></code></p></td>649</tr>650<tr class="row-even"><td><dl class="py data">651<dt class="sig sig-object py" id="token.TILDE">652<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">TILDE</span></span><a class="headerlink" href="#token.TILDE" title="Link to this definition">¶</a></dt>653<dd></dd></dl>654 655</td>656<td><p><code class="docutils literal notranslate"><span class="pre">"~"</span></code></p></td>657</tr>658<tr class="row-odd"><td><dl class="py data">659<dt class="sig sig-object py" id="token.CIRCUMFLEX">660<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">CIRCUMFLEX</span></span><a class="headerlink" href="#token.CIRCUMFLEX" title="Link to this definition">¶</a></dt>661<dd></dd></dl>662 663</td>664<td><p><code class="docutils literal notranslate"><span class="pre">"^"</span></code></p></td>665</tr>666<tr class="row-even"><td><dl class="py data">667<dt class="sig sig-object py" id="token.LEFTSHIFT">668<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LEFTSHIFT</span></span><a class="headerlink" href="#token.LEFTSHIFT" title="Link to this definition">¶</a></dt>669<dd></dd></dl>670 671</td>672<td><p><code class="docutils literal notranslate"><span class="pre">"<<"</span></code></p></td>673</tr>674<tr class="row-odd"><td><dl class="py data">675<dt class="sig sig-object py" id="token.RIGHTSHIFT">676<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">RIGHTSHIFT</span></span><a class="headerlink" href="#token.RIGHTSHIFT" title="Link to this definition">¶</a></dt>677<dd></dd></dl>678 679</td>680<td><p><code class="docutils literal notranslate"><span class="pre">">>"</span></code></p></td>681</tr>682<tr class="row-even"><td><dl class="py data">683<dt class="sig sig-object py" id="token.DOUBLESTAR">684<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">DOUBLESTAR</span></span><a class="headerlink" href="#token.DOUBLESTAR" title="Link to this definition">¶</a></dt>685<dd></dd></dl>686 687</td>688<td><p><code class="docutils literal notranslate"><span class="pre">"**"</span></code></p></td>689</tr>690<tr class="row-odd"><td><dl class="py data">691<dt class="sig sig-object py" id="token.PLUSEQUAL">692<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">PLUSEQUAL</span></span><a class="headerlink" href="#token.PLUSEQUAL" title="Link to this definition">¶</a></dt>693<dd></dd></dl>694 695</td>696<td><p><code class="docutils literal notranslate"><span class="pre">"+="</span></code></p></td>697</tr>698<tr class="row-even"><td><dl class="py data">699<dt class="sig sig-object py" id="token.MINEQUAL">700<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">MINEQUAL</span></span><a class="headerlink" href="#token.MINEQUAL" title="Link to this definition">¶</a></dt>701<dd></dd></dl>702 703</td>704<td><p><code class="docutils literal notranslate"><span class="pre">"-="</span></code></p></td>705</tr>706<tr class="row-odd"><td><dl class="py data">707<dt class="sig sig-object py" id="token.STAREQUAL">708<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">STAREQUAL</span></span><a class="headerlink" href="#token.STAREQUAL" title="Link to this definition">¶</a></dt>709<dd></dd></dl>710 711</td>712<td><p><code class="docutils literal notranslate"><span class="pre">"*="</span></code></p></td>713</tr>714<tr class="row-even"><td><dl class="py data">715<dt class="sig sig-object py" id="token.SLASHEQUAL">716<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">SLASHEQUAL</span></span><a class="headerlink" href="#token.SLASHEQUAL" title="Link to this definition">¶</a></dt>717<dd></dd></dl>718 719</td>720<td><p><code class="docutils literal notranslate"><span class="pre">"/="</span></code></p></td>721</tr>722<tr class="row-odd"><td><dl class="py data">723<dt class="sig sig-object py" id="token.PERCENTEQUAL">724<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">PERCENTEQUAL</span></span><a class="headerlink" href="#token.PERCENTEQUAL" title="Link to this definition">¶</a></dt>725<dd></dd></dl>726 727</td>728<td><p><code class="docutils literal notranslate"><span class="pre">"%="</span></code></p></td>729</tr>730<tr class="row-even"><td><dl class="py data">731<dt class="sig sig-object py" id="token.AMPEREQUAL">732<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">AMPEREQUAL</span></span><a class="headerlink" href="#token.AMPEREQUAL" title="Link to this definition">¶</a></dt>733<dd></dd></dl>734 735</td>736<td><p><code class="docutils literal notranslate"><span class="pre">"&="</span></code></p></td>737</tr>738<tr class="row-odd"><td><dl class="py data">739<dt class="sig sig-object py" id="token.VBAREQUAL">740<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">VBAREQUAL</span></span><a class="headerlink" href="#token.VBAREQUAL" title="Link to this definition">¶</a></dt>741<dd></dd></dl>742 743</td>744<td><p><code class="docutils literal notranslate"><span class="pre">"|="</span></code></p></td>745</tr>746<tr class="row-even"><td><dl class="py data">747<dt class="sig sig-object py" id="token.CIRCUMFLEXEQUAL">748<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">CIRCUMFLEXEQUAL</span></span><a class="headerlink" href="#token.CIRCUMFLEXEQUAL" title="Link to this definition">¶</a></dt>749<dd></dd></dl>750 751</td>752<td><p><code class="docutils literal notranslate"><span class="pre">"^="</span></code></p></td>753</tr>754<tr class="row-odd"><td><dl class="py data">755<dt class="sig sig-object py" id="token.LEFTSHIFTEQUAL">756<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">LEFTSHIFTEQUAL</span></span><a class="headerlink" href="#token.LEFTSHIFTEQUAL" title="Link to this definition">¶</a></dt>757<dd></dd></dl>758 759</td>760<td><p><code class="docutils literal notranslate"><span class="pre">"<<="</span></code></p></td>761</tr>762<tr class="row-even"><td><dl class="py data">763<dt class="sig sig-object py" id="token.RIGHTSHIFTEQUAL">764<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">RIGHTSHIFTEQUAL</span></span><a class="headerlink" href="#token.RIGHTSHIFTEQUAL" title="Link to this definition">¶</a></dt>765<dd></dd></dl>766 767</td>768<td><p><code class="docutils literal notranslate"><span class="pre">">>="</span></code></p></td>769</tr>770<tr class="row-odd"><td><dl class="py data">771<dt class="sig sig-object py" id="token.DOUBLESTAREQUAL">772<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">DOUBLESTAREQUAL</span></span><a class="headerlink" href="#token.DOUBLESTAREQUAL" title="Link to this definition">¶</a></dt>773<dd></dd></dl>774 775</td>776<td><p><code class="docutils literal notranslate"><span class="pre">"**="</span></code></p></td>777</tr>778<tr class="row-even"><td><dl class="py data">779<dt class="sig sig-object py" id="token.DOUBLESLASH">780<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">DOUBLESLASH</span></span><a class="headerlink" href="#token.DOUBLESLASH" title="Link to this definition">¶</a></dt>781<dd></dd></dl>782 783</td>784<td><p><code class="docutils literal notranslate"><span class="pre">"//"</span></code></p></td>785</tr>786<tr class="row-odd"><td><dl class="py data">787<dt class="sig sig-object py" id="token.DOUBLESLASHEQUAL">788<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">DOUBLESLASHEQUAL</span></span><a class="headerlink" href="#token.DOUBLESLASHEQUAL" title="Link to this definition">¶</a></dt>789<dd></dd></dl>790 791</td>792<td><p><code class="docutils literal notranslate"><span class="pre">"//="</span></code></p></td>793</tr>794<tr class="row-even"><td><dl class="py data">795<dt class="sig sig-object py" id="token.AT">796<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">AT</span></span><a class="headerlink" href="#token.AT" title="Link to this definition">¶</a></dt>797<dd></dd></dl>798 799</td>800<td><p><code class="docutils literal notranslate"><span class="pre">"@"</span></code></p></td>801</tr>802<tr class="row-odd"><td><dl class="py data">803<dt class="sig sig-object py" id="token.ATEQUAL">804<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ATEQUAL</span></span><a class="headerlink" href="#token.ATEQUAL" title="Link to this definition">¶</a></dt>805<dd></dd></dl>806 807</td>808<td><p><code class="docutils literal notranslate"><span class="pre">"@="</span></code></p></td>809</tr>810<tr class="row-even"><td><dl class="py data">811<dt class="sig sig-object py" id="token.RARROW">812<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">RARROW</span></span><a class="headerlink" href="#token.RARROW" title="Link to this definition">¶</a></dt>813<dd></dd></dl>814 815</td>816<td><p><code class="docutils literal notranslate"><span class="pre">"->"</span></code></p></td>817</tr>818<tr class="row-odd"><td><dl class="py data">819<dt class="sig sig-object py" id="token.ELLIPSIS">820<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">ELLIPSIS</span></span><a class="headerlink" href="#token.ELLIPSIS" title="Link to this definition">¶</a></dt>821<dd></dd></dl>822 823</td>824<td><p><code class="docutils literal notranslate"><span class="pre">"..."</span></code></p></td>825</tr>826<tr class="row-even"><td><dl class="py data">827<dt class="sig sig-object py" id="token.COLONEQUAL">828<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">COLONEQUAL</span></span><a class="headerlink" href="#token.COLONEQUAL" title="Link to this definition">¶</a></dt>829<dd></dd></dl>830 831</td>832<td><p><code class="docutils literal notranslate"><span class="pre">":="</span></code></p></td>833</tr>834<tr class="row-odd"><td><dl class="py data">835<dt class="sig sig-object py" id="token.EXCLAMATION">836<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">EXCLAMATION</span></span><a class="headerlink" href="#token.EXCLAMATION" title="Link to this definition">¶</a></dt>837<dd></dd></dl>838 839</td>840<td><p><code class="docutils literal notranslate"><span class="pre">"!"</span></code></p></td>841</tr>842</tbody>843</table>844<p>The following non-token constants are provided:</p>845<dl class="py data">846<dt class="sig sig-object py" id="token.N_TOKENS">847<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">N_TOKENS</span></span><a class="headerlink" href="#token.N_TOKENS" title="Link to this definition">¶</a></dt>848<dd><p>The number of token types defined in this module.</p>849</dd></dl>850 851<dl class="py data">852<dt class="sig sig-object py" id="token.EXACT_TOKEN_TYPES">853<span class="sig-prename descclassname"><span class="pre">token.</span></span><span class="sig-name descname"><span class="pre">EXACT_TOKEN_TYPES</span></span><a class="headerlink" href="#token.EXACT_TOKEN_TYPES" title="Link to this definition">¶</a></dt>854<dd><p>A dictionary mapping the string representation of a token to its numeric code.</p>855<div class="versionadded">856<p><span class="versionmodified added">Added in version 3.8.</span></p>857</div>858</dd></dl>859 860<div class="versionchanged">861<p><span class="versionmodified changed">Changed in version 3.5: </span>Added <code class="xref py py-data docutils literal notranslate"><span class="pre">AWAIT</span></code> and <code class="xref py py-data docutils literal notranslate"><span class="pre">ASYNC</span></code> tokens.</p>862</div>863<div class="versionchanged">864<p><span class="versionmodified changed">Changed in version 3.7: </span>Added <a class="reference internal" href="#token.COMMENT" title="token.COMMENT"><code class="xref py py-data docutils literal notranslate"><span class="pre">COMMENT</span></code></a>, <a class="reference internal" href="#token.NL" title="token.NL"><code class="xref py py-data docutils literal notranslate"><span class="pre">NL</span></code></a> and <a class="reference internal" href="#token.ENCODING" title="token.ENCODING"><code class="xref py py-data docutils literal notranslate"><span class="pre">ENCODING</span></code></a> tokens.</p>865</div>866<div class="versionchanged">867<p><span class="versionmodified changed">Changed in version 3.7: </span>Removed <code class="xref py py-data docutils literal notranslate"><span class="pre">AWAIT</span></code> and <code class="xref py py-data docutils literal notranslate"><span class="pre">ASYNC</span></code> tokens. “async” and “await” are868now tokenized as <a class="reference internal" href="#token.NAME" title="token.NAME"><code class="xref py py-data docutils literal notranslate"><span class="pre">NAME</span></code></a> tokens.</p>869</div>870<div class="versionchanged">871<p><span class="versionmodified changed">Changed in version 3.8: </span>Added <a class="reference internal" href="#token.TYPE_COMMENT" title="token.TYPE_COMMENT"><code class="xref py py-data docutils literal notranslate"><span class="pre">TYPE_COMMENT</span></code></a>, <a class="reference internal" href="#token.TYPE_IGNORE" title="token.TYPE_IGNORE"><code class="xref py py-data docutils literal notranslate"><span class="pre">TYPE_IGNORE</span></code></a>, <a class="reference internal" href="#token.COLONEQUAL" title="token.COLONEQUAL"><code class="xref py py-data docutils literal notranslate"><span class="pre">COLONEQUAL</span></code></a>.872Added <code class="xref py py-data docutils literal notranslate"><span class="pre">AWAIT</span></code> and <code class="xref py py-data docutils literal notranslate"><span class="pre">ASYNC</span></code> tokens back (they’re needed873to support parsing older Python versions for <a class="reference internal" href="ast.html#ast.parse" title="ast.parse"><code class="xref py py-func docutils literal notranslate"><span class="pre">ast.parse()</span></code></a> with874<code class="docutils literal notranslate"><span class="pre">feature_version</span></code> set to 6 or lower).</p>875</div>876<div class="versionchanged">877<p><span class="versionmodified changed">Changed in version 3.12: </span>Added <a class="reference internal" href="#token.EXCLAMATION" title="token.EXCLAMATION"><code class="xref py py-data docutils literal notranslate"><span class="pre">EXCLAMATION</span></code></a>.</p>878</div>879<div class="versionchanged">880<p><span class="versionmodified changed">Changed in version 3.13: </span>Removed <code class="xref py py-data docutils literal notranslate"><span class="pre">AWAIT</span></code> and <code class="xref py py-data docutils literal notranslate"><span class="pre">ASYNC</span></code> tokens again.</p>881</div>882</section>883 884 885 <div class="clearer"></div>886 </div>887 </div>888 </div>889 <div class="sphinxsidebar" role="navigation" aria-label="Main">890 <div class="sphinxsidebarwrapper">891 <div>892 <h4>Previous topic</h4>893 <p class="topless"><a href="symtable.html"894 title="previous chapter"><code class="xref py py-mod docutils literal notranslate"><span class="pre">symtable</span></code> — Access to the compiler’s symbol tables</a></p>895 </div>896 <div>897 <h4>Next topic</h4>898 <p class="topless"><a href="keyword.html"899 title="next chapter"><code class="xref py py-mod docutils literal notranslate"><span class="pre">keyword</span></code> — Testing for Python keywords</a></p>900 </div>901 <script>902 document.addEventListener('DOMContentLoaded', () => {903 const title = document.querySelector('meta[property="og:title"]').content;904 const elements = document.querySelectorAll('.improvepage');905 const pageurl = window.location.href.split('?')[0];906 elements.forEach(element => {907 const url = new URL(element.href.split('?')[0].replace("-nojs", ""));908 url.searchParams.set('pagetitle', title);909 url.searchParams.set('pageurl', pageurl);910 url.searchParams.set('pagesource', "library/token.rst");911 element.href = url.toString();912 });913 });914 </script>915 <div role="note" aria-label="source link">916 <h3>This page</h3>917 <ul class="this-page-menu">918 <li><a href="../bugs.html">Report a bug</a></li>919 <li><a class="improvepage" href="../improve-page-nojs.html">Improve this page</a></li>920 <li>921 <a href="https://github.com/python/cpython/blob/main/Doc/library/token.rst?plain=1"922 rel="nofollow">Show source923 </a>924 </li>925 926 </ul>927 </div>928 </div>929<div id="sidebarbutton" title="Collapse sidebar">930<span>«</span>931</div>932 933 </div>934 <div class="clearer"></div>935 </div> 936 <div class="related" role="navigation" aria-label="Related">937 <h3>Navigation</h3>938 <ul>939 <li class="right" style="margin-right: 10px">940 <a href="../genindex.html" title="General Index"941 >index</a></li>942 <li class="right" >943 <a href="../py-modindex.html" title="Python Module Index"944 >modules</a> |</li>945 <li class="right" >946 <a href="keyword.html" title="keyword — Testing for Python keywords"947 >next</a> |</li>948 <li class="right" >949 <a href="symtable.html" title="symtable — Access to the compiler’s symbol tables"950 >previous</a> |</li>951 952 <li><img src="../_static/py.svg" alt="Python logo" style="vertical-align: middle; margin-top: -1px"></li>953 <li><a href="https://www.python.org/">Python</a> »</li>954 <li class="switchers">955 <div class="language_switcher_placeholder"></div>956 <div class="version_switcher_placeholder"></div>957 </li>958 <li>959 960 </li>961 <li id="cpython-language-and-version">962 <a href="../index.html">3.15.0a6 Documentation</a> »963 </li>964 965 <li class="nav-item nav-item-1"><a href="index.html" >The Python Standard Library</a> »</li>966 <li class="nav-item nav-item-2"><a href="language.html" >Python Language Services</a> »</li>967 <li class="nav-item nav-item-this"><a href=""><code class="xref py py-mod docutils literal notranslate"><span class="pre">token</span></code> — Constants used with Python parse trees</a></li>968 <li class="right">969 970 971 <div class="inline-search" role="search">972 <form class="inline-search" action="../search.html" method="get">973 <input placeholder="Quick search" aria-label="Quick search" type="search" name="q" id="search-box">974 <input type="submit" value="Go">975 </form>976 </div>977 |978 </li>979 <li class="right">980<label class="theme-selector-label">981 Theme982 <select class="theme-selector" oninput="activateTheme(this.value)">983 <option value="auto" selected>Auto</option>984 <option value="light">Light</option>985 <option value="dark">Dark</option>986 </select>987</label> |</li>988 989 </ul>990 </div> 991 <div class="footer">992 © <a href="../copyright.html">Copyright</a> 2001 Python Software Foundation.993 <br>994 This page is licensed under the Python Software Foundation License Version 2.995 <br>996 Examples, recipes, and other code in the documentation are additionally licensed under the Zero Clause BSD License.997 <br>998 999 See <a href="/license.html">History and License</a> for more information.<br>1000 1001 1002 <br>1003 1004 The Python Software Foundation is a non-profit corporation.1005<a href="https://www.python.org/psf/donations/">Please donate.</a>1006<br>1007 <br>1008 Last updated on Mar 10, 2026 (08:58 UTC).1009 1010 <a href="/bugs.html">Found a bug</a>?1011 1012 <br>1013 1014 Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 8.2.3.1015 </div>1016 1017 </body>1018</html>