codekingpro/portable-devtools

1/*-------------------------------------------------------------------------2 *3 * scanner.h4 *		API for the core scanner (flex machine)5 *6 * The core scanner is also used by PL/pgSQL, so we provide a public API7 * for it.  However, the rest of the backend is only expected to use the8 * higher-level API provided by parser.h.9 *10 *11 * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group12 * Portions Copyright (c) 1994, Regents of the University of California13 *14 * src/include/parser/scanner.h15 *16 *-------------------------------------------------------------------------17 */18 19#ifndef SCANNER_H20#define SCANNER_H21 22#include "common/keywords.h"23 24/*25 * The scanner returns extra data about scanned tokens in this union type.26 * Note that this is a subset of the fields used in YYSTYPE of the bison27 * parsers built atop the scanner.28 */29typedef union core_YYSTYPE30{31	int			ival;			/* for integer literals */32	char	   *str;			/* for identifiers and non-integer literals */33	const char *keyword;		/* canonical spelling of keywords */34} core_YYSTYPE;35 36/*37 * We track token locations in terms of byte offsets from the start of the38 * source string, not the column number/line number representation that39 * bison uses by default.  Also, to minimize overhead we track only one40 * location (usually the first token location) for each construct, not41 * the beginning and ending locations as bison does by default.  It's42 * therefore sufficient to make YYLTYPE an int.43 */44#define YYLTYPE  int45 46/*47 * Another important component of the scanner's API is the token code numbers.48 * However, those are not defined in this file, because bison insists on49 * defining them for itself.  The token codes used by the core scanner are50 * the ASCII characters plus these:51 *	%token <str>	IDENT UIDENT FCONST SCONST USCONST BCONST XCONST Op52 *	%token <ival>	ICONST PARAM53 *	%token			TYPECAST DOT_DOT COLON_EQUALS EQUALS_GREATER54 *	%token			LESS_EQUALS GREATER_EQUALS NOT_EQUALS55 * The above token definitions *must* be the first ones declared in any56 * bison parser built atop this scanner, so that they will have consistent57 * numbers assigned to them (specifically, IDENT = 258 and so on).58 */59 60/*61 * The YY_EXTRA data that a flex scanner allows us to pass around.62 * Private state needed by the core scanner goes here.  Note that the actual63 * yy_extra struct may be larger and have this as its first component, thus64 * allowing the calling parser to keep some fields of its own in YY_EXTRA.65 */66typedef struct core_yy_extra_type67{68	/*69	 * The string the scanner is physically scanning.  We keep this mainly so70	 * that we can cheaply compute the offset of the current token (yytext).71	 */72	char	   *scanbuf;73	Size		scanbuflen;74 75	/*76	 * The keyword list to use, and the associated grammar token codes.77	 */78	const ScanKeywordList *keywordlist;79	const uint16 *keyword_tokens;80 81	/*82	 * Scanner settings to use.  These are initialized from the corresponding83	 * GUC variables by scanner_init().  Callers can modify them after84	 * scanner_init() if they don't want the scanner's behavior to follow the85	 * prevailing GUC settings.86	 */87	int			backslash_quote;88	bool		escape_string_warning;89	bool		standard_conforming_strings;90 91	/*92	 * literalbuf is used to accumulate literal values when multiple rules are93	 * needed to parse a single literal.  Call startlit() to reset buffer to94	 * empty, addlit() to add text.  NOTE: the string in literalbuf is NOT95	 * necessarily null-terminated, but there always IS room to add a trailing96	 * null at offset literallen.  We store a null only when we need it.97	 */98	char	   *literalbuf;		/* palloc'd expandable buffer */99	int			literallen;		/* actual current string length */100	int			literalalloc;	/* current allocated buffer size */101 102	/*103	 * Random assorted scanner state.104	 */105	int			state_before_str_stop;	/* start cond. before end quote */106	int			xcdepth;		/* depth of nesting in slash-star comments */107	char	   *dolqstart;		/* current $foo$ quote start string */108	YYLTYPE		save_yylloc;	/* one-element stack for PUSH_YYLLOC() */109 110	/* first part of UTF16 surrogate pair for Unicode escapes */111	int32		utf16_first_part;112 113	/* state variables for literal-lexing warnings */114	bool		warn_on_first_escape;115	bool		saw_non_ascii;116} core_yy_extra_type;117 118/*119 * The type of yyscanner is opaque outside scan.l.120 */121typedef void *core_yyscan_t;122 123/* Support for scanner_errposition_callback function */124typedef struct ScannerCallbackState125{126	core_yyscan_t yyscanner;127	int			location;128	ErrorContextCallback errcallback;129} ScannerCallbackState;130 131 132/* Constant data exported from parser/scan.l */133extern PGDLLIMPORT const uint16 ScanKeywordTokens[];134 135/* Entry points in parser/scan.l */136extern core_yyscan_t scanner_init(const char *str,137								  core_yy_extra_type *yyext,138								  const ScanKeywordList *keywordlist,139								  const uint16 *keyword_tokens);140extern void scanner_finish(core_yyscan_t yyscanner);141extern int	core_yylex(core_YYSTYPE *yylval_param, YYLTYPE *yylloc_param,142					   core_yyscan_t yyscanner);143extern int	scanner_errposition(int location, core_yyscan_t yyscanner);144extern void setup_scanner_errposition_callback(ScannerCallbackState *scbstate,145											   core_yyscan_t yyscanner,146											   int location);147extern void cancel_scanner_errposition_callback(ScannerCallbackState *scbstate);148extern void scanner_yyerror(const char *message, core_yyscan_t yyscanner) pg_attribute_noreturn();149 150#endif							/* SCANNER_H */151