AryaWu/sqlite
0
1/*2** 2015-08-123**4** The author disclaims copyright to this source code. In place of5** a legal notice, here is a blessing:6**7** May you do good and not evil.8** May you find forgiveness for yourself and forgive others.9** May you share freely, never taking more than you give.10**11******************************************************************************12**13** SQLite JSON functions.14**15** This file began as an extension in ext/misc/json1.c in 2015. That16** extension proved so useful that it has now been moved into the core.17**18** The original design stored all JSON as pure text, canonical RFC-8259.19** Support for JSON-5 extensions was added with version 3.42.0 (2023-05-16).20** All generated JSON text still conforms strictly to RFC-8259, but text21** with JSON-5 extensions is accepted as input.22**23** Beginning with version 3.45.0 (circa 2024-01-01), these routines also24** accept BLOB values that have JSON encoded using a binary representation25** called "JSONB". The name JSONB comes from PostgreSQL, however the on-disk26** format for SQLite-JSONB is completely different and incompatible with27** PostgreSQL-JSONB.28**29** Decoding and interpreting JSONB is still O(N) where N is the size of30** the input, the same as text JSON. However, the constant of proportionality31** for JSONB is much smaller due to faster parsing. The size of each32** element in JSONB is encoded in its header, so there is no need to search33** for delimiters using persnickety syntax rules. JSONB seems to be about34** 3x faster than text JSON as a result. JSONB is also tends to be slightly35** smaller than text JSON, by 5% or 10%, but there are corner cases where36** JSONB can be slightly larger. So you are not far mistaken to say that37** a JSONB blob is the same size as the equivalent RFC-8259 text.38**39**40** THE JSONB ENCODING:41**42** Every JSON element is encoded in JSONB as a header and a payload.43** The header is between 1 and 9 bytes in size. The payload is zero44** or more bytes.45**46** The lower 4 bits of the first byte of the header determines the47** element type:48**49** 0: NULL50** 1: TRUE51** 2: FALSE52** 3: INT -- RFC-8259 integer literal53** 4: INT5 -- JSON5 integer literal54** 5: FLOAT -- RFC-8259 floating point literal55** 6: FLOAT5 -- JSON5 floating point literal56** 7: TEXT -- Text literal acceptable to both SQL and JSON57** 8: TEXTJ -- Text containing RFC-8259 escapes58** 9: TEXT5 -- Text containing JSON5 and/or RFC-8259 escapes59** 10: TEXTRAW -- Text containing unescaped syntax characters60** 11: ARRAY61** 12: OBJECT62**63** The other three possible values (13-15) are reserved for future64** enhancements.65**66** The upper 4 bits of the first byte determine the size of the header67** and sometimes also the size of the payload. If X is the first byte68** of the element and if X>>4 is between 0 and 11, then the payload69** will be that many bytes in size and the header is exactly one byte70** in size. Other four values for X>>4 (12-15) indicate that the header71** is more than one byte in size and that the payload size is determined72** by the remainder of the header, interpreted as a unsigned big-endian73** integer.74**75** Value of X>>4 Size integer Total header size76** ------------- -------------------- -----------------77** 12 1 byte (0-255) 278** 13 2 byte (0-65535) 379** 14 4 byte (0-4294967295) 580** 15 8 byte (0-1.8e19) 981**82** The payload size need not be expressed in its minimal form. For example,83** if the payload size is 10, the size can be expressed in any of 5 different84** ways: (1) (X>>4)==10, (2) (X>>4)==12 following by one 0x0a byte,85** (3) (X>>4)==13 followed by 0x00 and 0x0a, (4) (X>>4)==14 followed by86** 0x00 0x00 0x00 0x0a, or (5) (X>>4)==15 followed by 7 bytes of 0x00 and87** a single byte of 0x0a. The shorter forms are preferred, of course, but88** sometimes when generating JSONB, the payload size is not known in advance89** and it is convenient to reserve sufficient header space to cover the90** largest possible payload size and then come back later and patch up91** the size when it becomes known, resulting in a non-minimal encoding.92**93** The value (X>>4)==15 is not actually used in the current implementation94** (as SQLite is currently unable to handle BLOBs larger than about 2GB)95** but is included in the design to allow for future enhancements.96**97** The payload follows the header. NULL, TRUE, and FALSE have no payload and98** their payload size must always be zero. The payload for INT, INT5,99** FLOAT, FLOAT5, TEXT, TEXTJ, TEXT5, and TEXTROW is text. Note that the100** "..." or '...' delimiters are omitted from the various text encodings.101** The payload for ARRAY and OBJECT is a list of additional elements that102** are the content for the array or object. The payload for an OBJECT103** must be an even number of elements. The first element of each pair is104** the label and must be of type TEXT, TEXTJ, TEXT5, or TEXTRAW.105**106** A valid JSONB blob consists of a single element, as described above.107** Usually this will be an ARRAY or OBJECT element which has many more108** elements as its content. But the overall blob is just a single element.109**110** Input validation for JSONB blobs simply checks that the element type111** code is between 0 and 12 and that the total size of the element112** (header plus payload) is the same as the size of the BLOB. If those113** checks are true, the BLOB is assumed to be JSONB and processing continues.114** Errors are only raised if some other miscoding is discovered during115** processing.116**117** Additional information can be found in the doc/jsonb.md file of the118** canonical SQLite source tree.119*/120#ifndef SQLITE_OMIT_JSON121#include "sqliteInt.h"122 123/* JSONB element types124*/125#define JSONB_NULL 0 /* "null" */126#define JSONB_TRUE 1 /* "true" */127#define JSONB_FALSE 2 /* "false" */128#define JSONB_INT 3 /* integer acceptable to JSON and SQL */129#define JSONB_INT5 4 /* integer in 0x000 notation */130#define JSONB_FLOAT 5 /* float acceptable to JSON and SQL */131#define JSONB_FLOAT5 6 /* float with JSON5 extensions */132#define JSONB_TEXT 7 /* Text compatible with both JSON and SQL */133#define JSONB_TEXTJ 8 /* Text with JSON escapes */134#define JSONB_TEXT5 9 /* Text with JSON-5 escape */135#define JSONB_TEXTRAW 10 /* SQL text that needs escaping for JSON */136#define JSONB_ARRAY 11 /* An array */137#define JSONB_OBJECT 12 /* An object */138 139/* Human-readable names for the JSONB values. The index for each140** string must correspond to the JSONB_* integer above.141*/142static const char * const jsonbType[] = {143 "null", "true", "false", "integer", "integer", 144 "real", "real", "text", "text", "text",145 "text", "array", "object", "", "", "", ""146};147 148/*149** Growing our own isspace() routine this way is twice as fast as150** the library isspace() function, resulting in a 7% overall performance151** increase for the text-JSON parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os).152*/153static const char jsonIsSpace[] = {154#ifdef SQLITE_ASCII155/*0 1 2 3 4 5 6 7 8 9 a b c d e f */156 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, /* 0 */157 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */158 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2 */159 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3 */160 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4 */161 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5 */162 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 6 */163 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7 */164 165 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 8 */166 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 9 */167 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* a */168 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* b */169 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* c */170 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* d */171 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* e */172 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* f */173#endif174#ifdef SQLITE_EBCDIC175/*0 1 2 3 4 5 6 7 8 9 a b c d e f */176 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, /* 0 */177 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */178 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2 */179 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3 */180 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4 */181 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5 */182 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 6 */183 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7 */184 185 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 8 */186 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 9 */187 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* a */188 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* b */189 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* c */190 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* d */191 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* e */192 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* f */193#endif194 195};196#define jsonIsspace(x) (jsonIsSpace[(unsigned char)x])197 198/*199** The set of all space characters recognized by jsonIsspace().200** Useful as the second argument to strspn().201*/202#ifdef SQLITE_ASCII203static const char jsonSpaces[] = "\011\012\015\040";204#endif205#ifdef SQLITE_EBCDIC206static const char jsonSpaces[] = "\005\045\015\100";207#endif208 209 210/*211** Characters that are special to JSON. Control characters,212** '"' and '\\' and '\''. Actually, '\'' is not special to213** canonical JSON, but it is special in JSON-5, so we include214** it in the set of special characters.215*/216static const char jsonIsOk[256] = {217#ifdef SQLITE_ASCII218/*0 1 2 3 4 5 6 7 8 9 a b c d e f */219 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0 */220 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */221 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, /* 2 */222 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 3 */223 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4 */224 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, /* 5 */225 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6 */226 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 7 */227 228 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 8 */229 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 9 */230 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* a */231 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* b */232 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* c */233 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* d */234 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* e */235 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 /* f */236#endif237#ifdef SQLITE_EBCDIC238/*0 1 2 3 4 5 6 7 8 9 a b c d e f */239 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0 */240 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */241 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2 */242 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, /* 3 */243 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4 */244 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 5 */245 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6 */246 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, /* 7 */247 248 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 8 */249 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 9 */250 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* a */251 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* b */252 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* c */253 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* d */254 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* e */255 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 /* f */256#endif257};258 259/* Objects */260typedef struct JsonCache JsonCache;261typedef struct JsonString JsonString;262typedef struct JsonParse JsonParse;263 264/*265** Magic number used for the JSON parse cache in sqlite3_get_auxdata()266*/267#define JSON_CACHE_ID (-429938) /* Cache entry */268#define JSON_CACHE_SIZE 4 /* Max number of cache entries */269 270/*271** jsonUnescapeOneChar() returns this invalid code point if it encounters272** a syntax error.273*/274#define JSON_INVALID_CHAR 0x99999275 276/* A cache mapping JSON text into JSONB blobs.277**278** Each cache entry is a JsonParse object with the following restrictions:279**280** * The bReadOnly flag must be set281**282** * The aBlob[] array must be owned by the JsonParse object. In other283** words, nBlobAlloc must be non-zero.284**285** * eEdit and delta must be zero.286**287** * zJson must be an RCStr. In other words bJsonIsRCStr must be true.288*/289struct JsonCache {290 sqlite3 *db; /* Database connection */291 int nUsed; /* Number of active entries in the cache */292 JsonParse *a[JSON_CACHE_SIZE]; /* One line for each cache entry */293};294 295/* An instance of this object represents a JSON string296** under construction. Really, this is a generic string accumulator297** that can be and is used to create strings other than JSON.298**299** If the generated string is longer than will fit into the zSpace[] buffer,300** then it will be an RCStr string. This aids with caching of large301** JSON strings.302*/303struct JsonString {304 sqlite3_context *pCtx; /* Function context - put error messages here */305 char *zBuf; /* Append JSON content here */306 u64 nAlloc; /* Bytes of storage available in zBuf[] */307 u64 nUsed; /* Bytes of zBuf[] currently used */308 u8 bStatic; /* True if zBuf is static space */309 u8 eErr; /* True if an error has been encountered */310 char zSpace[100]; /* Initial static space */311};312 313/* Allowed values for JsonString.eErr */314#define JSTRING_OOM 0x01 /* Out of memory */315#define JSTRING_MALFORMED 0x02 /* Malformed JSONB */316#define JSTRING_ERR 0x04 /* Error already sent to sqlite3_result */317 318/* The "subtype" set for text JSON values passed through using319** sqlite3_result_subtype() and sqlite3_value_subtype().320*/321#define JSON_SUBTYPE 74 /* Ascii for "J" */322 323/*324** Bit values for the flags passed into various SQL function implementations325** via the sqlite3_user_data() value.326*/327#define JSON_JSON 0x01 /* Result is always JSON */328#define JSON_SQL 0x02 /* Result is always SQL */329#define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */330#define JSON_ISSET 0x04 /* json_set(), not json_insert() */331#define JSON_BLOB 0x08 /* Use the BLOB output format */332 333 334/* A parsed JSON value. Lifecycle:335**336** 1. JSON comes in and is parsed into a JSONB value in aBlob. The337** original text is stored in zJson. This step is skipped if the338** input is JSONB instead of text JSON.339**340** 2. The aBlob[] array is searched using the JSON path notation, if needed.341** 342** 3. Zero or more changes are made to aBlob[] (via json_remove() or343** json_replace() or json_patch() or similar).344**345** 4. New JSON text is generated from the aBlob[] for output. This step346** is skipped if the function is one of the jsonb_* functions that347** returns JSONB instead of text JSON.348*/349struct JsonParse {350 u8 *aBlob; /* JSONB representation of JSON value */351 u32 nBlob; /* Bytes of aBlob[] actually used */352 u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */353 char *zJson; /* Json text used for parsing */354 sqlite3 *db; /* The database connection to which this object belongs */355 int nJson; /* Length of the zJson string in bytes */356 u32 nJPRef; /* Number of references to this object */357 u32 iErr; /* Error location in zJson[] */358 u16 iDepth; /* Nesting depth */359 u8 nErr; /* Number of errors seen */360 u8 oom; /* Set to true if out of memory */361 u8 bJsonIsRCStr; /* True if zJson is an RCStr */362 u8 hasNonstd; /* True if input uses non-standard features like JSON5 */363 u8 bReadOnly; /* Do not modify. */364 /* Search and edit information. See jsonLookupStep() */365 u8 eEdit; /* Edit operation to apply */366 int delta; /* Size change due to the edit */367 u32 nIns; /* Number of bytes to insert */368 u32 iLabel; /* Location of label if search landed on an object value */369 u8 *aIns; /* Content to be inserted */370};371 372/* Allowed values for JsonParse.eEdit */373#define JEDIT_DEL 1 /* Delete if exists */374#define JEDIT_REPL 2 /* Overwrite if exists */375#define JEDIT_INS 3 /* Insert if not exists */376#define JEDIT_SET 4 /* Insert or overwrite */377 378/*379** Maximum nesting depth of JSON for this implementation.380**381** This limit is needed to avoid a stack overflow in the recursive382** descent parser. A depth of 1000 is far deeper than any sane JSON383** should go. Historical note: This limit was 2000 prior to version 3.42.0384*/385#ifndef SQLITE_JSON_MAX_DEPTH386# define JSON_MAX_DEPTH 1000387#else388# define JSON_MAX_DEPTH SQLITE_JSON_MAX_DEPTH389#endif390 391/*392** Allowed values for the flgs argument to jsonParseFuncArg();393*/394#define JSON_EDITABLE 0x01 /* Generate a writable JsonParse object */395#define JSON_KEEPERROR 0x02 /* Return non-NULL even if there is an error */396 397/**************************************************************************398** Forward references399**************************************************************************/400static void jsonReturnStringAsBlob(JsonString*);401static int jsonArgIsJsonb(sqlite3_value *pJson, JsonParse *p);402static u32 jsonTranslateBlobToText(const JsonParse*,u32,JsonString*);403static void jsonReturnParse(sqlite3_context*,JsonParse*);404static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32);405static void jsonParseFree(JsonParse*);406static u32 jsonbPayloadSize(const JsonParse*, u32, u32*);407static u32 jsonUnescapeOneChar(const char*, u32, u32*);408 409/**************************************************************************410** Utility routines for dealing with JsonCache objects411**************************************************************************/412 413/*414** Free a JsonCache object.415*/416static void jsonCacheDelete(JsonCache *p){417 int i;418 for(i=0; i<p->nUsed; i++){419 jsonParseFree(p->a[i]);420 }421 sqlite3DbFree(p->db, p);422}423static void jsonCacheDeleteGeneric(void *p){424 jsonCacheDelete((JsonCache*)p);425}426 427/*428** Insert a new entry into the cache. If the cache is full, expel429** the least recently used entry. Return SQLITE_OK on success or a430** result code otherwise.431**432** Cache entries are stored in age order, oldest first.433*/434static int jsonCacheInsert(435 sqlite3_context *ctx, /* The SQL statement context holding the cache */436 JsonParse *pParse /* The parse object to be added to the cache */437){438 JsonCache *p;439 440 assert( pParse->zJson!=0 );441 assert( pParse->bJsonIsRCStr );442 assert( pParse->delta==0 );443 p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID);444 if( p==0 ){445 sqlite3 *db = sqlite3_context_db_handle(ctx);446 p = sqlite3DbMallocZero(db, sizeof(*p));447 if( p==0 ) return SQLITE_NOMEM;448 p->db = db;449 sqlite3_set_auxdata(ctx, JSON_CACHE_ID, p, jsonCacheDeleteGeneric);450 p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID);451 if( p==0 ) return SQLITE_NOMEM;452 }453 if( p->nUsed >= JSON_CACHE_SIZE ){454 jsonParseFree(p->a[0]);455 memmove(p->a, &p->a[1], (JSON_CACHE_SIZE-1)*sizeof(p->a[0]));456 p->nUsed = JSON_CACHE_SIZE-1;457 }458 assert( pParse->nBlobAlloc>0 );459 pParse->eEdit = 0;460 pParse->nJPRef++;461 pParse->bReadOnly = 1;462 p->a[p->nUsed] = pParse;463 p->nUsed++;464 return SQLITE_OK;465}466 467/*468** Search for a cached translation the json text supplied by pArg. Return469** the JsonParse object if found. Return NULL if not found.470**471** When a match if found, the matching entry is moved to become the472** most-recently used entry if it isn't so already.473**474** The JsonParse object returned still belongs to the Cache and might475** be deleted at any moment. If the caller wants the JsonParse to476** linger, it needs to increment the nPJRef reference counter.477*/478static JsonParse *jsonCacheSearch(479 sqlite3_context *ctx, /* The SQL statement context holding the cache */480 sqlite3_value *pArg /* Function argument containing SQL text */481){482 JsonCache *p;483 int i;484 const char *zJson;485 int nJson;486 487 if( sqlite3_value_type(pArg)!=SQLITE_TEXT ){488 return 0;489 }490 zJson = (const char*)sqlite3_value_text(pArg);491 if( zJson==0 ) return 0;492 nJson = sqlite3_value_bytes(pArg);493 494 p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID);495 if( p==0 ){496 return 0;497 }498 for(i=0; i<p->nUsed; i++){499 if( p->a[i]->zJson==zJson ) break;500 }501 if( i>=p->nUsed ){502 for(i=0; i<p->nUsed; i++){503 if( p->a[i]->nJson!=nJson ) continue;504 if( memcmp(p->a[i]->zJson, zJson, nJson)==0 ) break;505 }506 }507 if( i<p->nUsed ){508 if( i<p->nUsed-1 ){509 /* Make the matching entry the most recently used entry */510 JsonParse *tmp = p->a[i];511 memmove(&p->a[i], &p->a[i+1], (p->nUsed-i-1)*sizeof(tmp));512 p->a[p->nUsed-1] = tmp;513 i = p->nUsed - 1;514 }515 assert( p->a[i]->delta==0 );516 return p->a[i];517 }else{518 return 0;519 }520}521 522/**************************************************************************523** Utility routines for dealing with JsonString objects524**************************************************************************/525 526/* Turn uninitialized bulk memory into a valid JsonString object527** holding a zero-length string.528*/529static void jsonStringZero(JsonString *p){530 p->zBuf = p->zSpace;531 p->nAlloc = sizeof(p->zSpace);532 p->nUsed = 0;533 p->bStatic = 1;534}535 536/* Initialize the JsonString object537*/538static void jsonStringInit(JsonString *p, sqlite3_context *pCtx){539 p->pCtx = pCtx;540 p->eErr = 0;541 jsonStringZero(p);542}543 544/* Free all allocated memory and reset the JsonString object back to its545** initial state.546*/547static void jsonStringReset(JsonString *p){548 if( !p->bStatic ) sqlite3RCStrUnref(p->zBuf);549 jsonStringZero(p);550}551 552/* Report an out-of-memory (OOM) condition553*/554static void jsonStringOom(JsonString *p){555 p->eErr |= JSTRING_OOM;556 if( p->pCtx ) sqlite3_result_error_nomem(p->pCtx);557 jsonStringReset(p);558}559 560/* Enlarge pJson->zBuf so that it can hold at least N more bytes.561** Return zero on success. Return non-zero on an OOM error562*/563static int jsonStringGrow(JsonString *p, u32 N){564 u64 nTotal = N<p->nAlloc ? p->nAlloc*2 : p->nAlloc+N+10;565 char *zNew;566 if( p->bStatic ){567 if( p->eErr ) return 1;568 zNew = sqlite3RCStrNew(nTotal);569 if( zNew==0 ){570 jsonStringOom(p);571 return SQLITE_NOMEM;572 }573 memcpy(zNew, p->zBuf, (size_t)p->nUsed);574 p->zBuf = zNew;575 p->bStatic = 0;576 }else{577 p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal);578 if( p->zBuf==0 ){579 p->eErr |= JSTRING_OOM;580 jsonStringZero(p);581 return SQLITE_NOMEM;582 }583 }584 p->nAlloc = nTotal;585 return SQLITE_OK;586}587 588/* Append N bytes from zIn onto the end of the JsonString string.589*/590static SQLITE_NOINLINE void jsonStringExpandAndAppend(591 JsonString *p,592 const char *zIn,593 u32 N594){595 assert( N>0 );596 if( jsonStringGrow(p,N) ) return;597 memcpy(p->zBuf+p->nUsed, zIn, N);598 p->nUsed += N;599}600static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){601 if( N==0 ) return;602 if( N+p->nUsed >= p->nAlloc ){603 jsonStringExpandAndAppend(p,zIn,N);604 }else{605 memcpy(p->zBuf+p->nUsed, zIn, N);606 p->nUsed += N;607 }608}609static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){610 assert( N>0 );611 if( N+p->nUsed >= p->nAlloc ){612 jsonStringExpandAndAppend(p,zIn,N);613 }else{614 memcpy(p->zBuf+p->nUsed, zIn, N);615 p->nUsed += N;616 }617}618 619/* Append formatted text (not to exceed N bytes) to the JsonString.620*/621static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){622 va_list ap;623 if( (p->nUsed + N >= p->nAlloc) && jsonStringGrow(p, N) ) return;624 va_start(ap, zFormat);625 sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap);626 va_end(ap);627 p->nUsed += (int)strlen(p->zBuf+p->nUsed);628}629 630/* Append a single character631*/632static SQLITE_NOINLINE void jsonAppendCharExpand(JsonString *p, char c){633 if( jsonStringGrow(p,1) ) return;634 p->zBuf[p->nUsed++] = c;635}636static void jsonAppendChar(JsonString *p, char c){637 if( p->nUsed>=p->nAlloc ){638 jsonAppendCharExpand(p,c);639 }else{640 p->zBuf[p->nUsed++] = c;641 }642}643 644/* Remove a single character from the end of the string645*/646static void jsonStringTrimOneChar(JsonString *p){647 if( p->eErr==0 ){648 assert( p->nUsed>0 );649 p->nUsed--;650 }651}652 653 654/* Make sure there is a zero terminator on p->zBuf[]655**656** Return true on success. Return false if an OOM prevents this657** from happening.658*/659static int jsonStringTerminate(JsonString *p){660 jsonAppendChar(p, 0);661 jsonStringTrimOneChar(p);662 return p->eErr==0;663}664 665/* Append a comma separator to the output buffer, if the previous666** character is not '[' or '{'.667*/668static void jsonAppendSeparator(JsonString *p){669 char c;670 if( p->nUsed==0 ) return;671 c = p->zBuf[p->nUsed-1];672 if( c=='[' || c=='{' ) return;673 jsonAppendChar(p, ',');674}675 676/* c is a control character. Append the canonical JSON representation677** of that control character to p.678**679** This routine assumes that the output buffer has already been enlarged680** sufficiently to hold the worst-case encoding plus a nul terminator.681*/682static void jsonAppendControlChar(JsonString *p, u8 c){683 static const char aSpecial[] = {684 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0,685 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0686 };687 assert( sizeof(aSpecial)==32 );688 assert( aSpecial['\b']=='b' );689 assert( aSpecial['\f']=='f' );690 assert( aSpecial['\n']=='n' );691 assert( aSpecial['\r']=='r' );692 assert( aSpecial['\t']=='t' );693 assert( c>=0 && c<sizeof(aSpecial) );694 assert( p->nUsed+7 <= p->nAlloc );695 if( aSpecial[c] ){696 p->zBuf[p->nUsed] = '\\';697 p->zBuf[p->nUsed+1] = aSpecial[c];698 p->nUsed += 2;699 }else{700 p->zBuf[p->nUsed] = '\\';701 p->zBuf[p->nUsed+1] = 'u';702 p->zBuf[p->nUsed+2] = '0';703 p->zBuf[p->nUsed+3] = '0';704 p->zBuf[p->nUsed+4] = "0123456789abcdef"[c>>4];705 p->zBuf[p->nUsed+5] = "0123456789abcdef"[c&0xf];706 p->nUsed += 6;707 }708}709 710/* Append the N-byte string in zIn to the end of the JsonString string711** under construction. Enclose the string in double-quotes ("...") and712** escape any double-quotes or backslash characters contained within the713** string.714**715** This routine is a high-runner. There is a measurable performance716** increase associated with unwinding the jsonIsOk[] loop.717*/718static void jsonAppendString(JsonString *p, const char *zIn, u32 N){719 u32 k;720 u8 c;721 const u8 *z = (const u8*)zIn;722 if( z==0 ) return;723 if( (N+p->nUsed+2 >= p->nAlloc) && jsonStringGrow(p,N+2)!=0 ) return;724 p->zBuf[p->nUsed++] = '"';725 while( 1 /*exit-by-break*/ ){726 k = 0;727 /* The following while() is the 4-way unwound equivalent of728 **729 ** while( k<N && jsonIsOk[z[k]] ){ k++; }730 */731 while( 1 /* Exit by break */ ){732 if( k+3>=N ){733 while( k<N && jsonIsOk[z[k]] ){ k++; }734 break;735 }736 if( !jsonIsOk[z[k]] ){737 break;738 }739 if( !jsonIsOk[z[k+1]] ){740 k += 1;741 break;742 }743 if( !jsonIsOk[z[k+2]] ){744 k += 2;745 break;746 }747 if( !jsonIsOk[z[k+3]] ){748 k += 3;749 break;750 }else{751 k += 4;752 }753 }754 if( k>=N ){755 if( k>0 ){756 memcpy(&p->zBuf[p->nUsed], z, k);757 p->nUsed += k;758 }759 break;760 }761 if( k>0 ){762 memcpy(&p->zBuf[p->nUsed], z, k);763 p->nUsed += k;764 z += k;765 N -= k;766 }767 c = z[0];768 if( c=='"' || c=='\\' ){769 if( (p->nUsed+N+3 > p->nAlloc) && jsonStringGrow(p,N+3)!=0 ) return;770 p->zBuf[p->nUsed++] = '\\';771 p->zBuf[p->nUsed++] = c;772 }else if( c=='\'' ){773 p->zBuf[p->nUsed++] = c;774 }else{775 if( (p->nUsed+N+7 > p->nAlloc) && jsonStringGrow(p,N+7)!=0 ) return;776 jsonAppendControlChar(p, c);777 }778 z++;779 N--;780 }781 p->zBuf[p->nUsed++] = '"';782 assert( p->nUsed<p->nAlloc );783}784 785/*786** Append an sqlite3_value (such as a function parameter) to the JSON787** string under construction in p.788*/789static void jsonAppendSqlValue(790 JsonString *p, /* Append to this JSON string */791 sqlite3_value *pValue /* Value to append */792){793 switch( sqlite3_value_type(pValue) ){794 case SQLITE_NULL: {795 jsonAppendRawNZ(p, "null", 4);796 break;797 }798 case SQLITE_FLOAT: {799 jsonPrintf(100, p, "%!0.15g", sqlite3_value_double(pValue));800 break;801 }802 case SQLITE_INTEGER: {803 const char *z = (const char*)sqlite3_value_text(pValue);804 u32 n = (u32)sqlite3_value_bytes(pValue);805 jsonAppendRaw(p, z, n);806 break;807 }808 case SQLITE_TEXT: {809 const char *z = (const char*)sqlite3_value_text(pValue);810 u32 n = (u32)sqlite3_value_bytes(pValue);811 if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){812 jsonAppendRaw(p, z, n);813 }else{814 jsonAppendString(p, z, n);815 }816 break;817 }818 default: {819 JsonParse px;820 memset(&px, 0, sizeof(px));821 if( jsonArgIsJsonb(pValue, &px) ){822 jsonTranslateBlobToText(&px, 0, p);823 }else if( p->eErr==0 ){824 sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1);825 p->eErr = JSTRING_ERR;826 jsonStringReset(p);827 }828 break;829 }830 }831}832 833/* Make the text in p (which is probably a generated JSON text string)834** the result of the SQL function.835**836** The JsonString is reset.837**838** If pParse and ctx are both non-NULL, then the SQL string in p is839** loaded into the zJson field of the pParse object as a RCStr and the840** pParse is added to the cache.841*/842static void jsonReturnString(843 JsonString *p, /* String to return */844 JsonParse *pParse, /* JSONB source or NULL */845 sqlite3_context *ctx /* Where to cache */846){847 assert( (pParse!=0)==(ctx!=0) );848 assert( ctx==0 || ctx==p->pCtx );849 if( p->eErr==0 ){850 int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx));851 if( flags & JSON_BLOB ){852 jsonReturnStringAsBlob(p);853 }else if( p->bStatic ){854 sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed,855 SQLITE_TRANSIENT, SQLITE_UTF8);856 }else if( jsonStringTerminate(p) ){857 if( pParse && pParse->bJsonIsRCStr==0 && pParse->nBlobAlloc>0 ){858 int rc;859 pParse->zJson = sqlite3RCStrRef(p->zBuf);860 pParse->nJson = p->nUsed;861 pParse->bJsonIsRCStr = 1;862 rc = jsonCacheInsert(ctx, pParse);863 if( rc==SQLITE_NOMEM ){864 sqlite3_result_error_nomem(ctx);865 jsonStringReset(p);866 return;867 }868 }869 sqlite3_result_text64(p->pCtx, sqlite3RCStrRef(p->zBuf), p->nUsed,870 sqlite3RCStrUnref,871 SQLITE_UTF8);872 }else{873 sqlite3_result_error_nomem(p->pCtx);874 }875 }else if( p->eErr & JSTRING_OOM ){876 sqlite3_result_error_nomem(p->pCtx);877 }else if( p->eErr & JSTRING_MALFORMED ){878 sqlite3_result_error(p->pCtx, "malformed JSON", -1);879 }880 jsonStringReset(p);881}882 883/**************************************************************************884** Utility routines for dealing with JsonParse objects885**************************************************************************/886 887/*888** Reclaim all memory allocated by a JsonParse object. But do not889** delete the JsonParse object itself.890*/891static void jsonParseReset(JsonParse *pParse){892 assert( pParse->nJPRef<=1 );893 if( pParse->bJsonIsRCStr ){894 sqlite3RCStrUnref(pParse->zJson);895 pParse->zJson = 0;896 pParse->nJson = 0;897 pParse->bJsonIsRCStr = 0;898 }899 if( pParse->nBlobAlloc ){900 sqlite3DbFree(pParse->db, pParse->aBlob);901 pParse->aBlob = 0;902 pParse->nBlob = 0;903 pParse->nBlobAlloc = 0;904 }905}906 907/*908** Decrement the reference count on the JsonParse object. When the909** count reaches zero, free the object.910*/911static void jsonParseFree(JsonParse *pParse){912 if( pParse ){913 if( pParse->nJPRef>1 ){914 pParse->nJPRef--;915 }else{916 jsonParseReset(pParse);917 sqlite3DbFree(pParse->db, pParse);918 }919 }920}921 922/**************************************************************************923** Utility routines for the JSON text parser924**************************************************************************/925 926/*927** Translate a single byte of Hex into an integer.928** This routine only gives a correct answer if h really is a valid hexadecimal929** character: 0..9a..fA..F. But unlike sqlite3HexToInt(), it does not930** assert() if the digit is not hex.931*/932static u8 jsonHexToInt(int h){933#ifdef SQLITE_ASCII934 h += 9*(1&(h>>6));935#endif936#ifdef SQLITE_EBCDIC937 h += 9*(1&~(h>>4));938#endif939 return (u8)(h & 0xf);940}941 942/*943** Convert a 4-byte hex string into an integer944*/945static u32 jsonHexToInt4(const char *z){946 u32 v;947 v = (jsonHexToInt(z[0])<<12)948 + (jsonHexToInt(z[1])<<8)949 + (jsonHexToInt(z[2])<<4)950 + jsonHexToInt(z[3]);951 return v;952}953 954/*955** Return true if z[] begins with 2 (or more) hexadecimal digits956*/957static int jsonIs2Hex(const char *z){958 return sqlite3Isxdigit(z[0]) && sqlite3Isxdigit(z[1]);959}960 961/*962** Return true if z[] begins with 4 (or more) hexadecimal digits963*/964static int jsonIs4Hex(const char *z){965 return jsonIs2Hex(z) && jsonIs2Hex(&z[2]);966}967 968/*969** Return the number of bytes of JSON5 whitespace at the beginning of970** the input string z[].971**972** JSON5 whitespace consists of any of the following characters:973**974** Unicode UTF-8 Name975** U+0009 09 horizontal tab976** U+000a 0a line feed977** U+000b 0b vertical tab978** U+000c 0c form feed979** U+000d 0d carriage return980** U+0020 20 space981** U+00a0 c2 a0 non-breaking space982** U+1680 e1 9a 80 ogham space mark983** U+2000 e2 80 80 en quad984** U+2001 e2 80 81 em quad985** U+2002 e2 80 82 en space986** U+2003 e2 80 83 em space987** U+2004 e2 80 84 three-per-em space988** U+2005 e2 80 85 four-per-em space989** U+2006 e2 80 86 six-per-em space990** U+2007 e2 80 87 figure space991** U+2008 e2 80 88 punctuation space992** U+2009 e2 80 89 thin space993** U+200a e2 80 8a hair space994** U+2028 e2 80 a8 line separator995** U+2029 e2 80 a9 paragraph separator996** U+202f e2 80 af narrow no-break space (NNBSP)997** U+205f e2 81 9f medium mathematical space (MMSP)998** U+3000 e3 80 80 ideographical space999** U+FEFF ef bb bf byte order mark1000**1001** In addition, comments between '/', '*' and '*', '/' and1002** from '/', '/' to end-of-line are also considered to be whitespace.1003*/1004static int json5Whitespace(const char *zIn){1005 int n = 0;1006 const u8 *z = (u8*)zIn;1007 while( 1 /*exit by "goto whitespace_done"*/ ){1008 switch( z[n] ){1009 case 0x09:1010 case 0x0a:1011 case 0x0b:1012 case 0x0c:1013 case 0x0d:1014 case 0x20: {1015 n++;1016 break;1017 }1018 case '/': {1019 if( z[n+1]=='*' && z[n+2]!=0 ){1020 int j;1021 for(j=n+3; z[j]!='/' || z[j-1]!='*'; j++){1022 if( z[j]==0 ) goto whitespace_done;1023 }1024 n = j+1;1025 break;1026 }else if( z[n+1]=='/' ){1027 int j;1028 char c;1029 for(j=n+2; (c = z[j])!=0; j++){1030 if( c=='\n' || c=='\r' ) break;1031 if( 0xe2==(u8)c && 0x80==(u8)z[j+1]1032 && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])1033 ){1034 j += 2;1035 break;1036 }1037 }1038 n = j;1039 if( z[n] ) n++;1040 break;1041 }1042 goto whitespace_done;1043 }1044 case 0xc2: {1045 if( z[n+1]==0xa0 ){1046 n += 2;1047 break;1048 }1049 goto whitespace_done;1050 }1051 case 0xe1: {1052 if( z[n+1]==0x9a && z[n+2]==0x80 ){1053 n += 3;1054 break;1055 }1056 goto whitespace_done;1057 }1058 case 0xe2: {1059 if( z[n+1]==0x80 ){1060 u8 c = z[n+2];1061 if( c<0x80 ) goto whitespace_done;1062 if( c<=0x8a || c==0xa8 || c==0xa9 || c==0xaf ){1063 n += 3;1064 break;1065 }1066 }else if( z[n+1]==0x81 && z[n+2]==0x9f ){1067 n += 3;1068 break;1069 }1070 goto whitespace_done;1071 }1072 case 0xe3: {1073 if( z[n+1]==0x80 && z[n+2]==0x80 ){1074 n += 3;1075 break;1076 }1077 goto whitespace_done;1078 }1079 case 0xef: {1080 if( z[n+1]==0xbb && z[n+2]==0xbf ){1081 n += 3;1082 break;1083 }1084 goto whitespace_done;1085 }1086 default: {1087 goto whitespace_done;1088 }1089 }1090 }1091 whitespace_done:1092 return n;1093}1094 1095/*1096** Extra floating-point literals to allow in JSON.1097*/1098static const struct NanInfName {1099 char c1;1100 char c2;1101 char n;1102 char eType;1103 char nRepl;1104 char *zMatch;1105 char *zRepl;1106} aNanInfName[] = {1107 { 'i', 'I', 3, JSONB_FLOAT, 7, "inf", "9.0e999" },1108 { 'i', 'I', 8, JSONB_FLOAT, 7, "infinity", "9.0e999" },1109 { 'n', 'N', 3, JSONB_NULL, 4, "NaN", "null" },1110 { 'q', 'Q', 4, JSONB_NULL, 4, "QNaN", "null" },1111 { 's', 'S', 4, JSONB_NULL, 4, "SNaN", "null" },1112};1113 1114 1115/*1116** Report the wrong number of arguments for json_insert(), json_replace()1117** or json_set().1118*/1119static void jsonWrongNumArgs(1120 sqlite3_context *pCtx,1121 const char *zFuncName1122){1123 char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments",1124 zFuncName);1125 sqlite3_result_error(pCtx, zMsg, -1);1126 sqlite3_free(zMsg);1127}1128 1129/****************************************************************************1130** Utility routines for dealing with the binary BLOB representation of JSON1131****************************************************************************/1132 1133/*1134** Expand pParse->aBlob so that it holds at least N bytes.1135**1136** Return the number of errors.1137*/1138static int jsonBlobExpand(JsonParse *pParse, u32 N){1139 u8 *aNew;1140 u64 t;1141 assert( N>pParse->nBlobAlloc );1142 if( pParse->nBlobAlloc==0 ){1143 t = 100;1144 }else{1145 t = pParse->nBlobAlloc*2;1146 }1147 if( t<N ) t = N+100;1148 aNew = sqlite3DbRealloc(pParse->db, pParse->aBlob, t);1149 if( aNew==0 ){ pParse->oom = 1; return 1; }1150 assert( t<0x7fffffff );1151 pParse->aBlob = aNew;1152 pParse->nBlobAlloc = (u32)t;1153 return 0;1154}1155 1156/*1157** If pParse->aBlob is not previously editable (because it is taken1158** from sqlite3_value_blob(), as indicated by the fact that1159** pParse->nBlobAlloc==0 and pParse->nBlob>0) then make it editable1160** by making a copy into space obtained from malloc.1161**1162** Return true on success. Return false on OOM.1163*/1164static int jsonBlobMakeEditable(JsonParse *pParse, u32 nExtra){1165 u8 *aOld;1166 u32 nSize;1167 assert( !pParse->bReadOnly );1168 if( pParse->oom ) return 0;1169 if( pParse->nBlobAlloc>0 ) return 1;1170 aOld = pParse->aBlob;1171 nSize = pParse->nBlob + nExtra;1172 pParse->aBlob = 0;1173 if( jsonBlobExpand(pParse, nSize) ){1174 return 0;1175 }1176 assert( pParse->nBlobAlloc >= pParse->nBlob + nExtra );1177 memcpy(pParse->aBlob, aOld, pParse->nBlob);1178 return 1;1179}1180 1181/* Expand pParse->aBlob and append one bytes.1182*/1183static SQLITE_NOINLINE void jsonBlobExpandAndAppendOneByte(1184 JsonParse *pParse,1185 u8 c1186){1187 jsonBlobExpand(pParse, pParse->nBlob+1);1188 if( pParse->oom==0 ){1189 assert( pParse->nBlob+1<=pParse->nBlobAlloc );1190 pParse->aBlob[pParse->nBlob++] = c;1191 }1192}1193 1194/* Append a single character.1195*/1196static void jsonBlobAppendOneByte(JsonParse *pParse, u8 c){1197 if( pParse->nBlob >= pParse->nBlobAlloc ){1198 jsonBlobExpandAndAppendOneByte(pParse, c);1199 }else{1200 pParse->aBlob[pParse->nBlob++] = c;