AryaWu/sqlite
0
1/*2** 2015-04-173**4** The author disclaims copyright to this source code. In place of5** a legal notice, here is a blessing:6**7** May you do good and not evil.8** May you find forgiveness for yourself and forgive others.9** May you share freely, never taking more than you give.10**11*************************************************************************12**13** This is a utility program designed to aid running the SQLite library14** against an external fuzzer, such as American Fuzzy Lop (AFL)15** (http://lcamtuf.coredump.cx/afl/). Basically, this program reads16** SQL text from standard input and passes it through to SQLite for evaluation,17** just like the "sqlite3" command-line shell. Differences from the18** command-line shell:19**20** (1) The complex "dot-command" extensions are omitted. This21** prevents the fuzzer from discovering that it can run things22** like ".shell rm -rf ~"23**24** (2) The database is opened with the SQLITE_OPEN_MEMORY flag so that25** no disk I/O from the database is permitted. The ATTACH command26** with a filename still uses an in-memory database.27**28** (3) The main in-memory database can be initialized from a template29** disk database so that the fuzzer starts with a database containing30** content.31**32** (4) The eval() SQL function is added, allowing the fuzzer to do 33** interesting recursive operations.34**35** (5) An error is raised if there is a memory leak.36**37** The input text can be divided into separate test cases using comments38** of the form:39**40** |****<...>****|41**42** where the "..." is arbitrary text. (Except the "|" should really be "/".43** "|" is used here to avoid compiler errors about nested comments.)44** A separate in-memory SQLite database is created to run each test case.45** This feature allows the "queue" of AFL to be captured into a single big46** file using a command like this:47**48** (for i in id:*; do echo '|****<'$i'>****|'; cat $i; done) >~/all-queue.txt49**50** (Once again, change the "|" to "/") Then all elements of the AFL queue51** can be run in a single go (for regression testing, for example) by typing:52**53** fuzzershell -f ~/all-queue.txt54**55** After running each chunk of SQL, the database connection is closed. The56** program aborts if the close fails or if there is any unfreed memory after57** the close.58**59** New test cases can be appended to all-queue.txt at any time. If redundant60** test cases are added, they can be eliminated by running:61**62** fuzzershell -f ~/all-queue.txt --unique-cases ~/unique-cases.txt63*/64#include <stdio.h>65#include <stdlib.h>66#include <string.h>67#include <stdarg.h>68#include <ctype.h>69#include "sqlite3.h"70#define ISDIGIT(X) isdigit((unsigned char)(X))71 72/*73** All global variables are gathered into the "g" singleton.74*/75struct GlobalVars {76 const char *zArgv0; /* Name of program */77 sqlite3_mem_methods sOrigMem; /* Original memory methods */78 sqlite3_mem_methods sOomMem; /* Memory methods with OOM simulator */79 int iOomCntdown; /* Memory fails on 1 to 0 transition */80 int nOomFault; /* Increments for each OOM fault */81 int bOomOnce; /* Fail just once if true */82 int bOomEnable; /* True to enable OOM simulation */83 int nOomBrkpt; /* Number of calls to oomFault() */84 char zTestName[100]; /* Name of current test */85} g;86 87/*88** Maximum number of iterations for an OOM test89*/90#ifndef OOM_MAX91# define OOM_MAX 62592#endif93 94/*95** This routine is called when a simulated OOM occurs. It exists as a96** convenient place to set a debugger breakpoint.97*/98static void oomFault(void){99 g.nOomBrkpt++; /* Prevent oomFault() from being optimized out */100}101 102 103/* Versions of malloc() and realloc() that simulate OOM conditions */104static void *oomMalloc(int nByte){105 if( nByte>0 && g.bOomEnable && g.iOomCntdown>0 ){106 g.iOomCntdown--;107 if( g.iOomCntdown==0 ){108 if( g.nOomFault==0 ) oomFault();109 g.nOomFault++;110 if( !g.bOomOnce ) g.iOomCntdown = 1;111 return 0;112 }113 }114 return g.sOrigMem.xMalloc(nByte);115}116static void *oomRealloc(void *pOld, int nByte){117 if( nByte>0 && g.bOomEnable && g.iOomCntdown>0 ){118 g.iOomCntdown--;119 if( g.iOomCntdown==0 ){120 if( g.nOomFault==0 ) oomFault();121 g.nOomFault++;122 if( !g.bOomOnce ) g.iOomCntdown = 1;123 return 0;124 }125 }126 return g.sOrigMem.xRealloc(pOld, nByte);127}128 129/*130** Print an error message and abort in such a way to indicate to the131** fuzzer that this counts as a crash.132*/133static void abendError(const char *zFormat, ...){134 va_list ap;135 if( g.zTestName[0] ){136 fprintf(stderr, "%s (%s): ", g.zArgv0, g.zTestName);137 }else{138 fprintf(stderr, "%s: ", g.zArgv0);139 }140 va_start(ap, zFormat);141 vfprintf(stderr, zFormat, ap);142 va_end(ap);143 fprintf(stderr, "\n");144 abort();145}146/*147** Print an error message and quit, but not in a way that would look148** like a crash.149*/150static void fatalError(const char *zFormat, ...){151 va_list ap;152 if( g.zTestName[0] ){153 fprintf(stderr, "%s (%s): ", g.zArgv0, g.zTestName);154 }else{155 fprintf(stderr, "%s: ", g.zArgv0);156 }157 va_start(ap, zFormat);158 vfprintf(stderr, zFormat, ap);159 va_end(ap);160 fprintf(stderr, "\n");161 exit(1);162}163 164/*165** Evaluate some SQL. Abort if unable.166*/167static void sqlexec(sqlite3 *db, const char *zFormat, ...){168 va_list ap;169 char *zSql;170 char *zErrMsg = 0;171 int rc;172 va_start(ap, zFormat);173 zSql = sqlite3_vmprintf(zFormat, ap);174 va_end(ap);175 rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg);176 if( rc ) abendError("failed sql [%s]: %s", zSql, zErrMsg);177 sqlite3_free(zSql);178}179 180/*181** This callback is invoked by sqlite3_log().182*/183static void shellLog(void *pNotUsed, int iErrCode, const char *zMsg){184 printf("LOG: (%d) %s\n", iErrCode, zMsg);185 fflush(stdout);186}187static void shellLogNoop(void *pNotUsed, int iErrCode, const char *zMsg){188 return;189}190 191/*192** This callback is invoked by sqlite3_exec() to return query results.193*/194static int execCallback(void *NotUsed, int argc, char **argv, char **colv){195 int i;196 static unsigned cnt = 0;197 printf("ROW #%u:\n", ++cnt);198 if( argv ){199 for(i=0; i<argc; i++){200 printf(" %s=", colv[i]);201 if( argv[i] ){202 printf("[%s]\n", argv[i]);203 }else{204 printf("NULL\n");205 }206 }207 }208 fflush(stdout);209 return 0;210}211static int execNoop(void *NotUsed, int argc, char **argv, char **colv){212 return 0;213}214 215#ifndef SQLITE_OMIT_TRACE216/*217** This callback is invoked by sqlite3_trace() as each SQL statement218** starts.219*/220static void traceCallback(void *NotUsed, const char *zMsg){221 printf("TRACE: %s\n", zMsg);222 fflush(stdout);223}224static void traceNoop(void *NotUsed, const char *zMsg){225 return;226}227#endif228 229/***************************************************************************230** String accumulator object231*/232typedef struct Str Str;233struct Str {234 char *z; /* The string. Memory from malloc() */235 sqlite3_uint64 n; /* Bytes of input used */236 sqlite3_uint64 nAlloc; /* Bytes allocated to z[] */237 int oomErr; /* OOM error has been seen */238};239 240/* Initialize a Str object */241static void StrInit(Str *p){242 memset(p, 0, sizeof(*p));243}244 245/* Append text to the end of a Str object */246static void StrAppend(Str *p, const char *z){247 sqlite3_uint64 n = strlen(z);248 if( p->n + n >= p->nAlloc ){249 char *zNew;250 sqlite3_uint64 nNew;251 if( p->oomErr ) return;252 nNew = p->nAlloc*2 + 100 + n;253 zNew = sqlite3_realloc(p->z, (int)nNew);254 if( zNew==0 ){255 sqlite3_free(p->z);256 memset(p, 0, sizeof(*p));257 p->oomErr = 1;258 return;259 }260 p->z = zNew;261 p->nAlloc = nNew;262 }263 memcpy(p->z + p->n, z, (size_t)n);264 p->n += n;265 p->z[p->n] = 0;266}267 268/* Return the current string content */269static char *StrStr(Str *p){270 return p->z;271}272 273/* Free the string */274static void StrFree(Str *p){275 sqlite3_free(p->z);276 StrInit(p);277}278 279/***************************************************************************280** eval() implementation copied from ../ext/misc/eval.c281*/282/*283** Structure used to accumulate the output284*/285struct EvalResult {286 char *z; /* Accumulated output */287 const char *zSep; /* Separator */288 int szSep; /* Size of the separator string */289 sqlite3_int64 nAlloc; /* Number of bytes allocated for z[] */290 sqlite3_int64 nUsed; /* Number of bytes of z[] actually used */291};292 293/*294** Callback from sqlite_exec() for the eval() function.295*/296static int callback(void *pCtx, int argc, char **argv, char **colnames){297 struct EvalResult *p = (struct EvalResult*)pCtx;298 int i; 299 for(i=0; i<argc; i++){300 const char *z = argv[i] ? argv[i] : "";301 size_t sz = strlen(z);302 if( (sqlite3_int64)sz+p->nUsed+p->szSep+1 > p->nAlloc ){303 char *zNew;304 p->nAlloc = p->nAlloc*2 + sz + p->szSep + 1;305 /* Using sqlite3_realloc64() would be better, but it is a recent306 ** addition and will cause a segfault if loaded by an older version307 ** of SQLite. */308 zNew = p->nAlloc<=0x7fffffff ? sqlite3_realloc(p->z, (int)p->nAlloc) : 0;309 if( zNew==0 ){310 sqlite3_free(p->z);311 memset(p, 0, sizeof(*p));312 return 1;313 }314 p->z = zNew;315 }316 if( p->nUsed>0 ){317 memcpy(&p->z[p->nUsed], p->zSep, p->szSep);318 p->nUsed += p->szSep;319 }320 memcpy(&p->z[p->nUsed], z, sz);321 p->nUsed += sz;322 }323 return 0;324}325 326/*327** Implementation of the eval(X) and eval(X,Y) SQL functions.328**329** Evaluate the SQL text in X. Return the results, using string330** Y as the separator. If Y is omitted, use a single space character.331*/332static void sqlEvalFunc(333 sqlite3_context *context,334 int argc,335 sqlite3_value **argv336){337 const char *zSql;338 sqlite3 *db;339 char *zErr = 0;340 int rc;341 struct EvalResult x;342 343 memset(&x, 0, sizeof(x));344 x.zSep = " ";345 zSql = (const char*)sqlite3_value_text(argv[0]);346 if( zSql==0 ) return;347 if( argc>1 ){348 x.zSep = (const char*)sqlite3_value_text(argv[1]);349 if( x.zSep==0 ) return;350 }351 x.szSep = (int)strlen(x.zSep);352 db = sqlite3_context_db_handle(context);353 rc = sqlite3_exec(db, zSql, callback, &x, &zErr);354 if( rc!=SQLITE_OK ){355 sqlite3_result_error(context, zErr, -1);356 sqlite3_free(zErr);357 }else if( x.zSep==0 ){358 sqlite3_result_error_nomem(context);359 sqlite3_free(x.z);360 }else{361 sqlite3_result_text(context, x.z, (int)x.nUsed, sqlite3_free);362 }363}364/* End of the eval() implementation365******************************************************************************/366 367/******************************************************************************368** The generate_series(START,END,STEP) eponymous table-valued function.369**370** This code is copy/pasted from ext/misc/series.c in the SQLite source tree.371*/372/* series_cursor is a subclass of sqlite3_vtab_cursor which will373** serve as the underlying representation of a cursor that scans374** over rows of the result375*/376typedef struct series_cursor series_cursor;377struct series_cursor {378 sqlite3_vtab_cursor base; /* Base class - must be first */379 int isDesc; /* True to count down rather than up */380 sqlite3_int64 iRowid; /* The rowid */381 sqlite3_int64 iValue; /* Current value ("value") */382 sqlite3_int64 mnValue; /* Mimimum value ("start") */383 sqlite3_int64 mxValue; /* Maximum value ("stop") */384 sqlite3_int64 iStep; /* Increment ("step") */385};386 387/*388** The seriesConnect() method is invoked to create a new389** series_vtab that describes the generate_series virtual table.390**391** Think of this routine as the constructor for series_vtab objects.392**393** All this routine needs to do is:394**395** (1) Allocate the series_vtab object and initialize all fields.396**397** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the398** result set of queries against generate_series will look like.399*/400static int seriesConnect(401 sqlite3 *db,402 void *pAux,403 int argc, const char *const*argv,404 sqlite3_vtab **ppVtab,405 char **pzErr406){407 sqlite3_vtab *pNew;408 int rc;409 410/* Column numbers */411#define SERIES_COLUMN_VALUE 0412#define SERIES_COLUMN_START 1413#define SERIES_COLUMN_STOP 2414#define SERIES_COLUMN_STEP 3415 416 rc = sqlite3_declare_vtab(db,417 "CREATE TABLE x(value,start hidden,stop hidden,step hidden)");418 if( rc==SQLITE_OK ){419 pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) );420 if( pNew==0 ) return SQLITE_NOMEM;421 memset(pNew, 0, sizeof(*pNew));422 }423 return rc;424}425 426/*427** This method is the destructor for series_cursor objects.428*/429static int seriesDisconnect(sqlite3_vtab *pVtab){430 sqlite3_free(pVtab);431 return SQLITE_OK;432}433 434/*435** Constructor for a new series_cursor object.436*/437static int seriesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){438 series_cursor *pCur;439 pCur = sqlite3_malloc( sizeof(*pCur) );440 if( pCur==0 ) return SQLITE_NOMEM;441 memset(pCur, 0, sizeof(*pCur));442 *ppCursor = &pCur->base;443 return SQLITE_OK;444}445 446/*447** Destructor for a series_cursor.448*/449static int seriesClose(sqlite3_vtab_cursor *cur){450 sqlite3_free(cur);451 return SQLITE_OK;452}453 454 455/*456** Advance a series_cursor to its next row of output.457*/458static int seriesNext(sqlite3_vtab_cursor *cur){459 series_cursor *pCur = (series_cursor*)cur;460 if( pCur->isDesc ){461 pCur->iValue -= pCur->iStep;462 }else{463 pCur->iValue += pCur->iStep;464 }465 pCur->iRowid++;466 return SQLITE_OK;467}468 469/*470** Return values of columns for the row at which the series_cursor471** is currently pointing.472*/473static int seriesColumn(474 sqlite3_vtab_cursor *cur, /* The cursor */475 sqlite3_context *ctx, /* First argument to sqlite3_result_...() */476 int i /* Which column to return */477){478 series_cursor *pCur = (series_cursor*)cur;479 sqlite3_int64 x = 0;480 switch( i ){481 case SERIES_COLUMN_START: x = pCur->mnValue; break;482 case SERIES_COLUMN_STOP: x = pCur->mxValue; break;483 case SERIES_COLUMN_STEP: x = pCur->iStep; break;484 default: x = pCur->iValue; break;485 }486 sqlite3_result_int64(ctx, x);487 return SQLITE_OK;488}489 490/*491** Return the rowid for the current row. In this implementation, the492** rowid is the same as the output value.493*/494static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){495 series_cursor *pCur = (series_cursor*)cur;496 *pRowid = pCur->iRowid;497 return SQLITE_OK;498}499 500/*501** Return TRUE if the cursor has been moved off of the last502** row of output.503*/504static int seriesEof(sqlite3_vtab_cursor *cur){505 series_cursor *pCur = (series_cursor*)cur;506 if( pCur->isDesc ){507 return pCur->iValue < pCur->mnValue;508 }else{509 return pCur->iValue > pCur->mxValue;510 }511}512 513/* True to cause run-time checking of the start=, stop=, and/or step= 514** parameters. The only reason to do this is for testing the515** constraint checking logic for virtual tables in the SQLite core.516*/517#ifndef SQLITE_SERIES_CONSTRAINT_VERIFY518# define SQLITE_SERIES_CONSTRAINT_VERIFY 0519#endif520 521/*522** This method is called to "rewind" the series_cursor object back523** to the first row of output. This method is always called at least524** once prior to any call to seriesColumn() or seriesRowid() or 525** seriesEof().526**527** The query plan selected by seriesBestIndex is passed in the idxNum528** parameter. (idxStr is not used in this implementation.) idxNum529** is a bitmask showing which constraints are available:530**531** 1: start=VALUE532** 2: stop=VALUE533** 4: step=VALUE534**535** Also, if bit 8 is set, that means that the series should be output536** in descending order rather than in ascending order.537**538** This routine should initialize the cursor and position it so that it539** is pointing at the first row, or pointing off the end of the table540** (so that seriesEof() will return true) if the table is empty.541*/542static int seriesFilter(543 sqlite3_vtab_cursor *pVtabCursor, 544 int idxNum, const char *idxStr,545 int argc, sqlite3_value **argv546){547 series_cursor *pCur = (series_cursor *)pVtabCursor;548 int i = 0;549 if( idxNum & 1 ){550 pCur->mnValue = sqlite3_value_int64(argv[i++]);551 }else{552 pCur->mnValue = 0;553 }554 if( idxNum & 2 ){555 pCur->mxValue = sqlite3_value_int64(argv[i++]);556 }else{557 pCur->mxValue = 0xffffffff;558 }559 if( idxNum & 4 ){560 pCur->iStep = sqlite3_value_int64(argv[i++]);561 if( pCur->iStep<1 ) pCur->iStep = 1;562 }else{563 pCur->iStep = 1;564 }565 if( idxNum & 8 ){566 pCur->isDesc = 1;567 pCur->iValue = pCur->mxValue;568 if( pCur->iStep>0 ){569 pCur->iValue -= (pCur->mxValue - pCur->mnValue)%pCur->iStep;570 }571 }else{572 pCur->isDesc = 0;573 pCur->iValue = pCur->mnValue;574 }575 pCur->iRowid = 1;576 return SQLITE_OK;577}578 579/*580** SQLite will invoke this method one or more times while planning a query581** that uses the generate_series virtual table. This routine needs to create582** a query plan for each invocation and compute an estimated cost for that583** plan.584**585** In this implementation idxNum is used to represent the586** query plan. idxStr is unused.587**588** The query plan is represented by bits in idxNum:589**590** (1) start = $value -- constraint exists591** (2) stop = $value -- constraint exists592** (4) step = $value -- constraint exists593** (8) output in descending order594*/595static int seriesBestIndex(596 sqlite3_vtab *tab,597 sqlite3_index_info *pIdxInfo598){599 int i; /* Loop over constraints */600 int idxNum = 0; /* The query plan bitmask */601 int startIdx = -1; /* Index of the start= constraint, or -1 if none */602 int stopIdx = -1; /* Index of the stop= constraint, or -1 if none */603 int stepIdx = -1; /* Index of the step= constraint, or -1 if none */604 int nArg = 0; /* Number of arguments that seriesFilter() expects */605 606 const struct sqlite3_index_constraint *pConstraint;607 pConstraint = pIdxInfo->aConstraint;608 for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){609 if( pConstraint->usable==0 ) continue;610 if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;611 switch( pConstraint->iColumn ){612 case SERIES_COLUMN_START:613 startIdx = i;614 idxNum |= 1;615 break;616 case SERIES_COLUMN_STOP:617 stopIdx = i;618 idxNum |= 2;619 break;620 case SERIES_COLUMN_STEP:621 stepIdx = i;622 idxNum |= 4;623 break;624 }625 }626 if( startIdx>=0 ){627 pIdxInfo->aConstraintUsage[startIdx].argvIndex = ++nArg;628 pIdxInfo->aConstraintUsage[startIdx].omit= !SQLITE_SERIES_CONSTRAINT_VERIFY;629 }630 if( stopIdx>=0 ){631 pIdxInfo->aConstraintUsage[stopIdx].argvIndex = ++nArg;632 pIdxInfo->aConstraintUsage[stopIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;633 }634 if( stepIdx>=0 ){635 pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg;636 pIdxInfo->aConstraintUsage[stepIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;637 }638 if( (idxNum & 3)==3 ){639 /* Both start= and stop= boundaries are available. This is the 640 ** the preferred case */641 pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0));642 pIdxInfo->estimatedRows = 1000;643 if( pIdxInfo->nOrderBy==1 ){644 if( pIdxInfo->aOrderBy[0].desc ) idxNum |= 8;645 pIdxInfo->orderByConsumed = 1;646 }647 }else{648 /* If either boundary is missing, we have to generate a huge span649 ** of numbers. Make this case very expensive so that the query650 ** planner will work hard to avoid it. */651 pIdxInfo->estimatedCost = (double)2147483647;652 pIdxInfo->estimatedRows = 2147483647;653 }654 pIdxInfo->idxNum = idxNum;655 return SQLITE_OK;656}657 658/*659** This following structure defines all the methods for the 660** generate_series virtual table.661*/662static sqlite3_module seriesModule = {663 0, /* iVersion */664 0, /* xCreate */665 seriesConnect, /* xConnect */666 seriesBestIndex, /* xBestIndex */667 seriesDisconnect, /* xDisconnect */668 0, /* xDestroy */669 seriesOpen, /* xOpen - open a cursor */670 seriesClose, /* xClose - close a cursor */671 seriesFilter, /* xFilter - configure scan constraints */672 seriesNext, /* xNext - advance a cursor */673 seriesEof, /* xEof - check for end of scan */674 seriesColumn, /* xColumn - read data */675 seriesRowid, /* xRowid - read data */676 0, /* xUpdate */677 0, /* xBegin */678 0, /* xSync */679 0, /* xCommit */680 0, /* xRollback */681 0, /* xFindMethod */682 0, /* xRename */683 0, /* xSavepoint */684 0, /* xRelease */685 0, /* xRollbackTo */686 0, /* xShadowName */687 0 /* xIntegrity */688};689/* END the generate_series(START,END,STEP) implementation690*********************************************************************************/691 692/*693** Print sketchy documentation for this utility program694*/695static void showHelp(void){696 printf("Usage: %s [options] ?FILE...?\n", g.zArgv0);697 printf(698"Read SQL text from FILE... (or from standard input if FILE... is omitted)\n"699"and then evaluate each block of SQL contained therein.\n"700"Options:\n"701" --autovacuum Enable AUTOVACUUM mode\n"702" --database FILE Use database FILE instead of an in-memory database\n"703" --disable-lookaside Turn off lookaside memory\n"704" --heap SZ MIN Memory allocator uses SZ bytes & min allocation MIN\n"705" --help Show this help text\n" 706" --lookaside N SZ Configure lookaside for N slots of SZ bytes each\n"707" --oom Run each test multiple times in a simulated OOM loop\n"708" --pagesize N Set the page size to N\n"709" --pcache N SZ Configure N pages of pagecache each of size SZ bytes\n"710" -q Reduced output\n"711" --quiet Reduced output\n"712" --scratch N SZ Configure scratch memory for N slots of SZ bytes each\n"713" --unique-cases FILE Write all unique test cases to FILE\n"714" --utf16be Set text encoding to UTF-16BE\n"715" --utf16le Set text encoding to UTF-16LE\n"716" -v Increased output\n"717" --verbose Increased output\n"718 );719}720 721/*722** Return the value of a hexadecimal digit. Return -1 if the input723** is not a hex digit.724*/725static int hexDigitValue(char c){726 if( c>='0' && c<='9' ) return c - '0';727 if( c>='a' && c<='f' ) return c - 'a' + 10;728 if( c>='A' && c<='F' ) return c - 'A' + 10;729 return -1;730}731 732/*733** Interpret zArg as an integer value, possibly with suffixes.734*/735static int integerValue(const char *zArg){736 sqlite3_int64 v = 0;737 static const struct { char *zSuffix; int iMult; } aMult[] = {738 { "KiB", 1024 },739 { "MiB", 1024*1024 },740 { "GiB", 1024*1024*1024 },741 { "KB", 1000 },742 { "MB", 1000000 },743 { "GB", 1000000000 },744 { "K", 1000 },745 { "M", 1000000 },746 { "G", 1000000000 },747 };748 int i;749 int isNeg = 0;750 if( zArg[0]=='-' ){751 isNeg = 1;752 zArg++;753 }else if( zArg[0]=='+' ){754 zArg++;755 }756 if( zArg[0]=='0' && zArg[1]=='x' ){757 int x;758 zArg += 2;759 while( (x = hexDigitValue(zArg[0]))>=0 ){760 v = (v<<4) + x;761 zArg++;762 }763 }else{764 while( ISDIGIT(zArg[0]) ){765 v = v*10 + zArg[0] - '0';766 zArg++;767 }768 }769 for(i=0; i<sizeof(aMult)/sizeof(aMult[0]); i++){770 if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){771 v *= aMult[i].iMult;772 break;773 }774 }775 if( v>0x7fffffff ) abendError("parameter too large - max 2147483648");776 return (int)(isNeg? -v : v);777}778 779/* Return the current wall-clock time */780static sqlite3_int64 timeOfDay(void){781 static sqlite3_vfs *clockVfs = 0;782 sqlite3_int64 t;783 if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0);784 if( clockVfs->iVersion>=1 && clockVfs->xCurrentTimeInt64!=0 ){785 clockVfs->xCurrentTimeInt64(clockVfs, &t);786 }else{787 double r;788 clockVfs->xCurrentTime(clockVfs, &r);789 t = (sqlite3_int64)(r*86400000.0);790 }791 return t;792}793 794int main(int argc, char **argv){795 char *zIn = 0; /* Input text */796 int nAlloc = 0; /* Number of bytes allocated for zIn[] */797 int nIn = 0; /* Number of bytes of zIn[] used */798 size_t got; /* Bytes read from input */799 int rc = SQLITE_OK; /* Result codes from API functions */800 int i; /* Loop counter */801 int iNext; /* Next block of SQL */802 sqlite3 *db; /* Open database */803 char *zErrMsg = 0; /* Error message returned from sqlite3_exec() */804 const char *zEncoding = 0; /* --utf16be or --utf16le */805 int nHeap = 0, mnHeap = 0; /* Heap size from --heap */806 int nLook = 0, szLook = 0; /* --lookaside configuration */807 int nPCache = 0, szPCache = 0;/* --pcache configuration */808 int nScratch = 0, szScratch=0;/* --scratch configuration */809 int pageSize = 0; /* Desired page size. 0 means default */810 void *pHeap = 0; /* Allocated heap space */811 void *pLook = 0; /* Allocated lookaside space */812 void *pPCache = 0; /* Allocated storage for pcache */813 void *pScratch = 0; /* Allocated storage for scratch */814 int doAutovac = 0; /* True for --autovacuum */815 char *zSql; /* SQL to run */816 char *zToFree = 0; /* Call sqlite3_free() on this afte running zSql */817 int verboseFlag = 0; /* --verbose or -v flag */818 int quietFlag = 0; /* --quiet or -q flag */819 int nTest = 0; /* Number of test cases run */820 int multiTest = 0; /* True if there will be multiple test cases */821 int lastPct = -1; /* Previous percentage done output */822 sqlite3 *dataDb = 0; /* Database holding compacted input data */823 sqlite3_stmt *pStmt = 0; /* Statement to insert testcase into dataDb */824 const char *zDataOut = 0; /* Write compacted data to this output file */825 int nHeader = 0; /* Bytes of header comment text on input file */826 int oomFlag = 0; /* --oom */827 int oomCnt = 0; /* Counter for the OOM loop */828 char zErrBuf[200]; /* Space for the error message */829 const char *zFailCode; /* Value of the TEST_FAILURE environment var */830 const char *zPrompt; /* Initial prompt when large-file fuzzing */831 int nInFile = 0; /* Number of input files to read */832 char **azInFile = 0; /* Array of input file names */833 int jj; /* Loop counter for azInFile[] */834 sqlite3_int64 iBegin; /* Start time for the whole program */835 sqlite3_int64 iStart, iEnd; /* Start and end-times for a test case */836 const char *zDbName = 0; /* Name of an on-disk database file to open */837 838 iBegin = timeOfDay();839 sqlite3_shutdown();840 zFailCode = getenv("TEST_FAILURE");841 g.zArgv0 = argv[0];842 zPrompt = "<stdin>";843 for(i=1; i<argc; i++){844 const char *z = argv[i];845 if( z[0]=='-' ){846 z++;847 if( z[0]=='-' ) z++;848 if( strcmp(z,"autovacuum")==0 ){849 doAutovac = 1;850 }else851 if( strcmp(z,"database")==0 ){852 if( i>=argc-1 ) abendError("missing argument on %s\n", argv[i]);853 zDbName = argv[i+1];854 i += 1;855 }else856 if( strcmp(z,"disable-lookaside")==0 ){857 nLook = 1;858 szLook = 0;859 }else860 if( strcmp(z, "f")==0 && i+1<argc ){861 i++;862 goto addNewInFile;863 }else864 if( strcmp(z,"heap")==0 ){865 if( i>=argc-2 ) abendError("missing arguments on %s\n", argv[i]);866 nHeap = integerValue(argv[i+1]);867 mnHeap = integerValue(argv[i+2]);868 i += 2;869 }else870 if( strcmp(z,"help")==0 ){871 showHelp();872 return 0;873 }else874 if( strcmp(z,"lookaside")==0 ){875 if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]);876 nLook = integerValue(argv[i+1]);877 szLook = integerValue(argv[i+2]);878 i += 2;879 }else880 if( strcmp(z,"oom")==0 ){881 oomFlag = 1;882 }else883 if( strcmp(z,"pagesize")==0 ){884 if( i>=argc-1 ) abendError("missing argument on %s", argv[i]);885 pageSize = integerValue(argv[++i]);886 }else887 if( strcmp(z,"pcache")==0 ){888 if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]);889 nPCache = integerValue(argv[i+1]);890 szPCache = integerValue(argv[i+2]);891 i += 2;892 }else893 if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){894 quietFlag = 1;895 verboseFlag = 0;896 }else897 if( strcmp(z,"scratch")==0 ){898 if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]);899 nScratch = integerValue(argv[i+1]);900 szScratch = integerValue(argv[i+2]);901 i += 2;902 }else903 if( strcmp(z, "unique-cases")==0 ){904 if( i>=argc-1 ) abendError("missing arguments on %s", argv[i]);905 if( zDataOut ) abendError("only one --minimize allowed");906 zDataOut = argv[++i];907 }else908 if( strcmp(z,"utf16le")==0 ){909 zEncoding = "utf16le";910 }else911 if( strcmp(z,"utf16be")==0 ){912 zEncoding = "utf16be";913 }else914 if( strcmp(z,"verbose")==0 || strcmp(z,"v")==0 ){915 quietFlag = 0;916 verboseFlag = 1;917 }else918 {919 abendError("unknown option: %s", argv[i]);920 }921 }else{922 addNewInFile:923 nInFile++;924 azInFile = realloc(azInFile, sizeof(azInFile[0])*nInFile);925 if( azInFile==0 ) abendError("out of memory");926 azInFile[nInFile-1] = argv[i];927 }928 }929 930 /* Do global SQLite initialization */931 sqlite3_config(SQLITE_CONFIG_LOG, verboseFlag ? shellLog : shellLogNoop, 0);932 if( nHeap>0 ){933 pHeap = malloc( nHeap );934 if( pHeap==0 ) fatalError("cannot allocate %d-byte heap\n", nHeap);935 rc = sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nHeap, mnHeap);936 if( rc ) abendError("heap configuration failed: %d\n", rc);937 }938 if( oomFlag ){939 sqlite3_config(SQLITE_CONFIG_GETMALLOC, &g.sOrigMem);940 g.sOomMem = g.sOrigMem;941 g.sOomMem.xMalloc = oomMalloc;942 g.sOomMem.xRealloc = oomRealloc;943 sqlite3_config(SQLITE_CONFIG_MALLOC, &g.sOomMem);944 }945 if( nLook>0 ){946 sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0);947 if( szLook>0 ){948 pLook = malloc( nLook*szLook );949 if( pLook==0 ) fatalError("out of memory");950 }951 }952 if( nScratch>0 && szScratch>0 ){953 pScratch = malloc( nScratch*(sqlite3_int64)szScratch );954 if( pScratch==0 ) fatalError("cannot allocate %lld-byte scratch",955 nScratch*(sqlite3_int64)szScratch);956 rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, pScratch, szScratch, nScratch);957 if( rc ) abendError("scratch configuration failed: %d\n", rc);958 }959 if( nPCache>0 && szPCache>0 ){960 pPCache = malloc( nPCache*(sqlite3_int64)szPCache );961 if( pPCache==0 ) fatalError("cannot allocate %lld-byte pcache",962 nPCache*(sqlite3_int64)szPCache);963 rc = sqlite3_config(SQLITE_CONFIG_PAGECACHE, pPCache, szPCache, nPCache);964 if( rc ) abendError("pcache configuration failed: %d", rc);965 }966 967 /* If the --unique-cases option was supplied, open the database that will968 ** be used to gather unique test cases.969 */970 if( zDataOut ){971 rc = sqlite3_open(":memory:", &dataDb);972 if( rc ) abendError("cannot open :memory: database");973 rc = sqlite3_exec(dataDb,974 "CREATE TABLE testcase(sql BLOB PRIMARY KEY, tm) WITHOUT ROWID;",0,0,0);975 if( rc ) abendError("%s", sqlite3_errmsg(dataDb));976 rc = sqlite3_prepare_v2(dataDb,977 "INSERT OR IGNORE INTO testcase(sql,tm)VALUES(?1,?2)",978 -1, &pStmt, 0);979 if( rc ) abendError("%s", sqlite3_errmsg(dataDb));980 }981 982 /* Initialize the input buffer used to hold SQL text */983 if( nInFile==0 ) nInFile = 1;984 nAlloc = 1000;985 zIn = malloc(nAlloc);986 if( zIn==0 ) fatalError("out of memory");987 988 /* Loop over all input files */989 for(jj=0; jj<nInFile; jj++){990 991 /* Read the complete content of the next input file into zIn[] */992 FILE *in;993 if( azInFile ){994 int j, k;995 in = fopen(azInFile[jj],"rb");996 if( in==0 ){997 abendError("cannot open %s for reading", azInFile[jj]);998 }999 zPrompt = azInFile[jj];1000 for(j=k=0; zPrompt[j]; j++) if( zPrompt[j]=='/' ) k = j+1;1001 zPrompt += k;1002 }else{1003 in = stdin;1004 zPrompt = "<stdin>";1005 }1006 while( !feof(in) ){1007 got = fread(zIn+nIn, 1, nAlloc-nIn-1, in); 1008 nIn += (int)got;1009 zIn[nIn] = 0;1010 if( got==0 ) break;1011 if( nAlloc - nIn - 1 < 100 ){1012 nAlloc += nAlloc+1000;1013 zIn = realloc(zIn, nAlloc);1014 if( zIn==0 ) fatalError("out of memory");1015 }1016 }1017 if( in!=stdin ) fclose(in);1018 lastPct = -1;1019 1020 /* Skip initial lines of the input file that begin with "#" */1021 for(i=0; i<nIn; i=iNext+1){1022 if( zIn[i]!='#' ) break;1023 for(iNext=i+1; iNext<nIn && zIn[iNext]!='\n'; iNext++){}1024 }1025 nHeader = i;1026 1027 /* Process all test cases contained within the input file.1028 */1029 for(; i<nIn; i=iNext, nTest++, g.zTestName[0]=0){1030 char cSaved;1031 if( strncmp(&zIn[i], "/****<",6)==0 ){1032 char *z = strstr(&zIn[i], ">****/");1033 if( z ){1034 z += 6;1035 sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "%.*s", 1036 (int)(z-&zIn[i]) - 12, &zIn[i+6]);1037 if( verboseFlag ){1038 printf("%.*s\n", (int)(z-&zIn[i]), &zIn[i]);1039 fflush(stdout);1040 }1041 i += (int)(z-&zIn[i]);1042 multiTest = 1;1043 }1044 }1045 for(iNext=i; iNext<nIn && strncmp(&zIn[iNext],"/****<",6)!=0; iNext++){}1046 cSaved = zIn[iNext];1047 zIn[iNext] = 0;1048 1049 1050 /* Print out the SQL of the next test case is --verbose is enabled1051 */1052 zSql = &zIn[i];1053 if( verboseFlag ){1054 printf("INPUT (offset: %d, size: %d): [%s]\n",1055 i, (int)strlen(&zIn[i]), &zIn[i]);1056 }else if( multiTest && !quietFlag ){1057 if( oomFlag ){1058 printf("%s\n", g.zTestName);1059 }else{1060 int pct = (10*iNext)/nIn;1061 if( pct!=lastPct ){1062 if( lastPct<0 ) printf("%s:", zPrompt);1063 printf(" %d%%", pct*10);1064 lastPct = pct;1065 }1066 }1067 }else if( nInFile>1 ){1068 printf("%s\n", zPrompt);1069 }1070 fflush(stdout);1071 1072 /* Run the next test case. Run it multiple times in --oom mode1073 */1074 if( oomFlag ){1075 oomCnt = g.iOomCntdown = 1;1076 g.nOomFault = 0;1077 g.bOomOnce = 1;1078 if( verboseFlag ){1079 printf("Once.%d\n", oomCnt);1080 fflush(stdout);1081 }1082 }else{1083 oomCnt = 0;1084 }1085 do{1086 Str sql;1087 StrInit(&sql);1088 if( zDbName ){1089 rc = sqlite3_open_v2(zDbName, &db, SQLITE_OPEN_READWRITE, 0);1090 if( rc!=SQLITE_OK ){1091 abendError("Cannot open database file %s", zDbName);1092 }1093 }else{1094 rc = sqlite3_open_v2(1095 "main.db", &db,1096 SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY,1097 0);1098 if( rc!=SQLITE_OK ){1099 abendError("Unable to open the in-memory database");1100 }1101 }1102 if( pLook ){1103 rc = sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE,pLook,szLook,nLook);1104 if( rc!=SQLITE_OK ) abendError("lookaside configuration filed: %d", rc);1105 }1106 #ifndef SQLITE_OMIT_TRACE1107 sqlite3_trace(db, verboseFlag ? traceCallback : traceNoop, 0);1108 #endif1109 sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);1110 sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);1111 sqlite3_create_module(db, "generate_series", &seriesModule, 0);1112 sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 1000000);1113 if( zEncoding ) sqlexec(db, "PRAGMA encoding=%s", zEncoding);1114 if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize);1115 if( doAutovac ) sqlexec(db, "PRAGMA auto_vacuum=FULL");1116 iStart = timeOfDay();1117 1118 /* If using an input database file and that database contains a table1119 ** named "autoexec" with a column "sql", then replace the input SQL1120 ** with the concatenated text of the autoexec table. In this way,1121 ** if the database file is the input being fuzzed, the SQL text is1122 ** fuzzed at the same time. */1123 if( sqlite3_table_column_metadata(db,0,"autoexec","sql",0,0,0,0,0)==0 ){1124 sqlite3_stmt *pStmt2;1125 rc = sqlite3_prepare_v2(db,"SELECT sql FROM autoexec",-1,&pStmt2,0);1126 if( rc==SQLITE_OK ){1127 while( sqlite3_step(pStmt2)==SQLITE_ROW ){1128 StrAppend(&sql, (const char*)sqlite3_column_text(pStmt2, 0));1129 StrAppend(&sql, "\n");1130 }1131 }1132 sqlite3_finalize(pStmt2);1133 zSql = StrStr(&sql);1134 }1135 1136 g.bOomEnable = 1;1137 if( verboseFlag ){1138 zErrMsg = 0;1139 rc = sqlite3_exec(db, zSql, execCallback, 0, &zErrMsg);1140 if( zErrMsg ){1141 sqlite3_snprintf(sizeof(zErrBuf),zErrBuf,"%z", zErrMsg);1142 zErrMsg = 0;1143 }1144 }else {1145 rc = sqlite3_exec(db, zSql, execNoop, 0, 0);1146 }1147 g.bOomEnable = 0;1148 iEnd = timeOfDay();1149 StrFree(&sql);1150 rc = sqlite3_close(db);1151 if( rc ){1152 abendError("sqlite3_close() failed with rc=%d", rc);1153 }1154 if( !zDataOut && sqlite3_memory_used()>0 ){1155 abendError("memory in use after close: %lld bytes",sqlite3_memory_used());1156 }1157 if( oomFlag ){1158 /* Limit the number of iterations of the OOM loop to OOM_MAX. If the1159 ** first pass (single failure) exceeds 2/3rds of OOM_MAX this skip the1160 ** second pass (continuous failure after first) completely. */1161 if( g.nOomFault==0 || oomCnt>OOM_MAX ){1162 if( g.bOomOnce && oomCnt<=(OOM_MAX*2/3) ){1163 oomCnt = g.iOomCntdown = 1;1164 g.bOomOnce = 0;1165 }else{1166 oomCnt = 0;1167 }1168 }else{1169 g.iOomCntdown = ++oomCnt;1170 g.nOomFault = 0;1171 }1172 if( oomCnt ){1173 if( verboseFlag ){1174 printf("%s.%d\n", g.bOomOnce ? "Once" : "Multi", oomCnt);1175 fflush(stdout);1176 }1177 nTest++;1178 }1179 }1180 }while( oomCnt>0 );1181 1182 /* Store unique test cases in the in the dataDb database if the1183 ** --unique-cases flag is present1184 */1185 if( zDataOut ){1186 sqlite3_bind_blob(pStmt, 1, &zIn[i], iNext-i, SQLITE_STATIC);1187 sqlite3_bind_int64(pStmt, 2, iEnd - iStart);1188 rc = sqlite3_step(pStmt);1189 if( rc!=SQLITE_DONE ) abendError("%s", sqlite3_errmsg(dataDb));1190 sqlite3_reset(pStmt);1191 }1192 1193 /* Free the SQL from the current test case1194 */1195 if( zToFree ){1196 sqlite3_free(zToFree);1197 zToFree = 0;1198 }1199 zIn[iNext] = cSaved;1200