From 01713975057d39725fc2a73787094339fb02ac85 Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Tue, 1 Nov 2011 13:35:45 +0000 Subject: [PATCH] updated bundled library to 3.7.9 --- Changes | 3 + sqlite3.c | 3356 +++++++++++++++++++++++++++++++------------------- sqlite3.h | 61 +- sqlite3ext.h | 47 +- 4 files changed, 2169 insertions(+), 1298 deletions(-) diff --git a/Changes b/Changes index dfa3b43..64fdb8c 100644 --- a/Changes +++ b/Changes @@ -1,5 +1,8 @@ Changes for Perl extension DBD-SQLite +1.34_04 to be released + - Updated to SQLite 3.7.9 (ISHIGAKI) + 1.34_03 Tue 1 Nov 2011 - Adding an explicit dynamic_config => 1 (ADAMK) - bind_param SQL_INTEGER error now actually dies with RaiseError. diff --git a/sqlite3.c b/sqlite3.c index e60c457..3f5ce2b 100644 --- a/sqlite3.c +++ b/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.7.8. By combining all the individual C code files into this +** version 3.7.9. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -316,13 +316,6 @@ #include #endif -/* -** The number of samples of an index that SQLite takes in order to -** construct a histogram of the table content when running ANALYZE -** and with SQLITE_ENABLE_STAT2 -*/ -#define SQLITE_INDEX_SAMPLES 10 - /* ** The following macros are used to cast pointers to integers and ** integers to pointers. The way you do this varies from one compiler @@ -396,7 +389,7 @@ ** assertion will be triggered. ** ** (Historical note: There used to be several other options, but we've -** pared it down to just these two.) +** pared it down to just these three.) ** ** If none of the above are defined, then set SQLITE_SYSTEM_MALLOC as ** the default. @@ -656,9 +649,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.8" -#define SQLITE_VERSION_NUMBER 3007008 -#define SQLITE_SOURCE_ID "2011-09-19 14:49:19 3e0da808d2f5b4d12046e05980ca04578f581177" +#define SQLITE_VERSION "3.7.9" +#define SQLITE_VERSION_NUMBER 3007009 +#define SQLITE_SOURCE_ID "2011-11-01 00:52:41 c7c6050ef060877ebe77b41d959e9df13f8c9b5e" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -1295,7 +1288,7 @@ struct sqlite3_io_methods { ** retry counts and intervals for certain disk I/O operations for the ** windows [VFS] in order to work to provide robustness against ** anti-virus programs. By default, the windows VFS will retry file read, -** file write, and file delete opertions up to 10 times, with a delay +** file write, and file delete operations up to 10 times, with a delay ** of 25 milliseconds before the first retry and with the delay increasing ** by an additional 25 milliseconds with each subsequent retry. This ** opcode allows those to values (10 retries and 25 milliseconds of delay) @@ -1320,7 +1313,11 @@ struct sqlite3_io_methods { ** That integer is 0 to disable persistent WAL mode or 1 to enable persistent ** WAL mode. If the integer is -1, then it is overwritten with the current ** WAL persistence setting. -** +** +** ^The [SQLITE_FCNTL_OVERWRITE] opcode is invoked by SQLite after opening +** a write transaction to indicate that, unless it is rolled back for some +** reason, the entire database file will be overwritten by the current +** transaction. This is used by VACUUM operations. */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 @@ -1332,6 +1329,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_SYNC_OMITTED 8 #define SQLITE_FCNTL_WIN32_AV_RETRY 9 #define SQLITE_FCNTL_PERSIST_WAL 10 +#define SQLITE_FCNTL_OVERWRITE 11 /* ** CAPI3REF: Mutex Handle @@ -1948,8 +1946,8 @@ struct sqlite3_mem_methods { ** allocator is engaged to handle all of SQLites memory allocation needs. ** The first pointer (the memory pointer) must be aligned to an 8-byte ** boundary or subsequent behavior of SQLite will be undefined. -** The minimum allocation size is capped at 2^12. Reasonable values -** for the minimum allocation size are 2^5 through 2^8. +** The minimum allocation size is capped at 2**12. Reasonable values +** for the minimum allocation size are 2**5 through 2**8. ** ** [[SQLITE_CONFIG_MUTEX]]
SQLITE_CONFIG_MUTEX
**
^(This option takes a single argument which is a pointer to an @@ -3348,7 +3346,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** that the supplied string is nul-terminated, then there is a small ** performance advantage to be gained by passing an nByte parameter that ** is equal to the number of bytes in the input string including -** the nul-terminator bytes. +** the nul-terminator bytes as this saves SQLite from having to +** make a copy of the input string. ** ** ^If pzTail is not NULL then *pzTail is made to point to the first byte ** past the end of the first SQL statement in zSql. These routines only @@ -3399,7 +3398,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ^The specific value of WHERE-clause [parameter] might influence the ** choice of query plan if the parameter is the left-hand side of a [LIKE] ** or [GLOB] operator or if the parameter is compared to an indexed column -** and the [SQLITE_ENABLE_STAT2] compile-time option is enabled. +** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled. ** the ** ** @@ -3569,6 +3568,13 @@ typedef struct sqlite3_context sqlite3_context; ** number of bytes in the value, not the number of characters.)^ ** ^If the fourth parameter is negative, the length of the string is ** the number of bytes up to the first zero terminator. +** If a non-negative fourth parameter is provided to sqlite3_bind_text() +** or sqlite3_bind_text16() then that parameter must be the byte offset +** where the NUL terminator would occur assuming the string were NUL +** terminated. If any NUL characters occur at byte offsets less than +** the value of the fourth parameter then the resulting string value will +** contain embedded NULs. The result of expressions involving strings +** with embedded NULs is undefined. ** ** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and ** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or @@ -3902,6 +3908,12 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); ** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of ** interfaces) then sqlite3_data_count(P) returns 0. ** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** ^The sqlite3_data_count(P) routine returns 0 if the previous call to +** [sqlite3_step](P) returned [SQLITE_DONE]. ^The sqlite3_data_count(P) +** will return non-zero if previous call to [sqlite3_step](P) returned +** [SQLITE_ROW], except in the case of the [PRAGMA incremental_vacuum] +** where it always returns zero since each step of that multi-step +** pragma returns 0 columns of data. ** ** See also: [sqlite3_column_count()] */ @@ -4581,7 +4593,12 @@ typedef void (*sqlite3_destructor_type)(void*); ** ^If the 3rd parameter to the sqlite3_result_text* interfaces ** is non-negative, then as many bytes (not characters) of the text ** pointed to by the 2nd parameter are taken as the application-defined -** function result. +** function result. If the 3rd parameter is non-negative, then it +** must be the byte offset into the string where the NUL terminator would +** appear if the string where NUL terminated. If any NUL characters occur +** in the string at a byte offset that is less than the value of the 3rd +** parameter, then the resulting string will contain embedded NULs and the +** result of expressions operating on strings with embedded NULs is undefined. ** ^If the 4th parameter to the sqlite3_result_text* interfaces ** or sqlite3_result_blob is a non-NULL pointer, then SQLite calls that ** function as the destructor on the text or BLOB result when it has @@ -6364,6 +6381,18 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. **
+** +** [[SQLITE_DBSTATUS_CACHE_HIT]] ^(
SQLITE_DBSTATUS_CACHE_HIT
+**
This parameter returns the number of pager cache hits that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT +** is always 0. +**
+** +** [[SQLITE_DBSTATUS_CACHE_MISS]] ^(
SQLITE_DBSTATUS_CACHE_MISS
+**
This parameter returns the number of pager cache misses that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS +** is always 0. +**
** */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 @@ -6373,7 +6402,9 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r #define SQLITE_DBSTATUS_LOOKASIDE_HIT 4 #define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5 #define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 -#define SQLITE_DBSTATUS_MAX 6 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_CACHE_HIT 7 +#define SQLITE_DBSTATUS_CACHE_MISS 8 +#define SQLITE_DBSTATUS_MAX 8 /* Largest defined DBSTATUS */ /* @@ -6427,7 +6458,6 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** A non-zero value in this counter may indicate an opportunity to ** improvement performance by adding permanent indices that do not ** need to be reinitialized each time the statement is run. -** ** */ #define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 @@ -7713,6 +7743,18 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ */ #define SQLITE_MAX_U32 ((((u64)1)<<32)-1) +/* +** The datatype used to store estimates of the number of rows in a +** table or index. This is an unsigned integer type. For 99.9% of +** the world, a 32-bit integer is sufficient. But a 64-bit integer +** can be used at compile-time if desired. +*/ +#ifdef SQLITE_64BIT_STATS + typedef u64 tRowcnt; /* 64-bit only if requested at compile-time */ +#else + typedef u32 tRowcnt; /* 32-bit is the default */ +#endif + /* ** Macros to determine whether the machine is big or little endian, ** evaluated at runtime. @@ -8744,6 +8786,8 @@ SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager*); SQLITE_PRIVATE int sqlite3PagerNosync(Pager*); SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager*); SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager*); +SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, int *); +SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *); /* Functions used to truncate the database file. */ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno); @@ -9280,14 +9324,17 @@ SQLITE_PRIVATE int sqlite3OsCloseFree(sqlite3_file *); */ #define sqlite3_mutex_alloc(X) ((sqlite3_mutex*)8) #define sqlite3_mutex_free(X) -#define sqlite3_mutex_enter(X) +#define sqlite3_mutex_enter(X) #define sqlite3_mutex_try(X) SQLITE_OK -#define sqlite3_mutex_leave(X) +#define sqlite3_mutex_leave(X) #define sqlite3_mutex_held(X) ((void)(X),1) #define sqlite3_mutex_notheld(X) ((void)(X),1) #define sqlite3MutexAlloc(X) ((sqlite3_mutex*)8) #define sqlite3MutexInit() SQLITE_OK #define sqlite3MutexEnd() +#define MUTEX_LOGIC(X) +#else +#define MUTEX_LOGIC(X) X #endif /* defined(SQLITE_MUTEX_OMIT) */ /************** End of mutex.h ***********************************************/ @@ -9920,7 +9967,7 @@ struct Table { Column *aCol; /* Information about each column */ Index *pIndex; /* List of SQL indexes on this table. */ int tnum; /* Root BTree node for this table (see note above) */ - unsigned nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ + tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ Select *pSelect; /* NULL for tables. Points to definition if a view. */ u16 nRef; /* Number of pointers to this Table */ u8 tabFlags; /* Mask of TF_* values */ @@ -10119,7 +10166,7 @@ struct Index { char *zName; /* Name of this index */ int nColumn; /* Number of columns in the table used by this index */ int *aiColumn; /* Which columns are used by this index. 1st is 0 */ - unsigned *aiRowEst; /* Result of ANALYZE: Est. rows selected by each column */ + tRowcnt *aiRowEst; /* Result of ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ int tnum; /* Page containing root of this index in database file */ u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ @@ -10130,20 +10177,29 @@ struct Index { Schema *pSchema; /* Schema containing this index */ u8 *aSortOrder; /* Array of size Index.nColumn. True==DESC, False==ASC */ char **azColl; /* Array of collation sequence names for index */ - IndexSample *aSample; /* Array of SQLITE_INDEX_SAMPLES samples */ +#ifdef SQLITE_ENABLE_STAT3 + int nSample; /* Number of elements in aSample[] */ + tRowcnt avgEq; /* Average nEq value for key values not in aSample */ + IndexSample *aSample; /* Samples of the left-most key */ +#endif }; /* -** Each sample stored in the sqlite_stat2 table is represented in memory -** using a structure of this type. +** Each sample stored in the sqlite_stat3 table is represented in memory +** using a structure of this type. See documentation at the top of the +** analyze.c source file for additional information. */ struct IndexSample { union { char *z; /* Value if eType is SQLITE_TEXT or SQLITE_BLOB */ - double r; /* Value if eType is SQLITE_FLOAT or SQLITE_INTEGER */ + double r; /* Value if eType is SQLITE_FLOAT */ + i64 i; /* Value if eType is SQLITE_INTEGER */ } u; u8 eType; /* SQLITE_NULL, SQLITE_INTEGER ... etc. */ - u8 nByte; /* Size in byte of text or blob. */ + int nByte; /* Size in byte of text or blob. */ + tRowcnt nEq; /* Est. number of rows where the key equals this sample */ + tRowcnt nLt; /* Est. number of rows where key is less than this sample */ + tRowcnt nDLt; /* Est. number of distinct keys less than this sample */ }; /* @@ -10595,10 +10651,10 @@ struct WhereLevel { #define WHERE_ORDERBY_MAX 0x0002 /* ORDER BY processing for max() func */ #define WHERE_ONEPASS_DESIRED 0x0004 /* Want to do one-pass UPDATE/DELETE */ #define WHERE_DUPLICATES_OK 0x0008 /* Ok to return a row more than once */ -#define WHERE_OMIT_OPEN 0x0010 /* Table cursors are already open */ -#define WHERE_OMIT_CLOSE 0x0020 /* Omit close of table & index cursors */ -#define WHERE_FORCE_TABLE 0x0040 /* Do not use an index-only search */ -#define WHERE_ONETABLE_ONLY 0x0080 /* Only code the 1st table in pTabList */ +#define WHERE_OMIT_OPEN_CLOSE 0x0010 /* Table cursors are already open */ +#define WHERE_FORCE_TABLE 0x0020 /* Do not use an index-only search */ +#define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ +#define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ /* ** The WHERE clause processing routine has two halves. The @@ -11352,6 +11408,7 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse*,Table*); #endif SQLITE_PRIVATE void sqlite3DropTable(Parse*, SrcList*, int, int); +SQLITE_PRIVATE void sqlite3CodeDropTable(Parse*, Table*, int, int); SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3*, Table*); #ifndef SQLITE_OMIT_AUTOINCREMENT SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse); @@ -11608,7 +11665,7 @@ SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value*); SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *); SQLITE_PRIVATE char *sqlite3Utf16to8(sqlite3 *, const void*, int, u8); -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3 SQLITE_PRIVATE char *sqlite3Utf8to16(sqlite3 *, u8, char *, int, int *); #endif SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value **); @@ -11710,6 +11767,7 @@ SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char*); # define sqlite3VtabUnlock(X) # define sqlite3VtabUnlockList(X) # define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK +# define sqlite3GetVTable(X,Y) ((VTable*)0) #else SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table*); SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, char **); @@ -11719,6 +11777,7 @@ SQLITE_PRIVATE void sqlite3VtabLock(VTable *); SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *); SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3*); SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *, int, int); +SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3*, Table*); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); @@ -11738,7 +11797,6 @@ SQLITE_PRIVATE int sqlite3Reprepare(Vdbe*); SQLITE_PRIVATE void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *); SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3*); -SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3*, Table*); SQLITE_PRIVATE const char *sqlite3JournalModename(int); SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); SQLITE_PRIVATE int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); @@ -12232,8 +12290,8 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_ENABLE_RTREE "ENABLE_RTREE", #endif -#ifdef SQLITE_ENABLE_STAT2 - "ENABLE_STAT2", +#ifdef SQLITE_ENABLE_STAT3 + "ENABLE_STAT3", #endif #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY "ENABLE_UNLOCK_NOTIFY", @@ -12447,9 +12505,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_XFER_OPT "OMIT_XFER_OPT", #endif -#ifdef SQLITE_PAGECACHE_BLOCKALLOC - "PAGECACHE_BLOCKALLOC", -#endif #ifdef SQLITE_PERFORMANCE_TRACE "PERFORMANCE_TRACE", #endif @@ -12935,6 +12990,7 @@ SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem); +SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p); #ifdef SQLITE_OMIT_MERGE_SORT # define sqlite3VdbeSorterInit(Y,Z) SQLITE_OK @@ -13192,6 +13248,28 @@ SQLITE_API int sqlite3_db_status( break; } + /* + ** Set *pCurrent to the total cache hits or misses encountered by all + ** pagers the database handle is connected to. *pHighwater is always set + ** to zero. + */ + case SQLITE_DBSTATUS_CACHE_HIT: + case SQLITE_DBSTATUS_CACHE_MISS: { + int i; + int nRet = 0; + assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 ); + + for(i=0; inDb; i++){ + if( db->aDb[i].pBt ){ + Pager *pPager = sqlite3BtreePager(db->aDb[i].pBt); + sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet); + } + } + *pHighwater = 0; + *pCurrent = nRet; + break; + } + default: { rc = SQLITE_ERROR; } @@ -13492,12 +13570,18 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ } /* -** Set the time to the current time reported by the VFS +** Set the time to the current time reported by the VFS. +** +** Return the number of errors. */ -static void setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ +static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ sqlite3 *db = sqlite3_context_db_handle(context); - sqlite3OsCurrentTimeInt64(db->pVfs, &p->iJD); - p->validJD = 1; + if( sqlite3OsCurrentTimeInt64(db->pVfs, &p->iJD)==SQLITE_OK ){ + p->validJD = 1; + return 0; + }else{ + return 1; + } } /* @@ -13527,8 +13611,7 @@ static int parseDateOrTime( }else if( parseHhMmSs(zDate, p)==0 ){ return 0; }else if( sqlite3StrICmp(zDate,"now")==0){ - setDateTimeToCurrent(context, p); - return 0; + return setDateTimeToCurrent(context, p); }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){ p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); p->validJD = 1; @@ -13955,8 +14038,9 @@ static int isDate( int eType; memset(p, 0, sizeof(*p)); if( argc==0 ){ - setDateTimeToCurrent(context, p); - }else if( (eType = sqlite3_value_type(argv[0]))==SQLITE_FLOAT + return setDateTimeToCurrent(context, p); + } + if( (eType = sqlite3_value_type(argv[0]))==SQLITE_FLOAT || eType==SQLITE_INTEGER ){ p->iJD = (sqlite3_int64)(sqlite3_value_double(argv[0])*86400000.0 + 0.5); p->validJD = 1; @@ -14268,31 +14352,28 @@ static void currentTimeFunc( char *zFormat = (char *)sqlite3_user_data(context); sqlite3 *db; sqlite3_int64 iT; + struct tm *pTm; + struct tm sNow; char zBuf[20]; UNUSED_PARAMETER(argc); UNUSED_PARAMETER(argv); db = sqlite3_context_db_handle(context); - sqlite3OsCurrentTimeInt64(db->pVfs, &iT); + if( sqlite3OsCurrentTimeInt64(db->pVfs, &iT) ) return; t = iT/1000 - 10000*(sqlite3_int64)21086676; #ifdef HAVE_GMTIME_R - { - struct tm sNow; - gmtime_r(&t, &sNow); - strftime(zBuf, 20, zFormat, &sNow); - } + pTm = gmtime_r(&t, &sNow); #else - { - struct tm *pTm; - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - pTm = gmtime(&t); - strftime(zBuf, 20, zFormat, pTm); - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - } + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + pTm = gmtime(&t); + if( pTm ) memcpy(&sNow, pTm, sizeof(sNow)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); #endif - - sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + if( pTm ){ + strftime(zBuf, 20, zFormat, &sNow); + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + } } #endif @@ -14627,12 +14708,12 @@ static void vfsUnlink(sqlite3_vfs *pVfs){ ** true. */ SQLITE_API int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ - sqlite3_mutex *mutex = 0; + MUTEX_LOGIC(sqlite3_mutex *mutex;) #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); if( rc ) return rc; #endif - mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(mutex); vfsUnlink(pVfs); if( makeDflt || vfsList==0 ){ @@ -18880,48 +18961,10 @@ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ ** ************************************************************************** ** -** The following modules is an enhanced replacement for the "printf" subroutines -** found in the standard C library. The following enhancements are -** supported: -** -** + Additional functions. The standard set of "printf" functions -** includes printf, fprintf, sprintf, vprintf, vfprintf, and -** vsprintf. This module adds the following: -** -** * snprintf -- Works like sprintf, but has an extra argument -** which is the size of the buffer written to. -** -** * mprintf -- Similar to sprintf. Writes output to memory -** obtained from malloc. -** -** * xprintf -- Calls a function to dispose of output. -** -** * nprintf -- No output, but returns the number of characters -** that would have been output by printf. -** -** * A v- version (ex: vsnprintf) of every function is also -** supplied. -** -** + A few extensions to the formatting notation are supported: -** -** * The "=" flag (similar to "-") causes the output to be -** be centered in the appropriately sized field. -** -** * The %b field outputs an integer in binary notation. -** -** * The %c field now accepts a precision. The character output -** is repeated by the number of times the precision specifies. -** -** * The %' field works like %c, but takes as its character the -** next character of the format string, instead of the next -** argument. For example, printf("%.78'-") prints 78 minus -** signs, the same as printf("%.78c",'-'). -** -** + When compiled using GCC on a SPARC, this version of printf is -** faster than the library printf for SUN OS 4.1. -** -** + All functions are fully reentrant. -** +** This file contains code for a set of "printf"-like routines. These +** routines format strings much like the printf() from the standard C +** library, though the implementation here has enhancements to support +** SQLlite. */ /* @@ -19059,43 +19102,15 @@ static void appendSpace(StrAccum *pAccum, int N){ /* ** On machines with a small stack size, you can redefine the -** SQLITE_PRINT_BUF_SIZE to be less than 350. +** SQLITE_PRINT_BUF_SIZE to be something smaller, if desired. */ #ifndef SQLITE_PRINT_BUF_SIZE -# if defined(SQLITE_SMALL_STACK) -# define SQLITE_PRINT_BUF_SIZE 50 -# else -# define SQLITE_PRINT_BUF_SIZE 350 -# endif +# define SQLITE_PRINT_BUF_SIZE 70 #endif #define etBUFSIZE SQLITE_PRINT_BUF_SIZE /* Size of the output buffer */ /* -** The root program. All variations call this core. -** -** INPUTS: -** func This is a pointer to a function taking three arguments -** 1. A pointer to anything. Same as the "arg" parameter. -** 2. A pointer to the list of characters to be output -** (Note, this list is NOT null terminated.) -** 3. An integer number of characters to be output. -** (Note: This number might be zero.) -** -** arg This is the pointer to anything which will be passed as the -** first argument to "func". Use it for whatever you like. -** -** fmt This is the format string, as in the usual print. -** -** ap This is a pointer to a list of arguments. Same as in -** vfprint. -** -** OUTPUTS: -** The return value is the total number of characters sent to -** the function "func". Returns -1 on a error. -** -** Note that the order in which automatic variables are declared below -** seems to make a big difference in determining how fast this beast -** will run. +** Render a string given by "fmt" into the StrAccum object. */ SQLITE_PRIVATE void sqlite3VXPrintf( StrAccum *pAccum, /* Accumulate results here */ @@ -19118,23 +19133,23 @@ SQLITE_PRIVATE void sqlite3VXPrintf( etByte flag_long; /* True if "l" flag is present */ etByte flag_longlong; /* True if the "ll" flag is present */ etByte done; /* Loop termination flag */ + etByte xtype = 0; /* Conversion paradigm */ + char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ sqlite_uint64 longvalue; /* Value for integer types */ LONGDOUBLE_TYPE realvalue; /* Value for real types */ const et_info *infop; /* Pointer to the appropriate info structure */ - char buf[etBUFSIZE]; /* Conversion buffer */ - char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ - etByte xtype = 0; /* Conversion paradigm */ - char *zExtra; /* Extra memory used for etTCLESCAPE conversions */ + char *zOut; /* Rendering buffer */ + int nOut; /* Size of the rendering buffer */ + char *zExtra; /* Malloced memory used by some conversion */ #ifndef SQLITE_OMIT_FLOATING_POINT int exp, e2; /* exponent of real numbers */ + int nsd; /* Number of significant digits returned */ double rounder; /* Used for rounding floating point values */ etByte flag_dp; /* True if decimal point should be shown */ etByte flag_rtz; /* True if trailing zeros should be removed */ - etByte flag_exp; /* True to force display of the exponent */ - int nsd; /* Number of significant digits returned */ #endif + char buf[etBUFSIZE]; /* Conversion buffer */ - length = 0; bufpt = 0; for(; (c=(*fmt))!=0; ++fmt){ if( c!='%' ){ @@ -19179,9 +19194,6 @@ SQLITE_PRIVATE void sqlite3VXPrintf( c = *++fmt; } } - if( width > etBUFSIZE-10 ){ - width = etBUFSIZE-10; - } /* Get the precision */ if( c=='.' ){ precision = 0; @@ -19228,12 +19240,6 @@ SQLITE_PRIVATE void sqlite3VXPrintf( } zExtra = 0; - - /* Limit the precision to prevent overflowing buf[] during conversion */ - if( precision>etBUFSIZE-40 && (infop->flags & FLAG_STRING)==0 ){ - precision = etBUFSIZE-40; - } - /* ** At this point, variables are initialized as follows: ** @@ -19298,16 +19304,26 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( flag_zeropad && precisionmallocFailed = 1; + return; + } + } + bufpt = &zOut[nOut-1]; if( xtype==etORDINAL ){ static const char zOrd[] = "thstndrd"; int x = (int)(longvalue % 10); if( x>=4 || (longvalue/10)%10==1 ){ x = 0; } - buf[etBUFSIZE-3] = zOrd[x*2]; - buf[etBUFSIZE-2] = zOrd[x*2+1]; - bufpt -= 2; + *(--bufpt) = zOrd[x*2+1]; + *(--bufpt) = zOrd[x*2]; } { register const char *cset; /* Use registers for speed */ @@ -19319,7 +19335,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( longvalue = longvalue/base; }while( longvalue>0 ); } - length = (int)(&buf[etBUFSIZE-1]-bufpt); + length = (int)(&zOut[nOut-1]-bufpt); for(idx=precision-length; idx>0; idx--){ *(--bufpt) = '0'; /* Zero pad */ } @@ -19330,7 +19346,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( pre = &aPrefix[infop->prefix]; for(; (x=(*pre))!=0; pre++) *(--bufpt) = x; } - length = (int)(&buf[etBUFSIZE-1]-bufpt); + length = (int)(&zOut[nOut-1]-bufpt); break; case etFLOAT: case etEXP: @@ -19340,7 +19356,6 @@ SQLITE_PRIVATE void sqlite3VXPrintf( length = 0; #else if( precision<0 ) precision = 6; /* Set default precision */ - if( precision>etBUFSIZE/2-10 ) precision = etBUFSIZE/2-10; if( realvalue<0.0 ){ realvalue = -realvalue; prefix = '-'; @@ -19388,7 +19403,6 @@ SQLITE_PRIVATE void sqlite3VXPrintf( ** If the field type is etGENERIC, then convert to either etEXP ** or etFLOAT, as appropriate. */ - flag_exp = xtype==etEXP; if( xtype!=etFLOAT ){ realvalue += rounder; if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } @@ -19409,6 +19423,14 @@ SQLITE_PRIVATE void sqlite3VXPrintf( }else{ e2 = exp; } + if( e2+precision+width > etBUFSIZE - 15 ){ + bufpt = zExtra = sqlite3Malloc( e2+precision+width+15 ); + if( bufpt==0 ){ + pAccum->mallocFailed = 1; + return; + } + } + zOut = bufpt; nsd = 0; flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; /* The sign in front of the number */ @@ -19440,7 +19462,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( /* Remove trailing zeros and the "." if no digits follow the "." */ if( flag_rtz && flag_dp ){ while( bufpt[-1]=='0' ) *(--bufpt) = 0; - assert( bufpt>buf ); + assert( bufpt>zOut ); if( bufpt[-1]=='.' ){ if( flag_altform2 ){ *(bufpt++) = '0'; @@ -19450,7 +19472,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( } } /* Add the "eNNN" suffix */ - if( flag_exp || xtype==etEXP ){ + if( xtype==etEXP ){ *(bufpt++) = aDigits[infop->charset]; if( exp<0 ){ *(bufpt++) = '-'; exp = -exp; @@ -19469,8 +19491,8 @@ SQLITE_PRIVATE void sqlite3VXPrintf( /* The converted number is in buf[] and zero terminated. Output it. ** Note that the number is in the usual order, not reversed as with ** integer conversions. */ - length = (int)(bufpt-buf); - bufpt = buf; + length = (int)(bufpt-zOut); + bufpt = zOut; /* Special case: Add leading zeros if the flag_zeropad flag is ** set and we are not left justified */ @@ -19608,9 +19630,7 @@ SQLITE_PRIVATE void sqlite3VXPrintf( appendSpace(pAccum, nspace); } } - if( zExtra ){ - sqlite3_free(zExtra); - } + sqlite3_free(zExtra); }/* End for loop over the format string */ } /* End of function */ @@ -19624,6 +19644,7 @@ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ testcase(p->mallocFailed); return; } + assert( p->zText!=0 || p->nChar==0 ); if( N<0 ){ N = sqlite3Strlen30(z); } @@ -19655,7 +19676,7 @@ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ zNew = sqlite3_realloc(zOld, p->nAlloc); } if( zNew ){ - if( zOld==0 ) memcpy(zNew, p->zText, p->nChar); + if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); p->zText = zNew; }else{ p->mallocFailed = 1; @@ -19664,6 +19685,7 @@ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ } } } + assert( p->zText ); memcpy(&p->zText[p->nChar], z, N); p->nChar += N; } @@ -20513,7 +20535,7 @@ SQLITE_PRIVATE char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte, u8 e ** If a malloc failure occurs, NULL is returned and the db.mallocFailed ** flag set. */ -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3 SQLITE_PRIVATE char *sqlite3Utf8to16(sqlite3 *db, u8 enc, char *z, int n, int *pnOut){ Mem m; memset(&m, 0, sizeof(m)); @@ -20942,7 +20964,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en } /* copy digits to exponent */ while( z=342 ){ + if( esign<0 ){ + result = 0.0*s; + }else{ + result = 1e308*1e308*s; /* Infinity */ + } }else{ /* 1.0e+22 is the largest power of 10 than can be ** represented exactly. */ @@ -25104,7 +25132,7 @@ static int unixMutexHeld(void) { #endif -#ifdef SQLITE_DEBUG +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) /* ** Helper function for printing out trace information from debugging ** binaries. This returns the string represetation of the supplied @@ -25939,14 +25967,14 @@ static int unixLock(sqlite3_file *id, int eFileLock){ */ int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; - unixInodeInfo *pInode = pFile->pInode; + unixInodeInfo *pInode; struct flock lock; int tErrno = 0; assert( pFile ); OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (unix)\n", pFile->h, azFileLock(eFileLock), azFileLock(pFile->eFileLock), - azFileLock(pInode->eFileLock), pInode->nShared , getpid())); + azFileLock(pFile->pInode->eFileLock), pFile->pInode->nShared , getpid())); /* If there is already a lock of this type or more restrictive on the ** unixFile, do nothing. Don't use the end_lock: exit path, as @@ -26150,7 +26178,6 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ unixInodeInfo *pInode; struct flock lock; int rc = SQLITE_OK; - int h; assert( pFile ); OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (unix)\n", pFile->h, eFileLock, @@ -26162,14 +26189,10 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ return SQLITE_OK; } unixEnterMutex(); - h = pFile->h; pInode = pFile->pInode; assert( pInode->nShared!=0 ); if( pFile->eFileLock>SHARED_LOCK ){ assert( pInode->eFileLock==pFile->eFileLock ); - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); #ifndef NDEBUG /* When reducing a lock such that other processes can start @@ -26180,11 +26203,6 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ ** the file has changed and hence might not know to flush their ** cache. The use of a stale cache can lead to database corruption. */ -#if 0 - assert( pFile->inNormalWrite==0 - || pFile->dbUpdate==0 - || pFile->transCntrChng==1 ); -#endif pFile->inNormalWrite = 0; #endif @@ -26286,9 +26304,6 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = lock.l_len = 0L; - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); if( unixFileLock(pFile, &lock)==0 ){ pInode->eFileLock = NO_LOCK; }else{ @@ -28430,16 +28445,15 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } if( pInode->bProcessLock==0 ){ - pShmNode->h = robust_open(zShmFilename, O_RDWR|O_CREAT, - (sStat.st_mode & 0777)); + const char *zRO; + int openFlags = O_RDWR | O_CREAT; + zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); + if( zRO && sqlite3GetBoolean(zRO) ){ + openFlags = O_RDONLY; + pShmNode->isReadonly = 1; + } + pShmNode->h = robust_open(zShmFilename, openFlags, (sStat.st_mode&0777)); if( pShmNode->h<0 ){ - const char *zRO; - zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); - if( zRO && sqlite3GetBoolean(zRO) ){ - pShmNode->h = robust_open(zShmFilename, O_RDONLY, - (sStat.st_mode & 0777)); - pShmNode->isReadonly = 1; - } if( pShmNode->h<0 ){ rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); goto shm_open_err; @@ -29125,6 +29139,9 @@ static int fillInUnixFile( assert( zFilename==0 || zFilename[0]=='/' ); #endif + /* No locking occurs in temporary files */ + assert( zFilename!=0 || noLock ); + OSTRACE(("OPEN %-3d %s\n", h, zFilename)); pNew->h = h; pNew->zPath = zFilename; @@ -29226,6 +29243,7 @@ static int fillInUnixFile( */ char *zLockFile; int nFilename; + assert( zFilename!=0 ); nFilename = (int)strlen(zFilename) + 6; zLockFile = (char *)sqlite3_malloc(nFilename); if( zLockFile==0 ){ @@ -29460,12 +29478,20 @@ static int findCreateFileMode( ** "-journalNN" ** "-walNN" ** - ** where NN is a 4 digit decimal number. The NN naming schemes are + ** where NN is a decimal number. The NN naming schemes are ** used by the test_multiplex.c module. */ nDb = sqlite3Strlen30(zPath) - 1; - while( nDb>0 && zPath[nDb]!='-' ) nDb--; - if( nDb==0 ) return SQLITE_OK; +#ifdef SQLITE_ENABLE_8_3_NAMES + while( nDb>0 && !sqlite3Isalnum(zPath[nDb]) ) nDb--; + if( nDb==0 || zPath[nDb]!='-' ) return SQLITE_OK; +#else + while( zPath[nDb]!='-' ){ + assert( nDb>0 ); + assert( zPath[nDb]!='\n' ); + nDb--; + } +#endif memcpy(zDb, zPath, nDb); zDb[nDb] = '\0'; @@ -29997,10 +30023,12 @@ SQLITE_API int sqlite3_current_time = 0; /* Fake system time in seconds since 1 ** epoch of noon in Greenwich on November 24, 4714 B.C according to the ** proleptic Gregorian calendar. ** -** On success, return 0. Return 1 if the time and date cannot be found. +** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date +** cannot be found. */ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; + int rc = SQLITE_OK; #if defined(NO_GETTOD) time_t t; time(&t); @@ -30011,8 +30039,11 @@ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_nsec/1000000; #else struct timeval sNow; - gettimeofday(&sNow, 0); - *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; + if( gettimeofday(&sNow, 0)==0 ){ + *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; + }else{ + rc = SQLITE_ERROR; + } #endif #ifdef SQLITE_TEST @@ -30021,7 +30052,7 @@ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ } #endif UNUSED_PARAMETER(NotUsed); - return 0; + return rc; } /* @@ -30030,11 +30061,12 @@ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ ** return 0. Return 1 if the time and date cannot be found. */ static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ - sqlite3_int64 i; + sqlite3_int64 i = 0; + int rc; UNUSED_PARAMETER(NotUsed); - unixCurrentTimeInt64(0, &i); + rc = unixCurrentTimeInt64(0, &i); *prNow = i/86400000.0; - return 0; + return rc; } /* @@ -34171,7 +34203,7 @@ static int winOpen( pFile->lastErrno = GetLastError(); winLogError(SQLITE_CANTOPEN, "winOpen", zUtf8Name); free(zConverted); - if( isReadWrite ){ + if( isReadWrite && !isExclusive ){ return winOpen(pVfs, zName, id, ((flags|SQLITE_OPEN_READONLY)&~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); }else{ @@ -34537,7 +34569,7 @@ static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ UNUSED_PARAMETER(pVfs); getLastErrorMsg(nBuf, zBufOut); } -void (*winDlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol))(void){ +static void (*winDlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol))(void){ UNUSED_PARAMETER(pVfs); #if SQLITE_OS_WINCE /* The GetProcAddressA() routine is only available on wince. */ @@ -34548,7 +34580,7 @@ void (*winDlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol))(void){ return (void(*)(void))GetProcAddress((HANDLE)pHandle, zSymbol); #endif } -void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ +static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ UNUSED_PARAMETER(pVfs); FreeLibrary((HANDLE)pHandle); } @@ -34622,7 +34654,8 @@ SQLITE_API int sqlite3_current_time = 0; /* Fake system time in seconds since 1 ** epoch of noon in Greenwich on November 24, 4714 B.C according to the ** proleptic Gregorian calendar. ** -** On success, return 0. Return 1 if the time and date cannot be found. +** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date +** cannot be found. */ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ /* FILETIME structure is a 64-bit value representing the number of @@ -34642,7 +34675,7 @@ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ GetSystemTime(&time); /* if SystemTimeToFileTime() fails, it returns zero. */ if (!SystemTimeToFileTime(&time,&ft)){ - return 1; + return SQLITE_ERROR; } #else GetSystemTimeAsFileTime( &ft ); @@ -34658,7 +34691,7 @@ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ } #endif UNUSED_PARAMETER(pVfs); - return 0; + return SQLITE_OK; } /* @@ -34666,7 +34699,7 @@ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ ** current time and date as a Julian Day number into *prNow and ** return 0. Return 1 if the time and date cannot be found. */ -int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ +static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ int rc; sqlite3_int64 i; rc = winCurrentTimeInt64(pVfs, &i); @@ -35791,8 +35824,6 @@ typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; -typedef struct PGroupBlock PGroupBlock; -typedef struct PGroupBlockList PGroupBlockList; /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set ** of one or more PCaches that are able to recycle each others unpinned @@ -35823,66 +35854,8 @@ struct PGroup { int mxPinned; /* nMaxpage + 10 - nMinPage */ int nCurrentPage; /* Number of purgeable pages allocated */ PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ -#ifdef SQLITE_PAGECACHE_BLOCKALLOC - int isBusy; /* Do not run ReleaseMemory() if true */ - PGroupBlockList *pBlockList; /* List of block-lists for this group */ -#endif }; -/* -** If SQLITE_PAGECACHE_BLOCKALLOC is defined when the library is built, -** each PGroup structure has a linked list of the the following starting -** at PGroup.pBlockList. There is one entry for each distinct page-size -** currently used by members of the PGroup (i.e. 1024 bytes, 4096 bytes -** etc.). Variable PGroupBlockList.nByte is set to the actual allocation -** size requested by each pcache, which is the database page-size plus -** the various header structures used by the pcache, pager and btree layers. -** Usually around (pgsz+200) bytes. -** -** This size (pgsz+200) bytes is not allocated efficiently by some -** implementations of malloc. In particular, some implementations are only -** able to allocate blocks of memory chunks of 2^N bytes, where N is some -** integer value. Since the page-size is a power of 2, this means we -** end up wasting (pgsz-200) bytes in each allocation. -** -** If SQLITE_PAGECACHE_BLOCKALLOC is defined, the (pgsz+200) byte blocks -** are not allocated directly. Instead, blocks of roughly M*(pgsz+200) bytes -** are requested from malloc allocator. After a block is returned, -** sqlite3MallocSize() is used to determine how many (pgsz+200) byte -** allocations can fit in the space returned by malloc(). This value may -** be more than M. -** -** The blocks are stored in a doubly-linked list. Variable PGroupBlock.nEntry -** contains the number of allocations that will fit in the aData[] space. -** nEntry is limited to the number of bits in bitmask mUsed. If a slot -** within aData is in use, the corresponding bit in mUsed is set. Thus -** when (mUsed+1==(1 << nEntry)) the block is completely full. -** -** Each time a slot within a block is freed, the block is moved to the start -** of the linked-list. And if a block becomes completely full, then it is -** moved to the end of the list. As a result, when searching for a free -** slot, only the first block in the list need be examined. If it is full, -** then it is guaranteed that all blocks are full. -*/ -struct PGroupBlockList { - int nByte; /* Size of each allocation in bytes */ - PGroupBlock *pFirst; /* First PGroupBlock in list */ - PGroupBlock *pLast; /* Last PGroupBlock in list */ - PGroupBlockList *pNext; /* Next block-list attached to group */ -}; - -struct PGroupBlock { - Bitmask mUsed; /* Mask of used slots */ - int nEntry; /* Maximum number of allocations in aData[] */ - u8 *aData; /* Pointer to data block */ - PGroupBlock *pNext; /* Next PGroupBlock in list */ - PGroupBlock *pPrev; /* Previous PGroupBlock in list */ - PGroupBlockList *pList; /* Owner list */ -}; - -/* Minimum value for PGroupBlock.nEntry */ -#define PAGECACHE_BLOCKALLOC_MINENTRY 15 - /* Each page cache is an instance of the following object. Every ** open database file (including each in-memory database and each ** temporary or transient database) has a single page cache which @@ -35986,17 +35959,6 @@ static SQLITE_WSD struct PCacheGlobal { #define PGHDR1_TO_PAGE(p) (void*)(((char*)p) - p->pCache->szPage) #define PAGE_TO_PGHDR1(c, p) (PgHdr1*)(((char*)p) + c->szPage) -/* -** Blocks used by the SQLITE_PAGECACHE_BLOCKALLOC blocks to store/retrieve -** a PGroupBlock pointer based on a pointer to a page buffer. -*/ -#define PAGE_SET_BLOCKPTR(pCache, pPg, pBlock) \ - ( *(PGroupBlock **)&(((u8*)pPg)[sizeof(PgHdr1) + pCache->szPage]) = pBlock ) - -#define PAGE_GET_BLOCKPTR(pCache, pPg) \ - ( *(PGroupBlock **)&(((u8*)pPg)[sizeof(PgHdr1) + pCache->szPage]) ) - - /* ** Macros to enter and leave the PCache LRU mutex. */ @@ -36122,139 +36084,14 @@ static int pcache1MemSize(void *p){ } #endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT */ -#ifdef SQLITE_PAGECACHE_BLOCKALLOC -/* -** The block pBlock belongs to list pList but is not currently linked in. -** Insert it into the start of the list. -*/ -static void addBlockToList(PGroupBlockList *pList, PGroupBlock *pBlock){ - pBlock->pPrev = 0; - pBlock->pNext = pList->pFirst; - pList->pFirst = pBlock; - if( pBlock->pNext ){ - pBlock->pNext->pPrev = pBlock; - }else{ - assert( pList->pLast==0 ); - pList->pLast = pBlock; - } -} - -/* -** If there are no blocks in the list headed by pList, remove pList -** from the pGroup->pBlockList list and free it with sqlite3_free(). -*/ -static void freeListIfEmpty(PGroup *pGroup, PGroupBlockList *pList){ - assert( sqlite3_mutex_held(pGroup->mutex) ); - if( pList->pFirst==0 ){ - PGroupBlockList **pp; - for(pp=&pGroup->pBlockList; *pp!=pList; pp=&(*pp)->pNext); - *pp = (*pp)->pNext; - sqlite3_free(pList); - } -} -#endif /* SQLITE_PAGECACHE_BLOCKALLOC */ - /* ** Allocate a new page object initially associated with cache pCache. */ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ int nByte = sizeof(PgHdr1) + pCache->szPage; - void *pPg = 0; - PgHdr1 *p; + PgHdr1 *p = 0; + void *pPg; -#ifdef SQLITE_PAGECACHE_BLOCKALLOC - PGroup *pGroup = pCache->pGroup; - PGroupBlockList *pList; - PGroupBlock *pBlock; - int i; - - nByte += sizeof(PGroupBlockList *); - nByte = ROUND8(nByte); - - for(pList=pGroup->pBlockList; pList; pList=pList->pNext){ - if( pList->nByte==nByte ) break; - } - if( pList==0 ){ - PGroupBlockList *pNew; - assert( pGroup->isBusy==0 ); - assert( sqlite3_mutex_held(pGroup->mutex) ); - pGroup->isBusy = 1; /* Disable sqlite3PcacheReleaseMemory() */ - pNew = (PGroupBlockList *)sqlite3MallocZero(sizeof(PGroupBlockList)); - pGroup->isBusy = 0; /* Reenable sqlite3PcacheReleaseMemory() */ - if( pNew==0 ){ - /* malloc() failure. Return early. */ - return 0; - } -#ifdef SQLITE_DEBUG - for(pList=pGroup->pBlockList; pList; pList=pList->pNext){ - assert( pList->nByte!=nByte ); - } -#endif - pNew->nByte = nByte; - pNew->pNext = pGroup->pBlockList; - pGroup->pBlockList = pNew; - pList = pNew; - } - - pBlock = pList->pFirst; - if( pBlock==0 || pBlock->mUsed==(((Bitmask)1<nEntry)-1) ){ - int sz; - - /* Allocate a new block. Try to allocate enough space for the PGroupBlock - ** structure and MINENTRY allocations of nByte bytes each. If the - ** allocator returns more memory than requested, then more than MINENTRY - ** allocations may fit in it. */ - assert( sqlite3_mutex_held(pGroup->mutex) ); - pcache1LeaveMutex(pCache->pGroup); - sz = sizeof(PGroupBlock) + PAGECACHE_BLOCKALLOC_MINENTRY * nByte; - pBlock = (PGroupBlock *)sqlite3Malloc(sz); - pcache1EnterMutex(pCache->pGroup); - - if( !pBlock ){ - freeListIfEmpty(pGroup, pList); - return 0; - } - pBlock->nEntry = (sqlite3MallocSize(pBlock) - sizeof(PGroupBlock)) / nByte; - if( pBlock->nEntry>=BMS ){ - pBlock->nEntry = BMS-1; - } - pBlock->pList = pList; - pBlock->mUsed = 0; - pBlock->aData = (u8 *)&pBlock[1]; - addBlockToList(pList, pBlock); - - sz = sqlite3MallocSize(pBlock); - sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); - sqlite3_mutex_leave(pcache1.mutex); - } - - for(i=0; pPg==0 && ALWAYS(inEntry); i++){ - if( 0==(pBlock->mUsed & ((Bitmask)1<mUsed |= ((Bitmask)1<aData[pList->nByte * i]; - } - } - assert( pPg ); - PAGE_SET_BLOCKPTR(pCache, pPg, pBlock); - - /* If the block is now full, shift it to the end of the list */ - if( pBlock->mUsed==(((Bitmask)1<nEntry)-1) && pList->pLast!=pBlock ){ - assert( pList->pFirst==pBlock ); - assert( pBlock->pPrev==0 ); - assert( pList->pLast->pNext==0 ); - pList->pFirst = pBlock->pNext; - pList->pFirst->pPrev = 0; - pBlock->pPrev = pList->pLast; - pBlock->pNext = 0; - pList->pLast->pNext = pBlock; - pList->pLast = pBlock; - } - p = PAGE_TO_PGHDR1(pCache, pPg); - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage++; - } -#else /* The group mutex must be released before pcache1Alloc() is called. This ** is because it may call sqlite3_release_memory(), which assumes that ** this mutex is not held. */ @@ -36262,15 +36099,13 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ pcache1LeaveMutex(pCache->pGroup); pPg = pcache1Alloc(nByte); pcache1EnterMutex(pCache->pGroup); + if( pPg ){ p = PAGE_TO_PGHDR1(pCache, pPg); if( pCache->bPurgeable ){ pCache->pGroup->nCurrentPage++; } - }else{ - p = 0; } -#endif return p; } @@ -36284,49 +36119,8 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ static void pcache1FreePage(PgHdr1 *p){ if( ALWAYS(p) ){ PCache1 *pCache = p->pCache; - void *pPg = PGHDR1_TO_PAGE(p); - -#ifdef SQLITE_PAGECACHE_BLOCKALLOC - PGroupBlock *pBlock = PAGE_GET_BLOCKPTR(pCache, pPg); - PGroupBlockList *pList = pBlock->pList; - int i = ((u8 *)pPg - pBlock->aData) / pList->nByte; - - assert( pPg==(void *)&pBlock->aData[i*pList->nByte] ); - assert( pBlock->mUsed & ((Bitmask)1<mUsed &= ~((Bitmask)1<pFirst==pBlock ){ - pList->pFirst = pBlock->pNext; - if( pList->pFirst ) pList->pFirst->pPrev = 0; - }else{ - pBlock->pPrev->pNext = pBlock->pNext; - } - if( pList->pLast==pBlock ){ - pList->pLast = pBlock->pPrev; - if( pList->pLast ) pList->pLast->pNext = 0; - }else{ - pBlock->pNext->pPrev = pBlock->pPrev; - } - - if( pBlock->mUsed==0 ){ - PGroup *pGroup = p->pCache->pGroup; - - int sz = sqlite3MallocSize(pBlock); - sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -sz); - sqlite3_mutex_leave(pcache1.mutex); - freeListIfEmpty(pGroup, pList); - sqlite3_free(pBlock); - }else{ - addBlockToList(pList, pBlock); - } -#else assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); - pcache1Free(pPg); -#endif + pcache1Free(PGHDR1_TO_PAGE(p)); if( pCache->bPurgeable ){ pCache->pGroup->nCurrentPage--; } @@ -36937,9 +36731,6 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void){ */ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ int nFree = 0; -#ifdef SQLITE_PAGECACHE_BLOCKALLOC - if( pcache1.grp.isBusy ) return 0; -#endif assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); assert( sqlite3_mutex_notheld(pcache1.mutex) ); if( pcache1.pStart==0 ){ @@ -38202,8 +37993,8 @@ struct Pager { char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ void *pBusyHandlerArg; /* Context argument for xBusyHandler */ + int nHit, nMiss; /* Total cache hits and misses */ #ifdef SQLITE_TEST - int nHit, nMiss; /* Cache hits and missing */ int nRead, nWrite; /* Database pages read/written */ #endif void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ @@ -40235,7 +40026,6 @@ static int pager_playback(Pager *pPager, int isHot){ rc = pager_playback_one_page(pPager,&pPager->journalOff,0,1,0); if( rc!=SQLITE_OK ){ if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; pPager->journalOff = szJ; break; }else if( rc==SQLITE_IOERR_SHORT_READ ){ @@ -40497,6 +40287,7 @@ static int pagerWalFrames( #endif assert( pPager->pWal ); + assert( pList ); #ifdef SQLITE_DEBUG /* Verify that the page list is in accending order */ for(p=pList; p && p->pDirty; p=p->pDirty){ @@ -41701,7 +41492,7 @@ static int pagerStress(void *p, PgHdr *pPg){ ** ** Spilling is also prohibited when in an error state since that could ** lead to database corruption. In the current implementaton it - ** is impossible for sqlite3PCacheFetch() to be called with createFlag==1 + ** is impossible for sqlite3PcacheFetch() to be called with createFlag==1 ** while in the error state, hence it is impossible for this routine to ** be called in the error state. Nevertheless, we include a NEVER() ** test for the error state as a safeguard against future changes. @@ -42537,14 +42328,13 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); - PAGER_INCR(pPager->nHit); + pPager->nHit++; return SQLITE_OK; }else{ /* The pager cache has created a new page. Its content needs to ** be initialized. */ - PAGER_INCR(pPager->nMiss); pPg = *ppPage; pPg->pPager = pPager; @@ -42580,6 +42370,7 @@ SQLITE_PRIVATE int sqlite3PagerAcquire( IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ assert( pPg->pPager==pPager ); + pPager->nMiss++; rc = readDbPage(pPg); if( rc!=SQLITE_OK ){ goto pager_acquire_err; @@ -43614,6 +43405,31 @@ SQLITE_PRIVATE int *sqlite3PagerStats(Pager *pPager){ } #endif +/* +** Parameter eStat must be either SQLITE_DBSTATUS_CACHE_HIT or +** SQLITE_DBSTATUS_CACHE_MISS. Before returning, *pnVal is incremented by the +** current cache hit or miss count, according to the value of eStat. If the +** reset parameter is non-zero, the cache hit or miss count is zeroed before +** returning. +*/ +SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ + int *piStat; + + assert( eStat==SQLITE_DBSTATUS_CACHE_HIT + || eStat==SQLITE_DBSTATUS_CACHE_MISS + ); + if( eStat==SQLITE_DBSTATUS_CACHE_HIT ){ + piStat = &pPager->nHit; + }else{ + piStat = &pPager->nMiss; + } + + *pnVal += *piStat; + if( reset ){ + *piStat = 0; + } +} + /* ** Return true if this is an in-memory pager. */ @@ -44343,6 +44159,13 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ return rc; } +/* +** Unless this is an in-memory or temporary database, clear the pager cache. +*/ +SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *pPager){ + if( !MEMDB && pPager->tempFile==0 ) pager_reset(pPager); +} + #ifdef SQLITE_HAS_CODEC /* ** This function is called by the wal module when writing page content @@ -46708,7 +46531,7 @@ SQLITE_PRIVATE int sqlite3WalRead( int sz; i64 iOffset; sz = pWal->hdr.szPage; - sz = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + sz = (sz&0xfe00) + ((sz&0x0001)<<16); testcase( sz<=32768 ); testcase( sz>=65536 ); iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; @@ -50021,17 +49844,19 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( if( vfsFlags & SQLITE_OPEN_SHAREDCACHE ){ int nFullPathname = pVfs->mxPathname+1; char *zFullPathname = sqlite3Malloc(nFullPathname); - sqlite3_mutex *mutexShared; + MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) p->sharable = 1; if( !zFullPathname ){ sqlite3_free(p); return SQLITE_NOMEM; } sqlite3OsFullPathname(pVfs, zFilename, nFullPathname, zFullPathname); +#if SQLITE_THREADSAFE mutexOpen = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_OPEN); sqlite3_mutex_enter(mutexOpen); mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); sqlite3_mutex_enter(mutexShared); +#endif for(pBt=GLOBAL(BtShared*,sqlite3SharedCacheList); pBt; pBt=pBt->pNext){ assert( pBt->nRef>0 ); if( 0==strcmp(zFullPathname, sqlite3PagerFilename(pBt->pPager)) @@ -50137,9 +49962,9 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( /* Add the new BtShared object to the linked list sharable BtShareds. */ if( p->sharable ){ - sqlite3_mutex *mutexShared; + MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) pBt->nRef = 1; - mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + MUTEX_LOGIC( mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER);) if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); if( pBt->mutex==0 ){ @@ -50221,12 +50046,12 @@ btree_open_out: */ static int removeFromSharingList(BtShared *pBt){ #ifndef SQLITE_OMIT_SHARED_CACHE - sqlite3_mutex *pMaster; + MUTEX_LOGIC( sqlite3_mutex *pMaster; ) BtShared *pList; int removed = 0; assert( sqlite3_mutex_notheld(pBt->mutex) ); - pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(pMaster); pBt->nRef--; if( pBt->nRef<=0 ){ @@ -52193,21 +52018,55 @@ static int accessPayload( /* Need to read this page properly. It contains some of the ** range of data that is being read (eOp==0) or written (eOp!=0). */ - DbPage *pDbPage; +#ifdef SQLITE_DIRECT_OVERFLOW_READ + sqlite3_file *fd; +#endif int a = amt; - rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage); - if( rc==SQLITE_OK ){ - aPayload = sqlite3PagerGetData(pDbPage); - nextPage = get4byte(aPayload); - if( a + offset > ovflSize ){ - a = ovflSize - offset; - } - rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); - sqlite3PagerUnref(pDbPage); - offset = 0; - amt -= a; - pBuf += a; + if( a + offset > ovflSize ){ + a = ovflSize - offset; } + +#ifdef SQLITE_DIRECT_OVERFLOW_READ + /* If all the following are true: + ** + ** 1) this is a read operation, and + ** 2) data is required from the start of this overflow page, and + ** 3) the database is file-backed, and + ** 4) there is no open write-transaction, and + ** 5) the database is not a WAL database, + ** + ** then data can be read directly from the database file into the + ** output buffer, bypassing the page-cache altogether. This speeds + ** up loading large records that span many overflow pages. + */ + if( eOp==0 /* (1) */ + && offset==0 /* (2) */ + && pBt->inTransaction==TRANS_READ /* (4) */ + && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ + && pBt->pPage1->aData[19]==0x01 /* (5) */ + ){ + u8 aSave[4]; + u8 *aWrite = &pBuf[-4]; + memcpy(aSave, aWrite, 4); + rc = sqlite3OsRead(fd, aWrite, a+4, pBt->pageSize * (nextPage-1)); + nextPage = get4byte(aWrite); + memcpy(aWrite, aSave, 4); + }else +#endif + + { + DbPage *pDbPage; + rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage); + if( rc==SQLITE_OK ){ + aPayload = sqlite3PagerGetData(pDbPage); + nextPage = get4byte(aPayload); + rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); + sqlite3PagerUnref(pDbPage); + offset = 0; + } + } + amt -= a; + pBuf += a; } } } @@ -52806,7 +52665,6 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( if( c==0 ){ if( pPage->intKey && !pPage->leaf ){ lwr = idx; - upr = lwr - 1; break; }else{ *pRes = 0; @@ -52824,7 +52682,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( } pCur->aiIdx[pCur->iPage] = (u16)(idx = (lwr+upr)/2); } - assert( lwr==upr+1 ); + assert( lwr==upr+1 || (pPage->intKey && !pPage->leaf) ); assert( pPage->isInit ); if( pPage->leaf ){ chldPg = 0; @@ -53089,6 +52947,8 @@ static int allocateBtreePage( pTrunk = 0; goto end_allocate_page; } + assert( pTrunk!=0 ); + assert( pTrunk->aData!=0 ); k = get4byte(&pTrunk->aData[4]); /* # of leaves on this trunk page */ if( k==0 && !searchList ){ @@ -54216,13 +54076,15 @@ static int balance_nonroot( ** four bytes of the divider cell. So the pointer is safe to use ** later on. ** - ** Unless SQLite is compiled in secure-delete mode. In this case, + ** But not if we are in secure-delete mode. In secure-delete mode, ** the dropCell() routine will overwrite the entire cell with zeroes. ** In this case, temporarily copy the cell into the aOvflSpace[] ** buffer. It will be copied out again as soon as the aSpace[] buffer ** is allocated. */ if( pBt->secureDelete ){ - int iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); + int iOff; + + iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); if( (iOff+szNew[i])>(int)pBt->usableSize ){ rc = SQLITE_CORRUPT_BKPT; memset(apOld, 0, (i+1)*sizeof(MemPage*)); @@ -54642,6 +54504,7 @@ static int balance_nonroot( /* Cell i is the cell immediately following the last cell on old ** sibling page j. If the siblings are not leaf pages of an ** intkey b-tree, then cell i was a divider cell. */ + assert( j+1 < ArraySize(apCopy) ); pOld = apCopy[++j]; iNextOld = i + !leafData + pOld->nCell + pOld->nOverflow; if( pOld->nOverflow ){ @@ -56984,14 +56847,14 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ */ SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ sqlite3_backup **pp; /* Ptr to head of pagers backup list */ - sqlite3_mutex *mutex; /* Mutex to protect source database */ + MUTEX_LOGIC( sqlite3_mutex *mutex; ) /* Mutex to protect source database */ int rc; /* Value to return */ /* Enter the mutexes */ if( p==0 ) return SQLITE_OK; sqlite3_mutex_enter(p->pSrcDb->mutex); sqlite3BtreeEnter(p->pSrc); - mutex = p->pSrcDb->mutex; + MUTEX_LOGIC( mutex = p->pSrcDb->mutex; ) if( p->pDestDb ){ sqlite3_mutex_enter(p->pDestDb->mutex); } @@ -57110,10 +56973,18 @@ SQLITE_PRIVATE void sqlite3BackupRestart(sqlite3_backup *pBackup){ */ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ int rc; + sqlite3_file *pFd; /* File descriptor for database pTo */ sqlite3_backup b; sqlite3BtreeEnter(pTo); sqlite3BtreeEnter(pFrom); + assert( sqlite3BtreeIsInTrans(pTo) ); + pFd = sqlite3PagerFile(sqlite3BtreePager(pTo)); + if( pFd->pMethods ){ + i64 nByte = sqlite3BtreeGetPageSize(pFrom)*(i64)sqlite3BtreeLastPage(pFrom); + sqlite3OsFileControl(pFd, SQLITE_FCNTL_OVERWRITE, &nByte); + } + /* Set up an sqlite3_backup object. sqlite3_backup.pDestDb must be set ** to 0. This is used by the implementations of sqlite3_backup_step() ** and sqlite3_backup_finish() to detect that they are being called @@ -57137,8 +57008,11 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ rc = sqlite3_backup_finish(&b); if( rc==SQLITE_OK ){ pTo->pBt->pageSizeFixed = 0; + }else{ + sqlite3PagerClearCache(sqlite3BtreePager(b.pDest)); } + assert( sqlite3BtreeIsInTrans(pTo)==0 ); sqlite3BtreeLeave(pFrom); sqlite3BtreeLeave(pTo); return rc; @@ -58173,11 +58047,11 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( } op = pExpr->op; - /* op can only be TK_REGISTER if we have compiled with SQLITE_ENABLE_STAT2. + /* op can only be TK_REGISTER if we have compiled with SQLITE_ENABLE_STAT3. ** The ifdef here is to enable us to achieve 100% branch test coverage even - ** when SQLITE_ENABLE_STAT2 is omitted. + ** when SQLITE_ENABLE_STAT3 is omitted. */ -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3 if( op==TK_REGISTER ) op = pExpr->op2; #else if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; @@ -58876,8 +58750,8 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ ** the address of the next instruction to be coded. */ SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ - assert( addr>=0 ); - sqlite3VdbeChangeP2(p, addr, p->nOp); + assert( addr>=0 || p->db->mallocFailed ); + if( addr>=0 ) sqlite3VdbeChangeP2(p, addr, p->nOp); } @@ -59082,30 +58956,29 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int ** makes the code easier to read during debugging. None of this happens ** in a production build. */ -SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe *p, const char *zFormat, ...){ - va_list ap; - if( !p ) return; +static void vdbeVComment(Vdbe *p, const char *zFormat, va_list ap){ assert( p->nOp>0 || p->aOp==0 ); assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed ); if( p->nOp ){ - char **pz = &p->aOp[p->nOp-1].zComment; + assert( p->aOp ); + sqlite3DbFree(p->db, p->aOp[p->nOp-1].zComment); + p->aOp[p->nOp-1].zComment = sqlite3VMPrintf(p->db, zFormat, ap); + } +} +SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe *p, const char *zFormat, ...){ + va_list ap; + if( p ){ va_start(ap, zFormat); - sqlite3DbFree(p->db, *pz); - *pz = sqlite3VMPrintf(p->db, zFormat, ap); + vdbeVComment(p, zFormat, ap); va_end(ap); } } SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){ va_list ap; - if( !p ) return; - sqlite3VdbeAddOp0(p, OP_Noop); - assert( p->nOp>0 || p->aOp==0 ); - assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed ); - if( p->nOp ){ - char **pz = &p->aOp[p->nOp-1].zComment; + if( p ){ + sqlite3VdbeAddOp0(p, OP_Noop); va_start(ap, zFormat); - sqlite3DbFree(p->db, *pz); - *pz = sqlite3VMPrintf(p->db, zFormat, ap); + vdbeVComment(p, zFormat, ap); va_end(ap); } } @@ -59443,7 +59316,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( sqlite3 *db = p->db; /* The database connection */ int i; /* Loop counter */ int rc = SQLITE_OK; /* Return code */ - Mem *pMem = p->pResultSet = &p->aMem[1]; /* First Mem of result set */ + Mem *pMem = &p->aMem[1]; /* First Mem of result set */ assert( p->explain ); assert( p->magic==VDBE_MAGIC_RUN ); @@ -59454,6 +59327,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( ** sqlite3_column_text16(), causing a translation to UTF-16 encoding. */ releaseMemArray(pMem, 8); + p->pResultSet = 0; if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or @@ -59608,6 +59482,7 @@ SQLITE_PRIVATE int sqlite3VdbeList( } p->nResColumn = 8 - 4*(p->explain-1); + p->pResultSet = &p->aMem[1]; p->rc = SQLITE_OK; rc = SQLITE_ROW; } @@ -60610,6 +60485,30 @@ SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe *p){ p->rc = SQLITE_OK; } +/* +** Copy the error code and error message belonging to the VDBE passed +** as the first argument to its database handle (so that they will be +** returned by calls to sqlite3_errcode() and sqlite3_errmsg()). +** +** This function does not clear the VDBE error code or message, just +** copies them to the database handle. +*/ +SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p){ + sqlite3 *db = p->db; + int rc = p->rc; + if( p->zErrMsg ){ + u8 mallocFailed = db->mallocFailed; + sqlite3BeginBenignMalloc(); + sqlite3ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE_UTF8, SQLITE_TRANSIENT); + sqlite3EndBenignMalloc(); + db->mallocFailed = mallocFailed; + db->errCode = rc; + }else{ + sqlite3Error(db, rc, 0); + } + return rc; +} + /* ** Clean up a VDBE after execution but do not delete the VDBE just yet. ** Write any error messages into *pzErrMsg. Return the result code. @@ -60637,18 +60536,9 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ ** instructions yet, leave the main database error information unchanged. */ if( p->pc>=0 ){ - if( p->zErrMsg ){ - sqlite3BeginBenignMalloc(); - sqlite3ValueSetStr(db->pErr,-1,p->zErrMsg,SQLITE_UTF8,SQLITE_TRANSIENT); - sqlite3EndBenignMalloc(); - db->errCode = p->rc; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = 0; - }else if( p->rc ){ - sqlite3Error(db, p->rc, 0); - }else{ - sqlite3Error(db, SQLITE_OK, 0); - } + sqlite3VdbeTransferError(p); + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = 0; if( p->runOnlyOnce ) p->expired = 1; }else if( p->rc && p->expired ){ /* The expired flag was set on the VDBE before the first call @@ -61363,7 +61253,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ ** this code can safely assume that nCellKey is 32-bits */ assert( sqlite3BtreeCursorIsValid(pCur) ); - rc = sqlite3BtreeKeySize(pCur, &nCellKey); + VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey ); @@ -61438,7 +61328,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( Mem m; assert( sqlite3BtreeCursorIsValid(pCur) ); - rc = sqlite3BtreeKeySize(pCur, &nCellKey); + VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ /* nCellKey will always be between 0 and 0xffffffff because of the say ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ @@ -61994,7 +61884,7 @@ end_of_step: ** error has occured, then return the error code in p->rc to the ** caller. Set the error code in the database handle to the same value. */ - rc = db->errCode = p->rc; + rc = sqlite3VdbeTransferError(p); } return (rc&db->errMask); } @@ -65714,7 +65604,7 @@ case OP_Column: { u.am.zRec = (char*)u.am.pC->aRow; }else if( u.am.pC->isIndex ){ assert( sqlite3BtreeCursorIsValid(u.am.pCrsr) ); - rc = sqlite3BtreeKeySize(u.am.pCrsr, &u.am.payloadSize64); + VVA_ONLY(rc =) sqlite3BtreeKeySize(u.am.pCrsr, &u.am.payloadSize64); assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the ** payload size, so it is impossible for u.am.payloadSize64 to be @@ -65723,7 +65613,7 @@ case OP_Column: { u.am.payloadSize = (u32)u.am.payloadSize64; }else{ assert( sqlite3BtreeCursorIsValid(u.am.pCrsr) ); - rc = sqlite3BtreeDataSize(u.am.pCrsr, &u.am.payloadSize); + VVA_ONLY(rc =) sqlite3BtreeDataSize(u.am.pCrsr, &u.am.payloadSize); assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ } }else if( ALWAYS(u.am.pC->pseudoTableReg>0) ){ @@ -67775,14 +67665,14 @@ case OP_RowData: { if( u.bk.pC->isIndex ){ assert( !u.bk.pC->isTable ); - rc = sqlite3BtreeKeySize(u.bk.pCrsr, &u.bk.n64); + VVA_ONLY(rc =) sqlite3BtreeKeySize(u.bk.pCrsr, &u.bk.n64); assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ if( u.bk.n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } u.bk.n = (u32)u.bk.n64; }else{ - rc = sqlite3BtreeDataSize(u.bk.pCrsr, &u.bk.n); + VVA_ONLY(rc =) sqlite3BtreeDataSize(u.bk.pCrsr, &u.bk.n); assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ if( u.bk.n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; @@ -67894,9 +67784,8 @@ case OP_Last: { /* jump */ u.bn.pC = p->apCsr[pOp->p1]; assert( u.bn.pC!=0 ); u.bn.pCrsr = u.bn.pC->pCursor; - if( NEVER(u.bn.pCrsr==0) ){ - u.bn.res = 1; - }else{ + u.bn.res = 0; + if( ALWAYS(u.bn.pCrsr!=0) ){ rc = sqlite3BtreeLast(u.bn.pCrsr, &u.bn.res); } u.bn.pC->nullRow = (u8)u.bn.res; @@ -69097,7 +68986,7 @@ case OP_JournalMode: { /* out2-prerelease */ ** in temporary storage or if the VFS does not support shared memory */ if( u.ch.eNew==PAGER_JOURNALMODE_WAL - && (u.ch.zFilename[0]==0 /* Temp file */ + && (sqlite3Strlen30(u.ch.zFilename)==0 /* Temp file */ || !sqlite3PagerWalSupported(u.ch.pPager)) /* No shared-memory support */ ){ u.ch.eNew = u.ch.eOld; @@ -69532,10 +69421,15 @@ case OP_VRename: { assert( memIsValid(u.co.pName) ); REGISTER_TRACE(pOp->p1, u.co.pName); assert( u.co.pName->flags & MEM_Str ); - rc = u.co.pVtab->pModule->xRename(u.co.pVtab, u.co.pName->z); - importVtabErrMsg(p, u.co.pVtab); - p->expired = 0; - + testcase( u.co.pName->enc==SQLITE_UTF8 ); + testcase( u.co.pName->enc==SQLITE_UTF16BE ); + testcase( u.co.pName->enc==SQLITE_UTF16LE ); + rc = sqlite3VdbeChangeEncoding(u.co.pName, SQLITE_UTF8); + if( rc==SQLITE_OK ){ + rc = u.co.pVtab->pModule->xRename(u.co.pVtab, u.co.pName->z); + importVtabErrMsg(p, u.co.pVtab); + p->expired = 0; + } break; } #endif @@ -71894,6 +71788,24 @@ static void resolveAlias( sqlite3DbFree(db, pDup); } + +/* +** Return TRUE if the name zCol occurs anywhere in the USING clause. +** +** Return FALSE if the USING clause is NULL or if it does not contain +** zCol. +*/ +static int nameInUsingClause(IdList *pUsing, const char *zCol){ + if( pUsing ){ + int k; + for(k=0; knId; k++){ + if( sqlite3StrICmp(pUsing->a[k].zName, zCol)==0 ) return 1; + } + } + return 0; +} + + /* ** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up ** that name in the set of source tables in pSrcList and make the pExpr @@ -71985,7 +71897,14 @@ static int lookupName( } for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ - IdList *pUsing; + /* If there has been exactly one prior match and this match + ** is for the right-hand table of a NATURAL JOIN or is in a + ** USING clause, then skip this match. + */ + if( cnt==1 ){ + if( pItem->jointype & JT_NATURAL ) continue; + if( nameInUsingClause(pItem->pUsing, zCol) ) continue; + } cnt++; pExpr->iTable = pItem->iCursor; pExpr->pTab = pTab; @@ -71993,26 +71912,6 @@ static int lookupName( pSchema = pTab->pSchema; /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; - if( inSrc-1 ){ - if( pItem[1].jointype & JT_NATURAL ){ - /* If this match occurred in the left table of a natural join, - ** then skip the right table to avoid a duplicate match */ - pItem++; - i++; - }else if( (pUsing = pItem[1].pUsing)!=0 ){ - /* If this match occurs on a column that is in the USING clause - ** of a join, skip the search of the right table of the join - ** to avoid a duplicate match there. */ - int k; - for(k=0; knId; k++){ - if( sqlite3StrICmp(pUsing->a[k].zName, zCol)==0 ){ - pItem++; - i++; - break; - } - } - } - } break; } } @@ -73419,7 +73318,8 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( }else{ int c; pNew->u.zToken = (char*)&pNew[1]; - memcpy(pNew->u.zToken, pToken->z, pToken->n); + assert( pToken->z!=0 || pToken->n==0 ); + if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); pNew->u.zToken[pToken->n] = 0; if( dequote && nExtra>=3 && ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){ @@ -74458,11 +74358,19 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0); if( ALWAYS(pParse->nErr==0) && isCandidateForInOpt(p) ){ sqlite3 *db = pParse->db; /* Database connection */ - Expr *pExpr = p->pEList->a[0].pExpr; /* Expression */ - int iCol = pExpr->iColumn; /* Index of column */ Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ - Table *pTab = p->pSrc->a[0].pTab; /* Table . */ + Table *pTab; /* Table
. */ + Expr *pExpr; /* Expression */ + int iCol; /* Index of column */ int iDb; /* Database idx for pTab */ + + assert( p ); /* Because of isCandidateForInOpt(p) */ + assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ + assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ + assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ + pTab = p->pSrc->a[0].pTab; + pExpr = p->pEList->a[0].pExpr; + iCol = pExpr->iColumn; /* Code an OP_VerifyCookie and OP_TableLock for
. */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -76469,7 +76377,7 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB){ } }else if( pA->op!=TK_COLUMN && pA->u.zToken ){ if( ExprHasProperty(pB, EP_IntValue) || NEVER(pB->u.zToken==0) ) return 2; - if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ){ + if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ return 2; } } @@ -77612,22 +77520,124 @@ exit_begin_add_column: ** ************************************************************************* ** This file contains code associated with the ANALYZE command. +** +** The ANALYZE command gather statistics about the content of tables +** and indices. These statistics are made available to the query planner +** to help it make better decisions about how to perform queries. +** +** The following system tables are or have been supported: +** +** CREATE TABLE sqlite_stat1(tbl, idx, stat); +** CREATE TABLE sqlite_stat2(tbl, idx, sampleno, sample); +** CREATE TABLE sqlite_stat3(tbl, idx, nEq, nLt, nDLt, sample); +** +** Additional tables might be added in future releases of SQLite. +** The sqlite_stat2 table is not created or used unless the SQLite version +** is between 3.6.18 and 3.7.8, inclusive, and unless SQLite is compiled +** with SQLITE_ENABLE_STAT2. The sqlite_stat2 table is deprecated. +** The sqlite_stat2 table is superceded by sqlite_stat3, which is only +** created and used by SQLite versions 3.7.9 and later and with +** SQLITE_ENABLE_STAT3 defined. The fucntionality of sqlite_stat3 +** is a superset of sqlite_stat2. +** +** Format of sqlite_stat1: +** +** There is normally one row per index, with the index identified by the +** name in the idx column. The tbl column is the name of the table to +** which the index belongs. In each such row, the stat column will be +** a string consisting of a list of integers. The first integer in this +** list is the number of rows in the index and in the table. The second +** integer is the average number of rows in the index that have the same +** value in the first column of the index. The third integer is the average +** number of rows in the index that have the same value for the first two +** columns. The N-th integer (for N>1) is the average number of rows in +** the index which have the same value for the first N-1 columns. For +** a K-column index, there will be K+1 integers in the stat column. If +** the index is unique, then the last integer will be 1. +** +** The list of integers in the stat column can optionally be followed +** by the keyword "unordered". The "unordered" keyword, if it is present, +** must be separated from the last integer by a single space. If the +** "unordered" keyword is present, then the query planner assumes that +** the index is unordered and will not use the index for a range query. +** +** If the sqlite_stat1.idx column is NULL, then the sqlite_stat1.stat +** column contains a single integer which is the (estimated) number of +** rows in the table identified by sqlite_stat1.tbl. +** +** Format of sqlite_stat2: +** +** The sqlite_stat2 is only created and is only used if SQLite is compiled +** with SQLITE_ENABLE_STAT2 and if the SQLite version number is between +** 3.6.18 and 3.7.8. The "stat2" table contains additional information +** about the distribution of keys within an index. The index is identified by +** the "idx" column and the "tbl" column is the name of the table to which +** the index belongs. There are usually 10 rows in the sqlite_stat2 +** table for each index. +** +** The sqlite_stat2 entries for an index that have sampleno between 0 and 9 +** inclusive are samples of the left-most key value in the index taken at +** evenly spaced points along the index. Let the number of samples be S +** (10 in the standard build) and let C be the number of rows in the index. +** Then the sampled rows are given by: +** +** rownumber = (i*C*2 + C)/(S*2) +** +** For i between 0 and S-1. Conceptually, the index space is divided into +** S uniform buckets and the samples are the middle row from each bucket. +** +** The format for sqlite_stat2 is recorded here for legacy reference. This +** version of SQLite does not support sqlite_stat2. It neither reads nor +** writes the sqlite_stat2 table. This version of SQLite only supports +** sqlite_stat3. +** +** Format for sqlite_stat3: +** +** The sqlite_stat3 is an enhancement to sqlite_stat2. A new name is +** used to avoid compatibility problems. +** +** The format of the sqlite_stat3 table is similar to the format of +** the sqlite_stat2 table. There are multiple entries for each index. +** The idx column names the index and the tbl column is the table of the +** index. If the idx and tbl columns are the same, then the sample is +** of the INTEGER PRIMARY KEY. The sample column is a value taken from +** the left-most column of the index. The nEq column is the approximate +** number of entires in the index whose left-most column exactly matches +** the sample. nLt is the approximate number of entires whose left-most +** column is less than the sample. The nDLt column is the approximate +** number of distinct left-most entries in the index that are less than +** the sample. +** +** Future versions of SQLite might change to store a string containing +** multiple integers values in the nDLt column of sqlite_stat3. The first +** integer will be the number of prior index entires that are distinct in +** the left-most column. The second integer will be the number of prior index +** entries that are distinct in the first two columns. The third integer +** will be the number of prior index entries that are distinct in the first +** three columns. And so forth. With that extension, the nDLt field is +** similar in function to the sqlite_stat1.stat field. +** +** There can be an arbitrary number of sqlite_stat3 entries per index. +** The ANALYZE command will typically generate sqlite_stat3 tables +** that contain between 10 and 40 samples which are distributed across +** the key space, though not uniformly, and which include samples with +** largest possible nEq values. */ #ifndef SQLITE_OMIT_ANALYZE /* ** This routine generates code that opens the sqlite_stat1 table for ** writing with cursor iStatCur. If the library was built with the -** SQLITE_ENABLE_STAT2 macro defined, then the sqlite_stat2 table is +** SQLITE_ENABLE_STAT3 macro defined, then the sqlite_stat3 table is ** opened for writing using cursor (iStatCur+1) ** ** If the sqlite_stat1 tables does not previously exist, it is created. -** Similarly, if the sqlite_stat2 table does not exist and the library -** is compiled with SQLITE_ENABLE_STAT2 defined, it is created. +** Similarly, if the sqlite_stat3 table does not exist and the library +** is compiled with SQLITE_ENABLE_STAT3 defined, it is created. ** ** Argument zWhere may be a pointer to a buffer containing a table name, ** or it may be a NULL pointer. If it is not NULL, then all entries in -** the sqlite_stat1 and (if applicable) sqlite_stat2 tables associated +** the sqlite_stat1 and (if applicable) sqlite_stat3 tables associated ** with the named table are deleted. If zWhere==0, then code is generated ** to delete all stat table entries. */ @@ -77643,8 +77653,8 @@ static void openStatTable( const char *zCols; } aTable[] = { { "sqlite_stat1", "tbl,idx,stat" }, -#ifdef SQLITE_ENABLE_STAT2 - { "sqlite_stat2", "tbl,idx,sampleno,sample" }, +#ifdef SQLITE_ENABLE_STAT3 + { "sqlite_stat3", "tbl,idx,neq,nlt,ndlt,sample" }, #endif }; @@ -77660,6 +77670,9 @@ static void openStatTable( assert( sqlite3VdbeDb(v)==db ); pDb = &db->aDb[iDb]; + /* Create new statistic tables if they do not exist, or clear them + ** if they do already exist. + */ for(i=0; ia[0])*mxSample; + p = sqlite3_malloc( n ); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + memset(p, 0, n); + p->a = (struct Stat3Sample*)&p[1]; + p->nRow = nRow; + p->mxSample = mxSample; + p->nPSample = p->nRow/(mxSample/3+1) + 1; + sqlite3_randomness(sizeof(p->iPrn), &p->iPrn); + sqlite3_result_blob(context, p, sizeof(p), sqlite3_free); +} +static const FuncDef stat3InitFuncdef = { + 2, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + stat3Init, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat3_init", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ +}; + + +/* +** Implementation of the stat3_push(nEq,nLt,nDLt,rowid,P) SQL function. The +** arguments describe a single key instance. This routine makes the +** decision about whether or not to retain this key for the sqlite_stat3 +** table. +** +** The return value is NULL. +*/ +static void stat3Push( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Stat3Accum *p = (Stat3Accum*)sqlite3_value_blob(argv[4]); + tRowcnt nEq = sqlite3_value_int64(argv[0]); + tRowcnt nLt = sqlite3_value_int64(argv[1]); + tRowcnt nDLt = sqlite3_value_int64(argv[2]); + i64 rowid = sqlite3_value_int64(argv[3]); + u8 isPSample = 0; + u8 doInsert = 0; + int iMin = p->iMin; + struct Stat3Sample *pSample; + int i; + u32 h; + + UNUSED_PARAMETER(context); + UNUSED_PARAMETER(argc); + if( nEq==0 ) return; + h = p->iPrn = p->iPrn*1103515245 + 12345; + if( (nLt/p->nPSample)!=((nEq+nLt)/p->nPSample) ){ + doInsert = isPSample = 1; + }else if( p->nSamplemxSample ){ + doInsert = 1; + }else{ + if( nEq>p->a[iMin].nEq || (nEq==p->a[iMin].nEq && h>p->a[iMin].iHash) ){ + doInsert = 1; + } + } + if( !doInsert ) return; + if( p->nSample==p->mxSample ){ + assert( p->nSample - iMin - 1 >= 0 ); + memmove(&p->a[iMin], &p->a[iMin+1], sizeof(p->a[0])*(p->nSample-iMin-1)); + pSample = &p->a[p->nSample-1]; + }else{ + pSample = &p->a[p->nSample++]; + } + pSample->iRowid = rowid; + pSample->nEq = nEq; + pSample->nLt = nLt; + pSample->nDLt = nDLt; + pSample->iHash = h; + pSample->isPSample = isPSample; + + /* Find the new minimum */ + if( p->nSample==p->mxSample ){ + pSample = p->a; + i = 0; + while( pSample->isPSample ){ + i++; + pSample++; + assert( inSample ); + } + nEq = pSample->nEq; + h = pSample->iHash; + iMin = i; + for(i++, pSample++; inSample; i++, pSample++){ + if( pSample->isPSample ) continue; + if( pSample->nEqnEq==nEq && pSample->iHashnEq; + h = pSample->iHash; + } + } + p->iMin = iMin; + } +} +static const FuncDef stat3PushFuncdef = { + 5, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + stat3Push, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat3_push", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ +}; + +/* +** Implementation of the stat3_get(P,N,...) SQL function. This routine is +** used to query the results. Content is returned for the Nth sqlite_stat3 +** row where N is between 0 and S-1 and S is the number of samples. The +** value returned depends on the number of arguments. +** +** argc==2 result: rowid +** argc==3 result: nEq +** argc==4 result: nLt +** argc==5 result: nDLt +*/ +static void stat3Get( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int n = sqlite3_value_int(argv[1]); + Stat3Accum *p = (Stat3Accum*)sqlite3_value_blob(argv[0]); + + assert( p!=0 ); + if( p->nSample<=n ) return; + switch( argc ){ + case 2: sqlite3_result_int64(context, p->a[n].iRowid); break; + case 3: sqlite3_result_int64(context, p->a[n].nEq); break; + case 4: sqlite3_result_int64(context, p->a[n].nLt); break; + default: sqlite3_result_int64(context, p->a[n].nDLt); break; + } +} +static const FuncDef stat3GetFuncdef = { + -1, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + stat3Get, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat3_get", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ +}; +#endif /* SQLITE_ENABLE_STAT3 */ + + + + /* ** Generate code to do an analysis of all indices associated with ** a single table. @@ -77720,20 +77953,27 @@ static void analyzeOneTable( int iDb; /* Index of database containing pTab */ int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ - int regSampleno = iMem++; /* Register containing next sample number */ - int regCol = iMem++; /* Content of a column analyzed table */ + int regStat1 = iMem++; /* The stat column of sqlite_stat1 */ +#ifdef SQLITE_ENABLE_STAT3 + int regNumEq = regStat1; /* Number of instances. Same as regStat1 */ + int regNumLt = iMem++; /* Number of keys less than regSample */ + int regNumDLt = iMem++; /* Number of distinct keys less than regSample */ + int regSample = iMem++; /* The next sample value */ + int regRowid = regSample; /* Rowid of a sample */ + int regAccum = iMem++; /* Register to hold Stat3Accum object */ + int regLoop = iMem++; /* Loop counter */ + int regCount = iMem++; /* Number of rows in the table or index */ + int regTemp1 = iMem++; /* Intermediate register */ + int regTemp2 = iMem++; /* Intermediate register */ + int once = 1; /* One-time initialization */ + int shortJump = 0; /* Instruction address */ + int iTabCur = pParse->nTab++; /* Table cursor */ +#endif + int regCol = iMem++; /* Content of a column in analyzed table */ int regRec = iMem++; /* Register holding completed record */ int regTemp = iMem++; /* Temporary use register */ - int regRowid = iMem++; /* Rowid for the inserted record */ + int regNewRowid = iMem++; /* Rowid for the inserted record */ -#ifdef SQLITE_ENABLE_STAT2 - int addr = 0; /* Instruction address */ - int regTemp2 = iMem++; /* Temporary use register */ - int regSamplerecno = iMem++; /* Index of next sample to record */ - int regRecno = iMem++; /* Current sample index */ - int regLast = iMem++; /* Index of last sample to record */ - int regFirst = iMem++; /* Index of first sample to record */ -#endif v = sqlite3GetVdbe(pParse); if( v==0 || NEVER(pTab==0) ){ @@ -77766,9 +78006,14 @@ static void analyzeOneTable( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; KeyInfo *pKey; + int addrIfNot = 0; /* address of OP_IfNot */ + int *aChngAddr; /* Array of jump instruction addresses */ if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; + VdbeNoopComment((v, "Begin analysis of %s", pIdx->zName)); nCol = pIdx->nColumn; + aChngAddr = sqlite3DbMallocRaw(db, sizeof(int)*nCol); + if( aChngAddr==0 ) continue; pKey = sqlite3IndexKeyinfo(pParse, pIdx); if( iMem+1+(nCol*2)>pParse->nMem ){ pParse->nMem = iMem+1+(nCol*2); @@ -77783,31 +78028,20 @@ static void analyzeOneTable( /* Populate the register containing the index name. */ sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, pIdx->zName, 0); -#ifdef SQLITE_ENABLE_STAT2 - - /* If this iteration of the loop is generating code to analyze the - ** first index in the pTab->pIndex list, then register regLast has - ** not been populated. In this case populate it now. */ - if( pTab->pIndex==pIdx ){ - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES, regSamplerecno); - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES*2-1, regTemp); - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES*2, regTemp2); - - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regLast); - sqlite3VdbeAddOp2(v, OP_Null, 0, regFirst); - addr = sqlite3VdbeAddOp3(v, OP_Lt, regSamplerecno, 0, regLast); - sqlite3VdbeAddOp3(v, OP_Divide, regTemp2, regLast, regFirst); - sqlite3VdbeAddOp3(v, OP_Multiply, regLast, regTemp, regLast); - sqlite3VdbeAddOp2(v, OP_AddImm, regLast, SQLITE_INDEX_SAMPLES*2-2); - sqlite3VdbeAddOp3(v, OP_Divide, regTemp2, regLast, regLast); - sqlite3VdbeJumpHere(v, addr); +#ifdef SQLITE_ENABLE_STAT3 + if( once ){ + once = 0; + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); } - - /* Zero the regSampleno and regRecno registers. */ - sqlite3VdbeAddOp2(v, OP_Integer, 0, regSampleno); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regRecno); - sqlite3VdbeAddOp2(v, OP_Copy, regFirst, regSamplerecno); -#endif + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regCount); + sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_STAT3_SAMPLES, regTemp1); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regNumEq); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regNumLt); + sqlite3VdbeAddOp2(v, OP_Integer, -1, regNumDLt); + sqlite3VdbeAddOp4(v, OP_Function, 1, regCount, regAccum, + (char*)&stat3InitFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 2); +#endif /* SQLITE_ENABLE_STAT3 */ /* The block of memory cells initialized here is used as follows. ** @@ -77837,75 +78071,83 @@ static void analyzeOneTable( endOfLoop = sqlite3VdbeMakeLabel(v); sqlite3VdbeAddOp2(v, OP_Rewind, iIdxCur, endOfLoop); topOfLoop = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_AddImm, iMem, 1); + sqlite3VdbeAddOp2(v, OP_AddImm, iMem, 1); /* Increment row counter */ for(i=0; iazColl!=0 ); assert( pIdx->azColl[i]!=0 ); pColl = sqlite3LocateCollSeq(pParse, pIdx->azColl[i]); - sqlite3VdbeAddOp4(v, OP_Ne, regCol, 0, iMem+nCol+i+1, - (char*)pColl, P4_COLLSEQ); + aChngAddr[i] = sqlite3VdbeAddOp4(v, OP_Ne, regCol, 0, iMem+nCol+i+1, + (char*)pColl, P4_COLLSEQ); sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); - } - if( db->mallocFailed ){ - /* If a malloc failure has occurred, then the result of the expression - ** passed as the second argument to the call to sqlite3VdbeJumpHere() - ** below may be negative. Which causes an assert() to fail (or an - ** out-of-bounds write if SQLITE_DEBUG is not defined). */ - return; + VdbeComment((v, "jump if column %d changed", i)); +#ifdef SQLITE_ENABLE_STAT3 + if( i==0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, regNumEq, 1); + VdbeComment((v, "incr repeat count")); + } +#endif } sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfLoop); for(i=0; inColumn, regRowid); + sqlite3VdbeAddOp3(v, OP_Add, regNumEq, regNumLt, regNumLt); + sqlite3VdbeAddOp2(v, OP_AddImm, regNumDLt, 1); + sqlite3VdbeAddOp2(v, OP_Integer, 1, regNumEq); +#endif } - sqlite3VdbeJumpHere(v, addr2); /* Set jump dest for the OP_Ne */ sqlite3VdbeAddOp2(v, OP_AddImm, iMem+i+1, 1); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, iMem+nCol+i+1); } + sqlite3DbFree(db, aChngAddr); - /* End of the analysis loop. */ + /* Always jump here after updating the iMem+1...iMem+1+nCol counters */ sqlite3VdbeResolveLabel(v, endOfLoop); + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, topOfLoop); sqlite3VdbeAddOp1(v, OP_Close, iIdxCur); +#ifdef SQLITE_ENABLE_STAT3 + sqlite3VdbeAddOp4(v, OP_Function, 1, regNumEq, regTemp2, + (char*)&stat3PushFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 5); + sqlite3VdbeAddOp2(v, OP_Integer, -1, regLoop); + shortJump = + sqlite3VdbeAddOp2(v, OP_AddImm, regLoop, 1); + sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regTemp1, + (char*)&stat3GetFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 2); + sqlite3VdbeAddOp1(v, OP_IsNull, regTemp1); + sqlite3VdbeAddOp3(v, OP_NotExists, iTabCur, shortJump, regTemp1); + sqlite3VdbeAddOp3(v, OP_Column, iTabCur, pIdx->aiColumn[0], regSample); + sqlite3ColumnDefault(v, pTab, pIdx->aiColumn[0], regSample); + sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regNumEq, + (char*)&stat3GetFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 3); + sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regNumLt, + (char*)&stat3GetFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 4); + sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regNumDLt, + (char*)&stat3GetFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 5); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 6, regRec, "bbbbbb", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regRec, regNewRowid); + sqlite3VdbeAddOp2(v, OP_Goto, 0, shortJump); + sqlite3VdbeJumpHere(v, shortJump+2); +#endif /* Store the results in sqlite_stat1. ** @@ -77925,22 +78167,22 @@ static void analyzeOneTable( ** If K>0 then it is always the case the D>0 so division by zero ** is never possible. */ - sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regSampleno); + sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regStat1); if( jZeroRows<0 ){ jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); } for(i=0; ipIndex==0 ){ sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pTab->tnum, iDb); VdbeComment((v, "%s", pTab->zName)); - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regSampleno); + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat1); sqlite3VdbeAddOp1(v, OP_Close, iIdxCur); - jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regSampleno); + jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); }else{ sqlite3VdbeJumpHere(v, jZeroRows); jZeroRows = sqlite3VdbeAddOp0(v, OP_Goto); } sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regNewRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); if( pParse->nMemnMem = regRec; sqlite3VdbeJumpHere(v, jZeroRows); } + /* ** Generate code that will cause the most recent index analysis to ** be loaded into internal hash tables where is can be used. @@ -77989,7 +78232,7 @@ static void analyzeDatabase(Parse *pParse, int iDb){ sqlite3BeginWriteOperation(pParse, 0, iDb); iStatCur = pParse->nTab; - pParse->nTab += 2; + pParse->nTab += 3; openStatTable(pParse, iDb, iStatCur, 0, 0); iMem = pParse->nMem+1; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); @@ -78014,7 +78257,7 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); sqlite3BeginWriteOperation(pParse, 0, iDb); iStatCur = pParse->nTab; - pParse->nTab += 2; + pParse->nTab += 3; if( pOnlyIdx ){ openStatTable(pParse, iDb, iStatCur, pOnlyIdx->zName, "idx"); }else{ @@ -78119,7 +78362,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ Index *pIndex; Table *pTable; int i, c, n; - unsigned int v; + tRowcnt v; const char *z; assert( argc==3 ); @@ -78162,10 +78405,10 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** and its contents. */ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3 if( pIdx->aSample ){ int j; - for(j=0; jnSample; j++){ IndexSample *p = &pIdx->aSample[j]; if( p->eType==SQLITE_TEXT || p->eType==SQLITE_BLOB ){ sqlite3DbFree(db, p->u.z); @@ -78173,25 +78416,157 @@ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ } sqlite3DbFree(db, pIdx->aSample); } + if( db && db->pnBytesFreed==0 ){ + pIdx->nSample = 0; + pIdx->aSample = 0; + } #else UNUSED_PARAMETER(db); UNUSED_PARAMETER(pIdx); #endif } +#ifdef SQLITE_ENABLE_STAT3 /* -** Load the content of the sqlite_stat1 and sqlite_stat2 tables. The +** Load content from the sqlite_stat3 table into the Index.aSample[] +** arrays of all indices. +*/ +static int loadStat3(sqlite3 *db, const char *zDb){ + int rc; /* Result codes from subroutines */ + sqlite3_stmt *pStmt = 0; /* An SQL statement being run */ + char *zSql; /* Text of the SQL statement */ + Index *pPrevIdx = 0; /* Previous index in the loop */ + int idx = 0; /* slot in pIdx->aSample[] for next sample */ + int eType; /* Datatype of a sample */ + IndexSample *pSample; /* A slot in pIdx->aSample[] */ + + if( !sqlite3FindTable(db, "sqlite_stat3", zDb) ){ + return SQLITE_OK; + } + + zSql = sqlite3MPrintf(db, + "SELECT idx,count(*) FROM %Q.sqlite_stat3" + " GROUP BY idx", zDb); + if( !zSql ){ + return SQLITE_NOMEM; + } + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + sqlite3DbFree(db, zSql); + if( rc ) return rc; + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nSample; /* Number of samples */ + + zIndex = (char *)sqlite3_column_text(pStmt, 0); + if( zIndex==0 ) continue; + nSample = sqlite3_column_int(pStmt, 1); + pIdx = sqlite3FindIndex(db, zIndex, zDb); + if( pIdx==0 ) continue; + assert( pIdx->nSample==0 ); + pIdx->nSample = nSample; + pIdx->aSample = sqlite3MallocZero( nSample*sizeof(IndexSample) ); + pIdx->avgEq = pIdx->aiRowEst[1]; + if( pIdx->aSample==0 ){ + db->mallocFailed = 1; + sqlite3_finalize(pStmt); + return SQLITE_NOMEM; + } + } + rc = sqlite3_finalize(pStmt); + if( rc ) return rc; + + zSql = sqlite3MPrintf(db, + "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat3", zDb); + if( !zSql ){ + return SQLITE_NOMEM; + } + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + sqlite3DbFree(db, zSql); + if( rc ) return rc; + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int i; /* Loop counter */ + tRowcnt sumEq; /* Sum of the nEq values */ + + zIndex = (char *)sqlite3_column_text(pStmt, 0); + if( zIndex==0 ) continue; + pIdx = sqlite3FindIndex(db, zIndex, zDb); + if( pIdx==0 ) continue; + if( pIdx==pPrevIdx ){ + idx++; + }else{ + pPrevIdx = pIdx; + idx = 0; + } + assert( idxnSample ); + pSample = &pIdx->aSample[idx]; + pSample->nEq = (tRowcnt)sqlite3_column_int64(pStmt, 1); + pSample->nLt = (tRowcnt)sqlite3_column_int64(pStmt, 2); + pSample->nDLt = (tRowcnt)sqlite3_column_int64(pStmt, 3); + if( idx==pIdx->nSample-1 ){ + if( pSample->nDLt>0 ){ + for(i=0, sumEq=0; i<=idx-1; i++) sumEq += pIdx->aSample[i].nEq; + pIdx->avgEq = (pSample->nLt - sumEq)/pSample->nDLt; + } + if( pIdx->avgEq<=0 ) pIdx->avgEq = 1; + } + eType = sqlite3_column_type(pStmt, 4); + pSample->eType = (u8)eType; + switch( eType ){ + case SQLITE_INTEGER: { + pSample->u.i = sqlite3_column_int64(pStmt, 4); + break; + } + case SQLITE_FLOAT: { + pSample->u.r = sqlite3_column_double(pStmt, 4); + break; + } + case SQLITE_NULL: { + break; + } + default: assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); { + const char *z = (const char *)( + (eType==SQLITE_BLOB) ? + sqlite3_column_blob(pStmt, 4): + sqlite3_column_text(pStmt, 4) + ); + int n = z ? sqlite3_column_bytes(pStmt, 4) : 0; + pSample->nByte = n; + if( n < 1){ + pSample->u.z = 0; + }else{ + pSample->u.z = sqlite3Malloc(n); + if( pSample->u.z==0 ){ + db->mallocFailed = 1; + sqlite3_finalize(pStmt); + return SQLITE_NOMEM; + } + memcpy(pSample->u.z, z, n); + } + } + } + } + return sqlite3_finalize(pStmt); +} +#endif /* SQLITE_ENABLE_STAT3 */ + +/* +** Load the content of the sqlite_stat1 and sqlite_stat3 tables. The ** contents of sqlite_stat1 are used to populate the Index.aiRowEst[] -** arrays. The contents of sqlite_stat2 are used to populate the +** arrays. The contents of sqlite_stat3 are used to populate the ** Index.aSample[] arrays. ** ** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR -** is returned. In this case, even if SQLITE_ENABLE_STAT2 was defined -** during compilation and the sqlite_stat2 table is present, no data is +** is returned. In this case, even if SQLITE_ENABLE_STAT3 was defined +** during compilation and the sqlite_stat3 table is present, no data is ** read from it. ** -** If SQLITE_ENABLE_STAT2 was defined during compilation and the -** sqlite_stat2 table is not present in the database, SQLITE_ERROR is +** If SQLITE_ENABLE_STAT3 was defined during compilation and the +** sqlite_stat3 table is not present in the database, SQLITE_ERROR is ** returned. However, in this case, data is read from the sqlite_stat1 ** table (if it is present) before returning. ** @@ -78213,8 +78588,10 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3DefaultRowEst(pIdx); +#ifdef SQLITE_ENABLE_STAT3 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; +#endif } /* Check to make sure the sqlite_stat1 table exists */ @@ -78226,7 +78603,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load new statistics out of the sqlite_stat1 table */ zSql = sqlite3MPrintf(db, - "SELECT tbl, idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ @@ -78235,78 +78612,10 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ } - /* Load the statistics from the sqlite_stat2 table. */ -#ifdef SQLITE_ENABLE_STAT2 - if( rc==SQLITE_OK && !sqlite3FindTable(db, "sqlite_stat2", sInfo.zDatabase) ){ - rc = SQLITE_ERROR; - } + /* Load the statistics from the sqlite_stat3 table. */ +#ifdef SQLITE_ENABLE_STAT3 if( rc==SQLITE_OK ){ - sqlite3_stmt *pStmt = 0; - - zSql = sqlite3MPrintf(db, - "SELECT idx,sampleno,sample FROM %Q.sqlite_stat2", sInfo.zDatabase); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - sqlite3DbFree(db, zSql); - } - - if( rc==SQLITE_OK ){ - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - char *zIndex; /* Index name */ - Index *pIdx; /* Pointer to the index object */ - - zIndex = (char *)sqlite3_column_text(pStmt, 0); - pIdx = zIndex ? sqlite3FindIndex(db, zIndex, sInfo.zDatabase) : 0; - if( pIdx ){ - int iSample = sqlite3_column_int(pStmt, 1); - if( iSample=0 ){ - int eType = sqlite3_column_type(pStmt, 2); - - if( pIdx->aSample==0 ){ - static const int sz = sizeof(IndexSample)*SQLITE_INDEX_SAMPLES; - pIdx->aSample = (IndexSample *)sqlite3DbMallocRaw(0, sz); - if( pIdx->aSample==0 ){ - db->mallocFailed = 1; - break; - } - memset(pIdx->aSample, 0, sz); - } - - assert( pIdx->aSample ); - { - IndexSample *pSample = &pIdx->aSample[iSample]; - pSample->eType = (u8)eType; - if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - pSample->u.r = sqlite3_column_double(pStmt, 2); - }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ - const char *z = (const char *)( - (eType==SQLITE_BLOB) ? - sqlite3_column_blob(pStmt, 2): - sqlite3_column_text(pStmt, 2) - ); - int n = sqlite3_column_bytes(pStmt, 2); - if( n>24 ){ - n = 24; - } - pSample->nByte = (u8)n; - if( n < 1){ - pSample->u.z = 0; - }else{ - pSample->u.z = sqlite3DbStrNDup(0, z, n); - if( pSample->u.z==0 ){ - db->mallocFailed = 1; - break; - } - } - } - } - } - } - } - rc = sqlite3_finalize(pStmt); - } + rc = loadStat3(db, sInfo.zDatabase); } #endif @@ -81113,7 +81422,7 @@ static void destroyTable(Parse *pParse, Table *pTab){ } /* -** Remove entries from the sqlite_stat1 and sqlite_stat2 tables +** Remove entries from the sqlite_statN tables (for N in (1,2,3)) ** after a DROP INDEX or DROP TABLE command. */ static void sqlite3ClearStatTables( @@ -81122,19 +81431,90 @@ static void sqlite3ClearStatTables( const char *zType, /* "idx" or "tbl" */ const char *zName /* Name of index or table */ ){ - static const char *azStatTab[] = { "sqlite_stat1", "sqlite_stat2" }; int i; const char *zDbName = pParse->db->aDb[iDb].zName; - for(i=0; idb, azStatTab[i], zDbName) ){ + for(i=1; i<=3; i++){ + char zTab[24]; + sqlite3_snprintf(sizeof(zTab),zTab,"sqlite_stat%d",i); + if( sqlite3FindTable(pParse->db, zTab, zDbName) ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE %s=%Q", - zDbName, azStatTab[i], zType, zName + zDbName, zTab, zType, zName ); } } } +/* +** Generate code to drop a table. +*/ +SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ + Vdbe *v; + sqlite3 *db = pParse->db; + Trigger *pTrigger; + Db *pDb = &db->aDb[iDb]; + + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + sqlite3BeginWriteOperation(pParse, 1, iDb); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp0(v, OP_VBegin); + } +#endif + + /* Drop all triggers associated with the table being dropped. Code + ** is generated to remove entries from sqlite_master and/or + ** sqlite_temp_master if required. + */ + pTrigger = sqlite3TriggerList(pParse, pTab); + while( pTrigger ){ + assert( pTrigger->pSchema==pTab->pSchema || + pTrigger->pSchema==db->aDb[1].pSchema ); + sqlite3DropTriggerPtr(pParse, pTrigger); + pTrigger = pTrigger->pNext; + } + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* Remove any entries of the sqlite_sequence table associated with + ** the table being dropped. This is done before the table is dropped + ** at the btree level, in case the sqlite_sequence table needs to + ** move as a result of the drop (can happen in auto-vacuum mode). + */ + if( pTab->tabFlags & TF_Autoincrement ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.sqlite_sequence WHERE name=%Q", + pDb->zName, pTab->zName + ); + } +#endif + + /* Drop all SQLITE_MASTER table and index entries that refer to the + ** table. The program name loops through the master table and deletes + ** every row that refers to a table of the same name as the one being + ** dropped. Triggers are handled seperately because a trigger can be + ** created in the temp database that refers to a table in another + ** database. + */ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", + pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); + if( !isView && !IsVirtual(pTab) ){ + destroyTable(pParse, pTab); + } + + /* Remove the table entry from SQLite's internal schema and modify + ** the schema cookie. + */ + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0); + } + sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); + sqlite3ChangeCookie(pParse, iDb); + sqliteViewResetAll(db, iDb); +} + /* ** This routine is called to do the work of a DROP TABLE statement. ** pName is the name of the table to be dropped. @@ -81203,7 +81583,8 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, } } #endif - if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 + && sqlite3StrNICmp(pTab->zName, "sqlite_stat", 11)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); goto exit_drop_table; } @@ -81227,68 +81608,11 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, */ v = sqlite3GetVdbe(pParse); if( v ){ - Trigger *pTrigger; - Db *pDb = &db->aDb[iDb]; sqlite3BeginWriteOperation(pParse, 1, iDb); - -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pTab) ){ - sqlite3VdbeAddOp0(v, OP_VBegin); - } -#endif - sqlite3FkDropTable(pParse, pName, pTab); - - /* Drop all triggers associated with the table being dropped. Code - ** is generated to remove entries from sqlite_master and/or - ** sqlite_temp_master if required. - */ - pTrigger = sqlite3TriggerList(pParse, pTab); - while( pTrigger ){ - assert( pTrigger->pSchema==pTab->pSchema || - pTrigger->pSchema==db->aDb[1].pSchema ); - sqlite3DropTriggerPtr(pParse, pTrigger); - pTrigger = pTrigger->pNext; - } - -#ifndef SQLITE_OMIT_AUTOINCREMENT - /* Remove any entries of the sqlite_sequence table associated with - ** the table being dropped. This is done before the table is dropped - ** at the btree level, in case the sqlite_sequence table needs to - ** move as a result of the drop (can happen in auto-vacuum mode). - */ - if( pTab->tabFlags & TF_Autoincrement ){ - sqlite3NestedParse(pParse, - "DELETE FROM %s.sqlite_sequence WHERE name=%Q", - pDb->zName, pTab->zName - ); - } -#endif - - /* Drop all SQLITE_MASTER table and index entries that refer to the - ** table. The program name loops through the master table and deletes - ** every row that refers to a table of the same name as the one being - ** dropped. Triggers are handled seperately because a trigger can be - ** created in the temp database that refers to a table in another - ** database. - */ - sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", - pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); sqlite3ClearStatTables(pParse, iDb, "tbl", pTab->zName); - if( !isView && !IsVirtual(pTab) ){ - destroyTable(pParse, pTab); - } - - /* Remove the table entry from SQLite's internal schema and modify - ** the schema cookie. - */ - if( IsVirtual(pTab) ){ - sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0); - } - sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); - sqlite3ChangeCookie(pParse, iDb); + sqlite3FkDropTable(pParse, pName, pTab); + sqlite3CodeDropTable(pParse, pTab, iDb, isView); } - sqliteViewResetAll(db, iDb); exit_drop_table: sqlite3SrcListDelete(db, pName); @@ -81456,13 +81780,15 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ Table *pTab = pIndex->pTable; /* The table that is indexed */ int iTab = pParse->nTab++; /* Btree cursor used for pTab */ int iIdx = pParse->nTab++; /* Btree cursor used for pIndex */ - int iSorter = iTab; /* Cursor opened by OpenSorter (if in use) */ + int iSorter; /* Cursor opened by OpenSorter (if in use) */ int addr1; /* Address of top of loop */ int addr2; /* Address to jump to for next iteration */ int tnum; /* Root page of index */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ +#ifdef SQLITE_OMIT_MERGE_SORT int regIdxKey; /* Registers containing the index key */ +#endif int regRecord; /* Register holding assemblied index record */ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); @@ -81496,17 +81822,18 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); +#else + iSorter = iTab; #endif /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); - addr2 = addr1 + 1; regRecord = sqlite3GetTempReg(pParse); - regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); #ifndef SQLITE_OMIT_MERGE_SORT + sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); sqlite3VdbeJumpHere(v, addr1); @@ -81526,6 +81853,8 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); #else + regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); + addr2 = addr1 + 1; if( pIndex->onError!=OE_None ){ const int regRowid = regIdxKey + pIndex->nColumn; const int j2 = sqlite3VdbeCurrentAddr(v) + 2; @@ -81623,6 +81952,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( assert( pName1 && pName2 ); iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); if( iDb<0 ) goto exit_create_index; + assert( pName && pName->z ); #ifndef SQLITE_OMIT_TEMPDB /* If the index name was unqualified, check if the the table @@ -81650,6 +81980,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( assert( db->aDb[iDb].pSchema==pTab->pSchema ); }else{ assert( pName==0 ); + assert( pStart==0 ); pTab = pParse->pNewTable; if( !pTab ) goto exit_create_index; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -81692,6 +82023,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( if( pName ){ zName = sqlite3NameFromToken(db, pName); if( zName==0 ) goto exit_create_index; + assert( pName->z!=0 ); if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto exit_create_index; } @@ -81771,8 +82103,8 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( nCol = pList->nExpr; pIndex = sqlite3DbMallocZero(db, sizeof(Index) + /* Index structure */ + sizeof(tRowcnt)*(nCol+1) + /* Index.aiRowEst */ sizeof(int)*nCol + /* Index.aiColumn */ - sizeof(int)*(nCol+1) + /* Index.aiRowEst */ sizeof(char *)*nCol + /* Index.azColl */ sizeof(u8)*nCol + /* Index.aSortOrder */ nName + 1 + /* Index.zName */ @@ -81781,10 +82113,10 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( if( db->mallocFailed ){ goto exit_create_index; } - pIndex->azColl = (char**)(&pIndex[1]); + pIndex->aiRowEst = (tRowcnt*)(&pIndex[1]); + pIndex->azColl = (char**)(&pIndex->aiRowEst[nCol+1]); pIndex->aiColumn = (int *)(&pIndex->azColl[nCol]); - pIndex->aiRowEst = (unsigned *)(&pIndex->aiColumn[nCol]); - pIndex->aSortOrder = (u8 *)(&pIndex->aiRowEst[nCol+1]); + pIndex->aSortOrder = (u8 *)(&pIndex->aiColumn[nCol]); pIndex->zName = (char *)(&pIndex->aSortOrder[nCol]); zExtra = (char *)(&pIndex->zName[nName+1]); memcpy(pIndex->zName, zName, nName+1); @@ -82061,9 +82393,9 @@ exit_create_index: ** are based on typical values found in actual indices. */ SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ - unsigned *a = pIdx->aiRowEst; + tRowcnt *a = pIdx->aiRowEst; int i; - unsigned n; + tRowcnt n; assert( a!=0 ); a[0] = pIdx->pTable->nRowEst; if( a[0]<10 ) a[0] = 10; @@ -82547,13 +82879,10 @@ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ ** Commit a transaction */ SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ - sqlite3 *db; Vdbe *v; assert( pParse!=0 ); - db = pParse->db; - assert( db!=0 ); -/* if( db->aDb[0].pBt==0 ) return; */ + assert( pParse->db!=0 ); if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "COMMIT", 0, 0) ){ return; } @@ -82567,13 +82896,10 @@ SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ ** Rollback a transaction */ SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse *pParse){ - sqlite3 *db; Vdbe *v; assert( pParse!=0 ); - db = pParse->db; - assert( db!=0 ); -/* if( db->aDb[0].pBt==0 ) return; */ + assert( pParse->db!=0 ); if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "ROLLBACK", 0, 0) ){ return; } @@ -84379,16 +84705,15 @@ static void upperFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ if( z2 ){ z1 = contextMalloc(context, ((i64)n)+1); if( z1 ){ - memcpy(z1, z2, n+1); - for(i=0; z1[i]; i++){ - z1[i] = (char)sqlite3Toupper(z1[i]); + for(i=0; idb->flags & SQLITE_CountRows)!=0 ){ + return 0; + } /* If we get this far, it means either: ** @@ -88937,8 +89265,10 @@ struct sqlite3_api_routines { int (*busy_timeout)(sqlite3*,int ms); int (*changes)(sqlite3*); int (*close)(sqlite3*); - int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*,int eTextRep,const char*)); - int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*,int eTextRep,const void*)); + int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const char*)); + int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const void*)); const void * (*column_blob)(sqlite3_stmt*,int iCol); int (*column_bytes)(sqlite3_stmt*,int iCol); int (*column_bytes16)(sqlite3_stmt*,int iCol); @@ -88963,10 +89293,18 @@ struct sqlite3_api_routines { void * (*commit_hook)(sqlite3*,int(*)(void*),void*); int (*complete)(const char*sql); int (*complete16)(const void*sql); - int (*create_collation)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*)); - int (*create_collation16)(sqlite3*,const void*,int,void*,int(*)(void*,int,const void*,int,const void*)); - int (*create_function)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*)); - int (*create_function16)(sqlite3*,const void*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*)); + int (*create_collation)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_collation16)(sqlite3*,const void*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_function)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); + int (*create_function16)(sqlite3*,const void*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*); int (*data_count)(sqlite3_stmt*pStmt); sqlite3 * (*db_handle)(sqlite3_stmt*); @@ -89011,16 +89349,19 @@ struct sqlite3_api_routines { void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*)); void (*result_value)(sqlite3_context*,sqlite3_value*); void * (*rollback_hook)(sqlite3*,void(*)(void*),void*); - int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,const char*,const char*),void*); + int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, + const char*,const char*),void*); void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); char * (*snprintf)(int,char*,const char*,...); int (*step)(sqlite3_stmt*); - int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,char const**,char const**,int*,int*,int*); + int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, + char const**,char const**,int*,int*,int*); void (*thread_cleanup)(void); int (*total_changes)(sqlite3*); void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*); int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*); - void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*,sqlite_int64),void*); + void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*, + sqlite_int64),void*); void * (*user_data)(sqlite3_context*); const void * (*value_blob)(sqlite3_value*); int (*value_bytes)(sqlite3_value*); @@ -89042,15 +89383,19 @@ struct sqlite3_api_routines { int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); int (*clear_bindings)(sqlite3_stmt*); /* Added by 3.4.1 */ - int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,void (*xDestroy)(void *)); + int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*, + void (*xDestroy)(void *)); /* Added by 3.5.0 */ int (*bind_zeroblob)(sqlite3_stmt*,int,int); int (*blob_bytes)(sqlite3_blob*); int (*blob_close)(sqlite3_blob*); - int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,int,sqlite3_blob**); + int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64, + int,sqlite3_blob**); int (*blob_read)(sqlite3_blob*,void*,int,int); int (*blob_write)(sqlite3_blob*,const void*,int,int); - int (*create_collation_v2)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*),void(*)(void*)); + int (*create_collation_v2)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*), + void(*)(void*)); int (*file_control)(sqlite3*,const char*,int,void*); sqlite3_int64 (*memory_highwater)(int); sqlite3_int64 (*memory_used)(void); @@ -89086,7 +89431,11 @@ struct sqlite3_api_routines { int (*backup_step)(sqlite3_backup*,int); const char *(*compileoption_get)(int); int (*compileoption_used)(const char*); - int (*create_function_v2)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*),void(*xDestroy)(void*)); + int (*create_function_v2)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*)); int (*db_config)(sqlite3*,int,...); sqlite3_mutex *(*db_mutex)(sqlite3*); int (*db_status)(sqlite3*,int,int*,int*,int); @@ -89700,7 +90049,7 @@ static int sqlite3LoadExtension( int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); char *zErrmsg = 0; void **aHandle; - const int nMsg = 300; + int nMsg = 300 + sqlite3Strlen30(zFile); if( pzErrMsg ) *pzErrMsg = 0; @@ -89737,6 +90086,7 @@ static int sqlite3LoadExtension( sqlite3OsDlSym(pVfs, handle, zProc); if( xInit==0 ){ if( pzErrMsg ){ + nMsg += sqlite3Strlen30(zProc); *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, @@ -90422,7 +90772,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( sqlite3ReadSchema(pParse) ) goto pragma_out; sqlite3CodeVerifySchema(pParse, iDb); iReg = ++pParse->nMem; - if( zLeft[0]=='p' ){ + if( sqlite3Tolower(zLeft[0])=='p' ){ sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); }else{ sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, sqlite3Atoi(zRight)); @@ -90488,8 +90838,10 @@ SQLITE_PRIVATE void sqlite3Pragma( int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ int ii; /* Loop counter */ - /* Force the schema to be loaded on all databases. This cases all - ** database files to be opened and the journal_modes set. */ + /* Force the schema to be loaded on all databases. This causes all + ** database files to be opened and the journal_modes set. This is + ** necessary because subsequent processing must know if the databases + ** are in WAL mode. */ if( sqlite3ReadSchema(pParse) ){ goto pragma_out; } @@ -91033,7 +91385,7 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_ResultRow, 3, 1, 0}, }; - int isQuick = (zLeft[0]=='q'); + int isQuick = (sqlite3Tolower(zLeft[0])=='q'); /* Initialize the VDBE program */ if( sqlite3ReadSchema(pParse) ) goto pragma_out; @@ -92408,6 +92760,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); assert( db->mallocFailed || !pOffset || pLimit ); /* OFFSET implies LIMIT */ if( pNew==0 ){ + assert( db->mallocFailed ); pNew = &standin; memset(pNew, 0, sizeof(*pNew)); } @@ -92435,6 +92788,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( }else{ assert( pNew->pSrc!=0 || pParse->nErr>0 ); } + assert( pNew!=&standin ); return pNew; } @@ -93613,7 +93967,10 @@ static int selectColumnsFromExprList( }else{ Expr *pColExpr = p; /* The expression that is the result column name */ Table *pTab; /* Table associated with this expression */ - while( pColExpr->op==TK_DOT ) pColExpr = pColExpr->pRight; + while( pColExpr->op==TK_DOT ){ + pColExpr = pColExpr->pRight; + assert( pColExpr!=0 ); + } if( pColExpr->op==TK_COLUMN && ALWAYS(pColExpr->pTab!=0) ){ /* For columns use the column name name */ int iCol = pColExpr->iColumn; @@ -98611,6 +98968,7 @@ SQLITE_PRIVATE void sqlite3Update( } } for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + assert( aRegIdx ); if( openAll || aRegIdx[i]>0 ){ KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); sqlite3VdbeAddOp4(v, OP_OpenWrite, iCur+i+1, pIdx->tnum, iDb, @@ -98784,6 +99142,7 @@ SQLITE_PRIVATE void sqlite3Update( /* Close all tables */ for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + assert( aRegIdx ); if( openAll || aRegIdx[i]>0 ){ sqlite3VdbeAddOp2(v, OP_Close, iCur+i+1, 0); } @@ -98971,7 +99330,7 @@ static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ return sqlite3_errcode(db); } VVA_ONLY( rc = ) sqlite3_step(pStmt); - assert( rc!=SQLITE_ROW ); + assert( rc!=SQLITE_ROW || (db->flags&SQLITE_CountRows) ); return vacuumFinalize(db, pStmt, pzErrMsg); } @@ -99189,13 +99548,11 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ); if( rc ) goto end_of_vacuum; - /* At this point, unless the main db was completely empty, there is now a - ** transaction open on the vacuum database, but not on the main database. - ** Open a btree level transaction on the main database. This allows a - ** call to sqlite3BtreeCopyFile(). The main database btree level - ** transaction is then committed, so the SQL level never knows it was - ** opened for writing. This way, the SQL transaction used to create the - ** temporary database never needs to be committed. + /* At this point, there is a write transaction open on both the + ** vacuum database and the main database. Assuming no error occurs, + ** both transactions are closed by this block - the main database + ** transaction by sqlite3BtreeCopyFile() and the other by an explicit + ** call to sqlite3BtreeCommit(). */ { u32 meta; @@ -100164,7 +100521,7 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ for(i=0; rc==SQLITE_OK && inVTrans; i++){ VTable *pVTab = db->aVTrans[i]; const sqlite3_module *pMod = pVTab->pMod->pModule; - if( pMod->iVersion>=2 ){ + if( pVTab->pVtab && pMod->iVersion>=2 ){ int (*xMethod)(sqlite3_vtab *, int); switch( op ){ case SAVEPOINT_BEGIN: @@ -100179,7 +100536,7 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ break; } if( xMethod && pVTab->iSavepoint>iSavepoint ){ - rc = xMethod(db->aVTrans[i]->pVtab, iSavepoint); + rc = xMethod(pVTab->pVtab, iSavepoint); } } } @@ -100459,21 +100816,31 @@ struct WhereTerm { #define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */ #define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */ #define TERM_OR_OK 0x40 /* Used during OR-clause processing */ -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3 # define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */ #else -# define TERM_VNULL 0x00 /* Disabled if not using stat2 */ +# define TERM_VNULL 0x00 /* Disabled if not using stat3 */ #endif /* ** An instance of the following structure holds all information about a ** WHERE clause. Mostly this is a container for one or more WhereTerms. +** +** Explanation of pOuter: For a WHERE clause of the form +** +** a AND ((b AND c) OR (d AND e)) AND f +** +** There are separate WhereClause objects for the whole clause and for +** the subclauses "(b AND c)" and "(d AND e)". The pOuter field of the +** subclauses points to the WhereClause object for the whole clause. */ struct WhereClause { Parse *pParse; /* The parser context */ WhereMaskSet *pMaskSet; /* Mapping of table cursor numbers to bitmasks */ Bitmask vmask; /* Bitmask identifying virtual table cursors */ + WhereClause *pOuter; /* Outer conjunction */ u8 op; /* Split operator. TK_AND or TK_OR */ + u16 wctrlFlags; /* Might include WHERE_AND_ONLY */ int nTerm; /* Number of terms */ int nSlot; /* Number of entries in a[] */ WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ @@ -100602,14 +100969,17 @@ struct WhereCost { static void whereClauseInit( WhereClause *pWC, /* The WhereClause to be initialized */ Parse *pParse, /* The parsing context */ - WhereMaskSet *pMaskSet /* Mapping from table cursor numbers to bitmasks */ + WhereMaskSet *pMaskSet, /* Mapping from table cursor numbers to bitmasks */ + u16 wctrlFlags /* Might include WHERE_AND_ONLY */ ){ pWC->pParse = pParse; pWC->pMaskSet = pMaskSet; + pWC->pOuter = 0; pWC->nTerm = 0; pWC->nSlot = ArraySize(pWC->aStatic); pWC->a = pWC->aStatic; pWC->vmask = 0; + pWC->wctrlFlags = wctrlFlags; } /* Forward reference */ @@ -100925,36 +101295,38 @@ static WhereTerm *findTerm( int k; assert( iCur>=0 ); op &= WO_ALL; - for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ - if( pTerm->leftCursor==iCur - && (pTerm->prereqRight & notReady)==0 - && pTerm->u.leftColumn==iColumn - && (pTerm->eOperator & op)!=0 - ){ - if( pIdx && pTerm->eOperator!=WO_ISNULL ){ - Expr *pX = pTerm->pExpr; - CollSeq *pColl; - char idxaff; - int j; - Parse *pParse = pWC->pParse; - - idxaff = pIdx->pTable->aCol[iColumn].affinity; - if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue; - - /* Figure out the collation sequence required from an index for - ** it to be useful for optimising expression pX. Store this - ** value in variable pColl. - */ - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - assert(pColl || pParse->nErr); - - for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>=pIdx->nColumn) ) return 0; + for(; pWC; pWC=pWC->pOuter){ + for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ + if( pTerm->leftCursor==iCur + && (pTerm->prereqRight & notReady)==0 + && pTerm->u.leftColumn==iColumn + && (pTerm->eOperator & op)!=0 + ){ + if( pIdx && pTerm->eOperator!=WO_ISNULL ){ + Expr *pX = pTerm->pExpr; + CollSeq *pColl; + char idxaff; + int j; + Parse *pParse = pWC->pParse; + + idxaff = pIdx->pTable->aCol[iColumn].affinity; + if( !sqlite3IndexAffinityOk(pX, idxaff) ) continue; + + /* Figure out the collation sequence required from an index for + ** it to be useful for optimising expression pX. Store this + ** value in variable pColl. + */ + assert(pX->pLeft); + pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); + assert(pColl || pParse->nErr); + + for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ + if( NEVER(j>=pIdx->nColumn) ) return 0; + } + if( pColl && sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue; } - if( pColl && sqlite3StrICmp(pColl->zName, pIdx->azColl[j]) ) continue; + return pTerm; } - return pTerm; } } return 0; @@ -101031,7 +101403,7 @@ static int isLikeOrGlob( if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ z = (char *)sqlite3_value_text(pVal); } - sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); /* IMP: R-23257-02778 */ + sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); }else if( op==TK_STRING ){ z = pRight->u.zToken; @@ -101049,7 +101421,7 @@ static int isLikeOrGlob( *ppPrefix = pPrefix; if( op==TK_VARIABLE ){ Vdbe *v = pParse->pVdbe; - sqlite3VdbeSetVarmask(v, pRight->iColumn); /* IMP: R-23257-02778 */ + sqlite3VdbeSetVarmask(v, pRight->iColumn); if( *pisComplete && pRight->u.zToken[1] ){ /* If the rhs of the LIKE expression is a variable, and the current ** value of the variable means there is no need to invoke the LIKE @@ -101218,7 +101590,7 @@ static void exprAnalyzeOrTerm( if( pOrInfo==0 ) return; pTerm->wtFlags |= TERM_ORINFO; pOrWc = &pOrInfo->wc; - whereClauseInit(pOrWc, pWC->pParse, pMaskSet); + whereClauseInit(pOrWc, pWC->pParse, pMaskSet, pWC->wctrlFlags); whereSplit(pOrWc, pExpr, TK_OR); exprAnalyzeAll(pSrc, pOrWc); if( db->mallocFailed ) return; @@ -101245,9 +101617,10 @@ static void exprAnalyzeOrTerm( pOrTerm->wtFlags |= TERM_ANDINFO; pOrTerm->eOperator = WO_AND; pAndWC = &pAndInfo->wc; - whereClauseInit(pAndWC, pWC->pParse, pMaskSet); + whereClauseInit(pAndWC, pWC->pParse, pMaskSet, pWC->wctrlFlags); whereSplit(pAndWC, pOrTerm->pExpr, TK_AND); exprAnalyzeAll(pSrc, pAndWC); + pAndWC->pOuter = pWC; testcase( db->mallocFailed ); if( !db->mallocFailed ){ for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ @@ -101681,8 +102054,8 @@ static void exprAnalyze( } #endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifdef SQLITE_ENABLE_STAT2 - /* When sqlite_stat2 histogram data is available an operator of the +#ifdef SQLITE_ENABLE_STAT3 + /* When sqlite_stat3 histogram data is available an operator of the ** form "x IS NOT NULL" can sometimes be evaluated more efficiently ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a ** virtual term of that form. @@ -101720,7 +102093,7 @@ static void exprAnalyze( pNewTerm->prereqAll = pTerm->prereqAll; } } -#endif /* SQLITE_ENABLE_STAT2 */ +#endif /* SQLITE_ENABLE_STAT */ /* Prevent ON clause terms of a LEFT JOIN from being used to drive ** an index for tables to the left of the join. @@ -102142,11 +102515,14 @@ static void bestOrClauseIndex( WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */ WhereTerm *pTerm; /* A single term of the WHERE clause */ - /* No OR-clause optimization allowed if the INDEXED BY or NOT INDEXED clauses - ** are used */ + /* The OR-clause optimization is disallowed if the INDEXED BY or + ** NOT INDEXED clauses are used or if the WHERE_AND_ONLY bit is set. */ if( pSrc->notIndexed || pSrc->pIndex!=0 ){ return; } + if( pWC->wctrlFlags & WHERE_AND_ONLY ){ + return; + } /* Search the WHERE clause terms for a usable WO_OR term. */ for(pTerm=pWC->a; pTermpParse; tempWC.pMaskSet = pWC->pMaskSet; + tempWC.pOuter = pWC; tempWC.op = TK_AND; tempWC.a = pOrTerm; + tempWC.wctrlFlags = 0; tempWC.nTerm = 1; bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost); }else{ @@ -102768,67 +103146,85 @@ static void bestVirtualIndex( } #endif /* SQLITE_OMIT_VIRTUALTABLE */ +#ifdef SQLITE_ENABLE_STAT3 /* -** Argument pIdx is a pointer to an index structure that has an array of -** SQLITE_INDEX_SAMPLES evenly spaced samples of the first indexed column -** stored in Index.aSample. These samples divide the domain of values stored -** the index into (SQLITE_INDEX_SAMPLES+1) regions. -** Region 0 contains all values less than the first sample value. Region -** 1 contains values between the first and second samples. Region 2 contains -** values between samples 2 and 3. And so on. Region SQLITE_INDEX_SAMPLES -** contains values larger than the last sample. +** Estimate the location of a particular key among all keys in an +** index. Store the results in aStat as follows: ** -** If the index contains many duplicates of a single value, then it is -** possible that two or more adjacent samples can hold the same value. -** When that is the case, the smallest possible region code is returned -** when roundUp is false and the largest possible region code is returned -** when roundUp is true. +** aStat[0] Est. number of rows less than pVal +** aStat[1] Est. number of rows equal to pVal ** -** If successful, this function determines which of the regions value -** pVal lies in, sets *piRegion to the region index (a value between 0 -** and SQLITE_INDEX_SAMPLES+1, inclusive) and returns SQLITE_OK. -** Or, if an OOM occurs while converting text values between encodings, -** SQLITE_NOMEM is returned and *piRegion is undefined. +** Return SQLITE_OK on success. */ -#ifdef SQLITE_ENABLE_STAT2 -static int whereRangeRegion( +static int whereKeyStats( Parse *pParse, /* Database connection */ Index *pIdx, /* Index to consider domain of */ sqlite3_value *pVal, /* Value to consider */ - int roundUp, /* Return largest valid region if true */ - int *piRegion /* OUT: Region of domain in which value lies */ + int roundUp, /* Round up if true. Round down if false */ + tRowcnt *aStat /* OUT: stats written here */ ){ - assert( roundUp==0 || roundUp==1 ); - if( ALWAYS(pVal) ){ - IndexSample *aSample = pIdx->aSample; - int i = 0; - int eType = sqlite3_value_type(pVal); + tRowcnt n; + IndexSample *aSample; + int i, eType; + int isEq = 0; + i64 v; + double r, rS; - if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - double r = sqlite3_value_double(pVal); - for(i=0; i=SQLITE_TEXT ) break; - if( roundUp ){ - if( aSample[i].u.r>r ) break; - }else{ - if( aSample[i].u.r>=r ) break; + assert( roundUp==0 || roundUp==1 ); + assert( pIdx->nSample>0 ); + if( pVal==0 ) return SQLITE_ERROR; + n = pIdx->aiRowEst[0]; + aSample = pIdx->aSample; + eType = sqlite3_value_type(pVal); + + if( eType==SQLITE_INTEGER ){ + v = sqlite3_value_int64(pVal); + r = (i64)v; + for(i=0; inSample; i++){ + if( aSample[i].eType==SQLITE_NULL ) continue; + if( aSample[i].eType>=SQLITE_TEXT ) break; + if( aSample[i].eType==SQLITE_INTEGER ){ + if( aSample[i].u.i>=v ){ + isEq = aSample[i].u.i==v; + break; + } + }else{ + assert( aSample[i].eType==SQLITE_FLOAT ); + if( aSample[i].u.r>=r ){ + isEq = aSample[i].u.r==r; + break; } } - }else if( eType==SQLITE_NULL ){ - i = 0; - if( roundUp ){ - while( inSample; i++){ + if( aSample[i].eType==SQLITE_NULL ) continue; + if( aSample[i].eType>=SQLITE_TEXT ) break; + if( aSample[i].eType==SQLITE_FLOAT ){ + rS = aSample[i].u.r; + }else{ + rS = aSample[i].u.i; } - }else{ + if( rS>=r ){ + isEq = rS==r; + break; + } + } + }else if( eType==SQLITE_NULL ){ + i = 0; + if( aSample[0].eType==SQLITE_NULL ) isEq = 1; + }else{ + assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); + for(i=0; inSample; i++){ + if( aSample[i].eType==SQLITE_TEXT || aSample[i].eType==SQLITE_BLOB ){ + break; + } + } + if( inSample ){ sqlite3 *db = pParse->db; CollSeq *pColl; const u8 *z; - int n; - - /* pVal comes from sqlite3ValueFromExpr() so the type cannot be NULL */ - assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); - if( eType==SQLITE_BLOB ){ z = (const u8 *)sqlite3_value_blob(pVal); pColl = db->pDfltColl; @@ -102847,12 +103243,12 @@ static int whereRangeRegion( assert( z && pColl && pColl->xCmp ); } n = sqlite3ValueBytes(pVal, pColl->enc); - - for(i=0; inSample; i++){ int c; int eSampletype = aSample[i].eType; - if( eSampletype==SQLITE_NULL || eSampletypeenc!=SQLITE_UTF8 ){ int nSample; @@ -102870,16 +103266,47 @@ static int whereRangeRegion( { c = pColl->xCmp(pColl->pUser, aSample[i].nByte, aSample[i].u.z, n, z); } - if( c-roundUp>=0 ) break; + if( c>=0 ){ + if( c==0 ) isEq = 1; + break; + } } } + } - assert( i>=0 && i<=SQLITE_INDEX_SAMPLES ); - *piRegion = i; + /* At this point, aSample[i] is the first sample that is greater than + ** or equal to pVal. Or if i==pIdx->nSample, then all samples are less + ** than pVal. If aSample[i]==pVal, then isEq==1. + */ + if( isEq ){ + assert( inSample ); + aStat[0] = aSample[i].nLt; + aStat[1] = aSample[i].nEq; + }else{ + tRowcnt iLower, iUpper, iGap; + if( i==0 ){ + iLower = 0; + iUpper = aSample[0].nLt; + }else{ + iUpper = i>=pIdx->nSample ? n : aSample[i].nLt; + iLower = aSample[i-1].nEq + aSample[i-1].nLt; + } + aStat[1] = pIdx->avgEq; + if( iLower>=iUpper ){ + iGap = 0; + }else{ + iGap = iUpper - iLower; + } + if( roundUp ){ + iGap = (iGap*2)/3; + }else{ + iGap = iGap/3; + } + aStat[0] = iLower + iGap; } return SQLITE_OK; } -#endif /* #ifdef SQLITE_ENABLE_STAT2 */ +#endif /* SQLITE_ENABLE_STAT3 */ /* ** If expression pExpr represents a literal value, set *pp to point to @@ -102897,7 +103324,7 @@ static int whereRangeRegion( ** ** If an error occurs, return an error code. Otherwise, SQLITE_OK. */ -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3 static int valueFromExpr( Parse *pParse, Expr *pExpr, @@ -102908,7 +103335,7 @@ static int valueFromExpr( || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) ){ int iVar = pExpr->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); /* IMP: R-23257-02778 */ + sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); *pp = sqlite3VdbeGetValue(pParse->pReprepare, iVar, aff); return SQLITE_OK; } @@ -102945,17 +103372,15 @@ static int valueFromExpr( ** ** then nEq should be passed 0. ** -** The returned value is an integer between 1 and 100, inclusive. A return -** value of 1 indicates that the proposed range scan is expected to visit -** approximately 1/100th (1%) of the rows selected by the nEq equality -** constraints (if any). A return value of 100 indicates that it is expected -** that the range scan will visit every row (100%) selected by the equality -** constraints. +** The returned value is an integer divisor to reduce the estimated +** search space. A return value of 1 means that range constraints are +** no help at all. A return value of 2 means range constraints are +** expected to reduce the search space by half. And so forth... ** -** In the absence of sqlite_stat2 ANALYZE data, each range inequality -** reduces the search space by 3/4ths. Hence a single constraint (x>?) -** results in a return of 25 and a range constraint (x>? AND x?) +** results in a return of 4 and a range constraint (x>? AND xaCol[] of the range-compared column */ WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - int *piEst /* OUT: Return value */ + double *pRangeDiv /* OUT: Reduce search space by this divisor */ ){ int rc = SQLITE_OK; -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3 - if( nEq==0 && p->aSample ){ - sqlite3_value *pLowerVal = 0; - sqlite3_value *pUpperVal = 0; - int iEst; - int iLower = 0; - int iUpper = SQLITE_INDEX_SAMPLES; - int roundUpUpper = 0; - int roundUpLower = 0; + if( nEq==0 && p->nSample ){ + sqlite3_value *pRangeVal; + tRowcnt iLower = 0; + tRowcnt iUpper = p->aiRowEst[0]; + tRowcnt a[2]; u8 aff = p->pTable->aCol[p->aiColumn[0]].affinity; if( pLower ){ Expr *pExpr = pLower->pExpr->pRight; - rc = valueFromExpr(pParse, pExpr, aff, &pLowerVal); + rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); assert( pLower->eOperator==WO_GT || pLower->eOperator==WO_GE ); - roundUpLower = (pLower->eOperator==WO_GT) ?1:0; + if( rc==SQLITE_OK + && whereKeyStats(pParse, p, pRangeVal, 0, a)==SQLITE_OK + ){ + iLower = a[0]; + if( pLower->eOperator==WO_GT ) iLower += a[1]; + } + sqlite3ValueFree(pRangeVal); } if( rc==SQLITE_OK && pUpper ){ Expr *pExpr = pUpper->pExpr->pRight; - rc = valueFromExpr(pParse, pExpr, aff, &pUpperVal); + rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); assert( pUpper->eOperator==WO_LT || pUpper->eOperator==WO_LE ); - roundUpUpper = (pUpper->eOperator==WO_LE) ?1:0; - } - - if( rc!=SQLITE_OK || (pLowerVal==0 && pUpperVal==0) ){ - sqlite3ValueFree(pLowerVal); - sqlite3ValueFree(pUpperVal); - goto range_est_fallback; - }else if( pLowerVal==0 ){ - rc = whereRangeRegion(pParse, p, pUpperVal, roundUpUpper, &iUpper); - if( pLower ) iLower = iUpper/2; - }else if( pUpperVal==0 ){ - rc = whereRangeRegion(pParse, p, pLowerVal, roundUpLower, &iLower); - if( pUpper ) iUpper = (iLower + SQLITE_INDEX_SAMPLES + 1)/2; - }else{ - rc = whereRangeRegion(pParse, p, pUpperVal, roundUpUpper, &iUpper); - if( rc==SQLITE_OK ){ - rc = whereRangeRegion(pParse, p, pLowerVal, roundUpLower, &iLower); + if( rc==SQLITE_OK + && whereKeyStats(pParse, p, pRangeVal, 1, a)==SQLITE_OK + ){ + iUpper = a[0]; + if( pUpper->eOperator==WO_LE ) iUpper += a[1]; } + sqlite3ValueFree(pRangeVal); } - WHERETRACE(("range scan regions: %d..%d\n", iLower, iUpper)); - - iEst = iUpper - iLower; - testcase( iEst==SQLITE_INDEX_SAMPLES ); - assert( iEst<=SQLITE_INDEX_SAMPLES ); - if( iEst<1 ){ - *piEst = 50/SQLITE_INDEX_SAMPLES; - }else{ - *piEst = (iEst*100)/SQLITE_INDEX_SAMPLES; + if( rc==SQLITE_OK ){ + if( iUpper<=iLower ){ + *pRangeDiv = (double)p->aiRowEst[0]; + }else{ + *pRangeDiv = (double)p->aiRowEst[0]/(double)(iUpper - iLower); + } + WHERETRACE(("range scan regions: %u..%u div=%g\n", + (u32)iLower, (u32)iUpper, *pRangeDiv)); + return SQLITE_OK; } - sqlite3ValueFree(pLowerVal); - sqlite3ValueFree(pUpperVal); - return rc; } -range_est_fallback: #else UNUSED_PARAMETER(pParse); UNUSED_PARAMETER(p); UNUSED_PARAMETER(nEq); #endif assert( pLower || pUpper ); - *piEst = 100; - if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ) *piEst /= 4; - if( pUpper ) *piEst /= 4; + *pRangeDiv = (double)1; + if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ) *pRangeDiv *= (double)4; + if( pUpper ) *pRangeDiv *= (double)4; return rc; } -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3 /* ** Estimate the number of rows that will be returned based on ** an equality constraint x=VALUE and where that VALUE occurs in ** the histogram data. This only works when x is the left-most -** column of an index and sqlite_stat2 histogram data is available +** column of an index and sqlite_stat3 histogram data is available ** for that index. When pExpr==NULL that means the constraint is ** "x IS NULL" instead of "x=VALUE". ** @@ -103060,12 +103473,12 @@ static int whereEqualScanEst( double *pnRow /* Write the revised row estimate here */ ){ sqlite3_value *pRhs = 0; /* VALUE on right-hand side of pTerm */ - int iLower, iUpper; /* Range of histogram regions containing pRhs */ u8 aff; /* Column affinity */ int rc; /* Subfunction return code */ - double nRowEst; /* New estimate of the number of rows */ + tRowcnt a[2]; /* Statistics */ assert( p->aSample!=0 ); + assert( p->nSample>0 ); aff = p->pTable->aCol[p->aiColumn[0]].affinity; if( pExpr ){ rc = valueFromExpr(pParse, pExpr, aff, &pRhs); @@ -103074,26 +103487,18 @@ static int whereEqualScanEst( pRhs = sqlite3ValueNew(pParse->db); } if( pRhs==0 ) return SQLITE_NOTFOUND; - rc = whereRangeRegion(pParse, p, pRhs, 0, &iLower); - if( rc ) goto whereEqualScanEst_cancel; - rc = whereRangeRegion(pParse, p, pRhs, 1, &iUpper); - if( rc ) goto whereEqualScanEst_cancel; - WHERETRACE(("equality scan regions: %d..%d\n", iLower, iUpper)); - if( iLower>=iUpper ){ - nRowEst = p->aiRowEst[0]/(SQLITE_INDEX_SAMPLES*2); - if( nRowEst<*pnRow ) *pnRow = nRowEst; - }else{ - nRowEst = (iUpper-iLower)*p->aiRowEst[0]/SQLITE_INDEX_SAMPLES; - *pnRow = nRowEst; + rc = whereKeyStats(pParse, p, pRhs, 0, a); + if( rc==SQLITE_OK ){ + WHERETRACE(("equality scan regions: %d\n", (int)a[1])); + *pnRow = a[1]; } - whereEqualScanEst_cancel: sqlite3ValueFree(pRhs); return rc; } -#endif /* defined(SQLITE_ENABLE_STAT2) */ +#endif /* defined(SQLITE_ENABLE_STAT3) */ -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3 /* ** Estimate the number of rows that will be returned based on ** an IN constraint where the right-hand side of the IN operator @@ -103116,60 +103521,25 @@ static int whereInScanEst( ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ double *pnRow /* Write the revised row estimate here */ ){ - sqlite3_value *pVal = 0; /* One value from list */ - int iLower, iUpper; /* Range of histogram regions containing pRhs */ - u8 aff; /* Column affinity */ - int rc = SQLITE_OK; /* Subfunction return code */ - double nRowEst; /* New estimate of the number of rows */ - int nSpan = 0; /* Number of histogram regions spanned */ - int nSingle = 0; /* Histogram regions hit by a single value */ - int nNotFound = 0; /* Count of values that are not constants */ - int i; /* Loop counter */ - u8 aSpan[SQLITE_INDEX_SAMPLES+1]; /* Histogram regions that are spanned */ - u8 aSingle[SQLITE_INDEX_SAMPLES+1]; /* Histogram regions hit once */ + int rc = SQLITE_OK; /* Subfunction return code */ + double nEst; /* Number of rows for a single term */ + double nRowEst = (double)0; /* New estimate of the number of rows */ + int i; /* Loop counter */ assert( p->aSample!=0 ); - aff = p->pTable->aCol[p->aiColumn[0]].affinity; - memset(aSpan, 0, sizeof(aSpan)); - memset(aSingle, 0, sizeof(aSingle)); - for(i=0; inExpr; i++){ - sqlite3ValueFree(pVal); - rc = valueFromExpr(pParse, pList->a[i].pExpr, aff, &pVal); - if( rc ) break; - if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){ - nNotFound++; - continue; - } - rc = whereRangeRegion(pParse, p, pVal, 0, &iLower); - if( rc ) break; - rc = whereRangeRegion(pParse, p, pVal, 1, &iUpper); - if( rc ) break; - if( iLower>=iUpper ){ - aSingle[iLower] = 1; - }else{ - assert( iLower>=0 && iUpper<=SQLITE_INDEX_SAMPLES ); - while( iLowernExpr; i++){ + nEst = p->aiRowEst[0]; + rc = whereEqualScanEst(pParse, p, pList->a[i].pExpr, &nEst); + nRowEst += nEst; } if( rc==SQLITE_OK ){ - for(i=nSpan=0; i<=SQLITE_INDEX_SAMPLES; i++){ - if( aSpan[i] ){ - nSpan++; - }else if( aSingle[i] ){ - nSingle++; - } - } - nRowEst = (nSpan*2+nSingle)*p->aiRowEst[0]/(2*SQLITE_INDEX_SAMPLES) - + nNotFound*p->aiRowEst[1]; if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; *pnRow = nRowEst; - WHERETRACE(("IN row estimate: nSpan=%d, nSingle=%d, nNotFound=%d, est=%g\n", - nSpan, nSingle, nNotFound, nRowEst)); + WHERETRACE(("IN row estimate: est=%g\n", nRowEst)); } - sqlite3ValueFree(pVal); return rc; } -#endif /* defined(SQLITE_ENABLE_STAT2) */ +#endif /* defined(SQLITE_ENABLE_STAT3) */ /* @@ -103216,7 +103586,7 @@ static void bestBtreeIndex( int eqTermMask; /* Current mask of valid equality operators */ int idxEqTermMask; /* Index mask of valid equality operators */ Index sPk; /* A fake index object for the primary key */ - unsigned int aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ + tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ int wsFlagMask; /* Allowed flags in pCost->plan.wsFlag */ @@ -103271,10 +103641,10 @@ static void bestBtreeIndex( /* Loop over all indices looking for the best one to use */ for(; pProbe; pIdx=pProbe=pProbe->pNext){ - const unsigned int * const aiRowEst = pProbe->aiRowEst; + const tRowcnt * const aiRowEst = pProbe->aiRowEst; double cost; /* Cost of using pProbe */ double nRow; /* Estimated number of rows in result set */ - double log10N; /* base-10 logarithm of nRow (inexact) */ + double log10N = (double)1; /* base-10 logarithm of nRow (inexact) */ int rev; /* True to scan in reverse order */ int wsFlags = 0; Bitmask used = 0; @@ -103314,14 +103684,12 @@ static void bestBtreeIndex( ** IN operator must be a SELECT, not a value list, for this variable ** to be true. ** - ** estBound: - ** An estimate on the amount of the table that must be searched. A - ** value of 100 means the entire table is searched. Range constraints - ** might reduce this to a value less than 100 to indicate that only - ** a fraction of the table needs searching. In the absence of - ** sqlite_stat2 ANALYZE data, a single inequality reduces the search - ** space to 1/4rd its original size. So an x>? constraint reduces - ** estBound to 25. Two constraints (x>? AND xpWC!=pWC ); if( pTerm->eOperator & WO_IN ){ Expr *pExpr = pTerm->pExpr; wsFlags |= WHERE_COLUMN_IN; @@ -103376,28 +103745,30 @@ static void bestBtreeIndex( }else if( pTerm->eOperator & WO_ISNULL ){ wsFlags |= WHERE_COLUMN_NULL; } -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3 if( nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; #endif used |= pTerm->prereqRight; } - /* Determine the value of estBound. */ + /* Determine the value of rangeDiv */ if( nEqnColumn && pProbe->bUnordered==0 ){ int j = pProbe->aiColumn[nEq]; if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pIdx); WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pIdx); - whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &estBound); + whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &rangeDiv); if( pTop ){ nBound = 1; wsFlags |= WHERE_TOP_LIMIT; used |= pTop->prereqRight; + testcase( pTop->pWC!=pWC ); } if( pBtm ){ nBound++; wsFlags |= WHERE_BTM_LIMIT; used |= pBtm->prereqRight; + testcase( pBtm->pWC!=pWC ); } wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); } @@ -103460,7 +103831,7 @@ static void bestBtreeIndex( nInMul = (int)(nRow / aiRowEst[nEq]); } -#ifdef SQLITE_ENABLE_STAT2 +#ifdef SQLITE_ENABLE_STAT3 /* If the constraint is of the form x=VALUE or x IN (E1,E2,...) ** and we do not think that values of x are unique and if histogram ** data is available for column x, then it might be possible @@ -103468,20 +103839,22 @@ static void bestBtreeIndex( ** VALUE and how common that value is according to the histogram. */ if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 && aiRowEst[1]>1 ){ + assert( (pFirstTerm->eOperator & (WO_EQ|WO_ISNULL|WO_IN))!=0 ); if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ testcase( pFirstTerm->eOperator==WO_EQ ); testcase( pFirstTerm->eOperator==WO_ISNULL ); whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow); - }else if( pFirstTerm->eOperator==WO_IN && bInEst==0 ){ + }else if( bInEst==0 ){ + assert( pFirstTerm->eOperator==WO_IN ); whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &nRow); } } -#endif /* SQLITE_ENABLE_STAT2 */ +#endif /* SQLITE_ENABLE_STAT3 */ /* Adjust the number of output rows and downward to reflect rows ** that are excluded by range constraints. */ - nRow = (nRow * (double)estBound) / (double)100; + nRow = nRow/rangeDiv; if( nRow<1 ) nRow = 1; /* Experiments run on real SQLite databases show that the time needed @@ -103492,7 +103865,7 @@ static void bestBtreeIndex( ** slower with larger records, presumably because fewer records fit ** on one page and hence more pages have to be fetched. ** - ** The ANALYZE command and the sqlite_stat1 and sqlite_stat2 tables do + ** The ANALYZE command and the sqlite_stat1 and sqlite_stat3 tables do ** not give us data on the relative sizes of table and index records. ** So this computation assumes table records are about twice as big ** as index records @@ -103610,10 +103983,10 @@ static void bestBtreeIndex( WHERETRACE(( - "%s(%s): nEq=%d nInMul=%d estBound=%d bSort=%d bLookup=%d wsFlags=0x%x\n" + "%s(%s): nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%x\n" " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f used=0x%llx\n", pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), - nEq, nInMul, estBound, bSort, bLookup, wsFlags, + nEq, nInMul, (int)rangeDiv, bSort, bLookup, wsFlags, notReady, log10N, nRow, cost, used )); @@ -104117,7 +104490,8 @@ static Bitmask codeOneLoopStart( WhereInfo *pWInfo, /* Complete information about the WHERE clause */ int iLevel, /* Which level of pWInfo->a[] should be coded */ u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ - Bitmask notReady /* Which tables are currently available */ + Bitmask notReady, /* Which tables are currently available */ + Expr *pWhere /* Complete WHERE clause */ ){ int j, k; /* Loop counters */ int iCur; /* The VDBE cursor for the table */ @@ -104599,7 +104973,8 @@ static Bitmask codeOneLoopStart( int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ int iRetInit; /* Address of regReturn init */ int untestedTerms = 0; /* Some terms not completely tested */ - int ii; + int ii; /* Loop counter */ + Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ pTerm = pLevel->plan.u.pTerm; assert( pTerm!=0 ); @@ -104649,13 +105024,28 @@ static Bitmask codeOneLoopStart( } iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); + /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y + ** Then for every term xN, evaluate as the subexpression: xN AND z + ** That way, terms in y that are factored into the disjunction will + ** be picked up by the recursive calls to sqlite3WhereBegin() below. + */ + if( pWC->nTerm>1 ){ + pAndExpr = sqlite3ExprAlloc(pParse->db, TK_AND, 0, 0); + pAndExpr->pRight = pWhere; + } + for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ + Expr *pOrExpr = pOrTerm->pExpr; + if( pAndExpr ){ + pAndExpr->pLeft = pOrExpr; + pOrExpr = pAndExpr; + } /* Loop through table entries that match term pOrTerm. */ - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, 0, - WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, + WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY); if( pSubWInfo ){ explainOneScan( @@ -104683,6 +105073,7 @@ static Bitmask codeOneLoopStart( } } } + sqlite3DbFree(pParse->db, pAndExpr); sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); sqlite3VdbeResolveLabel(v, iLoopBody); @@ -104964,7 +105355,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); - whereClauseInit(pWC, pParse, pMaskSet); + whereClauseInit(pWC, pParse, pMaskSet, wctrlFlags); sqlite3ExprCodeConstants(pParse, pWhere); whereSplit(pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ @@ -105203,7 +105594,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( WHERETRACE(("*** Optimizer selects table %d for loop %d" " with cost=%g and nRow=%g\n", bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow)); - if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ + /* The ALWAYS() that follows was added to hush up clang scan-build */ + if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 && ALWAYS(ppOrderBy) ){ *ppOrderBy = 0; } if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){ @@ -105292,7 +105684,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( }else #endif if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 - && (wctrlFlags & WHERE_OMIT_OPEN)==0 ){ + && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ int op = pWInfo->okOnePass ? OP_OpenWrite : OP_OpenRead; sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); testcase( pTab->nCol==BMS-1 ); @@ -105337,7 +105729,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( for(i=0; ia[i]; explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags); - notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady); + notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady, pWhere); pWInfo->iContinue = pLevel->addrCont; } @@ -105472,7 +105864,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ assert( pTab!=0 ); if( (pTab->tabFlags & TF_Ephemeral)==0 && pTab->pSelect==0 - && (pWInfo->wctrlFlags & WHERE_OMIT_CLOSE)==0 + && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ int ws = pLevel->plan.wsFlags; if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){ @@ -108819,7 +109211,9 @@ SQLITE_PRIVATE void sqlite3Parser( ){ YYMINORTYPE yyminorunion; int yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) int yyendofinput; /* True if we are at the end of input */ +#endif #ifdef YYERRORSYMBOL int yyerrorhit = 0; /* True if yymajor has invoked an error */ #endif @@ -108842,7 +109236,9 @@ SQLITE_PRIVATE void sqlite3Parser( yypParser->yystack[0].major = 0; } yyminorunion.yy0 = yyminor; +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) yyendofinput = (yymajor==0); +#endif sqlite3ParserARG_STORE; #ifndef NDEBUG @@ -108854,7 +109250,6 @@ SQLITE_PRIVATE void sqlite3Parser( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); if( yyactyyerrcnt--; yymajor = YYNOCODE; @@ -110246,7 +110641,7 @@ SQLITE_API char *sqlite3_temp_directory = 0; ** without blocking. */ SQLITE_API int sqlite3_initialize(void){ - sqlite3_mutex *pMaster; /* The main static mutex */ + MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ int rc; /* Result code */ #ifdef SQLITE_OMIT_WSD @@ -110280,7 +110675,7 @@ SQLITE_API int sqlite3_initialize(void){ ** malloc subsystem - this implies that the allocation of a static ** mutex must not require support from the malloc subsystem. */ - pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(pMaster); sqlite3GlobalConfig.isMutexInit = 1; if( !sqlite3GlobalConfig.isMallocInit ){ @@ -111354,13 +111749,13 @@ SQLITE_API int sqlite3_overload_function( int nArg ){ int nName = sqlite3Strlen30(zName); - int rc; + int rc = SQLITE_OK; sqlite3_mutex_enter(db->mutex); if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ - sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, - 0, sqlite3InvalidFunction, 0, 0, 0); + rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, + 0, sqlite3InvalidFunction, 0, 0, 0); } - rc = sqlite3ApiExit(db, SQLITE_OK); + rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } @@ -112422,6 +112817,7 @@ opendb_out: sqlite3_mutex_leave(db->mutex); } rc = sqlite3_errcode(db); + assert( db!=0 || rc==SQLITE_NOMEM ); if( rc==SQLITE_NOMEM ){ sqlite3_close(db); db = 0; @@ -114151,6 +114547,13 @@ typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ #endif /* SQLITE_AMALGAMATION */ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); +# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() +#else +# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB +#endif + typedef struct Fts3Table Fts3Table; typedef struct Fts3Cursor Fts3Cursor; typedef struct Fts3Expr Fts3Expr; @@ -114178,6 +114581,7 @@ struct Fts3Table { int nColumn; /* number of named columns in virtual table */ char **azColumn; /* column names. malloced */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + char *zContentTbl; /* content=xxx option, or NULL */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. @@ -114218,7 +114622,7 @@ struct Fts3Table { int nPendingData; /* Current bytes of pending data */ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ -#if defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* State variables used for validating that the transaction control ** methods of the virtual table are called at appropriate times. These ** values do not contribution to the FTS computation; they are used for @@ -114303,6 +114707,7 @@ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ /* Variables above this point are populated when the expression is ** parsed (by code in fts3_expr.c). Below this point the variables are @@ -114421,6 +114826,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 #define FTS3_SEGMENT_PREFIX 0x00000008 #define FTS3_SEGMENT_SCAN 0x00000010 +#define FTS3_SEGMENT_FIRST 0x00000020 /* Type passed as 4th argument to SegmentReaderIterate() */ struct Fts3SegFilter { @@ -114460,8 +114866,8 @@ SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); - SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); +SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); /* fts3_tokenizer.c */ SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); @@ -114480,7 +114886,7 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const /* fts3_expr.c */ SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, - char **, int, int, const char *, int, Fts3Expr ** + char **, int, int, int, const char *, int, Fts3Expr ** ); SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST @@ -114651,7 +115057,7 @@ static void fts3GetReverseVarint( sqlite3_int64 *pVal ){ sqlite3_int64 iVal; - char *p = *pp; + char *p; /* Pointer p now points at the first byte past the varint we are ** interested in. So, unless the doclist is corrupt, the 0x80 bit is @@ -114681,6 +115087,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ sqlite3_free(p->zSegmentsTbl); sqlite3_free(p->zReadExprlist); sqlite3_free(p->zWriteExprlist); + sqlite3_free(p->zContentTbl); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); @@ -114720,16 +115127,19 @@ static void fts3DbExec( ** The xDestroy() virtual table method. */ static int fts3DestroyMethod(sqlite3_vtab *pVtab){ - int rc = SQLITE_OK; /* Return code */ Fts3Table *p = (Fts3Table *)pVtab; - sqlite3 *db = p->db; + int rc = SQLITE_OK; /* Return code */ + const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ + sqlite3 *db = p->db; /* Database handle */ /* Drop the shadow tables */ - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", p->zDb,p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", p->zDb, p->zName); + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); + } + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); /* If everything has worked, invoke fts3DisconnectMethod() to free the ** memory associated with the Fts3Table structure and return SQLITE_OK. @@ -114791,23 +115201,27 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ static int fts3CreateTables(Fts3Table *p){ int rc = SQLITE_OK; /* Return code */ int i; /* Iterator variable */ - char *zContentCols; /* Columns of %_content table */ sqlite3 *db = p->db; /* The database connection */ - /* Create a list of user columns for the content table */ - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); - for(i=0; zContentCols && inColumn; i++){ - char *z = p->azColumn[i]; - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); - } - if( zContentCols==0 ) rc = SQLITE_NOMEM; + if( p->zContentTbl==0 ){ + char *zContentCols; /* Columns of %_content table */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + } - /* Create the content table */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_content'(%s)", - p->zDb, p->zName, zContentCols - ); - sqlite3_free(zContentCols); /* Create other tables */ fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", @@ -114958,8 +115372,8 @@ static char *fts3QuoteId(char const *zInput){ } /* -** Return a list of comma separated SQL expressions that could be used -** in a SELECT statement such as the following: +** Return a list of comma separated SQL expressions and a FROM clause that +** could be used in a SELECT statement such as the following: ** ** SELECT FROM %_content AS x ... ** @@ -114970,7 +115384,7 @@ static char *fts3QuoteId(char const *zInput){ ** table has the three user-defined columns "a", "b", and "c", the following ** string is returned: ** -** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')" +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" ** ** The pointer returned points to a buffer allocated by sqlite3_malloc(). It ** is the responsibility of the caller to eventually free it. @@ -114986,16 +115400,28 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ char *zFunction; int i; - if( !zFunc ){ - zFunction = ""; + if( p->zContentTbl==0 ){ + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); + } + sqlite3_free(zFree); }else{ - zFree = zFunction = fts3QuoteId(zFunc); + fts3Appendf(pRc, &zRet, "rowid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + } } - fts3Appendf(pRc, &zRet, "docid"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); - } - sqlite3_free(zFree); + fts3Appendf(pRc, &zRet, "FROM '%q'.'%q%s' AS x", + p->zDb, + (p->zContentTbl ? p->zContentTbl : p->zName), + (p->zContentTbl ? "" : "_content") + ); return zRet; } @@ -115052,7 +115478,7 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ ** This function is used when parsing the "prefix=" FTS4 parameter. */ static int fts3GobbleInt(const char **pp, int *pnOut){ - const char *p = *pp; /* Iterator pointer */ + const char *p; /* Iterator pointer */ int nInt = 0; /* Output value */ for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ @@ -115119,6 +115545,91 @@ static int fts3PrefixParameter( return SQLITE_OK; } +/* +** This function is called when initializing an FTS4 table that uses the +** content=xxx option. It determines the number of and names of the columns +** of the new FTS4 table. +** +** The third argument passed to this function is the value passed to the +** config=xxx option (i.e. "xxx"). This function queries the database for +** a table of that name. If found, the output variables are populated +** as follows: +** +** *pnCol: Set to the number of columns table xxx has, +** +** *pnStr: Set to the total amount of space required to store a copy +** of each columns name, including the nul-terminator. +** +** *pazCol: Set to point to an array of *pnCol strings. Each string is +** the name of the corresponding column in table xxx. The array +** and its contents are allocated using a single allocation. It +** is the responsibility of the caller to free this allocation +** by eventually passing the *pazCol value to sqlite3_free(). +** +** If the table cannot be found, an error code is returned and the output +** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is +** returned (and the output variables are undefined). +*/ +static int fts3ContentColumns( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ + const char *zTbl, /* Name of content table */ + const char ***pazCol, /* OUT: Malloc'd array of column names */ + int *pnCol, /* OUT: Size of array *pazCol */ + int *pnStr /* OUT: Bytes of string content */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* "SELECT *" statement on zTbl */ + sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + const char **azCol; /* Output array */ + int nStr = 0; /* Size of all column names (incl. 0x00) */ + int nCol; /* Number of table columns */ + int i; /* Used to iterate through columns */ + + /* Loop through the returned columns. Set nStr to the number of bytes of + ** space required to store a copy of each column name, including the + ** nul-terminator byte. */ + nCol = sqlite3_column_count(pStmt); + for(i=0; i MATCHINFO */ - { "prefix", 6, 0 }, /* 1 -> PREFIX */ - { "compress", 8, 0 }, /* 2 -> COMPRESS */ - { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */ - { "order", 5, 0 } /* 4 -> ORDER */ + { "matchinfo", 9 }, /* 0 -> MATCHINFO */ + { "prefix", 6 }, /* 1 -> PREFIX */ + { "compress", 8 }, /* 2 -> COMPRESS */ + { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ + { "order", 5 }, /* 4 -> ORDER */ + { "content", 7 } /* 5 -> CONTENT */ }; int iOpt; @@ -115258,13 +115770,20 @@ static int fts3InitVtab( case 4: /* ORDER */ if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) - && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) ){ *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); rc = SQLITE_ERROR; } bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); break; + + default: /* CONTENT */ + assert( iOpt==5 ); + sqlite3_free(zUncompress); + zContent = zVal; + zVal = 0; + break; } } sqlite3_free(zVal); @@ -115277,6 +115796,26 @@ static int fts3InitVtab( aCol[nCol++] = z; } } + + /* If a content=xxx option was specified, the following: + ** + ** 1. Ignore any compress= and uncompress= options. + ** + ** 2. If no column names were specified as part of the CREATE VIRTUAL + ** TABLE statement, use all columns from the content table. + */ + if( rc==SQLITE_OK && zContent ){ + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + zCompress = 0; + zUncompress = 0; + if( nCol==0 ){ + sqlite3_free((void*)aCol); + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString); + } + assert( rc!=SQLITE_OK || nCol>0 ); + } if( rc!=SQLITE_OK ) goto fts3_init_out; if( nCol==0 ){ @@ -115321,6 +115860,8 @@ static int fts3InitVtab( p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; p->bDescIdx = bDescIdx; + p->zContentTbl = zContent; + zContent = 0; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); @@ -115382,6 +115923,7 @@ fts3_init_out: sqlite3_free(aIndex); sqlite3_free(zCompress); sqlite3_free(zUncompress); + sqlite3_free(zContent); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ @@ -115533,35 +116075,64 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } +/* +** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then +** compose and prepare an SQL statement of the form: +** +** "SELECT FROM %_content WHERE rowid = ?" +** +** (or the equivalent for a content=xxx table) and set pCsr->pStmt to +** it. If an error occurs, return an SQLite error code. +** +** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. +*/ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ + int rc = SQLITE_OK; + if( pCsr->pStmt==0 ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + char *zSql; + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + *ppStmt = pCsr->pStmt; + return rc; +} + /* ** Position the pCsr->pStmt statement so that it is on the row ** of the %_content table that contains the last match. Return ** SQLITE_OK on success. */ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc = SQLITE_OK; if( pCsr->isRequireSeek ){ - sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); - pCsr->isRequireSeek = 0; - if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ - return SQLITE_OK; - }else{ - int rc = sqlite3_reset(pCsr->pStmt); - if( rc==SQLITE_OK ){ - /* If no row was found and no error has occured, then the %_content - ** table is missing a row that is present in the full-text index. - ** The data structures are corrupt. - */ - rc = SQLITE_CORRUPT_VTAB; + sqlite3_stmt *pStmt = 0; + + rc = fts3CursorSeekStmt(pCsr, &pStmt); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + return SQLITE_OK; + }else{ + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ + /* If no row was found and no error has occured, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. */ + rc = FTS_CORRUPT_VTAB; + pCsr->isEof = 1; + } } - pCsr->isEof = 1; - if( pContext ){ - sqlite3_result_error_code(pContext, rc); - } - return rc; } - }else{ - return SQLITE_OK; } + + if( rc!=SQLITE_OK && pContext ){ + sqlite3_result_error_code(pContext, rc); + } + return rc; } /* @@ -115611,7 +116182,7 @@ static int fts3ScanInteriorNode( zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); if( zCsr>zEnd ){ - return SQLITE_CORRUPT_VTAB; + return FTS_CORRUPT_VTAB; } while( zCsrzEnd ){ - rc = SQLITE_CORRUPT_VTAB; + rc = FTS_CORRUPT_VTAB; goto finish_scan; } if( nPrefix+nSuffix>nAlloc ){ @@ -115642,6 +116213,7 @@ static int fts3ScanInteriorNode( } zBuffer = zNew; } + assert( zBuffer ); memcpy(&zBuffer[nPrefix], zCsr, nSuffix); nBuffer = nPrefix + nSuffix; zCsr += nSuffix; @@ -116000,7 +116572,7 @@ static int fts3PoslistPhraseMerge( char **pp1, /* IN/OUT: Left input list */ char **pp2 /* IN/OUT: Right input list */ ){ - char *p = (pp ? *pp : 0); + char *p = *pp; char *p1 = *pp1; char *p2 = *pp2; int iCol1 = 0; @@ -116009,7 +116581,7 @@ static int fts3PoslistPhraseMerge( /* Never set both isSaveLeft and isExact for the same invocation. */ assert( isSaveLeft==0 || isExact==0 ); - assert( *p1!=0 && *p2!=0 ); + assert( p!=0 && *p1!=0 && *p2!=0 ); if( *p1==POS_COLUMN ){ p1++; p1 += sqlite3Fts3GetVarint32(p1, &iCol1); @@ -116026,7 +116598,7 @@ static int fts3PoslistPhraseMerge( sqlite3_int64 iPos1 = 0; sqlite3_int64 iPos2 = 0; - if( pp && iCol1 ){ + if( iCol1 ){ *p++ = POS_COLUMN; p += sqlite3Fts3PutVarint(p, iCol1); } @@ -116041,16 +116613,10 @@ static int fts3PoslistPhraseMerge( || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) ){ sqlite3_int64 iSave; - if( !pp ){ - fts3PoslistCopy(0, &p2); - fts3PoslistCopy(0, &p1); - *pp1 = p1; - *pp2 = p2; - return 1; - } iSave = isSaveLeft ? iPos1 : iPos2; fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; pSave = 0; + assert( p ); } if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ if( (*p2&0xFE)==0 ) break; @@ -116099,7 +116665,7 @@ static int fts3PoslistPhraseMerge( fts3PoslistCopy(0, &p1); *pp1 = p1; *pp2 = p2; - if( !pp || *pp==p ){ + if( *pp==p ){ return 0; } *p++ = 0x00; @@ -116402,6 +116968,56 @@ static void fts3DoclistPhraseMerge( *pnRight = p - aOut; } +/* +** Argument pList points to a position list nList bytes in size. This +** function checks to see if the position list contains any entries for +** a token in position 0 (of any column). If so, it writes argument iDelta +** to the output buffer pOut, followed by a position list consisting only +** of the entries from pList at position 0, and terminated by an 0x00 byte. +** The value returned is the number of bytes written to pOut (if any). +*/ +SQLITE_PRIVATE int sqlite3Fts3FirstFilter( + sqlite3_int64 iDelta, /* Varint that may be written to pOut */ + char *pList, /* Position list (no 0x00 term) */ + int nList, /* Size of pList in bytes */ + char *pOut /* Write output here */ +){ + int nOut = 0; + int bWritten = 0; /* True once iDelta has been written */ + char *p = pList; + char *pEnd = &pList[nList]; + + if( *p!=0x01 ){ + if( *p==0x02 ){ + nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); + pOut[nOut++] = 0x02; + bWritten = 1; + } + fts3ColumnlistCopy(0, &p); + } + + while( pisPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = pTok->z; @@ -116898,8 +117515,8 @@ static int fts3FilterMethod( return SQLITE_NOMEM; } - rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, - iCol, zQuery, -1, &pCsr->pExpr + rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->bHasStat, + p->nColumn, iCol, zQuery, -1, &pCsr->pExpr ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ @@ -116926,23 +117543,24 @@ static int fts3FilterMethod( ** row by docid. */ if( idxNum==FTS3_FULLSCAN_SEARCH ){ - const char *zSort = (pCsr->bDesc ? "DESC" : "ASC"); - const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; - zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); - }else{ - const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?"; - zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName); + zSql = sqlite3_mprintf( + "SELECT %s ORDER BY rowid %s", + p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") + ); + if( zSql ){ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + }else if( idxNum==FTS3_DOCID_SEARCH ){ + rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + } } - if( !zSql ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); if( rc!=SQLITE_OK ) return rc; - if( idxNum==FTS3_DOCID_SEARCH ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); - if( rc!=SQLITE_OK ) return rc; - } - return fts3NextMethod(pCursor); } @@ -116994,7 +117612,7 @@ static int fts3ColumnMethod( sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); }else{ rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); } } @@ -117078,7 +117696,7 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ */ static void fts3ReversePoslist(char *pStart, char **ppPoslist){ char *p = &(*ppPoslist)[-2]; - char c; + char c = 0; while( p>pStart && (c=*p--)==0 ); while( p>pStart && (*p & 0x80) | c ){ @@ -117287,15 +117905,22 @@ static int fts3RenameMethod( sqlite3 *db = p->db; /* Database connection */ int rc; /* Return Code */ + /* As it happens, the pending terms table is always empty here. This is + ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction + ** always opens a savepoint transaction. And the xSavepoint() method + ** flushes the pending terms table. But leave the (no-op) call to + ** PendingTermsFlush() in in case that changes. + */ + assert( p->nPendingData==0 ); rc = sqlite3Fts3PendingTermsFlush(p); - if( rc!=SQLITE_OK ){ - return rc; + + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName + ); } - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", - p->zDb, p->zName, zName - ); if( p->bHasDocsize ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", @@ -117654,21 +118279,20 @@ static int fts3EvalPhraseLoad( */ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ int iToken; /* Used to iterate through phrase tokens */ - int rc = SQLITE_OK; /* Return code */ char *aPoslist = 0; /* Position list for deferred tokens */ int nPoslist = 0; /* Number of bytes in aPoslist */ int iPrev = -1; /* Token number of previous deferred token */ assert( pPhrase->doclist.bFreeList==0 ); - for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + for(iToken=0; iTokennToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; Fts3DeferredToken *pDeferred = pToken->pDeferred; if( pDeferred ){ char *pList; int nList; - rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); if( rc!=SQLITE_OK ) return rc; if( pList==0 ){ @@ -117769,6 +118393,7 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ && p->nToken==1 && pFirst->pSegcsr && pFirst->pSegcsr->bLookup + && pFirst->bFirst==0 ){ /* Use the incremental approach. */ int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); @@ -117998,7 +118623,7 @@ static void fts3EvalTokenCosts( Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ int *pRc /* IN/OUT: Error code */ ){ - if( *pRc==SQLITE_OK && pExpr ){ + if( *pRc==SQLITE_OK ){ if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int i; @@ -118012,6 +118637,11 @@ static void fts3EvalTokenCosts( *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); } }else if( pExpr->eType!=FTSQUERY_NOT ){ + assert( pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND + || pExpr->eType==FTSQUERY_NEAR + ); + assert( pExpr->pLeft && pExpr->pRight ); if( pExpr->eType==FTSQUERY_OR ){ pRoot = pExpr->pLeft; **ppOr = pRoot; @@ -118072,7 +118702,7 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ } if( nDoc==0 || nByte==0 ){ sqlite3_reset(pStmt); - return SQLITE_CORRUPT_VTAB; + return FTS_CORRUPT_VTAB; } pCsr->nDoc = nDoc; @@ -118116,6 +118746,15 @@ static int fts3EvalSelectDeferred( int nMinEst = 0; /* The minimum count for any phrase so far. */ int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + /* Tokens are never deferred for FTS tables created using the content=xxx + ** option. The reason being that it is not guaranteed that the content + ** table actually contains the same data as the index. To prevent this from + ** causing any problems, the deferred token optimization is completely + ** disabled for content=xxx tables. */ + if( pTab->zContentTbl ){ + return SQLITE_OK; + } + /* Count the tokens in this AND/NEAR cluster. If none of the doclists ** associated with the tokens spill onto overflow pages, or if there is ** only 1 token, exit early. No tokens to defer in this case. */ @@ -118178,7 +118817,11 @@ static int fts3EvalSelectDeferred( fts3SegReaderCursorFree(pToken->pSegcsr); pToken->pSegcsr = 0; }else{ - nLoad4 = nLoad4*4; + /* Set nLoad4 to the value of (4^nOther) for the next iteration of the + ** for-loop. Except, limit the value to 2^24 to prevent it from + ** overflowing the 32-bit integer it is stored in. */ + if( ii<12 ) nLoad4 = nLoad4*4; + if( ii==0 || pTC->pPhrase->nToken>1 ){ /* Either this is the cheapest token in the entire query, or it is ** part of a multi-token phrase. Either way, the entire doclist will @@ -118548,8 +119191,11 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ aPoslist = pExpr->pRight->pPhrase->doclist.pList; nToken = pExpr->pRight->pPhrase->nToken; for(p=pExpr->pLeft; p && res; p=p->pLeft){ - int nNear = p->pParent->nNear; - Fts3Phrase *pPhrase = ( + int nNear; + Fts3Phrase *pPhrase; + assert( p->pParent && p->pParent->pLeft==p ); + nNear = p->pParent->nNear; + pPhrase = ( p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase ); res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); @@ -119039,6 +119685,15 @@ SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ } } +/* +** Return SQLITE_CORRUPT_VTAB. +*/ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ + return SQLITE_CORRUPT_VTAB; +} +#endif + #if !SQLITE_CORE /* ** Initialize API pointer table, if required. @@ -119627,6 +120282,7 @@ typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ const char **azCol; /* Array of column names for fts3 table */ + int bFts4; /* True to allow FTS4-only syntax */ int nCol; /* Number of entries in azCol[] */ int iDefaultCol; /* Default column to query */ int isNot; /* True if getNextNode() sees a unary - */ @@ -119714,9 +120370,21 @@ static int getNextToken( pRet->pPhrase->aToken[0].isPrefix = 1; iEnd++; } - if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ - pParse->isNot = 1; + + while( 1 ){ + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' + ){ + pParse->isNot = 1; + iStart--; + }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ + pRet->pPhrase->aToken[0].bFirst = 1; + iStart--; + }else{ + break; + } } + } nConsumed = iEnd; } @@ -119815,6 +120483,7 @@ static int getNextString( pToken->n = nByte; pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); nToken = ii+1; } } @@ -119836,8 +120505,12 @@ static int getNextString( p->pPhrase->nToken = nToken; zBuf = (char *)&p->pPhrase->aToken[nToken]; - memcpy(zBuf, zTemp, nTemp); - sqlite3_free(zTemp); + if( zTemp ){ + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + }else{ + assert( nTemp==0 ); + } for(jj=0; jjpPhrase->nToken; jj++){ p->pPhrase->aToken[jj].z = zBuf; @@ -120262,6 +120935,7 @@ exprparse_out: SQLITE_PRIVATE int sqlite3Fts3ExprParse( sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ int nCol, /* Number of entries in azCol[] */ int iDefaultCol, /* Default column to query */ const char *z, int n, /* Text of MATCH query */ @@ -120275,6 +120949,7 @@ SQLITE_PRIVATE int sqlite3Fts3ExprParse( sParse.nCol = nCol; sParse.iDefaultCol = iDefaultCol; sParse.nNest = 0; + sParse.bFts4 = bFts4; if( z==0 ){ *ppExpr = 0; return SQLITE_OK; @@ -120464,7 +121139,7 @@ static void fts3ExprTest( } rc = sqlite3Fts3ExprParse( - pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr + pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr ); if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ sqlite3_result_error(context, "Error parsing expression", -1); @@ -122511,7 +123186,7 @@ static int fts3SqlStmt( /* 4 */ "DELETE FROM %Q.'%q_segdir'", /* 5 */ "DELETE FROM %Q.'%q_docsize'", /* 6 */ "DELETE FROM %Q.'%q_stat'", -/* 7 */ "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?", +/* 7 */ "SELECT %s WHERE rowid=?", /* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", /* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", @@ -122553,7 +123228,7 @@ static int fts3SqlStmt( if( eStmt==SQL_CONTENT_INSERT ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ - zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName); + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); } @@ -122596,7 +123271,7 @@ static int fts3SelectDocsize( rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ rc = sqlite3_reset(pStmt); - if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT_VTAB; + if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; pStmt = 0; }else{ rc = SQLITE_OK; @@ -122664,17 +123339,24 @@ static void fts3SqlExec( ** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can ** still happen if the user reads data directly from the %_segments or ** %_segdir tables instead of going through FTS3 though. +** +** This reasoning does not apply to a content=xxx table. */ SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *p){ int rc; /* Return code */ sqlite3_stmt *pStmt; /* Statement used to obtain lock */ - rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_null(pStmt, 1); - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); + if( p->zContentTbl==0 ){ + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + }else{ + rc = SQLITE_OK; } + return rc; } @@ -123035,6 +123717,18 @@ static int fts3InsertData( int rc; /* Return code */ sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ + if( p->zContentTbl ){ + sqlite3_value *pRowid = apVal[p->nColumn+3]; + if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ + pRowid = apVal[1]; + } + if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ + return SQLITE_CONSTRAINT; + } + *piDocid = sqlite3_value_int64(pRowid); + return SQLITE_OK; + } + /* Locate the statement handle used to insert data into the %_content ** table. The SQL for this statement is: ** @@ -123085,14 +123779,16 @@ static int fts3InsertData( ** Remove all data from the FTS3 table. Clear the hash table containing ** pending terms. */ -static int fts3DeleteAll(Fts3Table *p){ +static int fts3DeleteAll(Fts3Table *p, int bContent){ int rc = SQLITE_OK; /* Return code */ /* Discard the contents of the pending-terms hash table. */ sqlite3Fts3PendingTermsClear(p); - /* Delete everything from the %_content, %_segments and %_segdir tables. */ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); + /* Delete everything from the shadow tables. Except, leave %_content as + ** is if bContent is false. */ + assert( p->zContentTbl==0 || bContent==0 ); + if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); if( p->bHasDocsize ){ @@ -123400,7 +124096,7 @@ static int fts3SegReaderNext( if( nPrefix<0 || nSuffix<=0 || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] ){ - return SQLITE_CORRUPT_VTAB; + return FTS_CORRUPT_VTAB; } if( nPrefix+nSuffix>pReader->nTermAlloc ){ @@ -123430,7 +124126,7 @@ static int fts3SegReaderNext( if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) ){ - return SQLITE_CORRUPT_VTAB; + return FTS_CORRUPT_VTAB; } return SQLITE_OK; } @@ -124380,12 +125076,18 @@ static void fts3SegWriterFree(SegmentWriter *pWriter){ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ sqlite3_stmt *pStmt; int rc; - rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); - if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pisEmpty = sqlite3_column_int(pStmt, 0); + if( p->zContentTbl ){ + /* If using the content=xxx option, assume the table is never empty */ + *pisEmpty = 0; + rc = SQLITE_OK; + }else{ + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); } - rc = sqlite3_reset(pStmt); } return rc; } @@ -124737,6 +125439,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); + int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); Fts3SegReader **apSegment = pCsr->apSegment; int nSegment = pCsr->nSegment; @@ -124796,6 +125499,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); if( nMerge==1 && !isIgnoreEmpty + && !isFirst && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) ){ pCsr->nDoclist = apSegment[0]->nDoclist; @@ -124861,12 +125565,24 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( } pCsr->aBuffer = aNew; } - nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); - iPrev = iDocid; - if( isRequirePos ){ - memcpy(&pCsr->aBuffer[nDoclist], pList, nList); - nDoclist += nList; - pCsr->aBuffer[nDoclist++] = '\0'; + + if( isFirst ){ + char *a = &pCsr->aBuffer[nDoclist]; + int nWrite; + + nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); + if( nWrite ){ + iPrev = iDocid; + nDoclist += nWrite; + } + }else{ + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); + nDoclist += nList; + pCsr->aBuffer[nDoclist++] = '\0'; + } } } @@ -125042,9 +125758,9 @@ static void fts3DecodeIntArray( ** a blob of varints. */ static void fts3InsertDocsize( - int *pRC, /* Result code */ - Fts3Table *p, /* Table into which to insert */ - u32 *aSz /* Sizes of each column */ + int *pRC, /* Result code */ + Fts3Table *p, /* Table into which to insert */ + u32 *aSz /* Sizes of each column, in tokens */ ){ char *pBlob; /* The BLOB encoding of the document size */ int nBlob; /* Number of bytes in the BLOB */ @@ -125166,6 +125882,86 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; } +/* +** This function is called when the user executes the following statement: +** +** INSERT INTO () VALUES('rebuild'); +** +** The entire FTS index is discarded and rebuilt. If the table is one +** created using the content=xxx option, then the new index is based on +** the current contents of the xxx table. Otherwise, it is rebuilt based +** on the contents of the %_content table. +*/ +static int fts3DoRebuild(Fts3Table *p){ + int rc; /* Return Code */ + + rc = fts3DeleteAll(p, 0); + if( rc==SQLITE_OK ){ + u32 *aSz = 0; + u32 *aSzIns = 0; + u32 *aSzDel = 0; + sqlite3_stmt *pStmt = 0; + int nEntry = 0; + + /* Compose and prepare an SQL statement to loop through the content table */ + char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + int nByte = sizeof(u32) * (p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc(nByte); + if( aSz==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(aSz, 0, nByte); + aSzIns = &aSz[p->nColumn+1]; + aSzDel = &aSzIns[p->nColumn+1]; + } + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int iCol; + rc = fts3PendingTermsDocid(p, sqlite3_column_int64(pStmt, 0)); + aSz[p->nColumn] = 0; + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSz); + } + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + }else{ + nEntry++; + for(iCol=0; iCol<=p->nColumn; iCol++){ + aSzIns[iCol] += aSz[iCol]; + } + } + } + if( p->bHasStat ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); + } + sqlite3_free(aSz); + + if( pStmt ){ + int rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + return rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -125183,6 +125979,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ rc = fts3DoOptimize(p, 0); + }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ + rc = fts3DoRebuild(p); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); @@ -125263,6 +126061,7 @@ SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ Fts3PhraseToken *pPT = pDef->pToken; if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) ){ @@ -125354,14 +126153,18 @@ static int fts3DeleteByRowid( /* Deleting this row means the whole table is empty. In this case ** delete the contents of all three tables and throw away any ** data in the pendingTerms hash table. */ - rc = fts3DeleteAll(p); + rc = fts3DeleteAll(p, 1); *pnDoc = *pnDoc - 1; }else{ sqlite3_int64 iRemove = sqlite3_value_int64(pRowid); rc = fts3PendingTermsDocid(p, iRemove); fts3DeleteTerms(&rc, p, pRowid, aSzDel); - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); - if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + }else{ + *pnDoc = *pnDoc - 1; + } if( p->bHasDocsize ){ fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); } @@ -125384,7 +126187,6 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( Fts3Table *p = (Fts3Table *)pVtab; int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ - sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ u32 *aSzIns = 0; /* Sizes of inserted documents */ u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ @@ -125422,7 +126224,7 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( ** detect the conflict and return SQLITE_CONSTRAINT before beginning to ** modify the database file. */ - if( nArg>1 ){ + if( nArg>1 && p->zContentTbl==0 ){ /* Find the value object that holds the new rowid value. */ sqlite3_value *pNewRowid = apVal[3+p->nColumn]; if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ @@ -125467,19 +126269,21 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); isRemove = 1; - iRemove = sqlite3_value_int64(apVal[0]); } /* If this is an INSERT or UPDATE operation, insert the new record. */ if( nArg>1 && rc==SQLITE_OK ){ if( bInsertDone==0 ){ rc = fts3InsertData(p, apVal, pRowid); - if( rc==SQLITE_CONSTRAINT ) rc = SQLITE_CORRUPT_VTAB; + if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } } - if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){ + if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ rc = fts3PendingTermsDocid(p, *pRowid); } if( rc==SQLITE_OK ){ + assert( p->iPrevDocid==*pRowid ); rc = fts3InsertTerms(p, apVal, aSzIns); } if( p->bHasDocsize ){ @@ -125893,6 +126697,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iFirst = 0; pPhrase->pList = pCsr; fts3GetDeltaPosition(&pCsr, &iFirst); + assert( iFirst>=0 ); pPhrase->pHead = pCsr; pPhrase->pTail = pCsr; pPhrase->iHead = iFirst; @@ -126373,7 +127178,7 @@ static int fts3MatchinfoSelectDoctotal( a = sqlite3_column_blob(pStmt, 0); a += sqlite3Fts3GetVarint(a, &nDoc); - if( nDoc==0 ) return SQLITE_CORRUPT_VTAB; + if( nDoc==0 ) return FTS_CORRUPT_VTAB; *pnDoc = (u32)nDoc; if( paLen ) *paLen = a; @@ -126934,7 +127739,7 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( if( !pTerm ){ /* All offsets for this column have been gathered. */ - break; + rc = SQLITE_DONE; }else{ assert( iCurrent<=iMinPos ); if( 0==(0xFE&*pTerm->pList) ){ @@ -126951,8 +127756,8 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart ); rc = fts3StringAppend(&res, aBuffer, -1); - }else if( rc==SQLITE_DONE ){ - rc = SQLITE_CORRUPT_VTAB; + }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; } } } @@ -128293,7 +129098,8 @@ static int rtreeFilter( rc = SQLITE_NOMEM; }else{ memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); - assert( (idxStr==0 && argc==0) || (int)strlen(idxStr)==argc*2 ); + assert( (idxStr==0 && argc==0) + || (idxStr && (int)strlen(idxStr)==argc*2) ); for(ii=0; iiaConstraint[ii]; p->op = idxStr[ii*2]; @@ -128594,7 +129400,10 @@ static int ChooseLeaf( float fMinGrowth = 0.0; float fMinArea = 0.0; +#if VARIANT_RSTARTREE_CHOOSESUBTREE float fMinOverlap = 0.0; + float overlap; +#endif int nCell = NCELL(pNode); RtreeCell cell; @@ -128626,7 +129435,6 @@ static int ChooseLeaf( int bBest = 0; float growth; float area; - float overlap = 0.0; nodeGetCell(pRtree, pNode, iCell, &cell); growth = cellGrowth(pRtree, &cell, pCell); area = cellArea(pRtree, &cell); @@ -128634,6 +129442,8 @@ static int ChooseLeaf( #if VARIANT_RSTARTREE_CHOOSESUBTREE if( ii==(pRtree->iDepth-1) ){ overlap = cellOverlapEnlargement(pRtree,&cell,pCell,aCell,nCell,iCell); + }else{ + overlap = 0.0; } if( (iCell==0) || (overlap +** The minimum allocation size is capped at 2**12. Reasonable values +** for the minimum allocation size are 2**5 through 2**8. ** ** [[SQLITE_CONFIG_MUTEX]]
SQLITE_CONFIG_MUTEX
**
^(This option takes a single argument which is a pointer to an @@ -2799,7 +2804,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** that the supplied string is nul-terminated, then there is a small ** performance advantage to be gained by passing an nByte parameter that ** is equal to the number of bytes in the input string including -** the nul-terminator bytes. +** the nul-terminator bytes as this saves SQLite from having to +** make a copy of the input string. ** ** ^If pzTail is not NULL then *pzTail is made to point to the first byte ** past the end of the first SQL statement in zSql. These routines only @@ -2850,7 +2856,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ^The specific value of WHERE-clause [parameter] might influence the ** choice of query plan if the parameter is the left-hand side of a [LIKE] ** or [GLOB] operator or if the parameter is compared to an indexed column -** and the [SQLITE_ENABLE_STAT2] compile-time option is enabled. +** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled. ** the ** ** @@ -3020,6 +3026,13 @@ typedef struct sqlite3_context sqlite3_context; ** number of bytes in the value, not the number of characters.)^ ** ^If the fourth parameter is negative, the length of the string is ** the number of bytes up to the first zero terminator. +** If a non-negative fourth parameter is provided to sqlite3_bind_text() +** or sqlite3_bind_text16() then that parameter must be the byte offset +** where the NUL terminator would occur assuming the string were NUL +** terminated. If any NUL characters occur at byte offsets less than +** the value of the fourth parameter then the resulting string value will +** contain embedded NULs. The result of expressions involving strings +** with embedded NULs is undefined. ** ** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and ** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or @@ -3353,6 +3366,12 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); ** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of ** interfaces) then sqlite3_data_count(P) returns 0. ** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** ^The sqlite3_data_count(P) routine returns 0 if the previous call to +** [sqlite3_step](P) returned [SQLITE_DONE]. ^The sqlite3_data_count(P) +** will return non-zero if previous call to [sqlite3_step](P) returned +** [SQLITE_ROW], except in the case of the [PRAGMA incremental_vacuum] +** where it always returns zero since each step of that multi-step +** pragma returns 0 columns of data. ** ** See also: [sqlite3_column_count()] */ @@ -4032,7 +4051,12 @@ typedef void (*sqlite3_destructor_type)(void*); ** ^If the 3rd parameter to the sqlite3_result_text* interfaces ** is non-negative, then as many bytes (not characters) of the text ** pointed to by the 2nd parameter are taken as the application-defined -** function result. +** function result. If the 3rd parameter is non-negative, then it +** must be the byte offset into the string where the NUL terminator would +** appear if the string where NUL terminated. If any NUL characters occur +** in the string at a byte offset that is less than the value of the 3rd +** parameter, then the resulting string will contain embedded NULs and the +** result of expressions operating on strings with embedded NULs is undefined. ** ^If the 4th parameter to the sqlite3_result_text* interfaces ** or sqlite3_result_blob is a non-NULL pointer, then SQLite calls that ** function as the destructor on the text or BLOB result when it has @@ -5815,6 +5839,18 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. **
+** +** [[SQLITE_DBSTATUS_CACHE_HIT]] ^(
SQLITE_DBSTATUS_CACHE_HIT
+**
This parameter returns the number of pager cache hits that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT +** is always 0. +**
+** +** [[SQLITE_DBSTATUS_CACHE_MISS]] ^(
SQLITE_DBSTATUS_CACHE_MISS
+**
This parameter returns the number of pager cache misses that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS +** is always 0. +**
** */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 @@ -5824,7 +5860,9 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r #define SQLITE_DBSTATUS_LOOKASIDE_HIT 4 #define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5 #define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 -#define SQLITE_DBSTATUS_MAX 6 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_CACHE_HIT 7 +#define SQLITE_DBSTATUS_CACHE_MISS 8 +#define SQLITE_DBSTATUS_MAX 8 /* Largest defined DBSTATUS */ /* @@ -5878,7 +5916,6 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** A non-zero value in this counter may indicate an opportunity to ** improvement performance by adding permanent indices that do not ** need to be reinitialized each time the statement is run. -** ** */ #define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 diff --git a/sqlite3ext.h b/sqlite3ext.h index 50dd5b6..5abcde2 100644 --- a/sqlite3ext.h +++ b/sqlite3ext.h @@ -49,8 +49,10 @@ struct sqlite3_api_routines { int (*busy_timeout)(sqlite3*,int ms); int (*changes)(sqlite3*); int (*close)(sqlite3*); - int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*,int eTextRep,const char*)); - int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*,int eTextRep,const void*)); + int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const char*)); + int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const void*)); const void * (*column_blob)(sqlite3_stmt*,int iCol); int (*column_bytes)(sqlite3_stmt*,int iCol); int (*column_bytes16)(sqlite3_stmt*,int iCol); @@ -75,10 +77,18 @@ struct sqlite3_api_routines { void * (*commit_hook)(sqlite3*,int(*)(void*),void*); int (*complete)(const char*sql); int (*complete16)(const void*sql); - int (*create_collation)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*)); - int (*create_collation16)(sqlite3*,const void*,int,void*,int(*)(void*,int,const void*,int,const void*)); - int (*create_function)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*)); - int (*create_function16)(sqlite3*,const void*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*)); + int (*create_collation)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_collation16)(sqlite3*,const void*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_function)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); + int (*create_function16)(sqlite3*,const void*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*); int (*data_count)(sqlite3_stmt*pStmt); sqlite3 * (*db_handle)(sqlite3_stmt*); @@ -123,16 +133,19 @@ struct sqlite3_api_routines { void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*)); void (*result_value)(sqlite3_context*,sqlite3_value*); void * (*rollback_hook)(sqlite3*,void(*)(void*),void*); - int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,const char*,const char*),void*); + int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, + const char*,const char*),void*); void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); char * (*snprintf)(int,char*,const char*,...); int (*step)(sqlite3_stmt*); - int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,char const**,char const**,int*,int*,int*); + int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, + char const**,char const**,int*,int*,int*); void (*thread_cleanup)(void); int (*total_changes)(sqlite3*); void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*); int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*); - void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*,sqlite_int64),void*); + void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*, + sqlite_int64),void*); void * (*user_data)(sqlite3_context*); const void * (*value_blob)(sqlite3_value*); int (*value_bytes)(sqlite3_value*); @@ -154,15 +167,19 @@ struct sqlite3_api_routines { int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); int (*clear_bindings)(sqlite3_stmt*); /* Added by 3.4.1 */ - int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,void (*xDestroy)(void *)); + int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*, + void (*xDestroy)(void *)); /* Added by 3.5.0 */ int (*bind_zeroblob)(sqlite3_stmt*,int,int); int (*blob_bytes)(sqlite3_blob*); int (*blob_close)(sqlite3_blob*); - int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,int,sqlite3_blob**); + int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64, + int,sqlite3_blob**); int (*blob_read)(sqlite3_blob*,void*,int,int); int (*blob_write)(sqlite3_blob*,const void*,int,int); - int (*create_collation_v2)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*),void(*)(void*)); + int (*create_collation_v2)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*), + void(*)(void*)); int (*file_control)(sqlite3*,const char*,int,void*); sqlite3_int64 (*memory_highwater)(int); sqlite3_int64 (*memory_used)(void); @@ -198,7 +215,11 @@ struct sqlite3_api_routines { int (*backup_step)(sqlite3_backup*,int); const char *(*compileoption_get)(int); int (*compileoption_used)(const char*); - int (*create_function_v2)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*),void(*xDestroy)(void*)); + int (*create_function_v2)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*)); int (*db_config)(sqlite3*,int,...); sqlite3_mutex *(*db_mutex)(sqlite3*); int (*db_status)(sqlite3*,int,int*,int*,int);