From e366c24c230775c283ada29a50f7148d9b05a77e Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Thu, 29 Jan 2009 01:28:06 +0000 Subject: [PATCH] DBD::SQLite: updated bundled SQLite to 3.6.10, and bumped up the version requirement for installed SQLite to 3.6.0 as 3.6.x has backward incompatibility (cont.) --- Changes | 5 + MANIFEST | 254 +- MANIFEST.SKIP | 2 + Makefile.PL | 6 +- alter.c | 147 +- analyze.c | 155 +- attach.c | 141 +- auth.c | 11 +- bitvec.c | 386 ++ btmutex.c | 315 ++ btree.c | 3846 +++++++++----- btree.h | 94 +- btreeInt.h | 186 +- build.c | 936 ++-- callback.c | 213 +- complete.c | 24 +- date.c | 404 +- delete.c | 457 +- expr.c | 3351 +++++++------ fault.c | 91 + fts2.c => fts3.c | 2546 +++++++--- fts2.h => fts3.h | 4 +- fts3_expr.c | 890 ++++ fts3_expr.h | 96 + fts2_hash.c => fts3_hash.c | 176 +- fts2_hash.h => fts3_hash.h | 70 +- fts2_icu.c => fts3_icu.c | 15 +- fts2_porter.c => fts3_porter.c | 28 +- fts2_tokenizer.c => fts3_tokenizer.c | 61 +- fts2_tokenizer.h => fts3_tokenizer.h | 18 +- fts2_tokenizer1.c => fts3_tokenizer1.c | 29 +- func.c | 657 ++- global.c | 104 + hash.c | 236 +- hash.h | 44 +- hwtime.h | 87 + insert.c | 971 ++-- journal.c | 239 + keywordhash.h | 310 +- legacy.c | 40 +- loadext.c | 280 +- main.c | 1303 +++-- malloc.c | 1377 +++--- mem0.c | 61 + mem1.c | 145 + mem2.c | 440 ++ mem3.c | 688 +++ mem5.c | 488 ++ memjournal.c | 245 + mutex.c | 147 + mutex.h | 73 + mutex_noop.c | 186 + mutex_os2.c | 273 + mutex_unix.c | 328 ++ mutex_w32.c | 254 + opcodes.c | 149 +- opcodes.h | 325 +- os.c | 290 +- os.h | 444 +- os_common.h | 137 +- os_os2.c | 1247 ++--- os_unix.c | 6326 ++++++++++++++++-------- os_win.c | 1664 +++---- pager.c | 4564 ++++++++--------- pager.h | 68 +- parse.c | 4708 +++++++++--------- parse.h | 282 +- pcache.c | 580 +++ pcache.h | 157 + pcache1.c | 751 +++ ppport.h | 5473 +++++++++++++++++--- pragma.c | 714 ++- prepare.c | 352 +- printf.c | 407 +- random.c | 107 +- resolve.c | 1169 +++++ rowset.c | 238 + select.c | 4111 ++++++++------- shell.c | 2148 ++++++++ sqlite3.h | 6313 ++++++++++++++++++----- sqlite3ext.h | 92 +- sqliteInt.h | 1479 ++++-- sqliteLimit.h | 68 +- status.c | 122 + table.c | 50 +- tokenize.c | 102 +- trigger.c | 295 +- update.c | 468 +- utf.c | 243 +- util.c | 601 ++- vacuum.c | 70 +- vdbe.c | 4841 +++++++++--------- vdbe.h | 110 +- vdbeInt.h | 144 +- vdbeapi.c | 648 ++- vdbeaux.c | 1529 +++--- vdbeblob.c | 156 +- vdbefifo.c | 114 - vdbemem.c | 691 +-- vtab.c | 250 +- where.c | 2740 ++++++---- 101 files changed, 55510 insertions(+), 25960 deletions(-) create mode 100644 bitvec.c create mode 100644 btmutex.c create mode 100644 fault.c rename fts2.c => fts3.c (70%) rename fts2.h => fts3.h (86%) create mode 100644 fts3_expr.c create mode 100644 fts3_expr.h rename fts2_hash.c => fts3_hash.c (65%) rename fts2_hash.h => fts3_hash.h (61%) rename fts2_icu.c => fts3_icu.c (94%) rename fts2_porter.c => fts3_porter.c (96%) rename fts2_tokenizer.c => fts3_tokenizer.c (86%) rename fts2_tokenizer.h => fts3_tokenizer.h (91%) rename fts2_tokenizer1.c => fts3_tokenizer1.c (89%) create mode 100644 global.c create mode 100644 hwtime.h create mode 100644 journal.c create mode 100644 mem0.c create mode 100644 mem1.c create mode 100644 mem2.c create mode 100644 mem3.c create mode 100644 mem5.c create mode 100644 memjournal.c create mode 100644 mutex.c create mode 100644 mutex.h create mode 100644 mutex_noop.c create mode 100644 mutex_os2.c create mode 100644 mutex_unix.c create mode 100644 mutex_w32.c create mode 100644 pcache.c create mode 100644 pcache.h create mode 100644 pcache1.c create mode 100644 resolve.c create mode 100644 rowset.c create mode 100644 shell.c create mode 100644 status.c delete mode 100644 vdbefifo.c diff --git a/Changes b/Changes index 5c93475..169a8b8 100644 --- a/Changes +++ b/Changes @@ -1,5 +1,10 @@ Revision history for Perl extension DBD::SQLite. +1.15 + - Updated to SQLite 3.6.10, and bumped up the version + requirement for installed sqlite3 to 3.6.0 as 3.6.x + has backward incompatiblity (ISHIGAKI) + 1.14 - Updated to SQLite 3.4.2 - Switch to sqlite3_prepare_v2 which fixes a number of bugs diff --git a/MANIFEST b/MANIFEST index 53e1a9c..eeda6ba 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1,114 +1,140 @@ -alter.c -analyze.c -attach.c -auth.c -btree.c -btree.h -btreeInt.h -build.c -callback.c -Changes -complete.c -date.c -dbdimp.c -dbdimp.h -delete.c -expr.c -func.c -fts2.c -fts2.h -fts2_hash.c -fts2_hash.h -fts2_icu.c -fts2_porter.c -fts2_tokenizer.c -fts2_tokenizer.h -fts2_tokenizer1.c -getsqlite.pl -hash.c -hash.h -insert.c -keywordhash.h -legacy.c -lib/DBD/SQLite.pm -loadext.c -main.c -Makefile.PL -malloc.c -MANIFEST -MANIFEST.SKIP -META.yml Module meta-data (added by MakeMaker) -opcodes.c -opcodes.h -os.c -os.h -os_common.h -os_os2.c -os_os2.h -os_unix.c -os_win.c -pager.c -pager.h -parse.c -parse.h -ppport.h -pragma.c -prepare.c -printf.c -random.c -README -select.c -SQLite.xs -sqlite3.h -sqlite3ext.h -sqliteInt.h -sqliteLimit.h -SQLiteXS.h -t/00basic.t -t/01logon.t -t/02cr_table.t -t/03insert.t -t/04select.t -t/05tran.t -t/06error.t -t/07busy.t -t/08create_function.t -t/09create_aggregate.t -t/10dsnlist.t -t/11unicode.t -t/20createdrop.t -t/30insertfetch.t -t/40bindparam.t -t/40blobs.t -t/40blobtext.t -t/40listfields.t -t/40nulls.t -t/40numrows.t -t/40prepcached.t -t/50chopblanks.t -t/50commit.t -t/60metadata.t -t/70schemachange.t -t/90cppcomments.t -t/99cleanup.t -t/ak-dbd.t -t/dbdadmin.t -t/lib.pl -t/SQLite.dbtest -table.c -tokenize.c -trigger.c -update.c -utf.c -util.c -vacuum.c -vdbe.c -vdbe.h -vdbeapi.c -vdbeaux.c -vdbeblob.c -vdbefifo.c -vdbeInt.h -vdbemem.c -vtab.c -where.c +alter.c +analyze.c +attach.c +auth.c +bitvec.c +btmutex.c +btree.c +btree.h +btreeInt.h +build.c +callback.c +Changes +complete.c +date.c +dbdimp.c +dbdimp.h +delete.c +expr.c +fault.c +fts3.c +fts3.h +fts3_expr.c +fts3_expr.h +fts3_hash.c +fts3_hash.h +fts3_icu.c +fts3_porter.c +fts3_tokenizer.c +fts3_tokenizer.h +fts3_tokenizer1.c +func.c +getsqlite.pl +global.c +hash.c +hash.h +hwtime.h +insert.c +journal.c +keywordhash.h +legacy.c +lib/DBD/SQLite.pm +loadext.c +main.c +Makefile.PL +malloc.c +MANIFEST This list of files +MANIFEST.SKIP +mem0.c +mem1.c +mem2.c +mem3.c +mem5.c +memjournal.c +META.yml +mutex.c +mutex.h +mutex_noop.c +mutex_os2.c +mutex_unix.c +mutex_w32.c +opcodes.c +opcodes.h +os.c +os.h +os_common.h +os_os2.c +os_unix.c +os_win.c +pager.c +pager.h +parse.c +parse.h +pcache.c +pcache.h +pcache1.c +ppport.h +pragma.c +prepare.c +printf.c +random.c +README +resolve.c +rowset.c +select.c +shell.c +SQLite.xs +sqlite3.h +sqlite3ext.h +sqliteInt.h +sqliteLimit.h +SQLiteXS.h +status.c +t/00basic.t +t/01logon.t +t/02cr_table.t +t/03insert.t +t/04select.t +t/05tran.t +t/06error.t +t/07busy.t +t/08create_function.t +t/09create_aggregate.t +t/10dsnlist.t +t/11unicode.t +t/20createdrop.t +t/30insertfetch.t +t/40bindparam.t +t/40blobs.t +t/40blobtext.t +t/40listfields.t +t/40nulls.t +t/40numrows.t +t/40prepcached.t +t/50chopblanks.t +t/50commit.t +t/60metadata.t +t/70schemachange.t +t/90cppcomments.t +t/99cleanup.t +t/ak-dbd.t +t/dbdadmin.t +t/lib.pl +t/SQLite.dbtest +table.c +tokenize.c +trigger.c +update.c +utf.c +util.c +vacuum.c +vdbe.c +vdbe.h +vdbeapi.c +vdbeaux.c +vdbeblob.c +vdbeInt.h +vdbemem.c +vtab.c +walker.c +where.c diff --git a/MANIFEST.SKIP b/MANIFEST.SKIP index a535f37..066a8c5 100644 --- a/MANIFEST.SKIP +++ b/MANIFEST.SKIP @@ -18,3 +18,5 @@ output/.* ^Makefile\.[a-z]+$ ^pm_to_blib$ ~$ +^sqlite\- +\.svn diff --git a/Makefile.PL b/Makefile.PL index 17b8c76..619d5a2 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -89,8 +89,8 @@ unless ( $force_local ) { last if $version; } } - unless ( $version && ($version >= 3003009) ) { - warn "SQLite version must be at least 3.3.9. No header file at that\n"; + unless ( $version && ($version >= 3006000) ) { + warn "SQLite version must be at least 3.6.0. No header file at that\n"; warn "version or higher was found. Using the local version instead.\n"; $force_local = 1; undef $sqlite_lib; @@ -127,7 +127,7 @@ if ( $sqlite_inc ) { my @CC_DEFINE = ( '-DSQLITE_CORE', - '-DSQLITE_ENABLE_FTS2', + '-DSQLITE_ENABLE_FTS3', '-DNDEBUG=1', "-DSQLITE_PTR_SZ=$Config{ptrsize}" ); diff --git a/alter.c b/alter.c index b8c767a..9733c56 100644 --- a/alter.c +++ b/alter.c @@ -12,7 +12,7 @@ ** This file contains C code routines that used to generate VDBE code ** that implements the ALTER TABLE command. ** -** $Id: alter.c,v 1.27 2007/06/27 17:09:24 danielk1977 Exp $ +** $Id: alter.c,v 1.51 2008/12/10 19:26:22 drh Exp $ */ #include "sqliteInt.h" #include @@ -39,7 +39,7 @@ */ static void renameTableFunc( sqlite3_context *context, - int argc, + int NotUsed, sqlite3_value **argv ){ unsigned char const *zSql = sqlite3_value_text(argv[0]); @@ -51,9 +51,13 @@ static void renameTableFunc( int len = 0; char *zRet; + sqlite3 *db = sqlite3_context_db_handle(context); + + UNUSED_PARAMETER(NotUsed); + /* The principle used to locate the table name in the CREATE TABLE - ** statement is that the table name is the first token that is immediatedly - ** followed by a left parenthesis - TK_LP - or "USING" TK_USING. + ** statement is that the table name is the first non-space token that + ** is immediately followed by a TK_LP or TK_USING token. */ if( zSql ){ do { @@ -67,7 +71,7 @@ static void renameTableFunc( tname.n = len; /* Advance zCsr to the next token. Store that token type in 'token', - ** and it's length in 'len' (to be used next iteration of this loop). + ** and its length in 'len' (to be used next iteration of this loop). */ do { zCsr += len; @@ -76,9 +80,9 @@ static void renameTableFunc( assert( len>0 ); } while( token!=TK_LP && token!=TK_USING ); - zRet = sqlite3MPrintf("%.*s%Q%s", tname.z - zSql, zSql, + zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", tname.z - zSql, zSql, zTableName, tname.z+tname.n); - sqlite3_result_text(context, zRet, -1, sqlite3FreeX); + sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); } } @@ -92,7 +96,7 @@ static void renameTableFunc( */ static void renameTriggerFunc( sqlite3_context *context, - int argc, + int NotUsed, sqlite3_value **argv ){ unsigned char const *zSql = sqlite3_value_text(argv[0]); @@ -104,6 +108,9 @@ static void renameTriggerFunc( unsigned char const *zCsr = zSql; int len = 0; char *zRet; + sqlite3 *db = sqlite3_context_db_handle(context); + + UNUSED_PARAMETER(NotUsed); /* The principle used to locate the table name in the CREATE TRIGGER ** statement is that the table name is the first token that is immediatedly @@ -123,7 +130,7 @@ static void renameTriggerFunc( tname.n = len; /* Advance zCsr to the next token. Store that token type in 'token', - ** and it's length in 'len' (to be used next iteration of this loop). + ** and its length in 'len' (to be used next iteration of this loop). */ do { zCsr += len; @@ -149,9 +156,9 @@ static void renameTriggerFunc( /* Variable tname now contains the token that is the old table-name ** in the CREATE TRIGGER statement. */ - zRet = sqlite3MPrintf("%.*s%Q%s", tname.z - zSql, zSql, + zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", tname.z - zSql, zSql, zTableName, tname.z+tname.n); - sqlite3_result_text(context, zRet, -1, sqlite3FreeX); + sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); } } #endif /* !SQLITE_OMIT_TRIGGER */ @@ -160,22 +167,12 @@ static void renameTriggerFunc( ** Register built-in functions used to help implement ALTER TABLE */ void sqlite3AlterFunctions(sqlite3 *db){ - static const struct { - char *zName; - signed char nArg; - void (*xFunc)(sqlite3_context*,int,sqlite3_value **); - } aFuncs[] = { - { "sqlite_rename_table", 2, renameTableFunc}, + sqlite3CreateFunc(db, "sqlite_rename_table", 2, SQLITE_UTF8, 0, + renameTableFunc, 0, 0); #ifndef SQLITE_OMIT_TRIGGER - { "sqlite_rename_trigger", 2, renameTriggerFunc}, + sqlite3CreateFunc(db, "sqlite_rename_trigger", 2, SQLITE_UTF8, 0, + renameTriggerFunc, 0, 0); #endif - }; - int i; - - for(i=0; ipSchema!=pTempSchema ){ + sqlite3 *db = pParse->db; for( pTrig=pTab->pTrigger; pTrig; pTrig=pTrig->pNext ){ if( pTrig->pSchema==pTempSchema ){ if( !zWhere ){ - zWhere = sqlite3MPrintf("name=%Q", pTrig->name); + zWhere = sqlite3MPrintf(db, "name=%Q", pTrig->name); }else{ tmp = zWhere; - zWhere = sqlite3MPrintf("%s OR name=%Q", zWhere, pTrig->name); - sqliteFree(tmp); + zWhere = sqlite3MPrintf(db, "%s OR name=%Q", zWhere, pTrig->name); + sqlite3DbFree(db, tmp); } } } @@ -229,6 +227,7 @@ static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ v = sqlite3GetVdbe(pParse); if( !v ) return; + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); assert( iDb>=0 ); @@ -237,24 +236,24 @@ static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ for(pTrig=pTab->pTrigger; pTrig; pTrig=pTrig->pNext){ int iTrigDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); assert( iTrigDb==iDb || iTrigDb==1 ); - sqlite3VdbeOp3(v, OP_DropTrigger, iTrigDb, 0, pTrig->name, 0); + sqlite3VdbeAddOp4(v, OP_DropTrigger, iTrigDb, 0, 0, pTrig->name, 0); } #endif /* Drop the table and index from the internal schema */ - sqlite3VdbeOp3(v, OP_DropTable, iDb, 0, pTab->zName, 0); + sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); /* Reload the table, index and permanent trigger schemas. */ - zWhere = sqlite3MPrintf("tbl_name=%Q", zName); + zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName); if( !zWhere ) return; - sqlite3VdbeOp3(v, OP_ParseSchema, iDb, 0, zWhere, P3_DYNAMIC); + sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); #ifndef SQLITE_OMIT_TRIGGER /* Now, if the table is not stored in the temp database, reload any temp ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. */ if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ - sqlite3VdbeOp3(v, OP_ParseSchema, 1, 0, zWhere, P3_DYNAMIC); + sqlite3VdbeAddOp4(v, OP_ParseSchema, 1, 0, 0, zWhere, P4_DYNAMIC); } #endif } @@ -281,16 +280,17 @@ void sqlite3AlterRenameTable( #endif int isVirtualRename = 0; /* True if this is a v-table with an xRename() */ - if( sqlite3MallocFailed() ) goto exit_rename_table; + if( db->mallocFailed ) goto exit_rename_table; assert( pSrc->nSrc==1 ); + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - pTab = sqlite3LocateTable(pParse, pSrc->a[0].zName, pSrc->a[0].zDatabase); + pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); if( !pTab ) goto exit_rename_table; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); zDb = db->aDb[iDb].zName; /* Get a NULL terminated version of the new table name. */ - zName = sqlite3NameFromToken(pName); + zName = sqlite3NameFromToken(db, pName); if( !zName ) goto exit_rename_table; /* Check that a table or index named 'zName' does not already exist @@ -305,7 +305,9 @@ void sqlite3AlterRenameTable( /* Make sure it is not a system table being altered, or a reserved name ** that the table is being renamed to. */ - if( strlen(pTab->zName)>6 && 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) ){ + if( sqlite3Strlen30(pTab->zName)>6 + && 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) + ){ sqlite3ErrorMsg(pParse, "table %s may not be altered", pTab->zName); goto exit_rename_table; } @@ -313,6 +315,13 @@ void sqlite3AlterRenameTable( goto exit_rename_table; } +#ifndef SQLITE_OMIT_VIEW + if( pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "view %s may not be altered", pTab->zName); + goto exit_rename_table; + } +#endif + #ifndef SQLITE_OMIT_AUTHORIZATION /* Invoke the authorization callback. */ if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ @@ -339,7 +348,7 @@ void sqlite3AlterRenameTable( goto exit_rename_table; } sqlite3BeginWriteOperation(pParse, isVirtualRename, iDb); - sqlite3ChangeCookie(db, v, iDb); + sqlite3ChangeCookie(pParse, iDb); /* If this is a virtual table, invoke the xRename() function if ** one is defined. The xRename() callback will modify the names @@ -348,8 +357,9 @@ void sqlite3AlterRenameTable( */ #ifndef SQLITE_OMIT_VIRTUALTABLE if( isVirtualRename ){ - sqlite3VdbeOp3(v, OP_String8, 0, 0, zName, 0); - sqlite3VdbeOp3(v, OP_VRename, 0, 0, (const char*)pTab->pVtab, P3_VTAB); + int i = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_String8, 0, i, 0, zName, 0); + sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pTab->pVtab, P4_VTAB); } #endif @@ -371,7 +381,7 @@ void sqlite3AlterRenameTable( "name = CASE " "WHEN type='table' THEN %Q " "WHEN name LIKE 'sqlite_autoindex%%' AND type='index' THEN " - "'sqlite_autoindex_' || %Q || substr(name,%d+18,10) " + "'sqlite_autoindex_' || %Q || substr(name,%d+18) " "ELSE name END " "WHERE tbl_name=%Q AND " "(type='table' OR type='index' OR type='trigger');", @@ -388,7 +398,7 @@ void sqlite3AlterRenameTable( */ if( sqlite3FindTable(db, "sqlite_sequence", zDb) ){ sqlite3NestedParse(pParse, - "UPDATE %Q.sqlite_sequence set name = %Q WHERE name = %Q", + "UPDATE \"%w\".sqlite_sequence set name = %Q WHERE name = %Q", zDb, zName, pTab->zName); } #endif @@ -404,7 +414,7 @@ void sqlite3AlterRenameTable( "sql = sqlite_rename_trigger(sql, %Q), " "tbl_name = %Q " "WHERE %s;", zName, zName, zWhere); - sqliteFree(zWhere); + sqlite3DbFree(db, zWhere); } #endif @@ -412,8 +422,8 @@ void sqlite3AlterRenameTable( reloadTableSchema(pParse, pTab, zName); exit_rename_table: - sqlite3SrcListDelete(pSrc); - sqliteFree(zName); + sqlite3SrcListDelete(db, pSrc); + sqlite3DbFree(db, zName); } @@ -434,17 +444,20 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ char *zCol; /* Null-terminated column definition */ Column *pCol; /* The new column */ Expr *pDflt; /* Default value for the new column */ + sqlite3 *db; /* The database connection; */ - if( pParse->nErr ) return; + db = pParse->db; + if( pParse->nErr || db->mallocFailed ) return; pNew = pParse->pNewTable; assert( pNew ); - iDb = sqlite3SchemaToIndex(pParse->db, pNew->pSchema); - zDb = pParse->db->aDb[iDb].zName; + assert( sqlite3BtreeHoldsAllMutexes(db) ); + iDb = sqlite3SchemaToIndex(db, pNew->pSchema); + zDb = db->aDb[iDb].zName; zTab = pNew->zName; pCol = &pNew->aCol[pNew->nCol-1]; pDflt = pCol->pDflt; - pTab = sqlite3FindTable(pParse->db, zTab, zDb); + pTab = sqlite3FindTable(db, zTab, zDb); assert( pTab ); #ifndef SQLITE_OMIT_AUTHORIZATION @@ -485,8 +498,8 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ */ if( pDflt ){ sqlite3_value *pVal; - if( sqlite3ValueFromExpr(pDflt, SQLITE_UTF8, SQLITE_AFF_NONE, &pVal) ){ - /* malloc() has failed */ + if( sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_NONE, &pVal) ){ + db->mallocFailed = 1; return; } if( !pVal ){ @@ -497,20 +510,20 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ } /* Modify the CREATE TABLE statement. */ - zCol = sqliteStrNDup((char*)pColDef->z, pColDef->n); + zCol = sqlite3DbStrNDup(db, (char*)pColDef->z, pColDef->n); if( zCol ){ char *zEnd = &zCol[pColDef->n-1]; while( (zEnd>zCol && *zEnd==';') || isspace(*(unsigned char *)zEnd) ){ *zEnd-- = '\0'; } sqlite3NestedParse(pParse, - "UPDATE %Q.%s SET " - "sql = substr(sql,1,%d) || ', ' || %Q || substr(sql,%d,length(sql)) " + "UPDATE \"%w\".%s SET " + "sql = substr(sql,1,%d) || ', ' || %Q || substr(sql,%d) " "WHERE type = 'table' AND name = %Q", zDb, SCHEMA_TABLE(iDb), pNew->addColOffset, zCol, pNew->addColOffset+1, zTab ); - sqliteFree(zCol); + sqlite3DbFree(db, zCol); } /* If the default value of the new column is NULL, then set the file @@ -545,11 +558,13 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ int iDb; int i; int nAlloc; + sqlite3 *db = pParse->db; /* Look up the table being altered. */ assert( pParse->pNewTable==0 ); - if( sqlite3MallocFailed() ) goto exit_begin_add_column; - pTab = sqlite3LocateTable(pParse, pSrc->a[0].zName, pSrc->a[0].zDatabase); + assert( sqlite3BtreeHoldsAllMutexes(db) ); + if( db->mallocFailed ) goto exit_begin_add_column; + pTab = sqlite3LocateTable(pParse, 0, pSrc->a[0].zName, pSrc->a[0].zDatabase); if( !pTab ) goto exit_begin_add_column; #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -566,33 +581,35 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ } assert( pTab->addColOffset>0 ); - iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); /* Put a copy of the Table struct in Parse.pNewTable for the ** sqlite3AddColumn() function and friends to modify. */ - pNew = (Table *)sqliteMalloc(sizeof(Table)); + pNew = (Table*)sqlite3DbMallocZero(db, sizeof(Table)); if( !pNew ) goto exit_begin_add_column; pParse->pNewTable = pNew; pNew->nRef = 1; + pNew->db = db; pNew->nCol = pTab->nCol; assert( pNew->nCol>0 ); nAlloc = (((pNew->nCol-1)/8)*8)+8; assert( nAlloc>=pNew->nCol && nAlloc%8==0 && nAlloc-pNew->nCol<8 ); - pNew->aCol = (Column *)sqliteMalloc(sizeof(Column)*nAlloc); - pNew->zName = sqliteStrDup(pTab->zName); + pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); + pNew->zName = sqlite3DbStrDup(db, pTab->zName); if( !pNew->aCol || !pNew->zName ){ + db->mallocFailed = 1; goto exit_begin_add_column; } memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); for(i=0; inCol; i++){ Column *pCol = &pNew->aCol[i]; - pCol->zName = sqliteStrDup(pCol->zName); + pCol->zName = sqlite3DbStrDup(db, pCol->zName); pCol->zColl = 0; pCol->zType = 0; pCol->pDflt = 0; } - pNew->pSchema = pParse->db->aDb[iDb].pSchema; + pNew->pSchema = db->aDb[iDb].pSchema; pNew->addColOffset = pTab->addColOffset; pNew->nRef = 1; @@ -600,10 +617,10 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ sqlite3BeginWriteOperation(pParse, 0, iDb); v = sqlite3GetVdbe(pParse); if( !v ) goto exit_begin_add_column; - sqlite3ChangeCookie(pParse->db, v, iDb); + sqlite3ChangeCookie(pParse, iDb); exit_begin_add_column: - sqlite3SrcListDelete(pSrc); + sqlite3SrcListDelete(db, pSrc); return; } #endif /* SQLITE_ALTER_TABLE */ diff --git a/analyze.c b/analyze.c index b835aeb..b4422e7 100644 --- a/analyze.c +++ b/analyze.c @@ -11,7 +11,7 @@ ************************************************************************* ** This file contains code associated with the ANALYZE command. ** -** @(#) $Id: analyze.c,v 1.19 2007/06/20 13:37:31 drh Exp $ +** @(#) $Id: analyze.c,v 1.47 2008/12/10 16:45:51 drh Exp $ */ #ifndef SQLITE_OMIT_ANALYZE #include "sqliteInt.h" @@ -33,21 +33,25 @@ static void openStatTable( sqlite3 *db = pParse->db; Db *pDb; int iRootPage; + u8 createStat1 = 0; Table *pStat; Vdbe *v = sqlite3GetVdbe(pParse); if( v==0 ) return; + assert( sqlite3BtreeHoldsAllMutexes(db) ); + assert( sqlite3VdbeDb(v)==db ); pDb = &db->aDb[iDb]; if( (pStat = sqlite3FindTable(db, "sqlite_stat1", pDb->zName))==0 ){ /* The sqlite_stat1 tables does not exist. Create it. ** Note that a side-effect of the CREATE TABLE statement is to leave - ** the rootpage of the new table on the top of the stack. This is + ** the rootpage of the new table in register pParse->regRoot. This is ** important because the OpenWrite opcode below will be needing it. */ sqlite3NestedParse(pParse, "CREATE TABLE %Q.sqlite_stat1(tbl,idx,stat)", pDb->zName ); - iRootPage = 0; /* Cause rootpage to be taken from top of stack */ + iRootPage = pParse->regRoot; + createStat1 = 1; /* Cause rootpage to be taken from top of stack */ }else if( zWhere ){ /* The sqlite_stat1 table exists. Delete all entries associated with ** the table zWhere. */ @@ -59,7 +63,7 @@ static void openStatTable( }else{ /* The sqlite_stat1 table already exists. Delete all rows. */ iRootPage = pStat->tnum; - sqlite3VdbeAddOp(v, OP_Clear, pStat->tnum, iDb); + sqlite3VdbeAddOp2(v, OP_Clear, pStat->tnum, iDb); } /* Open the sqlite_stat1 table for writing. Unless it was created @@ -67,12 +71,12 @@ static void openStatTable( ** If this vdbe did create the sqlite_stat1 table, then it must have ** already obtained a schema-lock, making the write-lock redundant. */ - if( iRootPage>0 ){ + if( !createStat1 ){ sqlite3TableLock(pParse, iDb, iRootPage, 1, "sqlite_stat1"); } - sqlite3VdbeAddOp(v, OP_Integer, iDb, 0); - sqlite3VdbeAddOp(v, OP_OpenWrite, iStatCur, iRootPage); - sqlite3VdbeAddOp(v, OP_SetNumColumns, iStatCur, 3); + sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, 3); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iStatCur, iRootPage, iDb); + sqlite3VdbeChangeP5(v, createStat1); } /* @@ -82,11 +86,11 @@ static void openStatTable( static void analyzeOneTable( Parse *pParse, /* Parser context */ Table *pTab, /* Table whose indices are to be analyzed */ - int iStatCur, /* Cursor that writes to the sqlite_stat1 table */ + int iStatCur, /* Index of VdbeCursor that writes the sqlite_stat1 table */ int iMem /* Available memory locations begin here */ ){ Index *pIdx; /* An index to being analyzed */ - int iIdxCur; /* Cursor number for index being analyzed */ + int iIdxCur; /* Index of VdbeCursor for index being analyzed */ int nCol; /* Number of columns in the index */ Vdbe *v; /* The virtual machine being built up */ int i; /* Loop counter */ @@ -100,7 +104,7 @@ static void analyzeOneTable( /* Do no analysis for tables that have no indices */ return; } - + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); assert( iDb>=0 ); #ifndef SQLITE_OMIT_AUTHORIZATION @@ -116,19 +120,27 @@ static void analyzeOneTable( iIdxCur = pParse->nTab; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); + int regFields; /* Register block for building records */ + int regRec; /* Register holding completed record */ + int regTemp; /* Temporary use register */ + int regCol; /* Content of a column from the table being analyzed */ + int regRowid; /* Rowid for the inserted record */ + int regF2; /* Open a cursor to the index to be analyzed */ assert( iDb==sqlite3SchemaToIndex(pParse->db, pIdx->pSchema) ); - sqlite3VdbeAddOp(v, OP_Integer, iDb, 0); - VdbeComment((v, "# %s", pIdx->zName)); - sqlite3VdbeOp3(v, OP_OpenRead, iIdxCur, pIdx->tnum, - (char *)pKey, P3_KEYINFO_HANDOFF); nCol = pIdx->nColumn; - if( iMem+nCol*2>=pParse->nMem ){ - pParse->nMem = iMem+nCol*2+1; + sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, nCol+1); + sqlite3VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb, + (char *)pKey, P4_KEYINFO_HANDOFF); + VdbeComment((v, "%s", pIdx->zName)); + regFields = iMem+nCol*2; + regTemp = regRowid = regCol = regFields+3; + regRec = regCol+1; + if( regRec>pParse->nMem ){ + pParse->nMem = regRec; } - sqlite3VdbeAddOp(v, OP_SetNumColumns, iIdxCur, nCol+1); /* Memory cells are used as follows: ** @@ -144,33 +156,33 @@ static void analyzeOneTable( ** are initialized to NULL. */ for(i=0; i<=nCol; i++){ - sqlite3VdbeAddOp(v, OP_MemInt, 0, iMem+i); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iMem+i); } for(i=0; i0 then it is always the case the D>0 so division by zero ** is never possible. */ - sqlite3VdbeAddOp(v, OP_MemLoad, iMem, 0); - addr = sqlite3VdbeAddOp(v, OP_IfNot, 0, 0); - sqlite3VdbeAddOp(v, OP_NewRowid, iStatCur, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, pTab->zName, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, pIdx->zName, 0); - sqlite3VdbeAddOp(v, OP_MemLoad, iMem, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, " ", 0); + addr = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); + sqlite3VdbeAddOp4(v, OP_String8, 0, regFields, 0, pTab->zName, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, regFields+1, 0, pIdx->zName, 0); + regF2 = regFields+2; + sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regF2); for(i=0; inTab++; openStatTable(pParse, iDb, iStatCur, 0); - iMem = pParse->nMem; + iMem = pParse->nMem+1; for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); analyzeOneTable(pParse, pTab, iStatCur, iMem); @@ -258,11 +265,12 @@ static void analyzeTable(Parse *pParse, Table *pTab){ int iStatCur; assert( pTab!=0 ); + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); sqlite3BeginWriteOperation(pParse, 0, iDb); iStatCur = pParse->nTab++; openStatTable(pParse, iDb, iStatCur, pTab->zName); - analyzeOneTable(pParse, pTab, iStatCur, pParse->nMem); + analyzeOneTable(pParse, pTab, iStatCur, pParse->nMem+1); loadAnalysis(pParse, iDb); } @@ -288,6 +296,7 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ return; } @@ -304,11 +313,13 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ if( iDb>=0 ){ analyzeDatabase(pParse, iDb); }else{ - z = sqlite3NameFromToken(pName1); - pTab = sqlite3LocateTable(pParse, z, 0); - sqliteFree(z); - if( pTab ){ - analyzeTable(pParse, pTab); + z = sqlite3NameFromToken(db, pName1); + if( z ){ + pTab = sqlite3LocateTable(pParse, 0, z, 0); + sqlite3DbFree(db, z); + if( pTab ){ + analyzeTable(pParse, pTab); + } } } }else{ @@ -316,10 +327,10 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); if( iDb>=0 ){ zDb = db->aDb[iDb].zName; - z = sqlite3NameFromToken(pTableName); + z = sqlite3NameFromToken(db, pTableName); if( z ){ - pTab = sqlite3LocateTable(pParse, z, zDb); - sqliteFree(z); + pTab = sqlite3LocateTable(pParse, 0, z, zDb); + sqlite3DbFree(db, z); if( pTab ){ analyzeTable(pParse, pTab); } @@ -345,7 +356,7 @@ struct analysisInfo { ** argv[0] = name of the index ** argv[1] = results of analysis - on integer for each column */ -static int analysisLoader(void *pData, int argc, char **argv, char **azNotUsed){ +static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ analysisInfo *pInfo = (analysisInfo*)pData; Index *pIndex; int i, c; @@ -353,6 +364,8 @@ static int analysisLoader(void *pData, int argc, char **argv, char **azNotUsed){ const char *z; assert( argc==2 ); + UNUSED_PARAMETER2(NotUsed, argc); + if( argv==0 || argv[0]==0 || argv[1]==0 ){ return 0; } @@ -382,6 +395,10 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ char *zSql; int rc; + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pBt!=0 ); + assert( sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); + /* Clear any prior statistics */ for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); @@ -397,12 +414,12 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load new statistics out of the sqlite_stat1 table */ - zSql = sqlite3MPrintf("SELECT idx, stat FROM %Q.sqlite_stat1", + zSql = sqlite3MPrintf(db, "SELECT idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase); - sqlite3SafetyOff(db); + (void)sqlite3SafetyOff(db); rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); - sqlite3SafetyOn(db); - sqliteFree(zSql); + (void)sqlite3SafetyOn(db); + sqlite3DbFree(db, zSql); return rc; } diff --git a/attach.c b/attach.c index e3645d9..2a7525a 100644 --- a/attach.c +++ b/attach.c @@ -11,7 +11,7 @@ ************************************************************************* ** This file contains code used to implement the ATTACH and DETACH commands. ** -** $Id: attach.c,v 1.60 2007/05/09 20:31:30 drh Exp $ +** $Id: attach.c,v 1.81 2008/12/10 16:45:51 drh Exp $ */ #include "sqliteInt.h" @@ -39,7 +39,7 @@ static int resolveAttachExpr(NameContext *pName, Expr *pExpr) int rc = SQLITE_OK; if( pExpr ){ if( pExpr->op!=TK_ID ){ - rc = sqlite3ExprResolveNames(pName, pExpr); + rc = sqlite3ResolveExprNames(pName, pExpr); if( rc==SQLITE_OK && !sqlite3ExprIsConstant(pExpr) ){ sqlite3ErrorMsg(pName->pParse, "invalid name: \"%T\"", &pExpr->span); return SQLITE_ERROR; @@ -64,17 +64,19 @@ static int resolveAttachExpr(NameContext *pName, Expr *pExpr) */ static void attachFunc( sqlite3_context *context, - int argc, + int NotUsed, sqlite3_value **argv ){ int i; int rc = 0; - sqlite3 *db = sqlite3_user_data(context); + sqlite3 *db = sqlite3_context_db_handle(context); const char *zName; const char *zFile; Db *aNew; - char zErr[128]; char *zErrDyn = 0; + char zErr[128]; + + UNUSED_PARAMETER(NotUsed); zFile = (const char *)sqlite3_value_text(argv[0]); zName = (const char *)sqlite3_value_text(argv[1]); @@ -87,10 +89,10 @@ static void attachFunc( ** * Transaction currently open ** * Specified database name already being used. */ - if( db->nDb>=SQLITE_MAX_ATTACHED+2 ){ + if( db->nDb>=db->aLimit[SQLITE_LIMIT_ATTACHED]+2 ){ sqlite3_snprintf( sizeof(zErr), zErr, "too many attached databases - max %d", - SQLITE_MAX_ATTACHED + db->aLimit[SQLITE_LIMIT_ATTACHED] ); goto attach_error; } @@ -102,7 +104,8 @@ static void attachFunc( for(i=0; inDb; i++){ char *z = db->aDb[i].zName; if( z && zName && sqlite3StrICmp(z, zName)==0 ){ - sqlite3_snprintf(sizeof(zErr), zErr, "database %s is already in use", zName); + sqlite3_snprintf(sizeof(zErr), zErr, + "database %s is already in use", zName); goto attach_error; } } @@ -111,16 +114,12 @@ static void attachFunc( ** hash tables. */ if( db->aDb==db->aDbStatic ){ - aNew = sqliteMalloc( sizeof(db->aDb[0])*3 ); - if( aNew==0 ){ - return; - } + aNew = sqlite3DbMallocRaw(db, sizeof(db->aDb[0])*3 ); + if( aNew==0 ) return; memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); }else{ - aNew = sqliteRealloc(db->aDb, sizeof(db->aDb[0])*(db->nDb+1) ); - if( aNew==0 ){ - return; - } + aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(db->nDb+1) ); + if( aNew==0 ) return; } db->aDb = aNew; aNew = &db->aDb[db->nDb++]; @@ -130,9 +129,12 @@ static void attachFunc( ** it to obtain the database schema. At this point the schema may ** or may not be initialised. */ - rc = sqlite3BtreeFactory(db, zFile, 0, SQLITE_DEFAULT_CACHE_SIZE, &aNew->pBt); + rc = sqlite3BtreeFactory(db, zFile, 0, SQLITE_DEFAULT_CACHE_SIZE, + db->openFlags | SQLITE_OPEN_MAIN_DB, + &aNew->pBt); if( rc==SQLITE_OK ){ - aNew->pSchema = sqlite3SchemaGet(aNew->pBt); + Pager *pPager; + aNew->pSchema = sqlite3SchemaGet(db, aNew->pBt); if( !aNew->pSchema ){ rc = SQLITE_NOMEM; }else if( aNew->pSchema->file_format && aNew->pSchema->enc!=ENC(db) ){ @@ -140,9 +142,11 @@ static void attachFunc( "attached databases must use the same text encoding as main database"); goto attach_error; } - sqlite3PagerLockingMode(sqlite3BtreePager(aNew->pBt), db->dfltLockMode); + pPager = sqlite3BtreePager(aNew->pBt); + sqlite3PagerLockingMode(pPager, db->dfltLockMode); + sqlite3PagerJournalMode(pPager, db->dfltJournalMode); } - aNew->zName = sqliteStrDup(zName); + aNew->zName = sqlite3DbStrDup(db, zName); aNew->safety_level = 3; #if SQLITE_HAS_CODEC @@ -155,7 +159,7 @@ static void attachFunc( switch( t ){ case SQLITE_INTEGER: case SQLITE_FLOAT: - zErrDyn = sqliteStrDup("Invalid key value"); + zErrDyn = sqlite3DbStrDup(db, "Invalid key value"); rc = SQLITE_ERROR; break; @@ -181,9 +185,11 @@ static void attachFunc( ** we found it. */ if( rc==SQLITE_OK ){ - sqlite3SafetyOn(db); + (void)sqlite3SafetyOn(db); + sqlite3BtreeEnterAll(db); rc = sqlite3Init(db, &zErrDyn); - sqlite3SafetyOff(db); + sqlite3BtreeLeaveAll(db); + (void)sqlite3SafetyOff(db); } if( rc ){ int iDb = db->nDb - 1; @@ -195,8 +201,8 @@ static void attachFunc( } sqlite3ResetInternalSchema(db, 0); db->nDb = iDb; - if( rc==SQLITE_NOMEM ){ - sqlite3FailedMalloc(); + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + db->mallocFailed = 1; sqlite3_snprintf(sizeof(zErr),zErr, "out of memory"); }else{ sqlite3_snprintf(sizeof(zErr),zErr, "unable to open database: %s", zFile); @@ -210,11 +216,12 @@ attach_error: /* Return an error if we get here */ if( zErrDyn ){ sqlite3_result_error(context, zErrDyn, -1); - sqliteFree(zErrDyn); + sqlite3DbFree(db, zErrDyn); }else{ zErr[sizeof(zErr)-1] = 0; sqlite3_result_error(context, zErr, -1); } + if( rc ) sqlite3_result_error_code(context, rc); } /* @@ -227,15 +234,17 @@ attach_error: */ static void detachFunc( sqlite3_context *context, - int argc, + int NotUsed, sqlite3_value **argv ){ const char *zName = (const char *)sqlite3_value_text(argv[0]); - sqlite3 *db = sqlite3_user_data(context); + sqlite3 *db = sqlite3_context_db_handle(context); int i; Db *pDb = 0; char zErr[128]; + UNUSED_PARAMETER(NotUsed); + if( zName==0 ) zName = ""; for(i=0; inDb; i++){ pDb = &db->aDb[i]; @@ -278,8 +287,7 @@ detach_error: static void codeAttach( Parse *pParse, /* The parser context */ int type, /* Either SQLITE_ATTACH or SQLITE_DETACH */ - const char *zFunc, /* Either "sqlite_attach" or "sqlite_detach */ - int nFunc, /* Number of args to pass to zFunc */ + FuncDef *pFunc, /* FuncDef wrapper for detachFunc() or attachFunc() */ Expr *pAuthArg, /* Expression to pass to authorization callback */ Expr *pFilename, /* Name of database file */ Expr *pDbname, /* Name of the database to use internally */ @@ -288,18 +296,18 @@ static void codeAttach( int rc; NameContext sName; Vdbe *v; - FuncDef *pFunc; sqlite3* db = pParse->db; + int regArgs; #ifndef SQLITE_OMIT_AUTHORIZATION - assert( sqlite3MallocFailed() || pAuthArg ); + assert( db->mallocFailed || pAuthArg ); if( pAuthArg ){ - char *zAuthArg = sqlite3NameFromToken(&pAuthArg->span); + char *zAuthArg = sqlite3NameFromToken(db, &pAuthArg->span); if( !zAuthArg ){ goto attach_end; } rc = sqlite3AuthCheck(pParse, type, zAuthArg, 0, 0); - sqliteFree(zAuthArg); + sqlite3DbFree(db, zAuthArg); if(rc!=SQLITE_OK ){ goto attach_end; } @@ -319,27 +327,29 @@ static void codeAttach( } v = sqlite3GetVdbe(pParse); - sqlite3ExprCode(pParse, pFilename); - sqlite3ExprCode(pParse, pDbname); - sqlite3ExprCode(pParse, pKey); + regArgs = sqlite3GetTempRange(pParse, 4); + sqlite3ExprCode(pParse, pFilename, regArgs); + sqlite3ExprCode(pParse, pDbname, regArgs+1); + sqlite3ExprCode(pParse, pKey, regArgs+2); - assert( v || sqlite3MallocFailed() ); + assert( v || db->mallocFailed ); if( v ){ - sqlite3VdbeAddOp(v, OP_Function, 0, nFunc); - pFunc = sqlite3FindFunction(db, zFunc, strlen(zFunc), nFunc, SQLITE_UTF8,0); - sqlite3VdbeChangeP3(v, -1, (char *)pFunc, P3_FUNCDEF); + sqlite3VdbeAddOp3(v, OP_Function, 0, regArgs+3-pFunc->nArg, regArgs+3); + assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg ); + sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg)); + sqlite3VdbeChangeP4(v, -1, (char *)pFunc, P4_FUNCDEF); /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this ** statement only). For DETACH, set it to false (expire all existing ** statements). */ - sqlite3VdbeAddOp(v, OP_Expire, (type==SQLITE_ATTACH), 0); + sqlite3VdbeAddOp1(v, OP_Expire, (type==SQLITE_ATTACH)); } attach_end: - sqlite3ExprDelete(pFilename); - sqlite3ExprDelete(pDbname); - sqlite3ExprDelete(pKey); + sqlite3ExprDelete(db, pFilename); + sqlite3ExprDelete(db, pDbname); + sqlite3ExprDelete(db, pKey); } /* @@ -348,7 +358,19 @@ attach_end: ** DETACH pDbname */ void sqlite3Detach(Parse *pParse, Expr *pDbname){ - codeAttach(pParse, SQLITE_DETACH, "sqlite_detach", 1, pDbname, 0, 0, pDbname); + static FuncDef detach_func = { + 1, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + detachFunc, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "sqlite_detach", /* zName */ + 0 /* pHash */ + }; + codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); } /* @@ -357,21 +379,22 @@ void sqlite3Detach(Parse *pParse, Expr *pDbname){ ** ATTACH p AS pDbname KEY pKey */ void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *pKey){ - codeAttach(pParse, SQLITE_ATTACH, "sqlite_attach", 3, p, p, pDbname, pKey); + static FuncDef attach_func = { + 3, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + attachFunc, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "sqlite_attach", /* zName */ + 0 /* pHash */ + }; + codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); } #endif /* SQLITE_OMIT_ATTACH */ -/* -** Register the functions sqlite_attach and sqlite_detach. -*/ -void sqlite3AttachFunctions(sqlite3 *db){ -#ifndef SQLITE_OMIT_ATTACH - static const int enc = SQLITE_UTF8; - sqlite3CreateFunc(db, "sqlite_attach", 3, enc, db, attachFunc, 0, 0); - sqlite3CreateFunc(db, "sqlite_detach", 1, enc, db, detachFunc, 0, 0); -#endif -} - /* ** Initialize a DbFixer structure. This routine must be called prior ** to passing the structure to one of the sqliteFixAAAA() routines below. @@ -424,7 +447,7 @@ int sqlite3FixSrcList( zDb = pFix->zDb; for(i=0, pItem=pList->a; inSrc; i++, pItem++){ if( pItem->zDatabase==0 ){ - pItem->zDatabase = sqliteStrDup(zDb); + pItem->zDatabase = sqlite3DbStrDup(pFix->pParse->db, zDb); }else if( sqlite3StrICmp(pItem->zDatabase,zDb)!=0 ){ sqlite3ErrorMsg(pFix->pParse, "%s %T cannot reference objects in database %s", diff --git a/auth.c b/auth.c index 79940c2..5630c23 100644 --- a/auth.c +++ b/auth.c @@ -14,7 +14,7 @@ ** systems that do not need this facility may omit it by recompiling ** the library with -DSQLITE_OMIT_AUTHORIZATION=1 ** -** $Id: auth.c,v 1.26 2007/05/14 11:34:47 drh Exp $ +** $Id: auth.c,v 1.29 2007/09/18 15:55:07 drh Exp $ */ #include "sqliteInt.h" @@ -74,9 +74,11 @@ int sqlite3_set_authorizer( int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pArg ){ + sqlite3_mutex_enter(db->mutex); db->xAuth = xAuth; db->pAuthArg = pArg; sqlite3ExpirePreparedStatements(db); + sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -103,11 +105,12 @@ static void sqliteAuthBadReturnCode(Parse *pParse, int rc){ void sqlite3AuthRead( Parse *pParse, /* The parser context */ Expr *pExpr, /* The expression to check authorization on */ + Schema *pSchema, /* The schema of the expression */ SrcList *pTabList /* All table that pExpr might refer to */ ){ sqlite3 *db = pParse->db; int rc; - Table *pTab; /* The table being read */ + Table *pTab = 0; /* The table being read */ const char *zCol; /* Name of the column of the table */ int iSrc; /* Index in pTabList->a[] of table being read */ const char *zDBase; /* Name of database being accessed */ @@ -116,7 +119,7 @@ void sqlite3AuthRead( if( db->xAuth==0 ) return; if( pExpr->op!=TK_COLUMN ) return; - iDb = sqlite3SchemaToIndex(pParse->db, pExpr->pSchema); + iDb = sqlite3SchemaToIndex(pParse->db, pSchema); if( iDb<0 ){ /* An attempt to read a column out of a subquery or other ** temporary table. */ @@ -133,8 +136,6 @@ void sqlite3AuthRead( */ assert( pExpr->iTable==pStack->newIdx || pExpr->iTable==pStack->oldIdx ); pTab = pStack->pTab; - }else{ - return; } if( pTab==0 ) return; if( pExpr->iColumn>=0 ){ diff --git a/bitvec.c b/bitvec.c new file mode 100644 index 0000000..4098e57 --- /dev/null +++ b/bitvec.c @@ -0,0 +1,386 @@ +/* +** 2008 February 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements an object that represents a fixed-length +** bitmap. Bits are numbered starting with 1. +** +** A bitmap is used to record which pages of a database file have been +** journalled during a transaction, or which pages have the "dont-write" +** property. Usually only a few pages are meet either condition. +** So the bitmap is usually sparse and has low cardinality. +** But sometimes (for example when during a DROP of a large table) most +** or all of the pages in a database can get journalled. In those cases, +** the bitmap becomes dense with high cardinality. The algorithm needs +** to handle both cases well. +** +** The size of the bitmap is fixed when the object is created. +** +** All bits are clear when the bitmap is created. Individual bits +** may be set or cleared one at a time. +** +** Test operations are about 100 times more common that set operations. +** Clear operations are exceedingly rare. There are usually between +** 5 and 500 set operations per Bitvec object, though the number of sets can +** sometimes grow into tens of thousands or larger. The size of the +** Bitvec object is the number of pages in the database file at the +** start of a transaction, and is thus usually less than a few thousand, +** but can be as large as 2 billion for a really big database. +** +** @(#) $Id: bitvec.c,v 1.10 2009/01/02 21:39:39 drh Exp $ +*/ +#include "sqliteInt.h" + +/* Size of the Bitvec structure in bytes. */ +#define BITVEC_SZ 512 + +/* Round the union size down to the nearest pointer boundary, since that's how +** it will be aligned within the Bitvec struct. */ +#define BITVEC_USIZE (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) + +/* Type of the array "element" for the bitmap representation. +** Should be a power of 2, and ideally, evenly divide into BITVEC_USIZE. +** Setting this to the "natural word" size of your CPU may improve +** performance. */ +#define BITVEC_TELEM u8 +/* Size, in bits, of the bitmap element. */ +#define BITVEC_SZELEM 8 +/* Number of elements in a bitmap array. */ +#define BITVEC_NELEM (BITVEC_USIZE/sizeof(BITVEC_TELEM)) +/* Number of bits in the bitmap array. */ +#define BITVEC_NBIT (BITVEC_NELEM*BITVEC_SZELEM) + +/* Number of u32 values in hash table. */ +#define BITVEC_NINT (BITVEC_USIZE/sizeof(u32)) +/* Maximum number of entries in hash table before +** sub-dividing and re-hashing. */ +#define BITVEC_MXHASH (BITVEC_NINT/2) +/* Hashing function for the aHash representation. +** Empirical testing showed that the *37 multiplier +** (an arbitrary prime)in the hash function provided +** no fewer collisions than the no-op *1. */ +#define BITVEC_HASH(X) (((X)*1)%BITVEC_NINT) + +#define BITVEC_NPTR (BITVEC_USIZE/sizeof(Bitvec *)) + + +/* +** A bitmap is an instance of the following structure. +** +** This bitmap records the existance of zero or more bits +** with values between 1 and iSize, inclusive. +** +** There are three possible representations of the bitmap. +** If iSize<=BITVEC_NBIT, then Bitvec.u.aBitmap[] is a straight +** bitmap. The least significant bit is bit 1. +** +** If iSize>BITVEC_NBIT and iDivisor==0 then Bitvec.u.aHash[] is +** a hash table that will hold up to BITVEC_MXHASH distinct values. +** +** Otherwise, the value i is redirected into one of BITVEC_NPTR +** sub-bitmaps pointed to by Bitvec.u.apSub[]. Each subbitmap +** handles up to iDivisor separate values of i. apSub[0] holds +** values between 1 and iDivisor. apSub[1] holds values between +** iDivisor+1 and 2*iDivisor. apSub[N] holds values between +** N*iDivisor+1 and (N+1)*iDivisor. Each subbitmap is normalized +** to hold deal with values between 1 and iDivisor. +*/ +struct Bitvec { + u32 iSize; /* Maximum bit index. Max iSize is 4,294,967,296. */ + u32 nSet; /* Number of bits that are set - only valid for aHash element */ + /* Max nSet is BITVEC_NINT. For BITVEC_SZ of 512, this would be 125. */ + u32 iDivisor; /* Number of bits handled by each apSub[] entry. */ + /* Should >=0 for apSub element. */ + /* Max iDivisor is max(u32) / BITVEC_NPTR + 1. */ + /* For a BITVEC_SZ of 512, this would be 34,359,739. */ + union { + BITVEC_TELEM aBitmap[BITVEC_NELEM]; /* Bitmap representation */ + u32 aHash[BITVEC_NINT]; /* Hash table representation */ + Bitvec *apSub[BITVEC_NPTR]; /* Recursive representation */ + } u; +}; + +/* +** Create a new bitmap object able to handle bits between 0 and iSize, +** inclusive. Return a pointer to the new object. Return NULL if +** malloc fails. +*/ +Bitvec *sqlite3BitvecCreate(u32 iSize){ + Bitvec *p; + assert( sizeof(*p)==BITVEC_SZ ); + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ + p->iSize = iSize; + } + return p; +} + +/* +** Check to see if the i-th bit is set. Return true or false. +** If p is NULL (if the bitmap has not been created) or if +** i is out of range, then return false. +*/ +int sqlite3BitvecTest(Bitvec *p, u32 i){ + if( p==0 ) return 0; + if( i>p->iSize || i==0 ) return 0; + i--; + while( p->iDivisor ){ + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + p = p->u.apSub[bin]; + if (!p) { + return 0; + } + } + if( p->iSize<=BITVEC_NBIT ){ + return (p->u.aBitmap[i/BITVEC_SZELEM] & (1<<(i&(BITVEC_SZELEM-1))))!=0; + } else{ + u32 h = BITVEC_HASH(i++); + while( p->u.aHash[h] ){ + if( p->u.aHash[h]==i ) return 1; + h++; + if( h>=BITVEC_NINT ) h = 0; + } + return 0; + } +} + +/* +** Set the i-th bit. Return 0 on success and an error code if +** anything goes wrong. +** +** This routine might cause sub-bitmaps to be allocated. Failing +** to get the memory needed to hold the sub-bitmap is the only +** that can go wrong with an insert, assuming p and i are valid. +** +** The calling function must ensure that p is a valid Bitvec object +** and that the value for "i" is within range of the Bitvec object. +** Otherwise the behavior is undefined. +*/ +int sqlite3BitvecSet(Bitvec *p, u32 i){ + u32 h; + assert( p!=0 ); + assert( i>0 ); + assert( i<=p->iSize ); + i--; + while((p->iSize > BITVEC_NBIT) && p->iDivisor) { + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + if( p->u.apSub[bin]==0 ){ + p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor ); + if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM; + } + p = p->u.apSub[bin]; + } + if( p->iSize<=BITVEC_NBIT ){ + p->u.aBitmap[i/BITVEC_SZELEM] |= 1 << (i&(BITVEC_SZELEM-1)); + return SQLITE_OK; + } + h = BITVEC_HASH(i++); + /* if there wasn't a hash collision, and this doesn't */ + /* completely fill the hash, then just add it without */ + /* worring about sub-dividing and re-hashing. */ + if( !p->u.aHash[h] ){ + if (p->nSet<(BITVEC_NINT-1)) { + goto bitvec_set_end; + } else { + goto bitvec_set_rehash; + } + } + /* there was a collision, check to see if it's already */ + /* in hash, if not, try to find a spot for it */ + do { + if( p->u.aHash[h]==i ) return SQLITE_OK; + h++; + if( h>=BITVEC_NINT ) h = 0; + } while( p->u.aHash[h] ); + /* we didn't find it in the hash. h points to the first */ + /* available free spot. check to see if this is going to */ + /* make our hash too "full". */ +bitvec_set_rehash: + if( p->nSet>=BITVEC_MXHASH ){ + unsigned int j; + int rc; + u32 aiValues[BITVEC_NINT]; + memcpy(aiValues, p->u.aHash, sizeof(aiValues)); + memset(p->u.apSub, 0, sizeof(aiValues)); + p->iDivisor = (p->iSize + BITVEC_NPTR - 1)/BITVEC_NPTR; + rc = sqlite3BitvecSet(p, i); + for(j=0; jnSet++; + p->u.aHash[h] = i; + return SQLITE_OK; +} + +/* +** Clear the i-th bit. +*/ +void sqlite3BitvecClear(Bitvec *p, u32 i){ + assert( p!=0 ); + assert( i>0 ); + i--; + while( p->iDivisor ){ + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + p = p->u.apSub[bin]; + if (!p) { + return; + } + } + if( p->iSize<=BITVEC_NBIT ){ + p->u.aBitmap[i/BITVEC_SZELEM] &= ~(1 << (i&(BITVEC_SZELEM-1))); + }else{ + unsigned int j; + u32 aiValues[BITVEC_NINT]; + memcpy(aiValues, p->u.aHash, sizeof(aiValues)); + memset(p->u.aHash, 0, sizeof(aiValues)); + p->nSet = 0; + for(j=0; jnSet++; + while( p->u.aHash[h] ){ + h++; + if( h>=BITVEC_NINT ) h = 0; + } + p->u.aHash[h] = aiValues[j]; + } + } + } +} + +/* +** Destroy a bitmap object. Reclaim all memory used. +*/ +void sqlite3BitvecDestroy(Bitvec *p){ + if( p==0 ) return; + if( p->iDivisor ){ + unsigned int i; + for(i=0; iu.apSub[i]); + } + } + sqlite3_free(p); +} + +#ifndef SQLITE_OMIT_BUILTIN_TEST +/* +** Let V[] be an array of unsigned characters sufficient to hold +** up to N bits. Let I be an integer between 0 and N. 0<=I>3] |= (1<<(I&7)) +#define CLEARBIT(V,I) V[I>>3] &= ~(1<<(I&7)) +#define TESTBIT(V,I) (V[I>>3]&(1<<(I&7)))!=0 + +/* +** This routine runs an extensive test of the Bitvec code. +** +** The input is an array of integers that acts as a program +** to test the Bitvec. The integers are opcodes followed +** by 0, 1, or 3 operands, depending on the opcode. Another +** opcode follows immediately after the last operand. +** +** There are 6 opcodes numbered from 0 through 5. 0 is the +** "halt" opcode and causes the test to end. +** +** 0 Halt and return the number of errors +** 1 N S X Set N bits beginning with S and incrementing by X +** 2 N S X Clear N bits beginning with S and incrementing by X +** 3 N Set N randomly chosen bits +** 4 N Clear N randomly chosen bits +** 5 N S X Set N bits from S increment X in array only, not in bitvec +** +** The opcodes 1 through 4 perform set and clear operations are performed +** on both a Bitvec object and on a linear array of bits obtained from malloc. +** Opcode 5 works on the linear array only, not on the Bitvec. +** Opcode 5 is used to deliberately induce a fault in order to +** confirm that error detection works. +** +** At the conclusion of the test the linear array is compared +** against the Bitvec object. If there are any differences, +** an error is returned. If they are the same, zero is returned. +** +** If a memory allocation error occurs, return -1. +*/ +int sqlite3BitvecBuiltinTest(int sz, int *aOp){ + Bitvec *pBitvec = 0; + unsigned char *pV = 0; + int rc = -1; + int i, nx, pc, op; + + /* Allocate the Bitvec to be tested and a linear array of + ** bits to act as the reference */ + pBitvec = sqlite3BitvecCreate( sz ); + pV = sqlite3_malloc( (sz+7)/8 + 1 ); + if( pBitvec==0 || pV==0 ) goto bitvec_end; + memset(pV, 0, (sz+7)/8 + 1); + + /* Run the program */ + pc = 0; + while( (op = aOp[pc])!=0 ){ + switch( op ){ + case 1: + case 2: + case 5: { + nx = 4; + i = aOp[pc+2] - 1; + aOp[pc+2] += aOp[pc+3]; + break; + } + case 3: + case 4: + default: { + nx = 2; + sqlite3_randomness(sizeof(i), &i); + break; + } + } + if( (--aOp[pc+1]) > 0 ) nx = 0; + pc += nx; + i = (i & 0x7fffffff)%sz; + if( (op & 1)!=0 ){ + SETBIT(pV, (i+1)); + if( op!=5 ){ + if( sqlite3BitvecSet(pBitvec, i+1) ) goto bitvec_end; + } + }else{ + CLEARBIT(pV, (i+1)); + sqlite3BitvecClear(pBitvec, i+1); + } + } + + /* Test to make sure the linear array exactly matches the + ** Bitvec object. Start with the assumption that they do + ** match (rc==0). Change rc to non-zero if a discrepancy + ** is found. + */ + rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1) + + sqlite3BitvecTest(pBitvec, 0); + for(i=1; i<=sz; i++){ + if( (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){ + rc = i; + break; + } + } + + /* Free allocated structure */ +bitvec_end: + sqlite3_free(pV); + sqlite3BitvecDestroy(pBitvec); + return rc; +} +#endif /* SQLITE_OMIT_BUILTIN_TEST */ diff --git a/btmutex.c b/btmutex.c new file mode 100644 index 0000000..b1ab517 --- /dev/null +++ b/btmutex.c @@ -0,0 +1,315 @@ +/* +** 2007 August 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** $Id: btmutex.c,v 1.12 2008/11/17 19:18:55 danielk1977 Exp $ +** +** This file contains code used to implement mutexes on Btree objects. +** This code really belongs in btree.c. But btree.c is getting too +** big and we want to break it down some. This packaged seemed like +** a good breakout. +*/ +#include "btreeInt.h" +#if SQLITE_THREADSAFE && !defined(SQLITE_OMIT_SHARED_CACHE) + + +/* +** Enter a mutex on the given BTree object. +** +** If the object is not sharable, then no mutex is ever required +** and this routine is a no-op. The underlying mutex is non-recursive. +** But we keep a reference count in Btree.wantToLock so the behavior +** of this interface is recursive. +** +** To avoid deadlocks, multiple Btrees are locked in the same order +** by all database connections. The p->pNext is a list of other +** Btrees belonging to the same database connection as the p Btree +** which need to be locked after p. If we cannot get a lock on +** p, then first unlock all of the others on p->pNext, then wait +** for the lock to become available on p, then relock all of the +** subsequent Btrees that desire a lock. +*/ +void sqlite3BtreeEnter(Btree *p){ + Btree *pLater; + + /* Some basic sanity checking on the Btree. The list of Btrees + ** connected by pNext and pPrev should be in sorted order by + ** Btree.pBt value. All elements of the list should belong to + ** the same connection. Only shared Btrees are on the list. */ + assert( p->pNext==0 || p->pNext->pBt>p->pBt ); + assert( p->pPrev==0 || p->pPrev->pBtpBt ); + assert( p->pNext==0 || p->pNext->db==p->db ); + assert( p->pPrev==0 || p->pPrev->db==p->db ); + assert( p->sharable || (p->pNext==0 && p->pPrev==0) ); + + /* Check for locking consistency */ + assert( !p->locked || p->wantToLock>0 ); + assert( p->sharable || p->wantToLock==0 ); + + /* We should already hold a lock on the database connection */ + assert( sqlite3_mutex_held(p->db->mutex) ); + + if( !p->sharable ) return; + p->wantToLock++; + if( p->locked ) return; + + /* In most cases, we should be able to acquire the lock we + ** want without having to go throught the ascending lock + ** procedure that follows. Just be sure not to block. + */ + if( sqlite3_mutex_try(p->pBt->mutex)==SQLITE_OK ){ + p->locked = 1; + return; + } + + /* To avoid deadlock, first release all locks with a larger + ** BtShared address. Then acquire our lock. Then reacquire + ** the other BtShared locks that we used to hold in ascending + ** order. + */ + for(pLater=p->pNext; pLater; pLater=pLater->pNext){ + assert( pLater->sharable ); + assert( pLater->pNext==0 || pLater->pNext->pBt>pLater->pBt ); + assert( !pLater->locked || pLater->wantToLock>0 ); + if( pLater->locked ){ + sqlite3_mutex_leave(pLater->pBt->mutex); + pLater->locked = 0; + } + } + sqlite3_mutex_enter(p->pBt->mutex); + p->locked = 1; + for(pLater=p->pNext; pLater; pLater=pLater->pNext){ + if( pLater->wantToLock ){ + sqlite3_mutex_enter(pLater->pBt->mutex); + pLater->locked = 1; + } + } +} + +/* +** Exit the recursive mutex on a Btree. +*/ +void sqlite3BtreeLeave(Btree *p){ + if( p->sharable ){ + assert( p->wantToLock>0 ); + p->wantToLock--; + if( p->wantToLock==0 ){ + assert( p->locked ); + sqlite3_mutex_leave(p->pBt->mutex); + p->locked = 0; + } + } +} + +#ifndef NDEBUG +/* +** Return true if the BtShared mutex is held on the btree. +** +** This routine makes no determination one why or another if the +** database connection mutex is held. +** +** This routine is used only from within assert() statements. +*/ +int sqlite3BtreeHoldsMutex(Btree *p){ + return (p->sharable==0 || + (p->locked && p->wantToLock && sqlite3_mutex_held(p->pBt->mutex))); +} +#endif + + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** Enter and leave a mutex on a Btree given a cursor owned by that +** Btree. These entry points are used by incremental I/O and can be +** omitted if that module is not used. +*/ +void sqlite3BtreeEnterCursor(BtCursor *pCur){ + sqlite3BtreeEnter(pCur->pBtree); +} +void sqlite3BtreeLeaveCursor(BtCursor *pCur){ + sqlite3BtreeLeave(pCur->pBtree); +} +#endif /* SQLITE_OMIT_INCRBLOB */ + + +/* +** Enter the mutex on every Btree associated with a database +** connection. This is needed (for example) prior to parsing +** a statement since we will be comparing table and column names +** against all schemas and we do not want those schemas being +** reset out from under us. +** +** There is a corresponding leave-all procedures. +** +** Enter the mutexes in accending order by BtShared pointer address +** to avoid the possibility of deadlock when two threads with +** two or more btrees in common both try to lock all their btrees +** at the same instant. +*/ +void sqlite3BtreeEnterAll(sqlite3 *db){ + int i; + Btree *p, *pLater; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inDb; i++){ + p = db->aDb[i].pBt; + if( p && p->sharable ){ + p->wantToLock++; + if( !p->locked ){ + assert( p->wantToLock==1 ); + while( p->pPrev ) p = p->pPrev; + while( p->locked && p->pNext ) p = p->pNext; + for(pLater = p->pNext; pLater; pLater=pLater->pNext){ + if( pLater->locked ){ + sqlite3_mutex_leave(pLater->pBt->mutex); + pLater->locked = 0; + } + } + while( p ){ + sqlite3_mutex_enter(p->pBt->mutex); + p->locked++; + p = p->pNext; + } + } + } + } +} +void sqlite3BtreeLeaveAll(sqlite3 *db){ + int i; + Btree *p; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inDb; i++){ + p = db->aDb[i].pBt; + if( p && p->sharable ){ + assert( p->wantToLock>0 ); + p->wantToLock--; + if( p->wantToLock==0 ){ + assert( p->locked ); + sqlite3_mutex_leave(p->pBt->mutex); + p->locked = 0; + } + } + } +} + +#ifndef NDEBUG +/* +** Return true if the current thread holds the database connection +** mutex and all required BtShared mutexes. +** +** This routine is used inside assert() statements only. +*/ +int sqlite3BtreeHoldsAllMutexes(sqlite3 *db){ + int i; + if( !sqlite3_mutex_held(db->mutex) ){ + return 0; + } + for(i=0; inDb; i++){ + Btree *p; + p = db->aDb[i].pBt; + if( p && p->sharable && + (p->wantToLock==0 || !sqlite3_mutex_held(p->pBt->mutex)) ){ + return 0; + } + } + return 1; +} +#endif /* NDEBUG */ + +/* +** Add a new Btree pointer to a BtreeMutexArray. +** if the pointer can possibly be shared with +** another database connection. +** +** The pointers are kept in sorted order by pBtree->pBt. That +** way when we go to enter all the mutexes, we can enter them +** in order without every having to backup and retry and without +** worrying about deadlock. +** +** The number of shared btrees will always be small (usually 0 or 1) +** so an insertion sort is an adequate algorithm here. +*/ +void sqlite3BtreeMutexArrayInsert(BtreeMutexArray *pArray, Btree *pBtree){ + int i, j; + BtShared *pBt; + if( pBtree==0 || pBtree->sharable==0 ) return; +#ifndef NDEBUG + { + for(i=0; inMutex; i++){ + assert( pArray->aBtree[i]!=pBtree ); + } + } +#endif + assert( pArray->nMutex>=0 ); + assert( pArray->nMutexaBtree)-1 ); + pBt = pBtree->pBt; + for(i=0; inMutex; i++){ + assert( pArray->aBtree[i]!=pBtree ); + if( pArray->aBtree[i]->pBt>pBt ){ + for(j=pArray->nMutex; j>i; j--){ + pArray->aBtree[j] = pArray->aBtree[j-1]; + } + pArray->aBtree[i] = pBtree; + pArray->nMutex++; + return; + } + } + pArray->aBtree[pArray->nMutex++] = pBtree; +} + +/* +** Enter the mutex of every btree in the array. This routine is +** called at the beginning of sqlite3VdbeExec(). The mutexes are +** exited at the end of the same function. +*/ +void sqlite3BtreeMutexArrayEnter(BtreeMutexArray *pArray){ + int i; + for(i=0; inMutex; i++){ + Btree *p = pArray->aBtree[i]; + /* Some basic sanity checking */ + assert( i==0 || pArray->aBtree[i-1]->pBtpBt ); + assert( !p->locked || p->wantToLock>0 ); + + /* We should already hold a lock on the database connection */ + assert( sqlite3_mutex_held(p->db->mutex) ); + + p->wantToLock++; + if( !p->locked && p->sharable ){ + sqlite3_mutex_enter(p->pBt->mutex); + p->locked = 1; + } + } +} + +/* +** Leave the mutex of every btree in the group. +*/ +void sqlite3BtreeMutexArrayLeave(BtreeMutexArray *pArray){ + int i; + for(i=0; inMutex; i++){ + Btree *p = pArray->aBtree[i]; + /* Some basic sanity checking */ + assert( i==0 || pArray->aBtree[i-1]->pBtpBt ); + assert( p->locked || !p->sharable ); + assert( p->wantToLock>0 ); + + /* We should already hold a lock on the database connection */ + assert( sqlite3_mutex_held(p->db->mutex) ); + + p->wantToLock--; + if( p->wantToLock==0 && p->locked ){ + sqlite3_mutex_leave(p->pBt->mutex); + p->locked = 0; + } + } +} + + +#endif /* SQLITE_THREADSAFE && !SQLITE_OMIT_SHARED_CACHE */ diff --git a/btree.c b/btree.c index 1e3a43e..fbf886c 100644 --- a/btree.c +++ b/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.396 2007/08/13 14:56:44 drh Exp $ +** $Id: btree.c,v 1.558 2009/01/10 16:15:21 drh Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** See the header comment on "btreeInt.h" for additional information. @@ -23,19 +23,52 @@ */ static const char zMagicHeader[] = SQLITE_FILE_HEADER; - /* ** Set this global variable to 1 to enable tracing using the TRACE ** macro. */ -#if SQLITE_TEST -int sqlite3_btree_trace=0; /* True to enable tracing */ +#if 0 +int sqlite3BtreeTrace=0; /* True to enable tracing */ +# define TRACE(X) if(sqlite3BtreeTrace){printf X;fflush(stdout);} +#else +# define TRACE(X) #endif + + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** A list of BtShared objects that are eligible for participation +** in shared cache. This variable has file scope during normal builds, +** but the test harness needs to access it so we make it global for +** test builds. +*/ +#ifdef SQLITE_TEST +BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; +#else +static BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; +#endif +#endif /* SQLITE_OMIT_SHARED_CACHE */ + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Enable or disable the shared pager and schema features. +** +** This routine has no effect on existing database connections. +** The shared cache setting effects only future calls to +** sqlite3_open(), sqlite3_open16(), or sqlite3_open_v2(). +*/ +int sqlite3_enable_shared_cache(int enable){ + sqlite3GlobalConfig.sharedCacheEnabled = enable; + return SQLITE_OK; +} +#endif + + /* ** Forward declaration */ -static int checkReadLocks(Btree*,Pgno,BtCursor*); +static int checkReadLocks(Btree*, Pgno, BtCursor*, i64); #ifdef SQLITE_OMIT_SHARED_CACHE @@ -50,8 +83,9 @@ static int checkReadLocks(Btree*,Pgno,BtCursor*); #define queryTableLock(a,b,c) SQLITE_OK #define lockTable(a,b,c) SQLITE_OK #define unlockAllTables(a) -#else +#endif +#ifndef SQLITE_OMIT_SHARED_CACHE /* ** Query to see if btree handle p may obtain a lock of type eLock ** (READ_LOCK or WRITE_LOCK) on the table with root-page iTab. Return @@ -62,11 +96,22 @@ static int queryTableLock(Btree *p, Pgno iTab, u8 eLock){ BtShared *pBt = p->pBt; BtLock *pIter; + assert( sqlite3BtreeHoldsMutex(p) ); + assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); + assert( p->db!=0 ); + /* This is a no-op if the shared-cache is not enabled */ - if( 0==sqlite3ThreadDataReadOnly()->useSharedData ){ + if( !p->sharable ){ return SQLITE_OK; } + /* If some other connection is holding an exclusive lock, the + ** requested lock may not be obtained. + */ + if( pBt->pExclusive && pBt->pExclusive!=p ){ + return SQLITE_LOCKED; + } + /* This (along with lockTable()) is where the ReadUncommitted flag is ** dealt with. If the caller is querying for a read-lock and the flag is ** set, it is unconditionally granted - even if there are write-locks @@ -82,8 +127,7 @@ static int queryTableLock(Btree *p, Pgno iTab, u8 eLock){ ** write-cursor does not change. */ if( - !p->pSqlite || - 0==(p->pSqlite->flags&SQLITE_ReadUncommitted) || + 0==(p->db->flags&SQLITE_ReadUncommitted) || eLock==WRITE_LOCK || iTab==MASTER_ROOT ){ @@ -96,7 +140,9 @@ static int queryTableLock(Btree *p, Pgno iTab, u8 eLock){ } return SQLITE_OK; } +#endif /* !SQLITE_OMIT_SHARED_CACHE */ +#ifndef SQLITE_OMIT_SHARED_CACHE /* ** Add a lock on the table with root-page iTable to the shared-btree used ** by Btree handle p. Parameter eLock must be either READ_LOCK or @@ -110,8 +156,12 @@ static int lockTable(Btree *p, Pgno iTable, u8 eLock){ BtLock *pLock = 0; BtLock *pIter; + assert( sqlite3BtreeHoldsMutex(p) ); + assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); + assert( p->db!=0 ); + /* This is a no-op if the shared-cache is not enabled */ - if( 0==sqlite3ThreadDataReadOnly()->useSharedData ){ + if( !p->sharable ){ return SQLITE_OK; } @@ -123,8 +173,7 @@ static int lockTable(Btree *p, Pgno iTable, u8 eLock){ ** the ReadUncommitted flag. */ if( - (p->pSqlite) && - (p->pSqlite->flags&SQLITE_ReadUncommitted) && + (p->db->flags&SQLITE_ReadUncommitted) && (eLock==READ_LOCK) && iTable!=MASTER_ROOT ){ @@ -143,7 +192,7 @@ static int lockTable(Btree *p, Pgno iTable, u8 eLock){ ** with table iTable, allocate one and link it into the list. */ if( !pLock ){ - pLock = (BtLock *)sqliteMalloc(sizeof(BtLock)); + pLock = (BtLock *)sqlite3MallocZero(sizeof(BtLock)); if( !pLock ){ return SQLITE_NOMEM; } @@ -164,40 +213,56 @@ static int lockTable(Btree *p, Pgno iTable, u8 eLock){ return SQLITE_OK; } +#endif /* !SQLITE_OMIT_SHARED_CACHE */ +#ifndef SQLITE_OMIT_SHARED_CACHE /* ** Release all the table locks (locks obtained via calls to the lockTable() ** procedure) held by Btree handle p. */ static void unlockAllTables(Btree *p){ - BtLock **ppIter = &p->pBt->pLock; + BtShared *pBt = p->pBt; + BtLock **ppIter = &pBt->pLock; - /* If the shared-cache extension is not enabled, there should be no - ** locks in the BtShared.pLock list, making this procedure a no-op. Assert - ** that this is the case. - */ - assert( sqlite3ThreadDataReadOnly()->useSharedData || 0==*ppIter ); + assert( sqlite3BtreeHoldsMutex(p) ); + assert( p->sharable || 0==*ppIter ); while( *ppIter ){ BtLock *pLock = *ppIter; + assert( pBt->pExclusive==0 || pBt->pExclusive==pLock->pBtree ); if( pLock->pBtree==p ){ *ppIter = pLock->pNext; - sqliteFree(pLock); + sqlite3_free(pLock); }else{ ppIter = &pLock->pNext; } } + + if( pBt->pExclusive==p ){ + pBt->pExclusive = 0; + } } #endif /* SQLITE_OMIT_SHARED_CACHE */ static void releasePage(MemPage *pPage); /* Forward reference */ +/* +** Verify that the cursor holds a mutex on the BtShared +*/ +#ifndef NDEBUG +static int cursorHoldsMutex(BtCursor *p){ + return sqlite3_mutex_held(p->pBt->mutex); +} +#endif + + #ifndef SQLITE_OMIT_INCRBLOB /* ** Invalidate the overflow page-list cache for cursor pCur, if any. */ static void invalidateOverflowCache(BtCursor *pCur){ - sqliteFree(pCur->aOverflow); + assert( cursorHoldsMutex(pCur) ); + sqlite3_free(pCur->aOverflow); pCur->aOverflow = 0; } @@ -207,6 +272,7 @@ static void invalidateOverflowCache(BtCursor *pCur){ */ static void invalidateAllOverflowCache(BtShared *pBt){ BtCursor *p; + assert( sqlite3_mutex_held(pBt->mutex) ); for(p=pBt->pCursor; p; p=p->pNext){ invalidateOverflowCache(p); } @@ -225,6 +291,7 @@ static int saveCursorPosition(BtCursor *pCur){ assert( CURSOR_VALID==pCur->eState ); assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); @@ -234,24 +301,28 @@ static int saveCursorPosition(BtCursor *pCur){ ** table, then malloc space for and store the pCur->nKey bytes of key ** data. */ - if( rc==SQLITE_OK && 0==pCur->pPage->intKey){ - void *pKey = sqliteMalloc(pCur->nKey); + if( rc==SQLITE_OK && 0==pCur->apPage[0]->intKey){ + void *pKey = sqlite3Malloc( (int)pCur->nKey ); if( pKey ){ - rc = sqlite3BtreeKey(pCur, 0, pCur->nKey, pKey); + rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); if( rc==SQLITE_OK ){ pCur->pKey = pKey; }else{ - sqliteFree(pKey); + sqlite3_free(pKey); } }else{ rc = SQLITE_NOMEM; } } - assert( !pCur->pPage->intKey || !pCur->pKey ); + assert( !pCur->apPage[0]->intKey || !pCur->pKey ); if( rc==SQLITE_OK ){ - releasePage(pCur->pPage); - pCur->pPage = 0; + int i; + for(i=0; i<=pCur->iPage; i++){ + releasePage(pCur->apPage[i]); + pCur->apPage[i] = 0; + } + pCur->iPage = -1; pCur->eState = CURSOR_REQUIRESEEK; } @@ -266,6 +337,8 @@ static int saveCursorPosition(BtCursor *pCur){ */ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ BtCursor *p; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pExcept==0 || pExcept->pBt==pBt ); for(p=pBt->pCursor; p; p=p->pNext){ if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) && p->eState==CURSOR_VALID ){ @@ -281,8 +354,9 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ /* ** Clear the current cursor position. */ -static void clearCursorPosition(BtCursor *pCur){ - sqliteFree(pCur->pKey); +void sqlite3BtreeClearCursor(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + sqlite3_free(pCur->pKey); pCur->pKey = 0; pCur->eState = CURSOR_INVALID; } @@ -291,36 +365,55 @@ static void clearCursorPosition(BtCursor *pCur){ ** Restore the cursor to the position it was in (or as close to as possible) ** when saveCursorPosition() was called. Note that this call deletes the ** saved position info stored by saveCursorPosition(), so there can be -** at most one effective restoreOrClearCursorPosition() call after each +** at most one effective restoreCursorPosition() call after each ** saveCursorPosition(). -** -** If the second argument argument - doSeek - is false, then instead of -** returning the cursor to it's saved position, any saved position is deleted -** and the cursor state set to CURSOR_INVALID. */ -int sqlite3BtreeRestoreOrClearCursorPosition(BtCursor *pCur){ +int sqlite3BtreeRestoreCursorPosition(BtCursor *pCur){ int rc; - assert( pCur->eState==CURSOR_REQUIRESEEK ); -#ifndef SQLITE_OMIT_INCRBLOB - if( pCur->isIncrblobHandle ){ - return SQLITE_ABORT; + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState>=CURSOR_REQUIRESEEK ); + if( pCur->eState==CURSOR_FAULT ){ + return pCur->skip; } -#endif pCur->eState = CURSOR_INVALID; rc = sqlite3BtreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &pCur->skip); if( rc==SQLITE_OK ){ - sqliteFree(pCur->pKey); + sqlite3_free(pCur->pKey); pCur->pKey = 0; assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID ); } return rc; } -#define restoreOrClearCursorPosition(p) \ - (p->eState==CURSOR_REQUIRESEEK ? \ - sqlite3BtreeRestoreOrClearCursorPosition(p) : \ +#define restoreCursorPosition(p) \ + (p->eState>=CURSOR_REQUIRESEEK ? \ + sqlite3BtreeRestoreCursorPosition(p) : \ SQLITE_OK) +/* +** Determine whether or not a cursor has moved from the position it +** was last placed at. Cursors can move when the row they are pointing +** at is deleted out from under them. +** +** This routine returns an error code if something goes wrong. The +** integer *pHasMoved is set to one if the cursor has moved and 0 if not. +*/ +int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ + int rc; + + rc = restoreCursorPosition(pCur); + if( rc ){ + *pHasMoved = 1; + return rc; + } + if( pCur->eState!=CURSOR_VALID || pCur->skip!=0 ){ + *pHasMoved = 1; + }else{ + *pHasMoved = 0; + } + return SQLITE_OK; +} + #ifndef SQLITE_OMIT_AUTOVACUUM /* ** Given a page number of a regular database page, return the page @@ -328,9 +421,12 @@ int sqlite3BtreeRestoreOrClearCursorPosition(BtCursor *pCur){ ** input page number. */ static Pgno ptrmapPageno(BtShared *pBt, Pgno pgno){ - int nPagesPerMapPage = (pBt->usableSize/5)+1; - int iPtrMap = (pgno-2)/nPagesPerMapPage; - int ret = (iPtrMap*nPagesPerMapPage) + 2; + int nPagesPerMapPage; + Pgno iPtrMap, ret; + assert( sqlite3_mutex_held(pBt->mutex) ); + nPagesPerMapPage = (pBt->usableSize/5)+1; + iPtrMap = (pgno-2)/nPagesPerMapPage; + ret = (iPtrMap*nPagesPerMapPage) + 2; if( ret==PENDING_BYTE_PAGE(pBt) ){ ret++; } @@ -351,6 +447,7 @@ static int ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent){ int offset; /* Offset in pointer map page */ int rc; + assert( sqlite3_mutex_held(pBt->mutex) ); /* The master-journal page number must never be used as a pointer map page */ assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); @@ -363,7 +460,7 @@ static int ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent){ if( rc!=SQLITE_OK ){ return rc; } - offset = PTRMAP_PTROFFSET(pBt, key); + offset = PTRMAP_PTROFFSET(iPtrmap, key); pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ @@ -393,6 +490,8 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ int offset; /* Offset of entry in pointer map */ int rc; + assert( sqlite3_mutex_held(pBt->mutex) ); + iPtrmap = PTRMAP_PAGENO(pBt, key); rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage); if( rc!=0 ){ @@ -400,7 +499,7 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ } pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); - offset = PTRMAP_PTROFFSET(pBt, key); + offset = PTRMAP_PTROFFSET(iPtrmap, key); assert( pEType!=0 ); *pEType = pPtrmap[offset]; if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]); @@ -410,7 +509,11 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ return SQLITE_OK; } -#endif /* SQLITE_OMIT_AUTOVACUUM */ +#else /* if defined SQLITE_OMIT_AUTOVACUUM */ + #define ptrmapPut(w,x,y,z) SQLITE_OK + #define ptrmapGet(w,x,y,z) SQLITE_OK + #define ptrmapPutOvfl(y,z) SQLITE_OK +#endif /* ** Given a btree page and a cell index (0 means the first cell on @@ -419,22 +522,16 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ ** ** This routine works only for pages that do not contain overflow cells. */ -#define findCell(pPage, iCell) \ - ((pPage)->aData + get2byte(&(pPage)->aData[(pPage)->cellOffset+2*(iCell)])) -#ifdef SQLITE_TEST -u8 *sqlite3BtreeFindCell(MemPage *pPage, int iCell){ - assert( iCell>=0 ); - assert( iCellaData[pPage->hdrOffset+3]) ); - return findCell(pPage, iCell); -} -#endif +#define findCell(P,I) \ + ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)]))) /* -** This a more complex version of sqlite3BtreeFindCell() that works for +** This a more complex version of findCell() that works for ** pages that do contain overflow cells. See insert */ static u8 *findOverflowCell(MemPage *pPage, int iCell){ int i; + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); for(i=pPage->nOverflow-1; i>=0; i--){ int k; struct _OvflCell *pOvfl; @@ -464,41 +561,42 @@ void sqlite3BtreeParseCellPtr( u8 *pCell, /* Pointer to the cell text. */ CellInfo *pInfo /* Fill in this structure */ ){ - int n; /* Number bytes in cell content header */ + u16 n; /* Number bytes in cell content header */ u32 nPayload; /* Number of bytes of cell payload */ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + pInfo->pCell = pCell; assert( pPage->leaf==0 || pPage->leaf==1 ); n = pPage->childPtrSize; assert( n==4-4*pPage->leaf ); - if( pPage->hasData ){ - n += getVarint32(&pCell[n], &nPayload); - }else{ - nPayload = 0; - } - pInfo->nData = nPayload; if( pPage->intKey ){ - n += getVarint(&pCell[n], (u64 *)&pInfo->nKey); + if( pPage->hasData ){ + n += getVarint32(&pCell[n], nPayload); + }else{ + nPayload = 0; + } + n += getVarint(&pCell[n], (u64*)&pInfo->nKey); + pInfo->nData = nPayload; }else{ - u32 x; - n += getVarint32(&pCell[n], &x); - pInfo->nKey = x; - nPayload += x; + pInfo->nData = 0; + n += getVarint32(&pCell[n], nPayload); + pInfo->nKey = nPayload; } pInfo->nPayload = nPayload; pInfo->nHeader = n; - if( nPayload<=pPage->maxLocal ){ + if( likely(nPayload<=pPage->maxLocal) ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. */ int nSize; /* Total size of cell content in bytes */ - pInfo->nLocal = nPayload; - pInfo->iOverflow = 0; nSize = nPayload + n; - if( nSize<4 ){ + pInfo->nLocal = (u16)nPayload; + pInfo->iOverflow = 0; + if( (nSize & ~3)==0 ){ nSize = 4; /* Minimum cell size is 4 */ } - pInfo->nSize = nSize; + pInfo->nSize = (u16)nSize; }else{ /* If the payload will not fit completely on the local page, we have ** to decide how much to store locally and how much to spill onto @@ -517,11 +615,11 @@ void sqlite3BtreeParseCellPtr( maxLocal = pPage->maxLocal; surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize - 4); if( surplus <= maxLocal ){ - pInfo->nLocal = surplus; + pInfo->nLocal = (u16)surplus; }else{ - pInfo->nLocal = minLocal; + pInfo->nLocal = (u16)minLocal; } - pInfo->iOverflow = pInfo->nLocal + n; + pInfo->iOverflow = (u16)(pInfo->nLocal + n); pInfo->nSize = pInfo->iOverflow + 4; } } @@ -542,13 +640,13 @@ void sqlite3BtreeParseCell( ** the space used by the cell pointer. */ #ifndef NDEBUG -static int cellSize(MemPage *pPage, int iCell){ +static u16 cellSize(MemPage *pPage, int iCell){ CellInfo info; sqlite3BtreeParseCell(pPage, iCell, &info); return info.nSize; } #endif -static int cellSizePtr(MemPage *pPage, u8 *pCell){ +static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ CellInfo info; sqlite3BtreeParseCellPtr(pPage, pCell, &info); return info.nSize; @@ -561,14 +659,13 @@ static int cellSizePtr(MemPage *pPage, u8 *pCell){ ** for the overflow page. */ static int ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell){ - if( pCell ){ - CellInfo info; - sqlite3BtreeParseCellPtr(pPage, pCell, &info); - assert( (info.nData+(pPage->intKey?0:info.nKey))==info.nPayload ); - if( (info.nData+(pPage->intKey?0:info.nKey))>info.nLocal ){ - Pgno ovfl = get4byte(&pCell[info.iOverflow]); - return ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno); - } + CellInfo info; + assert( pCell!=0 ); + sqlite3BtreeParseCellPtr(pPage, pCell, &info); + assert( (info.nData+(pPage->intKey?0:info.nKey))==info.nPayload ); + if( (info.nData+(pPage->intKey?0:info.nKey))>info.nLocal ){ + Pgno ovfl = get4byte(&pCell[info.iOverflow]); + return ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno); } return SQLITE_OK; } @@ -579,6 +676,7 @@ static int ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell){ */ static int ptrmapPutOvfl(MemPage *pPage, int iCell){ u8 *pCell; + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); pCell = findOverflowCell(pPage, iCell); return ptrmapPutOvflPtr(pPage, pCell); } @@ -599,7 +697,7 @@ static int defragmentPage(MemPage *pPage){ int size; /* Size of a cell */ int usableSize; /* Number of usable bytes on a page */ int cellOffset; /* Offset to the cell pointer array */ - int brk; /* Offset to the cell content area */ + int cbrk; /* Offset to the cell content area */ int nCell; /* Number of cells on the page */ unsigned char *data; /* The page data */ unsigned char *temp; /* Temp area for cell content */ @@ -608,35 +706,44 @@ static int defragmentPage(MemPage *pPage){ assert( pPage->pBt!=0 ); assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); - temp = sqliteMalloc( pPage->pBt->pageSize ); - if( temp==0 ) return SQLITE_NOMEM; + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); usableSize = pPage->pBt->usableSize; - brk = get2byte(&data[hdr+5]); - memcpy(&temp[brk], &data[brk], usableSize - brk); - brk = usableSize; + cbrk = get2byte(&data[hdr+5]); + memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk); + cbrk = usableSize; for(i=0; ipBt->usableSize ); + if( pc>=usableSize ){ + return SQLITE_CORRUPT_BKPT; + } size = cellSizePtr(pPage, &temp[pc]); - brk -= size; - memcpy(&data[brk], &temp[pc], size); - put2byte(pAddr, brk); + cbrk -= size; + if( cbrkusableSize ){ + return SQLITE_CORRUPT_BKPT; + } + assert( cbrk+size<=usableSize && cbrk>=0 ); + memcpy(&data[cbrk], &temp[pc], size); + put2byte(pAddr, cbrk); } - assert( brk>=cellOffset+2*nCell ); - put2byte(&data[hdr+5], brk); + assert( cbrk>=cellOffset+2*nCell ); + put2byte(&data[hdr+5], cbrk); data[hdr+1] = 0; data[hdr+2] = 0; data[hdr+7] = 0; addr = cellOffset+2*nCell; - memset(&data[addr], 0, brk-addr); - sqliteFree(temp); + memset(&data[addr], 0, cbrk-addr); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + if( cbrk-addr!=pPage->nFree ){ + return SQLITE_CORRUPT_BKPT; + } return SQLITE_OK; } @@ -644,8 +751,8 @@ static int defragmentPage(MemPage *pPage){ ** Allocate nByte bytes of space on a page. ** ** Return the index into pPage->aData[] of the first byte of -** the new allocation. Or return 0 if there is not enough free -** space on the page to satisfy the allocation request. +** the new allocation. The caller guarantees that there is enough +** space. This routine will never fail. ** ** If the page contains nBytes of free space but does not contain ** nBytes of contiguous free space, then this routine automatically @@ -664,9 +771,11 @@ static int allocateSpace(MemPage *pPage, int nByte){ data = pPage->aData; assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt ); - if( nByte<4 ) nByte = 4; - if( pPage->nFreenOverflow>0 ) return 0; - pPage->nFree -= nByte; + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( nByte>=0 ); /* Minimum cell size is 4 */ + assert( pPage->nFree>=nByte ); + assert( pPage->nOverflow==0 ); + pPage->nFree -= (u16)nByte; hdr = pPage->hdrOffset; nFrag = data[hdr+7]; @@ -677,13 +786,14 @@ static int allocateSpace(MemPage *pPage, int nByte){ while( (pc = get2byte(&data[addr]))>0 ){ size = get2byte(&data[pc+2]); if( size>=nByte ){ + int x = size - nByte; if( sizecellOffset; if( nFrag>=60 || cellOffset + 2*nCell > top - nByte ){ - if( defragmentPage(pPage) ) return 0; + defragmentPage(pPage); top = get2byte(&data[hdr+5]); } top -= nByte; assert( cellOffset + 2*nCell <= top ); put2byte(&data[hdr+5], top); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); return top; } @@ -714,7 +825,7 @@ static int allocateSpace(MemPage *pPage, int nByte){ ** Most of the effort here is involved in coalesing adjacent ** free blocks into a single big free block. */ -static void freeSpace(MemPage *pPage, int start, int size){ +static int freeSpace(MemPage *pPage, int start, int size){ int addr, pbegin, hdr; unsigned char *data = pPage->aData; @@ -722,7 +833,8 @@ static void freeSpace(MemPage *pPage, int start, int size){ assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( start>=pPage->hdrOffset+6+(pPage->leaf?0:4) ); assert( (start + size)<=pPage->pBt->usableSize ); - if( size<4 ) size = 4; + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( size>=0 ); /* Minimum cell size is 4 */ #ifdef SQLITE_SECURE_DELETE /* Overwrite deleted information with zeros when the SECURE_DELETE @@ -735,30 +847,38 @@ static void freeSpace(MemPage *pPage, int start, int size){ addr = hdr + 1; while( (pbegin = get2byte(&data[addr]))0 ){ assert( pbegin<=pPage->pBt->usableSize-4 ); - assert( pbegin>addr ); + if( pbegin<=addr ) { + return SQLITE_CORRUPT_BKPT; + } addr = pbegin; } - assert( pbegin<=pPage->pBt->usableSize-4 ); + if ( pbegin>pPage->pBt->usableSize-4 ) { + return SQLITE_CORRUPT_BKPT; + } assert( pbegin>addr || pbegin==0 ); put2byte(&data[addr], start); put2byte(&data[start], pbegin); put2byte(&data[start+2], size); - pPage->nFree += size; + pPage->nFree += (u16)size; /* Coalesce adjacent free blocks */ addr = pPage->hdrOffset + 1; while( (pbegin = get2byte(&data[addr]))>0 ){ - int pnext, psize; + int pnext, psize, x; assert( pbegin>addr ); assert( pbegin<=pPage->pBt->usableSize-4 ); pnext = get2byte(&data[pbegin]); psize = get2byte(&data[pbegin+2]); if( pbegin + psize + 3 >= pnext && pnext>0 ){ int frag = pnext - (pbegin+psize); - assert( frag<=data[pPage->hdrOffset+7] ); - data[pPage->hdrOffset+7] -= frag; - put2byte(&data[pbegin], get2byte(&data[pnext])); - put2byte(&data[pbegin+2], pnext+get2byte(&data[pnext+2])-pbegin); + if( (frag<0) || (frag>(int)data[pPage->hdrOffset+7]) ){ + return SQLITE_CORRUPT_BKPT; + } + data[pPage->hdrOffset+7] -= (u8)frag; + x = get2byte(&data[pnext]); + put2byte(&data[pbegin], x); + x = pnext + get2byte(&data[pnext+2]) - pbegin; + put2byte(&data[pbegin+2], x); }else{ addr = pbegin; } @@ -769,119 +889,140 @@ static void freeSpace(MemPage *pPage, int start, int size){ int top; pbegin = get2byte(&data[hdr+1]); memcpy(&data[hdr+1], &data[pbegin], 2); - top = get2byte(&data[hdr+5]); - put2byte(&data[hdr+5], top + get2byte(&data[pbegin+2])); + top = get2byte(&data[hdr+5]) + get2byte(&data[pbegin+2]); + put2byte(&data[hdr+5], top); } + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + return SQLITE_OK; } /* ** Decode the flags byte (the first byte of the header) for a page ** and initialize fields of the MemPage structure accordingly. +** +** Only the following combinations are supported. Anything different +** indicates a corrupt database files: +** +** PTF_ZERODATA +** PTF_ZERODATA | PTF_LEAF +** PTF_LEAFDATA | PTF_INTKEY +** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF */ -static void decodeFlags(MemPage *pPage, int flagByte){ +static int decodeFlags(MemPage *pPage, int flagByte){ BtShared *pBt; /* A copy of pPage->pBt */ assert( pPage->hdrOffset==(pPage->pgno==1 ? 100 : 0) ); - pPage->intKey = (flagByte & (PTF_INTKEY|PTF_LEAFDATA))!=0; - pPage->zeroData = (flagByte & PTF_ZERODATA)!=0; - pPage->leaf = (flagByte & PTF_LEAF)!=0; - pPage->childPtrSize = 4*(pPage->leaf==0); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); + flagByte &= ~PTF_LEAF; + pPage->childPtrSize = 4-4*pPage->leaf; pBt = pPage->pBt; - if( flagByte & PTF_LEAFDATA ){ - pPage->leafData = 1; + if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ + pPage->intKey = 1; + pPage->hasData = pPage->leaf; pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; - }else{ - pPage->leafData = 0; + }else if( flagByte==PTF_ZERODATA ){ + pPage->intKey = 0; + pPage->hasData = 0; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; + }else{ + return SQLITE_CORRUPT_BKPT; } - pPage->hasData = !(pPage->zeroData || (!pPage->leaf && pPage->leafData)); + return SQLITE_OK; } /* ** Initialize the auxiliary information for a disk block. ** -** The pParent parameter must be a pointer to the MemPage which -** is the parent of the page being initialized. The root of a -** BTree has no parent and so for that page, pParent==NULL. -** ** Return SQLITE_OK on success. If we see that the page does ** not contain a well-formed database page, then return ** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not ** guarantee that the page is well-formed. It only shows that ** we failed to detect any corruption. */ -int sqlite3BtreeInitPage( - MemPage *pPage, /* The page to be initialized */ - MemPage *pParent /* The parent. Might be NULL */ -){ - int pc; /* Address of a freeblock within pPage->aData[] */ - int hdr; /* Offset to beginning of page header */ - u8 *data; /* Equal to pPage->aData */ - BtShared *pBt; /* The main btree structure */ - int usableSize; /* Amount of usable space on each page */ - int cellOffset; /* Offset from start of page to first cell pointer */ - int nFree; /* Number of unused bytes on the page */ - int top; /* First byte of the cell content area */ +int sqlite3BtreeInitPage(MemPage *pPage){ - pBt = pPage->pBt; - assert( pBt!=0 ); - assert( pParent==0 || pParent->pBt==pBt ); + assert( pPage->pBt!=0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); - assert( pPage->aData == &((unsigned char*)pPage)[-pBt->pageSize] ); - if( pPage->pParent!=pParent && (pPage->pParent!=0 || pPage->isInit) ){ - /* The parent page should never change unless the file is corrupt */ - return SQLITE_CORRUPT_BKPT; - } - if( pPage->isInit ) return SQLITE_OK; - if( pPage->pParent==0 && pParent!=0 ){ - pPage->pParent = pParent; - sqlite3PagerRef(pParent->pDbPage); - } - hdr = pPage->hdrOffset; - data = pPage->aData; - decodeFlags(pPage, data[hdr]); - pPage->nOverflow = 0; - pPage->idxShift = 0; - usableSize = pBt->usableSize; - pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; - top = get2byte(&data[hdr+5]); - pPage->nCell = get2byte(&data[hdr+3]); - if( pPage->nCell>MX_CELL(pBt) ){ - /* To many cells for a single page. The page must be corrupt */ - return SQLITE_CORRUPT_BKPT; - } - if( pPage->nCell==0 && pParent!=0 && pParent->pgno!=1 ){ - /* All pages must have at least one cell, except for root pages */ - return SQLITE_CORRUPT_BKPT; - } + assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); + assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); - /* Compute the total free space on the page */ - pc = get2byte(&data[hdr+1]); - nFree = data[hdr+7] + top - (cellOffset + 2*pPage->nCell); - while( pc>0 ){ - int next, size; - if( pc>usableSize-4 ){ - /* Free block is off the page */ + if( !pPage->isInit ){ + u16 pc; /* Address of a freeblock within pPage->aData[] */ + u8 hdr; /* Offset to beginning of page header */ + u8 *data; /* Equal to pPage->aData */ + BtShared *pBt; /* The main btree structure */ + u16 usableSize; /* Amount of usable space on each page */ + u16 cellOffset; /* Offset from start of page to first cell pointer */ + u16 nFree; /* Number of unused bytes on the page */ + u16 top; /* First byte of the cell content area */ + + pBt = pPage->pBt; + + hdr = pPage->hdrOffset; + data = pPage->aData; + if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; + assert( pBt->pageSize>=512 && pBt->pageSize<=32768 ); + pPage->maskPage = pBt->pageSize - 1; + pPage->nOverflow = 0; + usableSize = pBt->usableSize; + pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; + top = get2byte(&data[hdr+5]); + pPage->nCell = get2byte(&data[hdr+3]); + if( pPage->nCell>MX_CELL(pBt) ){ + /* To many cells for a single page. The page must be corrupt */ + return SQLITE_CORRUPT_BKPT; + } + + /* Compute the total free space on the page */ + pc = get2byte(&data[hdr+1]); + nFree = data[hdr+7] + top - (cellOffset + 2*pPage->nCell); + while( pc>0 ){ + u16 next, size; + if( pc>usableSize-4 ){ + /* Free block is off the page */ + return SQLITE_CORRUPT_BKPT; + } + next = get2byte(&data[pc]); + size = get2byte(&data[pc+2]); + if( next>0 && next<=pc+size+3 ){ + /* Free blocks must be in accending order */ + return SQLITE_CORRUPT_BKPT; + } + nFree += size; + pc = next; + } + pPage->nFree = (u16)nFree; + if( nFree>=usableSize ){ + /* Free space cannot exceed total page size */ return SQLITE_CORRUPT_BKPT; } - next = get2byte(&data[pc]); - size = get2byte(&data[pc+2]); - if( next>0 && next<=pc+size+3 ){ - /* Free blocks must be in accending order */ - return SQLITE_CORRUPT_BKPT; - } - nFree += size; - pc = next; - } - pPage->nFree = nFree; - if( nFree>=usableSize ){ - /* Free space cannot exceed total page size */ - return SQLITE_CORRUPT_BKPT; - } - pPage->isInit = 1; +#if 0 + /* Check that all the offsets in the cell offset array are within range. + ** + ** Omitting this consistency check and using the pPage->maskPage mask + ** to prevent overrunning the page buffer in findCell() results in a + ** 2.5% performance gain. + */ + { + u8 *pOff; /* Iterator used to check all cell offsets are in range */ + u8 *pEnd; /* Pointer to end of cell offset array */ + u8 mask; /* Mask of bits that must be zero in MSB of cell offsets */ + mask = ~(((u8)(pBt->pageSize>>8))-1); + pEnd = &data[cellOffset + pPage->nCell*2]; + for(pOff=&data[cellOffset]; pOff!=pEnd && !((*pOff)&mask); pOff+=2); + if( pOff!=pEnd ){ + return SQLITE_CORRUPT_BKPT; + } + } +#endif + + pPage->isInit = 1; + } return SQLITE_OK; } @@ -892,15 +1033,17 @@ int sqlite3BtreeInitPage( static void zeroPage(MemPage *pPage, int flags){ unsigned char *data = pPage->aData; BtShared *pBt = pPage->pBt; - int hdr = pPage->hdrOffset; - int first; + u8 hdr = pPage->hdrOffset; + u16 first; assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno ); - assert( &data[pBt->pageSize] == (unsigned char*)pPage ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage) == data ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - memset(&data[hdr], 0, pBt->usableSize - hdr); - data[hdr] = flags; - first = hdr + 8 + 4*((flags&PTF_LEAF)==0); + assert( sqlite3_mutex_held(pBt->mutex) ); + /*memset(&data[hdr], 0, pBt->usableSize - hdr);*/ + data[hdr] = (char)flags; + first = hdr + 8 + 4*((flags&PTF_LEAF)==0 ?1:0); memset(&data[hdr+1], 0, 4); data[hdr+7] = 0; put2byte(&data[hdr+5], pBt->usableSize); @@ -909,11 +1052,27 @@ static void zeroPage(MemPage *pPage, int flags){ pPage->hdrOffset = hdr; pPage->cellOffset = first; pPage->nOverflow = 0; - pPage->idxShift = 0; + assert( pBt->pageSize>=512 && pBt->pageSize<=32768 ); + pPage->maskPage = pBt->pageSize - 1; pPage->nCell = 0; pPage->isInit = 1; } + +/* +** Convert a DbPage obtained from the pager into a MemPage used by +** the btree layer. +*/ +static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ + MemPage *pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); + pPage->aData = sqlite3PagerGetData(pDbPage); + pPage->pDbPage = pDbPage; + pPage->pBt = pBt; + pPage->pgno = pgno; + pPage->hdrOffset = pPage->pgno==1 ? 100 : 0; + return pPage; +} + /* ** Get a page from the pager. Initialize the MemPage.pBt and ** MemPage.aData elements if needed. @@ -932,21 +1091,28 @@ int sqlite3BtreeGetPage( int noContent /* Do not load page content if true */ ){ int rc; - MemPage *pPage; DbPage *pDbPage; + assert( sqlite3_mutex_held(pBt->mutex) ); rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent); if( rc ) return rc; - pPage = (MemPage *)sqlite3PagerGetExtra(pDbPage); - pPage->aData = sqlite3PagerGetData(pDbPage); - pPage->pDbPage = pDbPage; - pPage->pBt = pBt; - pPage->pgno = pgno; - pPage->hdrOffset = pPage->pgno==1 ? 100 : 0; - *ppPage = pPage; + *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; } +/* +** Return the size of the database file in pages. If there is any kind of +** error, return ((unsigned int)-1). +*/ +static Pgno pagerPagecount(BtShared *pBt){ + int nPage = -1; + int rc; + assert( pBt->pPage1 ); + rc = sqlite3PagerPagecount(pBt->pPager, &nPage); + assert( rc==SQLITE_OK || nPage==-1 ); + return (Pgno)nPage; +} + /* ** Get a page from the pager and initialize it. This routine ** is just a convenience wrapper around separate calls to @@ -955,16 +1121,42 @@ int sqlite3BtreeGetPage( static int getAndInitPage( BtShared *pBt, /* The database file */ Pgno pgno, /* Number of the page to get */ - MemPage **ppPage, /* Write the page pointer here */ - MemPage *pParent /* Parent of the page */ + MemPage **ppPage /* Write the page pointer here */ ){ int rc; + DbPage *pDbPage; + MemPage *pPage; + + assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; } - rc = sqlite3BtreeGetPage(pBt, pgno, ppPage, 0); - if( rc==SQLITE_OK && (*ppPage)->isInit==0 ){ - rc = sqlite3BtreeInitPage(*ppPage, pParent); + + /* It is often the case that the page we want is already in cache. + ** If so, get it directly. This saves us from having to call + ** pagerPagecount() to make sure pgno is within limits, which results + ** in a measureable performance improvements. + */ + pDbPage = sqlite3PagerLookup(pBt->pPager, pgno); + if( pDbPage ){ + /* Page is already in cache */ + *ppPage = pPage = btreePageFromDbPage(pDbPage, pgno, pBt); + rc = SQLITE_OK; + }else{ + /* Page not in cache. Acquire it. */ + if( pgno>pagerPagecount(pBt) ){ + return SQLITE_CORRUPT_BKPT; + } + rc = sqlite3BtreeGetPage(pBt, pgno, ppPage, 0); + if( rc ) return rc; + pPage = *ppPage; + } + if( !pPage->isInit ){ + rc = sqlite3BtreeInitPage(pPage); + } + if( rc!=SQLITE_OK ){ + releasePage(pPage); + *ppPage = 0; } return rc; } @@ -975,30 +1167,16 @@ static int getAndInitPage( */ static void releasePage(MemPage *pPage){ if( pPage ){ + assert( pPage->nOverflow==0 || sqlite3PagerPageRefcount(pPage->pDbPage)>1 ); assert( pPage->aData ); assert( pPage->pBt ); - assert( &pPage->aData[pPage->pBt->pageSize]==(unsigned char*)pPage ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); sqlite3PagerUnref(pPage->pDbPage); } } -/* -** This routine is called when the reference count for a page -** reaches zero. We need to unref the pParent pointer when that -** happens. -*/ -static void pageDestructor(DbPage *pData, int pageSize){ - MemPage *pPage; - assert( (pageSize & 7)==0 ); - pPage = (MemPage *)sqlite3PagerGetExtra(pData); - if( pPage->pParent ){ - MemPage *pParent = pPage->pParent; - pPage->pParent = 0; - releasePage(pParent); - } - pPage->isInit = 0; -} - /* ** During a rollback, when the pager reloads information into the cache ** so that the cache is restored to its original state at the start of @@ -1007,37 +1185,50 @@ static void pageDestructor(DbPage *pData, int pageSize){ ** This routine needs to reset the extra data section at the end of the ** page to agree with the restored data. */ -static void pageReinit(DbPage *pData, int pageSize){ +static void pageReinit(DbPage *pData){ MemPage *pPage; - assert( (pageSize & 7)==0 ); pPage = (MemPage *)sqlite3PagerGetExtra(pData); if( pPage->isInit ){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); pPage->isInit = 0; - sqlite3BtreeInitPage(pPage, pPage->pParent); + if( sqlite3PagerPageRefcount(pData)>0 ){ + sqlite3BtreeInitPage(pPage); + } } } +/* +** Invoke the busy handler for a btree. +*/ +static int btreeInvokeBusyHandler(void *pArg){ + BtShared *pBt = (BtShared*)pArg; + assert( pBt->db ); + assert( sqlite3_mutex_held(pBt->db->mutex) ); + return sqlite3InvokeBusyHandler(&pBt->db->busyHandler); +} + /* ** Open a database file. ** ** zFilename is the name of the database file. If zFilename is NULL ** a new database with a random name is created. This randomly named ** database file will be deleted when sqlite3BtreeClose() is called. +** If zFilename is ":memory:" then an in-memory database is created +** that is automatically destroyed when it is closed. */ int sqlite3BtreeOpen( const char *zFilename, /* Name of the file containing the BTree database */ - sqlite3 *pSqlite, /* Associated database handle */ + sqlite3 *db, /* Associated database handle */ Btree **ppBtree, /* Pointer to new Btree object written here */ - int flags /* Options */ + int flags, /* Options */ + int vfsFlags /* Flags passed through to sqlite3_vfs.xOpen() */ ){ - BtShared *pBt; /* Shared part of btree structure */ + sqlite3_vfs *pVfs; /* The VFS to use for this btree */ + BtShared *pBt = 0; /* Shared part of btree structure */ Btree *p; /* Handle to return */ int rc = SQLITE_OK; - int nReserve; + u8 nReserve; unsigned char zDbHeader[100]; -#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) - const ThreadData *pTsdro; -#endif /* Set the variable isMemdb to true for an in-memory database, or ** false for a file-based database. This symbol is only required if @@ -1052,113 +1243,179 @@ int sqlite3BtreeOpen( #endif #endif - p = sqliteMalloc(sizeof(Btree)); + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + + pVfs = db->pVfs; + p = sqlite3MallocZero(sizeof(Btree)); if( !p ){ return SQLITE_NOMEM; } p->inTrans = TRANS_NONE; - p->pSqlite = pSqlite; + p->db = db; - /* Try to find an existing Btree structure opened on zFilename. */ #if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) - pTsdro = sqlite3ThreadDataReadOnly(); - if( pTsdro->useSharedData && zFilename && !isMemdb ){ - char *zFullPathname = sqlite3OsFullPathname(zFilename); - if( !zFullPathname ){ - sqliteFree(p); - return SQLITE_NOMEM; + /* + ** If this Btree is a candidate for shared cache, try to find an + ** existing BtShared object that we can share with + */ + if( isMemdb==0 + && (db->flags & SQLITE_Vtab)==0 + && zFilename && zFilename[0] + ){ + if( sqlite3GlobalConfig.sharedCacheEnabled ){ + int nFullPathname = pVfs->mxPathname+1; + char *zFullPathname = sqlite3Malloc(nFullPathname); + sqlite3_mutex *mutexShared; + p->sharable = 1; + db->flags |= SQLITE_SharedCache; + if( !zFullPathname ){ + sqlite3_free(p); + return SQLITE_NOMEM; + } + sqlite3OsFullPathname(pVfs, zFilename, nFullPathname, zFullPathname); + mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(mutexShared); + for(pBt=GLOBAL(BtShared*,sqlite3SharedCacheList); pBt; pBt=pBt->pNext){ + assert( pBt->nRef>0 ); + if( 0==strcmp(zFullPathname, sqlite3PagerFilename(pBt->pPager)) + && sqlite3PagerVfs(pBt->pPager)==pVfs ){ + p->pBt = pBt; + pBt->nRef++; + break; + } + } + sqlite3_mutex_leave(mutexShared); + sqlite3_free(zFullPathname); } - for(pBt=pTsdro->pBtree; pBt; pBt=pBt->pNext){ - assert( pBt->nRef>0 ); - if( 0==strcmp(zFullPathname, sqlite3PagerFilename(pBt->pPager)) ){ - p->pBt = pBt; - *ppBtree = p; - pBt->nRef++; - sqliteFree(zFullPathname); - return SQLITE_OK; +#ifdef SQLITE_DEBUG + else{ + /* In debug mode, we mark all persistent databases as sharable + ** even when they are not. This exercises the locking code and + ** gives more opportunity for asserts(sqlite3_mutex_held()) + ** statements to find locking problems. + */ + p->sharable = 1; + } +#endif + } +#endif + if( pBt==0 ){ + /* + ** The following asserts make sure that structures used by the btree are + ** the right size. This is to guard against size changes that result + ** when compiling on a different architecture. + */ + assert( sizeof(i64)==8 || sizeof(i64)==4 ); + assert( sizeof(u64)==8 || sizeof(u64)==4 ); + assert( sizeof(u32)==4 ); + assert( sizeof(u16)==2 ); + assert( sizeof(Pgno)==4 ); + + pBt = sqlite3MallocZero( sizeof(*pBt) ); + if( pBt==0 ){ + rc = SQLITE_NOMEM; + goto btree_open_out; + } + rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, + EXTRA_SIZE, flags, vfsFlags); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); + } + if( rc!=SQLITE_OK ){ + goto btree_open_out; + } + sqlite3PagerSetBusyhandler(pBt->pPager, btreeInvokeBusyHandler, pBt); + p->pBt = pBt; + + sqlite3PagerSetReiniter(pBt->pPager, pageReinit); + pBt->pCursor = 0; + pBt->pPage1 = 0; + pBt->readOnly = sqlite3PagerIsreadonly(pBt->pPager); + pBt->pageSize = get2byte(&zDbHeader[16]); + if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE + || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ + pBt->pageSize = 0; + sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize); +#ifndef SQLITE_OMIT_AUTOVACUUM + /* If the magic name ":memory:" will create an in-memory database, then + ** leave the autoVacuum mode at 0 (do not auto-vacuum), even if + ** SQLITE_DEFAULT_AUTOVACUUM is true. On the other hand, if + ** SQLITE_OMIT_MEMORYDB has been defined, then ":memory:" is just a + ** regular file-name. In this case the auto-vacuum applies as per normal. + */ + if( zFilename && !isMemdb ){ + pBt->autoVacuum = (SQLITE_DEFAULT_AUTOVACUUM ? 1 : 0); + pBt->incrVacuum = (SQLITE_DEFAULT_AUTOVACUUM==2 ? 1 : 0); + } +#endif + nReserve = 0; + }else{ + nReserve = zDbHeader[20]; + pBt->pageSizeFixed = 1; +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->autoVacuum = (get4byte(&zDbHeader[36 + 4*4])?1:0); + pBt->incrVacuum = (get4byte(&zDbHeader[36 + 7*4])?1:0); +#endif + } + pBt->usableSize = pBt->pageSize - nReserve; + assert( (pBt->pageSize & 7)==0 ); /* 8-byte alignment of pageSize */ + sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize); + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* Add the new BtShared object to the linked list sharable BtShareds. + */ + if( p->sharable ){ + sqlite3_mutex *mutexShared; + pBt->nRef = 1; + mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ + pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); + if( pBt->mutex==0 ){ + rc = SQLITE_NOMEM; + db->mallocFailed = 0; + goto btree_open_out; + } + } + sqlite3_mutex_enter(mutexShared); + pBt->pNext = GLOBAL(BtShared*,sqlite3SharedCacheList); + GLOBAL(BtShared*,sqlite3SharedCacheList) = pBt; + sqlite3_mutex_leave(mutexShared); + } +#endif + } + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* If the new Btree uses a sharable pBtShared, then link the new + ** Btree into the list of all sharable Btrees for the same connection. + ** The list is kept in ascending order by pBt address. + */ + if( p->sharable ){ + int i; + Btree *pSib; + for(i=0; inDb; i++){ + if( (pSib = db->aDb[i].pBt)!=0 && pSib->sharable ){ + while( pSib->pPrev ){ pSib = pSib->pPrev; } + if( p->pBtpBt ){ + p->pNext = pSib; + p->pPrev = 0; + pSib->pPrev = p; + }else{ + while( pSib->pNext && pSib->pNext->pBtpBt ){ + pSib = pSib->pNext; + } + p->pNext = pSib->pNext; + p->pPrev = pSib; + if( p->pNext ){ + p->pNext->pPrev = p; + } + pSib->pNext = p; + } + break; } } - sqliteFree(zFullPathname); } #endif - - /* - ** The following asserts make sure that structures used by the btree are - ** the right size. This is to guard against size changes that result - ** when compiling on a different architecture. - */ - assert( sizeof(i64)==8 || sizeof(i64)==4 ); - assert( sizeof(u64)==8 || sizeof(u64)==4 ); - assert( sizeof(u32)==4 ); - assert( sizeof(u16)==2 ); - assert( sizeof(Pgno)==4 ); - - pBt = sqliteMalloc( sizeof(*pBt) ); - if( pBt==0 ){ - rc = SQLITE_NOMEM; - goto btree_open_out; - } - rc = sqlite3PagerOpen(&pBt->pPager, zFilename, EXTRA_SIZE, flags); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); - } - if( rc!=SQLITE_OK ){ - goto btree_open_out; - } - p->pBt = pBt; - - sqlite3PagerSetDestructor(pBt->pPager, pageDestructor); - sqlite3PagerSetReiniter(pBt->pPager, pageReinit); - pBt->pCursor = 0; - pBt->pPage1 = 0; - pBt->readOnly = sqlite3PagerIsreadonly(pBt->pPager); - pBt->pageSize = get2byte(&zDbHeader[16]); - if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE - || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ - pBt->pageSize = SQLITE_DEFAULT_PAGE_SIZE; - pBt->maxEmbedFrac = 64; /* 25% */ - pBt->minEmbedFrac = 32; /* 12.5% */ - pBt->minLeafFrac = 32; /* 12.5% */ -#ifndef SQLITE_OMIT_AUTOVACUUM - /* If the magic name ":memory:" will create an in-memory database, then - ** leave the autoVacuum mode at 0 (do not auto-vacuum), even if - ** SQLITE_DEFAULT_AUTOVACUUM is true. On the other hand, if - ** SQLITE_OMIT_MEMORYDB has been defined, then ":memory:" is just a - ** regular file-name. In this case the auto-vacuum applies as per normal. - */ - if( zFilename && !isMemdb ){ - pBt->autoVacuum = (SQLITE_DEFAULT_AUTOVACUUM ? 1 : 0); - pBt->incrVacuum = (SQLITE_DEFAULT_AUTOVACUUM==2 ? 1 : 0); - } -#endif - nReserve = 0; - }else{ - nReserve = zDbHeader[20]; - pBt->maxEmbedFrac = zDbHeader[21]; - pBt->minEmbedFrac = zDbHeader[22]; - pBt->minLeafFrac = zDbHeader[23]; - pBt->pageSizeFixed = 1; -#ifndef SQLITE_OMIT_AUTOVACUUM - pBt->autoVacuum = (get4byte(&zDbHeader[36 + 4*4])?1:0); - pBt->incrVacuum = (get4byte(&zDbHeader[36 + 7*4])?1:0); -#endif - } - pBt->usableSize = pBt->pageSize - nReserve; - assert( (pBt->pageSize & 7)==0 ); /* 8-byte alignment of pageSize */ - sqlite3PagerSetPagesize(pBt->pPager, pBt->pageSize); - -#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) - /* Add the new btree to the linked list starting at ThreadData.pBtree. - ** There is no chance that a malloc() may fail inside of the - ** sqlite3ThreadData() call, as the ThreadData structure must have already - ** been allocated for pTsdro->useSharedData to be non-zero. - */ - if( pTsdro->useSharedData && zFilename && !isMemdb ){ - pBt->pNext = pTsdro->pBtree; - sqlite3ThreadData()->pBtree = pBt; - } -#endif - pBt->nRef = 1; *ppBtree = p; btree_open_out: @@ -1166,13 +1423,71 @@ btree_open_out: if( pBt && pBt->pPager ){ sqlite3PagerClose(pBt->pPager); } - sqliteFree(pBt); - sqliteFree(p); + sqlite3_free(pBt); + sqlite3_free(p); *ppBtree = 0; } return rc; } +/* +** Decrement the BtShared.nRef counter. When it reaches zero, +** remove the BtShared structure from the sharing list. Return +** true if the BtShared.nRef counter reaches zero and return +** false if it is still positive. +*/ +static int removeFromSharingList(BtShared *pBt){ +#ifndef SQLITE_OMIT_SHARED_CACHE + sqlite3_mutex *pMaster; + BtShared *pList; + int removed = 0; + + assert( sqlite3_mutex_notheld(pBt->mutex) ); + pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(pMaster); + pBt->nRef--; + if( pBt->nRef<=0 ){ + if( GLOBAL(BtShared*,sqlite3SharedCacheList)==pBt ){ + GLOBAL(BtShared*,sqlite3SharedCacheList) = pBt->pNext; + }else{ + pList = GLOBAL(BtShared*,sqlite3SharedCacheList); + while( ALWAYS(pList) && pList->pNext!=pBt ){ + pList=pList->pNext; + } + if( ALWAYS(pList) ){ + pList->pNext = pBt->pNext; + } + } + if( SQLITE_THREADSAFE ){ + sqlite3_mutex_free(pBt->mutex); + } + removed = 1; + } + sqlite3_mutex_leave(pMaster); + return removed; +#else + return 1; +#endif +} + +/* +** Make sure pBt->pTmpSpace points to an allocation of +** MX_CELL_SIZE(pBt) bytes. +*/ +static void allocateTempSpace(BtShared *pBt){ + if( !pBt->pTmpSpace ){ + pBt->pTmpSpace = sqlite3PageMalloc( pBt->pageSize ); + } +} + +/* +** Free the pBt->pTmpSpace allocation +*/ +static void freeTempSpace(BtShared *pBt){ + sqlite3PageFree( pBt->pTmpSpace); + pBt->pTmpSpace = 0; +} + /* ** Close an open database and invalidate all cursors. */ @@ -1180,11 +1495,10 @@ int sqlite3BtreeClose(Btree *p){ BtShared *pBt = p->pBt; BtCursor *pCur; -#ifndef SQLITE_OMIT_SHARED_CACHE - ThreadData *pTsd; -#endif - /* Close all cursors opened via this handle. */ + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + pBt->db = p->db; pCur = pBt->pCursor; while( pCur ){ BtCursor *pTmp = pCur; @@ -1199,55 +1513,37 @@ int sqlite3BtreeClose(Btree *p){ ** this handle. */ sqlite3BtreeRollback(p); - sqliteFree(p); + sqlite3BtreeLeave(p); -#ifndef SQLITE_OMIT_SHARED_CACHE /* If there are still other outstanding references to the shared-btree ** structure, return now. The remainder of this procedure cleans ** up the shared-btree. */ - assert( pBt->nRef>0 ); - pBt->nRef--; - if( pBt->nRef ){ - return SQLITE_OK; + assert( p->wantToLock==0 && p->locked==0 ); + if( !p->sharable || removeFromSharingList(pBt) ){ + /* The pBt is no longer on the sharing list, so we can access + ** it without having to hold the mutex. + ** + ** Clean out and delete the BtShared object. + */ + assert( !pBt->pCursor ); + sqlite3PagerClose(pBt->pPager); + if( pBt->xFreeSchema && pBt->pSchema ){ + pBt->xFreeSchema(pBt->pSchema); + } + sqlite3_free(pBt->pSchema); + freeTempSpace(pBt); + sqlite3_free(pBt); } - /* Remove the shared-btree from the thread wide list. Call - ** ThreadDataReadOnly() and then cast away the const property of the - ** pointer to avoid allocating thread data if it is not really required. - */ - pTsd = (ThreadData *)sqlite3ThreadDataReadOnly(); - if( pTsd->pBtree==pBt ){ - assert( pTsd==sqlite3ThreadData() ); - pTsd->pBtree = pBt->pNext; - }else{ - BtShared *pPrev; - for(pPrev=pTsd->pBtree; pPrev && pPrev->pNext!=pBt; pPrev=pPrev->pNext){} - if( pPrev ){ - assert( pTsd==sqlite3ThreadData() ); - pPrev->pNext = pBt->pNext; - } - } +#ifndef SQLITE_OMIT_SHARED_CACHE + assert( p->wantToLock==0 ); + assert( p->locked==0 ); + if( p->pPrev ) p->pPrev->pNext = p->pNext; + if( p->pNext ) p->pNext->pPrev = p->pPrev; #endif - /* Close the pager and free the shared-btree structure */ - assert( !pBt->pCursor ); - sqlite3PagerClose(pBt->pPager); - if( pBt->xFreeSchema && pBt->pSchema ){ - pBt->xFreeSchema(pBt->pSchema); - } - sqliteFree(pBt->pSchema); - sqliteFree(pBt); - return SQLITE_OK; -} - -/* -** Change the busy handler callback function. -*/ -int sqlite3BtreeSetBusyHandler(Btree *p, BusyHandler *pHandler){ - BtShared *pBt = p->pBt; - pBt->pBusyHandler = pHandler; - sqlite3PagerSetBusyhandler(pBt->pPager, pHandler); + sqlite3_free(p); return SQLITE_OK; } @@ -1268,7 +1564,10 @@ int sqlite3BtreeSetBusyHandler(Btree *p, BusyHandler *pHandler){ */ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); sqlite3PagerSetCachesize(pBt->pPager, mxPage); + sqlite3BtreeLeave(p); return SQLITE_OK; } @@ -1283,7 +1582,10 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ #ifndef SQLITE_OMIT_PAGER_PRAGMAS int sqlite3BtreeSetSafetyLevel(Btree *p, int level, int fullSync){ BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); sqlite3PagerSetSafetyLevel(pBt->pPager, level, fullSync); + sqlite3BtreeLeave(p); return SQLITE_OK; } #endif @@ -1294,8 +1596,13 @@ int sqlite3BtreeSetSafetyLevel(Btree *p, int level, int fullSync){ */ int sqlite3BtreeSyncDisabled(Btree *p){ BtShared *pBt = p->pBt; + int rc; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); assert( pBt && pBt->pPager ); - return sqlite3PagerNosync(pBt->pPager); + rc = sqlite3PagerNosync(pBt->pPager); + sqlite3BtreeLeave(p); + return rc; } #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) @@ -1315,21 +1622,29 @@ int sqlite3BtreeSyncDisabled(Btree *p){ ** bytes per page is left unchanged. */ int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve){ + int rc = SQLITE_OK; BtShared *pBt = p->pBt; + assert( nReserve>=-1 && nReserve<=255 ); + sqlite3BtreeEnter(p); if( pBt->pageSizeFixed ){ + sqlite3BtreeLeave(p); return SQLITE_READONLY; } if( nReserve<0 ){ nReserve = pBt->pageSize - pBt->usableSize; } + assert( nReserve>=0 && nReserve<=255 ); if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && ((pageSize-1)&pageSize)==0 ){ assert( (pageSize & 7)==0 ); assert( !pBt->pPage1 && !pBt->pCursor ); - pBt->pageSize = sqlite3PagerSetPagesize(pBt->pPager, pageSize); + pBt->pageSize = (u16)pageSize; + freeTempSpace(pBt); + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize); } - pBt->usableSize = pBt->pageSize - nReserve; - return SQLITE_OK; + pBt->usableSize = pBt->pageSize - (u16)nReserve; + sqlite3BtreeLeave(p); + return rc; } /* @@ -1339,7 +1654,11 @@ int sqlite3BtreeGetPageSize(Btree *p){ return p->pBt->pageSize; } int sqlite3BtreeGetReserve(Btree *p){ - return p->pBt->pageSize - p->pBt->usableSize; + int n; + sqlite3BtreeEnter(p); + n = p->pBt->pageSize - p->pBt->usableSize; + sqlite3BtreeLeave(p); + return n; } /* @@ -1348,7 +1667,11 @@ int sqlite3BtreeGetReserve(Btree *p){ ** Regardless of the value of mxPage, return the maximum page count. */ int sqlite3BtreeMaxPageCount(Btree *p, int mxPage){ - return sqlite3PagerMaxPageCount(p->pBt->pPager, mxPage); + int n; + sqlite3BtreeEnter(p); + n = sqlite3PagerMaxPageCount(p->pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return n; } #endif /* !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) */ @@ -1363,12 +1686,17 @@ int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ return SQLITE_READONLY; #else BtShared *pBt = p->pBt; - int av = (autoVacuum?1:0); + int rc = SQLITE_OK; + u8 av = autoVacuum ?1:0; + + sqlite3BtreeEnter(p); if( pBt->pageSizeFixed && av!=pBt->autoVacuum ){ - return SQLITE_READONLY; + rc = SQLITE_READONLY; + }else{ + pBt->autoVacuum = av; } - pBt->autoVacuum = av; - return SQLITE_OK; + sqlite3BtreeLeave(p); + return rc; #endif } @@ -1380,11 +1708,15 @@ int sqlite3BtreeGetAutoVacuum(Btree *p){ #ifdef SQLITE_OMIT_AUTOVACUUM return BTREE_AUTOVACUUM_NONE; #else - return ( + int rc; + sqlite3BtreeEnter(p); + rc = ( (!p->pBt->autoVacuum)?BTREE_AUTOVACUUM_NONE: (!p->pBt->incrVacuum)?BTREE_AUTOVACUUM_FULL: BTREE_AUTOVACUUM_INCR ); + sqlite3BtreeLeave(p); + return rc; #endif } @@ -1399,19 +1731,26 @@ int sqlite3BtreeGetAutoVacuum(Btree *p){ ** is returned if we run out of memory. */ static int lockBtree(BtShared *pBt){ - int rc, pageSize; + int rc; MemPage *pPage1; + int nPage; + + assert( sqlite3_mutex_held(pBt->mutex) ); if( pBt->pPage1 ) return SQLITE_OK; rc = sqlite3BtreeGetPage(pBt, 1, &pPage1, 0); if( rc!=SQLITE_OK ) return rc; - /* Do some checking to help insure the file we opened really is ** a valid database file. */ - rc = SQLITE_NOTADB; - if( sqlite3PagerPagecount(pBt->pPager)>0 ){ + rc = sqlite3PagerPagecount(pBt->pPager, &nPage); + if( rc!=SQLITE_OK ){ + goto page1_init_failed; + }else if( nPage>0 ){ + int pageSize; + int usableSize; u8 *page1 = pPage1->aData; + rc = SQLITE_NOTADB; if( memcmp(page1, zMagicHeader, 16)!=0 ){ goto page1_init_failed; } @@ -1421,19 +1760,42 @@ static int lockBtree(BtShared *pBt){ if( page1[19]>1 ){ goto page1_init_failed; } + + /* The maximum embedded fraction must be exactly 25%. And the minimum + ** embedded fraction must be 12.5% for both leaf-data and non-leaf-data. + ** The original design allowed these amounts to vary, but as of + ** version 3.6.0, we require them to be fixed. + */ + if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ + goto page1_init_failed; + } pageSize = get2byte(&page1[16]); - if( ((pageSize-1)&pageSize)!=0 || pageSize<512 ){ + if( ((pageSize-1)&pageSize)!=0 || pageSize<512 || + (SQLITE_MAX_PAGE_SIZE<32768 && pageSize>SQLITE_MAX_PAGE_SIZE) + ){ goto page1_init_failed; } assert( (pageSize & 7)==0 ); - pBt->pageSize = pageSize; - pBt->usableSize = pageSize - page1[20]; - if( pBt->usableSize<500 ){ + usableSize = pageSize - page1[20]; + if( pageSize!=pBt->pageSize ){ + /* After reading the first page of the database assuming a page size + ** of BtShared.pageSize, we have discovered that the page-size is + ** actually pageSize. Unlock the database, leave pBt->pPage1 at + ** zero and return SQLITE_OK. The caller will call this function + ** again with the correct page-size. + */ + releasePage(pPage1); + pBt->usableSize = (u16)usableSize; + pBt->pageSize = (u16)pageSize; + freeTempSpace(pBt); + sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize); + return SQLITE_OK; + } + if( usableSize<500 ){ goto page1_init_failed; } - pBt->maxEmbedFrac = page1[21]; - pBt->minEmbedFrac = page1[22]; - pBt->minLeafFrac = page1[23]; + pBt->pageSize = (u16)pageSize; + pBt->usableSize = (u16)usableSize; #ifndef SQLITE_OMIT_AUTOVACUUM pBt->autoVacuum = (get4byte(&page1[36 + 4*4])?1:0); pBt->incrVacuum = (get4byte(&page1[36 + 7*4])?1:0); @@ -1453,13 +1815,10 @@ static int lockBtree(BtShared *pBt){ ** 17 bytes long, 0 to N bytes of payload, and an optional 4 byte overflow ** page pointer. */ - pBt->maxLocal = (pBt->usableSize-12)*pBt->maxEmbedFrac/255 - 23; - pBt->minLocal = (pBt->usableSize-12)*pBt->minEmbedFrac/255 - 23; + pBt->maxLocal = (pBt->usableSize-12)*64/255 - 23; + pBt->minLocal = (pBt->usableSize-12)*32/255 - 23; pBt->maxLeaf = pBt->usableSize - 35; - pBt->minLeaf = (pBt->usableSize-12)*pBt->minLeafFrac/255 - 23; - if( pBt->minLocal>pBt->maxLocal || pBt->maxLocal<0 ){ - goto page1_init_failed; - } + pBt->minLeaf = (pBt->usableSize-12)*32/255 - 23; assert( pBt->maxLeaf + 23 <= MX_CELL_SIZE(pBt) ); pBt->pPage1 = pPage1; return SQLITE_OK; @@ -1476,6 +1835,8 @@ page1_init_failed: */ static int lockBtreeWithRetry(Btree *pRef){ int rc = SQLITE_OK; + + assert( sqlite3BtreeHoldsMutex(pRef) ); if( pRef->inTrans==TRANS_NONE ){ u8 inTransaction = pRef->pBt->inTransaction; btreeIntegrity(pRef); @@ -1502,14 +1863,18 @@ static int lockBtreeWithRetry(Btree *pRef){ ** If there is a transaction in progress, this routine is a no-op. */ static void unlockBtreeIfUnused(BtShared *pBt){ + assert( sqlite3_mutex_held(pBt->mutex) ); if( pBt->inTransaction==TRANS_NONE && pBt->pCursor==0 && pBt->pPage1!=0 ){ if( sqlite3PagerRefcount(pBt->pPager)>=1 ){ + assert( pBt->pPage1->aData ); +#if 0 if( pBt->pPage1->aData==0 ){ MemPage *pPage = pBt->pPage1; - pPage->aData = &((u8*)pPage)[-pBt->pageSize]; + pPage->aData = sqlite3PagerGetData(pPage->pDbPage); pPage->pBt = pBt; pPage->pgno = 1; } +#endif releasePage(pBt->pPage1); } pBt->pPage1 = 0; @@ -1525,7 +1890,13 @@ static int newDatabase(BtShared *pBt){ MemPage *pP1; unsigned char *data; int rc; - if( sqlite3PagerPagecount(pBt->pPager)>0 ) return SQLITE_OK; + int nPage; + + assert( sqlite3_mutex_held(pBt->mutex) ); + rc = sqlite3PagerPagecount(pBt->pPager, &nPage); + if( rc!=SQLITE_OK || nPage>0 ){ + return rc; + } pP1 = pBt->pPage1; assert( pP1!=0 ); data = pP1->aData; @@ -1536,10 +1907,11 @@ static int newDatabase(BtShared *pBt){ put2byte(&data[16], pBt->pageSize); data[18] = 1; data[19] = 1; - data[20] = pBt->pageSize - pBt->usableSize; - data[21] = pBt->maxEmbedFrac; - data[22] = pBt->minEmbedFrac; - data[23] = pBt->minLeafFrac; + assert( pBt->usableSize<=pBt->pageSize && pBt->usableSize+255>=pBt->pageSize); + data[20] = (u8)(pBt->pageSize - pBt->usableSize); + data[21] = 64; + data[22] = 32; + data[23] = 32; memset(&data[24], 0, 100-24); zeroPage(pP1, PTF_INTKEY|PTF_LEAF|PTF_LEAFDATA ); pBt->pageSizeFixed = 1; @@ -1591,6 +1963,8 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ BtShared *pBt = p->pBt; int rc = SQLITE_OK; + sqlite3BtreeEnter(p); + pBt->db = p->db; btreeIntegrity(p); /* If the btree is already in a write-transaction, or it @@ -1598,12 +1972,13 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ ** is requested, this is a no-op. */ if( p->inTrans==TRANS_WRITE || (p->inTrans==TRANS_READ && !wrflag) ){ - return SQLITE_OK; + goto trans_begun; } /* Write transactions are not possible on a read-only database */ if( pBt->readOnly && wrflag ){ - return SQLITE_READONLY; + rc = SQLITE_READONLY; + goto trans_begun; } /* If another database handle has already opened a write transaction @@ -1611,12 +1986,27 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ ** requested, return SQLITE_BUSY. */ if( pBt->inTransaction==TRANS_WRITE && wrflag ){ - return SQLITE_BUSY; + rc = SQLITE_BUSY; + goto trans_begun; } +#ifndef SQLITE_OMIT_SHARED_CACHE + if( wrflag>1 ){ + BtLock *pIter; + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + if( pIter->pBtree!=p ){ + rc = SQLITE_BUSY; + goto trans_begun; + } + } + } +#endif + do { if( pBt->pPage1==0 ){ - rc = lockBtree(pBt); + do{ + rc = lockBtree(pBt); + }while( pBt->pPage1==0 && rc==SQLITE_OK ); } if( rc==SQLITE_OK && wrflag ){ @@ -1636,7 +2026,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ unlockBtreeIfUnused(pBt); } }while( rc==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && - sqlite3InvokeBusyHandler(pBt->pBusyHandler) ); + btreeInvokeBusyHandler(pBt) ); if( rc==SQLITE_OK ){ if( p->inTrans==TRANS_NONE ){ @@ -1646,9 +2036,26 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( p->inTrans>pBt->inTransaction ){ pBt->inTransaction = p->inTrans; } +#ifndef SQLITE_OMIT_SHARED_CACHE + if( wrflag>1 ){ + assert( !pBt->pExclusive ); + pBt->pExclusive = p; + } +#endif + } + + +trans_begun: + if( rc==SQLITE_OK && wrflag ){ + /* This call makes sure that the pager has the correct number of + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ + rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); } btreeIntegrity(p); + sqlite3BtreeLeave(p); return rc; } @@ -1664,10 +2071,11 @@ static int setChildPtrmaps(MemPage *pPage){ int nCell; /* Number of cells in page pPage */ int rc; /* Return code */ BtShared *pBt = pPage->pBt; - int isInitOrig = pPage->isInit; + u8 isInitOrig = pPage->isInit; Pgno pgno = pPage->pgno; - rc = sqlite3BtreeInitPage(pPage, pPage->pParent); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + rc = sqlite3BtreeInitPage(pPage); if( rc!=SQLITE_OK ){ goto set_child_ptrmaps_out; } @@ -1714,6 +2122,8 @@ set_child_ptrmaps_out: ** overflow page in the list. */ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); if( eType==PTRMAP_OVERFLOW2 ){ /* The pointer is always the first 4 bytes of the page in this case. */ if( get4byte(pPage->aData)!=iFrom ){ @@ -1721,11 +2131,11 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ } put4byte(pPage->aData, iTo); }else{ - int isInitOrig = pPage->isInit; + u8 isInitOrig = pPage->isInit; int i; int nCell; - sqlite3BtreeInitPage(pPage, 0); + sqlite3BtreeInitPage(pPage); nCell = pPage->nCell; for(i=0; ipgno; @@ -1779,11 +2190,13 @@ static int relocatePage( assert( eType==PTRMAP_OVERFLOW2 || eType==PTRMAP_OVERFLOW1 || eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ); + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pDbPage->pBt==pBt ); - /* Move page iDbPage from it's current location to page number iFreePage */ + /* Move page iDbPage from its current location to page number iFreePage */ TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", iDbPage, iFreePage, iPtrPage, eType)); - rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage); + rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); if( rc!=SQLITE_OK ){ return rc; } @@ -1853,14 +2266,10 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); ** number of pages the database file will contain after this ** process is complete. */ -static int incrVacuumStep(BtShared *pBt, Pgno nFin){ - Pgno iLastPg; /* Last page in the database */ +static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){ Pgno nFreeList; /* Number of pages still on the free-list */ - iLastPg = pBt->nTrunc; - if( iLastPg==0 ){ - iLastPg = sqlite3PagerPagecount(pBt->pPager); - } + assert( sqlite3_mutex_held(pBt->mutex) ); if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){ int rc; @@ -1924,20 +2333,22 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin){ assert( iFreePgpDbPage); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg); + if( rc==SQLITE_OK ){ + rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, nFin!=0); + } releasePage(pLastPg); if( rc!=SQLITE_OK ){ return rc; - } + } } } - pBt->nTrunc = iLastPg - 1; - while( pBt->nTrunc==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, pBt->nTrunc) ){ - pBt->nTrunc--; + if( nFin==0 ){ + iLastPg--; + while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){ + iLastPg--; + } + sqlite3PagerTruncateImage(pBt->pPager, iLastPg); } return SQLITE_OK; } @@ -1951,13 +2362,20 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin){ ** SQLITE_OK is returned. Otherwise an SQLite error code. */ int sqlite3BtreeIncrVacuum(Btree *p){ + int rc; BtShared *pBt = p->pBt; + + sqlite3BtreeEnter(p); + pBt->db = p->db; assert( pBt->inTransaction==TRANS_WRITE && p->inTrans==TRANS_WRITE ); if( !pBt->autoVacuum ){ - return SQLITE_DONE; + rc = SQLITE_DONE; + }else{ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, 0, sqlite3PagerImageSize(pBt->pPager)); } - invalidateAllOverflowCache(pBt); - return incrVacuumStep(pBt, 0); + sqlite3BtreeLeave(p); + return rc; } /* @@ -1969,68 +2387,58 @@ int sqlite3BtreeIncrVacuum(Btree *p){ ** i.e. the database has been reorganized so that only the first *pnTrunc ** pages are in use. */ -static int autoVacuumCommit(BtShared *pBt, Pgno *pnTrunc){ +static int autoVacuumCommit(BtShared *pBt){ int rc = SQLITE_OK; Pager *pPager = pBt->pPager; -#ifndef NDEBUG - int nRef = sqlite3PagerRefcount(pPager); -#endif + VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager) ); + assert( sqlite3_mutex_held(pBt->mutex) ); invalidateAllOverflowCache(pBt); assert(pBt->autoVacuum); if( !pBt->incrVacuum ){ - Pgno nFin = 0; + Pgno nFin; + Pgno nFree; + Pgno nPtrmap; + Pgno iFree; + const int pgsz = pBt->pageSize; + Pgno nOrig = pagerPagecount(pBt); - if( pBt->nTrunc==0 ){ - Pgno nFree; - Pgno nPtrmap; - const int pgsz = pBt->pageSize; - Pgno nOrig = sqlite3PagerPagecount(pBt->pPager); - - if( PTRMAP_ISPAGE(pBt, nOrig) ){ - return SQLITE_CORRUPT_BKPT; - } - if( nOrig==PENDING_BYTE_PAGE(pBt) ){ - nOrig--; - } - nFree = get4byte(&pBt->pPage1->aData[36]); - nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+pgsz/5)/(pgsz/5); - nFin = nOrig - nFree - nPtrmap; - if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){ - nFin--; - } - while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){ - nFin--; - } + if( PTRMAP_ISPAGE(pBt, nOrig) ){ + return SQLITE_CORRUPT_BKPT; + } + if( nOrig==PENDING_BYTE_PAGE(pBt) ){ + nOrig--; + } + nFree = get4byte(&pBt->pPage1->aData[36]); + nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+pgsz/5)/(pgsz/5); + nFin = nOrig - nFree - nPtrmap; + if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){ + nFin--; + } + while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){ + nFin--; } - while( rc==SQLITE_OK ){ - rc = incrVacuumStep(pBt, nFin); + for(iFree=nOrig; iFree>nFin && rc==SQLITE_OK; iFree--){ + rc = incrVacuumStep(pBt, nFin, iFree); } - if( rc==SQLITE_DONE ){ - assert(nFin==0 || pBt->nTrunc==0 || nFin<=pBt->nTrunc); + if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ rc = SQLITE_OK; - if( pBt->nTrunc ){ - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - put4byte(&pBt->pPage1->aData[32], 0); - put4byte(&pBt->pPage1->aData[36], 0); - pBt->nTrunc = nFin; - } + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[32], 0); + put4byte(&pBt->pPage1->aData[36], 0); + sqlite3PagerTruncateImage(pBt->pPager, nFin); } if( rc!=SQLITE_OK ){ sqlite3PagerRollback(pPager); } } - if( rc==SQLITE_OK ){ - *pnTrunc = pBt->nTrunc; - pBt->nTrunc = 0; - } assert( nRef==sqlite3PagerRefcount(pPager) ); return rc; } -#endif +#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */ /* ** This routine does the first phase of a two-phase commit. This routine @@ -2062,16 +2470,19 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ int rc = SQLITE_OK; if( p->inTrans==TRANS_WRITE ){ BtShared *pBt = p->pBt; - Pgno nTrunc = 0; + sqlite3BtreeEnter(p); + pBt->db = p->db; #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - rc = autoVacuumCommit(pBt, &nTrunc); + rc = autoVacuumCommit(pBt); if( rc!=SQLITE_OK ){ + sqlite3BtreeLeave(p); return rc; } } #endif - rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, nTrunc); + rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0); + sqlite3BtreeLeave(p); } return rc; } @@ -2093,6 +2504,8 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ int sqlite3BtreeCommitPhaseTwo(Btree *p){ BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + pBt->db = p->db; btreeIntegrity(p); /* If the handle has a write-transaction open, commit the shared-btrees @@ -2104,6 +2517,7 @@ int sqlite3BtreeCommitPhaseTwo(Btree *p){ assert( pBt->nTransaction>0 ); rc = sqlite3PagerCommitPhaseTwo(pBt->pPager); if( rc!=SQLITE_OK ){ + sqlite3BtreeLeave(p); return rc; } pBt->inTransaction = TRANS_READ; @@ -2130,6 +2544,7 @@ int sqlite3BtreeCommitPhaseTwo(Btree *p){ unlockBtreeIfUnused(pBt); btreeIntegrity(p); + sqlite3BtreeLeave(p); return SQLITE_OK; } @@ -2138,10 +2553,12 @@ int sqlite3BtreeCommitPhaseTwo(Btree *p){ */ int sqlite3BtreeCommit(Btree *p){ int rc; + sqlite3BtreeEnter(p); rc = sqlite3BtreeCommitPhaseOne(p, 0); if( rc==SQLITE_OK ){ rc = sqlite3BtreeCommitPhaseTwo(p); } + sqlite3BtreeLeave(p); return rc; } @@ -2150,17 +2567,55 @@ int sqlite3BtreeCommit(Btree *p){ ** Return the number of write-cursors open on this handle. This is for use ** in assert() expressions, so it is only compiled if NDEBUG is not ** defined. +** +** For the purposes of this routine, a write-cursor is any cursor that +** is capable of writing to the databse. That means the cursor was +** originally opened for writing and the cursor has not be disabled +** by having its state changed to CURSOR_FAULT. */ static int countWriteCursors(BtShared *pBt){ BtCursor *pCur; int r = 0; for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ - if( pCur->wrFlag ) r++; + if( pCur->wrFlag && pCur->eState!=CURSOR_FAULT ) r++; } return r; } #endif +/* +** This routine sets the state to CURSOR_FAULT and the error +** code to errCode for every cursor on BtShared that pBtree +** references. +** +** Every cursor is tripped, including cursors that belong +** to other database connections that happen to be sharing +** the cache with pBtree. +** +** This routine gets called when a rollback occurs. +** All cursors using the same cache must be tripped +** to prevent them from trying to use the btree after +** the rollback. The rollback may have deleted tables +** or moved root pages, so it is not sufficient to +** save the state of the cursor. The cursor must be +** invalidated. +*/ +void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ + BtCursor *p; + sqlite3BtreeEnter(pBtree); + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + int i; + sqlite3BtreeClearCursor(p); + p->eState = CURSOR_FAULT; + p->skip = errCode; + for(i=0; i<=p->iPage; i++){ + releasePage(p->apPage[i]); + p->apPage[i] = 0; + } + } + sqlite3BtreeLeave(pBtree); +} + /* ** Rollback the transaction in progress. All cursors will be ** invalided by this operation. Any attempt to use a cursor @@ -2175,6 +2630,8 @@ int sqlite3BtreeRollback(Btree *p){ BtShared *pBt = p->pBt; MemPage *pPage1; + sqlite3BtreeEnter(p); + pBt->db = p->db; rc = saveAllCursors(pBt, 0, 0); #ifndef SQLITE_OMIT_SHARED_CACHE if( rc!=SQLITE_OK ){ @@ -2185,12 +2642,7 @@ int sqlite3BtreeRollback(Btree *p){ ** we cannot simply return the error to the caller. Instead, abort ** all queries that may be using any of the cursors that failed to save. */ - while( pBt->pCursor ){ - sqlite3 *db = pBt->pCursor->pBtree->pSqlite; - if( db ){ - sqlite3AbortOtherActiveVdbes(db, 0); - } - } + sqlite3BtreeTripAllCursors(p, rc); } #endif btreeIntegrity(p); @@ -2199,10 +2651,6 @@ int sqlite3BtreeRollback(Btree *p){ if( p->inTrans==TRANS_WRITE ){ int rc2; -#ifndef SQLITE_OMIT_AUTOVACUUM - pBt->nTrunc = 0; -#endif - assert( TRANS_WRITE==pBt->inTransaction ); rc2 = sqlite3PagerRollback(pBt->pPager); if( rc2!=SQLITE_OK ){ @@ -2232,6 +2680,7 @@ int sqlite3BtreeRollback(Btree *p){ unlockBtreeIfUnused(pBt); btreeIntegrity(p); + sqlite3BtreeLeave(p); return rc; } @@ -2253,16 +2702,27 @@ int sqlite3BtreeRollback(Btree *p){ int sqlite3BtreeBeginStmt(Btree *p){ int rc; BtShared *pBt = p->pBt; - if( (p->inTrans!=TRANS_WRITE) || pBt->inStmt ){ - return pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR; + sqlite3BtreeEnter(p); + pBt->db = p->db; + assert( p->inTrans==TRANS_WRITE ); + assert( !pBt->inStmt ); + assert( pBt->readOnly==0 ); + if( NEVER(p->inTrans!=TRANS_WRITE || pBt->inStmt || pBt->readOnly) ){ + rc = SQLITE_INTERNAL; + }else{ + assert( pBt->inTransaction==TRANS_WRITE ); + /* At the pager level, a statement transaction is a savepoint with + ** an index greater than all savepoints created explicitly using + ** SQL statements. It is illegal to open, release or rollback any + ** such savepoints while the statement transaction savepoint is active. + */ + rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint+1); + pBt->inStmt = 1; } - assert( pBt->inTransaction==TRANS_WRITE ); - rc = pBt->readOnly ? SQLITE_OK : sqlite3PagerStmtBegin(pBt->pPager); - pBt->inStmt = 1; + sqlite3BtreeLeave(p); return rc; } - /* ** Commit the statment subtransaction currently in progress. If no ** subtransaction is active, this is a no-op. @@ -2270,12 +2730,17 @@ int sqlite3BtreeBeginStmt(Btree *p){ int sqlite3BtreeCommitStmt(Btree *p){ int rc; BtShared *pBt = p->pBt; - if( pBt->inStmt && !pBt->readOnly ){ - rc = sqlite3PagerStmtCommit(pBt->pPager); + sqlite3BtreeEnter(p); + pBt->db = p->db; + assert( pBt->readOnly==0 ); + if( pBt->inStmt ){ + int iStmtpoint = p->db->nSavepoint; + rc = sqlite3PagerSavepoint(pBt->pPager, SAVEPOINT_RELEASE, iStmtpoint); }else{ rc = SQLITE_OK; } pBt->inStmt = 0; + sqlite3BtreeLeave(p); return rc; } @@ -2290,31 +2755,49 @@ int sqlite3BtreeCommitStmt(Btree *p){ int sqlite3BtreeRollbackStmt(Btree *p){ int rc = SQLITE_OK; BtShared *pBt = p->pBt; - sqlite3MallocDisallow(); - if( pBt->inStmt && !pBt->readOnly ){ - rc = sqlite3PagerStmtRollback(pBt->pPager); - assert( countWriteCursors(pBt)==0 ); + sqlite3BtreeEnter(p); + pBt->db = p->db; + assert( pBt->readOnly==0 ); + if( pBt->inStmt ){ + int iStmtpoint = p->db->nSavepoint; + rc = sqlite3PagerSavepoint(pBt->pPager, SAVEPOINT_ROLLBACK, iStmtpoint); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSavepoint(pBt->pPager, SAVEPOINT_RELEASE, iStmtpoint); + } pBt->inStmt = 0; } - sqlite3MallocAllow(); + sqlite3BtreeLeave(p); return rc; } /* -** Default key comparison function to be used if no comparison function -** is specified on the sqlite3BtreeCursor() call. +** The second argument to this function, op, is always SAVEPOINT_ROLLBACK +** or SAVEPOINT_RELEASE. This function either releases or rolls back the +** savepoint identified by parameter iSavepoint, depending on the value +** of op. +** +** Normally, iSavepoint is greater than or equal to zero. However, if op is +** SAVEPOINT_ROLLBACK, then iSavepoint may also be -1. In this case the +** contents of the entire transaction are rolled back. This is different +** from a normal transaction rollback, as no locks are released and the +** transaction remains open. */ -static int dfltCompare( - void *NotUsed, /* User data is not used */ - int n1, const void *p1, /* First key to compare */ - int n2, const void *p2 /* Second key to compare */ -){ - int c; - c = memcmp(p1, p2, n1inTrans==TRANS_WRITE ){ + BtShared *pBt = p->pBt; + assert( pBt->inStmt==0 ); + assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); + assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); + sqlite3BtreeEnter(p); + pBt->db = p->db; + rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + if( rc==SQLITE_OK ){ + rc = newDatabase(pBt); + } + sqlite3BtreeLeave(p); } - return c; + return rc; } /* @@ -2344,30 +2827,28 @@ static int dfltCompare( ** root page of a b-tree. If it is not, then the cursor acquired ** will not work correctly. ** -** The comparison function must be logically the same for every cursor -** on a particular table. Changing the comparison function will result -** in incorrect operations. If the comparison function is NULL, a -** default comparison function is used. The comparison function is -** always ignored for INTKEY tables. +** It is assumed that the sqlite3BtreeCursorSize() bytes of memory +** pointed to by pCur have been zeroed by the caller. */ -int sqlite3BtreeCursor( - Btree *p, /* The btree */ - int iTable, /* Root page of table to open */ - int wrFlag, /* 1 to write. 0 read-only */ - int (*xCmp)(void*,int,const void*,int,const void*), /* Key Comparison func */ - void *pArg, /* First arg to xCompare() */ - BtCursor **ppCur /* Write new cursor here */ +static int btreeCursor( + Btree *p, /* The btree */ + int iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to comparison function */ + BtCursor *pCur /* Space for new cursor */ ){ int rc; - BtCursor *pCur; + Pgno nPage; BtShared *pBt = p->pBt; - *ppCur = 0; + assert( sqlite3BtreeHoldsMutex(p) ); + assert( wrFlag==0 || wrFlag==1 ); if( wrFlag ){ - if( pBt->readOnly ){ + assert( !pBt->readOnly ); + if( NEVER(pBt->readOnly) ){ return SQLITE_READONLY; } - if( checkReadLocks(p, iTable, 0) ){ + if( checkReadLocks(p, iTable, 0, 0) ){ return SQLITE_LOCKED; } } @@ -2377,21 +2858,17 @@ int sqlite3BtreeCursor( if( rc!=SQLITE_OK ){ return rc; } - if( pBt->readOnly && wrFlag ){ - return SQLITE_READONLY; - } - } - pCur = sqliteMalloc( sizeof(*pCur) ); - if( pCur==0 ){ - rc = SQLITE_NOMEM; - goto create_cursor_exception; } pCur->pgnoRoot = (Pgno)iTable; - if( iTable==1 && sqlite3PagerPagecount(pBt->pPager)==0 ){ + rc = sqlite3PagerPagecount(pBt->pPager, (int *)&nPage); + if( rc!=SQLITE_OK ){ + return rc; + } + if( iTable==1 && nPage==0 ){ rc = SQLITE_EMPTY; goto create_cursor_exception; } - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->pPage, 0); + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); if( rc!=SQLITE_OK ){ goto create_cursor_exception; } @@ -2400,47 +2877,72 @@ int sqlite3BtreeCursor( ** variables, link the cursor into the BtShared list and set *ppCur (the ** output argument to this function). */ - pCur->xCompare = xCmp ? xCmp : dfltCompare; - pCur->pArg = pArg; + pCur->pKeyInfo = pKeyInfo; pCur->pBtree = p; - pCur->wrFlag = wrFlag; + pCur->pBt = pBt; + pCur->wrFlag = (u8)wrFlag; pCur->pNext = pBt->pCursor; if( pCur->pNext ){ pCur->pNext->pPrev = pCur; } pBt->pCursor = pCur; pCur->eState = CURSOR_INVALID; - *ppCur = pCur; return SQLITE_OK; + create_cursor_exception: - if( pCur ){ - releasePage(pCur->pPage); - sqliteFree(pCur); - } + releasePage(pCur->apPage[0]); unlockBtreeIfUnused(pBt); return rc; } +int sqlite3BtreeCursor( + Btree *p, /* The btree */ + int iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to xCompare() */ + BtCursor *pCur /* Write new cursor here */ +){ + int rc; + sqlite3BtreeEnter(p); + p->pBt->db = p->db; + rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + sqlite3BtreeLeave(p); + return rc; +} +int sqlite3BtreeCursorSize(){ + return sizeof(BtCursor); +} + + /* ** Close a cursor. The read lock on the database file is released ** when the last cursor is closed. */ int sqlite3BtreeCloseCursor(BtCursor *pCur){ - BtShared *pBt = pCur->pBtree->pBt; - clearCursorPosition(pCur); - if( pCur->pPrev ){ - pCur->pPrev->pNext = pCur->pNext; - }else{ - pBt->pCursor = pCur->pNext; + Btree *pBtree = pCur->pBtree; + if( pBtree ){ + int i; + BtShared *pBt = pCur->pBt; + sqlite3BtreeEnter(pBtree); + pBt->db = pBtree->db; + sqlite3BtreeClearCursor(pCur); + if( pCur->pPrev ){ + pCur->pPrev->pNext = pCur->pNext; + }else{ + pBt->pCursor = pCur->pNext; + } + if( pCur->pNext ){ + pCur->pNext->pPrev = pCur->pPrev; + } + for(i=0; i<=pCur->iPage; i++){ + releasePage(pCur->apPage[i]); + } + unlockBtreeIfUnused(pBt); + invalidateOverflowCache(pCur); + /* sqlite3_free(pCur); */ + sqlite3BtreeLeave(pBtree); } - if( pCur->pNext ){ - pCur->pNext->pPrev = pCur->pPrev; - } - releasePage(pCur->pPage); - unlockBtreeIfUnused(pBt); - invalidateOverflowCache(pCur); - sqliteFree(pCur); return SQLITE_OK; } @@ -2449,12 +2951,15 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){ ** The temporary cursor is not on the cursor list for the Btree. */ void sqlite3BtreeGetTempCursor(BtCursor *pCur, BtCursor *pTempCur){ - memcpy(pTempCur, pCur, sizeof(*pCur)); + int i; + assert( cursorHoldsMutex(pCur) ); + memcpy(pTempCur, pCur, sizeof(BtCursor)); pTempCur->pNext = 0; pTempCur->pPrev = 0; - if( pTempCur->pPage ){ - sqlite3PagerRef(pTempCur->pPage->pDbPage); + for(i=0; i<=pTempCur->iPage; i++){ + sqlite3PagerRef(pTempCur->apPage[i]->pDbPage); } + assert( pTempCur->pKey==0 ); } /* @@ -2462,9 +2967,12 @@ void sqlite3BtreeGetTempCursor(BtCursor *pCur, BtCursor *pTempCur){ ** function above. */ void sqlite3BtreeReleaseTempCursor(BtCursor *pCur){ - if( pCur->pPage ){ - sqlite3PagerUnref(pCur->pPage->pDbPage); + int i; + assert( cursorHoldsMutex(pCur) ); + for(i=0; i<=pCur->iPage; i++){ + sqlite3PagerUnref(pCur->apPage[i]->pDbPage); } + sqlite3_free(pCur->pKey); } /* @@ -2485,8 +2993,9 @@ void sqlite3BtreeReleaseTempCursor(BtCursor *pCur){ #ifndef NDEBUG static void assertCellInfo(BtCursor *pCur){ CellInfo info; + int iPage = pCur->iPage; memset(&info, 0, sizeof(info)); - sqlite3BtreeParseCell(pCur->pPage, pCur->idx, &info); + sqlite3BtreeParseCell(pCur->apPage[iPage], pCur->aiIdx[iPage], &info); assert( memcmp(&info, &pCur->info, sizeof(info))==0 ); } #else @@ -2496,18 +3005,22 @@ void sqlite3BtreeReleaseTempCursor(BtCursor *pCur){ /* Use a real function in MSVC to work around bugs in that compiler. */ static void getCellInfo(BtCursor *pCur){ if( pCur->info.nSize==0 ){ - sqlite3BtreeParseCell(pCur->pPage, pCur->idx, &pCur->info); + int iPage = pCur->iPage; + sqlite3BtreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); + pCur->validNKey = 1; }else{ assertCellInfo(pCur); } } #else /* if not _MSC_VER */ /* Use a macro in all other compilers so that the function is inlined */ -#define getCellInfo(pCur) \ - if( pCur->info.nSize==0 ){ \ - sqlite3BtreeParseCell(pCur->pPage, pCur->idx, &pCur->info); \ - }else{ \ - assertCellInfo(pCur); \ +#define getCellInfo(pCur) \ + if( pCur->info.nSize==0 ){ \ + int iPage = pCur->iPage; \ + sqlite3BtreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); \ + pCur->validNKey = 1; \ + }else{ \ + assertCellInfo(pCur); \ } #endif /* _MSC_VER */ @@ -2520,7 +3033,10 @@ void sqlite3BtreeReleaseTempCursor(BtCursor *pCur){ ** itself, not the number of bytes in the key. */ int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ - int rc = restoreOrClearCursorPosition(pCur); + int rc; + + assert( cursorHoldsMutex(pCur) ); + rc = restoreCursorPosition(pCur); if( rc==SQLITE_OK ){ assert( pCur->eState==CURSOR_INVALID || pCur->eState==CURSOR_VALID ); if( pCur->eState==CURSOR_INVALID ){ @@ -2541,7 +3057,10 @@ int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ ** the database is empty) then *pSize is set to 0. */ int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ - int rc = restoreOrClearCursorPosition(pCur); + int rc; + + assert( cursorHoldsMutex(pCur) ); + rc = restoreCursorPosition(pCur); if( rc==SQLITE_OK ){ assert( pCur->eState==CURSOR_INVALID || pCur->eState==CURSOR_VALID ); if( pCur->eState==CURSOR_INVALID ){ @@ -2565,7 +3084,7 @@ int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ ** ** Unless pPgnoNext is NULL, the page number of the next overflow ** page in the linked list is written to *pPgnoNext. If page ovfl -** is the last page in it's linked list, *pPgnoNext is set to zero. +** is the last page in its linked list, *pPgnoNext is set to zero. ** ** If ppPage is not NULL, *ppPage is set to the MemPage* handle ** for page ovfl. The underlying pager page may have been requested @@ -2579,8 +3098,9 @@ static int getOverflowPage( Pgno *pPgnoNext /* OUT: Next overflow page number */ ){ Pgno next = 0; - int rc; + int rc = SQLITE_OK; + assert( sqlite3_mutex_held(pBt->mutex) ); /* One of these must not be NULL. Otherwise, why call this function? */ assert(ppPage || pPgnoNext); @@ -2607,7 +3127,7 @@ static int getOverflowPage( iGuess++; } - if( iGuess<=sqlite3PagerPagecount(pBt->pPager) ){ + if( iGuess<=pagerPagecount(pBt) ){ rc = ptrmapGet(pBt, iGuess, &eType, &pgno); if( rc!=SQLITE_OK ){ return rc; @@ -2703,8 +3223,8 @@ static int copyPayload( */ static int accessPayload( BtCursor *pCur, /* Cursor pointing to entry to read from */ - int offset, /* Begin reading this far into payload */ - int amt, /* Read this many bytes */ + u32 offset, /* Begin reading this far into payload */ + u32 amt, /* Read this many bytes */ unsigned char *pBuf, /* Write the bytes into this buffer */ int skipKey, /* offset begins at data if this is true */ int eOp /* zero to read. non-zero to write. */ @@ -2713,24 +3233,26 @@ static int accessPayload( int rc = SQLITE_OK; u32 nKey; int iIdx = 0; - MemPage *pPage = pCur->pPage; /* Btree page of current cursor entry */ - BtShared *pBt = pCur->pBtree->pBt; /* Btree this cursor belongs to */ + MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ + BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ assert( pPage ); assert( pCur->eState==CURSOR_VALID ); - assert( pCur->idx>=0 && pCur->idxnCell ); - assert( offset>=0 ); + assert( pCur->aiIdx[pCur->iPage]nCell ); + assert( cursorHoldsMutex(pCur) ); getCellInfo(pCur); aPayload = pCur->info.pCell + pCur->info.nHeader; - nKey = (pPage->intKey ? 0 : pCur->info.nKey); + nKey = (pPage->intKey ? 0 : (int)pCur->info.nKey); if( skipKey ){ offset += nKey; } - if( offset+amt > nKey+pCur->info.nData ){ + if( offset+amt > nKey+pCur->info.nData + || &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] + ){ /* Trying to read or write past the end of the data is an error */ - return SQLITE_ERROR; + return SQLITE_CORRUPT_BKPT; } /* Check if data must be read/written to/from the btree page itself. */ @@ -2748,7 +3270,7 @@ static int accessPayload( } if( rc==SQLITE_OK && amt>0 ){ - const int ovflSize = pBt->usableSize - 4; /* Bytes content per ovfl page */ + const u32 ovflSize = pBt->usableSize - 4; /* Bytes content per ovfl page */ Pgno nextPage; nextPage = get4byte(&aPayload[pCur->info.nLocal]); @@ -2763,7 +3285,7 @@ static int accessPayload( */ if( pCur->isIncrblobHandle && !pCur->aOverflow ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; - pCur->aOverflow = (Pgno *)sqliteMalloc(sizeof(Pgno)*nOvfl); + pCur->aOverflow = (Pgno *)sqlite3MallocZero(sizeof(Pgno)*nOvfl); if( nOvfl && !pCur->aOverflow ){ rc = SQLITE_NOMEM; } @@ -2843,15 +3365,17 @@ static int accessPayload( ** the available payload. */ int sqlite3BtreeKey(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ - int rc = restoreOrClearCursorPosition(pCur); + int rc; + + assert( cursorHoldsMutex(pCur) ); + rc = restoreCursorPosition(pCur); if( rc==SQLITE_OK ){ assert( pCur->eState==CURSOR_VALID ); - assert( pCur->pPage!=0 ); - if( pCur->pPage->intKey ){ + assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); + if( pCur->apPage[0]->intKey ){ return SQLITE_CORRUPT_BKPT; } - assert( pCur->pPage->intKey==0 ); - assert( pCur->idx>=0 && pCur->idxpPage->nCell ); + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); rc = accessPayload(pCur, offset, amt, (unsigned char*)pBuf, 0, 0); } return rc; @@ -2867,11 +3391,20 @@ int sqlite3BtreeKey(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ ** the available payload. */ int sqlite3BtreeData(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ - int rc = restoreOrClearCursorPosition(pCur); + int rc; + +#ifndef SQLITE_OMIT_INCRBLOB + if ( pCur->eState==CURSOR_INVALID ){ + return SQLITE_ABORT; + } +#endif + + assert( cursorHoldsMutex(pCur) ); + rc = restoreCursorPosition(pCur); if( rc==SQLITE_OK ){ assert( pCur->eState==CURSOR_VALID ); - assert( pCur->pPage!=0 ); - assert( pCur->idx>=0 && pCur->idxpPage->nCell ); + assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); rc = accessPayload(pCur, offset, amt, pBuf, 1, 0); } return rc; @@ -2904,19 +3437,20 @@ static const unsigned char *fetchPayload( unsigned char *aPayload; MemPage *pPage; u32 nKey; - int nLocal; + u32 nLocal; - assert( pCur!=0 && pCur->pPage!=0 ); + assert( pCur!=0 && pCur->iPage>=0 && pCur->apPage[pCur->iPage]); assert( pCur->eState==CURSOR_VALID ); - pPage = pCur->pPage; - assert( pCur->idx>=0 && pCur->idxnCell ); + assert( cursorHoldsMutex(pCur) ); + pPage = pCur->apPage[pCur->iPage]; + assert( pCur->aiIdx[pCur->iPage]nCell ); getCellInfo(pCur); aPayload = pCur->info.pCell; aPayload += pCur->info.nHeader; if( pPage->intKey ){ nKey = 0; }else{ - nKey = pCur->info.nKey; + nKey = (int)pCur->info.nKey; } if( skipKey ){ aPayload += nKey; @@ -2938,18 +3472,23 @@ static const unsigned char *fetchPayload( ** b-tree page. Write the number of available bytes into *pAmt. ** ** The pointer returned is ephemeral. The key/data may move -** or be destroyed on the next call to any Btree routine. +** or be destroyed on the next call to any Btree routine, +** including calls from other threads against the same cache. +** Hence, a mutex on the BtShared should be held prior to calling +** this routine. ** ** These routines is used to get quick access to key and data ** in the common case where no overflow pages are used. */ const void *sqlite3BtreeKeyFetch(BtCursor *pCur, int *pAmt){ + assert( cursorHoldsMutex(pCur) ); if( pCur->eState==CURSOR_VALID ){ return (const void*)fetchPayload(pCur, pAmt, 0); } return 0; } const void *sqlite3BtreeDataFetch(BtCursor *pCur, int *pAmt){ + assert( cursorHoldsMutex(pCur) ); if( pCur->eState==CURSOR_VALID ){ return (const void*)fetchPayload(pCur, pAmt, 1); } @@ -2963,42 +3502,49 @@ const void *sqlite3BtreeDataFetch(BtCursor *pCur, int *pAmt){ */ static int moveToChild(BtCursor *pCur, u32 newPgno){ int rc; + int i = pCur->iPage; MemPage *pNewPage; - MemPage *pOldPage; - BtShared *pBt = pCur->pBtree->pBt; + BtShared *pBt = pCur->pBt; + assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - rc = getAndInitPage(pBt, newPgno, &pNewPage, pCur->pPage); + assert( pCur->iPageiPage>=(BTCURSOR_MAX_DEPTH-1) ){ + return SQLITE_CORRUPT_BKPT; + } + rc = getAndInitPage(pBt, newPgno, &pNewPage); if( rc ) return rc; - pNewPage->idxParent = pCur->idx; - pOldPage = pCur->pPage; - pOldPage->idxShift = 0; - releasePage(pOldPage); - pCur->pPage = pNewPage; - pCur->idx = 0; + pCur->apPage[i+1] = pNewPage; + pCur->aiIdx[i+1] = 0; + pCur->iPage++; + pCur->info.nSize = 0; + pCur->validNKey = 0; if( pNewPage->nCell<1 ){ return SQLITE_CORRUPT_BKPT; } return SQLITE_OK; } +#ifndef NDEBUG /* -** Return true if the page is the virtual root of its table. -** -** The virtual root page is the root page for most tables. But -** for the table rooted on page 1, sometime the real root page -** is empty except for the right-pointer. In such cases the -** virtual root page is the page that the right-pointer of page -** 1 is pointing to. +** Page pParent is an internal (non-leaf) tree page. This function +** asserts that page number iChild is the left-child if the iIdx'th +** cell in page pParent. Or, if iIdx is equal to the total number of +** cells in pParent, that page number iChild is the right-child of +** the page. */ -int sqlite3BtreeIsRootPage(MemPage *pPage){ - MemPage *pParent = pPage->pParent; - if( pParent==0 ) return 1; - if( pParent->pgno>1 ) return 0; - if( get2byte(&pParent->aData[pParent->hdrOffset+3])==0 ) return 1; - return 0; +static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ + assert( iIdx<=pParent->nCell ); + if( iIdx==pParent->nCell ){ + assert( get4byte(&pParent->aData[pParent->hdrOffset+8])==iChild ); + }else{ + assert( get4byte(findCell(pParent, iIdx))==iChild ); + } } +#else +# define assertParentIndex(x,y,z) +#endif /* ** Move the cursor up to the parent page. @@ -3009,23 +3555,19 @@ int sqlite3BtreeIsRootPage(MemPage *pPage){ ** the largest cell index. */ void sqlite3BtreeMoveToParent(BtCursor *pCur){ - MemPage *pParent; - MemPage *pPage; - int idxParent; - + assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - pPage = pCur->pPage; - assert( pPage!=0 ); - assert( !sqlite3BtreeIsRootPage(pPage) ); - pParent = pPage->pParent; - assert( pParent!=0 ); - idxParent = pPage->idxParent; - sqlite3PagerRef(pParent->pDbPage); - releasePage(pPage); - pCur->pPage = pParent; + assert( pCur->iPage>0 ); + assert( pCur->apPage[pCur->iPage] ); + assertParentIndex( + pCur->apPage[pCur->iPage-1], + pCur->aiIdx[pCur->iPage-1], + pCur->apPage[pCur->iPage]->pgno + ); + releasePage(pCur->apPage[pCur->iPage]); + pCur->iPage--; pCur->info.nSize = 0; - assert( pParent->idxShift==0 ); - pCur->idx = idxParent; + pCur->validNKey = 0; } /* @@ -3034,26 +3576,42 @@ void sqlite3BtreeMoveToParent(BtCursor *pCur){ static int moveToRoot(BtCursor *pCur){ MemPage *pRoot; int rc = SQLITE_OK; - BtShared *pBt = pCur->pBtree->pBt; + Btree *p = pCur->pBtree; + BtShared *pBt = p->pBt; - if( pCur->eState==CURSOR_REQUIRESEEK ){ - clearCursorPosition(pCur); + assert( cursorHoldsMutex(pCur) ); + assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); + assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); + assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + if( pCur->eState==CURSOR_FAULT ){ + return pCur->skip; + } + sqlite3BtreeClearCursor(pCur); } - pRoot = pCur->pPage; - if( pRoot && pRoot->pgno==pCur->pgnoRoot ){ - assert( pRoot->isInit ); + + if( pCur->iPage>=0 ){ + int i; + for(i=1; i<=pCur->iPage; i++){ + releasePage(pCur->apPage[i]); + } }else{ if( - SQLITE_OK!=(rc = getAndInitPage(pBt, pCur->pgnoRoot, &pRoot, 0)) + SQLITE_OK!=(rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0])) ){ pCur->eState = CURSOR_INVALID; return rc; } - releasePage(pCur->pPage); - pCur->pPage = pRoot; } - pCur->idx = 0; + + pRoot = pCur->apPage[0]; + assert( pRoot->pgno==pCur->pgnoRoot ); + pCur->iPage = 0; + pCur->aiIdx[0] = 0; pCur->info.nSize = 0; + pCur->atLast = 0; + pCur->validNKey = 0; + if( pRoot->nCell==0 && !pRoot->leaf ){ Pgno subpage; assert( pRoot->pgno==1 ); @@ -3061,8 +3619,9 @@ static int moveToRoot(BtCursor *pCur){ assert( subpage>0 ); pCur->eState = CURSOR_VALID; rc = moveToChild(pCur, subpage); + }else{ + pCur->eState = ((pRoot->nCell>0)?CURSOR_VALID:CURSOR_INVALID); } - pCur->eState = ((pCur->pPage->nCell>0)?CURSOR_VALID:CURSOR_INVALID); return rc; } @@ -3075,17 +3634,17 @@ static int moveToRoot(BtCursor *pCur){ */ static int moveToLeftmost(BtCursor *pCur){ Pgno pgno; - int rc; + int rc = SQLITE_OK; MemPage *pPage; + assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - while( !(pPage = pCur->pPage)->leaf ){ - assert( pCur->idx>=0 && pCur->idxnCell ); - pgno = get4byte(findCell(pPage, pCur->idx)); + while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ + assert( pCur->aiIdx[pCur->iPage]nCell ); + pgno = get4byte(findCell(pPage, pCur->aiIdx[pCur->iPage])); rc = moveToChild(pCur, pgno); - if( rc ) return rc; } - return SQLITE_OK; + return rc; } /* @@ -3100,19 +3659,22 @@ static int moveToLeftmost(BtCursor *pCur){ */ static int moveToRightmost(BtCursor *pCur){ Pgno pgno; - int rc; - MemPage *pPage; + int rc = SQLITE_OK; + MemPage *pPage = 0; + assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - while( !(pPage = pCur->pPage)->leaf ){ + while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); - pCur->idx = pPage->nCell; + pCur->aiIdx[pCur->iPage] = pPage->nCell; rc = moveToChild(pCur, pgno); - if( rc ) return rc; } - pCur->idx = pPage->nCell - 1; - pCur->info.nSize = 0; - return SQLITE_OK; + if( rc==SQLITE_OK ){ + pCur->aiIdx[pCur->iPage] = pPage->nCell-1; + pCur->info.nSize = 0; + pCur->validNKey = 0; + } + return rc; } /* Move the cursor to the first entry in the table. Return SQLITE_OK @@ -3121,16 +3683,21 @@ static int moveToRightmost(BtCursor *pCur){ */ int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ int rc; + + assert( cursorHoldsMutex(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); rc = moveToRoot(pCur); - if( rc ) return rc; - if( pCur->eState==CURSOR_INVALID ){ - assert( pCur->pPage->nCell==0 ); - *pRes = 1; - return SQLITE_OK; + if( rc==SQLITE_OK ){ + if( pCur->eState==CURSOR_INVALID ){ + assert( pCur->apPage[pCur->iPage]->nCell==0 ); + *pRes = 1; + rc = SQLITE_OK; + }else{ + assert( pCur->apPage[pCur->iPage]->nCell>0 ); + *pRes = 0; + rc = moveToLeftmost(pCur); + } } - assert( pCur->pPage->nCell>0 ); - *pRes = 0; - rc = moveToLeftmost(pCur); return rc; } @@ -3140,131 +3707,170 @@ int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ */ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ int rc; + + assert( cursorHoldsMutex(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); rc = moveToRoot(pCur); - if( rc ) return rc; - if( CURSOR_INVALID==pCur->eState ){ - assert( pCur->pPage->nCell==0 ); - *pRes = 1; - return SQLITE_OK; + if( rc==SQLITE_OK ){ + if( CURSOR_INVALID==pCur->eState ){ + assert( pCur->apPage[pCur->iPage]->nCell==0 ); + *pRes = 1; + }else{ + assert( pCur->eState==CURSOR_VALID ); + *pRes = 0; + rc = moveToRightmost(pCur); + getCellInfo(pCur); + pCur->atLast = rc==SQLITE_OK ?1:0; + } } - assert( pCur->eState==CURSOR_VALID ); - *pRes = 0; - rc = moveToRightmost(pCur); return rc; } -/* Move the cursor so that it points to an entry near pKey/nKey. -** Return a success code. +/* Move the cursor so that it points to an entry near the key +** specified by pIdxKey or intKey. Return a success code. ** -** For INTKEY tables, only the nKey parameter is used. pKey is -** ignored. For other tables, nKey is the number of bytes of data -** in pKey. The comparison function specified when the cursor was -** created is used to compare keys. +** For INTKEY tables, the intKey parameter is used. pIdxKey +** must be NULL. For index tables, pIdxKey is used and intKey +** is ignored. ** ** If an exact match is not found, then the cursor is always ** left pointing at a leaf page which would hold the entry if it ** were present. The cursor might point to an entry that comes ** before or after the key. ** -** The result of comparing the key with the entry to which the -** cursor is written to *pRes if pRes!=NULL. The meaning of -** this value is as follows: +** An integer is written into *pRes which is the result of +** comparing the key with the entry to which the cursor is +** pointing. The meaning of the integer written into +** *pRes is as follows: ** ** *pRes<0 The cursor is left pointing at an entry that -** is smaller than pKey or if the table is empty +** is smaller than intKey/pIdxKey or if the table is empty ** and the cursor is therefore left point to nothing. ** ** *pRes==0 The cursor is left pointing at an entry that -** exactly matches pKey. +** exactly matches intKey/pIdxKey. ** ** *pRes>0 The cursor is left pointing at an entry that -** is larger than pKey. +** is larger than intKey/pIdxKey. +** */ -int sqlite3BtreeMoveto( - BtCursor *pCur, /* The cursor to be moved */ - const void *pKey, /* The key content for indices. Not used by tables */ - i64 nKey, /* Size of pKey. Or the key for tables */ - int biasRight, /* If true, bias the search to the high end */ - int *pRes /* Search result flag */ +int sqlite3BtreeMovetoUnpacked( + BtCursor *pCur, /* The cursor to be moved */ + UnpackedRecord *pIdxKey, /* Unpacked index key */ + i64 intKey, /* The table key */ + int biasRight, /* If true, bias the search to the high end */ + int *pRes /* Write search results here */ ){ int rc; + + assert( cursorHoldsMutex(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + + /* If the cursor is already positioned at the point we are trying + ** to move to, then just return without doing any work */ + if( pCur->eState==CURSOR_VALID && pCur->validNKey + && pCur->apPage[0]->intKey + ){ + if( pCur->info.nKey==intKey ){ + *pRes = 0; + return SQLITE_OK; + } + if( pCur->atLast && pCur->info.nKeypPage ); - assert( pCur->pPage->isInit ); + if( rc ){ + return rc; + } + assert( pCur->apPage[pCur->iPage] ); + assert( pCur->apPage[pCur->iPage]->isInit ); if( pCur->eState==CURSOR_INVALID ){ *pRes = -1; - assert( pCur->pPage->nCell==0 ); + assert( pCur->apPage[pCur->iPage]->nCell==0 ); return SQLITE_OK; } + assert( pCur->apPage[0]->intKey || pIdxKey ); for(;;){ int lwr, upr; Pgno chldPg; - MemPage *pPage = pCur->pPage; + MemPage *pPage = pCur->apPage[pCur->iPage]; int c = -1; /* pRes return if table is empty must be -1 */ lwr = 0; upr = pPage->nCell-1; - if( !pPage->intKey && pKey==0 ){ - return SQLITE_CORRUPT_BKPT; + if( (!pPage->intKey && pIdxKey==0) || upr<0 ){ + rc = SQLITE_CORRUPT_BKPT; + goto moveto_finish; } if( biasRight ){ - pCur->idx = upr; + pCur->aiIdx[pCur->iPage] = (u16)upr; }else{ - pCur->idx = (upr+lwr)/2; + pCur->aiIdx[pCur->iPage] = (u16)((upr+lwr)/2); } - if( lwr<=upr ) for(;;){ + for(;;){ void *pCellKey; i64 nCellKey; + int idx = pCur->aiIdx[pCur->iPage]; pCur->info.nSize = 0; + pCur->validNKey = 1; if( pPage->intKey ){ u8 *pCell; - pCell = findCell(pPage, pCur->idx) + pPage->childPtrSize; + pCell = findCell(pPage, idx) + pPage->childPtrSize; if( pPage->hasData ){ u32 dummy; - pCell += getVarint32(pCell, &dummy); + pCell += getVarint32(pCell, dummy); } - getVarint(pCell, (u64 *)&nCellKey); - if( nCellKeynKey ){ - c = +1; - }else{ + getVarint(pCell, (u64*)&nCellKey); + if( nCellKey==intKey ){ c = 0; + }else if( nCellKeyintKey ); + c = +1; } }else{ int available; pCellKey = (void *)fetchPayload(pCur, &available, 0); nCellKey = pCur->info.nKey; if( available>=nCellKey ){ - c = pCur->xCompare(pCur->pArg, nCellKey, pCellKey, nKey, pKey); + c = sqlite3VdbeRecordCompare((int)nCellKey, pCellKey, pIdxKey); }else{ - pCellKey = sqliteMallocRaw( nCellKey ); - if( pCellKey==0 ) return SQLITE_NOMEM; - rc = sqlite3BtreeKey(pCur, 0, nCellKey, (void *)pCellKey); - c = pCur->xCompare(pCur->pArg, nCellKey, pCellKey, nKey, pKey); - sqliteFree(pCellKey); - if( rc ) return rc; + pCellKey = sqlite3Malloc( (int)nCellKey ); + if( pCellKey==0 ){ + rc = SQLITE_NOMEM; + goto moveto_finish; + } + rc = sqlite3BtreeKey(pCur, 0, (int)nCellKey, (void*)pCellKey); + c = sqlite3VdbeRecordCompare((int)nCellKey, pCellKey, pIdxKey); + sqlite3_free(pCellKey); + if( rc ) goto moveto_finish; } } if( c==0 ){ - if( pPage->leafData && !pPage->leaf ){ - lwr = pCur->idx; + pCur->info.nKey = nCellKey; + if( pPage->intKey && !pPage->leaf ){ + lwr = idx; upr = lwr - 1; break; }else{ - if( pRes ) *pRes = 0; - return SQLITE_OK; + *pRes = 0; + rc = SQLITE_OK; + goto moveto_finish; } } if( c<0 ){ - lwr = pCur->idx+1; + lwr = idx+1; }else{ - upr = pCur->idx-1; + upr = idx-1; } if( lwr>upr ){ + pCur->info.nKey = nCellKey; break; } - pCur->idx = (lwr+upr)/2; + pCur->aiIdx[pCur->iPage] = (u16)((lwr+upr)/2); } assert( lwr==upr+1 ); assert( pPage->isInit ); @@ -3276,20 +3882,53 @@ int sqlite3BtreeMoveto( chldPg = get4byte(findCell(pPage, lwr)); } if( chldPg==0 ){ - assert( pCur->idx>=0 && pCur->idxpPage->nCell ); + assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); if( pRes ) *pRes = c; - return SQLITE_OK; + rc = SQLITE_OK; + goto moveto_finish; } - pCur->idx = lwr; + pCur->aiIdx[pCur->iPage] = (u16)lwr; pCur->info.nSize = 0; + pCur->validNKey = 0; rc = moveToChild(pCur, chldPg); - if( rc ){ - return rc; - } + if( rc ) goto moveto_finish; } - /* NOT REACHED */ +moveto_finish: + return rc; } +/* +** In this version of BtreeMoveto, pKey is a packed index record +** such as is generated by the OP_MakeRecord opcode. Unpack the +** record and then call BtreeMovetoUnpacked() to do the work. +*/ +int sqlite3BtreeMoveto( + BtCursor *pCur, /* Cursor open on the btree to be searched */ + const void *pKey, /* Packed key if the btree is an index */ + i64 nKey, /* Integer key for tables. Size of pKey for indices */ + int bias, /* Bias search to the high end */ + int *pRes /* Write search results here */ +){ + int rc; /* Status code */ + UnpackedRecord *pIdxKey; /* Unpacked index key */ + UnpackedRecord aSpace[16]; /* Temp space for pIdxKey - to avoid a malloc */ + + if( pKey ){ + assert( nKey==(i64)(int)nKey ); + pIdxKey = sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, + aSpace, sizeof(aSpace)); + if( pIdxKey==0 ) return SQLITE_NOMEM; + }else{ + pIdxKey = 0; + } + rc = sqlite3BtreeMovetoUnpacked(pCur, pIdxKey, nKey, bias, pRes); + if( pKey ){ + sqlite3VdbeDeleteUnpackedRecord(pIdxKey); + } + return rc; +} + + /* ** Return TRUE if the cursor is not pointing at an entry of the table. ** @@ -3305,6 +3944,14 @@ int sqlite3BtreeEof(BtCursor *pCur){ return (CURSOR_VALID!=pCur->eState); } +/* +** Return the database connection handle for a cursor. +*/ +sqlite3 *sqlite3BtreeCursorDb(const BtCursor *pCur){ + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + return pCur->pBtree->db; +} + /* ** Advance the cursor to the next entry in the database. If ** successful then set *pRes=0. If the cursor @@ -3313,14 +3960,15 @@ int sqlite3BtreeEof(BtCursor *pCur){ */ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ int rc; + int idx; MemPage *pPage; - rc = restoreOrClearCursorPosition(pCur); + assert( cursorHoldsMutex(pCur) ); + rc = restoreCursorPosition(pCur); if( rc!=SQLITE_OK ){ return rc; } assert( pRes!=0 ); - pPage = pCur->pPage; if( CURSOR_INVALID==pCur->eState ){ *pRes = 1; return SQLITE_OK; @@ -3332,12 +3980,14 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ } pCur->skip = 0; + pPage = pCur->apPage[pCur->iPage]; + idx = ++pCur->aiIdx[pCur->iPage]; assert( pPage->isInit ); - assert( pCur->idxnCell ); + assert( idx<=pPage->nCell ); - pCur->idx++; pCur->info.nSize = 0; - if( pCur->idx>=pPage->nCell ){ + pCur->validNKey = 0; + if( idx>=pPage->nCell ){ if( !pPage->leaf ){ rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); if( rc ) return rc; @@ -3346,16 +3996,16 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ return rc; } do{ - if( sqlite3BtreeIsRootPage(pPage) ){ + if( pCur->iPage==0 ){ *pRes = 1; pCur->eState = CURSOR_INVALID; return SQLITE_OK; } sqlite3BtreeMoveToParent(pCur); - pPage = pCur->pPage; - }while( pCur->idx>=pPage->nCell ); + pPage = pCur->apPage[pCur->iPage]; + }while( pCur->aiIdx[pCur->iPage]>=pPage->nCell ); *pRes = 0; - if( pPage->leafData ){ + if( pPage->intKey ){ rc = sqlite3BtreeNext(pCur, pRes); }else{ rc = SQLITE_OK; @@ -3370,6 +4020,7 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ return rc; } + /* ** Step the cursor to the back to the previous entry in the database. If ** successful then set *pRes=0. If the cursor @@ -3378,13 +4029,14 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ */ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ int rc; - Pgno pgno; MemPage *pPage; - rc = restoreOrClearCursorPosition(pCur); + assert( cursorHoldsMutex(pCur) ); + rc = restoreCursorPosition(pCur); if( rc!=SQLITE_OK ){ return rc; } + pCur->atLast = 0; if( CURSOR_INVALID==pCur->eState ){ *pRes = 1; return SQLITE_OK; @@ -3396,27 +4048,30 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ } pCur->skip = 0; - pPage = pCur->pPage; + pPage = pCur->apPage[pCur->iPage]; assert( pPage->isInit ); - assert( pCur->idx>=0 ); if( !pPage->leaf ){ - pgno = get4byte( findCell(pPage, pCur->idx) ); - rc = moveToChild(pCur, pgno); - if( rc ) return rc; + int idx = pCur->aiIdx[pCur->iPage]; + rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); + if( rc ){ + return rc; + } rc = moveToRightmost(pCur); }else{ - while( pCur->idx==0 ){ - if( sqlite3BtreeIsRootPage(pPage) ){ + while( pCur->aiIdx[pCur->iPage]==0 ){ + if( pCur->iPage==0 ){ pCur->eState = CURSOR_INVALID; *pRes = 1; return SQLITE_OK; } sqlite3BtreeMoveToParent(pCur); - pPage = pCur->pPage; } - pCur->idx--; pCur->info.nSize = 0; - if( pPage->leafData && !pPage->leaf ){ + pCur->validNKey = 0; + + pCur->aiIdx[pCur->iPage]--; + pPage = pCur->apPage[pCur->iPage]; + if( pPage->intKey && !pPage->leaf ){ rc = sqlite3BtreePrevious(pCur, pRes); }else{ rc = SQLITE_OK; @@ -3461,6 +4116,7 @@ static int allocateBtreePage( MemPage *pTrunk = 0; MemPage *pPrevTrunk = 0; + assert( sqlite3_mutex_held(pBt->mutex) ); pPage1 = pBt->pPage1; n = get4byte(&pPage1->aData[36]); if( n>0 ){ @@ -3473,7 +4129,7 @@ static int allocateBtreePage( ** the entire-list will be searched for that page. */ #ifndef SQLITE_OMIT_AUTOVACUUM - if( exact && nearby<=sqlite3PagerPagecount(pBt->pPager) ){ + if( exact && nearby<=pagerPagecount(pBt) ){ u8 eType; assert( nearby>0 ); assert( pBt->autoVacuum ); @@ -3525,7 +4181,7 @@ static int allocateBtreePage( *ppPage = pTrunk; pTrunk = 0; TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); - }else if( k>pBt->usableSize/4 - 8 ){ + }else if( k>pBt->usableSize/4 - 2 ){ /* Value of k is out of range. Database corruption */ rc = SQLITE_CORRUPT_BKPT; goto end_allocate_page; @@ -3568,6 +4224,7 @@ static int allocateBtreePage( memcpy(&pNewTrunk->aData[8], &pTrunk->aData[12], (k-1)*4); releasePage(pNewTrunk); if( !pPrevTrunk ){ + assert( sqlite3PagerIswriteable(pPage1->pDbPage) ); put4byte(&pPage1->aData[32], iNewTrunk); }else{ rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); @@ -3608,10 +4265,13 @@ static int allocateBtreePage( iPage = get4byte(&aData[8+closest*4]); if( !searchList || iPage==nearby ){ + Pgno nPage; *pPgno = iPage; - if( *pPgno>sqlite3PagerPagecount(pBt->pPager) ){ + nPage = pagerPagecount(pBt); + if( *pPgno>nPage ){ /* Free page off the end of the file */ - return SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_BKPT; + goto end_allocate_page; } TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d" ": %d more free pages\n", @@ -3620,6 +4280,7 @@ static int allocateBtreePage( memcpy(&aData[8+closest*4], &aData[4+k*4], 4); } put4byte(&aData[4], k-1); + assert( sqlite3PagerIswriteable(pTrunk->pDbPage) ); rc = sqlite3BtreeGetPage(pBt, *pPgno, ppPage, 1); if( rc==SQLITE_OK ){ sqlite3PagerDontRollback((*ppPage)->pDbPage); @@ -3637,19 +4298,10 @@ static int allocateBtreePage( }else{ /* There are no pages on the freelist, so create a new page at the ** end of the file */ - *pPgno = sqlite3PagerPagecount(pBt->pPager) + 1; + int nPage = pagerPagecount(pBt); + *pPgno = nPage + 1; #ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->nTrunc ){ - /* An incr-vacuum has already run within this transaction. So the - ** page to allocate is not from the physical end of the file, but - ** at pBt->nTrunc. - */ - *pPgno = pBt->nTrunc+1; - if( *pPgno==PENDING_BYTE_PAGE(pBt) ){ - (*pPgno)++; - } - } if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, *pPgno) ){ /* If *pPgno refers to a pointer-map page, allocate two new pages ** at the end of the file instead of one. The first allocated page @@ -3658,9 +4310,7 @@ static int allocateBtreePage( TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", *pPgno)); assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); (*pPgno)++; - } - if( pBt->nTrunc ){ - pBt->nTrunc = *pPgno; + if( *pPgno==PENDING_BYTE_PAGE(pBt) ){ (*pPgno)++; } } #endif @@ -3679,6 +4329,13 @@ static int allocateBtreePage( end_allocate_page: releasePage(pTrunk); releasePage(pPrevTrunk); + if( rc==SQLITE_OK ){ + if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ + releasePage(*ppPage); + return SQLITE_CORRUPT_BKPT; + } + (*ppPage)->isInit = 0; + } return rc; } @@ -3693,10 +4350,9 @@ static int freePage(MemPage *pPage){ int rc, n, k; /* Prepare the page for freeing */ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->pgno>1 ); pPage->isInit = 0; - releasePage(pPage->pParent); - pPage->pParent = 0; /* Increment the free page count on pPage1 */ rc = sqlite3PagerWrite(pPage1->pDbPage); @@ -3713,15 +4369,13 @@ static int freePage(MemPage *pPage){ memset(pPage->aData, 0, pPage->pBt->pageSize); #endif -#ifndef SQLITE_OMIT_AUTOVACUUM /* If the database supports auto-vacuum, write an entry in the pointer-map ** to indicate that the page is free. */ - if( pBt->autoVacuum ){ + if( ISAUTOVACUUM ){ rc = ptrmapPut(pBt, pPage->pgno, PTRMAP_FREEPAGE, 0); if( rc ) return rc; } -#endif if( n==0 ){ /* This is the first free page */ @@ -3739,14 +4393,29 @@ static int freePage(MemPage *pPage){ k = get4byte(&pTrunk->aData[4]); if( k>=pBt->usableSize/4 - 8 ){ /* The trunk is full. Turn the page being freed into a new - ** trunk page with no leaves. */ + ** trunk page with no leaves. + ** + ** Note that the trunk page is not really full until it contains + ** usableSize/4 - 2 entries, not usableSize/4 - 8 entries as we have + ** coded. But due to a coding error in versions of SQLite prior to + ** 3.6.0, databases with freelist trunk pages holding more than + ** usableSize/4 - 8 entries will be reported as corrupt. In order + ** to maintain backwards compatibility with older versions of SQLite, + ** we will contain to restrict the number of entries to usableSize/4 - 8 + ** for now. At some point in the future (once everyone has upgraded + ** to 3.6.0 or later) we should consider fixing the conditional above + ** to read "usableSize/4-2" instead of "usableSize/4-8". + */ rc = sqlite3PagerWrite(pPage->pDbPage); - if( rc ) return rc; - put4byte(pPage->aData, pTrunk->pgno); - put4byte(&pPage->aData[4], 0); - put4byte(&pPage1->aData[32], pPage->pgno); - TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", - pPage->pgno, pTrunk->pgno)); + if( rc==SQLITE_OK ){ + put4byte(pPage->aData, pTrunk->pgno); + put4byte(&pPage->aData[4], 0); + put4byte(&pPage1->aData[32], pPage->pgno); + TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", + pPage->pgno, pTrunk->pgno)); + } + }else if( k<0 ){ + rc = SQLITE_CORRUPT; }else{ /* Add the newly freed page as a leaf on the current trunk */ rc = sqlite3PagerWrite(pTrunk->pDbPage); @@ -3754,7 +4423,7 @@ static int freePage(MemPage *pPage){ put4byte(&pTrunk->aData[4], k+1); put4byte(&pTrunk->aData[8+k*4], pPage->pgno); #ifndef SQLITE_SECURE_DELETE - sqlite3PagerDontWrite(pPage->pDbPage); + rc = sqlite3PagerDontWrite(pPage->pDbPage); #endif } TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); @@ -3775,6 +4444,7 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ int nOvfl; int ovflPageSize; + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); sqlite3BtreeParseCellPtr(pPage, pCell, &info); if( info.iOverflow==0 ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ @@ -3785,7 +4455,7 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ assert( ovflPgno==0 || nOvfl>0 ); while( nOvfl-- ){ MemPage *pOvfl; - if( ovflPgno==0 || ovflPgno>sqlite3PagerPagecount(pBt->pPager) ){ + if( ovflPgno==0 || ovflPgno>pagerPagecount(pBt) ){ return SQLITE_CORRUPT_BKPT; } @@ -3831,6 +4501,13 @@ static int fillInCell( int nHeader; CellInfo info; + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + + /* pPage is not necessarily writeable since pCell might be auxiliary + ** buffer space that is separate from the pPage buffer area */ + assert( pCellaData || pCell>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + /* Fill in the header. */ nHeader = 0; if( !pPage->leaf ){ @@ -3845,7 +4522,7 @@ static int fillInCell( sqlite3BtreeParseCellPtr(pPage, pCell, &info); assert( info.nHeader==nHeader ); assert( info.nKey==nKey ); - assert( info.nData==nData+nZero ); + assert( info.nData==(u32)(nData+nZero) ); /* Fill in the payload */ nPayload = nData + nZero; @@ -3853,10 +4530,11 @@ static int fillInCell( pSrc = pData; nSrc = nData; nData = 0; - }else{ - nPayload += nKey; + }else{ + /* TBD: Perhaps raise SQLITE_CORRUPT if nKey is larger than 31 bits? */ + nPayload += (int)nKey; pSrc = pKey; - nSrc = nKey; + nSrc = (int)nKey; } *pnSize = info.nSize; spaceLeft = info.nLocal; @@ -3865,7 +4543,6 @@ static int fillInCell( while( nPayload>0 ){ if( spaceLeft==0 ){ - int isExact = 0; #ifndef SQLITE_OMIT_AUTOVACUUM Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */ if( pBt->autoVacuum ){ @@ -3874,12 +4551,9 @@ static int fillInCell( } while( PTRMAP_ISPAGE(pBt, pgnoOvfl) || pgnoOvfl==PENDING_BYTE_PAGE(pBt) ); - if( pgnoOvfl>1 ){ - /* isExact = 1; */ - } } #endif - rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, isExact); + rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0); #ifndef SQLITE_OMIT_AUTOVACUUM /* If the database supports auto-vacuum, and the second or subsequent ** overflow page is being allocated, add an entry to the pointer-map @@ -3903,6 +4577,16 @@ static int fillInCell( releasePage(pToRelease); return rc; } + + /* If pToRelease is not zero than pPrior points into the data area + ** of pToRelease. Make sure pToRelease is still writeable. */ + assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); + + /* If pPrior is part of the data area of pPage, then make sure pPage + ** is still writeable */ + assert( pPrioraData || pPrior>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + put4byte(pPrior, pgnoOvfl); releasePage(pToRelease); pToRelease = pOvfl; @@ -3913,6 +4597,16 @@ static int fillInCell( } n = nPayload; if( n>spaceLeft ) n = spaceLeft; + + /* If pToRelease is not zero than pPayload points into the data area + ** of pToRelease. Make sure pToRelease is still writeable. */ + assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); + + /* If pPayload is part of the data area of pPage, then make sure pPage + ** is still writeable */ + assert( pPayloadaData || pPayload>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + if( nSrc>0 ){ if( n>nSrc ) n = nSrc; assert( pSrc ); @@ -3934,75 +4628,6 @@ static int fillInCell( return SQLITE_OK; } -/* -** Change the MemPage.pParent pointer on the page whose number is -** given in the second argument so that MemPage.pParent holds the -** pointer in the third argument. -*/ -static int reparentPage(BtShared *pBt, Pgno pgno, MemPage *pNewParent, int idx){ - MemPage *pThis; - DbPage *pDbPage; - - assert( pNewParent!=0 ); - if( pgno==0 ) return SQLITE_OK; - assert( pBt->pPager!=0 ); - pDbPage = sqlite3PagerLookup(pBt->pPager, pgno); - if( pDbPage ){ - pThis = (MemPage *)sqlite3PagerGetExtra(pDbPage); - if( pThis->isInit ){ - assert( pThis->aData==(sqlite3PagerGetData(pDbPage)) ); - if( pThis->pParent!=pNewParent ){ - if( pThis->pParent ) sqlite3PagerUnref(pThis->pParent->pDbPage); - pThis->pParent = pNewParent; - sqlite3PagerRef(pNewParent->pDbPage); - } - pThis->idxParent = idx; - } - sqlite3PagerUnref(pDbPage); - } - -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - return ptrmapPut(pBt, pgno, PTRMAP_BTREE, pNewParent->pgno); - } -#endif - return SQLITE_OK; -} - - - -/* -** Change the pParent pointer of all children of pPage to point back -** to pPage. -** -** In other words, for every child of pPage, invoke reparentPage() -** to make sure that each child knows that pPage is its parent. -** -** This routine gets called after you memcpy() one page into -** another. -*/ -static int reparentChildPages(MemPage *pPage){ - int i; - BtShared *pBt = pPage->pBt; - int rc = SQLITE_OK; - - if( pPage->leaf ) return SQLITE_OK; - - for(i=0; inCell; i++){ - u8 *pCell = findCell(pPage, i); - if( !pPage->leaf ){ - rc = reparentPage(pBt, get4byte(pCell), pPage, i); - if( rc!=SQLITE_OK ) return rc; - } - } - if( !pPage->leaf ){ - rc = reparentPage(pBt, get4byte(&pPage->aData[pPage->hdrOffset+8]), - pPage, i); - pPage->idxShift = 0; - } - return rc; -} - /* ** Remove the i-th cell from pPage. This routine effects pPage only. ** The cell content is not freed or deallocated. It is assumed that @@ -4011,20 +4636,28 @@ static int reparentChildPages(MemPage *pPage){ ** ** "sz" must be the number of bytes in the cell. */ -static void dropCell(MemPage *pPage, int idx, int sz){ +static int dropCell(MemPage *pPage, int idx, int sz){ int i; /* Loop counter */ int pc; /* Offset to cell content of cell being deleted */ u8 *data; /* pPage->aData */ u8 *ptr; /* Used to move bytes around within data[] */ + int rc; /* The return code */ assert( idx>=0 && idxnCell ); assert( sz==cellSize(pPage, idx) ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); data = pPage->aData; ptr = &data[pPage->cellOffset + 2*idx]; pc = get2byte(ptr); - assert( pc>10 && pc+sz<=pPage->pBt->usableSize ); - freeSpace(pPage, pc, sz); + if( (pchdrOffset+6+(pPage->leaf?0:4)) + || (pc+sz>pPage->pBt->usableSize) ){ + return SQLITE_CORRUPT_BKPT; + } + rc = freeSpace(pPage, pc, sz); + if( rc!=SQLITE_OK ){ + return rc; + } for(i=idx+1; inCell; i++, ptr+=2){ ptr[0] = ptr[2]; ptr[1] = ptr[3]; @@ -4032,7 +4665,7 @@ static void dropCell(MemPage *pPage, int idx, int sz){ pPage->nCell--; put2byte(&data[pPage->hdrOffset+3], pPage->nCell); pPage->nFree += 2; - pPage->idxShift = 1; + return SQLITE_OK; } /* @@ -4071,19 +4704,26 @@ static int insertCell( u8 *ptr; /* Used for moving information around in data[] */ assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=5460 ); + assert( pPage->nOverflow<=ArraySize(pPage->aOvfl) ); assert( sz==cellSizePtr(pPage, pCell) ); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp+nSkip, pCell+nSkip, sz-nSkip); pCell = pTemp; } j = pPage->nOverflow++; - assert( jaOvfl)/sizeof(pPage->aOvfl[0]) ); + assert( j<(int)(sizeof(pPage->aOvfl)/sizeof(pPage->aOvfl[0])) ); pPage->aOvfl[j].pCell = pCell; - pPage->aOvfl[j].idx = i; + pPage->aOvfl[j].idx = (u16)i; pPage->nFree = 0; }else{ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); data = pPage->aData; hdr = pPage->hdrOffset; top = get2byte(&data[hdr+5]); @@ -4091,14 +4731,19 @@ static int insertCell( end = cellOffset + 2*pPage->nCell + 2; ins = cellOffset + 2*i; if( end > top - sz ){ - int rc = defragmentPage(pPage); - if( rc!=SQLITE_OK ) return rc; + rc = defragmentPage(pPage); + if( rc!=SQLITE_OK ){ + return rc; + } top = get2byte(&data[hdr+5]); assert( end + sz <= top ); } idx = allocateSpace(pPage, sz); assert( idx>0 ); assert( end <= get2byte(&data[hdr+5]) ); + if (idx+sz > pPage->pBt->usableSize) { + return SQLITE_CORRUPT_BKPT; + } pPage->nCell++; pPage->nFree -= 2; memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip); @@ -4108,7 +4753,6 @@ static int insertCell( } put2byte(&data[ins], idx); put2byte(&data[hdr+3], pPage->nCell); - pPage->idxShift = 1; #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ /* The cell may contain a pointer to an overflow page. If so, write @@ -4119,7 +4763,7 @@ static int insertCell( assert( (info.nData+(pPage->intKey?0:info.nKey))==info.nPayload ); if( (info.nData+(pPage->intKey?0:info.nKey))>info.nLocal ){ Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); - int rc = ptrmapPut(pPage->pBt, pgnoOvfl, PTRMAP_OVERFLOW1, pPage->pgno); + rc = ptrmapPut(pPage->pBt, pgnoOvfl, PTRMAP_OVERFLOW1, pPage->pgno); if( rc!=SQLITE_OK ) return rc; } } @@ -4137,7 +4781,7 @@ static void assemblePage( MemPage *pPage, /* The page to be assemblied */ int nCell, /* The number of cells to add to this page */ u8 **apCell, /* Pointers to cell bodies */ - int *aSize /* Sizes of the cells */ + u16 *aSize /* Sizes of the cells */ ){ int i; /* Loop counter */ int totalSize; /* Total size of all cells */ @@ -4147,12 +4791,15 @@ static void assemblePage( u8 *data; /* Data for the page */ assert( pPage->nOverflow==0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( nCell>=0 && nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=5460 ); totalSize = 0; for(i=0; inFree ); assert( pPage->nCell==0 ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); cellptr = pPage->cellOffset; data = pPage->aData; hdr = pPage->hdrOffset; @@ -4170,7 +4817,7 @@ static void assemblePage( } assert( cellbody==pPage->pBt->usableSize ); } - pPage->nCell = nCell; + pPage->nCell = (u16)nCell; } /* @@ -4189,7 +4836,7 @@ static void assemblePage( #define NB (NN*2+1) /* Total pages involved in the balance */ /* Forward reference */ -static int balance(MemPage*, int); +static int balance(BtCursor*, int); #ifndef SQLITE_OMIT_QUICKBALANCE /* @@ -4209,76 +4856,99 @@ static int balance(MemPage*, int); ** pParent is its parent. pPage must have a single overflow entry ** which is also the right-most entry on the page. */ -static int balance_quick(MemPage *pPage, MemPage *pParent){ +static int balance_quick(BtCursor *pCur){ int rc; - MemPage *pNew; + MemPage *pNew = 0; Pgno pgnoNew; u8 *pCell; - int szCell; + u16 szCell; CellInfo info; + MemPage *pPage = pCur->apPage[pCur->iPage]; + MemPage *pParent = pCur->apPage[pCur->iPage-1]; BtShared *pBt = pPage->pBt; int parentIdx = pParent->nCell; /* pParent new divider cell index */ int parentSize; /* Size of new divider cell */ u8 parentCell[64]; /* Space for the new divider cell */ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + /* Allocate a new page. Insert the overflow cell from pPage ** into it. Then remove the overflow cell from pPage. */ rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - pCell = pPage->aOvfl[0].pCell; - szCell = cellSizePtr(pPage, pCell); - zeroPage(pNew, pPage->aData[0]); - assemblePage(pNew, 1, &pCell, &szCell); - pPage->nOverflow = 0; - - /* Set the parent of the newly allocated page to pParent. */ - pNew->pParent = pParent; - sqlite3PagerRef(pParent->pDbPage); - - /* pPage is currently the right-child of pParent. Change this - ** so that the right-child is the new page allocated above and - ** pPage is the next-to-right child. - */ - assert( pPage->nCell>0 ); - pCell = findCell(pPage, pPage->nCell-1); - sqlite3BtreeParseCellPtr(pPage, pCell, &info); - rc = fillInCell(pParent, parentCell, 0, info.nKey, 0, 0, 0, &parentSize); - if( rc!=SQLITE_OK ){ - return rc; - } - assert( parentSize<64 ); - rc = insertCell(pParent, parentIdx, parentCell, parentSize, 0, 4); - if( rc!=SQLITE_OK ){ - return rc; - } - put4byte(findOverflowCell(pParent,parentIdx), pPage->pgno); - put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); - -#ifndef SQLITE_OMIT_AUTOVACUUM - /* If this is an auto-vacuum database, update the pointer map - ** with entries for the new page, and any pointer from the - ** cell on the page to an overflow page. - */ - if( pBt->autoVacuum ){ - rc = ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno); - if( rc==SQLITE_OK ){ - rc = ptrmapPutOvfl(pNew, 0); + if( rc==SQLITE_OK ){ + pCell = pPage->aOvfl[0].pCell; + szCell = cellSizePtr(pPage, pCell); + assert( sqlite3PagerIswriteable(pNew->pDbPage) ); + zeroPage(pNew, pPage->aData[0]); + assemblePage(pNew, 1, &pCell, &szCell); + pPage->nOverflow = 0; + + /* pPage is currently the right-child of pParent. Change this + ** so that the right-child is the new page allocated above and + ** pPage is the next-to-right child. + ** + ** Ignore the return value of the call to fillInCell(). fillInCell() + ** may only return other than SQLITE_OK if it is required to allocate + ** one or more overflow pages. Since an internal table B-Tree cell + ** may never spill over onto an overflow page (it is a maximum of + ** 13 bytes in size), it is not neccessary to check the return code. + ** + ** Similarly, the insertCell() function cannot fail if the page + ** being inserted into is already writable and the cell does not + ** contain an overflow pointer. So ignore this return code too. + */ + assert( pPage->nCell>0 ); + pCell = findCell(pPage, pPage->nCell-1); + sqlite3BtreeParseCellPtr(pPage, pCell, &info); + fillInCell(pParent, parentCell, 0, info.nKey, 0, 0, 0, &parentSize); + assert( parentSize<64 ); + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + insertCell(pParent, parentIdx, parentCell, parentSize, 0, 4); + put4byte(findOverflowCell(pParent,parentIdx), pPage->pgno); + put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); + + /* If this is an auto-vacuum database, update the pointer map + ** with entries for the new page, and any pointer from the + ** cell on the page to an overflow page. + */ + if( ISAUTOVACUUM ){ + rc = ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno); + if( rc==SQLITE_OK ){ + rc = ptrmapPutOvfl(pNew, 0); + } } - if( rc!=SQLITE_OK ){ - releasePage(pNew); - return rc; - } - } -#endif - /* Release the reference to the new page and balance the parent page, - ** in case the divider cell inserted caused it to become overfull. + /* Release the reference to the new page. */ + releasePage(pNew); + } + + /* At this point the pPage->nFree variable is not set correctly with + ** respect to the content of the page (because it was set to 0 by + ** insertCell). So call sqlite3BtreeInitPage() to make sure it is + ** correct. + ** + ** This has to be done even if an error will be returned. Normally, if + ** an error occurs during tree balancing, the contents of MemPage are + ** not important, as they will be recalculated when the page is rolled + ** back. But here, in balance_quick(), it is possible that pPage has + ** not yet been marked dirty or written into the journal file. Therefore + ** it will not be rolled back and so it is important to make sure that + ** the page data and contents of MemPage are consistent. */ - releasePage(pNew); - return balance(pParent, 0); + pPage->isInit = 0; + sqlite3BtreeInitPage(pPage); + assert( pPage->nOverflow==0 ); + + /* If everything else succeeded, balance the parent page, in + ** case the divider cell inserted caused it to become overfull. + */ + if( rc==SQLITE_OK ){ + releasePage(pPage); + pCur->iPage--; + rc = balance(pCur, 0); + } + return rc; } #endif /* SQLITE_OMIT_QUICKBALANCE */ @@ -4311,13 +4981,14 @@ static int balance_quick(MemPage *pPage, MemPage *pParent){ ** in a corrupted state. So if this routine fails, the database should ** be rolled back. */ -static int balance_nonroot(MemPage *pPage){ +static int balance_nonroot(BtCursor *pCur){ + MemPage *pPage; /* The over or underfull page to balance */ MemPage *pParent; /* The parent of pPage */ BtShared *pBt; /* The whole database */ int nCell = 0; /* Number of cells in apCell[] */ int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ - int nOld; /* Number of pages in apOld[] */ - int nNew; /* Number of pages in apNew[] */ + int nOld = 0; /* Number of pages in apOld[] */ + int nNew = 0; /* Number of pages in apNew[] */ int nDiv; /* Number of cells in apDiv[] */ int i, j, k; /* Loop counters */ int idx; /* Index of pPage in pParent->aCell[] */ @@ -4328,7 +4999,9 @@ static int balance_nonroot(MemPage *pPage){ int usableSpace; /* Bytes in pPage beyond the header */ int pageFlags; /* Value of pPage->aData[0] */ int subtotal; /* Subtotal of bytes in cells on one page */ - int iSpace = 0; /* First unused byte of aSpace[] */ + int iSpace1 = 0; /* First unused byte of aSpace1[] */ + int iSpace2 = 0; /* First unused byte of aSpace2[] */ + int szScratch; /* Size of scratch memory requested */ MemPage *apOld[NB]; /* pPage and up to two siblings */ Pgno pgnoOld[NB]; /* Page numbers for each page in apOld[] */ MemPage *apCopy[NB]; /* Private copies of apOld[] pages */ @@ -4338,24 +5011,29 @@ static int balance_nonroot(MemPage *pPage){ int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ int szNew[NB+2]; /* Combined size of cells place on i-th page */ u8 **apCell = 0; /* All cells begin balanced */ - int *szCell; /* Local size of all cells in apCell[] */ - u8 *aCopy[NB]; /* Space for holding data of apCopy[] */ - u8 *aSpace; /* Space to hold copies of dividers cells */ -#ifndef SQLITE_OMIT_AUTOVACUUM + u16 *szCell; /* Local size of all cells in apCell[] */ + u8 *aCopy[NB]; /* Space for holding data of apCopy[] */ + u8 *aSpace1; /* Space for copies of dividers cells before balance */ + u8 *aSpace2 = 0; /* Space for overflow dividers cells after balance */ u8 *aFrom = 0; -#endif + + pPage = pCur->apPage[pCur->iPage]; + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + VVA_ONLY( pCur->pagesShuffled = 1 ); /* ** Find the parent page. */ + assert( pCur->iPage>0 ); assert( pPage->isInit ); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) || pPage->nOverflow==1 ); pBt = pPage->pBt; - pParent = pPage->pParent; + pParent = pCur->apPage[pCur->iPage-1]; assert( pParent ); if( SQLITE_OK!=(rc = sqlite3PagerWrite(pParent->pDbPage)) ){ - return rc; + goto balance_cleanup; } + TRACE(("BALANCE: begin page %d child of %d\n", pPage->pgno, pParent->pgno)); #ifndef SQLITE_OMIT_QUICKBALANCE @@ -4369,46 +5047,31 @@ static int balance_nonroot(MemPage *pPage){ */ if( pPage->leaf && pPage->intKey && - pPage->leafData && pPage->nOverflow==1 && pPage->aOvfl[0].idx==pPage->nCell && - pPage->pParent->pgno!=1 && + pParent->pgno!=1 && get4byte(&pParent->aData[pParent->hdrOffset+8])==pPage->pgno ){ + assert( pPage->intKey ); /* ** TODO: Check the siblings to the left of pPage. It may be that ** they are not full and no new page is required. */ - return balance_quick(pPage, pParent); + return balance_quick(pCur); } #endif + if( SQLITE_OK!=(rc = sqlite3PagerWrite(pPage->pDbPage)) ){ + goto balance_cleanup; + } + /* ** Find the cell in the parent page whose left child points back ** to pPage. The "idx" variable is the index of that cell. If pPage ** is the rightmost child of pParent then set idx to pParent->nCell */ - if( pParent->idxShift ){ - Pgno pgno; - pgno = pPage->pgno; - assert( pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); - for(idx=0; idxnCell; idx++){ - if( get4byte(findCell(pParent, idx))==pgno ){ - break; - } - } - assert( idxnCell - || get4byte(&pParent->aData[pParent->hdrOffset+8])==pgno ); - }else{ - idx = pPage->idxParent; - } - - /* - ** Initialize variables so that it will be safe to jump - ** directly to balance_cleanup at any moment. - */ - nOld = nNew = 0; - sqlite3PagerRef(pParent->pDbPage); + idx = pCur->aiIdx[pCur->iPage-1]; + assertParentIndex(pParent, idx, pPage->pgno); /* ** Find sibling pages to pPage and the cells in pParent that divide @@ -4436,47 +5099,50 @@ static int balance_nonroot(MemPage *pPage){ }else{ break; } - rc = getAndInitPage(pBt, pgnoOld[i], &apOld[i], pParent); + rc = getAndInitPage(pBt, pgnoOld[i], &apOld[i]); if( rc ) goto balance_cleanup; - apOld[i]->idxParent = k; + /* apOld[i]->idxParent = k; */ apCopy[i] = 0; assert( i==nOld ); nOld++; nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow; } - /* Make nMaxCells a multiple of 2 in order to preserve 8-byte + /* Make nMaxCells a multiple of 4 in order to preserve 8-byte ** alignment */ - nMaxCells = (nMaxCells + 1)&~1; + nMaxCells = (nMaxCells + 3)&~3; /* ** Allocate space for memory structures */ - apCell = sqliteMallocRaw( - nMaxCells*sizeof(u8*) /* apCell */ - + nMaxCells*sizeof(int) /* szCell */ - + ROUND8(sizeof(MemPage))*NB /* aCopy */ - + pBt->pageSize*(5+NB) /* aSpace */ - + (ISAUTOVACUUM ? nMaxCells : 0) /* aFrom */ - ); + szScratch = + nMaxCells*sizeof(u8*) /* apCell */ + + nMaxCells*sizeof(u16) /* szCell */ + + (ROUND8(sizeof(MemPage))+pBt->pageSize)*NB /* aCopy */ + + pBt->pageSize /* aSpace1 */ + + (ISAUTOVACUUM ? nMaxCells : 0); /* aFrom */ + apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; goto balance_cleanup; } - szCell = (int*)&apCell[nMaxCells]; + szCell = (u16*)&apCell[nMaxCells]; aCopy[0] = (u8*)&szCell[nMaxCells]; - assert( ((aCopy[0] - (u8*)apCell) & 7)==0 ); /* 8-byte alignment required */ + assert( ((aCopy[0] - (u8*)0) & 7)==0 ); /* 8-byte alignment required */ for(i=1; ipageSize+ROUND8(sizeof(MemPage))]; - assert( ((aCopy[i] - (u8*)apCell) & 7)==0 ); /* 8-byte alignment required */ + assert( ((aCopy[i] - (u8*)0) & 7)==0 ); /* 8-byte alignment required */ } - aSpace = &aCopy[NB-1][pBt->pageSize+ROUND8(sizeof(MemPage))]; - assert( ((aSpace - (u8*)apCell) & 7)==0 ); /* 8-byte alignment required */ -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - aFrom = &aSpace[5*pBt->pageSize]; + aSpace1 = &aCopy[NB-1][pBt->pageSize+ROUND8(sizeof(MemPage))]; + assert( ((aSpace1 - (u8*)0) & 7)==0 ); /* 8-byte alignment required */ + if( ISAUTOVACUUM ){ + aFrom = &aSpace1[pBt->pageSize]; + } + aSpace2 = sqlite3PageMalloc(pBt->pageSize); + if( aSpace2==0 ){ + rc = SQLITE_NOMEM; + goto balance_cleanup; } -#endif /* ** Make copies of the content of pPage and its siblings into aOld[]. @@ -4485,23 +5151,21 @@ static int balance_nonroot(MemPage *pPage){ ** process of being overwritten. */ for(i=0; ipageSize]; - p->aData = &((u8*)p)[-pBt->pageSize]; - memcpy(p->aData, apOld[i]->aData, pBt->pageSize + sizeof(MemPage)); - /* The memcpy() above changes the value of p->aData so we have to - ** set it again. */ - p->aData = &((u8*)p)[-pBt->pageSize]; + MemPage *p = apCopy[i] = (MemPage*)aCopy[i]; + memcpy(p, apOld[i], sizeof(MemPage)); + p->aData = (void*)&p[1]; + memcpy(p->aData, apOld[i]->aData, pBt->pageSize); } /* ** Load pointers to all cells on sibling pages and the divider cells ** into the local apCell[] array. Make copies of the divider cells - ** into space obtained form aSpace[] and remove the the divider Cells + ** into space obtained form aSpace1[] and remove the the divider Cells ** from pParent. ** ** If the siblings are on leaf pages, then the child pointers of the ** divider cells are stripped from the cells before they are copied - ** into aSpace[]. In this way, all cells in apCell[] are without + ** into aSpace1[]. In this way, all cells in apCell[] are without ** child pointers. If siblings are not leaves, then all cell in ** apCell[] include child pointers. Either way, all cells in apCell[] ** are alike. @@ -4511,7 +5175,7 @@ static int balance_nonroot(MemPage *pPage){ */ nCell = 0; leafCorrection = pPage->leaf*4; - leafData = pPage->leafData && pPage->leaf; + leafData = pPage->hasData; for(i=0; inCell+pOld->nOverflow; @@ -4519,10 +5183,9 @@ static int balance_nonroot(MemPage *pPage){ assert( nCellautoVacuum ){ + if( ISAUTOVACUUM ){ int a; - aFrom[nCell] = i; + aFrom[nCell] = (u8)i; assert( i>=0 && i<6 ); for(a=0; anOverflow; a++){ if( pOld->aOvfl[a].pCell==apCell[nCell] ){ aFrom[nCell] = 0xFF; @@ -4530,11 +5193,10 @@ static int balance_nonroot(MemPage *pPage){ } } } -#endif nCell++; } if( ipageSize*5 ); + pTemp = &aSpace1[iSpace1]; + iSpace1 += sz; + assert( sz<=pBt->pageSize/4 ); + assert( iSpace1<=pBt->pageSize ); memcpy(pTemp, apDiv[i], sz); apCell[nCell] = pTemp+leafCorrection; -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ + if( ISAUTOVACUUM ){ aFrom[nCell] = 0xFF; } -#endif dropCell(pParent, nxDiv, sz); - szCell[nCell] -= leafCorrection; + assert( leafCorrection==0 || leafCorrection==4 ); + szCell[nCell] -= (u16)leafCorrection; assert( get4byte(pTemp)==pgnoOld[i] ); if( !pOld->leaf ){ assert( leafCorrection==0 ); @@ -4666,7 +5328,6 @@ static int balance_nonroot(MemPage *pPage){ apNew[i] = pNew; nNew++; } - zeroPage(pNew, pageFlags); } /* Free any old pages that were not reused as new pages. @@ -4733,28 +5394,30 @@ static int balance_nonroot(MemPage *pPage){ MemPage *pNew = apNew[i]; assert( jpgno==pgnoNew[i] ); + zeroPage(pNew, pageFlags); assemblePage(pNew, cntNew[i]-j, &apCell[j], &szCell[j]); assert( pNew->nCell>0 || (nNew==1 && cntNew[0]==0) ); assert( pNew->nOverflow==0 ); -#ifndef SQLITE_OMIT_AUTOVACUUM /* If this is an auto-vacuum database, update the pointer map entries ** that point to the siblings that were rearranged. These can be: left ** children of cells, the right-child of the page, or overflow pages ** pointed to by cells. */ - if( pBt->autoVacuum ){ + if( ISAUTOVACUUM ){ for(k=j; kpgno!=pNew->pgno ){ rc = ptrmapPutOvfl(pNew, k-j); + if( rc==SQLITE_OK && leafCorrection==0 ){ + rc = ptrmapPut(pBt, get4byte(apCell[k]), PTRMAP_BTREE, pNew->pgno); + } if( rc!=SQLITE_OK ){ goto balance_cleanup; } } } } -#endif j = cntNew[i]; @@ -4769,9 +5432,17 @@ static int balance_nonroot(MemPage *pPage){ assert( jleaf ){ memcpy(&pNew->aData[8], pCell, 4); - pTemp = 0; + if( ISAUTOVACUUM + && (aFrom[j]==0xFF || apCopy[aFrom[j]]->pgno!=pNew->pgno) + ){ + rc = ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno); + if( rc!=SQLITE_OK ){ + goto balance_cleanup; + } + } }else if( leafData ){ /* If the tree is a leaf-data tree, and the siblings are leaves, ** then there is no divider cell in apCell[]. Instead, the divider @@ -4781,21 +5452,16 @@ static int balance_nonroot(MemPage *pPage){ CellInfo info; j--; sqlite3BtreeParseCellPtr(pNew, apCell[j], &info); - pCell = &aSpace[iSpace]; + pCell = pTemp; fillInCell(pParent, pCell, 0, info.nKey, 0, 0, 0, &sz); - iSpace += sz; - assert( iSpace<=pBt->pageSize*5 ); pTemp = 0; }else{ pCell -= 4; - pTemp = &aSpace[iSpace]; - iSpace += sz; - assert( iSpace<=pBt->pageSize*5 ); /* Obscure case for non-leaf-data trees: If the cell at pCell was - ** previously stored on a leaf node, and it's reported size was 4 + ** previously stored on a leaf node, and its reported size was 4 ** bytes, then it may actually be smaller than this ** (see sqlite3BtreeParseCellPtr(), 4 bytes is the minimum size of - ** any cell). But it's important to pass the correct size to + ** any cell). But it is important to pass the correct size to ** insertCell(), so reparse the cell now. ** ** Note that this can never happen in an SQLite data file, as all @@ -4807,31 +5473,50 @@ static int balance_nonroot(MemPage *pPage){ sz = cellSizePtr(pParent, pCell); } } + iSpace2 += sz; + assert( sz<=pBt->pageSize/4 ); + assert( iSpace2<=pBt->pageSize ); rc = insertCell(pParent, nxDiv, pCell, sz, pTemp, 4); if( rc!=SQLITE_OK ) goto balance_cleanup; + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); put4byte(findOverflowCell(pParent,nxDiv), pNew->pgno); -#ifndef SQLITE_OMIT_AUTOVACUUM + /* If this is an auto-vacuum database, and not a leaf-data tree, ** then update the pointer map with an entry for the overflow page ** that the cell just inserted points to (if any). */ - if( pBt->autoVacuum && !leafData ){ + if( ISAUTOVACUUM && !leafData ){ rc = ptrmapPutOvfl(pParent, nxDiv); if( rc!=SQLITE_OK ){ goto balance_cleanup; } } -#endif j++; nxDiv++; } + + /* Set the pointer-map entry for the new sibling page. */ + if( ISAUTOVACUUM ){ + rc = ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno); + if( rc!=SQLITE_OK ){ + goto balance_cleanup; + } + } } assert( j==nCell ); assert( nOld>0 ); assert( nNew>0 ); if( (pageFlags & PTF_LEAF)==0 ){ - memcpy(&apNew[nNew-1]->aData[8], &apCopy[nOld-1]->aData[8], 4); + u8 *zChild = &apCopy[nOld-1]->aData[8]; + memcpy(&apNew[nNew-1]->aData[8], zChild, 4); + if( ISAUTOVACUUM ){ + rc = ptrmapPut(pBt, get4byte(zChild), PTRMAP_BTREE, apNew[nNew-1]->pgno); + if( rc!=SQLITE_OK ){ + goto balance_cleanup; + } + } } + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); if( nxDiv==pParent->nCell+pParent->nOverflow ){ /* Right-most sibling is the right-most child of pParent */ put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew[nNew-1]); @@ -4841,38 +5526,35 @@ static int balance_nonroot(MemPage *pPage){ put4byte(findOverflowCell(pParent, nxDiv), pgnoNew[nNew-1]); } - /* - ** Reparent children of all cells. - */ - for(i=0; iisInit ); - rc = balance(pParent, 0); + sqlite3ScratchFree(apCell); + apCell = 0; + TRACE(("BALANCE: finished with %d: old=%d new=%d cells=%d\n", + pPage->pgno, nOld, nNew, nCell)); + pPage->nOverflow = 0; + releasePage(pPage); + pCur->iPage--; + rc = balance(pCur, 0); /* ** Cleanup before returning. */ balance_cleanup: - sqliteFree(apCell); + sqlite3PageFree(aSpace2); + sqlite3ScratchFree(apCell); for(i=0; ipgno, nOld, nNew, nCell)); + pCur->apPage[pCur->iPage]->nOverflow = 0; + return rc; } @@ -4881,22 +5563,26 @@ balance_cleanup: ** page contains no cells. This is an opportunity to make the tree ** shallower by one level. */ -static int balance_shallower(MemPage *pPage){ +static int balance_shallower(BtCursor *pCur){ + MemPage *pPage; /* Root page of B-Tree */ MemPage *pChild; /* The only child page of pPage */ Pgno pgnoChild; /* Page number for pChild */ int rc = SQLITE_OK; /* Return code from subprocedures */ BtShared *pBt; /* The main BTree structure */ int mxCellPerPage; /* Maximum number of cells per page */ u8 **apCell; /* All cells from pages being balanced */ - int *szCell; /* Local size of all cells */ + u16 *szCell; /* Local size of all cells */ + + assert( pCur->iPage==0 ); + pPage = pCur->apPage[0]; - assert( pPage->pParent==0 ); assert( pPage->nCell==0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); pBt = pPage->pBt; mxCellPerPage = MX_CELL(pBt); - apCell = sqliteMallocRaw( mxCellPerPage*(sizeof(u8*)+sizeof(int)) ); + apCell = sqlite3Malloc( mxCellPerPage*(sizeof(u8*)+sizeof(u16)) ); if( apCell==0 ) return SQLITE_NOMEM; - szCell = (int*)&apCell[mxCellPerPage]; + szCell = (u16*)&apCell[mxCellPerPage]; if( pPage->leaf ){ /* The table is completely empty */ TRACE(("BALANCE: empty table %d\n", pPage->pgno)); @@ -4913,13 +5599,14 @@ static int balance_shallower(MemPage *pPage){ ** for the right-pointer to the child page. The child page becomes ** the virtual root of the tree. */ + VVA_ONLY( pCur->pagesShuffled = 1 ); pgnoChild = get4byte(&pPage->aData[pPage->hdrOffset+8]); assert( pgnoChild>0 ); - assert( pgnoChild<=sqlite3PagerPagecount(pPage->pBt->pPager) ); + assert( pgnoChild<=pagerPagecount(pPage->pBt) ); rc = sqlite3BtreeGetPage(pPage->pBt, pgnoChild, &pChild, 0); if( rc ) goto end_shallow_balance; if( pPage->pgno==1 ){ - rc = sqlite3BtreeInitPage(pChild, pPage); + rc = sqlite3BtreeInitPage(pChild); if( rc ) goto end_shallow_balance; assert( pChild->nOverflow==0 ); if( pChild->nFree>=100 ){ @@ -4933,9 +5620,10 @@ static int balance_shallower(MemPage *pPage){ } assemblePage(pPage, pChild->nCell, apCell, szCell); /* Copy the right-pointer of the child to the parent. */ + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); put4byte(&pPage->aData[pPage->hdrOffset+8], get4byte(&pChild->aData[pChild->hdrOffset+8])); - freePage(pChild); + rc = freePage(pChild); TRACE(("BALANCE: child %d transfer to page 1\n", pChild->pgno)); }else{ /* The child has more information that will fit on the root. @@ -4945,30 +5633,22 @@ static int balance_shallower(MemPage *pPage){ }else{ memcpy(pPage->aData, pChild->aData, pPage->pBt->usableSize); pPage->isInit = 0; - pPage->pParent = 0; - rc = sqlite3BtreeInitPage(pPage, 0); + rc = sqlite3BtreeInitPage(pPage); assert( rc==SQLITE_OK ); freePage(pChild); TRACE(("BALANCE: transfer child %d into root %d\n", pChild->pgno, pPage->pgno)); } - rc = reparentChildPages(pPage); assert( pPage->nOverflow==0 ); #ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - int i; - for(i=0; inCell; i++){ - rc = ptrmapPutOvfl(pPage, i); - if( rc!=SQLITE_OK ){ - goto end_shallow_balance; - } - } + if( ISAUTOVACUUM && rc==SQLITE_OK ){ + rc = setChildPtrmaps(pPage); } #endif releasePage(pChild); } end_shallow_balance: - sqliteFree(apCell); + sqlite3_free(apCell); return rc; } @@ -4982,8 +5662,9 @@ end_shallow_balance: ** child. Finally, call balance_internal() on the new child ** to cause it to split. */ -static int balance_deeper(MemPage *pPage){ +static int balance_deeper(BtCursor *pCur){ int rc; /* Return value from subprocedures */ + MemPage *pPage; /* Pointer to the root page */ MemPage *pChild; /* Pointer to a new child page */ Pgno pgnoChild; /* Page number of the new child page */ BtShared *pBt; /* The BTree */ @@ -4991,70 +5672,96 @@ static int balance_deeper(MemPage *pPage){ u8 *data; /* Content of the parent page */ u8 *cdata; /* Content of the child page */ int hdr; /* Offset to page header in parent */ - int brk; /* Offset to content of first cell in parent */ + int cbrk; /* Offset to content of first cell in parent */ - assert( pPage->pParent==0 ); - assert( pPage->nOverflow>0 ); + assert( pCur->iPage==0 ); + assert( pCur->apPage[0]->nOverflow>0 ); + + VVA_ONLY( pCur->pagesShuffled = 1 ); + pPage = pCur->apPage[0]; pBt = pPage->pBt; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); rc = allocateBtreePage(pBt, &pChild, &pgnoChild, pPage->pgno, 0); if( rc ) return rc; assert( sqlite3PagerIswriteable(pChild->pDbPage) ); usableSize = pBt->usableSize; data = pPage->aData; hdr = pPage->hdrOffset; - brk = get2byte(&data[hdr+5]); + cbrk = get2byte(&data[hdr+5]); cdata = pChild->aData; memcpy(cdata, &data[hdr], pPage->cellOffset+2*pPage->nCell-hdr); - memcpy(&cdata[brk], &data[brk], usableSize-brk); + memcpy(&cdata[cbrk], &data[cbrk], usableSize-cbrk); + assert( pChild->isInit==0 ); - rc = sqlite3BtreeInitPage(pChild, pPage); - if( rc ) goto balancedeeper_out; - memcpy(pChild->aOvfl, pPage->aOvfl, pPage->nOverflow*sizeof(pPage->aOvfl[0])); - pChild->nOverflow = pPage->nOverflow; - if( pChild->nOverflow ){ - pChild->nFree = 0; - } - assert( pChild->nCell==pPage->nCell ); - zeroPage(pPage, pChild->aData[0] & ~PTF_LEAF); - put4byte(&pPage->aData[pPage->hdrOffset+8], pgnoChild); - TRACE(("BALANCE: copy root %d into %d\n", pPage->pgno, pChild->pgno)); + rc = sqlite3BtreeInitPage(pChild); + if( rc==SQLITE_OK ){ + int nCopy = pPage->nOverflow*sizeof(pPage->aOvfl[0]); + memcpy(pChild->aOvfl, pPage->aOvfl, nCopy); + pChild->nOverflow = pPage->nOverflow; + if( pChild->nOverflow ){ + pChild->nFree = 0; + } + assert( pChild->nCell==pPage->nCell ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + zeroPage(pPage, pChild->aData[0] & ~PTF_LEAF); + put4byte(&pPage->aData[pPage->hdrOffset+8], pgnoChild); + TRACE(("BALANCE: copy root %d into %d\n", pPage->pgno, pChild->pgno)); + if( ISAUTOVACUUM ){ + rc = ptrmapPut(pBt, pChild->pgno, PTRMAP_BTREE, pPage->pgno); #ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->autoVacuum ){ - int i; - rc = ptrmapPut(pBt, pChild->pgno, PTRMAP_BTREE, pPage->pgno); - if( rc ) goto balancedeeper_out; - for(i=0; inCell; i++){ - rc = ptrmapPutOvfl(pChild, i); - if( rc!=SQLITE_OK ){ - return rc; + if( rc==SQLITE_OK ){ + rc = setChildPtrmaps(pChild); } + if( rc ){ + pChild->nOverflow = 0; + } +#endif } } -#endif - rc = balance_nonroot(pChild); -balancedeeper_out: - releasePage(pChild); + if( rc==SQLITE_OK ){ + pCur->iPage++; + pCur->apPage[1] = pChild; + pCur->aiIdx[0] = 0; + rc = balance_nonroot(pCur); + }else{ + releasePage(pChild); + } + return rc; } /* -** Decide if the page pPage needs to be balanced. If balancing is -** required, call the appropriate balancing routine. +** The page that pCur currently points to has just been modified in +** some way. This function figures out if this modification means the +** tree needs to be balanced, and if so calls the appropriate balancing +** routine. +** +** Parameter isInsert is true if a new cell was just inserted into the +** page, or false otherwise. */ -static int balance(MemPage *pPage, int insert){ +static int balance(BtCursor *pCur, int isInsert){ int rc = SQLITE_OK; - if( pPage->pParent==0 ){ - if( pPage->nOverflow>0 ){ - rc = balance_deeper(pPage); + MemPage *pPage = pCur->apPage[pCur->iPage]; + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + if( pCur->iPage==0 ){ + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc==SQLITE_OK && pPage->nOverflow>0 ){ + rc = balance_deeper(pCur); + assert( pCur->apPage[0]==pPage ); + assert( pPage->nOverflow==0 || rc!=SQLITE_OK ); } if( rc==SQLITE_OK && pPage->nCell==0 ){ - rc = balance_shallower(pPage); + rc = balance_shallower(pCur); + assert( pCur->apPage[0]==pPage ); + assert( pPage->nOverflow==0 || rc!=SQLITE_OK ); } }else{ if( pPage->nOverflow>0 || - (!insert && pPage->nFree>pPage->pBt->usableSize*2/3) ){ - rc = balance_nonroot(pPage); + (!isInsert && pPage->nFree>pPage->pBt->usableSize*2/3) ){ + rc = balance_nonroot(pCur); } } return rc; @@ -5068,30 +5775,62 @@ static int balance(MemPage *pPage, int insert){ ** is not in the ReadUncommmitted state, then this routine returns ** SQLITE_LOCKED. ** -** In addition to checking for read-locks (where a read-lock -** means a cursor opened with wrFlag==0) this routine also moves -** all write cursors so that they are pointing to the -** first Cell on the root page. This is necessary because an insert -** or delete might change the number of cells on a page or delete -** a page entirely and we do not want to leave any cursors -** pointing to non-existant pages or cells. +** As well as cursors with wrFlag==0, cursors with wrFlag==1 and +** isIncrblobHandle==1 are also considered 'read' cursors. Incremental +** blob cursors are used for both reading and writing. +** +** When pgnoRoot is the root page of an intkey table, this function is also +** responsible for invalidating incremental blob cursors when the table row +** on which they are opened is deleted or modified. Cursors are invalidated +** according to the following rules: +** +** 1) When BtreeClearTable() is called to completely delete the contents +** of a B-Tree table, pExclude is set to zero and parameter iRow is +** set to non-zero. In this case all incremental blob cursors open +** on the table rooted at pgnoRoot are invalidated. +** +** 2) When BtreeInsert(), BtreeDelete() or BtreePutData() is called to +** modify a table row via an SQL statement, pExclude is set to the +** write cursor used to do the modification and parameter iRow is set +** to the integer row id of the B-Tree entry being modified. Unless +** pExclude is itself an incremental blob cursor, then all incremental +** blob cursors open on row iRow of the B-Tree are invalidated. +** +** 3) If both pExclude and iRow are set to zero, no incremental blob +** cursors are invalidated. */ -static int checkReadLocks(Btree *pBtree, Pgno pgnoRoot, BtCursor *pExclude){ +static int checkReadLocks( + Btree *pBtree, + Pgno pgnoRoot, + BtCursor *pExclude, + i64 iRow +){ BtCursor *p; BtShared *pBt = pBtree->pBt; - sqlite3 *db = pBtree->pSqlite; + sqlite3 *db = pBtree->db; + assert( sqlite3BtreeHoldsMutex(pBtree) ); for(p=pBt->pCursor; p; p=p->pNext){ if( p==pExclude ) continue; - if( p->eState!=CURSOR_VALID ) continue; if( p->pgnoRoot!=pgnoRoot ) continue; - if( p->wrFlag==0 ){ - sqlite3 *dbOther = p->pBtree->pSqlite; +#ifndef SQLITE_OMIT_INCRBLOB + if( p->isIncrblobHandle && ( + (!pExclude && iRow) + || (pExclude && !pExclude->isIncrblobHandle && p->info.nKey==iRow) + )){ + p->eState = CURSOR_INVALID; + } +#endif + if( p->eState!=CURSOR_VALID ) continue; + if( p->wrFlag==0 +#ifndef SQLITE_OMIT_INCRBLOB + || p->isIncrblobHandle +#endif + ){ + sqlite3 *dbOther = p->pBtree->db; if( dbOther==0 || (dbOther!=db && (dbOther->flags & SQLITE_ReadUncommitted)==0) ){ return SQLITE_LOCKED; } - }else if( p->pPage->pgno!=p->pgnoRoot ){ - moveToRoot(p); } } return SQLITE_OK; @@ -5116,25 +5855,26 @@ int sqlite3BtreeInsert( int rc; int loc; int szNew; + int idx; MemPage *pPage; - BtShared *pBt = pCur->pBtree->pBt; + Btree *p = pCur->pBtree; + BtShared *pBt = p->pBt; unsigned char *oldCell; unsigned char *newCell = 0; - if( pBt->inTransaction!=TRANS_WRITE ){ - /* Must start a transaction before doing an insert */ - return pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR; - } + assert( cursorHoldsMutex(pCur) ); + assert( pBt->inTransaction==TRANS_WRITE ); assert( !pBt->readOnly ); - if( !pCur->wrFlag ){ - return SQLITE_PERM; /* Cursor not open for writing */ - } - if( checkReadLocks(pCur->pBtree, pCur->pgnoRoot, pCur) ){ + assert( pCur->wrFlag ); + if( checkReadLocks(pCur->pBtree, pCur->pgnoRoot, pCur, nKey) ){ return SQLITE_LOCKED; /* The table pCur points to has a read lock */ } + if( pCur->eState==CURSOR_FAULT ){ + return pCur->skip; + } /* Save the positions of any other cursors open on this table */ - clearCursorPosition(pCur); + sqlite3BtreeClearCursor(pCur); if( SQLITE_OK!=(rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur)) || SQLITE_OK!=(rc = sqlite3BtreeMoveto(pCur, pKey, nKey, appendBias, &loc)) @@ -5142,76 +5882,88 @@ int sqlite3BtreeInsert( return rc; } - pPage = pCur->pPage; + pPage = pCur->apPage[pCur->iPage]; assert( pPage->intKey || nKey>=0 ); - assert( pPage->leaf || !pPage->leafData ); + assert( pPage->leaf || !pPage->intKey ); TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", pCur->pgnoRoot, nKey, nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit ); - rc = sqlite3PagerWrite(pPage->pDbPage); - if( rc ) return rc; - newCell = sqliteMallocRaw( MX_CELL_SIZE(pBt) ); + allocateTempSpace(pBt); + newCell = pBt->pTmpSpace; if( newCell==0 ) return SQLITE_NOMEM; rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); if( rc ) goto end_insert; assert( szNew==cellSizePtr(pPage, newCell) ); assert( szNew<=MX_CELL_SIZE(pBt) ); + idx = pCur->aiIdx[pCur->iPage]; if( loc==0 && CURSOR_VALID==pCur->eState ){ - int szOld; - assert( pCur->idx>=0 && pCur->idxnCell ); - oldCell = findCell(pPage, pCur->idx); + u16 szOld; + assert( idxnCell ); + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ){ + goto end_insert; + } + oldCell = findCell(pPage, idx); if( !pPage->leaf ){ memcpy(newCell, oldCell, 4); } szOld = cellSizePtr(pPage, oldCell); rc = clearCell(pPage, oldCell); if( rc ) goto end_insert; - dropCell(pPage, pCur->idx, szOld); + rc = dropCell(pPage, idx, szOld); + if( rc!=SQLITE_OK ) { + goto end_insert; + } }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); - pCur->idx++; + idx = ++pCur->aiIdx[pCur->iPage]; pCur->info.nSize = 0; + pCur->validNKey = 0; }else{ assert( pPage->leaf ); } - rc = insertCell(pPage, pCur->idx, newCell, szNew, 0, 0); - if( rc!=SQLITE_OK ) goto end_insert; - rc = balance(pPage, 1); - /* sqlite3BtreePageDump(pCur->pBt, pCur->pgnoRoot, 1); */ - /* fflush(stdout); */ + rc = insertCell(pPage, idx, newCell, szNew, 0, 0); + if( rc==SQLITE_OK ){ + rc = balance(pCur, 1); + } + + /* Must make sure nOverflow is reset to zero even if the balance() + ** fails. Internal data structure corruption will result otherwise. */ + pCur->apPage[pCur->iPage]->nOverflow = 0; + if( rc==SQLITE_OK ){ moveToRoot(pCur); } end_insert: - sqliteFree(newCell); return rc; } /* ** Delete the entry that the cursor is pointing to. The cursor -** is left pointing at a random location. +** is left pointing at a arbitrary location. */ int sqlite3BtreeDelete(BtCursor *pCur){ - MemPage *pPage = pCur->pPage; + MemPage *pPage = pCur->apPage[pCur->iPage]; + int idx; unsigned char *pCell; int rc; Pgno pgnoChild = 0; - BtShared *pBt = pCur->pBtree->pBt; + Btree *p = pCur->pBtree; + BtShared *pBt = p->pBt; + assert( cursorHoldsMutex(pCur) ); assert( pPage->isInit ); - if( pBt->inTransaction!=TRANS_WRITE ){ - /* Must start a transaction before doing a delete */ - return pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR; - } + assert( pBt->inTransaction==TRANS_WRITE ); assert( !pBt->readOnly ); - if( pCur->idx >= pPage->nCell ){ + if( pCur->eState==CURSOR_FAULT ){ + return pCur->skip; + } + if( NEVER(pCur->aiIdx[pCur->iPage]>=pPage->nCell) ){ return SQLITE_ERROR; /* The cursor is not pointing to anything */ } - if( !pCur->wrFlag ){ - return SQLITE_PERM; /* Did not open this cursor for writing */ - } - if( checkReadLocks(pCur->pBtree, pCur->pgnoRoot, pCur) ){ + assert( pCur->wrFlag ); + if( checkReadLocks(pCur->pBtree, pCur->pgnoRoot, pCur, pCur->info.nKey) ){ return SQLITE_LOCKED; /* The table pCur points to has a read lock */ } @@ -5221,23 +5973,26 @@ int sqlite3BtreeDelete(BtCursor *pCur){ ** that the entry will be deleted from. */ if( - (rc = restoreOrClearCursorPosition(pCur))!=0 || + (rc = restoreCursorPosition(pCur))!=0 || (rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur))!=0 || (rc = sqlite3PagerWrite(pPage->pDbPage))!=0 ){ return rc; } - /* Locate the cell within it's page and leave pCell pointing to the + /* Locate the cell within its page and leave pCell pointing to the ** data. The clearCell() call frees any overflow pages associated with the ** cell. The cell itself is still intact. */ - pCell = findCell(pPage, pCur->idx); + idx = pCur->aiIdx[pCur->iPage]; + pCell = findCell(pPage, idx); if( !pPage->leaf ){ pgnoChild = get4byte(pCell); } rc = clearCell(pPage, pCell); - if( rc ) return rc; + if( rc ){ + return rc; + } if( !pPage->leaf ){ /* @@ -5248,48 +6003,135 @@ int sqlite3BtreeDelete(BtCursor *pCur){ ** to be a leaf so we can use it. */ BtCursor leafCur; + MemPage *pLeafPage = 0; + unsigned char *pNext; - int szNext; /* The compiler warning is wrong: szNext is always - ** initialized before use. Adding an extra initialization - ** to silence the compiler slows down the code. */ int notUsed; unsigned char *tempCell = 0; - assert( !pPage->leafData ); + assert( !pPage->intKey ); sqlite3BtreeGetTempCursor(pCur, &leafCur); rc = sqlite3BtreeNext(&leafCur, ¬Used); if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(leafCur.pPage->pDbPage); + assert( leafCur.aiIdx[leafCur.iPage]==0 ); + pLeafPage = leafCur.apPage[leafCur.iPage]; + rc = sqlite3PagerWrite(pLeafPage->pDbPage); } if( rc==SQLITE_OK ){ + int leafCursorInvalid = 0; + u16 szNext; TRACE(("DELETE: table=%d delete internal from %d replace from leaf %d\n", - pCur->pgnoRoot, pPage->pgno, leafCur.pPage->pgno)); - dropCell(pPage, pCur->idx, cellSizePtr(pPage, pCell)); - pNext = findCell(leafCur.pPage, leafCur.idx); - szNext = cellSizePtr(leafCur.pPage, pNext); + pCur->pgnoRoot, pPage->pgno, pLeafPage->pgno)); + dropCell(pPage, idx, cellSizePtr(pPage, pCell)); + pNext = findCell(pLeafPage, 0); + szNext = cellSizePtr(pLeafPage, pNext); assert( MX_CELL_SIZE(pBt)>=szNext+4 ); - tempCell = sqliteMallocRaw( MX_CELL_SIZE(pBt) ); + allocateTempSpace(pBt); + tempCell = pBt->pTmpSpace; if( tempCell==0 ){ rc = SQLITE_NOMEM; } + if( rc==SQLITE_OK ){ + rc = insertCell(pPage, idx, pNext-4, szNext+4, tempCell, 0); + } + + + /* The "if" statement in the next code block is critical. The + ** slightest error in that statement would allow SQLite to operate + ** correctly most of the time but produce very rare failures. To + ** guard against this, the following macros help to verify that + ** the "if" statement is well tested. + */ + testcase( pPage->nOverflow==0 && pPage->nFreeusableSize*2/3 + && pLeafPage->nFree+2+szNext > pBt->usableSize*2/3 ); + testcase( pPage->nOverflow==0 && pPage->nFree==pBt->usableSize*2/3 + && pLeafPage->nFree+2+szNext > pBt->usableSize*2/3 ); + testcase( pPage->nOverflow==0 && pPage->nFree==pBt->usableSize*2/3+1 + && pLeafPage->nFree+2+szNext > pBt->usableSize*2/3 ); + testcase( pPage->nOverflow>0 && pPage->nFree<=pBt->usableSize*2/3 + && pLeafPage->nFree+2+szNext > pBt->usableSize*2/3 ); + testcase( (pPage->nOverflow>0 || (pPage->nFree > pBt->usableSize*2/3)) + && pLeafPage->nFree+2+szNext == pBt->usableSize*2/3 ); + + + if( (pPage->nOverflow>0 || (pPage->nFree > pBt->usableSize*2/3)) && + (pLeafPage->nFree+2+szNext > pBt->usableSize*2/3) + ){ + /* This branch is taken if the internal node is now either overflowing + ** or underfull and the leaf node will be underfull after the just cell + ** copied to the internal node is deleted from it. This is a special + ** case because the call to balance() to correct the internal node + ** may change the tree structure and invalidate the contents of + ** the leafCur.apPage[] and leafCur.aiIdx[] arrays, which will be + ** used by the balance() required to correct the underfull leaf + ** node. + ** + ** The formula used in the expression above are based on facets of + ** the SQLite file-format that do not change over time. + */ + testcase( pPage->nFree==pBt->usableSize*2/3+1 ); + testcase( pLeafPage->nFree+2+szNext==pBt->usableSize*2/3+1 ); + leafCursorInvalid = 1; + } + + if( rc==SQLITE_OK ){ + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + put4byte(findOverflowCell(pPage, idx), pgnoChild); + VVA_ONLY( pCur->pagesShuffled = 0 ); + rc = balance(pCur, 0); + } + + if( rc==SQLITE_OK && leafCursorInvalid ){ + /* The leaf-node is now underfull and so the tree needs to be + ** rebalanced. However, the balance() operation on the internal + ** node above may have modified the structure of the B-Tree and + ** so the current contents of leafCur.apPage[] and leafCur.aiIdx[] + ** may not be trusted. + ** + ** It is not possible to copy the ancestry from pCur, as the same + ** balance() call has invalidated the pCur->apPage[] and aiIdx[] + ** arrays. + ** + ** The call to saveCursorPosition() below internally saves the + ** key that leafCur is currently pointing to. Currently, there + ** are two copies of that key in the tree - one here on the leaf + ** page and one on some internal node in the tree. The copy on + ** the leaf node is always the next key in tree-order after the + ** copy on the internal node. So, the call to sqlite3BtreeNext() + ** calls restoreCursorPosition() to point the cursor to the copy + ** stored on the internal node, then advances to the next entry, + ** which happens to be the copy of the key on the internal node. + ** Net effect: leafCur is pointing back to the duplicate cell + ** that needs to be removed, and the leafCur.apPage[] and + ** leafCur.aiIdx[] arrays are correct. + */ + VVA_ONLY( Pgno leafPgno = pLeafPage->pgno ); + rc = saveCursorPosition(&leafCur); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeNext(&leafCur, ¬Used); + } + pLeafPage = leafCur.apPage[leafCur.iPage]; + assert( pLeafPage->pgno==leafPgno ); + assert( leafCur.aiIdx[leafCur.iPage]==0 ); + } + + if( SQLITE_OK==rc + && SQLITE_OK==(rc = sqlite3PagerWrite(pLeafPage->pDbPage)) + ){ + dropCell(pLeafPage, 0, szNext); + VVA_ONLY( leafCur.pagesShuffled = 0 ); + rc = balance(&leafCur, 0); + assert( leafCursorInvalid || !leafCur.pagesShuffled + || !pCur->pagesShuffled ); + } } - if( rc==SQLITE_OK ){ - rc = insertCell(pPage, pCur->idx, pNext-4, szNext+4, tempCell, 0); - } - if( rc==SQLITE_OK ){ - put4byte(findOverflowCell(pPage, pCur->idx), pgnoChild); - rc = balance(pPage, 0); - } - if( rc==SQLITE_OK ){ - dropCell(leafCur.pPage, leafCur.idx, szNext); - rc = balance(leafCur.pPage, 0); - } - sqliteFree(tempCell); sqlite3BtreeReleaseTempCursor(&leafCur); }else{ TRACE(("DELETE: table=%d delete from leaf %d\n", pCur->pgnoRoot, pPage->pgno)); - dropCell(pPage, pCur->idx, cellSizePtr(pPage, pCell)); - rc = balance(pPage, 0); + rc = dropCell(pPage, idx, cellSizePtr(pPage, pCell)); + if( rc==SQLITE_OK ){ + rc = balance(pCur, 0); + } } if( rc==SQLITE_OK ){ moveToRoot(pCur); @@ -5308,20 +6150,21 @@ int sqlite3BtreeDelete(BtCursor *pCur){ ** BTREE_INTKEY|BTREE_LEAFDATA Used for SQL tables with rowid keys ** BTREE_ZERODATA Used for SQL indices */ -int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){ +static int btreeCreateTable(Btree *p, int *piTable, int flags){ BtShared *pBt = p->pBt; MemPage *pRoot; Pgno pgnoRoot; int rc; - if( pBt->inTransaction!=TRANS_WRITE ){ - /* Must start a transaction first */ - return pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR; - } + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( pBt->inTransaction==TRANS_WRITE ); assert( !pBt->readOnly ); #ifdef SQLITE_OMIT_AUTOVACUUM rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); - if( rc ) return rc; + if( rc ){ + return rc; + } #else if( pBt->autoVacuum ){ Pgno pgnoMove; /* Move a page here to make room for the root-page */ @@ -5339,13 +6182,15 @@ int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){ ** created so far, so the new root-page is (meta[3]+1). */ rc = sqlite3BtreeGetMeta(p, 4, &pgnoRoot); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + return rc; + } pgnoRoot++; /* The new root-page may not be allocated on a pointer-map page, or the ** PENDING_BYTE page. */ - if( pgnoRoot==PTRMAP_PAGENO(pBt, pgnoRoot) || + while( pgnoRoot==PTRMAP_PAGENO(pBt, pgnoRoot) || pgnoRoot==PENDING_BYTE_PAGE(pBt) ){ pgnoRoot++; } @@ -5389,7 +6234,7 @@ int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){ releasePage(pRoot); return rc; } - rc = relocatePage(pBt, pRoot, eType, iPtrPage, pgnoMove); + rc = relocatePage(pBt, pRoot, eType, iPtrPage, pgnoMove, 0); releasePage(pRoot); /* Obtain the page at pgnoRoot */ @@ -5432,6 +6277,14 @@ int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){ *piTable = (int)pgnoRoot; return SQLITE_OK; } +int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){ + int rc; + sqlite3BtreeEnter(p); + p->pBt->db = p->db; + rc = btreeCreateTable(p, piTable, flags); + sqlite3BtreeLeave(p); + return rc; +} /* ** Erase the given database page and all its children. Return @@ -5440,32 +6293,36 @@ int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){ static int clearDatabasePage( BtShared *pBt, /* The BTree that contains the table */ Pgno pgno, /* Page number to clear */ - MemPage *pParent, /* Parent page. NULL for the root */ - int freePageFlag /* Deallocate page if true */ + int freePageFlag, /* Deallocate page if true */ + int *pnChange ){ MemPage *pPage = 0; int rc; unsigned char *pCell; int i; - if( pgno>sqlite3PagerPagecount(pBt->pPager) ){ + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno>pagerPagecount(pBt) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, pgno, &pPage, pParent); + rc = getAndInitPage(pBt, pgno, &pPage); if( rc ) goto cleardatabasepage_out; for(i=0; inCell; i++){ pCell = findCell(pPage, i); if( !pPage->leaf ){ - rc = clearDatabasePage(pBt, get4byte(pCell), pPage->pParent, 1); + rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); if( rc ) goto cleardatabasepage_out; } rc = clearCell(pPage, pCell); if( rc ) goto cleardatabasepage_out; } if( !pPage->leaf ){ - rc = clearDatabasePage(pBt, get4byte(&pPage->aData[8]), pPage->pParent, 1); + rc = clearDatabasePage(pBt, get4byte(&pPage->aData[8]), 1, pnChange); if( rc ) goto cleardatabasepage_out; + }else if( pnChange ){ + assert( pPage->intKey ); + *pnChange += pPage->nCell; } if( freePageFlag ){ rc = freePage(pPage); @@ -5486,24 +6343,26 @@ cleardatabasepage_out: ** This routine will fail with SQLITE_LOCKED if there are any open ** read cursors on the table. Open write cursors are moved to the ** root of the table. +** +** If pnChange is not NULL, then table iTable must be an intkey table. The +** integer value pointed to by pnChange is incremented by the number of +** entries in the table. */ -int sqlite3BtreeClearTable(Btree *p, int iTable){ +int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ int rc; BtShared *pBt = p->pBt; - if( p->inTrans!=TRANS_WRITE ){ - return pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR; + sqlite3BtreeEnter(p); + pBt->db = p->db; + assert( p->inTrans==TRANS_WRITE ); + if( (rc = checkReadLocks(p, iTable, 0, 1))!=SQLITE_OK ){ + /* nothing to do */ + }else if( SQLITE_OK!=(rc = saveAllCursors(pBt, iTable, 0)) ){ + /* nothing to do */ + }else{ + rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); } - rc = checkReadLocks(p, iTable, 0); - if( rc ){ - return rc; - } - - /* Save the position of all cursors open on this table */ - if( SQLITE_OK!=(rc = saveAllCursors(pBt, iTable, 0)) ){ - return rc; - } - - return clearDatabasePage(pBt, (Pgno)iTable, 0, 0); + sqlite3BtreeLeave(p); + return rc; } /* @@ -5526,14 +6385,13 @@ int sqlite3BtreeClearTable(Btree *p, int iTable){ ** The last root page is recorded in meta[3] and the value of ** meta[3] is updated by this procedure. */ -int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ +static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ int rc; MemPage *pPage = 0; BtShared *pBt = p->pBt; - if( p->inTrans!=TRANS_WRITE ){ - return pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR; - } + assert( sqlite3BtreeHoldsMutex(p) ); + assert( p->inTrans==TRANS_WRITE ); /* It is illegal to drop a table if any cursors are open on the ** database. This is because in auto-vacuum mode the backend may @@ -5547,7 +6405,7 @@ int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ rc = sqlite3BtreeGetPage(pBt, (Pgno)iTable, &pPage, 0); if( rc ) return rc; - rc = sqlite3BtreeClearTable(p, iTable); + rc = sqlite3BtreeClearTable(p, iTable, 0); if( rc ){ releasePage(pPage); return rc; @@ -5588,7 +6446,7 @@ int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ if( rc!=SQLITE_OK ){ return rc; } - rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable); + rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); releasePage(pMove); if( rc!=SQLITE_OK ){ return rc; @@ -5632,6 +6490,14 @@ int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ } return rc; } +int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ + int rc; + sqlite3BtreeEnter(p); + p->pBt->db = p->db; + rc = btreeDropTable(p, iTable, piMoved); + sqlite3BtreeLeave(p); + return rc; +} /* @@ -5645,11 +6511,14 @@ int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ ** free pages is not visible. So Cookie[0] is the same as Meta[1]. */ int sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ - DbPage *pDbPage; + DbPage *pDbPage = 0; int rc; unsigned char *pP1; BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + pBt->db = p->db; + /* Reading a meta-data value requires a read-lock on page 1 (and hence ** the sqlite_master table. We grab this lock regardless of whether or ** not the SQLITE_ReadUncommitted flag is set (the table rooted at page @@ -5657,15 +6526,37 @@ int sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ */ rc = queryTableLock(p, 1, READ_LOCK); if( rc!=SQLITE_OK ){ + sqlite3BtreeLeave(p); return rc; } assert( idx>=0 && idx<=15 ); - rc = sqlite3PagerGet(pBt->pPager, 1, &pDbPage); - if( rc ) return rc; - pP1 = (unsigned char *)sqlite3PagerGetData(pDbPage); + if( pBt->pPage1 ){ + /* The b-tree is already holding a reference to page 1 of the database + ** file. In this case the required meta-data value can be read directly + ** from the page data of this reference. This is slightly faster than + ** requesting a new reference from the pager layer. + */ + pP1 = (unsigned char *)pBt->pPage1->aData; + }else{ + /* The b-tree does not have a reference to page 1 of the database file. + ** Obtain one from the pager layer. + */ + rc = sqlite3PagerGet(pBt->pPager, 1, &pDbPage); + if( rc ){ + sqlite3BtreeLeave(p); + return rc; + } + pP1 = (unsigned char *)sqlite3PagerGetData(pDbPage); + } *pMeta = get4byte(&pP1[36 + idx*4]); - sqlite3PagerUnref(pDbPage); + + /* If the b-tree is not holding a reference to page 1, then one was + ** requested from the pager layer in the above block. Release it now. + */ + if( !pBt->pPage1 ){ + sqlite3PagerUnref(pDbPage); + } /* If autovacuumed is disabled in this build but we are trying to ** access an autovacuumed database, then make the database readonly. @@ -5676,6 +6567,7 @@ int sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ /* Grab the read-lock on page 1. */ rc = lockTable(p, 1, READ_LOCK); + sqlite3BtreeLeave(p); return rc; } @@ -5688,20 +6580,24 @@ int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ unsigned char *pP1; int rc; assert( idx>=1 && idx<=15 ); - if( p->inTrans!=TRANS_WRITE ){ - return pBt->readOnly ? SQLITE_READONLY : SQLITE_ERROR; - } + sqlite3BtreeEnter(p); + pBt->db = p->db; + assert( p->inTrans==TRANS_WRITE ); assert( pBt->pPage1!=0 ); pP1 = pBt->pPage1->aData; rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - if( rc ) return rc; - put4byte(&pP1[36 + idx*4], iMeta); - if( idx==7 ){ - assert( pBt->autoVacuum || iMeta==0 ); - assert( iMeta==0 || iMeta==1 ); - pBt->incrVacuum = iMeta; + if( rc==SQLITE_OK ){ + put4byte(&pP1[36 + idx*4], iMeta); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( idx==7 ){ + assert( pBt->autoVacuum || iMeta==0 ); + assert( iMeta==0 || iMeta==1 ); + pBt->incrVacuum = (u8)iMeta; + } +#endif } - return SQLITE_OK; + sqlite3BtreeLeave(p); + return rc; } /* @@ -5710,10 +6606,15 @@ int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ */ int sqlite3BtreeFlags(BtCursor *pCur){ /* TODO: What about CURSOR_REQUIRESEEK state? Probably need to call - ** restoreOrClearCursorPosition() here. + ** restoreCursorPosition() here. */ - MemPage *pPage = pCur->pPage; - return pPage ? pPage->aData[pPage->hdrOffset] : 0; + MemPage *pPage; + restoreCursorPosition(pCur); + pPage = pCur->apPage[pCur->iPage]; + assert( cursorHoldsMutex(pCur) ); + assert( pPage!=0 ); + assert( pPage->pBt==pCur->pBt ); + return pPage->aData[pPage->hdrOffset]; } @@ -5736,23 +6637,21 @@ static void checkAppendMsg( ... ){ va_list ap; - char *zMsg2; if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; va_start(ap, zFormat); - zMsg2 = sqlite3VMPrintf(zFormat, ap); - va_end(ap); - if( zMsg1==0 ) zMsg1 = ""; - if( pCheck->zErrMsg ){ - char *zOld = pCheck->zErrMsg; - pCheck->zErrMsg = 0; - sqlite3SetString(&pCheck->zErrMsg, zOld, "\n", zMsg1, zMsg2, (char*)0); - sqliteFree(zOld); - }else{ - sqlite3SetString(&pCheck->zErrMsg, zMsg1, zMsg2, (char*)0); + if( pCheck->errMsg.nChar ){ + sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); + } + if( zMsg1 ){ + sqlite3StrAccumAppend(&pCheck->errMsg, zMsg1, -1); + } + sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap); + va_end(ap); + if( pCheck->errMsg.mallocFailed ){ + pCheck->mallocFailed = 1; } - sqliteFree(zMsg2); } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -5765,9 +6664,9 @@ static void checkAppendMsg( ** ** Also check that the page number is in bounds. */ -static int checkRef(IntegrityCk *pCheck, int iPage, char *zContext){ +static int checkRef(IntegrityCk *pCheck, Pgno iPage, char *zContext){ if( iPage==0 ) return 1; - if( iPage>pCheck->nPage || iPage<0 ){ + if( iPage>pCheck->nPage ){ checkAppendMsg(pCheck, zContext, "invalid page number %d", iPage); return 1; } @@ -5797,6 +6696,7 @@ static void checkPtrmap( rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) pCheck->mallocFailed = 1; checkAppendMsg(pCheck, zContext, "Failed to read ptrmap key=%d", iChild); return; } @@ -5845,7 +6745,7 @@ static void checkList( checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext); } #endif - if( n>pCheck->pBt->usableSize/4-8 ){ + if( n>pCheck->pBt->usableSize/4-2 ){ checkAppendMsg(pCheck, zContext, "freelist leaf count too big on page %d", iPage); N--; @@ -5902,7 +6802,6 @@ static void checkList( static int checkTreePage( IntegrityCk *pCheck, /* Context for the sanity check */ int iPage, /* Page number of the page to check */ - MemPage *pParent, /* Parent page */ char *zParentContext /* Parent context */ ){ MemPage *pPage; @@ -5913,7 +6812,7 @@ static int checkTreePage( BtShared *pBt; int usableSize; char zContext[100]; - char *hit; + char *hit = 0; sqlite3_snprintf(sizeof(zContext), zContext, "Page %d: ", iPage); @@ -5924,11 +6823,13 @@ static int checkTreePage( if( iPage==0 ) return 0; if( checkRef(pCheck, iPage, zParentContext) ) return 0; if( (rc = sqlite3BtreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ + if( rc==SQLITE_NOMEM ) pCheck->mallocFailed = 1; checkAppendMsg(pCheck, zContext, "unable to get the page. error code=%d", rc); return 0; } - if( (rc = sqlite3BtreeInitPage(pPage, pParent))!=0 ){ + if( (rc = sqlite3BtreeInitPage(pPage))!=0 ){ + assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ checkAppendMsg(pCheck, zContext, "sqlite3BtreeInitPage() returns error code %d", rc); releasePage(pPage); @@ -5940,7 +6841,7 @@ static int checkTreePage( depth = 0; for(i=0; inCell && pCheck->mxErr; i++){ u8 *pCell; - int sz; + u32 sz; CellInfo info; /* Check payload overflow pages @@ -5950,7 +6851,7 @@ static int checkTreePage( pCell = findCell(pPage,i); sqlite3BtreeParseCellPtr(pPage, pCell, &info); sz = info.nData; - if( !pPage->intKey ) sz += info.nKey; + if( !pPage->intKey ) sz += (int)info.nKey; assert( sz==info.nPayload ); if( sz>info.nLocal ){ int nPage = (sz - info.nLocal + usableSize - 5)/(usableSize - 4); @@ -5972,7 +6873,7 @@ static int checkTreePage( checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); } #endif - d2 = checkTreePage(pCheck,pgno,pPage,zContext); + d2 = checkTreePage(pCheck, pgno, zContext); if( i>0 && d2!=depth ){ checkAppendMsg(pCheck, zContext, "Child page depth differs"); } @@ -5988,22 +6889,34 @@ static int checkTreePage( checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, 0); } #endif - checkTreePage(pCheck, pgno, pPage, zContext); + checkTreePage(pCheck, pgno, zContext); } /* Check for complete coverage of the page */ data = pPage->aData; hdr = pPage->hdrOffset; - hit = sqliteMalloc( usableSize ); - if( hit ){ - memset(hit, 1, get2byte(&data[hdr+5])); + hit = sqlite3PageMalloc( pBt->pageSize ); + if( hit==0 ){ + pCheck->mallocFailed = 1; + }else{ + u16 contentOffset = get2byte(&data[hdr+5]); + if (contentOffset > usableSize) { + checkAppendMsg(pCheck, 0, + "Corruption detected in header on page %d",iPage,0); + goto check_page_abort; + } + memset(hit+contentOffset, 0, usableSize-contentOffset); + memset(hit, 1, contentOffset); nCell = get2byte(&data[hdr+3]); cellStart = hdr + 12 - 4*pPage->leaf; for(i=0; i=usableSize || pc<0 ){ checkAppendMsg(pCheck, 0, "Corruption detected in cell %d on page %d",i,iPage,0); @@ -6038,7 +6951,8 @@ static int checkTreePage( cnt, data[hdr+7], iPage); } } - sqliteFree(hit); +check_page_abort: + if (hit) sqlite3PageFree(hit); releasePage(pPage); return depth+1; @@ -6051,10 +6965,10 @@ static int checkTreePage( ** an array of pages numbers were each page number is the root page of ** a table. nRoot is the number of entries in aRoot. ** -** If everything checks out, this routine returns NULL. If something is -** amiss, an error message is written into memory obtained from malloc() -** and a pointer to that error message is returned. The calling function -** is responsible for freeing the error message when it is done. +** Write the number of error seen in *pnErr. Except for some memory +** allocation errors, an error message held in memory obtained from +** malloc is returned if *pnErr is non-zero. If *pnErr==0 then NULL is +** returned. If a memory allocation error occurs, NULL is returned. */ char *sqlite3BtreeIntegrityCheck( Btree *p, /* The btree to be checked */ @@ -6063,43 +6977,45 @@ char *sqlite3BtreeIntegrityCheck( int mxErr, /* Stop reporting errors after this many */ int *pnErr /* Write number of errors seen to this variable */ ){ - int i; + Pgno i; int nRef; IntegrityCk sCheck; BtShared *pBt = p->pBt; + char zErr[100]; + sqlite3BtreeEnter(p); + pBt->db = p->db; nRef = sqlite3PagerRefcount(pBt->pPager); if( lockBtreeWithRetry(p)!=SQLITE_OK ){ - return sqliteStrDup("Unable to acquire a read lock on the database"); + *pnErr = 1; + sqlite3BtreeLeave(p); + return sqlite3DbStrDup(0, "cannot acquire a read lock on the database"); } sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; - sCheck.nPage = sqlite3PagerPagecount(sCheck.pPager); + sCheck.nPage = pagerPagecount(sCheck.pBt); sCheck.mxErr = mxErr; sCheck.nErr = 0; + sCheck.mallocFailed = 0; *pnErr = 0; -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pBt->nTrunc!=0 ){ - sCheck.nPage = pBt->nTrunc; - } -#endif if( sCheck.nPage==0 ){ unlockBtreeIfUnused(pBt); + sqlite3BtreeLeave(p); return 0; } - sCheck.anRef = sqliteMallocRaw( (sCheck.nPage+1)*sizeof(sCheck.anRef[0]) ); + sCheck.anRef = sqlite3Malloc( (sCheck.nPage+1)*sizeof(sCheck.anRef[0]) ); if( !sCheck.anRef ){ unlockBtreeIfUnused(pBt); *pnErr = 1; - return sqlite3MPrintf("Unable to malloc %d bytes", - (sCheck.nPage+1)*sizeof(sCheck.anRef[0])); + sqlite3BtreeLeave(p); + return 0; } for(i=0; i<=sCheck.nPage; i++){ sCheck.anRef[i] = 0; } i = PENDING_BYTE_PAGE(pBt); if( i<=sCheck.nPage ){ sCheck.anRef[i] = 1; } - sCheck.zErrMsg = 0; + sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), 20000); /* Check the integrity of the freelist */ @@ -6108,14 +7024,14 @@ char *sqlite3BtreeIntegrityCheck( /* Check all the tables. */ - for(i=0; iautoVacuum && aRoot[i]>1 ){ checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0); } #endif - checkTreePage(&sCheck, aRoot[i], 0, "List of tree roots: "); + checkTreePage(&sCheck, aRoot[i], "List of tree roots: "); } /* Make sure every page in the file is referenced @@ -6140,10 +7056,12 @@ char *sqlite3BtreeIntegrityCheck( #endif } - /* Make sure this analysis did not leave any unref() pages + /* Make sure this analysis did not leave any unref() pages. + ** This is an internal consistency check; an integrity check + ** of the integrity check. */ unlockBtreeIfUnused(pBt); - if( nRef != sqlite3PagerRefcount(pBt->pPager) ){ + if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){ checkAppendMsg(&sCheck, 0, "Outstanding page count goes from %d to %d during this analysis", nRef, sqlite3PagerRefcount(pBt->pPager) @@ -6152,14 +7070,24 @@ char *sqlite3BtreeIntegrityCheck( /* Clean up and report errors. */ - sqliteFree(sCheck.anRef); + sqlite3BtreeLeave(p); + sqlite3_free(sCheck.anRef); + if( sCheck.mallocFailed ){ + sqlite3StrAccumReset(&sCheck.errMsg); + *pnErr = sCheck.nErr+1; + return 0; + } *pnErr = sCheck.nErr; - return sCheck.zErrMsg; + if( sCheck.nErr==0 ) sqlite3StrAccumReset(&sCheck.errMsg); + return sqlite3StrAccumFinish(&sCheck.errMsg); } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ /* ** Return the full pathname of the underlying database file. +** +** The pager filename is invariant as long as the pager is +** open so it is safe to access without the BtShared mutex. */ const char *sqlite3BtreeGetFilename(Btree *p){ assert( p->pBt->pPager!=0 ); @@ -6168,6 +7096,9 @@ const char *sqlite3BtreeGetFilename(Btree *p){ /* ** Return the pathname of the directory that contains the database file. +** +** The pager directory name is invariant as long as the pager is +** open so it is safe to access without the BtShared mutex. */ const char *sqlite3BtreeGetDirname(Btree *p){ assert( p->pBt->pPager!=0 ); @@ -6178,6 +7109,9 @@ const char *sqlite3BtreeGetDirname(Btree *p){ ** Return the pathname of the journal file for this database. The return ** value of this routine is the same regardless of whether the journal file ** has been created or not. +** +** The pager journal filename is invariant as long as the pager is +** open so it is safe to access without the BtShared mutex. */ const char *sqlite3BtreeGetJournalname(Btree *p){ assert( p->pBt->pPager!=0 ); @@ -6189,67 +7123,221 @@ const char *sqlite3BtreeGetJournalname(Btree *p){ ** Copy the complete content of pBtFrom into pBtTo. A transaction ** must be active for both files. ** -** The size of file pBtFrom may be reduced by this operation. -** If anything goes wrong, the transaction on pBtFrom is rolled back. +** The size of file pTo may be reduced by this operation. +** If anything goes wrong, the transaction on pTo is rolled back. +** +** If successful, CommitPhaseOne() may be called on pTo before returning. +** The caller should finish committing the transaction on pTo by calling +** sqlite3BtreeCommit(). */ -int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ +static int btreeCopyFile(Btree *pTo, Btree *pFrom){ int rc = SQLITE_OK; - Pgno i, nPage, nToPage, iSkip; + Pgno i; + + Pgno nFromPage; /* Number of pages in pFrom */ + Pgno nToPage; /* Number of pages in pTo */ + Pgno nNewPage; /* Number of pages in pTo after the copy */ + + Pgno iSkip; /* Pending byte page in pTo */ + int nToPageSize; /* Page size of pTo in bytes */ + int nFromPageSize; /* Page size of pFrom in bytes */ BtShared *pBtTo = pTo->pBt; BtShared *pBtFrom = pFrom->pBt; + pBtTo->db = pTo->db; + pBtFrom->db = pFrom->db; - if( pTo->inTrans!=TRANS_WRITE || pFrom->inTrans!=TRANS_WRITE ){ - return SQLITE_ERROR; + nToPageSize = pBtTo->pageSize; + nFromPageSize = pBtFrom->pageSize; + + assert( pTo->inTrans==TRANS_WRITE ); + assert( pFrom->inTrans==TRANS_WRITE ); + if( NEVER(pBtTo->pCursor) ){ + return SQLITE_BUSY; } - if( pBtTo->pCursor ) return SQLITE_BUSY; - nToPage = sqlite3PagerPagecount(pBtTo->pPager); - nPage = sqlite3PagerPagecount(pBtFrom->pPager); + + nToPage = pagerPagecount(pBtTo); + nFromPage = pagerPagecount(pBtFrom); iSkip = PENDING_BYTE_PAGE(pBtTo); - for(i=1; rc==SQLITE_OK && i<=nPage; i++){ - DbPage *pDbPage; - if( i==iSkip ) continue; - rc = sqlite3PagerGet(pBtFrom->pPager, i, &pDbPage); - if( rc ) break; - rc = sqlite3PagerOverwrite(pBtTo->pPager, i, sqlite3PagerGetData(pDbPage)); - sqlite3PagerUnref(pDbPage); - } - /* If the file is shrinking, journal the pages that are being truncated - ** so that they can be rolled back if the commit fails. + /* Variable nNewPage is the number of pages required to store the + ** contents of pFrom using the current page-size of pTo. */ - for(i=nPage+1; rc==SQLITE_OK && i<=nToPage; i++){ - DbPage *pDbPage; - if( i==iSkip ) continue; - rc = sqlite3PagerGet(pBtTo->pPager, i, &pDbPage); - if( rc ) break; - rc = sqlite3PagerWrite(pDbPage); - sqlite3PagerDontWrite(pDbPage); - /* Yeah. It seems wierd to call DontWrite() right after Write(). But - ** that is because the names of those procedures do not exactly - ** represent what they do. Write() really means "put this page in the - ** rollback journal and mark it as dirty so that it will be written - ** to the database file later." DontWrite() undoes the second part of - ** that and prevents the page from being written to the database. The - ** page is still on the rollback journal, though. And that is the whole - ** point of this loop: to put pages on the rollback journal. */ - sqlite3PagerUnref(pDbPage); - } - if( !rc && nPagepPager, nPage); + nNewPage = (Pgno) + (((i64)nFromPage*(i64)nFromPageSize+(i64)nToPageSize-1)/(i64)nToPageSize); + + for(i=1; rc==SQLITE_OK && (i<=nToPage || i<=nNewPage); i++){ + + /* Journal the original page. + ** + ** iSkip is the page number of the locking page (PENDING_BYTE_PAGE) + ** in database *pTo (before the copy). This page is never written + ** into the journal file. Unless i==iSkip or the page was not + ** present in pTo before the copy operation, journal page i from pTo. + */ + if( i!=iSkip && i<=nToPage ){ + DbPage *pDbPage = 0; + rc = sqlite3PagerGet(pBtTo->pPager, i, &pDbPage); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pDbPage); + if( rc==SQLITE_OK && i>nFromPage ){ + /* Yeah. It seems wierd to call DontWrite() right after Write(). But + ** that is because the names of those procedures do not exactly + ** represent what they do. Write() really means "put this page in the + ** rollback journal and mark it as dirty so that it will be written + ** to the database file later." DontWrite() undoes the second part of + ** that and prevents the page from being written to the database. The + ** page is still on the rollback journal, though. And that is the + ** whole point of this block: to put pages on the rollback journal. + */ + rc = sqlite3PagerDontWrite(pDbPage); + } + sqlite3PagerUnref(pDbPage); + } + } + + /* Overwrite the data in page i of the target database */ + if( rc==SQLITE_OK && i!=iSkip && i<=nNewPage ){ + + DbPage *pToPage = 0; + sqlite3_int64 iOff; + + rc = sqlite3PagerGet(pBtTo->pPager, i, &pToPage); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pToPage); + } + + for( + iOff=(i-1)*nToPageSize; + rc==SQLITE_OK && iOffpPager, iFrom, &pFromPage); + if( rc==SQLITE_OK ){ + char *zTo = sqlite3PagerGetData(pToPage); + char *zFrom = sqlite3PagerGetData(pFromPage); + int nCopy; + + if( nFromPageSize>=nToPageSize ){ + zFrom += ((i-1)*nToPageSize - ((iFrom-1)*nFromPageSize)); + nCopy = nToPageSize; + }else{ + zTo += (((iFrom-1)*nFromPageSize) - (i-1)*nToPageSize); + nCopy = nFromPageSize; + } + + memcpy(zTo, zFrom, nCopy); + sqlite3PagerUnref(pFromPage); + } + } + + if( pToPage ){ + MemPage *p = (MemPage *)sqlite3PagerGetExtra(pToPage); + p->isInit = 0; + sqlite3PagerUnref(pToPage); + } + } } - if( rc ){ + /* If things have worked so far, the database file may need to be + ** truncated. The complex part is that it may need to be truncated to + ** a size that is not an integer multiple of nToPageSize - the current + ** page size used by the pager associated with B-Tree pTo. + ** + ** For example, say the page-size of pTo is 2048 bytes and the original + ** number of pages is 5 (10 KB file). If pFrom has a page size of 1024 + ** bytes and 9 pages, then the file needs to be truncated to 9KB. + */ + if( rc==SQLITE_OK ){ + sqlite3_file *pFile = sqlite3PagerFile(pBtTo->pPager); + i64 iSize = (i64)nFromPageSize * (i64)nFromPage; + i64 iNow = (i64)((nToPage>nNewPage)?nToPage:nNewPage) * (i64)nToPageSize; + i64 iPending = ((i64)PENDING_BYTE_PAGE(pBtTo)-1) *(i64)nToPageSize; + + assert( iSize<=iNow ); + + /* Commit phase one syncs the journal file associated with pTo + ** containing the original data. It does not sync the database file + ** itself. After doing this it is safe to use OsTruncate() and other + ** file APIs on the database file directly. + */ + pBtTo->db = pTo->db; + rc = sqlite3PagerCommitPhaseOne(pBtTo->pPager, 0, 1); + if( iSizeiPending){ + i64 iOff; + for( + iOff=iPending; + rc==SQLITE_OK && iOff<(iPending+nToPageSize); + iOff += nFromPageSize + ){ + DbPage *pFromPage = 0; + Pgno iFrom = (Pgno)(iOff/nFromPageSize)+1; + + if( iFrom==PENDING_BYTE_PAGE(pBtFrom) || iFrom>nFromPage ){ + continue; + } + + rc = sqlite3PagerGet(pBtFrom->pPager, iFrom, &pFromPage); + if( rc==SQLITE_OK ){ + char *zFrom = sqlite3PagerGetData(pFromPage); + rc = sqlite3OsWrite(pFile, zFrom, nFromPageSize, iOff); + sqlite3PagerUnref(pFromPage); + } + } + } + } + + /* Sync the database file */ + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSync(pBtTo->pPager); + } + if( rc==SQLITE_OK ){ + pBtTo->pageSizeFixed = 0; + }else{ sqlite3BtreeRollback(pTo); } - return rc; + + return rc; } +int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ + int rc; + sqlite3BtreeEnter(pTo); + sqlite3BtreeEnter(pFrom); + rc = btreeCopyFile(pTo, pFrom); + sqlite3BtreeLeave(pFrom); + sqlite3BtreeLeave(pTo); + return rc; +} + #endif /* SQLITE_OMIT_VACUUM */ /* ** Return non-zero if a transaction is active. */ int sqlite3BtreeIsInTrans(Btree *p){ + assert( p==0 || sqlite3_mutex_held(p->db->mutex) ); return (p && (p->inTrans==TRANS_WRITE)); } @@ -6257,19 +7345,22 @@ int sqlite3BtreeIsInTrans(Btree *p){ ** Return non-zero if a statement transaction is active. */ int sqlite3BtreeIsInStmt(Btree *p){ - return (p->pBt && p->pBt->inStmt); + assert( sqlite3BtreeHoldsMutex(p) ); + return ALWAYS(p->pBt) && p->pBt->inStmt; } /* ** Return non-zero if a read (or write) transaction is active. */ int sqlite3BtreeIsInReadTrans(Btree *p){ - return (p && (p->inTrans!=TRANS_NONE)); + assert( p ); + assert( sqlite3_mutex_held(p->db->mutex) ); + return p->inTrans!=TRANS_NONE; } /* ** This function returns a pointer to a blob of memory associated with -** a single shared-btree. The memory is used by client code for it's own +** a single shared-btree. The memory is used by client code for its own ** purposes (for example, to store a high-level schema associated with ** the shared-btree). The btree layer manages reference counting issues. ** @@ -6278,17 +7369,23 @@ int sqlite3BtreeIsInReadTrans(Btree *p){ ** call the nBytes parameter is ignored and a pointer to the same blob ** of memory returned. ** +** If the nBytes parameter is 0 and the blob of memory has not yet been +** allocated, a null pointer is returned. If the blob has already been +** allocated, it is returned as normal. +** ** Just before the shared-btree is closed, the function passed as the ** xFree argument when the memory allocation was made is invoked on the -** blob of allocated memory. This function should not call sqliteFree() +** blob of allocated memory. This function should not call sqlite3_free() ** on the memory, the btree layer does that. */ void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ BtShared *pBt = p->pBt; - if( !pBt->pSchema ){ - pBt->pSchema = sqliteMalloc(nBytes); + sqlite3BtreeEnter(p); + if( !pBt->pSchema && nBytes ){ + pBt->pSchema = sqlite3MallocZero(nBytes); pBt->xFreeSchema = xFree; } + sqlite3BtreeLeave(p); return pBt->pSchema; } @@ -6297,7 +7394,12 @@ void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ ** handle holds an exclusive lock on the sqlite_master table. */ int sqlite3BtreeSchemaLocked(Btree *p){ - return (queryTableLock(p, MASTER_ROOT, READ_LOCK)!=SQLITE_OK); + int rc; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + rc = (queryTableLock(p, MASTER_ROOT, READ_LOCK)!=SQLITE_OK); + sqlite3BtreeLeave(p); + return rc; } @@ -6309,10 +7411,16 @@ int sqlite3BtreeSchemaLocked(Btree *p){ */ int sqlite3BtreeLockTable(Btree *p, int iTab, u8 isWriteLock){ int rc = SQLITE_OK; - u8 lockType = (isWriteLock?WRITE_LOCK:READ_LOCK); - rc = queryTableLock(p, iTab, lockType); - if( rc==SQLITE_OK ){ - rc = lockTable(p, iTab, lockType); + if( p->sharable ){ + u8 lockType = READ_LOCK + isWriteLock; + assert( READ_LOCK+1==WRITE_LOCK ); + assert( isWriteLock==0 || isWriteLock==1 ); + sqlite3BtreeEnter(p); + rc = queryTableLock(p, iTab, lockType); + if( rc==SQLITE_OK ){ + rc = lockTable(p, iTab, lockType); + } + sqlite3BtreeLeave(p); } return rc; } @@ -6327,9 +7435,13 @@ int sqlite3BtreeLockTable(Btree *p, int iTab, u8 isWriteLock){ ** to change the length of the data stored. */ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ - + assert( cursorHoldsMutex(pCsr) ); + assert( sqlite3_mutex_held(pCsr->pBtree->db->mutex) ); assert(pCsr->isIncrblobHandle); - if( pCsr->eState==CURSOR_REQUIRESEEK ){ + + restoreCursorPosition(pCsr); + assert( pCsr->eState!=CURSOR_REQUIRESEEK ); + if( pCsr->eState!=CURSOR_VALID ){ return SQLITE_ABORT; } @@ -6341,12 +7453,12 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ if( !pCsr->wrFlag ){ return SQLITE_READONLY; } - assert( !pCsr->pBtree->pBt->readOnly - && pCsr->pBtree->pBt->inTransaction==TRANS_WRITE ); - if( checkReadLocks(pCsr->pBtree, pCsr->pgnoRoot, pCsr) ){ + assert( !pCsr->pBt->readOnly + && pCsr->pBt->inTransaction==TRANS_WRITE ); + if( checkReadLocks(pCsr->pBtree, pCsr->pgnoRoot, pCsr, 0) ){ return SQLITE_LOCKED; /* The table pCur points to has a read lock */ } - if( pCsr->eState==CURSOR_INVALID || !pCsr->pPage->intKey ){ + if( pCsr->eState==CURSOR_INVALID || !pCsr->apPage[pCsr->iPage]->intKey ){ return SQLITE_ERROR; } @@ -6364,6 +7476,8 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ ** sqlite3BtreePutData()). */ void sqlite3BtreeCacheOverflow(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert(!pCur->isIncrblobHandle); assert(!pCur->aOverflow); pCur->isIncrblobHandle = 1; diff --git a/btree.h b/btree.h index bf8929d..070059c 100644 --- a/btree.h +++ b/btree.h @@ -13,7 +13,7 @@ ** subsystem. See comments in the source code for a detailed description ** of what each interface routine does. ** -** @(#) $Id: btree.h,v 1.82 2007/05/08 21:45:27 drh Exp $ +** @(#) $Id: btree.h,v 1.106 2008/12/17 17:30:26 danielk1977 Exp $ */ #ifndef _BTREE_H_ #define _BTREE_H_ @@ -41,13 +41,26 @@ typedef struct Btree Btree; typedef struct BtCursor BtCursor; typedef struct BtShared BtShared; +typedef struct BtreeMutexArray BtreeMutexArray; + +/* +** This structure records all of the Btrees that need to hold +** a mutex before we enter sqlite3VdbeExec(). The Btrees are +** are placed in aBtree[] in order of aBtree[]->pBt. That way, +** we can always lock and unlock them all quickly. +*/ +struct BtreeMutexArray { + int nMutex; + Btree *aBtree[SQLITE_MAX_ATTACHED+1]; +}; int sqlite3BtreeOpen( const char *zFilename, /* Name of database file to open */ sqlite3 *db, /* Associated database connection */ Btree **, /* Return open Btree* here */ - int flags /* Flags */ + int flags, /* Flags */ + int vfsFlags /* Flags passed through to VFS open */ ); /* The flags parameter to sqlite3BtreeOpen can be the bitwise or of the @@ -59,9 +72,11 @@ int sqlite3BtreeOpen( #define BTREE_OMIT_JOURNAL 1 /* Do not use journal. No argument */ #define BTREE_NO_READLOCK 2 /* Omit readlocks on readonly files */ #define BTREE_MEMORY 4 /* In-memory DB. No argument */ +#define BTREE_READONLY 8 /* Open the database in read-only mode */ +#define BTREE_READWRITE 16 /* Open for both reading and writing */ +#define BTREE_CREATE 32 /* Create the database if it does not exist */ int sqlite3BtreeClose(Btree*); -int sqlite3BtreeSetBusyHandler(Btree*,BusyHandler*); int sqlite3BtreeSetCacheSize(Btree*,int); int sqlite3BtreeSetSafetyLevel(Btree*,int,int); int sqlite3BtreeSyncDisabled(Btree*); @@ -86,6 +101,7 @@ int sqlite3BtreeIsInReadTrans(Btree*); void *sqlite3BtreeSchema(Btree *, int, void(*)(void *)); int sqlite3BtreeSchemaLocked(Btree *); int sqlite3BtreeLockTable(Btree *, int, u8); +int sqlite3BtreeSavepoint(Btree *, int, int); const char *sqlite3BtreeGetFilename(Btree *); const char *sqlite3BtreeGetDirname(Btree *); @@ -102,21 +118,36 @@ int sqlite3BtreeIncrVacuum(Btree *); #define BTREE_LEAFDATA 4 /* Data stored in leaves only. Implies INTKEY */ int sqlite3BtreeDropTable(Btree*, int, int*); -int sqlite3BtreeClearTable(Btree*, int); +int sqlite3BtreeClearTable(Btree*, int, int*); int sqlite3BtreeGetMeta(Btree*, int idx, u32 *pValue); int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); +void sqlite3BtreeTripAllCursors(Btree*, int); int sqlite3BtreeCursor( Btree*, /* BTree containing table to open */ int iTable, /* Index of root page */ int wrFlag, /* 1 for writing. 0 for read-only */ - int(*)(void*,int,const void*,int,const void*), /* Key comparison function */ - void*, /* First argument to compare function */ - BtCursor **ppCursor /* Returned cursor */ + struct KeyInfo*, /* First argument to compare function */ + BtCursor *pCursor /* Space to write cursor structure */ ); +int sqlite3BtreeCursorSize(void); int sqlite3BtreeCloseCursor(BtCursor*); -int sqlite3BtreeMoveto(BtCursor*,const void *pKey,i64 nKey,int bias,int *pRes); +int sqlite3BtreeMoveto( + BtCursor*, + const void *pKey, + i64 nKey, + int bias, + int *pRes +); +int sqlite3BtreeMovetoUnpacked( + BtCursor*, + UnpackedRecord *pUnKey, + i64 intKey, + int bias, + int *pRes +); +int sqlite3BtreeCursorHasMoved(BtCursor*, int*); int sqlite3BtreeDelete(BtCursor*); int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, const void *pData, int nData, @@ -129,6 +160,7 @@ int sqlite3BtreeFlags(BtCursor*); int sqlite3BtreePrevious(BtCursor*, int *pRes); int sqlite3BtreeKeySize(BtCursor*, i64 *pSize); int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*); +sqlite3 *sqlite3BtreeCursorDb(const BtCursor*); const void *sqlite3BtreeKeyFetch(BtCursor*, int *pAmt); const void *sqlite3BtreeDataFetch(BtCursor*, int *pAmt); int sqlite3BtreeDataSize(BtCursor*, u32 *pSize); @@ -139,11 +171,55 @@ struct Pager *sqlite3BtreePager(Btree*); int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); void sqlite3BtreeCacheOverflow(BtCursor *); +void sqlite3BtreeClearCursor(BtCursor *); #ifdef SQLITE_TEST int sqlite3BtreeCursorInfo(BtCursor*, int*, int); void sqlite3BtreeCursorList(Btree*); -int sqlite3BtreePageDump(Btree*, int, int recursive); #endif +/* +** If we are not using shared cache, then there is no need to +** use mutexes to access the BtShared structures. So make the +** Enter and Leave procedures no-ops. +*/ +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE + void sqlite3BtreeEnter(Btree*); + void sqlite3BtreeLeave(Btree*); +#ifndef NDEBUG + /* This routine is used inside assert() statements only. */ + int sqlite3BtreeHoldsMutex(Btree*); +#endif + void sqlite3BtreeEnterCursor(BtCursor*); + void sqlite3BtreeLeaveCursor(BtCursor*); + void sqlite3BtreeEnterAll(sqlite3*); + void sqlite3BtreeLeaveAll(sqlite3*); +#ifndef NDEBUG + /* This routine is used inside assert() statements only. */ + int sqlite3BtreeHoldsAllMutexes(sqlite3*); +#endif + void sqlite3BtreeMutexArrayEnter(BtreeMutexArray*); + void sqlite3BtreeMutexArrayLeave(BtreeMutexArray*); + void sqlite3BtreeMutexArrayInsert(BtreeMutexArray*, Btree*); +#else +# define sqlite3BtreeEnter(X) +# define sqlite3BtreeLeave(X) +#ifndef NDEBUG + /* This routine is used inside assert() statements only. */ +# define sqlite3BtreeHoldsMutex(X) 1 +#endif +# define sqlite3BtreeEnterCursor(X) +# define sqlite3BtreeLeaveCursor(X) +# define sqlite3BtreeEnterAll(X) +# define sqlite3BtreeLeaveAll(X) +#ifndef NDEBUG + /* This routine is used inside assert() statements only. */ +# define sqlite3BtreeHoldsAllMutexes(X) 1 +#endif +# define sqlite3BtreeMutexArrayEnter(X) +# define sqlite3BtreeMutexArrayLeave(X) +# define sqlite3BtreeMutexArrayInsert(X,Y) +#endif + + #endif /* _BTREE_H_ */ diff --git a/btreeInt.h b/btreeInt.h index 2964ad3..17453d2 100644 --- a/btreeInt.h +++ b/btreeInt.h @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btreeInt.h,v 1.5 2007/06/15 12:06:59 drh Exp $ +** $Id: btreeInt.h,v 1.38 2008/12/27 15:23:13 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to @@ -221,10 +221,11 @@ #define MX_CELL_SIZE(pBt) (pBt->pageSize-8) /* The maximum number of cells on a single page of the database. This -** assumes a minimum cell size of 3 bytes. Such small cells will be -** exceedingly rare, but they are possible. +** assumes a minimum cell size of 6 bytes (4 bytes for the cell itself +** plus 2 bytes for the index to the cell in the page header). Such +** small cells will be rare, but they are possible. */ -#define MX_CELL(pBt) ((pBt->pageSize-8)/3) +#define MX_CELL(pBt) ((pBt->pageSize-8)/6) /* Forward declarations */ typedef struct MemPage MemPage; @@ -248,7 +249,7 @@ typedef struct BtLock BtLock; /* ** Page type flags. An ORed combination of these flags appear as the -** first byte of every BTree page. +** first byte of on-disk image of every BTree page. */ #define PTF_INTKEY 0x01 #define PTF_ZERODATA 0x02 @@ -264,33 +265,32 @@ typedef struct BtLock BtLock; ** walk up the BTree from any leaf to the root. Care must be taken to ** unref() the parent page pointer when this page is no longer referenced. ** The pageDestructor() routine handles that chore. +** +** Access to all fields of this structure is controlled by the mutex +** stored in MemPage.pBt->mutex. */ struct MemPage { u8 isInit; /* True if previously initialized. MUST BE FIRST! */ - u8 idxShift; /* True if Cell indices have changed */ u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ u8 intKey; /* True if intkey flag is set */ u8 leaf; /* True if leaf flag is set */ - u8 zeroData; /* True if table stores keys only */ - u8 leafData; /* True if tables stores data on leaves only */ u8 hasData; /* True if this page stores data */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ - u16 maxLocal; /* Copy of Btree.maxLocal or Btree.maxLeaf */ - u16 minLocal; /* Copy of Btree.minLocal or Btree.minLeaf */ + u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ + u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ u16 cellOffset; /* Index in aData of first cell pointer */ - u16 idxParent; /* Index in parent of this node */ u16 nFree; /* Number of free bytes on the page */ u16 nCell; /* Number of cells on this page, local and ovfl */ + u16 maskPage; /* Mask for page offset */ struct _OvflCell { /* Cells that will not fit on aData[] */ u8 *pCell; /* Pointers to the body of the overflow cell */ u16 idx; /* Insert this cell before idx-th non-overflow cell */ } aOvfl[5]; - BtShared *pBt; /* Pointer back to BTree structure */ - u8 *aData; /* Pointer back to the start of the page */ + BtShared *pBt; /* Pointer to BtShared that this page is part of */ + u8 *aData; /* Pointer to disk image of the page data */ DbPage *pDbPage; /* Pager page handle */ Pgno pgno; /* Page number for this page */ - MemPage *pParent; /* The parent of this page. NULL for root */ }; /* @@ -300,11 +300,36 @@ struct MemPage { */ #define EXTRA_SIZE sizeof(MemPage) -/* Btree handle */ +/* A Btree handle +** +** A database connection contains a pointer to an instance of +** this object for every database file that it has open. This structure +** is opaque to the database connection. The database connection cannot +** see the internals of this structure and only deals with pointers to +** this structure. +** +** For some database files, the same underlying database cache might be +** shared between multiple connections. In that case, each contection +** has it own pointer to this object. But each instance of this object +** points to the same BtShared object. The database cache and the +** schema associated with the database file are all contained within +** the BtShared object. +** +** All fields in this structure are accessed under sqlite3.mutex. +** The pBt pointer itself may not be changed while there exists cursors +** in the referenced BtShared that point back to this Btree since those +** cursors have to do go through this Btree to find their BtShared and +** they often do so without holding sqlite3.mutex. +*/ struct Btree { - sqlite3 *pSqlite; - BtShared *pBt; - u8 inTrans; /* TRANS_NONE, TRANS_READ or TRANS_WRITE */ + sqlite3 *db; /* The database connection holding this btree */ + BtShared *pBt; /* Sharable content of this btree */ + u8 inTrans; /* TRANS_NONE, TRANS_READ or TRANS_WRITE */ + u8 sharable; /* True if we can share pBt with another db */ + u8 locked; /* True if db currently has pBt locked */ + int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ + Btree *pNext; /* List of other sharable Btrees from the same db */ + Btree *pPrev; /* Back pointer of the same list */ }; /* @@ -312,47 +337,59 @@ struct Btree { ** ** If the shared-data extension is enabled, there may be multiple users ** of the Btree structure. At most one of these may open a write transaction, -** but any number may have active read transactions. Variable Btree.pDb -** points to the handle that owns any current write-transaction. +** but any number may have active read transactions. */ #define TRANS_NONE 0 #define TRANS_READ 1 #define TRANS_WRITE 2 /* -** Everything we need to know about an open database +** An instance of this object represents a single database file. +** +** A single database file can be in use as the same time by two +** or more database connections. When two or more connections are +** sharing the same database file, each connection has it own +** private Btree object for the file and each of those Btrees points +** to this one BtShared object. BtShared.nRef is the number of +** connections currently sharing this database file. +** +** Fields in this structure are accessed under the BtShared.mutex +** mutex, except for nRef and pNext which are accessed under the +** global SQLITE_MUTEX_STATIC_MASTER mutex. The pPager field +** may not be modified once it is initially set as long as nRef>0. +** The pSchema field may be set once under BtShared.mutex and +** thereafter is unchanged as long as nRef>0. */ struct BtShared { Pager *pPager; /* The page cache */ + sqlite3 *db; /* Database connection currently using this Btree */ BtCursor *pCursor; /* A list of all open cursors */ MemPage *pPage1; /* First page of the database */ u8 inStmt; /* True if we are in a statement subtransaction */ u8 readOnly; /* True if the underlying file is readonly */ - u8 maxEmbedFrac; /* Maximum payload as % of total page size */ - u8 minEmbedFrac; /* Minimum payload as % of total page size */ - u8 minLeafFrac; /* Minimum leaf payload as % of total page size */ u8 pageSizeFixed; /* True if the page size can no longer be changed */ #ifndef SQLITE_OMIT_AUTOVACUUM u8 autoVacuum; /* True if auto-vacuum is enabled */ u8 incrVacuum; /* True if incr-vacuum is enabled */ - Pgno nTrunc; /* Non-zero if the db will be truncated (incr vacuum) */ #endif u16 pageSize; /* Total number of bytes on a page */ u16 usableSize; /* Number of usable bytes on each page */ - int maxLocal; /* Maximum local payload in non-LEAFDATA tables */ - int minLocal; /* Minimum local payload in non-LEAFDATA tables */ - int maxLeaf; /* Maximum local payload in a LEAFDATA table */ - int minLeaf; /* Minimum local payload in a LEAFDATA table */ - BusyHandler *pBusyHandler; /* Callback for when there is lock contention */ + u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ + u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ + u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */ + u16 minLeaf; /* Minimum local payload in a LEAFDATA table */ u8 inTransaction; /* Transaction state */ - int nRef; /* Number of references to this structure */ int nTransaction; /* Number of open transactions (read + write) */ void *pSchema; /* Pointer to space allocated by sqlite3BtreeSchema() */ void (*xFreeSchema)(void*); /* Destructor for BtShared.pSchema */ + sqlite3_mutex *mutex; /* Non-recursive mutex required to access this struct */ #ifndef SQLITE_OMIT_SHARED_CACHE + int nRef; /* Number of references to this structure */ + BtShared *pNext; /* Next on a list of sharable BtShared structs */ BtLock *pLock; /* List of locks held on this shared-btree struct */ - BtShared *pNext; /* Next in ThreadData.pBtree linked list */ + Btree *pExclusive; /* Btree with an EXCLUSIVE lock on the whole db */ #endif + u8 *pTmpSpace; /* BtShared.pageSize bytes of space for tmp use */ }; /* @@ -373,20 +410,40 @@ struct CellInfo { }; /* -** A cursor is a pointer to a particular entry in the BTree. +** Maximum depth of an SQLite B-Tree structure. Any B-Tree deeper than +** this will be declared corrupt. This value is calculated based on a +** maximum database size of 2^31 pages a minimum fanout of 2 for a +** root-node and 3 for all other internal nodes. +** +** If a tree that appears to be taller than this is encountered, it is +** assumed that the database is corrupt. +*/ +#define BTCURSOR_MAX_DEPTH 20 + +/* +** A cursor is a pointer to a particular entry within a particular +** b-tree within a database file. +** ** The entry is identified by its MemPage and the index in ** MemPage.aCell[] of the entry. +** +** When a single database file can shared by two more database connections, +** but cursors cannot be shared. Each cursor is associated with a +** particular database connection identified BtCursor.pBtree.db. +** +** Fields in this structure are accessed under the BtShared.mutex +** found at self->pBt->mutex. */ struct BtCursor { Btree *pBtree; /* The Btree to which this cursor belongs */ + BtShared *pBt; /* The BtShared this cursor points to */ BtCursor *pNext, *pPrev; /* Forms a linked list of all cursors */ - int (*xCompare)(void*,int,const void*,int,const void*); /* Key comp func */ - void *pArg; /* First arg to xCompare() */ + struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ Pgno pgnoRoot; /* The root page of this tree */ - MemPage *pPage; /* Page that contains the entry */ - int idx; /* Index of the entry in pPage->aCell[] */ CellInfo info; /* A parse of the cell we are pointing at */ u8 wrFlag; /* True if writable */ + u8 atLast; /* Cursor pointing to the last entry */ + u8 validNKey; /* True if info.nKey is valid */ u8 eState; /* One of the CURSOR_XXX constants (see below) */ void *pKey; /* Saved key that was cursor's last known position */ i64 nKey; /* Size of pKey, or last integer key */ @@ -395,6 +452,12 @@ struct BtCursor { u8 isIncrblobHandle; /* True if this cursor is an incr. io handle */ Pgno *aOverflow; /* Cache of overflow page locations */ #endif +#ifndef NDEBUG + u8 pagesShuffled; /* True if Btree pages are rearranged by balance()*/ +#endif + i16 iPage; /* Index of current page in apPage */ + MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ + u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */ }; /* @@ -412,32 +475,20 @@ struct BtCursor { ** The table that this cursor was opened on still exists, but has been ** modified since the cursor was last used. The cursor position is saved ** in variables BtCursor.pKey and BtCursor.nKey. When a cursor is in -** this state, restoreOrClearCursorPosition() can be called to attempt to +** this state, restoreCursorPosition() can be called to attempt to ** seek the cursor to the saved position. +** +** CURSOR_FAULT: +** A unrecoverable error (an I/O error or a malloc failure) has occurred +** on a different connection that shares the BtShared cache with this +** cursor. The error has left the cache in an inconsistent state. +** Do nothing else with this cursor. Any attempt to use the cursor +** should return the error code stored in BtCursor.skip */ #define CURSOR_INVALID 0 #define CURSOR_VALID 1 #define CURSOR_REQUIRESEEK 2 - -/* -** The TRACE macro will print high-level status information about the -** btree operation when the global variable sqlite3_btree_trace is -** enabled. -*/ -#if SQLITE_TEST -# define TRACE(X) if( sqlite3_btree_trace ){ printf X; fflush(stdout); } -#else -# define TRACE(X) -#endif - -/* -** Routines to read and write variable-length integers. These used to -** be defined locally, but now we use the varint routines in the util.c -** file. -*/ -#define getVarint sqlite3GetVarint -#define getVarint32(A,B) ((*B=*(A))<=0x7f?1:sqlite3GetVarint32(A,B)) -#define putVarint sqlite3PutVarint +#define CURSOR_FAULT 3 /* The database page the PENDING_BYTE occupies. This page is never used. ** TODO: This macro is very similary to PAGER_MJ_PGNO() in pager.c. They @@ -449,7 +500,7 @@ struct BtCursor { #ifdef SQLITE_OMIT_DISKIO # define PENDING_BYTE_PAGE(pBt) 0x7fffffff #else -# define PENDING_BYTE_PAGE(pBt) ((PENDING_BYTE/(pBt)->pageSize)+1) +# define PENDING_BYTE_PAGE(pBt) ((Pgno)((PENDING_BYTE/(pBt)->pageSize)+1)) #endif /* @@ -486,7 +537,7 @@ struct BtLock { ** this test. */ #define PTRMAP_PAGENO(pBt, pgno) ptrmapPageno(pBt, pgno) -#define PTRMAP_PTROFFSET(pBt, pgno) (5*(pgno-ptrmapPageno(pBt, pgno)-1)) +#define PTRMAP_PTROFFSET(pgptrmap, pgno) (5*(pgno-pgptrmap-1)) #define PTRMAP_ISPAGE(pBt, pgno) (PTRMAP_PAGENO((pBt),(pgno))==(pgno)) /* @@ -530,8 +581,6 @@ struct BtLock { ** of handle p (type Btree*) are internally consistent. */ #define btreeIntegrity(p) \ - assert( p->inTrans!=TRANS_NONE || p->pBt->nTransactionpBt->nRef ); \ - assert( p->pBt->nTransaction<=p->pBt->nRef ); \ assert( p->pBt->inTransaction!=TRANS_NONE || p->pBt->nTransaction==0 ); \ assert( p->pBt->inTransaction>=p->inTrans ); @@ -558,18 +607,19 @@ typedef struct IntegrityCk IntegrityCk; struct IntegrityCk { BtShared *pBt; /* The tree being checked out */ Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ - int nPage; /* Number of pages in the database */ + Pgno nPage; /* Number of pages in the database */ int *anRef; /* Number of times each page is referenced */ int mxErr; /* Stop accumulating errors when this reaches zero */ - char *zErrMsg; /* An error message. NULL if no errors seen. */ int nErr; /* Number of messages written to zErrMsg so far */ + int mallocFailed; /* A memory allocation error has occurred */ + StrAccum errMsg; /* Accumulate the error message text here */ }; /* ** Read or write a two- and four-byte big-endian integer values. */ #define get2byte(x) ((x)[0]<<8 | (x)[1]) -#define put2byte(p,v) ((p)[0] = (v)>>8, (p)[1] = (v)) +#define put2byte(p,v) ((p)[0] = (u8)((v)>>8), (p)[1] = (u8)(v)) #define get4byte sqlite3Get4byte #define put4byte sqlite3Put4byte @@ -577,12 +627,10 @@ struct IntegrityCk { ** Internal routines that should be accessed by the btree layer only. */ int sqlite3BtreeGetPage(BtShared*, Pgno, MemPage**, int); -int sqlite3BtreeInitPage(MemPage *pPage, MemPage *pParent); +int sqlite3BtreeInitPage(MemPage *pPage); void sqlite3BtreeParseCellPtr(MemPage*, u8*, CellInfo*); void sqlite3BtreeParseCell(MemPage*, int, CellInfo*); -u8 *sqlite3BtreeFindCell(MemPage *pPage, int iCell); -int sqlite3BtreeRestoreOrClearCursorPosition(BtCursor *pCur); +int sqlite3BtreeRestoreCursorPosition(BtCursor *pCur); void sqlite3BtreeGetTempCursor(BtCursor *pCur, BtCursor *pTempCur); void sqlite3BtreeReleaseTempCursor(BtCursor *pCur); -int sqlite3BtreeIsRootPage(MemPage *pPage); void sqlite3BtreeMoveToParent(BtCursor *pCur); diff --git a/build.c b/build.c index 0fd4180..c61fbb3 100644 --- a/build.c +++ b/build.c @@ -22,7 +22,7 @@ ** COMMIT ** ROLLBACK ** -** $Id: build.c,v 1.433 2007/07/02 19:31:27 drh Exp $ +** $Id: build.c,v 1.511 2008/12/30 06:24:58 danielk1977 Exp $ */ #include "sqliteInt.h" #include @@ -32,7 +32,7 @@ ** be parsed. Initialize the pParse structure as needed. */ void sqlite3BeginParse(Parse *pParse, int explainFlag){ - pParse->explain = explainFlag; + pParse->explain = (u8)explainFlag; pParse->nVar = 0; } @@ -69,7 +69,7 @@ void sqlite3TableLock( int nBytes; TableLock *p; - if( 0==sqlite3ThreadDataReadOnly()->useSharedData || iDb<0 ){ + if( iDb<0 ){ return; } @@ -82,13 +82,17 @@ void sqlite3TableLock( } nBytes = sizeof(TableLock) * (pParse->nTableLock+1); - pParse->aTableLock = sqliteReallocOrFree(pParse->aTableLock, nBytes); + pParse->aTableLock = + sqlite3DbReallocOrFree(pParse->db, pParse->aTableLock, nBytes); if( pParse->aTableLock ){ p = &pParse->aTableLock[pParse->nTableLock++]; p->iDb = iDb; p->iTab = iTab; p->isWriteLock = isWriteLock; p->zName = zName; + }else{ + pParse->nTableLock = 0; + pParse->db->mallocFailed = 1; } } @@ -99,7 +103,6 @@ void sqlite3TableLock( static void codeTableLocks(Parse *pParse){ int i; Vdbe *pVdbe; - assert( sqlite3ThreadDataReadOnly()->useSharedData || pParse->nTableLock==0 ); if( 0==(pVdbe = sqlite3GetVdbe(pParse)) ){ return; @@ -108,10 +111,8 @@ static void codeTableLocks(Parse *pParse){ for(i=0; inTableLock; i++){ TableLock *p = &pParse->aTableLock[i]; int p1 = p->iDb; - if( p->isWriteLock ){ - p1 = -1*(p1+1); - } - sqlite3VdbeOp3(pVdbe, OP_TableLock, p1, p->iTab, p->zName, P3_STATIC); + sqlite3VdbeAddOp4(pVdbe, OP_TableLock, p1, p->iTab, p->isWriteLock, + p->zName, P4_STATIC); } } #else @@ -132,22 +133,17 @@ void sqlite3FinishCoding(Parse *pParse){ sqlite3 *db; Vdbe *v; - if( sqlite3MallocFailed() ) return; + db = pParse->db; + if( db->mallocFailed ) return; if( pParse->nested ) return; - if( !pParse->pVdbe ){ - if( pParse->rc==SQLITE_OK && pParse->nErr ){ - pParse->rc = SQLITE_ERROR; - return; - } - } + if( pParse->nErr ) return; /* Begin by generating some termination code at the end of the ** vdbe program */ - db = pParse->db; v = sqlite3GetVdbe(pParse); if( v ){ - sqlite3VdbeAddOp(v, OP_Halt, 0, 0); + sqlite3VdbeAddOp0(v, OP_Halt); /* The cookie mask contains one bit for each database file open. ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are @@ -161,13 +157,18 @@ void sqlite3FinishCoding(Parse *pParse){ sqlite3VdbeJumpHere(v, pParse->cookieGoto-1); for(iDb=0, mask=1; iDbnDb; mask<<=1, iDb++){ if( (mask & pParse->cookieMask)==0 ) continue; - sqlite3VdbeAddOp(v, OP_Transaction, iDb, (mask & pParse->writeMask)!=0); - sqlite3VdbeAddOp(v, OP_VerifyCookie, iDb, pParse->cookieValue[iDb]); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v,OP_Transaction, iDb, (mask & pParse->writeMask)!=0); + sqlite3VdbeAddOp2(v,OP_VerifyCookie, iDb, pParse->cookieValue[iDb]); } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pParse->pVirtualLock ){ - char *vtab = (char *)pParse->pVirtualLock->pVtab; - sqlite3VdbeOp3(v, OP_VBegin, 0, 0, vtab, P3_VTAB); + { + int i; + for(i=0; inVtabLock; i++){ + char *vtab = (char *)pParse->apVtabLock[i]->pVtab; + sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); + } + pParse->nVtabLock = 0; } #endif @@ -176,28 +177,32 @@ void sqlite3FinishCoding(Parse *pParse){ ** shared-cache feature is enabled. */ codeTableLocks(pParse); - sqlite3VdbeAddOp(v, OP_Goto, 0, pParse->cookieGoto); + sqlite3VdbeAddOp2(v, OP_Goto, 0, pParse->cookieGoto); } #ifndef SQLITE_OMIT_TRACE - /* Add a No-op that contains the complete text of the compiled SQL - ** statement as its P3 argument. This does not change the functionality - ** of the program. - ** - ** This is used to implement sqlite3_trace(). - */ - sqlite3VdbeOp3(v, OP_Noop, 0, 0, pParse->zSql, pParse->zTail-pParse->zSql); + if( !db->init.busy ){ + /* Change the P4 argument of the first opcode (which will always be + ** an OP_Trace) to be the complete text of the current SQL statement. + */ + VdbeOp *pOp = sqlite3VdbeGetOp(v, 0); + if( pOp && pOp->opcode==OP_Trace ){ + sqlite3VdbeChangeP4(v, 0, pParse->zSql, + (int)(pParse->zTail - pParse->zSql)); + } + } #endif /* SQLITE_OMIT_TRACE */ } /* Get the VDBE program ready for execution */ - if( v && pParse->nErr==0 && !sqlite3MallocFailed() ){ + if( v && pParse->nErr==0 && !db->mallocFailed ){ #ifdef SQLITE_DEBUG FILE *trace = (db->flags & SQLITE_VdbeTrace)!=0 ? stdout : 0; sqlite3VdbeTrace(v, trace); #endif + assert( pParse->disableColCache==0 ); /* Disables and re-enables match */ sqlite3VdbeMakeReady(v, pParse->nVar, pParse->nMem+3, pParse->nTab+3, pParse->explain); pParse->rc = SQLITE_DONE; @@ -228,13 +233,15 @@ void sqlite3FinishCoding(Parse *pParse){ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ va_list ap; char *zSql; + char *zErrMsg = 0; + sqlite3 *db = pParse->db; # define SAVE_SZ (sizeof(Parse) - offsetof(Parse,nVar)) char saveBuf[SAVE_SZ]; if( pParse->nErr ) return; assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ va_start(ap, zFormat); - zSql = sqlite3VMPrintf(zFormat, ap); + zSql = sqlite3VMPrintf(db, zFormat, ap); va_end(ap); if( zSql==0 ){ return; /* A malloc must have failed */ @@ -242,8 +249,9 @@ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ pParse->nested++; memcpy(saveBuf, &pParse->nVar, SAVE_SZ); memset(&pParse->nVar, 0, SAVE_SZ); - sqlite3RunParser(pParse, zSql, 0); - sqliteFree(zSql); + sqlite3RunParser(pParse, zSql, &zErrMsg); + sqlite3DbFree(db, zErrMsg); + sqlite3DbFree(db, zSql); memcpy(&pParse->nVar, saveBuf, SAVE_SZ); pParse->nested--; } @@ -263,11 +271,13 @@ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ Table *p = 0; int i; + int nName; assert( zName!=0 ); + nName = sqlite3Strlen(db, zName) + 1; for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; - p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName, strlen(zName)+1); + p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName, nName); if( p ) break; } return p; @@ -283,7 +293,12 @@ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ ** routine leaves an error message in pParse->zErrMsg where ** sqlite3FindTable() does not. */ -Table *sqlite3LocateTable(Parse *pParse, const char *zName, const char *zDbase){ +Table *sqlite3LocateTable( + Parse *pParse, /* context in which to report errors */ + int isView, /* True if looking for a VIEW rather than a TABLE */ + const char *zName, /* Name of the table we are looking for */ + const char *zDbase /* Name of the database. Might be NULL */ +){ Table *p; /* Read the database schema. If an error occurs, leave an error message @@ -294,10 +309,11 @@ Table *sqlite3LocateTable(Parse *pParse, const char *zName, const char *zDbase){ p = sqlite3FindTable(pParse->db, zName, zDbase); if( p==0 ){ + const char *zMsg = isView ? "no such view" : "no such table"; if( zDbase ){ - sqlite3ErrorMsg(pParse, "no such table: %s.%s", zDbase, zName); + sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); }else{ - sqlite3ErrorMsg(pParse, "no such table: %s", zName); + sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); } pParse->checkSchema = 1; } @@ -319,13 +335,14 @@ Table *sqlite3LocateTable(Parse *pParse, const char *zName, const char *zDbase){ Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ Index *p = 0; int i; + int nName = sqlite3Strlen(db, zName)+1; for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ Schema *pSchema = db->aDb[j].pSchema; if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue; assert( pSchema || (j==1 && !db->aDb[1].pBt) ); if( pSchema ){ - p = sqlite3HashFind(&pSchema->idxHash, zName, strlen(zName)+1); + p = sqlite3HashFind(&pSchema->idxHash, zName, nName); } if( p ) break; } @@ -336,8 +353,9 @@ Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ ** Reclaim the memory used by an index */ static void freeIndex(Index *p){ - sqliteFree(p->zColAff); - sqliteFree(p); + sqlite3 *db = p->pTable->db; + sqlite3DbFree(db, p->zColAff); + sqlite3DbFree(db, p); } /* @@ -352,7 +370,8 @@ static void sqliteDeleteIndex(Index *p){ Index *pOld; const char *zName = p->zName; - pOld = sqlite3HashInsert(&p->pSchema->idxHash, zName, strlen( zName)+1, 0); + pOld = sqlite3HashInsert(&p->pSchema->idxHash, zName, + sqlite3Strlen30(zName)+1, 0); assert( pOld==0 || pOld==p ); freeIndex(p); } @@ -368,7 +387,7 @@ void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){ int len; Hash *pHash = &db->aDb[iDb].pSchema->idxHash; - len = strlen(zIdxName); + len = sqlite3Strlen(db, zIdxName); pIndex = sqlite3HashInsert(pHash, zIdxName, len+1, 0); if( pIndex ){ if( pIndex->pTable->pIndex==pIndex ){ @@ -398,17 +417,22 @@ void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){ */ void sqlite3ResetInternalSchema(sqlite3 *db, int iDb){ int i, j; - assert( iDb>=0 && iDbnDb ); + + if( iDb==0 ){ + sqlite3BtreeEnterAll(db); + } for(i=iDb; inDb; i++){ Db *pDb = &db->aDb[i]; if( pDb->pSchema ){ + assert(i==1 || (pDb->pBt && sqlite3BtreeHoldsMutex(pDb->pBt))); sqlite3SchemaFree(pDb->pSchema); } if( iDb>0 ) return; } assert( iDb==0 ); db->flags &= ~SQLITE_InternChanges; + sqlite3BtreeLeaveAll(db); /* If one or more of the auxiliary database files has been closed, ** then remove them from the auxiliary database list. We take the @@ -426,7 +450,7 @@ void sqlite3ResetInternalSchema(sqlite3 *db, int iDb){ for(i=j=2; inDb; i++){ struct Db *pDb = &db->aDb[i]; if( pDb->pBt==0 ){ - sqliteFree(pDb->zName); + sqlite3DbFree(db, pDb->zName); pDb->zName = 0; continue; } @@ -439,7 +463,7 @@ void sqlite3ResetInternalSchema(sqlite3 *db, int iDb){ db->nDb = j; if( db->nDb<=2 && db->aDb!=db->aDbStatic ){ memcpy(db->aDbStatic, db->aDb, 2*sizeof(db->aDb[0])); - sqliteFree(db->aDb); + sqlite3DbFree(db, db->aDb); db->aDb = db->aDbStatic; } } @@ -457,15 +481,16 @@ void sqlite3CommitInternalChanges(sqlite3 *db){ static void sqliteResetColumnNames(Table *pTable){ int i; Column *pCol; + sqlite3 *db = pTable->db; assert( pTable!=0 ); if( (pCol = pTable->aCol)!=0 ){ for(i=0; inCol; i++, pCol++){ - sqliteFree(pCol->zName); - sqlite3ExprDelete(pCol->pDflt); - sqliteFree(pCol->zType); - sqliteFree(pCol->zColl); + sqlite3DbFree(db, pCol->zName); + sqlite3ExprDelete(db, pCol->pDflt); + sqlite3DbFree(db, pCol->zType); + sqlite3DbFree(db, pCol->zColl); } - sqliteFree(pTable->aCol); + sqlite3DbFree(db, pTable->aCol); } pTable->aCol = 0; pTable->nCol = 0; @@ -484,8 +509,10 @@ static void sqliteResetColumnNames(Table *pTable){ void sqlite3DeleteTable(Table *pTable){ Index *pIndex, *pNext; FKey *pFKey, *pNextFKey; + sqlite3 *db; if( pTable==0 ) return; + db = pTable->db; /* Do not delete the table until the reference count reaches zero. */ pTable->nRef--; @@ -509,22 +536,22 @@ void sqlite3DeleteTable(Table *pTable){ for(pFKey=pTable->pFKey; pFKey; pFKey=pNextFKey){ pNextFKey = pFKey->pNextFrom; assert( sqlite3HashFind(&pTable->pSchema->aFKey, - pFKey->zTo, strlen(pFKey->zTo)+1)!=pFKey ); - sqliteFree(pFKey); + pFKey->zTo, sqlite3Strlen30(pFKey->zTo)+1)!=pFKey ); + sqlite3DbFree(db, pFKey); } #endif /* Delete the Table structure itself. */ sqliteResetColumnNames(pTable); - sqliteFree(pTable->zName); - sqliteFree(pTable->zColAff); - sqlite3SelectDelete(pTable->pSelect); + sqlite3DbFree(db, pTable->zName); + sqlite3DbFree(db, pTable->zColAff); + sqlite3SelectDelete(db, pTable->pSelect); #ifndef SQLITE_OMIT_CHECK - sqlite3ExprDelete(pTable->pCheck); + sqlite3ExprDelete(db, pTable->pCheck); #endif sqlite3VtabClear(pTable); - sqliteFree(pTable); + sqlite3DbFree(db, pTable); } /* @@ -540,11 +567,12 @@ void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){ assert( iDb>=0 && iDbnDb ); assert( zTabName && zTabName[0] ); pDb = &db->aDb[iDb]; - p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, strlen(zTabName)+1,0); + p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, + sqlite3Strlen30(zTabName)+1,0); if( p ){ #ifndef SQLITE_OMIT_FOREIGN_KEY for(pF1=p->pFKey; pF1; pF1=pF1->pNextFrom){ - int nTo = strlen(pF1->zTo) + 1; + int nTo = sqlite3Strlen30(pF1->zTo) + 1; pF2 = sqlite3HashFind(&pDb->pSchema->aFKey, pF1->zTo, nTo); if( pF2==pF1 ){ sqlite3HashInsert(&pDb->pSchema->aFKey, pF1->zTo, nTo, pF1->pNextTo); @@ -571,10 +599,10 @@ void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){ ** are not \000 terminated and are not persistent. The returned string ** is \000 terminated and is persistent. */ -char *sqlite3NameFromToken(Token *pName){ +char *sqlite3NameFromToken(sqlite3 *db, Token *pName){ char *zName; if( pName ){ - zName = sqliteStrNDup((char*)pName->z, pName->n); + zName = sqlite3DbStrNDup(db, (char*)pName->z, pName->n); sqlite3Dequote(zName); }else{ zName = 0; @@ -589,9 +617,8 @@ char *sqlite3NameFromToken(Token *pName){ void sqlite3OpenMasterTable(Parse *p, int iDb){ Vdbe *v = sqlite3GetVdbe(p); sqlite3TableLock(p, iDb, MASTER_ROOT, 1, SCHEMA_TABLE(iDb)); - sqlite3VdbeAddOp(v, OP_Integer, iDb, 0); - sqlite3VdbeAddOp(v, OP_OpenWrite, 0, MASTER_ROOT); - sqlite3VdbeAddOp(v, OP_SetNumColumns, 0, 5); /* sqlite_master has 5 columns */ + sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, 5);/* sqlite_master has 5 columns */ + sqlite3VdbeAddOp3(v, OP_OpenWrite, 0, MASTER_ROOT, iDb); } /* @@ -606,16 +633,16 @@ int sqlite3FindDb(sqlite3 *db, Token *pName){ Db *pDb; /* A database whose name space is being searched */ char *zName; /* Name we are searching for */ - zName = sqlite3NameFromToken(pName); + zName = sqlite3NameFromToken(db, pName); if( zName ){ - n = strlen(zName); + n = sqlite3Strlen30(zName); for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ - if( (!OMIT_TEMPDB || i!=1 ) && n==strlen(pDb->zName) && + if( (!OMIT_TEMPDB || i!=1 ) && n==sqlite3Strlen30(pDb->zName) && 0==sqlite3StrICmp(pDb->zName, zName) ){ break; } } - sqliteFree(zName); + sqlite3DbFree(db, zName); } return i; } @@ -646,7 +673,11 @@ int sqlite3TwoPartName( sqlite3 *db = pParse->db; if( pName2 && pName2->n>0 ){ - assert( !db->init.busy ); + if( db->init.busy ) { + sqlite3ErrorMsg(pParse, "corrupt database"); + pParse->nErr++; + return -1; + } *pUnqual = pName2; iDb = sqlite3FindDb(db, pName1); if( iDb<0 ){ @@ -738,7 +769,7 @@ void sqlite3StartTable( if( !OMIT_TEMPDB && isTemp ) iDb = 1; pParse->sNameToken = *pName; - zName = sqlite3NameFromToken(pName); + zName = sqlite3NameFromToken(db, pName); if( zName==0 ) return; if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto begin_table_error; @@ -795,8 +826,9 @@ void sqlite3StartTable( } } - pTable = sqliteMalloc( sizeof(Table) ); + pTable = sqlite3DbMallocZero(db, sizeof(Table)); if( pTable==0 ){ + db->mallocFailed = 1; pParse->rc = SQLITE_NOMEM; pParse->nErr++; goto begin_table_error; @@ -805,6 +837,7 @@ void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; + pTable->db = db; if( pParse->pNewTable ) sqlite3DeleteTable(pParse->pNewTable); pParse->pNewTable = pTable; @@ -827,29 +860,33 @@ void sqlite3StartTable( ** now. */ if( !db->init.busy && (v = sqlite3GetVdbe(pParse))!=0 ){ - int lbl; + int j1; int fileFormat; + int reg1, reg2, reg3; sqlite3BeginWriteOperation(pParse, 0, iDb); #ifndef SQLITE_OMIT_VIRTUALTABLE if( isVirtual ){ - sqlite3VdbeAddOp(v, OP_VBegin, 0, 0); + sqlite3VdbeAddOp0(v, OP_VBegin); } #endif /* If the file format and encoding in the database have not been set, ** set them now. */ - sqlite3VdbeAddOp(v, OP_ReadCookie, iDb, 1); /* file_format */ - lbl = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp(v, OP_If, 0, lbl); + reg1 = pParse->regRowid = ++pParse->nMem; + reg2 = pParse->regRoot = ++pParse->nMem; + reg3 = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, 1); /* file_format */ + sqlite3VdbeUsesBtree(v, iDb); + j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? 1 : SQLITE_MAX_FILE_FORMAT; - sqlite3VdbeAddOp(v, OP_Integer, fileFormat, 0); - sqlite3VdbeAddOp(v, OP_SetCookie, iDb, 1); - sqlite3VdbeAddOp(v, OP_Integer, ENC(db), 0); - sqlite3VdbeAddOp(v, OP_SetCookie, iDb, 4); - sqlite3VdbeResolveLabel(v, lbl); + sqlite3VdbeAddOp2(v, OP_Integer, fileFormat, reg3); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, 1, reg3); + sqlite3VdbeAddOp2(v, OP_Integer, ENC(db), reg3); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, 4, reg3); + sqlite3VdbeJumpHere(v, j1); /* This just creates a place-holder record in the sqlite_master table. ** The record created does not contain anything yet. It will be replaced @@ -861,19 +898,18 @@ void sqlite3StartTable( */ #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) if( isView || isVirtual ){ - sqlite3VdbeAddOp(v, OP_Integer, 0, 0); + sqlite3VdbeAddOp2(v, OP_Integer, 0, reg2); }else #endif { - sqlite3VdbeAddOp(v, OP_CreateTable, iDb, 0); + sqlite3VdbeAddOp2(v, OP_CreateTable, iDb, reg2); } sqlite3OpenMasterTable(pParse, iDb); - sqlite3VdbeAddOp(v, OP_NewRowid, 0, 0); - sqlite3VdbeAddOp(v, OP_Dup, 0, 0); - sqlite3VdbeAddOp(v, OP_Null, 0, 0); - sqlite3VdbeAddOp(v, OP_Insert, 0, OPFLAG_APPEND); - sqlite3VdbeAddOp(v, OP_Close, 0, 0); - sqlite3VdbeAddOp(v, OP_Pull, 1, 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); + sqlite3VdbeAddOp2(v, OP_Null, 0, reg3); + sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3VdbeAddOp0(v, OP_Close); } /* Normal (non-error) return. */ @@ -881,7 +917,7 @@ void sqlite3StartTable( /* If an error occurs, we jump here */ begin_table_error: - sqliteFree(zName); + sqlite3DbFree(db, zName); return; } @@ -911,25 +947,28 @@ void sqlite3AddColumn(Parse *pParse, Token *pName){ int i; char *z; Column *pCol; + sqlite3 *db = pParse->db; if( (p = pParse->pNewTable)==0 ) return; - if( p->nCol+1>SQLITE_MAX_COLUMN ){ +#if SQLITE_MAX_COLUMN + if( p->nCol+1>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName); return; } - z = sqlite3NameFromToken(pName); +#endif + z = sqlite3NameFromToken(db, pName); if( z==0 ) return; for(i=0; inCol; i++){ if( STRICMP(z, p->aCol[i].zName) ){ sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); - sqliteFree(z); + sqlite3DbFree(db, z); return; } } if( (p->nCol & 0x7)==0 ){ Column *aNew; - aNew = sqliteRealloc( p->aCol, (p->nCol+8)*sizeof(p->aCol[0])); + aNew = sqlite3DbRealloc(db,p->aCol,(p->nCol+8)*sizeof(p->aCol[0])); if( aNew==0 ){ - sqliteFree(z); + sqlite3DbFree(db, z); return; } p->aCol = aNew; @@ -957,7 +996,7 @@ void sqlite3AddNotNull(Parse *pParse, int onError){ int i; if( (p = pParse->pNewTable)==0 ) return; i = p->nCol-1; - if( i>=0 ) p->aCol[i].notNull = onError; + if( i>=0 ) p->aCol[i].notNull = (u8)onError; } /* @@ -1036,13 +1075,15 @@ void sqlite3AddColumnType(Parse *pParse, Token *pType){ Table *p; int i; Column *pCol; + sqlite3 *db; if( (p = pParse->pNewTable)==0 ) return; i = p->nCol-1; if( i<0 ) return; pCol = &p->aCol[i]; - sqliteFree(pCol->zType); - pCol->zType = sqlite3NameFromToken(pType); + db = pParse->db; + sqlite3DbFree(db, pCol->zType); + pCol->zType = sqlite3NameFromToken(db, pType); pCol->affinity = sqlite3AffinityType(pType); } @@ -1059,6 +1100,7 @@ void sqlite3AddColumnType(Parse *pParse, Token *pType){ void sqlite3AddDefaultValue(Parse *pParse, Expr *pExpr){ Table *p; Column *pCol; + sqlite3 *db = pParse->db; if( (p = pParse->pNewTable)!=0 ){ pCol = &(p->aCol[p->nCol-1]); if( !sqlite3ExprIsConstantOrFunction(pExpr) ){ @@ -1066,14 +1108,14 @@ void sqlite3AddDefaultValue(Parse *pParse, Expr *pExpr){ pCol->zName); }else{ Expr *pCopy; - sqlite3ExprDelete(pCol->pDflt); - pCol->pDflt = pCopy = sqlite3ExprDup(pExpr); + sqlite3ExprDelete(db, pCol->pDflt); + pCol->pDflt = pCopy = sqlite3ExprDup(db, pExpr); if( pCopy ){ - sqlite3TokenCopy(&pCopy->span, &pExpr->span); + sqlite3TokenCopy(db, &pCopy->span, &pExpr->span); } } } - sqlite3ExprDelete(pExpr); + sqlite3ExprDelete(db, pExpr); } /* @@ -1105,12 +1147,12 @@ void sqlite3AddPrimaryKey( char *zType = 0; int iCol = -1, i; if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; - if( pTab->hasPrimKey ){ + if( pTab->tabFlags & TF_HasPrimaryKey ){ sqlite3ErrorMsg(pParse, "table \"%s\" has more than one primary key", pTab->zName); goto primary_key_exit; } - pTab->hasPrimKey = 1; + pTab->tabFlags |= TF_HasPrimaryKey; if( pList==0 ){ iCol = pTab->nCol - 1; pTab->aCol[iCol].isPrimKey = 1; @@ -1133,8 +1175,9 @@ void sqlite3AddPrimaryKey( if( zType && sqlite3StrICmp(zType, "INTEGER")==0 && sortOrder==SQLITE_SO_ASC ){ pTab->iPKey = iCol; - pTab->keyConf = onError; - pTab->autoInc = autoInc; + pTab->keyConf = (u8)onError; + assert( autoInc==0 || autoInc==1 ); + pTab->tabFlags |= autoInc*TF_Autoincrement; }else if( autoInc ){ #ifndef SQLITE_OMIT_AUTOINCREMENT sqlite3ErrorMsg(pParse, "AUTOINCREMENT is only allowed on an " @@ -1146,7 +1189,7 @@ void sqlite3AddPrimaryKey( } primary_key_exit: - sqlite3ExprListDelete(pList); + sqlite3ExprListDelete(pParse->db, pList); return; } @@ -1157,32 +1200,39 @@ void sqlite3AddCheckConstraint( Parse *pParse, /* Parsing context */ Expr *pCheckExpr /* The check expression */ ){ + sqlite3 *db = pParse->db; #ifndef SQLITE_OMIT_CHECK Table *pTab = pParse->pNewTable; if( pTab && !IN_DECLARE_VTAB ){ /* The CHECK expression must be duplicated so that tokens refer ** to malloced space and not the (ephemeral) text of the CREATE TABLE ** statement */ - pTab->pCheck = sqlite3ExprAnd(pTab->pCheck, sqlite3ExprDup(pCheckExpr)); + pTab->pCheck = sqlite3ExprAnd(db, pTab->pCheck, + sqlite3ExprDup(db, pCheckExpr)); } #endif - sqlite3ExprDelete(pCheckExpr); + sqlite3ExprDelete(db, pCheckExpr); } /* ** Set the collation function of the most recently parsed table column ** to the CollSeq given. */ -void sqlite3AddCollateType(Parse *pParse, const char *zType, int nType){ +void sqlite3AddCollateType(Parse *pParse, Token *pToken){ Table *p; int i; + char *zColl; /* Dequoted name of collation sequence */ + sqlite3 *db; if( (p = pParse->pNewTable)==0 ) return; i = p->nCol-1; + db = pParse->db; + zColl = sqlite3NameFromToken(db, pToken); + if( !zColl ) return; - if( sqlite3LocateCollSeq(pParse, zType, nType) ){ + if( sqlite3LocateCollSeq(pParse, zColl, -1) ){ Index *pIdx; - p->aCol[i].zColl = sqliteStrNDup(zType, nType); + p->aCol[i].zColl = zColl; /* If the column is declared as " PRIMARY KEY COLLATE ", ** then an index may have been created on this column before the @@ -1194,6 +1244,8 @@ void sqlite3AddCollateType(Parse *pParse, const char *zType, int nType){ pIdx->azColl[0] = p->aCol[i].zColl; } } + }else{ + sqlite3DbFree(db, zColl); } } @@ -1226,7 +1278,7 @@ CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName, int nName){ pColl = sqlite3GetCollSeq(db, pColl, zName, nName); if( !pColl ){ if( nName<0 ){ - nName = strlen(zName); + nName = sqlite3Strlen(db, zName); } sqlite3ErrorMsg(pParse, "no such collation sequence: %.*s", nName, zName); pColl = 0; @@ -1253,9 +1305,13 @@ CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName, int nName){ ** and the probability of hitting the same cookie value is only ** 1 chance in 2^32. So we're safe enough. */ -void sqlite3ChangeCookie(sqlite3 *db, Vdbe *v, int iDb){ - sqlite3VdbeAddOp(v, OP_Integer, db->aDb[iDb].pSchema->schema_cookie+1, 0); - sqlite3VdbeAddOp(v, OP_SetCookie, iDb, 0); +void sqlite3ChangeCookie(Parse *pParse, int iDb){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + sqlite3VdbeAddOp2(v, OP_Integer, db->aDb[iDb].pSchema->schema_cookie+1, r1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, 0, r1); + sqlite3ReleaseTempReg(pParse, r1); } /* @@ -1302,7 +1358,7 @@ static void identPut(char *z, int *pIdx, char *zSignedIdent){ ** table. Memory to hold the text of the statement is obtained ** from sqliteMalloc() and must be freed by the calling function. */ -static char *createTableStmt(Table *p, int isTemp){ +static char *createTableStmt(sqlite3 *db, Table *p, int isTemp){ int i, k, n; char *zStmt; char *zSep, *zSep2, *zEnd, *z; @@ -1312,7 +1368,7 @@ static char *createTableStmt(Table *p, int isTemp){ n += identLength(pCol->zName); z = pCol->zType; if( z ){ - n += (strlen(z) + 1); + n += (sqlite3Strlen30(z) + 1); } } n += identLength(p->zName); @@ -1326,23 +1382,26 @@ static char *createTableStmt(Table *p, int isTemp){ zEnd = "\n)"; } n += 35 + 6*p->nCol; - zStmt = sqliteMallocRaw( n ); - if( zStmt==0 ) return 0; + zStmt = sqlite3Malloc( n ); + if( zStmt==0 ){ + db->mallocFailed = 1; + return 0; + } sqlite3_snprintf(n, zStmt, !OMIT_TEMPDB&&isTemp ? "CREATE TEMP TABLE ":"CREATE TABLE "); - k = strlen(zStmt); + k = sqlite3Strlen30(zStmt); identPut(zStmt, &k, p->zName); zStmt[k++] = '('; for(pCol=p->aCol, i=0; inCol; i++, pCol++){ sqlite3_snprintf(n-k, &zStmt[k], zSep); - k += strlen(&zStmt[k]); + k += sqlite3Strlen30(&zStmt[k]); zSep = zSep2; identPut(zStmt, &k, pCol->zName); if( (z = pCol->zType)!=0 ){ zStmt[k++] = ' '; - assert( strlen(z)+k+1<=n ); + assert( (int)(sqlite3Strlen30(z)+k+1)<=n ); sqlite3_snprintf(n-k, &zStmt[k], "%s", z); - k += strlen(z); + k += sqlite3Strlen30(z); } } sqlite3_snprintf(n-k, &zStmt[k], "%s", zEnd); @@ -1379,7 +1438,7 @@ void sqlite3EndTable( sqlite3 *db = pParse->db; int iDb; - if( (pEnd==0 && pSelect==0) || pParse->nErr || sqlite3MallocFailed() ) { + if( (pEnd==0 && pSelect==0) || pParse->nErr || db->mallocFailed ) { return; } p = pParse->pNewTable; @@ -1405,7 +1464,7 @@ void sqlite3EndTable( sNC.pParse = pParse; sNC.pSrcList = &sSrc; sNC.isCheck = 1; - if( sqlite3ExprResolveNames(&sNC, p->pCheck) ){ + if( sqlite3ResolveExprNames(&sNC, p->pCheck) ){ return; } } @@ -1438,7 +1497,7 @@ void sqlite3EndTable( v = sqlite3GetVdbe(pParse); if( v==0 ) return; - sqlite3VdbeAddOp(v, OP_Close, 0, 0); + sqlite3VdbeAddOp1(v, OP_Close, 0); /* Create the rootpage for the new table and push it onto the stack. ** A view has no rootpage, so just push a zero onto the stack for @@ -1470,15 +1529,18 @@ void sqlite3EndTable( ** be redundant. */ if( pSelect ){ + SelectDest dest; Table *pSelTab; - sqlite3VdbeAddOp(v, OP_Dup, 0, 0); - sqlite3VdbeAddOp(v, OP_Integer, iDb, 0); - sqlite3VdbeAddOp(v, OP_OpenWrite, 1, 0); + + assert(pParse->nTab==0); + sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); + sqlite3VdbeChangeP5(v, 1); pParse->nTab = 2; - sqlite3Select(pParse, pSelect, SRT_Table, 1, 0, 0, 0, 0); - sqlite3VdbeAddOp(v, OP_Close, 1, 0); + sqlite3SelectDestInit(&dest, SRT_Table, 1); + sqlite3Select(pParse, pSelect, &dest); + sqlite3VdbeAddOp1(v, OP_Close, 1); if( pParse->nErr==0 ){ - pSelTab = sqlite3ResultSetOfSelect(pParse, 0, pSelect); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); if( pSelTab==0 ) return; assert( p->aCol==0 ); p->nCol = pSelTab->nCol; @@ -1491,10 +1553,12 @@ void sqlite3EndTable( /* Compute the complete text of the CREATE statement */ if( pSelect ){ - zStmt = createTableStmt(p, p->pSchema==pParse->db->aDb[1].pSchema); + zStmt = createTableStmt(db, p, p->pSchema==db->aDb[1].pSchema); }else{ - n = pEnd->z - pParse->sNameToken.z + 1; - zStmt = sqlite3MPrintf("CREATE %s %.*s", zType2, n, pParse->sNameToken.z); + n = (int)(pEnd->z - pParse->sNameToken.z) + 1; + zStmt = sqlite3MPrintf(db, + "CREATE %s %.*s", zType2, n, pParse->sNameToken.z + ); } /* A slot for the record has already been allocated in the @@ -1505,22 +1569,24 @@ void sqlite3EndTable( */ sqlite3NestedParse(pParse, "UPDATE %Q.%s " - "SET type='%s', name=%Q, tbl_name=%Q, rootpage=#0, sql=%Q " - "WHERE rowid=#1", + "SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q " + "WHERE rowid=#%d", db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zType, p->zName, p->zName, - zStmt + pParse->regRoot, + zStmt, + pParse->regRowid ); - sqliteFree(zStmt); - sqlite3ChangeCookie(db, v, iDb); + sqlite3DbFree(db, zStmt); + sqlite3ChangeCookie(pParse, iDb); #ifndef SQLITE_OMIT_AUTOINCREMENT /* Check to see if we need to create an sqlite_sequence table for ** keeping track of autoincrement keys. */ - if( p->autoInc ){ + if( p->tabFlags & TF_Autoincrement ){ Db *pDb = &db->aDb[iDb]; if( pDb->pSchema->pSeqTab==0 ){ sqlite3NestedParse(pParse, @@ -1532,8 +1598,8 @@ void sqlite3EndTable( #endif /* Reparse everything to update our internal data structures */ - sqlite3VdbeOp3(v, OP_ParseSchema, iDb, 0, - sqlite3MPrintf("tbl_name='%q'",p->zName), P3_DYNAMIC); + sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, + sqlite3MPrintf(db, "tbl_name='%q'",p->zName), P4_DYNAMIC); } @@ -1543,16 +1609,22 @@ void sqlite3EndTable( Table *pOld; FKey *pFKey; Schema *pSchema = p->pSchema; - pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, strlen(p->zName)+1,p); + pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, + sqlite3Strlen30(p->zName)+1,p); if( pOld ){ assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ + db->mallocFailed = 1; return; } #ifndef SQLITE_OMIT_FOREIGN_KEY for(pFKey=p->pFKey; pFKey; pFKey=pFKey->pNextFrom){ - int nTo = strlen(pFKey->zTo) + 1; + void *data; + int nTo = sqlite3Strlen30(pFKey->zTo) + 1; pFKey->pNextTo = sqlite3HashFind(&pSchema->aFKey, pFKey->zTo, nTo); - sqlite3HashInsert(&pSchema->aFKey, pFKey->zTo, nTo, pFKey); + data = sqlite3HashInsert(&pSchema->aFKey, pFKey->zTo, nTo, pFKey); + if( data==(void *)pFKey ){ + db->mallocFailed = 1; + } } #endif pParse->pNewTable = 0; @@ -1567,7 +1639,7 @@ void sqlite3EndTable( if( pCons->z==0 ){ pCons = pEnd; } - nName = (const char *)pCons->z - zName; + nName = (int)((const char *)pCons->z - zName); p->addColOffset = 13 + sqlite3Utf8CharLen(zName, nName); } #endif @@ -1594,24 +1666,25 @@ void sqlite3CreateView( DbFixer sFix; Token *pName; int iDb; + sqlite3 *db = pParse->db; if( pParse->nVar>0 ){ sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); - sqlite3SelectDelete(pSelect); + sqlite3SelectDelete(db, pSelect); return; } sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); p = pParse->pNewTable; if( p==0 || pParse->nErr ){ - sqlite3SelectDelete(pSelect); + sqlite3SelectDelete(db, pSelect); return; } sqlite3TwoPartName(pParse, pName1, pName2, &pName); - iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + iDb = sqlite3SchemaToIndex(db, p->pSchema); if( sqlite3FixInit(&sFix, pParse, iDb, "view", pName) && sqlite3FixSelect(&sFix, pSelect) ){ - sqlite3SelectDelete(pSelect); + sqlite3SelectDelete(db, pSelect); return; } @@ -1620,12 +1693,12 @@ void sqlite3CreateView( ** allocated rather than point to the input string - which means that ** they will persist after the current sqlite3_exec() call returns. */ - p->pSelect = sqlite3SelectDup(pSelect); - sqlite3SelectDelete(pSelect); - if( sqlite3MallocFailed() ){ + p->pSelect = sqlite3SelectDup(db, pSelect); + sqlite3SelectDelete(db, pSelect); + if( db->mallocFailed ){ return; } - if( !pParse->db->init.busy ){ + if( !db->init.busy ){ sqlite3ViewGetColumnNames(pParse, p); } @@ -1637,7 +1710,7 @@ void sqlite3CreateView( sEnd.z += sEnd.n; } sEnd.n = 0; - n = sEnd.z - pBegin->z; + n = (int)(sEnd.z - pBegin->z); z = (const unsigned char*)pBegin->z; while( n>0 && (z[n-1]==';' || isspace(z[n-1])) ){ n--; } sEnd.z = &z[n-1]; @@ -1660,6 +1733,8 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ Select *pSel; /* Copy of the SELECT that implements the view */ int nErr = 0; /* Number of errors encountered */ int n; /* Temporarily holds the number of cursors assigned */ + sqlite3 *db = pParse->db; /* Database connection for malloc errors */ + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); assert( pTable ); @@ -1700,12 +1775,19 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ ** statement that defines the view. */ assert( pTable->pSelect ); - pSel = sqlite3SelectDup(pTable->pSelect); + pSel = sqlite3SelectDup(db, pTable->pSelect); if( pSel ){ n = pParse->nTab; sqlite3SrcListAssignCursors(pParse, pSel->pSrc); pTable->nCol = -1; - pSelTab = sqlite3ResultSetOfSelect(pParse, 0, pSel); +#ifndef SQLITE_OMIT_AUTHORIZATION + xAuth = db->xAuth; + db->xAuth = 0; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + db->xAuth = xAuth; +#else + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); +#endif pParse->nTab = n; if( pSelTab ){ assert( pTable->aCol==0 ); @@ -1719,7 +1801,7 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ pTable->nCol = 0; nErr++; } - sqlite3SelectDelete(pSel); + sqlite3SelectDelete(db, pSel); } else { nErr++; } @@ -1794,20 +1876,22 @@ void sqlite3RootPageMoved(Db *pDb, int iFrom, int iTo){ */ static void destroyRootPage(Parse *pParse, int iTable, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); - sqlite3VdbeAddOp(v, OP_Destroy, iTable, iDb); + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); #ifndef SQLITE_OMIT_AUTOVACUUM - /* OP_Destroy pushes an integer onto the stack. If this integer + /* OP_Destroy stores an in integer r1. If this integer ** is non-zero, then it is the root page number of a table moved to ** location iTable. The following code modifies the sqlite_master table to ** reflect this. ** - ** The "#0" in the SQL is a special constant that means whatever value + ** The "#%d" in the SQL is a special constant that means whatever value ** is on the top of the stack. See sqlite3RegisterExpr(). */ sqlite3NestedParse(pParse, - "UPDATE %Q.%s SET rootpage=%d WHERE #0 AND rootpage=#0", - pParse->db->aDb[iDb].zName, SCHEMA_TABLE(iDb), iTable); + "UPDATE %Q.%s SET rootpage=%d WHERE #%d AND rootpage=#%d", + pParse->db->aDb[iDb].zName, SCHEMA_TABLE(iDb), iTable, r1, r1); #endif + sqlite3ReleaseTempReg(pParse, r1); } /* @@ -1879,11 +1963,12 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ sqlite3 *db = pParse->db; int iDb; - if( pParse->nErr || sqlite3MallocFailed() ){ + if( pParse->nErr || db->mallocFailed ){ goto exit_drop_table; } assert( pName->nSrc==1 ); - pTab = sqlite3LocateTable(pParse, pName->a[0].zName, pName->a[0].zDatabase); + pTab = sqlite3LocateTable(pParse, isView, + pName->a[0].zName, pName->a[0].zDatabase); if( pTab==0 ){ if( noErr ){ @@ -1893,6 +1978,13 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDb>=0 && iDbnDb ); + + /* If pTab is a virtual table, call ViewGetColumnNames() to ensure + ** it is initialized. + */ + if( IsVirtual(pTab) && sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto exit_drop_table; + } #ifndef SQLITE_OMIT_AUTHORIZATION { int code; @@ -1910,9 +2002,6 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ } #ifndef SQLITE_OMIT_VIRTUALTABLE }else if( IsVirtual(pTab) ){ - if( sqlite3ViewGetColumnNames(pParse, pTab) ){ - goto exit_drop_table; - } code = SQLITE_DROP_VTABLE; zArg2 = pTab->pMod->zName; #endif @@ -1931,7 +2020,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ } } #endif - if( pTab->readOnly || pTab==db->aDb[iDb].pSchema->pSeqTab ){ + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ sqlite3ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); goto exit_drop_table; } @@ -1957,13 +2046,12 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ if( v ){ Trigger *pTrigger; Db *pDb = &db->aDb[iDb]; - sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3BeginWriteOperation(pParse, 1, iDb); #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pTab) ){ - Vdbe *v = sqlite3GetVdbe(pParse); if( v ){ - sqlite3VdbeAddOp(v, OP_VBegin, 0, 0); + sqlite3VdbeAddOp0(v, OP_VBegin); } } #endif @@ -1986,7 +2074,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ ** at the btree level, in case the sqlite_sequence table needs to ** move as a result of the drop (can happen in auto-vacuum mode). */ - if( pTab->autoInc ){ + if( pTab->tabFlags & TF_Autoincrement ){ sqlite3NestedParse(pParse, "DELETE FROM %s.sqlite_sequence WHERE name=%Q", pDb->zName, pTab->zName @@ -2004,6 +2092,14 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); + + /* Drop any statistics from the sqlite_stat1 table, if it exists */ + if( sqlite3FindTable(db, "sqlite_stat1", db->aDb[iDb].zName) ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.sqlite_stat1 WHERE tbl=%Q", pDb->zName, pTab->zName + ); + } + if( !isView && !IsVirtual(pTab) ){ destroyTable(pParse, pTab); } @@ -2012,15 +2108,15 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ ** the schema cookie. */ if( IsVirtual(pTab) ){ - sqlite3VdbeOp3(v, OP_VDestroy, iDb, 0, pTab->zName, 0); + sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0); } - sqlite3VdbeOp3(v, OP_DropTable, iDb, 0, pTab->zName, 0); - sqlite3ChangeCookie(db, v, iDb); + sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); + sqlite3ChangeCookie(pParse, iDb); } sqliteViewResetAll(db, iDb); exit_drop_table: - sqlite3SrcListDelete(pName); + sqlite3SrcListDelete(db, pName); } /* @@ -2048,6 +2144,7 @@ void sqlite3CreateForeignKey( ExprList *pToCol, /* Columns in the other table */ int flags /* Conflict resolution algorithms. */ ){ + sqlite3 *db = pParse->db; #ifndef SQLITE_OMIT_FOREIGN_KEY FKey *pFKey = 0; Table *p = pParse->pNewTable; @@ -2079,11 +2176,13 @@ void sqlite3CreateForeignKey( nByte = sizeof(*pFKey) + nCol*sizeof(pFKey->aCol[0]) + pTo->n + 1; if( pToCol ){ for(i=0; inExpr; i++){ - nByte += strlen(pToCol->a[i].zName) + 1; + nByte += sqlite3Strlen30(pToCol->a[i].zName) + 1; } } - pFKey = sqliteMalloc( nByte ); - if( pFKey==0 ) goto fk_end; + pFKey = sqlite3DbMallocZero(db, nByte ); + if( pFKey==0 ){ + goto fk_end; + } pFKey->pFrom = p; pFKey->pNextFrom = p->pFKey; z = (char*)&pFKey[1]; @@ -2116,7 +2215,7 @@ void sqlite3CreateForeignKey( } if( pToCol ){ for(i=0; ia[i].zName); + int n = sqlite3Strlen30(pToCol->a[i].zName); pFKey->aCol[i].zCol = z; memcpy(z, pToCol->a[i].zName, n); z[n] = 0; @@ -2124,9 +2223,9 @@ void sqlite3CreateForeignKey( } } pFKey->isDeferred = 0; - pFKey->deleteConf = flags & 0xff; - pFKey->updateConf = (flags >> 8 ) & 0xff; - pFKey->insertConf = (flags >> 16 ) & 0xff; + pFKey->deleteConf = (u8)(flags & 0xff); + pFKey->updateConf = (u8)((flags >> 8 ) & 0xff); + pFKey->insertConf = (u8)((flags >> 16 ) & 0xff); /* Link the foreign key to the table as the last step. */ @@ -2134,10 +2233,10 @@ void sqlite3CreateForeignKey( pFKey = 0; fk_end: - sqliteFree(pFKey); + sqlite3DbFree(db, pFKey); #endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ - sqlite3ExprListDelete(pFromCol); - sqlite3ExprListDelete(pToCol); + sqlite3ExprListDelete(db, pFromCol); + sqlite3ExprListDelete(db, pToCol); } /* @@ -2152,7 +2251,8 @@ void sqlite3DeferForeignKey(Parse *pParse, int isDeferred){ Table *pTab; FKey *pFKey; if( (pTab = pParse->pNewTable)==0 || (pFKey = pTab->pFKey)==0 ) return; - pFKey->isDeferred = isDeferred; + assert( isDeferred==0 || isDeferred==1 ); + pFKey->isDeferred = (u8)isDeferred; #endif } @@ -2162,7 +2262,7 @@ void sqlite3DeferForeignKey(Parse *pParse, int isDeferred){ ** content of an index in response to a REINDEX command. ** ** if memRootPage is not negative, it means that the index is newly -** created. The memory cell specified by memRootPage contains the +** created. The register specified by memRootPage contains the ** root page number of the index. If memRootPage is negative, then ** the index already exists and must be cleared before being refilled and ** the root page number of the index is taken from pIndex->tnum. @@ -2175,11 +2275,14 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int tnum; /* Root page of index */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ - int iDb = sqlite3SchemaToIndex(pParse->db, pIndex->pSchema); + int regIdxKey; /* Registers containing the index key */ + int regRecord; /* Register holding assemblied index record */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, - pParse->db->aDb[iDb].zName ) ){ + db->aDb[iDb].zName ) ){ return; } #endif @@ -2190,34 +2293,40 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ v = sqlite3GetVdbe(pParse); if( v==0 ) return; if( memRootPage>=0 ){ - sqlite3VdbeAddOp(v, OP_MemLoad, memRootPage, 0); - tnum = 0; + tnum = memRootPage; }else{ tnum = pIndex->tnum; - sqlite3VdbeAddOp(v, OP_Clear, tnum, iDb); + sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); } - sqlite3VdbeAddOp(v, OP_Integer, iDb, 0); pKey = sqlite3IndexKeyinfo(pParse, pIndex); - sqlite3VdbeOp3(v, OP_OpenWrite, iIdx, tnum, (char *)pKey, P3_KEYINFO_HANDOFF); - sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); - addr1 = sqlite3VdbeAddOp(v, OP_Rewind, iTab, 0); - sqlite3GenerateIndexKey(v, pIndex, iTab); - if( pIndex->onError!=OE_None ){ - int curaddr = sqlite3VdbeCurrentAddr(v); - int addr2 = curaddr+4; - sqlite3VdbeChangeP2(v, curaddr-1, addr2); - sqlite3VdbeAddOp(v, OP_Rowid, iTab, 0); - sqlite3VdbeAddOp(v, OP_AddImm, 1, 0); - sqlite3VdbeAddOp(v, OP_IsUnique, iIdx, addr2); - sqlite3VdbeOp3(v, OP_Halt, SQLITE_CONSTRAINT, OE_Abort, - "indexed columns are not unique", P3_STATIC); - assert( sqlite3MallocFailed() || addr2==sqlite3VdbeCurrentAddr(v) ); + sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, tnum, iDb, + (char *)pKey, P4_KEYINFO_HANDOFF); + if( memRootPage>=0 ){ + sqlite3VdbeChangeP5(v, 1); } - sqlite3VdbeAddOp(v, OP_IdxInsert, iIdx, 0); - sqlite3VdbeAddOp(v, OP_Next, iTab, addr1+1); + sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); + regRecord = sqlite3GetTempReg(pParse); + regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); + if( pIndex->onError!=OE_None ){ + int j1, j2; + int regRowid; + + regRowid = regIdxKey + pIndex->nColumn; + j1 = sqlite3VdbeAddOp3(v, OP_IsNull, regIdxKey, 0, pIndex->nColumn); + j2 = sqlite3VdbeAddOp4(v, OP_IsUnique, iIdx, + 0, regRowid, SQLITE_INT_TO_PTR(regRecord), P4_INT32); + sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, OE_Abort, 0, + "indexed columns are not unique", P4_STATIC); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, j2); + } + sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdx, regRecord); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp(v, OP_Close, iTab, 0); - sqlite3VdbeAddOp(v, OP_Close, iIdx, 0); + sqlite3VdbeAddOp1(v, OP_Close, iTab); + sqlite3VdbeAddOp1(v, OP_Close, iIdx); } /* @@ -2261,7 +2370,7 @@ void sqlite3CreateIndex( int nExtra = 0; char *zExtra; - if( pParse->nErr || sqlite3MallocFailed() || IN_DECLARE_VTAB ){ + if( pParse->nErr || db->mallocFailed || IN_DECLARE_VTAB ){ goto exit_create_index; } @@ -2280,11 +2389,14 @@ void sqlite3CreateIndex( #ifndef SQLITE_OMIT_TEMPDB /* If the index name was unqualified, check if the the table - ** is a temp table. If so, set the database to 1. + ** is a temp table. If so, set the database to 1. Do not do this + ** if initialising a database schema. */ - pTab = sqlite3SrcListLookup(pParse, pTblName); - if( pName2 && pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ - iDb = 1; + if( !db->init.busy ){ + pTab = sqlite3SrcListLookup(pParse, pTblName); + if( pName2 && pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ + iDb = 1; + } } #endif @@ -2295,9 +2407,9 @@ void sqlite3CreateIndex( ** sqlite3FixSrcList can never fail. */ assert(0); } - pTab = sqlite3LocateTable(pParse, pTblName->a[0].zName, + pTab = sqlite3LocateTable(pParse, 0, pTblName->a[0].zName, pTblName->a[0].zDatabase); - if( !pTab ) goto exit_create_index; + if( !pTab || db->mallocFailed ) goto exit_create_index; assert( db->aDb[iDb].pSchema==pTab->pSchema ); }else{ assert( pName==0 ); @@ -2308,7 +2420,7 @@ void sqlite3CreateIndex( pDb = &db->aDb[iDb]; if( pTab==0 || pParse->nErr ) goto exit_create_index; - if( pTab->readOnly ){ + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; } @@ -2339,7 +2451,7 @@ void sqlite3CreateIndex( ** own name. */ if( pName ){ - zName = sqlite3NameFromToken(pName); + zName = sqlite3NameFromToken(db, pName); if( SQLITE_OK!=sqlite3ReadSchema(pParse) ) goto exit_create_index; if( zName==0 ) goto exit_create_index; if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ @@ -2359,14 +2471,13 @@ void sqlite3CreateIndex( goto exit_create_index; } }else{ - char zBuf[30]; int n; Index *pLoop; for(pLoop=pTab->pIndex, n=1; pLoop; pLoop=pLoop->pNext, n++){} - sqlite3_snprintf(sizeof(zBuf),zBuf,"_%d",n); - zName = 0; - sqlite3SetString(&zName, "sqlite_autoindex_", pTab->zName, zBuf, (char*)0); - if( zName==0 ) goto exit_create_index; + zName = sqlite3MPrintf(db, "sqlite_autoindex_%s_%d", pTab->zName, n); + if( zName==0 ){ + goto exit_create_index; + } } /* Check for authorization to create an index. @@ -2391,28 +2502,29 @@ void sqlite3CreateIndex( */ if( pList==0 ){ nullId.z = (u8*)pTab->aCol[pTab->nCol-1].zName; - nullId.n = strlen((char*)nullId.z); - pList = sqlite3ExprListAppend(0, 0, &nullId); + nullId.n = sqlite3Strlen30((char*)nullId.z); + pList = sqlite3ExprListAppend(pParse, 0, 0, &nullId); if( pList==0 ) goto exit_create_index; - pList->a[0].sortOrder = sortOrder; + pList->a[0].sortOrder = (u8)sortOrder; } /* Figure out how many bytes of space are required to store explicitly ** specified collation sequence names. */ for(i=0; inExpr; i++){ - Expr *pExpr = pList->a[i].pExpr; - if( pExpr ){ - nExtra += (1 + strlen(pExpr->pColl->zName)); + Expr *pExpr; + CollSeq *pColl; + if( (pExpr = pList->a[i].pExpr)!=0 && (pColl = pExpr->pColl)!=0 ){ + nExtra += (1 + sqlite3Strlen30(pColl->zName)); } } /* ** Allocate the index structure. */ - nName = strlen(zName); + nName = sqlite3Strlen30(zName); nCol = pList->nExpr; - pIndex = sqliteMalloc( + pIndex = sqlite3DbMallocZero(db, sizeof(Index) + /* Index structure */ sizeof(int)*nCol + /* Index.aiColumn */ sizeof(int)*(nCol+1) + /* Index.aiRowEst */ @@ -2421,7 +2533,9 @@ void sqlite3CreateIndex( nName + 1 + /* Index.zName */ nExtra /* Collation sequence names */ ); - if( sqlite3MallocFailed() ) goto exit_create_index; + if( db->mallocFailed ){ + goto exit_create_index; + } pIndex->azColl = (char**)(&pIndex[1]); pIndex->aiColumn = (int *)(&pIndex->azColl[nCol]); pIndex->aiRowEst = (unsigned *)(&pIndex->aiColumn[nCol]); @@ -2431,8 +2545,8 @@ void sqlite3CreateIndex( memcpy(pIndex->zName, zName, nName+1); pIndex->pTable = pTab; pIndex->nColumn = pList->nExpr; - pIndex->onError = onError; - pIndex->autoIndex = pName==0; + pIndex->onError = (u8)onError; + pIndex->autoIndex = (u8)(pName==0); pIndex->pSchema = db->aDb[iDb].pSchema; /* Check to see if we should honor DESC requests on index columns @@ -2468,11 +2582,11 @@ void sqlite3CreateIndex( ** break backwards compatibility - it needs to be a warning. */ pIndex->aiColumn[i] = j; - if( pListItem->pExpr ){ + if( pListItem->pExpr && pListItem->pExpr->pColl ){ assert( pListItem->pExpr->pColl ); zColl = zExtra; sqlite3_snprintf(nExtra, zExtra, "%s", pListItem->pExpr->pColl->zName); - zExtra += (strlen(zColl) + 1); + zExtra += (sqlite3Strlen30(zColl) + 1); }else{ zColl = pTab->aCol[j].zColl; if( !zColl ){ @@ -2484,7 +2598,7 @@ void sqlite3CreateIndex( } pIndex->azColl[i] = zColl; requestedSortOrder = pListItem->sortOrder & sortOrderMask; - pIndex->aSortOrder[i] = requestedSortOrder; + pIndex->aSortOrder[i] = (u8)requestedSortOrder; } sqlite3DefaultRowEst(pIndex); @@ -2545,9 +2659,11 @@ void sqlite3CreateIndex( if( db->init.busy ){ Index *p; p = sqlite3HashInsert(&pIndex->pSchema->idxHash, - pIndex->zName, strlen(pIndex->zName)+1, pIndex); + pIndex->zName, sqlite3Strlen30(pIndex->zName)+1, + pIndex); if( p ){ assert( p==pIndex ); /* Malloc must have failed */ + db->mallocFailed = 1; goto exit_create_index; } db->flags |= SQLITE_InternChanges; @@ -2574,7 +2690,7 @@ void sqlite3CreateIndex( else if( db->init.busy==0 ){ Vdbe *v; char *zStmt; - int iMem = pParse->nMem++; + int iMem = ++pParse->nMem; v = sqlite3GetVdbe(pParse); if( v==0 ) goto exit_create_index; @@ -2583,15 +2699,14 @@ void sqlite3CreateIndex( /* Create the rootpage for the index */ sqlite3BeginWriteOperation(pParse, 1, iDb); - sqlite3VdbeAddOp(v, OP_CreateIndex, iDb, 0); - sqlite3VdbeAddOp(v, OP_MemStore, iMem, 0); + sqlite3VdbeAddOp2(v, OP_CreateIndex, iDb, iMem); /* Gather the complete text of the CREATE INDEX statement into ** the zStmt variable */ if( pStart && pEnd ){ /* A named index with an explicit CREATE INDEX statement */ - zStmt = sqlite3MPrintf("CREATE%s INDEX %.*s", + zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", onError==OE_None ? "" : " UNIQUE", pEnd->z - pName->z + 1, pName->z); @@ -2604,24 +2719,24 @@ void sqlite3CreateIndex( /* Add an entry in sqlite_master for this index */ sqlite3NestedParse(pParse, - "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#0,%Q);", + "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName, pTab->zName, + iMem, zStmt ); - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); - sqliteFree(zStmt); + sqlite3DbFree(db, zStmt); /* Fill the index with data and reparse the schema. Code an OP_Expire ** to invalidate all pre-compiled statements. */ if( pTblName ){ sqlite3RefillIndex(pParse, pIndex, iMem); - sqlite3ChangeCookie(db, v, iDb); - sqlite3VdbeOp3(v, OP_ParseSchema, iDb, 0, - sqlite3MPrintf("name='%q'", pIndex->zName), P3_DYNAMIC); - sqlite3VdbeAddOp(v, OP_Expire, 0, 0); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, + sqlite3MPrintf(db, "name='%q'", pIndex->zName), P4_DYNAMIC); + sqlite3VdbeAddOp1(v, OP_Expire, 0); } } @@ -2651,9 +2766,9 @@ exit_create_index: if( pIndex ){ freeIndex(pIndex); } - sqlite3ExprListDelete(pList); - sqlite3SrcListDelete(pTblName); - sqliteFree(zName); + sqlite3ExprListDelete(db, pList); + sqlite3SrcListDelete(db, pTblName); + sqlite3DbFree(db, zName); return; } @@ -2665,11 +2780,17 @@ void sqlite3MinimumFileFormat(Parse *pParse, int iDb, int minFormat){ Vdbe *v; v = sqlite3GetVdbe(pParse); if( v ){ - sqlite3VdbeAddOp(v, OP_ReadCookie, iDb, 1); - sqlite3VdbeAddOp(v, OP_Integer, minFormat, 0); - sqlite3VdbeAddOp(v, OP_Ge, 0, sqlite3VdbeCurrentAddr(v)+3); - sqlite3VdbeAddOp(v, OP_Integer, minFormat, 0); - sqlite3VdbeAddOp(v, OP_SetCookie, iDb, 1); + int r1 = sqlite3GetTempReg(pParse); + int r2 = sqlite3GetTempReg(pParse); + int j1; + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, 1); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_Integer, minFormat, r2); + j1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, 1, r2); + sqlite3VdbeJumpHere(v, j1); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempReg(pParse, r2); } } @@ -2718,7 +2839,7 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ sqlite3 *db = pParse->db; int iDb; - if( pParse->nErr || sqlite3MallocFailed() ){ + if( pParse->nErr || db->mallocFailed ){ goto exit_drop_index; } assert( pName->nSrc==1 ); @@ -2758,18 +2879,25 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ /* Generate code to remove the index and from the master table */ v = sqlite3GetVdbe(pParse); if( v ){ + sqlite3BeginWriteOperation(pParse, 1, iDb); sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE name=%Q", db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName ); - sqlite3ChangeCookie(db, v, iDb); + if( sqlite3FindTable(db, "sqlite_stat1", db->aDb[iDb].zName) ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.sqlite_stat1 WHERE idx=%Q", + db->aDb[iDb].zName, pIndex->zName + ); + } + sqlite3ChangeCookie(pParse, iDb); destroyRootPage(pParse, pIndex->tnum, iDb); - sqlite3VdbeOp3(v, OP_DropIndex, iDb, 0, pIndex->zName, 0); + sqlite3VdbeAddOp4(v, OP_DropIndex, iDb, 0, 0, pIndex->zName, 0); } exit_drop_index: - sqlite3SrcListDelete(pName); + sqlite3SrcListDelete(db, pName); } /* @@ -2788,6 +2916,7 @@ exit_drop_index: ** pointer if the array was resized. */ void *sqlite3ArrayAllocate( + sqlite3 *db, /* Connection to notify of malloc failures */ void *pArray, /* Array of objects. Might be reallocated */ int szEntry, /* Size of each object in the array */ int initSize, /* Suggested initial allocation, in elements */ @@ -2800,12 +2929,12 @@ void *sqlite3ArrayAllocate( void *pNew; int newSize; newSize = (*pnAlloc)*2 + initSize; - pNew = sqliteRealloc(pArray, newSize*szEntry); + pNew = sqlite3DbRealloc(db, pArray, newSize*szEntry); if( pNew==0 ){ *pIdx = -1; return pArray; } - *pnAlloc = newSize; + *pnAlloc = sqlite3DbMallocSize(db, pNew)/szEntry; pArray = pNew; } z = (char*)pArray; @@ -2821,14 +2950,15 @@ void *sqlite3ArrayAllocate( ** ** A new IdList is returned, or NULL if malloc() fails. */ -IdList *sqlite3IdListAppend(IdList *pList, Token *pToken){ +IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pToken){ int i; if( pList==0 ){ - pList = sqliteMalloc( sizeof(IdList) ); + pList = sqlite3DbMallocZero(db, sizeof(IdList) ); if( pList==0 ) return 0; pList->nAlloc = 0; } pList->a = sqlite3ArrayAllocate( + db, pList->a, sizeof(pList->a[0]), 5, @@ -2837,24 +2967,24 @@ IdList *sqlite3IdListAppend(IdList *pList, Token *pToken){ &i ); if( i<0 ){ - sqlite3IdListDelete(pList); + sqlite3IdListDelete(db, pList); return 0; } - pList->a[i].zName = sqlite3NameFromToken(pToken); + pList->a[i].zName = sqlite3NameFromToken(db, pToken); return pList; } /* ** Delete an IdList. */ -void sqlite3IdListDelete(IdList *pList){ +void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ int i; if( pList==0 ) return; for(i=0; inId; i++){ - sqliteFree(pList->a[i].zName); + sqlite3DbFree(db, pList->a[i].zName); } - sqliteFree(pList->a); - sqliteFree(pList); + sqlite3DbFree(db, pList->a); + sqlite3DbFree(db, pList); } /* @@ -2870,11 +3000,83 @@ int sqlite3IdListIndex(IdList *pList, const char *zName){ return -1; } +/* +** Expand the space allocated for the given SrcList object by +** creating nExtra new slots beginning at iStart. iStart is zero based. +** New slots are zeroed. +** +** For example, suppose a SrcList initially contains two entries: A,B. +** To append 3 new entries onto the end, do this: +** +** sqlite3SrcListEnlarge(db, pSrclist, 3, 2); +** +** After the call above it would contain: A, B, nil, nil, nil. +** If the iStart argument had been 1 instead of 2, then the result +** would have been: A, nil, nil, nil, B. To prepend the new slots, +** the iStart value would be 0. The result then would +** be: nil, nil, nil, A, B. +** +** If a memory allocation fails the SrcList is unchanged. The +** db->mallocFailed flag will be set to true. +*/ +SrcList *sqlite3SrcListEnlarge( + sqlite3 *db, /* Database connection to notify of OOM errors */ + SrcList *pSrc, /* The SrcList to be enlarged */ + int nExtra, /* Number of new slots to add to pSrc->a[] */ + int iStart /* Index in pSrc->a[] of first new slot */ +){ + int i; + + /* Sanity checking on calling parameters */ + assert( iStart>=0 ); + assert( nExtra>=1 ); + if( pSrc==0 || iStart>pSrc->nSrc ){ + assert( db->mallocFailed ); + return pSrc; + } + + /* Allocate additional space if needed */ + if( pSrc->nSrc+nExtra>pSrc->nAlloc ){ + SrcList *pNew; + int nAlloc = pSrc->nSrc+nExtra; + int nGot; + pNew = sqlite3DbRealloc(db, pSrc, + sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); + if( pNew==0 ){ + assert( db->mallocFailed ); + return pSrc; + } + pSrc = pNew; + nGot = (sqlite3DbMallocSize(db, pNew) - sizeof(*pSrc))/sizeof(pSrc->a[0])+1; + pSrc->nAlloc = (u16)nGot; + } + + /* Move existing slots that come after the newly inserted slots + ** out of the way */ + for(i=pSrc->nSrc-1; i>=iStart; i--){ + pSrc->a[i+nExtra] = pSrc->a[i]; + } + pSrc->nSrc += (i16)nExtra; + + /* Zero the newly allocated slots */ + memset(&pSrc->a[iStart], 0, sizeof(pSrc->a[0])*nExtra); + for(i=iStart; ia[i].iCursor = -1; + } + + /* Return a pointer to the enlarged SrcList */ + return pSrc; +} + + /* ** Append a new table name to the given SrcList. Create a new SrcList if ** need be. A new entry is created in the SrcList even if pToken is NULL. ** -** A new SrcList is returned, or NULL if malloc() fails. +** A SrcList is returned, or NULL if there is an OOM error. The returned +** SrcList might be the same as the SrcList that was input or it might be +** a new one. If an OOM error does occurs, then the prior value of pList +** that is input to this routine is automatically freed. ** ** If pDatabase is not null, it means that the table has an optional ** database name prefix. Like this: "database.table". The pDatabase @@ -2886,35 +3088,33 @@ int sqlite3IdListIndex(IdList *pList, const char *zName){ ** ** In other words, if call like this: ** -** sqlite3SrcListAppend(A,B,0); +** sqlite3SrcListAppend(D,A,B,0); ** ** Then B is a table name and the database name is unspecified. If called ** like this: ** -** sqlite3SrcListAppend(A,B,C); +** sqlite3SrcListAppend(D,A,B,C); ** ** Then C is the table name and B is the database name. */ -SrcList *sqlite3SrcListAppend(SrcList *pList, Token *pTable, Token *pDatabase){ +SrcList *sqlite3SrcListAppend( + sqlite3 *db, /* Connection to notify of malloc failures */ + SrcList *pList, /* Append to this SrcList. NULL creates a new SrcList */ + Token *pTable, /* Table to append */ + Token *pDatabase /* Database of the table */ +){ struct SrcList_item *pItem; if( pList==0 ){ - pList = sqliteMalloc( sizeof(SrcList) ); + pList = sqlite3DbMallocZero(db, sizeof(SrcList) ); if( pList==0 ) return 0; pList->nAlloc = 1; } - if( pList->nSrc>=pList->nAlloc ){ - SrcList *pNew; - pList->nAlloc *= 2; - pNew = sqliteRealloc(pList, - sizeof(*pList) + (pList->nAlloc-1)*sizeof(pList->a[0]) ); - if( pNew==0 ){ - sqlite3SrcListDelete(pList); - return 0; - } - pList = pNew; + pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc); + if( db->mallocFailed ){ + sqlite3SrcListDelete(db, pList); + return 0; } - pItem = &pList->a[pList->nSrc]; - memset(pItem, 0, sizeof(pList->a[0])); + pItem = &pList->a[pList->nSrc-1]; if( pDatabase && pDatabase->z==0 ){ pDatabase = 0; } @@ -2923,21 +3123,18 @@ SrcList *sqlite3SrcListAppend(SrcList *pList, Token *pTable, Token *pDatabase){ pDatabase = pTable; pTable = pTemp; } - pItem->zName = sqlite3NameFromToken(pTable); - pItem->zDatabase = sqlite3NameFromToken(pDatabase); - pItem->iCursor = -1; - pItem->isPopulated = 0; - pList->nSrc++; + pItem->zName = sqlite3NameFromToken(db, pTable); + pItem->zDatabase = sqlite3NameFromToken(db, pDatabase); return pList; } /* -** Assign cursors to all tables in a SrcList +** Assign VdbeCursor index numbers to all tables in a SrcList */ void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ int i; struct SrcList_item *pItem; - assert(pList || sqlite3MallocFailed() ); + assert(pList || pParse->db->mallocFailed ); if( pList ){ for(i=0, pItem=pList->a; inSrc; i++, pItem++){ if( pItem->iCursor>=0 ) break; @@ -2952,20 +3149,21 @@ void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ /* ** Delete an entire SrcList including all its substructure. */ -void sqlite3SrcListDelete(SrcList *pList){ +void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ int i; struct SrcList_item *pItem; if( pList==0 ) return; for(pItem=pList->a, i=0; inSrc; i++, pItem++){ - sqliteFree(pItem->zDatabase); - sqliteFree(pItem->zName); - sqliteFree(pItem->zAlias); + sqlite3DbFree(db, pItem->zDatabase); + sqlite3DbFree(db, pItem->zName); + sqlite3DbFree(db, pItem->zAlias); + sqlite3DbFree(db, pItem->zIndex); sqlite3DeleteTable(pItem->pTab); - sqlite3SelectDelete(pItem->pSelect); - sqlite3ExprDelete(pItem->pOn); - sqlite3IdListDelete(pItem->pUsing); + sqlite3SelectDelete(db, pItem->pSelect); + sqlite3ExprDelete(db, pItem->pOn); + sqlite3IdListDelete(db, pItem->pUsing); } - sqliteFree(pList); + sqlite3DbFree(db, pList); } /* @@ -2985,6 +3183,7 @@ void sqlite3SrcListDelete(SrcList *pList){ ** term added. */ SrcList *sqlite3SrcListAppendFromTerm( + Parse *pParse, /* Parsing context */ SrcList *p, /* The left part of the FROM clause already seen */ Token *pTable, /* Name of the table to add to the FROM clause */ Token *pDatabase, /* Name of the database containing pTable */ @@ -2994,16 +3193,17 @@ SrcList *sqlite3SrcListAppendFromTerm( IdList *pUsing /* The USING clause of a join */ ){ struct SrcList_item *pItem; - p = sqlite3SrcListAppend(p, pTable, pDatabase); + sqlite3 *db = pParse->db; + p = sqlite3SrcListAppend(db, p, pTable, pDatabase); if( p==0 || p->nSrc==0 ){ - sqlite3ExprDelete(pOn); - sqlite3IdListDelete(pUsing); - sqlite3SelectDelete(pSubquery); + sqlite3ExprDelete(db, pOn); + sqlite3IdListDelete(db, pUsing); + sqlite3SelectDelete(db, pSubquery); return p; } pItem = &p->a[p->nSrc-1]; if( pAlias && pAlias->n ){ - pItem->zAlias = sqlite3NameFromToken(pAlias); + pItem->zAlias = sqlite3NameFromToken(db, pAlias); } pItem->pSelect = pSubquery; pItem->pOn = pOn; @@ -3011,6 +3211,24 @@ SrcList *sqlite3SrcListAppendFromTerm( return p; } +/* +** Add an INDEXED BY or NOT INDEXED clause to the most recently added +** element of the source-list passed as the second argument. +*/ +void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ + if( pIndexedBy && p && p->nSrc>0 ){ + struct SrcList_item *pItem = &p->a[p->nSrc-1]; + assert( pItem->notIndexed==0 && pItem->zIndex==0 ); + if( pIndexedBy->n==1 && !pIndexedBy->z ){ + /* A "NOT INDEXED" clause was supplied. See parse.y + ** construct "indexed_opt" for details. */ + pItem->notIndexed = 1; + }else{ + pItem->zIndex = sqlite3NameFromToken(pParse->db, pIndexedBy); + } + } +} + /* ** When building up a FROM clause in the parser, the join operator ** is initially attached to the left operand. But the code generator @@ -3045,17 +3263,18 @@ void sqlite3BeginTransaction(Parse *pParse, int type){ int i; if( pParse==0 || (db=pParse->db)==0 || db->aDb[0].pBt==0 ) return; - if( pParse->nErr || sqlite3MallocFailed() ) return; + if( pParse->nErr || db->mallocFailed ) return; if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "BEGIN", 0, 0) ) return; v = sqlite3GetVdbe(pParse); if( !v ) return; if( type!=TK_DEFERRED ){ for(i=0; inDb; i++){ - sqlite3VdbeAddOp(v, OP_Transaction, i, (type==TK_EXCLUSIVE)+1); + sqlite3VdbeAddOp2(v, OP_Transaction, i, (type==TK_EXCLUSIVE)+1); + sqlite3VdbeUsesBtree(v, i); } } - sqlite3VdbeAddOp(v, OP_AutoCommit, 0, 0); + sqlite3VdbeAddOp2(v, OP_AutoCommit, 0, 0); } /* @@ -3066,12 +3285,12 @@ void sqlite3CommitTransaction(Parse *pParse){ Vdbe *v; if( pParse==0 || (db=pParse->db)==0 || db->aDb[0].pBt==0 ) return; - if( pParse->nErr || sqlite3MallocFailed() ) return; + if( pParse->nErr || db->mallocFailed ) return; if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "COMMIT", 0, 0) ) return; v = sqlite3GetVdbe(pParse); if( v ){ - sqlite3VdbeAddOp(v, OP_AutoCommit, 1, 0); + sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 0); } } @@ -3083,12 +3302,32 @@ void sqlite3RollbackTransaction(Parse *pParse){ Vdbe *v; if( pParse==0 || (db=pParse->db)==0 || db->aDb[0].pBt==0 ) return; - if( pParse->nErr || sqlite3MallocFailed() ) return; + if( pParse->nErr || db->mallocFailed ) return; if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "ROLLBACK", 0, 0) ) return; v = sqlite3GetVdbe(pParse); if( v ){ - sqlite3VdbeAddOp(v, OP_AutoCommit, 1, 1); + sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 1); + } +} + +/* +** This function is called by the parser when it parses a command to create, +** release or rollback an SQL savepoint. +*/ +void sqlite3Savepoint(Parse *pParse, int op, Token *pName){ + char *zName = sqlite3NameFromToken(pParse->db, pName); + if( zName ){ + Vdbe *v = sqlite3GetVdbe(pParse); +#ifndef SQLITE_OMIT_AUTHORIZATION + static const char *az[] = { "BEGIN", "RELEASE", "ROLLBACK" }; + assert( !SAVEPOINT_BEGIN && SAVEPOINT_RELEASE==1 && SAVEPOINT_ROLLBACK==2 ); +#endif + if( !v || sqlite3AuthCheck(pParse, SQLITE_SAVEPOINT, az[op], zName, 0) ){ + sqlite3DbFree(pParse->db, zName); + return; + } + sqlite3VdbeAddOp4(v, OP_Savepoint, op, 0, 0, zName, P4_DYNAMIC); } } @@ -3099,7 +3338,15 @@ void sqlite3RollbackTransaction(Parse *pParse){ int sqlite3OpenTempDatabase(Parse *pParse){ sqlite3 *db = pParse->db; if( db->aDb[1].pBt==0 && !pParse->explain ){ - int rc = sqlite3BtreeFactory(db, 0, 0, SQLITE_DEFAULT_CACHE_SIZE, + int rc; + static const int flags = + SQLITE_OPEN_READWRITE | + SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_TEMP_DB; + + rc = sqlite3BtreeFactory(db, 0, 0, SQLITE_DEFAULT_CACHE_SIZE, flags, &db->aDb[1].pBt); if( rc!=SQLITE_OK ){ sqlite3ErrorMsg(pParse, "unable to open a temporary database " @@ -3107,16 +3354,10 @@ int sqlite3OpenTempDatabase(Parse *pParse){ pParse->rc = rc; return 1; } - if( db->flags & !db->autoCommit ){ - rc = sqlite3BtreeBeginTrans(db->aDb[1].pBt, 1); - if( rc!=SQLITE_OK ){ - sqlite3ErrorMsg(pParse, "unable to get a write lock on " - "the temporary database file"); - pParse->rc = rc; - return 1; - } - } + assert( (db->flags & SQLITE_InTrans)==0 || db->autoCommit ); assert( db->aDb[1].pSchema ); + sqlite3PagerJournalMode(sqlite3BtreePager(db->aDb[1].pBt), + db->dfltJournalMode); } return 0; } @@ -3152,7 +3393,7 @@ void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ if( v==0 ) return; /* This only happens if there was a prior error */ db = pParse->db; if( pParse->cookieGoto==0 ){ - pParse->cookieGoto = sqlite3VdbeAddOp(v, OP_Goto, 0, 0)+1; + pParse->cookieGoto = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0)+1; } if( iDb>=0 ){ assert( iDbnDb ); @@ -3193,7 +3434,7 @@ void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){ sqlite3CodeVerifySchema(pParse, iDb); pParse->writeMask |= 1<nested==0 ){ - sqlite3VdbeAddOp(v, OP_Statement, iDb, 0); + sqlite3VdbeAddOp1(v, OP_Statement, iDb); } if( (OMIT_TEMPDB || iDb!=1) && pParse->db->aDb[1].pBt!=0 ){ sqlite3BeginWriteOperation(pParse, setStatement, 1); @@ -3292,30 +3533,33 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ reindexDatabases(pParse, 0); return; }else if( pName2==0 || pName2->z==0 ){ + char *zColl; assert( pName1->z ); - pColl = sqlite3FindCollSeq(db, ENC(db), (char*)pName1->z, pName1->n, 0); + zColl = sqlite3NameFromToken(pParse->db, pName1); + if( !zColl ) return; + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, -1, 0); if( pColl ){ - char *zColl = sqliteStrNDup((const char *)pName1->z, pName1->n); if( zColl ){ reindexDatabases(pParse, zColl); - sqliteFree(zColl); + sqlite3DbFree(db, zColl); } return; } + sqlite3DbFree(db, zColl); } iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pObjName); if( iDb<0 ) return; - z = sqlite3NameFromToken(pObjName); + z = sqlite3NameFromToken(db, pObjName); if( z==0 ) return; zDb = db->aDb[iDb].zName; pTab = sqlite3FindTable(db, z, zDb); if( pTab ){ reindexTable(pParse, pTab, 0); - sqliteFree(z); + sqlite3DbFree(db, z); return; } pIndex = sqlite3FindIndex(db, z, zDb); - sqliteFree(z); + sqlite3DbFree(db, z); if( pIndex ){ sqlite3BeginWriteOperation(pParse, 0, iDb); sqlite3RefillIndex(pParse, pIndex, -1); @@ -3330,7 +3574,7 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ ** with OP_OpenRead or OP_OpenWrite to access database index pIdx. ** ** If successful, a pointer to the new structure is returned. In this case -** the caller is responsible for calling sqliteFree() on the returned +** the caller is responsible for calling sqlite3DbFree(db, ) on the returned ** pointer. If an error occurs (out of memory or missing collation ** sequence), NULL is returned and the state of pParse updated to reflect ** the error. @@ -3339,9 +3583,11 @@ KeyInfo *sqlite3IndexKeyinfo(Parse *pParse, Index *pIdx){ int i; int nCol = pIdx->nColumn; int nBytes = sizeof(KeyInfo) + (nCol-1)*sizeof(CollSeq*) + nCol; - KeyInfo *pKey = (KeyInfo *)sqliteMalloc(nBytes); + sqlite3 *db = pParse->db; + KeyInfo *pKey = (KeyInfo *)sqlite3DbMallocZero(db, nBytes); if( pKey ){ + pKey->db = pParse->db; pKey->aSortOrder = (u8 *)&(pKey->aColl[nCol]); assert( &pKey->aSortOrder[nCol]==&(((u8 *)pKey)[nBytes]) ); for(i=0; iaColl[i] = sqlite3LocateCollSeq(pParse, zColl, -1); pKey->aSortOrder[i] = pIdx->aSortOrder[i]; } - pKey->nField = nCol; + pKey->nField = (u16)nCol; } if( pParse->nErr ){ - sqliteFree(pKey); + sqlite3DbFree(db, pKey); pKey = 0; } return pKey; diff --git a/callback.c b/callback.c index f3d4188..7204d17 100644 --- a/callback.c +++ b/callback.c @@ -13,7 +13,7 @@ ** This file contains functions used to access the internal hash tables ** of user defined functions and collation sequences. ** -** $Id: callback.c,v 1.18 2007/05/07 09:32:45 danielk1977 Exp $ +** $Id: callback.c,v 1.34 2008/12/10 21:19:57 drh Exp $ */ #include "sqliteInt.h" @@ -25,17 +25,17 @@ */ static void callCollNeeded(sqlite3 *db, const char *zName, int nName){ assert( !db->xCollNeeded || !db->xCollNeeded16 ); - if( nName<0 ) nName = strlen(zName); + if( nName<0 ) nName = sqlite3Strlen(db, zName); if( db->xCollNeeded ){ - char *zExternal = sqliteStrNDup(zName, nName); + char *zExternal = sqlite3DbStrNDup(db, zName, nName); if( !zExternal ) return; db->xCollNeeded(db->pCollNeededArg, db, (int)ENC(db), zExternal); - sqliteFree(zExternal); + sqlite3DbFree(db, zExternal); } #ifndef SQLITE_OMIT_UTF16 if( db->xCollNeeded16 ){ char const *zExternal; - sqlite3_value *pTmp = sqlite3ValueNew(); + sqlite3_value *pTmp = sqlite3ValueNew(db); sqlite3ValueSetStr(pTmp, nName, zName, SQLITE_UTF8, SQLITE_STATIC); zExternal = sqlite3ValueText(pTmp, SQLITE_UTF16NATIVE); if( zExternal ){ @@ -56,7 +56,7 @@ static void callCollNeeded(sqlite3 *db, const char *zName, int nName){ static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ CollSeq *pColl2; char *z = pColl->zName; - int n = strlen(z); + int n = sqlite3Strlen30(z); int i; static const u8 aEnc[] = { SQLITE_UTF16BE, SQLITE_UTF16LE, SQLITE_UTF8 }; for(i=0; i<3; i++){ @@ -158,11 +158,11 @@ static CollSeq *findCollSeqEntry( int create ){ CollSeq *pColl; - if( nName<0 ) nName = strlen(zName); + if( nName<0 ) nName = sqlite3Strlen(db, zName); pColl = sqlite3HashFind(&db->aCollSeq, zName, nName); if( 0==pColl && create ){ - pColl = sqliteMalloc( 3*sizeof(*pColl) + nName + 1 ); + pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1 ); if( pColl ){ CollSeq *pDel = 0; pColl[0].zName = (char*)&pColl[3]; @@ -179,9 +179,10 @@ static CollSeq *findCollSeqEntry( ** return the pColl pointer to be deleted (because it wasn't added ** to the hash table). */ - assert( !pDel || (sqlite3MallocFailed() && pDel==pColl) ); - if( pDel ){ - sqliteFree(pDel); + assert( pDel==0 || pDel==pColl ); + if( pDel!=0 ){ + db->mallocFailed = 1; + sqlite3DbFree(db, pDel); pColl = 0; } } @@ -221,6 +222,87 @@ CollSeq *sqlite3FindCollSeq( return pColl; } +/* During the search for the best function definition, this procedure +** is called to test how well the function passed as the first argument +** matches the request for a function with nArg arguments in a system +** that uses encoding enc. The value returned indicates how well the +** request is matched. A higher value indicates a better match. +** +** The returned value is always between 1 and 6, as follows: +** +** 1: A variable arguments function that prefers UTF-8 when a UTF-16 +** encoding is requested, or vice versa. +** 2: A variable arguments function that uses UTF-16BE when UTF-16LE is +** requested, or vice versa. +** 3: A variable arguments function using the same text encoding. +** 4: A function with the exact number of arguments requested that +** prefers UTF-8 when a UTF-16 encoding is requested, or vice versa. +** 5: A function with the exact number of arguments requested that +** prefers UTF-16LE when UTF-16BE is requested, or vice versa. +** 6: An exact match. +** +*/ +static int matchQuality(FuncDef *p, int nArg, u8 enc){ + int match = 0; + if( p->nArg==-1 || p->nArg==nArg || nArg==-1 ){ + match = 1; + if( p->nArg==nArg || nArg==-1 ){ + match = 4; + } + if( enc==p->iPrefEnc ){ + match += 2; + } + else if( (enc==SQLITE_UTF16LE && p->iPrefEnc==SQLITE_UTF16BE) || + (enc==SQLITE_UTF16BE && p->iPrefEnc==SQLITE_UTF16LE) ){ + match += 1; + } + } + return match; +} + +/* +** Search a FuncDefHash for a function with the given name. Return +** a pointer to the matching FuncDef if found, or 0 if there is no match. +*/ +static FuncDef *functionSearch( + FuncDefHash *pHash, /* Hash table to search */ + int h, /* Hash of the name */ + const char *zFunc, /* Name of function */ + int nFunc /* Number of bytes in zFunc */ +){ + FuncDef *p; + for(p=pHash->a[h]; p; p=p->pHash){ + if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 && p->zName[nFunc]==0 ){ + return p; + } + } + return 0; +} + +/* +** Insert a new FuncDef into a FuncDefHash hash table. +*/ +void sqlite3FuncDefInsert( + FuncDefHash *pHash, /* The hash table into which to insert */ + FuncDef *pDef /* The function definition to insert */ +){ + FuncDef *pOther; + int nName = sqlite3Strlen30(pDef->zName); + u8 c1 = (u8)pDef->zName[0]; + int h = (sqlite3UpperToLower[c1] + nName) % ArraySize(pHash->a); + pOther = functionSearch(pHash, h, pDef->zName, nName); + if( pOther ){ + pDef->pNext = pOther->pNext; + pOther->pNext = pDef; + }else{ + pDef->pNext = 0; + pDef->pHash = pHash->a[h]; + pHash->a[h] = pDef; + } +} + + + /* ** Locate a user function given a name, a number of arguments and a flag ** indicating whether the function prefers UTF-16 over UTF-8. Return a @@ -250,69 +332,60 @@ FuncDef *sqlite3FindFunction( int createFlag /* Create new entry if true and does not otherwise exist */ ){ FuncDef *p; /* Iterator variable */ - FuncDef *pFirst; /* First function with this name */ FuncDef *pBest = 0; /* Best match found so far */ - int bestmatch = 0; + int bestScore = 0; /* Score of best match */ + int h; /* Hash value */ assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); if( nArg<-1 ) nArg = -1; + h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % ArraySize(db->aFunc.a); - pFirst = (FuncDef*)sqlite3HashFind(&db->aFunc, zName, nName); - for(p=pFirst; p; p=p->pNext){ - /* During the search for the best function definition, bestmatch is set - ** as follows to indicate the quality of the match with the definition - ** pointed to by pBest: - ** - ** 0: pBest is NULL. No match has been found. - ** 1: A variable arguments function that prefers UTF-8 when a UTF-16 - ** encoding is requested, or vice versa. - ** 2: A variable arguments function that uses UTF-16BE when UTF-16LE is - ** requested, or vice versa. - ** 3: A variable arguments function using the same text encoding. - ** 4: A function with the exact number of arguments requested that - ** prefers UTF-8 when a UTF-16 encoding is requested, or vice versa. - ** 5: A function with the exact number of arguments requested that - ** prefers UTF-16LE when UTF-16BE is requested, or vice versa. - ** 6: An exact match. - ** - ** A larger value of 'matchqual' indicates a more desirable match. - */ - if( p->nArg==-1 || p->nArg==nArg || nArg==-1 ){ - int match = 1; /* Quality of this match */ - if( p->nArg==nArg || nArg==-1 ){ - match = 4; - } - if( enc==p->iPrefEnc ){ - match += 2; - } - else if( (enc==SQLITE_UTF16LE && p->iPrefEnc==SQLITE_UTF16BE) || - (enc==SQLITE_UTF16BE && p->iPrefEnc==SQLITE_UTF16LE) ){ - match += 1; - } + /* First search for a match amongst the application-defined functions. + */ + p = functionSearch(&db->aFunc, h, zName, nName); + while( p ){ + int score = matchQuality(p, nArg, enc); + if( score>bestScore ){ + pBest = p; + bestScore = score; + } + p = p->pNext; + } - if( match>bestmatch ){ + /* If no match is found, search the built-in functions. + ** + ** Except, if createFlag is true, that means that we are trying to + ** install a new function. Whatever FuncDef structure is returned will + ** have fields overwritten with new information appropriate for the + ** new function. But the FuncDefs for built-in functions are read-only. + ** So we must not search for built-ins when creating a new function. + */ + if( !createFlag && !pBest ){ + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + p = functionSearch(pHash, h, zName, nName); + while( p ){ + int score = matchQuality(p, nArg, enc); + if( score>bestScore ){ pBest = p; - bestmatch = match; + bestScore = score; } + p = p->pNext; } } - /* If the createFlag parameter is true, and the seach did not reveal an + /* If the createFlag parameter is true and the search did not reveal an ** exact match for the name, number of arguments and encoding, then add a ** new entry to the hash table and return it. */ - if( createFlag && bestmatch<6 && - (pBest = sqliteMalloc(sizeof(*pBest)+nName))!=0 ){ - pBest->nArg = nArg; - pBest->pNext = pFirst; + if( createFlag && (bestScore<6 || pBest->nArg!=nArg) && + (pBest = sqlite3DbMallocZero(db, sizeof(*pBest)+nName+1))!=0 ){ + pBest->zName = (char *)&pBest[1]; + pBest->nArg = (u16)nArg; pBest->iPrefEnc = enc; memcpy(pBest->zName, zName, nName); pBest->zName[nName] = 0; - if( pBest==sqlite3HashInsert(&db->aFunc,pBest->zName,nName,(void*)pBest) ){ - sqliteFree(pBest); - return 0; - } + sqlite3FuncDefInsert(&db->aFunc, pBest); } if( pBest && (pBest->xStep || pBest->xFunc || createFlag) ){ @@ -323,9 +396,11 @@ FuncDef *sqlite3FindFunction( /* ** Free all resources held by the schema structure. The void* argument points -** at a Schema struct. This function does not call sqliteFree() on the +** at a Schema struct. This function does not call sqlite3DbFree(db, ) on the ** pointer itself, it just cleans up subsiduary resources (i.e. the contents ** of the schema hash tables). +** +** The Schema.cache_size variable is not cleared. */ void sqlite3SchemaFree(void *p){ Hash temp1; @@ -335,14 +410,14 @@ void sqlite3SchemaFree(void *p){ temp1 = pSchema->tblHash; temp2 = pSchema->trigHash; - sqlite3HashInit(&pSchema->trigHash, SQLITE_HASH_STRING, 0); + sqlite3HashInit(&pSchema->trigHash, 0); sqlite3HashClear(&pSchema->aFKey); sqlite3HashClear(&pSchema->idxHash); for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ - sqlite3DeleteTrigger((Trigger*)sqliteHashData(pElem)); + sqlite3DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); } sqlite3HashClear(&temp2); - sqlite3HashInit(&pSchema->tblHash, SQLITE_HASH_STRING, 0); + sqlite3HashInit(&pSchema->tblHash, 0); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ Table *pTab = sqliteHashData(pElem); sqlite3DeleteTable(pTab); @@ -356,18 +431,20 @@ void sqlite3SchemaFree(void *p){ ** Find and return the schema associated with a BTree. Create ** a new one if necessary. */ -Schema *sqlite3SchemaGet(Btree *pBt){ +Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ Schema * p; if( pBt ){ - p = (Schema *)sqlite3BtreeSchema(pBt,sizeof(Schema),sqlite3SchemaFree); + p = (Schema *)sqlite3BtreeSchema(pBt, sizeof(Schema), sqlite3SchemaFree); }else{ - p = (Schema *)sqliteMalloc(sizeof(Schema)); + p = (Schema *)sqlite3MallocZero(sizeof(Schema)); } - if( p && 0==p->file_format ){ - sqlite3HashInit(&p->tblHash, SQLITE_HASH_STRING, 0); - sqlite3HashInit(&p->idxHash, SQLITE_HASH_STRING, 0); - sqlite3HashInit(&p->trigHash, SQLITE_HASH_STRING, 0); - sqlite3HashInit(&p->aFKey, SQLITE_HASH_STRING, 1); + if( !p ){ + db->mallocFailed = 1; + }else if ( 0==p->file_format ){ + sqlite3HashInit(&p->tblHash, 0); + sqlite3HashInit(&p->idxHash, 0); + sqlite3HashInit(&p->trigHash, 0); + sqlite3HashInit(&p->aFKey, 1); p->enc = SQLITE_UTF8; } return p; diff --git a/complete.c b/complete.c index 536ffad..8e2dbc2 100644 --- a/complete.c +++ b/complete.c @@ -16,7 +16,7 @@ ** separating it out, the code will be automatically omitted from ** static links that do not use it. ** -** $Id: complete.c,v 1.3 2006/01/18 15:25:17 danielk1977 Exp $ +** $Id: complete.c,v 1.7 2008/06/13 18:24:27 drh Exp $ */ #include "sqliteInt.h" #ifndef SQLITE_OMIT_COMPLETE @@ -24,8 +24,16 @@ /* ** This is defined in tokenize.c. We just have to import the definition. */ -extern const char sqlite3IsIdChar[]; -#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsIdChar[c-0x20])) +#ifndef SQLITE_AMALGAMATION +#ifdef SQLITE_ASCII +extern const char sqlite3IsAsciiIdChar[]; +#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsAsciiIdChar[c-0x20])) +#endif +#ifdef SQLITE_EBCDIC +extern const char sqlite3IsEbcdicIdChar[]; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif +#endif /* SQLITE_AMALGAMATION */ /* @@ -248,13 +256,19 @@ int sqlite3_complete(const char *zSql){ int sqlite3_complete16(const void *zSql){ sqlite3_value *pVal; char const *zSql8; - int rc = 0; + int rc = SQLITE_NOMEM; - pVal = sqlite3ValueNew(); +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + pVal = sqlite3ValueNew(0); sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); if( zSql8 ){ rc = sqlite3_complete(zSql8); + }else{ + rc = SQLITE_NOMEM; } sqlite3ValueFree(pVal); return sqlite3ApiExit(0, rc); diff --git a/date.c b/date.c index bf4bc56..ae99125 100644 --- a/date.c +++ b/date.c @@ -16,7 +16,7 @@ ** sqlite3RegisterDateTimeFunctions() found at the bottom of the file. ** All other code has file scope. ** -** $Id: date.c,v 1.66 2007/05/08 21:56:00 drh Exp $ +** $Id: date.c,v 1.99 2008/12/20 13:18:50 drh Exp $ ** ** SQLite processes all times and dates as Julian Day numbers. The ** dates and times are stored as the number of days since noon @@ -46,7 +46,6 @@ ** Richmond, Virginia (USA) */ #include "sqliteInt.h" -#include "os.h" #include #include #include @@ -54,20 +53,37 @@ #ifndef SQLITE_OMIT_DATETIME_FUNCS +/* +** On recent Windows platforms, the localtime_s() function is available +** as part of the "Secure CRT". It is essentially equivalent to +** localtime_r() available under most POSIX platforms, except that the +** order of the parameters is reversed. +** +** See http://msdn.microsoft.com/en-us/library/a442x3ye(VS.80).aspx. +** +** If the user has not indicated to use localtime_r() or localtime_s() +** already, check for an MSVC build environment that provides +** localtime_s(). +*/ +#if !defined(HAVE_LOCALTIME_R) && !defined(HAVE_LOCALTIME_S) && \ + defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) +#define HAVE_LOCALTIME_S 1 +#endif + /* ** A structure for holding a single date and time. */ typedef struct DateTime DateTime; struct DateTime { - double rJD; /* The julian day number */ - int Y, M, D; /* Year, month, and day */ - int h, m; /* Hour and minutes */ - int tz; /* Timezone offset in minutes */ - double s; /* Seconds */ - char validYMD; /* True if Y,M,D are valid */ - char validHMS; /* True if h,m,s are valid */ - char validJD; /* True if rJD is valid */ - char validTZ; /* True if tz is valid */ + sqlite3_int64 iJD; /* The julian day number times 86400000 */ + int Y, M, D; /* Year, month, and day */ + int h, m; /* Hour and minutes */ + int tz; /* Timezone offset in minutes */ + double s; /* Seconds */ + char validYMD; /* True (1) if Y,M,D are valid */ + char validHMS; /* True (1) if h,m,s are valid */ + char validJD; /* True (1) if iJD is valid */ + char validTZ; /* True (1) if tz is valid */ }; @@ -132,23 +148,32 @@ end_getDigits: ** ** (+/-)HH:MM ** +** Or the "zulu" notation: +** +** Z +** ** If the parse is successful, write the number of minutes -** of change in *pnMin and return 0. If a parser error occurs, -** return 0. +** of change in p->tz and return 0. If a parser error occurs, +** return non-zero. ** ** A missing specifier is not considered an error. */ static int parseTimezone(const char *zDate, DateTime *p){ int sgn = 0; int nHr, nMn; + int c; while( isspace(*(u8*)zDate) ){ zDate++; } p->tz = 0; - if( *zDate=='-' ){ + c = *zDate; + if( c=='-' ){ sgn = -1; - }else if( *zDate=='+' ){ + }else if( c=='+' ){ sgn = +1; + }else if( c=='Z' || c=='z' ){ + zDate++; + goto zulu_time; }else{ - return *zDate!=0; + return c!=0; } zDate++; if( getDigits(zDate, 2, 0, 14, ':', &nHr, 2, 0, 59, 0, &nMn)!=2 ){ @@ -156,6 +181,7 @@ static int parseTimezone(const char *zDate, DateTime *p){ } zDate += 5; p->tz = sgn*(nMn + nHr*60); +zulu_time: while( isspace(*(u8*)zDate) ){ zDate++; } return *zDate!=0; } @@ -199,7 +225,7 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ p->m = m; p->s = s + ms; if( parseTimezone(zDate, p) ) return 1; - p->validTZ = p->tz!=0; + p->validTZ = (p->tz!=0)?1:0; return 0; } @@ -228,14 +254,14 @@ static void computeJD(DateTime *p){ } A = Y/100; B = 2 - A + (A/4); - X1 = 365.25*(Y+4716); - X2 = 30.6001*(M+1); - p->rJD = X1 + X2 + D + B - 1524.5; + X1 = 36525*(Y+4716)/100; + X2 = 306001*(M+1)/10000; + p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); p->validJD = 1; if( p->validHMS ){ - p->rJD += (p->h*3600.0 + p->m*60.0 + p->s)/86400.0; + p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000); if( p->validTZ ){ - p->rJD -= p->tz*60/86400.0; + p->iJD -= p->tz*60000; p->validYMD = 0; p->validHMS = 0; p->validTZ = 0; @@ -287,6 +313,17 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ return 0; } +/* +** Set the time to the current time reported by the VFS +*/ +static void setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ + double r; + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3OsCurrentTime(db->pVfs, &r); + p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); + p->validJD = 1; +} + /* ** Attempt to parse the given string into a Julian Day Number. Return ** the number of errors. @@ -303,20 +340,22 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ ** as there is a time string. The time string can be omitted as long ** as there is a year and date. */ -static int parseDateOrTime(const char *zDate, DateTime *p){ - memset(p, 0, sizeof(*p)); +static int parseDateOrTime( + sqlite3_context *context, + const char *zDate, + DateTime *p +){ if( parseYyyyMmDd(zDate,p)==0 ){ return 0; }else if( parseHhMmSs(zDate, p)==0 ){ return 0; }else if( sqlite3StrICmp(zDate,"now")==0){ - double r; - sqlite3OsCurrentTime(&r); - p->rJD = r; - p->validJD = 1; + setDateTimeToCurrent(context, p); return 0; }else if( sqlite3IsNumber(zDate, 0, SQLITE_UTF8) ){ - getValue(zDate, &p->rJD); + double r; + getValue(zDate, &r); + p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); p->validJD = 1; return 0; } @@ -334,14 +373,14 @@ static void computeYMD(DateTime *p){ p->M = 1; p->D = 1; }else{ - Z = p->rJD + 0.5; - A = (Z - 1867216.25)/36524.25; + Z = (int)((p->iJD + 43200000)/86400000); + A = (int)((Z - 1867216.25)/36524.25); A = Z + 1 + A - (A/4); B = A + 1524; - C = (B - 122.1)/365.25; - D = 365.25*C; - E = (B-D)/30.6001; - X1 = 30.6001*E; + C = (int)((B - 122.1)/365.25); + D = (36525*C)/100; + E = (int)((B-D)/30.6001); + X1 = (int)(30.6001*E); p->D = B - D - X1; p->M = E<14 ? E-1 : E-13; p->Y = p->M>2 ? C - 4716 : C - 4715; @@ -353,13 +392,12 @@ static void computeYMD(DateTime *p){ ** Compute the Hour, Minute, and Seconds from the julian day number. */ static void computeHMS(DateTime *p){ - int Z, s; + int s; if( p->validHMS ) return; computeJD(p); - Z = p->rJD + 0.5; - s = (p->rJD + 0.5 - Z)*86400000.0 + 0.5; - p->s = 0.001*s; - s = p->s; + s = (int)((p->iJD + 43200000) % 86400000); + p->s = s/1000.0; + s = (int)p->s; p->s -= s; p->h = s/3600; s -= p->h*3600; @@ -385,11 +423,13 @@ static void clearYMD_HMS_TZ(DateTime *p){ p->validTZ = 0; } +#ifndef SQLITE_OMIT_LOCALTIME /* -** Compute the difference (in days) between localtime and UTC (a.k.a. GMT) +** Compute the difference (in milliseconds) +** between localtime and UTC (a.k.a. GMT) ** for the time value p where p is in UTC. */ -static double localtimeOffset(DateTime *p){ +static sqlite3_int64 localtimeOffset(DateTime *p){ DateTime x, y; time_t t; x = *p; @@ -402,13 +442,13 @@ static double localtimeOffset(DateTime *p){ x.m = 0; x.s = 0.0; } else { - int s = x.s + 0.5; + int s = (int)(x.s + 0.5); x.s = s; } x.tz = 0; x.validJD = 0; computeJD(&x); - t = (x.rJD-2440587.5)*86400.0 + 0.5; + t = x.iJD/1000 - 21086676*(i64)10000; #ifdef HAVE_LOCALTIME_R { struct tm sLocal; @@ -420,10 +460,21 @@ static double localtimeOffset(DateTime *p){ y.m = sLocal.tm_min; y.s = sLocal.tm_sec; } +#elif defined(HAVE_LOCALTIME_S) + { + struct tm sLocal; + localtime_s(&sLocal, &t); + y.Y = sLocal.tm_year + 1900; + y.M = sLocal.tm_mon + 1; + y.D = sLocal.tm_mday; + y.h = sLocal.tm_hour; + y.m = sLocal.tm_min; + y.s = sLocal.tm_sec; + } #else { struct tm *pTm; - sqlite3OsEnterMutex(); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); pTm = localtime(&t); y.Y = pTm->tm_year + 1900; y.M = pTm->tm_mon + 1; @@ -431,7 +482,7 @@ static double localtimeOffset(DateTime *p){ y.h = pTm->tm_hour; y.m = pTm->tm_min; y.s = pTm->tm_sec; - sqlite3OsLeaveMutex(); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } #endif y.validYMD = 1; @@ -439,8 +490,9 @@ static double localtimeOffset(DateTime *p){ y.validJD = 0; y.validTZ = 0; computeJD(&y); - return y.rJD - x.rJD; + return y.iJD - x.iJD; } +#endif /* SQLITE_OMIT_LOCALTIME */ /* ** Process a modifier to a date-time stamp. The modifiers are @@ -469,11 +521,12 @@ static int parseModifier(const char *zMod, DateTime *p){ double r; char *z, zBuf[30]; z = zBuf; - for(n=0; nrJD += localtimeOffset(p); + p->iJD += localtimeOffset(p); clearYMD_HMS_TZ(p); rc = 0; } break; } +#endif case 'u': { /* ** unixepoch ** - ** Treat the current value of p->rJD as the number of + ** Treat the current value of p->iJD as the number of ** seconds since 1970. Convert to a real julian day number. */ if( strcmp(z, "unixepoch")==0 && p->validJD ){ - p->rJD = p->rJD/86400.0 + 2440587.5; + p->iJD = p->iJD/86400 + 21086676*(i64)10000000; clearYMD_HMS_TZ(p); rc = 0; - }else if( strcmp(z, "utc")==0 ){ - double c1; - computeJD(p); - c1 = localtimeOffset(p); - p->rJD -= c1; - clearYMD_HMS_TZ(p); - p->rJD += c1 - localtimeOffset(p); - rc = 0; } +#ifndef SQLITE_OMIT_LOCALTIME + else if( strcmp(z, "utc")==0 ){ + sqlite3_int64 c1; + computeJD(p); + c1 = localtimeOffset(p); + p->iJD -= c1; + clearYMD_HMS_TZ(p); + p->iJD += c1 - localtimeOffset(p); + rc = 0; + } +#endif break; } case 'w': { @@ -519,16 +576,15 @@ static int parseModifier(const char *zMod, DateTime *p){ ** date is already on the appropriate weekday, this is a no-op. */ if( strncmp(z, "weekday ", 8)==0 && getValue(&z[8],&r)>0 - && (n=r)==r && n>=0 && r<7 ){ - int Z; + && (n=(int)r)==r && n>=0 && r<7 ){ + sqlite3_int64 Z; computeYMD_HMS(p); p->validTZ = 0; p->validJD = 0; computeJD(p); - Z = p->rJD + 1.5; - Z %= 7; + Z = ((p->iJD + 129600000)/86400000) % 7; if( Z>n ) Z -= 7; - p->rJD += n - Z; + p->iJD += (n - Z)*86400000; clearYMD_HMS_TZ(p); rc = 0; } @@ -584,52 +640,52 @@ static int parseModifier(const char *zMod, DateTime *p){ */ const char *z2 = z; DateTime tx; - int day; + sqlite3_int64 day; if( !isdigit(*(u8*)z2) ) z2++; memset(&tx, 0, sizeof(tx)); if( parseHhMmSs(z2, &tx) ) break; computeJD(&tx); - tx.rJD -= 0.5; - day = (int)tx.rJD; - tx.rJD -= day; - if( z[0]=='-' ) tx.rJD = -tx.rJD; + tx.iJD -= 43200000; + day = tx.iJD/86400000; + tx.iJD -= day*86400000; + if( z[0]=='-' ) tx.iJD = -tx.iJD; computeJD(p); clearYMD_HMS_TZ(p); - p->rJD += tx.rJD; + p->iJD += tx.iJD; rc = 0; break; } z += n; while( isspace(*(u8*)z) ) z++; - n = strlen(z); + n = sqlite3Strlen30(z); if( n>10 || n<3 ) break; if( z[n-1]=='s' ){ z[n-1] = 0; n--; } computeJD(p); rc = 0; if( n==3 && strcmp(z,"day")==0 ){ - p->rJD += r; + p->iJD += (sqlite3_int64)(r*86400000.0 + 0.5); }else if( n==4 && strcmp(z,"hour")==0 ){ - p->rJD += r/24.0; + p->iJD += (sqlite3_int64)(r*(86400000.0/24.0) + 0.5); }else if( n==6 && strcmp(z,"minute")==0 ){ - p->rJD += r/(24.0*60.0); + p->iJD += (sqlite3_int64)(r*(86400000.0/(24.0*60.0)) + 0.5); }else if( n==6 && strcmp(z,"second")==0 ){ - p->rJD += r/(24.0*60.0*60.0); + p->iJD += (sqlite3_int64)(r*(86400000.0/(24.0*60.0*60.0)) + 0.5); }else if( n==5 && strcmp(z,"month")==0 ){ int x, y; computeYMD_HMS(p); - p->M += r; + p->M += (int)r; x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; p->Y += x; p->M -= x*12; p->validJD = 0; computeJD(p); - y = r; + y = (int)r; if( y!=r ){ - p->rJD += (r - y)*30.0; + p->iJD += (sqlite3_int64)((r - y)*30.0*86400000.0 + 0.5); } }else if( n==4 && strcmp(z,"year")==0 ){ computeYMD_HMS(p); - p->Y += r; + p->Y += (int)r; p->validJD = 0; computeJD(p); }else{ @@ -650,13 +706,31 @@ static int parseModifier(const char *zMod, DateTime *p){ ** argv[1] and following are modifiers. Parse them all and write ** the resulting time into the DateTime structure p. Return 0 ** on success and 1 if there are any errors. +** +** If there are zero parameters (if even argv[0] is undefined) +** then assume a default value of "now" for argv[0]. */ -static int isDate(int argc, sqlite3_value **argv, DateTime *p){ +static int isDate( + sqlite3_context *context, + int argc, + sqlite3_value **argv, + DateTime *p +){ int i; const unsigned char *z; - if( argc==0 ) return 1; - if( (z = sqlite3_value_text(argv[0]))==0 || parseDateOrTime((char*)z, p) ){ - return 1; + int eType; + memset(p, 0, sizeof(*p)); + if( argc==0 ){ + setDateTimeToCurrent(context, p); + }else if( (eType = sqlite3_value_type(argv[0]))==SQLITE_FLOAT + || eType==SQLITE_INTEGER ){ + p->iJD = (sqlite3_int64)(sqlite3_value_double(argv[0])*86400000.0 + 0.5); + p->validJD = 1; + }else{ + z = sqlite3_value_text(argv[0]); + if( !z || parseDateOrTime(context, (char*)z, p) ){ + return 1; + } } for(i=1; iSQLITE_MAX_LENGTH ){ + }else if( n>(u64)db->aLimit[SQLITE_LIMIT_LENGTH] ){ sqlite3_result_error_toobig(context); return; }else{ - z = sqliteMalloc( n ); - if( z==0 ) return; + z = sqlite3DbMallocRaw(db, (int)n); + if( z==0 ){ + sqlite3_result_error_nomem(context); + return; + } } computeJD(&x); computeYMD_HMS(&x); @@ -833,7 +912,7 @@ static void strftimeFunc( double s = x.s; if( s>59.999 ) s = 59.999; sqlite3_snprintf(7, &z[j],"%06.3f", s); - j += strlen(&z[j]); + j += sqlite3Strlen30(&z[j]); break; } case 'H': sqlite3_snprintf(3, &z[j],"%02d",x.h); j+=2; break; @@ -845,10 +924,10 @@ static void strftimeFunc( y.M = 1; y.D = 1; computeJD(&y); - nDay = x.rJD - y.rJD + 0.5; + nDay = (int)((x.iJD-y.iJD+43200000)/86400000); if( zFmt[i]=='W' ){ int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */ - wd = ((int)(x.rJD+0.5)) % 7; + wd = (int)(((x.iJD+43200000)/86400000)%7); sqlite3_snprintf(3, &z[j],"%02d",(nDay+7-wd)/7); j += 2; }else{ @@ -858,30 +937,34 @@ static void strftimeFunc( break; } case 'J': { - sqlite3_snprintf(20, &z[j],"%.16g",x.rJD); - j+=strlen(&z[j]); + sqlite3_snprintf(20, &z[j],"%.16g",x.iJD/86400000.0); + j+=sqlite3Strlen30(&z[j]); break; } case 'm': sqlite3_snprintf(3, &z[j],"%02d",x.M); j+=2; break; case 'M': sqlite3_snprintf(3, &z[j],"%02d",x.m); j+=2; break; case 's': { sqlite3_snprintf(30,&z[j],"%d", - (int)((x.rJD-2440587.5)*86400.0 + 0.5)); - j += strlen(&z[j]); + (int)(x.iJD/1000.0 - 210866760000.0)); + j += sqlite3Strlen30(&z[j]); break; } case 'S': sqlite3_snprintf(3,&z[j],"%02d",(int)x.s); j+=2; break; - case 'w': z[j++] = (((int)(x.rJD+1.5)) % 7) + '0'; break; - case 'Y': sqlite3_snprintf(5,&z[j],"%04d",x.Y); j+=strlen(&z[j]);break; - case '%': z[j++] = '%'; break; + case 'w': { + z[j++] = (char)(((x.iJD+129600000)/86400000) % 7) + '0'; + break; + } + case 'Y': { + sqlite3_snprintf(5,&z[j],"%04d",x.Y); j+=sqlite3Strlen30(&z[j]); + break; + } + default: z[j++] = '%'; break; } } } z[j] = 0; - sqlite3_result_text(context, z, -1, SQLITE_TRANSIENT); - if( z!=zBuf ){ - sqliteFree(z); - } + sqlite3_result_text(context, z, -1, + z==zBuf ? SQLITE_TRANSIENT : SQLITE_DYNAMIC); } /* @@ -891,15 +974,11 @@ static void strftimeFunc( */ static void ctimeFunc( sqlite3_context *context, - int argc, - sqlite3_value **argv + int NotUsed, + sqlite3_value **NotUsed2 ){ - sqlite3_value *pVal = sqlite3ValueNew(); - if( pVal ){ - sqlite3ValueSetStr(pVal, -1, "now", SQLITE_UTF8, SQLITE_STATIC); - timeFunc(context, 1, &pVal); - sqlite3ValueFree(pVal); - } + UNUSED_PARAMETER2(NotUsed, NotUsed2); + timeFunc(context, 0, 0); } /* @@ -909,15 +988,11 @@ static void ctimeFunc( */ static void cdateFunc( sqlite3_context *context, - int argc, - sqlite3_value **argv + int NotUsed, + sqlite3_value **NotUsed2 ){ - sqlite3_value *pVal = sqlite3ValueNew(); - if( pVal ){ - sqlite3ValueSetStr(pVal, -1, "now", SQLITE_UTF8, SQLITE_STATIC); - dateFunc(context, 1, &pVal); - sqlite3ValueFree(pVal); - } + UNUSED_PARAMETER2(NotUsed, NotUsed2); + dateFunc(context, 0, 0); } /* @@ -927,15 +1002,11 @@ static void cdateFunc( */ static void ctimestampFunc( sqlite3_context *context, - int argc, - sqlite3_value **argv + int NotUsed, + sqlite3_value **NotUsed2 ){ - sqlite3_value *pVal = sqlite3ValueNew(); - if( pVal ){ - sqlite3ValueSetStr(pVal, -1, "now", SQLITE_UTF8, SQLITE_STATIC); - datetimeFunc(context, 1, &pVal); - sqlite3ValueFree(pVal); - } + UNUSED_PARAMETER2(NotUsed, NotUsed2); + datetimeFunc(context, 0, 0); } #endif /* !defined(SQLITE_OMIT_DATETIME_FUNCS) */ @@ -958,18 +1029,13 @@ static void currentTimeFunc( ){ time_t t; char *zFormat = (char *)sqlite3_user_data(context); + sqlite3 *db; + double rT; char zBuf[20]; - time(&t); -#ifdef SQLITE_TEST - { - extern int sqlite3_current_time; /* See os_XXX.c */ - if( sqlite3_current_time ){ - t = sqlite3_current_time; - } - } -#endif - + db = sqlite3_context_db_handle(context); + sqlite3OsCurrentTime(db->pVfs, &rT); + t = 86400.0*(rT - 2440587.5) + 0.5; #ifdef HAVE_GMTIME_R { struct tm sNow; @@ -979,10 +1045,10 @@ static void currentTimeFunc( #else { struct tm *pTm; - sqlite3OsEnterMutex(); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); pTm = gmtime(&t); strftime(zBuf, 20, zFormat, pTm); - sqlite3OsLeaveMutex(); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } #endif @@ -995,42 +1061,28 @@ static void currentTimeFunc( ** functions. This should be the only routine in this file with ** external linkage. */ -void sqlite3RegisterDateTimeFunctions(sqlite3 *db){ +void sqlite3RegisterDateTimeFunctions(void){ + static SQLITE_WSD FuncDef aDateTimeFuncs[] = { #ifndef SQLITE_OMIT_DATETIME_FUNCS - static const struct { - char *zName; - int nArg; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } aFuncs[] = { - { "julianday", -1, juliandayFunc }, - { "date", -1, dateFunc }, - { "time", -1, timeFunc }, - { "datetime", -1, datetimeFunc }, - { "strftime", -1, strftimeFunc }, - { "current_time", 0, ctimeFunc }, - { "current_timestamp", 0, ctimestampFunc }, - { "current_date", 0, cdateFunc }, - }; - int i; - - for(i=0; ia; inSrc; i++, pItem++){ - pTab = sqlite3LocateTable(pParse, pItem->zName, pItem->zDatabase); - sqlite3DeleteTable(pItem->pTab); - pItem->pTab = pTab; - if( pTab ){ - pTab->nRef++; - } + struct SrcList_item *pItem = pSrc->a; + Table *pTab; + assert( pItem && pSrc->nSrc==1 ); + pTab = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); + sqlite3DeleteTable(pItem->pTab); + pItem->pTab = pTab; + if( pTab ){ + pTab->nRef++; + } + if( sqlite3IndexedByLookup(pParse, pItem) ){ + pTab = 0; } return pTab; } @@ -42,7 +43,8 @@ Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ ** writable return 0; */ int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ - if( (pTab->readOnly && (pParse->db->flags & SQLITE_WriteSchema)==0 + if( ((pTab->tabFlags & TF_Readonly)!=0 + && (pParse->db->flags & SQLITE_WriteSchema)==0 && pParse->nested==0) #ifndef SQLITE_OMIT_VIRTUALTABLE || (pTab->pMod && pTab->pMod->pModule->xUpdate==0) @@ -74,14 +76,138 @@ void sqlite3OpenTable( if( IsVirtual(pTab) ) return; v = sqlite3GetVdbe(p); assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); - sqlite3TableLock(p, iDb, pTab->tnum, (opcode==OP_OpenWrite), pTab->zName); - sqlite3VdbeAddOp(v, OP_Integer, iDb, 0); - VdbeComment((v, "# %s", pTab->zName)); - sqlite3VdbeAddOp(v, opcode, iCur, pTab->tnum); - sqlite3VdbeAddOp(v, OP_SetNumColumns, iCur, pTab->nCol); + sqlite3TableLock(p, iDb, pTab->tnum, (opcode==OP_OpenWrite)?1:0, pTab->zName); + sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, pTab->nCol); + sqlite3VdbeAddOp3(v, opcode, iCur, pTab->tnum, iDb); + VdbeComment((v, "%s", pTab->zName)); } +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) +/* +** Evaluate a view and store its result in an ephemeral table. The +** pWhere argument is an optional WHERE clause that restricts the +** set of rows in the view that are to be added to the ephemeral table. +*/ +void sqlite3MaterializeView( + Parse *pParse, /* Parsing context */ + Table *pView, /* View definition */ + Expr *pWhere, /* Optional WHERE clause to be added */ + int iCur /* Cursor number for ephemerial table */ +){ + SelectDest dest; + Select *pDup; + sqlite3 *db = pParse->db; + + pDup = sqlite3SelectDup(db, pView->pSelect); + if( pWhere ){ + SrcList *pFrom; + Token viewName; + + pWhere = sqlite3ExprDup(db, pWhere); + viewName.z = (u8*)pView->zName; + viewName.n = (unsigned int)sqlite3Strlen30((const char*)viewName.z); + pFrom = sqlite3SrcListAppendFromTerm(pParse, 0, 0, 0, &viewName, pDup, 0,0); + pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); + } + sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); + sqlite3Select(pParse, pDup, &dest); + sqlite3SelectDelete(db, pDup); +} +#endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */ + +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) +/* +** Generate an expression tree to implement the WHERE, ORDER BY, +** and LIMIT/OFFSET portion of DELETE and UPDATE statements. +** +** DELETE FROM table_wxyz WHERE a<5 ORDER BY a LIMIT 1; +** \__________________________/ +** pLimitWhere (pInClause) +*/ +Expr *sqlite3LimitWhere( + Parse *pParse, /* The parser context */ + SrcList *pSrc, /* the FROM clause -- which tables to scan */ + Expr *pWhere, /* The WHERE clause. May be null */ + ExprList *pOrderBy, /* The ORDER BY clause. May be null */ + Expr *pLimit, /* The LIMIT clause. May be null */ + Expr *pOffset, /* The OFFSET clause. May be null */ + char *zStmtType /* Either DELETE or UPDATE. For error messages. */ +){ + Expr *pWhereRowid = NULL; /* WHERE rowid .. */ + Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ + Expr *pSelectRowid = NULL; /* SELECT rowid ... */ + ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ + SrcList *pSelectSrc = NULL; /* SELECT rowid FROM x ... (dup of pSrc) */ + Select *pSelect = NULL; /* Complete SELECT tree */ + + /* Check that there isn't an ORDER BY without a LIMIT clause. + */ + if( pOrderBy && (pLimit == 0) ) { + sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); + pParse->parseError = 1; + goto limit_where_cleanup_2; + } + + /* We only need to generate a select expression if there + ** is a limit/offset term to enforce. + */ + if( pLimit == 0 ) { + /* if pLimit is null, pOffset will always be null as well. */ + assert( pOffset == 0 ); + return pWhere; + } + + /* Generate a select expression tree to enforce the limit/offset + ** term for the DELETE or UPDATE statement. For example: + ** DELETE FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 + ** becomes: + ** DELETE FROM table_a WHERE rowid IN ( + ** SELECT rowid FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 + ** ); + */ + + pSelectRowid = sqlite3Expr(pParse->db, TK_ROW, 0, 0, 0); + if( pSelectRowid == 0 ) goto limit_where_cleanup_2; + pEList = sqlite3ExprListAppend(pParse, 0, pSelectRowid, 0); + if( pEList == 0 ) goto limit_where_cleanup_2; + + /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree + ** and the SELECT subtree. */ + pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc); + if( pSelectSrc == 0 ) { + sqlite3ExprListDelete(pParse->db, pEList); + goto limit_where_cleanup_2; + } + + /* generate the SELECT expression tree. */ + pSelect = sqlite3SelectNew(pParse,pEList,pSelectSrc,pWhere,0,0,pOrderBy,0,pLimit,pOffset); + if( pSelect == 0 ) return 0; + + /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ + pWhereRowid = sqlite3Expr(pParse->db, TK_ROW, 0, 0, 0); + if( pWhereRowid == 0 ) goto limit_where_cleanup_1; + pInClause = sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0, 0); + if( pInClause == 0 ) goto limit_where_cleanup_1; + + pInClause->pSelect = pSelect; + sqlite3ExprSetHeight(pParse, pInClause); + return pInClause; + + /* something went wrong. clean up anything allocated. */ +limit_where_cleanup_1: + sqlite3SelectDelete(pParse->db, pSelect); + return 0; + +limit_where_cleanup_2: + sqlite3ExprDelete(pParse->db, pWhere); + sqlite3ExprListDelete(pParse->db, pOrderBy); + sqlite3ExprDelete(pParse->db, pLimit); + sqlite3ExprDelete(pParse->db, pOffset); + return 0; +} +#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ + /* ** Generate code for a DELETE FROM statement. ** @@ -107,18 +233,24 @@ void sqlite3DeleteFrom( int oldIdx = -1; /* Cursor for the OLD table of AFTER triggers */ NameContext sNC; /* Name context to resolve expressions in */ int iDb; /* Database number */ - int memCnt = 0; /* Memory cell used for change counting */ + int memCnt = -1; /* Memory cell used for change counting */ + int rcauth; /* Value returned by authorization callback */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ int triggers_exist = 0; /* True if any triggers exist */ #endif + int iBeginAfterTrigger = 0; /* Address of after trigger program */ + int iEndAfterTrigger = 0; /* Exit of after trigger program */ + int iBeginBeforeTrigger = 0; /* Address of before trigger program */ + int iEndBeforeTrigger = 0; /* Exit of before trigger program */ + u32 old_col_mask = 0; /* Mask of OLD.* columns in use */ sContext.pParse = 0; - if( pParse->nErr || sqlite3MallocFailed() ){ + db = pParse->db; + if( pParse->nErr || db->mallocFailed ){ goto delete_from_cleanup; } - db = pParse->db; assert( pTabList->nSrc==1 ); /* Locate the table which we want to delete. This table has to be @@ -133,7 +265,7 @@ void sqlite3DeleteFrom( ** deleted from is a view */ #ifndef SQLITE_OMIT_TRIGGER - triggers_exist = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0); + triggers_exist = sqlite3TriggersExist(pTab, TK_DELETE, 0); isView = pTab->pSelect!=0; #else # define triggers_exist 0 @@ -150,9 +282,12 @@ void sqlite3DeleteFrom( iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDbnDb ); zDb = db->aDb[iDb].zName; - if( sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb) ){ + rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb); + assert( rcauth==SQLITE_OK || rcauth==SQLITE_DENY || rcauth==SQLITE_IGNORE ); + if( rcauth==SQLITE_DENY ){ goto delete_from_cleanup; } + assert(!isView || triggers_exist); /* If pTab is really a view, make sure it has been initialized. */ @@ -166,15 +301,12 @@ void sqlite3DeleteFrom( oldIdx = pParse->nTab++; } - /* Resolve the column names in the WHERE clause. + /* Assign cursor number to the table and all its indices. */ assert( pTabList->nSrc==1 ); iCur = pTabList->a[0].iCursor = pParse->nTab++; - memset(&sNC, 0, sizeof(sNC)); - sNC.pParse = pParse; - sNC.pSrcList = pTabList; - if( sqlite3ExprResolveNames(&sNC, pWhere) ){ - goto delete_from_cleanup; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + pParse->nTab++; } /* Start the view context @@ -192,79 +324,90 @@ void sqlite3DeleteFrom( if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); sqlite3BeginWriteOperation(pParse, triggers_exist, iDb); + if( triggers_exist ){ + int orconf = ((pParse->trigStack)?pParse->trigStack->orconf:OE_Default); + int iGoto = sqlite3VdbeAddOp0(v, OP_Goto); + addr = sqlite3VdbeMakeLabel(v); + + iBeginBeforeTrigger = sqlite3VdbeCurrentAddr(v); + (void)sqlite3CodeRowTrigger(pParse, TK_DELETE, 0, TRIGGER_BEFORE, pTab, + -1, oldIdx, orconf, addr, &old_col_mask, 0); + iEndBeforeTrigger = sqlite3VdbeAddOp0(v, OP_Goto); + + iBeginAfterTrigger = sqlite3VdbeCurrentAddr(v); + (void)sqlite3CodeRowTrigger(pParse, TK_DELETE, 0, TRIGGER_AFTER, pTab, -1, + oldIdx, orconf, addr, &old_col_mask, 0); + iEndAfterTrigger = sqlite3VdbeAddOp0(v, OP_Goto); + + sqlite3VdbeJumpHere(v, iGoto); + } + /* If we are trying to delete from a view, realize that view into ** a ephemeral table. */ +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ - Select *pView = sqlite3SelectDup(pTab->pSelect); - sqlite3Select(pParse, pView, SRT_EphemTab, iCur, 0, 0, 0, 0); - sqlite3SelectDelete(pView); + sqlite3MaterializeView(pParse, pTab, pWhere, iCur); + } +#endif + + /* Resolve the column names in the WHERE clause. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + if( sqlite3ResolveExprNames(&sNC, pWhere) ){ + goto delete_from_cleanup; } /* Initialize the counter of the number of rows deleted, if ** we are counting rows. */ if( db->flags & SQLITE_CountRows ){ - memCnt = pParse->nMem++; - sqlite3VdbeAddOp(v, OP_MemInt, 0, memCnt); + memCnt = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, memCnt); } +#ifndef SQLITE_OMIT_TRUNCATE_OPTIMIZATION /* Special case: A DELETE without a WHERE clause deletes everything. ** It is easier just to erase the whole table. Note, however, that ** this means that the row change count will be incorrect. */ - if( pWhere==0 && !triggers_exist && !IsVirtual(pTab) ){ - if( db->flags & SQLITE_CountRows ){ - /* If counting rows deleted, just count the total number of - ** entries in the table. */ - int endOfLoop = sqlite3VdbeMakeLabel(v); - int addr2; - if( !isView ){ - sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); - } - sqlite3VdbeAddOp(v, OP_Rewind, iCur, sqlite3VdbeCurrentAddr(v)+2); - addr2 = sqlite3VdbeAddOp(v, OP_MemIncr, 1, memCnt); - sqlite3VdbeAddOp(v, OP_Next, iCur, addr2); - sqlite3VdbeResolveLabel(v, endOfLoop); - sqlite3VdbeAddOp(v, OP_Close, iCur, 0); + if( rcauth==SQLITE_OK && pWhere==0 && !triggers_exist && !IsVirtual(pTab) ){ + assert( !isView ); + sqlite3VdbeAddOp3(v, OP_Clear, pTab->tnum, iDb, memCnt); + if( !pParse->nested ){ + sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_STATIC); } - if( !isView ){ - sqlite3VdbeAddOp(v, OP_Clear, pTab->tnum, iDb); - if( !pParse->nested ){ - sqlite3VdbeChangeP3(v, -1, pTab->zName, P3_STATIC); - } - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - assert( pIdx->pSchema==pTab->pSchema ); - sqlite3VdbeAddOp(v, OP_Clear, pIdx->tnum, iDb); - } + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->pSchema==pTab->pSchema ); + sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); } - } + }else +#endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ /* The usual case: There is a WHERE clause so we have to scan through ** the table and pick which records to delete. */ - else{ - /* Begin the database scan + { + int iRowid = ++pParse->nMem; /* Used for storing rowid values. */ + int iRowSet = ++pParse->nMem; /* Register for rowset of rows to delete */ + + /* Collect rowids of every row to be deleted. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0); + sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, + WHERE_FILL_ROWSET, iRowSet); if( pWInfo==0 ) goto delete_from_cleanup; - - /* Remember the rowid of every item to be deleted. - */ - sqlite3VdbeAddOp(v, IsVirtual(pTab) ? OP_VRowid : OP_Rowid, iCur, 0); - sqlite3VdbeAddOp(v, OP_FifoWrite, 0, 0); if( db->flags & SQLITE_CountRows ){ - sqlite3VdbeAddOp(v, OP_MemIncr, 1, memCnt); + sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); } - - /* End the database scan loop. - */ sqlite3WhereEnd(pWInfo); /* Open the pseudo-table used to store OLD if there are triggers. */ if( triggers_exist ){ - sqlite3VdbeAddOp(v, OP_OpenPseudo, oldIdx, 0); - sqlite3VdbeAddOp(v, OP_SetNumColumns, oldIdx, pTab->nCol); + sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, pTab->nCol); + sqlite3VdbeAddOp1(v, OP_OpenPseudo, oldIdx); } /* Delete every item whose key was written to the list during the @@ -273,52 +416,53 @@ void sqlite3DeleteFrom( */ end = sqlite3VdbeMakeLabel(v); - /* This is the beginning of the delete loop when there are - ** row triggers. + if( !isView ){ + /* Open cursors for the table we are deleting from and + ** all its indices. + */ + sqlite3OpenTableAndIndices(pParse, pTab, iCur, OP_OpenWrite); + } + + /* This is the beginning of the delete loop. If a trigger encounters + ** an IGNORE constraint, it jumps back to here. */ if( triggers_exist ){ - addr = sqlite3VdbeAddOp(v, OP_FifoRead, 0, end); - if( !isView ){ - sqlite3VdbeAddOp(v, OP_Dup, 0, 0); - sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); - } - sqlite3VdbeAddOp(v, OP_MoveGe, iCur, 0); - sqlite3VdbeAddOp(v, OP_Rowid, iCur, 0); - sqlite3VdbeAddOp(v, OP_RowData, iCur, 0); - sqlite3VdbeAddOp(v, OP_Insert, oldIdx, 0); - if( !isView ){ - sqlite3VdbeAddOp(v, OP_Close, iCur, 0); - } + sqlite3VdbeResolveLabel(v, addr); + } + addr = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, end, iRowid); - (void)sqlite3CodeRowTrigger(pParse, TK_DELETE, 0, TRIGGER_BEFORE, pTab, - -1, oldIdx, (pParse->trigStack)?pParse->trigStack->orconf:OE_Default, - addr); + if( triggers_exist ){ + int iData = ++pParse->nMem; /* For storing row data of OLD table */ + + /* If the record is no longer present in the table, jump to the + ** next iteration of the loop through the contents of the fifo. + */ + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addr, iRowid); + + /* Populate the OLD.* pseudo-table */ + if( old_col_mask ){ + sqlite3VdbeAddOp2(v, OP_RowData, iCur, iData); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, iData); + } + sqlite3VdbeAddOp3(v, OP_Insert, oldIdx, iData, iRowid); + + /* Jump back and run the BEFORE triggers */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, iBeginBeforeTrigger); + sqlite3VdbeJumpHere(v, iEndBeforeTrigger); } if( !isView ){ - /* Open cursors for the table we are deleting from and all its - ** indices. If there are row triggers, this happens inside the - ** OP_FifoRead loop because the cursor have to all be closed - ** before the trigger fires. If there are no row triggers, the - ** cursors are opened only once on the outside the loop. - */ - sqlite3OpenTableAndIndices(pParse, pTab, iCur, OP_OpenWrite); - - /* This is the beginning of the delete loop when there are no - ** row triggers */ - if( !triggers_exist ){ - addr = sqlite3VdbeAddOp(v, OP_FifoRead, 0, end); - } - /* Delete the row */ #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pTab) ){ - pParse->pVirtualLock = pTab; - sqlite3VdbeOp3(v, OP_VUpdate, 0, 1, (const char*)pTab->pVtab, P3_VTAB); + const char *pVtab = (const char *)pTab->pVtab; + sqlite3VtabMakeWritable(pParse, pTab); + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iRowid, pVtab, P4_VTAB); }else #endif { - sqlite3GenerateRowDelete(db, v, pTab, iCur, pParse->nested==0); + sqlite3GenerateRowDelete(pParse, pTab, iCur, iRowid, pParse->nested==0); } } @@ -326,27 +470,21 @@ void sqlite3DeleteFrom( ** the AFTER triggers */ if( triggers_exist ){ - if( !isView ){ - for(i=1, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ - sqlite3VdbeAddOp(v, OP_Close, iCur + i, pIdx->tnum); - } - sqlite3VdbeAddOp(v, OP_Close, iCur, 0); - } - (void)sqlite3CodeRowTrigger(pParse, TK_DELETE, 0, TRIGGER_AFTER, pTab, -1, - oldIdx, (pParse->trigStack)?pParse->trigStack->orconf:OE_Default, - addr); + /* Jump back and run the AFTER triggers */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, iBeginAfterTrigger); + sqlite3VdbeJumpHere(v, iEndAfterTrigger); } /* End of the delete loop */ - sqlite3VdbeAddOp(v, OP_Goto, 0, addr); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); sqlite3VdbeResolveLabel(v, end); /* Close the cursors after the loop if there are no row triggers */ - if( !triggers_exist && !IsVirtual(pTab) ){ + if( !isView && !IsVirtual(pTab) ){ for(i=1, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ - sqlite3VdbeAddOp(v, OP_Close, iCur + i, pIdx->tnum); + sqlite3VdbeAddOp2(v, OP_Close, iCur + i, pIdx->tnum); } - sqlite3VdbeAddOp(v, OP_Close, iCur, 0); + sqlite3VdbeAddOp1(v, OP_Close, iCur); } } @@ -356,16 +494,15 @@ void sqlite3DeleteFrom( ** invoke the callback function. */ if( db->flags & SQLITE_CountRows && pParse->nested==0 && !pParse->trigStack ){ - sqlite3VdbeAddOp(v, OP_MemLoad, memCnt, 0); - sqlite3VdbeAddOp(v, OP_Callback, 1, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, memCnt, 1); sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", P3_STATIC); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", SQLITE_STATIC); } delete_from_cleanup: sqlite3AuthContextPop(&sContext); - sqlite3SrcListDelete(pTabList); - sqlite3ExprDelete(pWhere); + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprDelete(db, pWhere); return; } @@ -382,26 +519,29 @@ delete_from_cleanup: ** 2. Read/write cursors for all indices of pTab must be open as ** cursor number base+i for the i-th index. ** -** 3. The record number of the row to be deleted must be on the top -** of the stack. +** 3. The record number of the row to be deleted must be stored in +** memory cell iRowid. ** ** This routine pops the top of the stack to remove the record number ** and then generates code to remove both the table record and all index ** entries that point to that record. */ void sqlite3GenerateRowDelete( - sqlite3 *db, /* The database containing the index */ - Vdbe *v, /* Generate code into this VDBE */ + Parse *pParse, /* Parsing context */ Table *pTab, /* Table containing the row to be deleted */ int iCur, /* Cursor number for the table */ + int iRowid, /* Memory cell that contains the rowid to delete */ int count /* Increment the row change counter */ ){ int addr; - addr = sqlite3VdbeAddOp(v, OP_NotExists, iCur, 0); - sqlite3GenerateRowIndexDelete(v, pTab, iCur, 0); - sqlite3VdbeAddOp(v, OP_Delete, iCur, (count?OPFLAG_NCHANGE:0)); + Vdbe *v; + + v = pParse->pVdbe; + addr = sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowid); + sqlite3GenerateRowIndexDelete(pParse, pTab, iCur, 0); + sqlite3VdbeAddOp2(v, OP_Delete, iCur, (count?OPFLAG_NCHANGE:0)); if( count ){ - sqlite3VdbeChangeP3(v, -1, pTab->zName, P3_STATIC); + sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_STATIC); } sqlite3VdbeJumpHere(v, addr); } @@ -423,45 +563,68 @@ void sqlite3GenerateRowDelete( ** deleted. */ void sqlite3GenerateRowIndexDelete( - Vdbe *v, /* Generate code into this VDBE */ + Parse *pParse, /* Parsing and code generating context */ Table *pTab, /* Table containing the row to be deleted */ int iCur, /* Cursor number for the table */ - char *aIdxUsed /* Only delete if aIdxUsed!=0 && aIdxUsed[i]!=0 */ + int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ ){ int i; Index *pIdx; + int r1; for(i=1, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ - if( aIdxUsed!=0 && aIdxUsed[i-1]==0 ) continue; - sqlite3GenerateIndexKey(v, pIdx, iCur); - sqlite3VdbeAddOp(v, OP_IdxDelete, iCur+i, 0); + if( aRegIdx!=0 && aRegIdx[i-1]==0 ) continue; + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iCur, 0, 0); + sqlite3VdbeAddOp3(pParse->pVdbe, OP_IdxDelete, iCur+i, r1,pIdx->nColumn+1); } } /* -** Generate code that will assemble an index key and put it on the top -** of the tack. The key with be for index pIdx which is an index on pTab. +** Generate code that will assemble an index key and put it in register +** regOut. The key with be for index pIdx which is an index on pTab. ** iCur is the index of a cursor open on the pTab table and pointing to ** the entry that needs indexing. +** +** Return a register number which is the first in a block of +** registers that holds the elements of the index key. The +** block of registers has already been deallocated by the time +** this routine returns. */ -void sqlite3GenerateIndexKey( - Vdbe *v, /* Generate code into this VDBE */ +int sqlite3GenerateIndexKey( + Parse *pParse, /* Parsing context */ Index *pIdx, /* The index for which to generate a key */ - int iCur /* Cursor number for the pIdx->pTable table */ + int iCur, /* Cursor number for the pIdx->pTable table */ + int regOut, /* Write the new index key to this register */ + int doMakeRec /* Run the OP_MakeRecord instruction if true */ ){ + Vdbe *v = pParse->pVdbe; int j; Table *pTab = pIdx->pTable; + int regBase; + int nCol; - sqlite3VdbeAddOp(v, OP_Rowid, iCur, 0); - for(j=0; jnColumn; j++){ + nCol = pIdx->nColumn; + regBase = sqlite3GetTempRange(pParse, nCol+1); + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, regBase+nCol); + for(j=0; jaiColumn[j]; if( idx==pTab->iPKey ){ - sqlite3VdbeAddOp(v, OP_Dup, j, 0); + sqlite3VdbeAddOp2(v, OP_SCopy, regBase+nCol, regBase+j); }else{ - sqlite3VdbeAddOp(v, OP_Column, iCur, idx); + sqlite3VdbeAddOp3(v, OP_Column, iCur, idx, regBase+j); sqlite3ColumnDefault(v, pTab, idx); } } - sqlite3VdbeAddOp(v, OP_MakeIdxRec, pIdx->nColumn, 0); - sqlite3IndexAffinityStr(v, pIdx); + if( doMakeRec ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol+1, regOut); + sqlite3IndexAffinityStr(v, pIdx); + sqlite3ExprCacheAffinityChange(pParse, regBase, nCol+1); + } + sqlite3ReleaseTempRange(pParse, regBase, nCol+1); + return regBase; } + +/* Make sure "isView" gets undefined in case this file becomes part of +** the amalgamation - so that subsequent files do not see isView as a +** macro. */ +#undef isView diff --git a/expr.c b/expr.c index b640bec..6e11471 100644 --- a/expr.c +++ b/expr.c @@ -12,7 +12,7 @@ ** This file contains routines used for analyzing expressions and ** for generating VDBE code that evaluates expressions in SQLite. ** -** $Id: expr.c,v 1.303 2007/08/07 17:13:04 drh Exp $ +** $Id: expr.c,v 1.409 2009/01/10 13:24:51 drh Exp $ */ #include "sqliteInt.h" #include @@ -43,6 +43,16 @@ char sqlite3ExprAffinity(Expr *pExpr){ return sqlite3AffinityType(&pExpr->token); } #endif + if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER) + && pExpr->pTab!=0 + ){ + /* op==TK_REGISTER && pExpr->pTab!=0 happens when pExpr was originally + ** a TK_COLUMN but was previously evaluated and cached in a register */ + int j = pExpr->iColumn; + if( j<0 ) return SQLITE_AFF_INTEGER; + assert( pExpr->pTab && jpTab->nCol ); + return pExpr->pTab->aCol[j].affinity; + } return pExpr->affinity; } @@ -53,14 +63,19 @@ char sqlite3ExprAffinity(Expr *pExpr){ ** flag. An explicit collating sequence will override implicit ** collating sequences. */ -Expr *sqlite3ExprSetColl(Parse *pParse, Expr *pExpr, Token *pName){ +Expr *sqlite3ExprSetColl(Parse *pParse, Expr *pExpr, Token *pCollName){ + char *zColl = 0; /* Dequoted name of collation sequence */ CollSeq *pColl; - if( pExpr==0 ) return 0; - pColl = sqlite3LocateCollSeq(pParse, (char*)pName->z, pName->n); - if( pColl ){ - pExpr->pColl = pColl; - pExpr->flags |= EP_ExpCollate; + sqlite3 *db = pParse->db; + zColl = sqlite3NameFromToken(db, pCollName); + if( pExpr && zColl ){ + pColl = sqlite3LocateCollSeq(pParse, zColl, -1); + if( pColl ){ + pExpr->pColl = pColl; + pExpr->flags |= EP_ExpCollate; + } } + sqlite3DbFree(db, zColl); return pExpr; } @@ -70,13 +85,29 @@ Expr *sqlite3ExprSetColl(Parse *pParse, Expr *pExpr, Token *pName){ */ CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ CollSeq *pColl = 0; - if( pExpr ){ + Expr *p = pExpr; + while( p ){ int op; - pColl = pExpr->pColl; - op = pExpr->op; - if( (op==TK_CAST || op==TK_UPLUS) && !pColl ){ - return sqlite3ExprCollSeq(pParse, pExpr->pLeft); + pColl = p->pColl; + if( pColl ) break; + op = p->op; + if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER) && p->pTab!=0 ){ + /* op==TK_REGISTER && p->pTab!=0 happens when pExpr was originally + ** a TK_COLUMN but was previously evaluated and cached in a register */ + const char *zColl; + int j = p->iColumn; + if( j>=0 ){ + sqlite3 *db = pParse->db; + zColl = p->pTab->aCol[j].zColl; + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, -1, 0); + pExpr->pColl = pColl; + } + break; } + if( op!=TK_CAST && op!=TK_UPLUS ){ + break; + } + p = p->pLeft; } if( sqlite3CheckCollSeq(pParse, pColl) ){ pColl = 0; @@ -154,15 +185,13 @@ int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity){ } /* -** Return the P1 value that should be used for a binary comparison +** Return the P5 value that should be used for a binary comparison ** opcode (OP_Eq, OP_Ge etc.) used to compare pExpr1 and pExpr2. -** If jumpIfNull is true, then set the low byte of the returned -** P1 value to tell the opcode to jump if either expression -** evaluates to NULL. */ -static int binaryCompareP1(Expr *pExpr1, Expr *pExpr2, int jumpIfNull){ - char aff = sqlite3ExprAffinity(pExpr2); - return ((int)sqlite3CompareAffinity(pExpr1, aff))+(jumpIfNull?0x100:0); +static u8 binaryCompareP5(Expr *pExpr1, Expr *pExpr2, int jumpIfNull){ + u8 aff = (char)sqlite3ExprAffinity(pExpr2); + aff = (u8)sqlite3CompareAffinity(pExpr1, aff) | (u8)jumpIfNull; + return aff; } /* @@ -199,6 +228,30 @@ CollSeq *sqlite3BinaryCompareCollSeq( return pColl; } +/* +** Generate the operands for a comparison operation. Before +** generating the code for each operand, set the EP_AnyAff +** flag on the expression so that it will be able to used a +** cached column value that has previously undergone an +** affinity change. +*/ +static void codeCompareOperands( + Parse *pParse, /* Parsing and code generating context */ + Expr *pLeft, /* The left operand */ + int *pRegLeft, /* Register where left operand is stored */ + int *pFreeLeft, /* Free this register when done */ + Expr *pRight, /* The right operand */ + int *pRegRight, /* Register where right operand is stored */ + int *pFreeRight /* Write temp register for right operand there */ +){ + while( pLeft->op==TK_UPLUS ) pLeft = pLeft->pLeft; + pLeft->flags |= EP_AnyAff; + *pRegLeft = sqlite3ExprCodeTemp(pParse, pLeft, pFreeLeft); + while( pRight->op==TK_UPLUS ) pRight = pRight->pLeft; + pRight->flags |= EP_AnyAff; + *pRegRight = sqlite3ExprCodeTemp(pParse, pRight, pFreeRight); +} + /* ** Generate code for a comparison operator. */ @@ -207,480 +260,44 @@ static int codeCompare( Expr *pLeft, /* The left operand */ Expr *pRight, /* The right operand */ int opcode, /* The comparison opcode */ + int in1, int in2, /* Register holding operands */ int dest, /* Jump here if true. */ int jumpIfNull /* If true, jump if either operand is NULL */ ){ - int p1 = binaryCompareP1(pLeft, pRight, jumpIfNull); - CollSeq *p3 = sqlite3BinaryCompareCollSeq(pParse, pLeft, pRight); - return sqlite3VdbeOp3(pParse->pVdbe, opcode, p1, dest, (void*)p3, P3_COLLSEQ); -} + int p5; + int addr; + CollSeq *p4; -/* -** Construct a new expression node and return a pointer to it. Memory -** for this node is obtained from sqliteMalloc(). The calling function -** is responsible for making sure the node eventually gets freed. -*/ -Expr *sqlite3Expr(int op, Expr *pLeft, Expr *pRight, const Token *pToken){ - Expr *pNew; - pNew = sqliteMalloc( sizeof(Expr) ); - if( pNew==0 ){ - /* When malloc fails, delete pLeft and pRight. Expressions passed to - ** this function must always be allocated with sqlite3Expr() for this - ** reason. - */ - sqlite3ExprDelete(pLeft); - sqlite3ExprDelete(pRight); - return 0; + p4 = sqlite3BinaryCompareCollSeq(pParse, pLeft, pRight); + p5 = binaryCompareP5(pLeft, pRight, jumpIfNull); + addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1, + (void*)p4, P4_COLLSEQ); + sqlite3VdbeChangeP5(pParse->pVdbe, (u8)p5); + if( (p5 & SQLITE_AFF_MASK)!=SQLITE_AFF_NONE ){ + sqlite3ExprCacheAffinityChange(pParse, in1, 1); + sqlite3ExprCacheAffinityChange(pParse, in2, 1); } - pNew->op = op; - pNew->pLeft = pLeft; - pNew->pRight = pRight; - pNew->iAgg = -1; - if( pToken ){ - assert( pToken->dyn==0 ); - pNew->span = pNew->token = *pToken; - }else if( pLeft ){ - if( pRight ){ - sqlite3ExprSpan(pNew, &pLeft->span, &pRight->span); - if( pRight->flags & EP_ExpCollate ){ - pNew->flags |= EP_ExpCollate; - pNew->pColl = pRight->pColl; - } - } - if( pLeft->flags & EP_ExpCollate ){ - pNew->flags |= EP_ExpCollate; - pNew->pColl = pLeft->pColl; - } - } - - sqlite3ExprSetHeight(pNew); - return pNew; + return addr; } -/* -** Works like sqlite3Expr() but frees its pLeft and pRight arguments -** if it fails due to a malloc problem. -*/ -Expr *sqlite3ExprOrFree(int op, Expr *pLeft, Expr *pRight, const Token *pToken){ - Expr *pNew = sqlite3Expr(op, pLeft, pRight, pToken); - if( pNew==0 ){ - sqlite3ExprDelete(pLeft); - sqlite3ExprDelete(pRight); - } - return pNew; -} - -/* -** When doing a nested parse, you can include terms in an expression -** that look like this: #0 #1 #2 ... These terms refer to elements -** on the stack. "#0" means the top of the stack. -** "#1" means the next down on the stack. And so forth. -** -** This routine is called by the parser to deal with on of those terms. -** It immediately generates code to store the value in a memory location. -** The returns an expression that will code to extract the value from -** that memory location as needed. -*/ -Expr *sqlite3RegisterExpr(Parse *pParse, Token *pToken){ - Vdbe *v = pParse->pVdbe; - Expr *p; - int depth; - if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", pToken); - return sqlite3Expr(TK_NULL, 0, 0, 0); - } - if( v==0 ) return 0; - p = sqlite3Expr(TK_REGISTER, 0, 0, pToken); - if( p==0 ){ - return 0; /* Malloc failed */ - } - depth = atoi((char*)&pToken->z[1]); - p->iTable = pParse->nMem++; - sqlite3VdbeAddOp(v, OP_Dup, depth, 0); - sqlite3VdbeAddOp(v, OP_MemStore, p->iTable, 1); - return p; -} - -/* -** Join two expressions using an AND operator. If either expression is -** NULL, then just return the other expression. -*/ -Expr *sqlite3ExprAnd(Expr *pLeft, Expr *pRight){ - if( pLeft==0 ){ - return pRight; - }else if( pRight==0 ){ - return pLeft; - }else{ - return sqlite3Expr(TK_AND, pLeft, pRight, 0); - } -} - -/* -** Set the Expr.span field of the given expression to span all -** text between the two given tokens. -*/ -void sqlite3ExprSpan(Expr *pExpr, Token *pLeft, Token *pRight){ - assert( pRight!=0 ); - assert( pLeft!=0 ); - if( !sqlite3MallocFailed() && pRight->z && pLeft->z ){ - assert( pLeft->dyn==0 || pLeft->z[pLeft->n]==0 ); - if( pLeft->dyn==0 && pRight->dyn==0 ){ - pExpr->span.z = pLeft->z; - pExpr->span.n = pRight->n + (pRight->z - pLeft->z); - }else{ - pExpr->span.z = 0; - } - } -} - -/* -** Construct a new expression node for a function with multiple -** arguments. -*/ -Expr *sqlite3ExprFunction(ExprList *pList, Token *pToken){ - Expr *pNew; - assert( pToken ); - pNew = sqliteMalloc( sizeof(Expr) ); - if( pNew==0 ){ - sqlite3ExprListDelete(pList); /* Avoid leaking memory when malloc fails */ - return 0; - } - pNew->op = TK_FUNCTION; - pNew->pList = pList; - assert( pToken->dyn==0 ); - pNew->token = *pToken; - pNew->span = pNew->token; - - sqlite3ExprSetHeight(pNew); - return pNew; -} - -/* -** Assign a variable number to an expression that encodes a wildcard -** in the original SQL statement. -** -** Wildcards consisting of a single "?" are assigned the next sequential -** variable number. -** -** Wildcards of the form "?nnn" are assigned the number "nnn". We make -** sure "nnn" is not too be to avoid a denial of service attack when -** the SQL statement comes from an external source. -** -** Wildcards of the form ":aaa" or "$aaa" are assigned the same number -** as the previous instance of the same wildcard. Or if this is the first -** instance of the wildcard, the next sequenial variable number is -** assigned. -*/ -void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ - Token *pToken; - if( pExpr==0 ) return; - pToken = &pExpr->token; - assert( pToken->n>=1 ); - assert( pToken->z!=0 ); - assert( pToken->z[0]!=0 ); - if( pToken->n==1 ){ - /* Wildcard of the form "?". Assign the next variable number */ - pExpr->iTable = ++pParse->nVar; - }else if( pToken->z[0]=='?' ){ - /* Wildcard of the form "?nnn". Convert "nnn" to an integer and - ** use it as the variable number */ - int i; - pExpr->iTable = i = atoi((char*)&pToken->z[1]); - if( i<1 || i>SQLITE_MAX_VARIABLE_NUMBER ){ - sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", - SQLITE_MAX_VARIABLE_NUMBER); - } - if( i>pParse->nVar ){ - pParse->nVar = i; - } - }else{ - /* Wildcards of the form ":aaa" or "$aaa". Reuse the same variable - ** number as the prior appearance of the same name, or if the name - ** has never appeared before, reuse the same variable number - */ - int i, n; - n = pToken->n; - for(i=0; inVarExpr; i++){ - Expr *pE; - if( (pE = pParse->apVarExpr[i])!=0 - && pE->token.n==n - && memcmp(pE->token.z, pToken->z, n)==0 ){ - pExpr->iTable = pE->iTable; - break; - } - } - if( i>=pParse->nVarExpr ){ - pExpr->iTable = ++pParse->nVar; - if( pParse->nVarExpr>=pParse->nVarExprAlloc-1 ){ - pParse->nVarExprAlloc += pParse->nVarExprAlloc + 10; - pParse->apVarExpr = sqliteReallocOrFree(pParse->apVarExpr, - pParse->nVarExprAlloc*sizeof(pParse->apVarExpr[0]) ); - } - if( !sqlite3MallocFailed() ){ - assert( pParse->apVarExpr!=0 ); - pParse->apVarExpr[pParse->nVarExpr++] = pExpr; - } - } - } - if( !pParse->nErr && pParse->nVar>SQLITE_MAX_VARIABLE_NUMBER ){ - sqlite3ErrorMsg(pParse, "too many SQL variables"); - } -} - -/* -** Recursively delete an expression tree. -*/ -void sqlite3ExprDelete(Expr *p){ - if( p==0 ) return; - if( p->span.dyn ) sqliteFree((char*)p->span.z); - if( p->token.dyn ) sqliteFree((char*)p->token.z); - sqlite3ExprDelete(p->pLeft); - sqlite3ExprDelete(p->pRight); - sqlite3ExprListDelete(p->pList); - sqlite3SelectDelete(p->pSelect); - sqliteFree(p); -} - -/* -** The Expr.token field might be a string literal that is quoted. -** If so, remove the quotation marks. -*/ -void sqlite3DequoteExpr(Expr *p){ - if( ExprHasAnyProperty(p, EP_Dequoted) ){ - return; - } - ExprSetProperty(p, EP_Dequoted); - if( p->token.dyn==0 ){ - sqlite3TokenCopy(&p->token, &p->token); - } - sqlite3Dequote((char*)p->token.z); -} - - -/* -** The following group of routines make deep copies of expressions, -** expression lists, ID lists, and select statements. The copies can -** be deleted (by being passed to their respective ...Delete() routines) -** without effecting the originals. -** -** The expression list, ID, and source lists return by sqlite3ExprListDup(), -** sqlite3IdListDup(), and sqlite3SrcListDup() can not be further expanded -** by subsequent calls to sqlite*ListAppend() routines. -** -** Any tables that the SrcList might point to are not duplicated. -*/ -Expr *sqlite3ExprDup(Expr *p){ - Expr *pNew; - if( p==0 ) return 0; - pNew = sqliteMallocRaw( sizeof(*p) ); - if( pNew==0 ) return 0; - memcpy(pNew, p, sizeof(*pNew)); - if( p->token.z!=0 ){ - pNew->token.z = (u8*)sqliteStrNDup((char*)p->token.z, p->token.n); - pNew->token.dyn = 1; - }else{ - assert( pNew->token.z==0 ); - } - pNew->span.z = 0; - pNew->pLeft = sqlite3ExprDup(p->pLeft); - pNew->pRight = sqlite3ExprDup(p->pRight); - pNew->pList = sqlite3ExprListDup(p->pList); - pNew->pSelect = sqlite3SelectDup(p->pSelect); - return pNew; -} -void sqlite3TokenCopy(Token *pTo, Token *pFrom){ - if( pTo->dyn ) sqliteFree((char*)pTo->z); - if( pFrom->z ){ - pTo->n = pFrom->n; - pTo->z = (u8*)sqliteStrNDup((char*)pFrom->z, pFrom->n); - pTo->dyn = 1; - }else{ - pTo->z = 0; - } -} -ExprList *sqlite3ExprListDup(ExprList *p){ - ExprList *pNew; - struct ExprList_item *pItem, *pOldItem; - int i; - if( p==0 ) return 0; - pNew = sqliteMalloc( sizeof(*pNew) ); - if( pNew==0 ) return 0; - pNew->nExpr = pNew->nAlloc = p->nExpr; - pNew->a = pItem = sqliteMalloc( p->nExpr*sizeof(p->a[0]) ); - if( pItem==0 ){ - sqliteFree(pNew); - return 0; - } - pOldItem = p->a; - for(i=0; inExpr; i++, pItem++, pOldItem++){ - Expr *pNewExpr, *pOldExpr; - pItem->pExpr = pNewExpr = sqlite3ExprDup(pOldExpr = pOldItem->pExpr); - if( pOldExpr->span.z!=0 && pNewExpr ){ - /* Always make a copy of the span for top-level expressions in the - ** expression list. The logic in SELECT processing that determines - ** the names of columns in the result set needs this information */ - sqlite3TokenCopy(&pNewExpr->span, &pOldExpr->span); - } - assert( pNewExpr==0 || pNewExpr->span.z!=0 - || pOldExpr->span.z==0 - || sqlite3MallocFailed() ); - pItem->zName = sqliteStrDup(pOldItem->zName); - pItem->sortOrder = pOldItem->sortOrder; - pItem->isAgg = pOldItem->isAgg; - pItem->done = 0; - } - return pNew; -} - -/* -** If cursors, triggers, views and subqueries are all omitted from -** the build, then none of the following routines, except for -** sqlite3SelectDup(), can be called. sqlite3SelectDup() is sometimes -** called with a NULL argument. -*/ -#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) \ - || !defined(SQLITE_OMIT_SUBQUERY) -SrcList *sqlite3SrcListDup(SrcList *p){ - SrcList *pNew; - int i; - int nByte; - if( p==0 ) return 0; - nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); - pNew = sqliteMallocRaw( nByte ); - if( pNew==0 ) return 0; - pNew->nSrc = pNew->nAlloc = p->nSrc; - for(i=0; inSrc; i++){ - struct SrcList_item *pNewItem = &pNew->a[i]; - struct SrcList_item *pOldItem = &p->a[i]; - Table *pTab; - pNewItem->zDatabase = sqliteStrDup(pOldItem->zDatabase); - pNewItem->zName = sqliteStrDup(pOldItem->zName); - pNewItem->zAlias = sqliteStrDup(pOldItem->zAlias); - pNewItem->jointype = pOldItem->jointype; - pNewItem->iCursor = pOldItem->iCursor; - pNewItem->isPopulated = pOldItem->isPopulated; - pTab = pNewItem->pTab = pOldItem->pTab; - if( pTab ){ - pTab->nRef++; - } - pNewItem->pSelect = sqlite3SelectDup(pOldItem->pSelect); - pNewItem->pOn = sqlite3ExprDup(pOldItem->pOn); - pNewItem->pUsing = sqlite3IdListDup(pOldItem->pUsing); - pNewItem->colUsed = pOldItem->colUsed; - } - return pNew; -} -IdList *sqlite3IdListDup(IdList *p){ - IdList *pNew; - int i; - if( p==0 ) return 0; - pNew = sqliteMallocRaw( sizeof(*pNew) ); - if( pNew==0 ) return 0; - pNew->nId = pNew->nAlloc = p->nId; - pNew->a = sqliteMallocRaw( p->nId*sizeof(p->a[0]) ); - if( pNew->a==0 ){ - sqliteFree(pNew); - return 0; - } - for(i=0; inId; i++){ - struct IdList_item *pNewItem = &pNew->a[i]; - struct IdList_item *pOldItem = &p->a[i]; - pNewItem->zName = sqliteStrDup(pOldItem->zName); - pNewItem->idx = pOldItem->idx; - } - return pNew; -} -Select *sqlite3SelectDup(Select *p){ - Select *pNew; - if( p==0 ) return 0; - pNew = sqliteMallocRaw( sizeof(*p) ); - if( pNew==0 ) return 0; - pNew->isDistinct = p->isDistinct; - pNew->pEList = sqlite3ExprListDup(p->pEList); - pNew->pSrc = sqlite3SrcListDup(p->pSrc); - pNew->pWhere = sqlite3ExprDup(p->pWhere); - pNew->pGroupBy = sqlite3ExprListDup(p->pGroupBy); - pNew->pHaving = sqlite3ExprDup(p->pHaving); - pNew->pOrderBy = sqlite3ExprListDup(p->pOrderBy); - pNew->op = p->op; - pNew->pPrior = sqlite3SelectDup(p->pPrior); - pNew->pLimit = sqlite3ExprDup(p->pLimit); - pNew->pOffset = sqlite3ExprDup(p->pOffset); - pNew->iLimit = -1; - pNew->iOffset = -1; - pNew->isResolved = p->isResolved; - pNew->isAgg = p->isAgg; - pNew->usesEphm = 0; - pNew->disallowOrderBy = 0; - pNew->pRightmost = 0; - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; - pNew->addrOpenEphm[2] = -1; - return pNew; -} -#else -Select *sqlite3SelectDup(Select *p){ - assert( p==0 ); - return 0; -} -#endif - - -/* -** Add a new element to the end of an expression list. If pList is -** initially NULL, then create a new expression list. -*/ -ExprList *sqlite3ExprListAppend(ExprList *pList, Expr *pExpr, Token *pName){ - if( pList==0 ){ - pList = sqliteMalloc( sizeof(ExprList) ); - if( pList==0 ){ - goto no_mem; - } - assert( pList->nAlloc==0 ); - } - if( pList->nAlloc<=pList->nExpr ){ - struct ExprList_item *a; - int n = pList->nAlloc*2 + 4; - a = sqliteRealloc(pList->a, n*sizeof(pList->a[0])); - if( a==0 ){ - goto no_mem; - } - pList->a = a; - pList->nAlloc = n; - } - assert( pList->a!=0 ); - if( pExpr || pName ){ - struct ExprList_item *pItem = &pList->a[pList->nExpr++]; - memset(pItem, 0, sizeof(*pItem)); - pItem->zName = sqlite3NameFromToken(pName); - pItem->pExpr = pExpr; - } - return pList; - -no_mem: - /* Avoid leaking memory if malloc has failed. */ - sqlite3ExprDelete(pExpr); - sqlite3ExprListDelete(pList); - return 0; -} - -/* -** If the expression list pEList contains more than iLimit elements, -** leave an error message in pParse. -*/ -void sqlite3ExprListCheckLength( - Parse *pParse, - ExprList *pEList, - int iLimit, - const char *zObject -){ - if( pEList && pEList->nExpr>iLimit ){ - sqlite3ErrorMsg(pParse, "too many columns in %s", zObject); - } -} - - #if SQLITE_MAX_EXPR_DEPTH>0 +/* +** Check that argument nHeight is less than or equal to the maximum +** expression depth allowed. If it is not, leave an error message in +** pParse. +*/ +int sqlite3ExprCheckHeight(Parse *pParse, int nHeight){ + int rc = SQLITE_OK; + int mxHeight = pParse->db->aLimit[SQLITE_LIMIT_EXPR_DEPTH]; + if( nHeight>mxHeight ){ + sqlite3ErrorMsg(pParse, + "Expression tree is too large (maximum depth %d)", mxHeight + ); + rc = SQLITE_ERROR; + } + return rc; +} + /* The following three functions, heightOfExpr(), heightOfExprList() ** and heightOfSelect(), are used to determine the maximum height ** of any expression tree referenced by the structure passed as the @@ -725,7 +342,7 @@ static void heightOfSelect(Select *p, int *pnHeight){ ** has a height equal to the maximum height of any other ** referenced Expr plus one. */ -void sqlite3ExprSetHeight(Expr *p){ +static void exprSetHeight(Expr *p){ int nHeight = 0; heightOfExpr(p->pLeft, &nHeight); heightOfExpr(p->pRight, &nHeight); @@ -734,6 +351,16 @@ void sqlite3ExprSetHeight(Expr *p){ p->nHeight = nHeight + 1; } +/* +** Set the Expr.nHeight variable using the exprSetHeight() function. If +** the height is greater than the maximum allowed expression depth, +** leave an error in pParse. +*/ +void sqlite3ExprSetHeight(Parse *pParse, Expr *p){ + exprSetHeight(p); + sqlite3ExprCheckHeight(pParse, p->nHeight); +} + /* ** Return the maximum height of any expression tree referenced ** by the select statement passed as an argument. @@ -743,132 +370,587 @@ int sqlite3SelectExprHeight(Select *p){ heightOfSelect(p, &nHeight); return nHeight; } +#else + #define exprSetHeight(y) +#endif /* SQLITE_MAX_EXPR_DEPTH>0 */ + +/* +** Construct a new expression node and return a pointer to it. Memory +** for this node is obtained from sqlite3_malloc(). The calling function +** is responsible for making sure the node eventually gets freed. +*/ +Expr *sqlite3Expr( + sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ + int op, /* Expression opcode */ + Expr *pLeft, /* Left operand */ + Expr *pRight, /* Right operand */ + const Token *pToken /* Argument token */ +){ + Expr *pNew; + pNew = sqlite3DbMallocZero(db, sizeof(Expr)); + if( pNew==0 ){ + /* When malloc fails, delete pLeft and pRight. Expressions passed to + ** this function must always be allocated with sqlite3Expr() for this + ** reason. + */ + sqlite3ExprDelete(db, pLeft); + sqlite3ExprDelete(db, pRight); + return 0; + } + pNew->op = (u8)op; + pNew->pLeft = pLeft; + pNew->pRight = pRight; + pNew->iAgg = -1; + pNew->span.z = (u8*)""; + if( pToken ){ + assert( pToken->dyn==0 ); + pNew->span = pNew->token = *pToken; + }else if( pLeft ){ + if( pRight ){ + if( pRight->span.dyn==0 && pLeft->span.dyn==0 ){ + sqlite3ExprSpan(pNew, &pLeft->span, &pRight->span); + } + if( pRight->flags & EP_ExpCollate ){ + pNew->flags |= EP_ExpCollate; + pNew->pColl = pRight->pColl; + } + } + if( pLeft->flags & EP_ExpCollate ){ + pNew->flags |= EP_ExpCollate; + pNew->pColl = pLeft->pColl; + } + } + + exprSetHeight(pNew); + return pNew; +} + +/* +** Works like sqlite3Expr() except that it takes an extra Parse* +** argument and notifies the associated connection object if malloc fails. +*/ +Expr *sqlite3PExpr( + Parse *pParse, /* Parsing context */ + int op, /* Expression opcode */ + Expr *pLeft, /* Left operand */ + Expr *pRight, /* Right operand */ + const Token *pToken /* Argument token */ +){ + Expr *p = sqlite3Expr(pParse->db, op, pLeft, pRight, pToken); + if( p ){ + sqlite3ExprCheckHeight(pParse, p->nHeight); + } + return p; +} + +/* +** When doing a nested parse, you can include terms in an expression +** that look like this: #1 #2 ... These terms refer to registers +** in the virtual machine. #N is the N-th register. +** +** This routine is called by the parser to deal with on of those terms. +** It immediately generates code to store the value in a memory location. +** The returns an expression that will code to extract the value from +** that memory location as needed. +*/ +Expr *sqlite3RegisterExpr(Parse *pParse, Token *pToken){ + Vdbe *v = pParse->pVdbe; + Expr *p; + if( pParse->nested==0 ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", pToken); + return sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); + } + if( v==0 ) return 0; + p = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, pToken); + if( p==0 ){ + return 0; /* Malloc failed */ + } + p->iTable = atoi((char*)&pToken->z[1]); + return p; +} + +/* +** Join two expressions using an AND operator. If either expression is +** NULL, then just return the other expression. +*/ +Expr *sqlite3ExprAnd(sqlite3 *db, Expr *pLeft, Expr *pRight){ + if( pLeft==0 ){ + return pRight; + }else if( pRight==0 ){ + return pLeft; + }else{ + return sqlite3Expr(db, TK_AND, pLeft, pRight, 0); + } +} + +/* +** Set the Expr.span field of the given expression to span all +** text between the two given tokens. Both tokens must be pointing +** at the same string. +*/ +void sqlite3ExprSpan(Expr *pExpr, Token *pLeft, Token *pRight){ + assert( pRight!=0 ); + assert( pLeft!=0 ); + if( pExpr ){ + pExpr->span.z = pLeft->z; + pExpr->span.n = pRight->n + (pRight->z - pLeft->z); + } +} + +/* +** Construct a new expression node for a function with multiple +** arguments. +*/ +Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ + Expr *pNew; + sqlite3 *db = pParse->db; + assert( pToken ); + pNew = sqlite3DbMallocZero(db, sizeof(Expr) ); + if( pNew==0 ){ + sqlite3ExprListDelete(db, pList); /* Avoid leaking memory when malloc fails */ + return 0; + } + pNew->op = TK_FUNCTION; + pNew->pList = pList; + assert( pToken->dyn==0 ); + pNew->token = *pToken; + pNew->span = pNew->token; + + sqlite3ExprSetHeight(pParse, pNew); + return pNew; +} + +/* +** Assign a variable number to an expression that encodes a wildcard +** in the original SQL statement. +** +** Wildcards consisting of a single "?" are assigned the next sequential +** variable number. +** +** Wildcards of the form "?nnn" are assigned the number "nnn". We make +** sure "nnn" is not too be to avoid a denial of service attack when +** the SQL statement comes from an external source. +** +** Wildcards of the form ":aaa" or "$aaa" are assigned the same number +** as the previous instance of the same wildcard. Or if this is the first +** instance of the wildcard, the next sequenial variable number is +** assigned. +*/ +void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ + Token *pToken; + sqlite3 *db = pParse->db; + + if( pExpr==0 ) return; + pToken = &pExpr->token; + assert( pToken->n>=1 ); + assert( pToken->z!=0 ); + assert( pToken->z[0]!=0 ); + if( pToken->n==1 ){ + /* Wildcard of the form "?". Assign the next variable number */ + pExpr->iTable = ++pParse->nVar; + }else if( pToken->z[0]=='?' ){ + /* Wildcard of the form "?nnn". Convert "nnn" to an integer and + ** use it as the variable number */ + int i; + pExpr->iTable = i = atoi((char*)&pToken->z[1]); + testcase( i==0 ); + testcase( i==1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); + if( i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", + db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); + } + if( i>pParse->nVar ){ + pParse->nVar = i; + } + }else{ + /* Wildcards of the form ":aaa" or "$aaa". Reuse the same variable + ** number as the prior appearance of the same name, or if the name + ** has never appeared before, reuse the same variable number + */ + int i; + u32 n; + n = pToken->n; + for(i=0; inVarExpr; i++){ + Expr *pE; + if( (pE = pParse->apVarExpr[i])!=0 + && pE->token.n==n + && memcmp(pE->token.z, pToken->z, n)==0 ){ + pExpr->iTable = pE->iTable; + break; + } + } + if( i>=pParse->nVarExpr ){ + pExpr->iTable = ++pParse->nVar; + if( pParse->nVarExpr>=pParse->nVarExprAlloc-1 ){ + pParse->nVarExprAlloc += pParse->nVarExprAlloc + 10; + pParse->apVarExpr = + sqlite3DbReallocOrFree( + db, + pParse->apVarExpr, + pParse->nVarExprAlloc*sizeof(pParse->apVarExpr[0]) + ); + } + if( !db->mallocFailed ){ + assert( pParse->apVarExpr!=0 ); + pParse->apVarExpr[pParse->nVarExpr++] = pExpr; + } + } + } + if( !pParse->nErr && pParse->nVar>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "too many SQL variables"); + } +} + +/* +** Clear an expression structure without deleting the structure itself. +** Substructure is deleted. +*/ +void sqlite3ExprClear(sqlite3 *db, Expr *p){ + if( p->span.dyn ) sqlite3DbFree(db, (char*)p->span.z); + if( p->token.dyn ) sqlite3DbFree(db, (char*)p->token.z); + sqlite3ExprDelete(db, p->pLeft); + sqlite3ExprDelete(db, p->pRight); + sqlite3ExprListDelete(db, p->pList); + sqlite3SelectDelete(db, p->pSelect); +} + +/* +** Recursively delete an expression tree. +*/ +void sqlite3ExprDelete(sqlite3 *db, Expr *p){ + if( p==0 ) return; + sqlite3ExprClear(db, p); + sqlite3DbFree(db, p); +} + +/* +** The Expr.token field might be a string literal that is quoted. +** If so, remove the quotation marks. +*/ +void sqlite3DequoteExpr(sqlite3 *db, Expr *p){ + if( ExprHasAnyProperty(p, EP_Dequoted) ){ + return; + } + ExprSetProperty(p, EP_Dequoted); + if( p->token.dyn==0 ){ + sqlite3TokenCopy(db, &p->token, &p->token); + } + sqlite3Dequote((char*)p->token.z); +} + +/* +** The following group of routines make deep copies of expressions, +** expression lists, ID lists, and select statements. The copies can +** be deleted (by being passed to their respective ...Delete() routines) +** without effecting the originals. +** +** The expression list, ID, and source lists return by sqlite3ExprListDup(), +** sqlite3IdListDup(), and sqlite3SrcListDup() can not be further expanded +** by subsequent calls to sqlite*ListAppend() routines. +** +** Any tables that the SrcList might point to are not duplicated. +*/ +Expr *sqlite3ExprDup(sqlite3 *db, Expr *p){ + Expr *pNew; + if( p==0 ) return 0; + pNew = sqlite3DbMallocRaw(db, sizeof(*p) ); + if( pNew==0 ) return 0; + memcpy(pNew, p, sizeof(*pNew)); + if( p->token.z!=0 ){ + pNew->token.z = (u8*)sqlite3DbStrNDup(db, (char*)p->token.z, p->token.n); + pNew->token.dyn = 1; + }else{ + assert( pNew->token.z==0 ); + } + pNew->span.z = 0; + pNew->pLeft = sqlite3ExprDup(db, p->pLeft); + pNew->pRight = sqlite3ExprDup(db, p->pRight); + pNew->pList = sqlite3ExprListDup(db, p->pList); + pNew->pSelect = sqlite3SelectDup(db, p->pSelect); + return pNew; +} +void sqlite3TokenCopy(sqlite3 *db, Token *pTo, Token *pFrom){ + if( pTo->dyn ) sqlite3DbFree(db, (char*)pTo->z); + if( pFrom->z ){ + pTo->n = pFrom->n; + pTo->z = (u8*)sqlite3DbStrNDup(db, (char*)pFrom->z, pFrom->n); + pTo->dyn = 1; + }else{ + pTo->z = 0; + } +} +ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p){ + ExprList *pNew; + struct ExprList_item *pItem, *pOldItem; + int i; + if( p==0 ) return 0; + pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + if( pNew==0 ) return 0; + pNew->iECursor = 0; + pNew->nExpr = pNew->nAlloc = p->nExpr; + pNew->a = pItem = sqlite3DbMallocRaw(db, p->nExpr*sizeof(p->a[0]) ); + if( pItem==0 ){ + sqlite3DbFree(db, pNew); + return 0; + } + pOldItem = p->a; + for(i=0; inExpr; i++, pItem++, pOldItem++){ + Expr *pNewExpr, *pOldExpr; + pItem->pExpr = pNewExpr = sqlite3ExprDup(db, pOldExpr = pOldItem->pExpr); + if( pOldExpr->span.z!=0 && pNewExpr ){ + /* Always make a copy of the span for top-level expressions in the + ** expression list. The logic in SELECT processing that determines + ** the names of columns in the result set needs this information */ + sqlite3TokenCopy(db, &pNewExpr->span, &pOldExpr->span); + } + assert( pNewExpr==0 || pNewExpr->span.z!=0 + || pOldExpr->span.z==0 + || db->mallocFailed ); + pItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pItem->sortOrder = pOldItem->sortOrder; + pItem->done = 0; + pItem->iCol = pOldItem->iCol; + pItem->iAlias = pOldItem->iAlias; + } + return pNew; +} + +/* +** If cursors, triggers, views and subqueries are all omitted from +** the build, then none of the following routines, except for +** sqlite3SelectDup(), can be called. sqlite3SelectDup() is sometimes +** called with a NULL argument. +*/ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) \ + || !defined(SQLITE_OMIT_SUBQUERY) +SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p){ + SrcList *pNew; + int i; + int nByte; + if( p==0 ) return 0; + nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); + pNew = sqlite3DbMallocRaw(db, nByte ); + if( pNew==0 ) return 0; + pNew->nSrc = pNew->nAlloc = p->nSrc; + for(i=0; inSrc; i++){ + struct SrcList_item *pNewItem = &pNew->a[i]; + struct SrcList_item *pOldItem = &p->a[i]; + Table *pTab; + pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); + pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); + pNewItem->jointype = pOldItem->jointype; + pNewItem->iCursor = pOldItem->iCursor; + pNewItem->isPopulated = pOldItem->isPopulated; + pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); + pNewItem->notIndexed = pOldItem->notIndexed; + pNewItem->pIndex = pOldItem->pIndex; + pTab = pNewItem->pTab = pOldItem->pTab; + if( pTab ){ + pTab->nRef++; + } + pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect); + pNewItem->pOn = sqlite3ExprDup(db, pOldItem->pOn); + pNewItem->pUsing = sqlite3IdListDup(db, pOldItem->pUsing); + pNewItem->colUsed = pOldItem->colUsed; + } + return pNew; +} +IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ + IdList *pNew; + int i; + if( p==0 ) return 0; + pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + if( pNew==0 ) return 0; + pNew->nId = pNew->nAlloc = p->nId; + pNew->a = sqlite3DbMallocRaw(db, p->nId*sizeof(p->a[0]) ); + if( pNew->a==0 ){ + sqlite3DbFree(db, pNew); + return 0; + } + for(i=0; inId; i++){ + struct IdList_item *pNewItem = &pNew->a[i]; + struct IdList_item *pOldItem = &p->a[i]; + pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pNewItem->idx = pOldItem->idx; + } + return pNew; +} +Select *sqlite3SelectDup(sqlite3 *db, Select *p){ + Select *pNew; + if( p==0 ) return 0; + pNew = sqlite3DbMallocRaw(db, sizeof(*p) ); + if( pNew==0 ) return 0; + pNew->pEList = sqlite3ExprListDup(db, p->pEList); + pNew->pSrc = sqlite3SrcListDup(db, p->pSrc); + pNew->pWhere = sqlite3ExprDup(db, p->pWhere); + pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy); + pNew->pHaving = sqlite3ExprDup(db, p->pHaving); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy); + pNew->op = p->op; + pNew->pPrior = sqlite3SelectDup(db, p->pPrior); + pNew->pLimit = sqlite3ExprDup(db, p->pLimit); + pNew->pOffset = sqlite3ExprDup(db, p->pOffset); + pNew->iLimit = 0; + pNew->iOffset = 0; + pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; + pNew->pRightmost = 0; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->addrOpenEphm[2] = -1; + return pNew; +} +#else +Select *sqlite3SelectDup(sqlite3 *db, Select *p){ + assert( p==0 ); + return 0; +} #endif + +/* +** Add a new element to the end of an expression list. If pList is +** initially NULL, then create a new expression list. +*/ +ExprList *sqlite3ExprListAppend( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + Expr *pExpr, /* Expression to be appended */ + Token *pName /* AS keyword for the expression */ +){ + sqlite3 *db = pParse->db; + if( pList==0 ){ + pList = sqlite3DbMallocZero(db, sizeof(ExprList) ); + if( pList==0 ){ + goto no_mem; + } + assert( pList->nAlloc==0 ); + } + if( pList->nAlloc<=pList->nExpr ){ + struct ExprList_item *a; + int n = pList->nAlloc*2 + 4; + a = sqlite3DbRealloc(db, pList->a, n*sizeof(pList->a[0])); + if( a==0 ){ + goto no_mem; + } + pList->a = a; + pList->nAlloc = sqlite3DbMallocSize(db, a)/sizeof(a[0]); + } + assert( pList->a!=0 ); + if( pExpr || pName ){ + struct ExprList_item *pItem = &pList->a[pList->nExpr++]; + memset(pItem, 0, sizeof(*pItem)); + pItem->zName = sqlite3NameFromToken(db, pName); + pItem->pExpr = pExpr; + pItem->iAlias = 0; + } + return pList; + +no_mem: + /* Avoid leaking memory if malloc has failed. */ + sqlite3ExprDelete(db, pExpr); + sqlite3ExprListDelete(db, pList); + return 0; +} + +/* +** If the expression list pEList contains more than iLimit elements, +** leave an error message in pParse. +*/ +void sqlite3ExprListCheckLength( + Parse *pParse, + ExprList *pEList, + const char *zObject +){ + int mx = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; + testcase( pEList && pEList->nExpr==mx ); + testcase( pEList && pEList->nExpr==mx+1 ); + if( pEList && pEList->nExpr>mx ){ + sqlite3ErrorMsg(pParse, "too many columns in %s", zObject); + } +} + /* ** Delete an entire expression list. */ -void sqlite3ExprListDelete(ExprList *pList){ +void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ int i; struct ExprList_item *pItem; if( pList==0 ) return; assert( pList->a!=0 || (pList->nExpr==0 && pList->nAlloc==0) ); assert( pList->nExpr<=pList->nAlloc ); for(pItem=pList->a, i=0; inExpr; i++, pItem++){ - sqlite3ExprDelete(pItem->pExpr); - sqliteFree(pItem->zName); + sqlite3ExprDelete(db, pItem->pExpr); + sqlite3DbFree(db, pItem->zName); } - sqliteFree(pList->a); - sqliteFree(pList); + sqlite3DbFree(db, pList->a); + sqlite3DbFree(db, pList); } /* -** Walk an expression tree. Call xFunc for each node visited. +** These routines are Walker callbacks. Walker.u.pi is a pointer +** to an integer. These routines are checking an expression to see +** if it is a constant. Set *Walker.u.pi to 0 if the expression is +** not constant. ** -** The return value from xFunc determines whether the tree walk continues. -** 0 means continue walking the tree. 1 means do not walk children -** of the current node but continue with siblings. 2 means abandon -** the tree walk completely. +** These callback routines are used to implement the following: ** -** The return value from this routine is 1 to abandon the tree walk -** and 0 to continue. +** sqlite3ExprIsConstant() +** sqlite3ExprIsConstantNotJoin() +** sqlite3ExprIsConstantOrFunction() ** -** NOTICE: This routine does *not* descend into subqueries. */ -static int walkExprList(ExprList *, int (*)(void *, Expr*), void *); -static int walkExprTree(Expr *pExpr, int (*xFunc)(void*,Expr*), void *pArg){ - int rc; - if( pExpr==0 ) return 0; - rc = (*xFunc)(pArg, pExpr); - if( rc==0 ){ - if( walkExprTree(pExpr->pLeft, xFunc, pArg) ) return 1; - if( walkExprTree(pExpr->pRight, xFunc, pArg) ) return 1; - if( walkExprList(pExpr->pList, xFunc, pArg) ) return 1; - } - return rc>1; -} +static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ -/* -** Call walkExprTree() for every expression in list p. -*/ -static int walkExprList(ExprList *p, int (*xFunc)(void *, Expr*), void *pArg){ - int i; - struct ExprList_item *pItem; - if( !p ) return 0; - for(i=p->nExpr, pItem=p->a; i>0; i--, pItem++){ - if( walkExprTree(pItem->pExpr, xFunc, pArg) ) return 1; - } - return 0; -} - -/* -** Call walkExprTree() for every expression in Select p, not including -** expressions that are part of sub-selects in any FROM clause or the LIMIT -** or OFFSET expressions.. -*/ -static int walkSelectExpr(Select *p, int (*xFunc)(void *, Expr*), void *pArg){ - walkExprList(p->pEList, xFunc, pArg); - walkExprTree(p->pWhere, xFunc, pArg); - walkExprList(p->pGroupBy, xFunc, pArg); - walkExprTree(p->pHaving, xFunc, pArg); - walkExprList(p->pOrderBy, xFunc, pArg); - if( p->pPrior ){ - walkSelectExpr(p->pPrior, xFunc, pArg); - } - return 0; -} - - -/* -** This routine is designed as an xFunc for walkExprTree(). -** -** pArg is really a pointer to an integer. If we can tell by looking -** at pExpr that the expression that contains pExpr is not a constant -** expression, then set *pArg to 0 and return 2 to abandon the tree walk. -** If pExpr does does not disqualify the expression from being a constant -** then do nothing. -** -** After walking the whole tree, if no nodes are found that disqualify -** the expression as constant, then we assume the whole expression -** is constant. See sqlite3ExprIsConstant() for additional information. -*/ -static int exprNodeIsConstant(void *pArg, Expr *pExpr){ - int *pN = (int*)pArg; - - /* If *pArg is 3 then any term of the expression that comes from + /* If pWalker->u.i is 3 then any term of the expression that comes from ** the ON or USING clauses of a join disqualifies the expression ** from being considered constant. */ - if( (*pN)==3 && ExprHasAnyProperty(pExpr, EP_FromJoin) ){ - *pN = 0; - return 2; + if( pWalker->u.i==3 && ExprHasAnyProperty(pExpr, EP_FromJoin) ){ + pWalker->u.i = 0; + return WRC_Abort; } switch( pExpr->op ){ /* Consider functions to be constant if all their arguments are constant - ** and *pArg==2 */ + ** and pWalker->u.i==2 */ case TK_FUNCTION: - if( (*pN)==2 ) return 0; + if( pWalker->u.i==2 ) return 0; /* Fall through */ case TK_ID: case TK_COLUMN: - case TK_DOT: case TK_AGG_FUNCTION: case TK_AGG_COLUMN: #ifndef SQLITE_OMIT_SUBQUERY case TK_SELECT: case TK_EXISTS: + testcase( pExpr->op==TK_SELECT ); + testcase( pExpr->op==TK_EXISTS ); #endif - *pN = 0; - return 2; - case TK_IN: - if( pExpr->pSelect ){ - *pN = 0; - return 2; - } + testcase( pExpr->op==TK_ID ); + testcase( pExpr->op==TK_COLUMN ); + testcase( pExpr->op==TK_AGG_FUNCTION ); + testcase( pExpr->op==TK_AGG_COLUMN ); + pWalker->u.i = 0; + return WRC_Abort; default: - return 0; + return WRC_Continue; } } +static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ + UNUSED_PARAMETER(NotUsed); + pWalker->u.i = 0; + return WRC_Abort; +} +static int exprIsConst(Expr *p, int initFlag){ + Walker w; + w.u.i = initFlag; + w.xExprCallback = exprNodeIsConstant; + w.xSelectCallback = selectNodeIsConstant; + sqlite3WalkExpr(&w, p); + return w.u.i; +} /* ** Walk an expression tree. Return 1 if the expression is constant @@ -879,9 +961,7 @@ static int exprNodeIsConstant(void *pArg, Expr *pExpr){ ** a constant. */ int sqlite3ExprIsConstant(Expr *p){ - int isConst = 1; - walkExprTree(p, exprNodeIsConstant, &isConst); - return isConst; + return exprIsConst(p, 1); } /* @@ -891,9 +971,7 @@ int sqlite3ExprIsConstant(Expr *p){ ** an ON or USING clause. */ int sqlite3ExprIsConstantNotJoin(Expr *p){ - int isConst = 3; - walkExprTree(p, exprNodeIsConstant, &isConst); - return isConst!=0; + return exprIsConst(p, 3); } /* @@ -906,9 +984,7 @@ int sqlite3ExprIsConstantNotJoin(Expr *p){ ** a constant. */ int sqlite3ExprIsConstantOrFunction(Expr *p){ - int isConst = 2; - walkExprTree(p, exprNodeIsConstant, &isConst); - return isConst!=0; + return exprIsConst(p, 2); } /* @@ -918,27 +994,36 @@ int sqlite3ExprIsConstantOrFunction(Expr *p){ ** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged. */ int sqlite3ExprIsInteger(Expr *p, int *pValue){ + int rc = 0; + if( p->flags & EP_IntValue ){ + *pValue = p->iTable; + return 1; + } switch( p->op ){ case TK_INTEGER: { - if( sqlite3GetInt32((char*)p->token.z, pValue) ){ - return 1; - } + rc = sqlite3GetInt32((char*)p->token.z, pValue); break; } case TK_UPLUS: { - return sqlite3ExprIsInteger(p->pLeft, pValue); + rc = sqlite3ExprIsInteger(p->pLeft, pValue); + break; } case TK_UMINUS: { int v; if( sqlite3ExprIsInteger(p->pLeft, &v) ){ *pValue = -v; - return 1; + rc = 1; } break; } default: break; } - return 0; + if( rc ){ + p->op = TK_INTEGER; + p->flags |= EP_IntValue; + p->iTable = *pValue; + } + return rc; } /* @@ -952,547 +1037,202 @@ int sqlite3IsRowid(const char *z){ } /* -** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up -** that name in the set of source tables in pSrcList and make the pExpr -** expression node refer back to that source column. The following changes -** are made to pExpr: +** Return true if the IN operator optimization is enabled and +** the SELECT statement p exists and is of the +** simple form: ** -** pExpr->iDb Set the index in db->aDb[] of the database holding -** the table. -** pExpr->iTable Set to the cursor number for the table obtained -** from pSrcList. -** pExpr->iColumn Set to the column number within the table. -** pExpr->op Set to TK_COLUMN. -** pExpr->pLeft Any expression this points to is deleted -** pExpr->pRight Any expression this points to is deleted. +** SELECT FROM ** -** The pDbToken is the name of the database (the "X"). This value may be -** NULL meaning that name is of the form Y.Z or Z. Any available database -** can be used. The pTableToken is the name of the table (the "Y"). This -** value can be NULL if pDbToken is also NULL. If pTableToken is NULL it -** means that the form of the name is Z and that columns from any table -** can be used. -** -** If the name cannot be resolved unambiguously, leave an error message -** in pParse and return non-zero. Return zero on success. +** If this is the case, it may be possible to use an existing table +** or index instead of generating an epheremal table. */ -static int lookupName( - Parse *pParse, /* The parsing context */ - Token *pDbToken, /* Name of the database containing table, or NULL */ - Token *pTableToken, /* Name of table containing column, or NULL */ - Token *pColumnToken, /* Name of the column. */ - NameContext *pNC, /* The name context used to resolve the name */ - Expr *pExpr /* Make this EXPR node point to the selected column */ -){ - char *zDb = 0; /* Name of the database. The "X" in X.Y.Z */ - char *zTab = 0; /* Name of the table. The "Y" in X.Y.Z or Y.Z */ - char *zCol = 0; /* Name of the column. The "Z" */ - int i, j; /* Loop counters */ - int cnt = 0; /* Number of matching column names */ - int cntTab = 0; /* Number of matching table names */ - sqlite3 *db = pParse->db; /* The database */ - struct SrcList_item *pItem; /* Use for looping over pSrcList items */ - struct SrcList_item *pMatch = 0; /* The matching pSrcList item */ - NameContext *pTopNC = pNC; /* First namecontext in the list */ - - assert( pColumnToken && pColumnToken->z ); /* The Z in X.Y.Z cannot be NULL */ - zDb = sqlite3NameFromToken(pDbToken); - zTab = sqlite3NameFromToken(pTableToken); - zCol = sqlite3NameFromToken(pColumnToken); - if( sqlite3MallocFailed() ){ - goto lookupname_end; - } - - pExpr->iTable = -1; - while( pNC && cnt==0 ){ - ExprList *pEList; - SrcList *pSrcList = pNC->pSrcList; - - if( pSrcList ){ - for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ - Table *pTab; - int iDb; - Column *pCol; - - pTab = pItem->pTab; - assert( pTab!=0 ); - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - assert( pTab->nCol>0 ); - if( zTab ){ - if( pItem->zAlias ){ - char *zTabName = pItem->zAlias; - if( sqlite3StrICmp(zTabName, zTab)!=0 ) continue; - }else{ - char *zTabName = pTab->zName; - if( zTabName==0 || sqlite3StrICmp(zTabName, zTab)!=0 ) continue; - if( zDb!=0 && sqlite3StrICmp(db->aDb[iDb].zName, zDb)!=0 ){ - continue; - } - } - } - if( 0==(cntTab++) ){ - pExpr->iTable = pItem->iCursor; - pExpr->pSchema = pTab->pSchema; - pMatch = pItem; - } - for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ - if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ - const char *zColl = pTab->aCol[j].zColl; - IdList *pUsing; - cnt++; - pExpr->iTable = pItem->iCursor; - pMatch = pItem; - pExpr->pSchema = pTab->pSchema; - /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ - pExpr->iColumn = j==pTab->iPKey ? -1 : j; - pExpr->affinity = pTab->aCol[j].affinity; - if( (pExpr->flags & EP_ExpCollate)==0 ){ - pExpr->pColl = sqlite3FindCollSeq(db, ENC(db), zColl,-1, 0); - } - if( inSrc-1 ){ - if( pItem[1].jointype & JT_NATURAL ){ - /* If this match occurred in the left table of a natural join, - ** then skip the right table to avoid a duplicate match */ - pItem++; - i++; - }else if( (pUsing = pItem[1].pUsing)!=0 ){ - /* If this match occurs on a column that is in the USING clause - ** of a join, skip the search of the right table of the join - ** to avoid a duplicate match there. */ - int k; - for(k=0; knId; k++){ - if( sqlite3StrICmp(pUsing->a[k].zName, zCol)==0 ){ - pItem++; - i++; - break; - } - } - } - } - break; - } - } - } - } - -#ifndef SQLITE_OMIT_TRIGGER - /* If we have not already resolved the name, then maybe - ** it is a new.* or old.* trigger argument reference - */ - if( zDb==0 && zTab!=0 && cnt==0 && pParse->trigStack!=0 ){ - TriggerStack *pTriggerStack = pParse->trigStack; - Table *pTab = 0; - if( pTriggerStack->newIdx != -1 && sqlite3StrICmp("new", zTab) == 0 ){ - pExpr->iTable = pTriggerStack->newIdx; - assert( pTriggerStack->pTab ); - pTab = pTriggerStack->pTab; - }else if( pTriggerStack->oldIdx != -1 && sqlite3StrICmp("old", zTab)==0 ){ - pExpr->iTable = pTriggerStack->oldIdx; - assert( pTriggerStack->pTab ); - pTab = pTriggerStack->pTab; - } - - if( pTab ){ - int iCol; - Column *pCol = pTab->aCol; - - pExpr->pSchema = pTab->pSchema; - cntTab++; - for(iCol=0; iCol < pTab->nCol; iCol++, pCol++) { - if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ - const char *zColl = pTab->aCol[iCol].zColl; - cnt++; - pExpr->iColumn = iCol==pTab->iPKey ? -1 : iCol; - pExpr->affinity = pTab->aCol[iCol].affinity; - if( (pExpr->flags & EP_ExpCollate)==0 ){ - pExpr->pColl = sqlite3FindCollSeq(db, ENC(db), zColl,-1, 0); - } - pExpr->pTab = pTab; - break; - } - } - } - } -#endif /* !defined(SQLITE_OMIT_TRIGGER) */ - - /* - ** Perhaps the name is a reference to the ROWID - */ - if( cnt==0 && cntTab==1 && sqlite3IsRowid(zCol) ){ - cnt = 1; - pExpr->iColumn = -1; - pExpr->affinity = SQLITE_AFF_INTEGER; - } - - /* - ** If the input is of the form Z (not Y.Z or X.Y.Z) then the name Z - ** might refer to an result-set alias. This happens, for example, when - ** we are resolving names in the WHERE clause of the following command: - ** - ** SELECT a+b AS x FROM table WHERE x<10; - ** - ** In cases like this, replace pExpr with a copy of the expression that - ** forms the result set entry ("a+b" in the example) and return immediately. - ** Note that the expression in the result set should have already been - ** resolved by the time the WHERE clause is resolved. - */ - if( cnt==0 && (pEList = pNC->pEList)!=0 && zTab==0 ){ - for(j=0; jnExpr; j++){ - char *zAs = pEList->a[j].zName; - if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ - Expr *pDup, *pOrig; - assert( pExpr->pLeft==0 && pExpr->pRight==0 ); - assert( pExpr->pList==0 ); - assert( pExpr->pSelect==0 ); - pOrig = pEList->a[j].pExpr; - if( !pNC->allowAgg && ExprHasProperty(pOrig, EP_Agg) ){ - sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); - sqliteFree(zCol); - return 2; - } - pDup = sqlite3ExprDup(pOrig); - if( pExpr->flags & EP_ExpCollate ){ - pDup->pColl = pExpr->pColl; - pDup->flags |= EP_ExpCollate; - } - if( pExpr->span.dyn ) sqliteFree((char*)pExpr->span.z); - if( pExpr->token.dyn ) sqliteFree((char*)pExpr->token.z); - memcpy(pExpr, pDup, sizeof(*pExpr)); - sqliteFree(pDup); - cnt = 1; - pMatch = 0; - assert( zTab==0 && zDb==0 ); - goto lookupname_end_2; - } - } - } - - /* Advance to the next name context. The loop will exit when either - ** we have a match (cnt>0) or when we run out of name contexts. - */ - if( cnt==0 ){ - pNC = pNC->pNext; - } - } - - /* - ** If X and Y are NULL (in other words if only the column name Z is - ** supplied) and the value of Z is enclosed in double-quotes, then - ** Z is a string literal if it doesn't match any column names. In that - ** case, we need to return right away and not make any changes to - ** pExpr. - ** - ** Because no reference was made to outer contexts, the pNC->nRef - ** fields are not changed in any context. - */ - if( cnt==0 && zTab==0 && pColumnToken->z[0]=='"' ){ - sqliteFree(zCol); - return 0; - } - - /* - ** cnt==0 means there was not match. cnt>1 means there were two or - ** more matches. Either way, we have an error. - */ - if( cnt!=1 ){ - char *z = 0; - char *zErr; - zErr = cnt==0 ? "no such column: %s" : "ambiguous column name: %s"; - if( zDb ){ - sqlite3SetString(&z, zDb, ".", zTab, ".", zCol, (char*)0); - }else if( zTab ){ - sqlite3SetString(&z, zTab, ".", zCol, (char*)0); - }else{ - z = sqliteStrDup(zCol); - } - sqlite3ErrorMsg(pParse, zErr, z); - sqliteFree(z); - pTopNC->nErr++; - } - - /* If a column from a table in pSrcList is referenced, then record - ** this fact in the pSrcList.a[].colUsed bitmask. Column 0 causes - ** bit 0 to be set. Column 1 sets bit 1. And so forth. If the - ** column number is greater than the number of bits in the bitmask - ** then set the high-order bit of the bitmask. - */ - if( pExpr->iColumn>=0 && pMatch!=0 ){ - int n = pExpr->iColumn; - if( n>=sizeof(Bitmask)*8 ){ - n = sizeof(Bitmask)*8-1; - } - assert( pMatch->iCursor==pExpr->iTable ); - pMatch->colUsed |= ((Bitmask)1)<pLeft); - pExpr->pLeft = 0; - sqlite3ExprDelete(pExpr->pRight); - pExpr->pRight = 0; - pExpr->op = TK_COLUMN; -lookupname_end_2: - sqliteFree(zCol); - if( cnt==1 ){ - assert( pNC!=0 ); - sqlite3AuthRead(pParse, pExpr, pNC->pSrcList); - if( pMatch && !pMatch->pSelect ){ - pExpr->pTab = pMatch->pTab; - } - /* Increment the nRef value on all name contexts from TopNC up to - ** the point where the name matched. */ - for(;;){ - assert( pTopNC!=0 ); - pTopNC->nRef++; - if( pTopNC==pNC ) break; - pTopNC = pTopNC->pNext; - } - return 0; - } else { - return 1; - } -} - -/* -** This routine is designed as an xFunc for walkExprTree(). -** -** Resolve symbolic names into TK_COLUMN operators for the current -** node in the expression tree. Return 0 to continue the search down -** the tree or 2 to abort the tree walk. -** -** This routine also does error checking and name resolution for -** function names. The operator for aggregate functions is changed -** to TK_AGG_FUNCTION. -*/ -static int nameResolverStep(void *pArg, Expr *pExpr){ - NameContext *pNC = (NameContext*)pArg; - Parse *pParse; - - if( pExpr==0 ) return 1; - assert( pNC!=0 ); - pParse = pNC->pParse; - - if( ExprHasAnyProperty(pExpr, EP_Resolved) ) return 1; - ExprSetProperty(pExpr, EP_Resolved); -#ifndef NDEBUG - if( pNC->pSrcList && pNC->pSrcList->nAlloc>0 ){ - SrcList *pSrcList = pNC->pSrcList; - int i; - for(i=0; ipSrcList->nSrc; i++){ - assert( pSrcList->a[i].iCursor>=0 && pSrcList->a[i].iCursornTab); - } - } -#endif - switch( pExpr->op ){ - /* Double-quoted strings (ex: "abc") are used as identifiers if - ** possible. Otherwise they remain as strings. Single-quoted - ** strings (ex: 'abc') are always string literals. - */ - case TK_STRING: { - if( pExpr->token.z[0]=='\'' ) break; - /* Fall thru into the TK_ID case if this is a double-quoted string */ - } - /* A lone identifier is the name of a column. - */ - case TK_ID: { - lookupName(pParse, 0, 0, &pExpr->token, pNC, pExpr); - return 1; - } - - /* A table name and column name: ID.ID - ** Or a database, table and column: ID.ID.ID - */ - case TK_DOT: { - Token *pColumn; - Token *pTable; - Token *pDb; - Expr *pRight; - - /* if( pSrcList==0 ) break; */ - pRight = pExpr->pRight; - if( pRight->op==TK_ID ){ - pDb = 0; - pTable = &pExpr->pLeft->token; - pColumn = &pRight->token; - }else{ - assert( pRight->op==TK_DOT ); - pDb = &pExpr->pLeft->token; - pTable = &pRight->pLeft->token; - pColumn = &pRight->pRight->token; - } - lookupName(pParse, pDb, pTable, pColumn, pNC, pExpr); - return 1; - } - - /* Resolve function names - */ - case TK_CONST_FUNC: - case TK_FUNCTION: { - ExprList *pList = pExpr->pList; /* The argument list */ - int n = pList ? pList->nExpr : 0; /* Number of arguments */ - int no_such_func = 0; /* True if no such function exists */ - int wrong_num_args = 0; /* True if wrong number of arguments */ - int is_agg = 0; /* True if is an aggregate function */ - int i; - int auth; /* Authorization to use the function */ - int nId; /* Number of characters in function name */ - const char *zId; /* The function name. */ - FuncDef *pDef; /* Information about the function */ - int enc = ENC(pParse->db); /* The database encoding */ - - zId = (char*)pExpr->token.z; - nId = pExpr->token.n; - pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); - if( pDef==0 ){ - pDef = sqlite3FindFunction(pParse->db, zId, nId, -1, enc, 0); - if( pDef==0 ){ - no_such_func = 1; - }else{ - wrong_num_args = 1; - } - }else{ - is_agg = pDef->xFunc==0; - } -#ifndef SQLITE_OMIT_AUTHORIZATION - if( pDef ){ - auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0); - if( auth!=SQLITE_OK ){ - if( auth==SQLITE_DENY ){ - sqlite3ErrorMsg(pParse, "not authorized to use function: %s", - pDef->zName); - pNC->nErr++; - } - pExpr->op = TK_NULL; - return 1; - } - } -#endif - if( is_agg && !pNC->allowAgg ){ - sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); - pNC->nErr++; - is_agg = 0; - }else if( no_such_func ){ - sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); - pNC->nErr++; - }else if( wrong_num_args ){ - sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", - nId, zId); - pNC->nErr++; - } - if( is_agg ){ - pExpr->op = TK_AGG_FUNCTION; - pNC->hasAgg = 1; - } - if( is_agg ) pNC->allowAgg = 0; - for(i=0; pNC->nErr==0 && ia[i].pExpr, nameResolverStep, pNC); - } - if( is_agg ) pNC->allowAgg = 1; - /* FIX ME: Compute pExpr->affinity based on the expected return - ** type of the function - */ - return is_agg; - } #ifndef SQLITE_OMIT_SUBQUERY - case TK_SELECT: - case TK_EXISTS: -#endif - case TK_IN: { - if( pExpr->pSelect ){ - int nRef = pNC->nRef; -#ifndef SQLITE_OMIT_CHECK - if( pNC->isCheck ){ - sqlite3ErrorMsg(pParse,"subqueries prohibited in CHECK constraints"); - } -#endif - sqlite3SelectResolve(pParse, pExpr->pSelect, pNC); - assert( pNC->nRef>=nRef ); - if( nRef!=pNC->nRef ){ - ExprSetProperty(pExpr, EP_VarSelect); - } - } - break; - } -#ifndef SQLITE_OMIT_CHECK - case TK_VARIABLE: { - if( pNC->isCheck ){ - sqlite3ErrorMsg(pParse,"parameters prohibited in CHECK constraints"); - } - break; - } -#endif +static int isCandidateForInOpt(Select *p){ + SrcList *pSrc; + ExprList *pEList; + Table *pTab; + if( p==0 ) return 0; /* right-hand side of IN is SELECT */ + if( p->pPrior ) return 0; /* Not a compound SELECT */ + if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ + return 0; /* No DISTINCT keyword and no aggregate functions */ } - return 0; + if( p->pGroupBy ) return 0; /* Has no GROUP BY clause */ + if( p->pLimit ) return 0; /* Has no LIMIT clause */ + if( p->pOffset ) return 0; + if( p->pWhere ) return 0; /* Has no WHERE clause */ + pSrc = p->pSrc; + assert( pSrc!=0 ); + if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ + if( pSrc->a[0].pSelect ) return 0; /* FROM clause is not a subquery */ + pTab = pSrc->a[0].pTab; + if( pTab==0 ) return 0; + if( pTab->pSelect ) return 0; /* FROM clause is not a view */ + if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ + pEList = p->pEList; + if( pEList->nExpr!=1 ) return 0; /* One column in the result set */ + if( pEList->a[0].pExpr->op!=TK_COLUMN ) return 0; /* Result is a column */ + return 1; } +#endif /* SQLITE_OMIT_SUBQUERY */ /* -** This routine walks an expression tree and resolves references to -** table columns. Nodes of the form ID.ID or ID resolve into an -** index to the table in the table list and a column offset. The -** Expr.opcode for such nodes is changed to TK_COLUMN. The Expr.iTable -** value is changed to the index of the referenced table in pTabList -** plus the "base" value. The base value will ultimately become the -** VDBE cursor number for a cursor that is pointing into the referenced -** table. The Expr.iColumn value is changed to the index of the column -** of the referenced table. The Expr.iColumn value for the special -** ROWID column is -1. Any INTEGER PRIMARY KEY column is tried as an -** alias for ROWID. +** This function is used by the implementation of the IN (...) operator. +** It's job is to find or create a b-tree structure that may be used +** either to test for membership of the (...) set or to iterate through +** its members, skipping duplicates. ** -** Also resolve function names and check the functions for proper -** usage. Make sure all function names are recognized and all functions -** have the correct number of arguments. Leave an error message -** in pParse->zErrMsg if anything is amiss. Return the number of errors. +** The cursor opened on the structure (database table, database index +** or ephermal table) is stored in pX->iTable before this function returns. +** The returned value indicates the structure type, as follows: ** -** If the expression contains aggregate functions then set the EP_Agg -** property on the expression. +** IN_INDEX_ROWID - The cursor was opened on a database table. +** IN_INDEX_INDEX - The cursor was opened on a database index. +** IN_INDEX_EPH - The cursor was opened on a specially created and +** populated epheremal table. +** +** An existing structure may only be used if the SELECT is of the simple +** form: +** +** SELECT FROM
+** +** If prNotFound parameter is 0, then the structure will be used to iterate +** through the set members, skipping any duplicates. In this case an +** epheremal table must be used unless the selected is guaranteed +** to be unique - either because it is an INTEGER PRIMARY KEY or it +** is unique by virtue of a constraint or implicit index. +** +** If the prNotFound parameter is not 0, then the structure will be used +** for fast set membership tests. In this case an epheremal table must +** be used unless is an INTEGER PRIMARY KEY or an index can +** be found with as its left-most column. +** +** When the structure is being used for set membership tests, the user +** needs to know whether or not the structure contains an SQL NULL +** value in order to correctly evaluate expressions like "X IN (Y, Z)". +** If there is a chance that the structure may contain a NULL value at +** runtime, then a register is allocated and the register number written +** to *prNotFound. If there is no chance that the structure contains a +** NULL value, then *prNotFound is left unchanged. +** +** If a register is allocated and its location stored in *prNotFound, then +** its initial value is NULL. If the structure does not remain constant +** for the duration of the query (i.e. the set is a correlated sub-select), +** the value of the allocated register is reset to NULL each time the +** structure is repopulated. This allows the caller to use vdbe code +** equivalent to the following: +** +** if( register==NULL ){ +** has_null = +** register = 1 +** } +** +** in order to avoid running the +** test more often than is necessary. */ -int sqlite3ExprResolveNames( - NameContext *pNC, /* Namespace to resolve expressions in. */ - Expr *pExpr /* The expression to be analyzed. */ -){ - int savedHasAgg; - if( pExpr==0 ) return 0; -#if SQLITE_MAX_EXPR_DEPTH>0 - if( (pExpr->nHeight+pNC->pParse->nHeight)>SQLITE_MAX_EXPR_DEPTH ){ - sqlite3ErrorMsg(pNC->pParse, - "Expression tree is too large (maximum depth %d)", - SQLITE_MAX_EXPR_DEPTH - ); - return 1; +#ifndef SQLITE_OMIT_SUBQUERY +int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ + Select *p; + int eType = 0; + int iTab = pParse->nTab++; + int mustBeUnique = !prNotFound; + + /* The follwing if(...) expression is true if the SELECT is of the + ** simple form: + ** + ** SELECT FROM
+ ** + ** If this is the case, it may be possible to use an existing table + ** or index instead of generating an epheremal table. + */ + p = pX->pSelect; + if( isCandidateForInOpt(p) ){ + sqlite3 *db = pParse->db; + Index *pIdx; + Expr *pExpr = p->pEList->a[0].pExpr; + int iCol = pExpr->iColumn; + Vdbe *v = sqlite3GetVdbe(pParse); + + /* This function is only called from two places. In both cases the vdbe + ** has already been allocated. So assume sqlite3GetVdbe() is always + ** successful here. + */ + assert(v); + if( iCol<0 ){ + int iMem = ++pParse->nMem; + int iAddr; + Table *pTab = p->pSrc->a[0].pTab; + int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + sqlite3VdbeUsesBtree(v, iDb); + + iAddr = sqlite3VdbeAddOp1(v, OP_If, iMem); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iMem); + + sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); + eType = IN_INDEX_ROWID; + + sqlite3VdbeJumpHere(v, iAddr); + }else{ + /* The collation sequence used by the comparison. If an index is to + ** be used in place of a temp-table, it must be ordered according + ** to this collation sequence. + */ + CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pExpr); + + /* Check that the affinity that will be used to perform the + ** comparison is the same as the affinity of the column. If + ** it is not, it is not possible to use any index. + */ + Table *pTab = p->pSrc->a[0].pTab; + char aff = comparisonAffinity(pX); + int affinity_ok = (pTab->aCol[iCol].affinity==aff||aff==SQLITE_AFF_NONE); + + for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ + if( (pIdx->aiColumn[0]==iCol) + && (pReq==sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], -1, 0)) + && (!mustBeUnique || (pIdx->nColumn==1 && pIdx->onError!=OE_None)) + ){ + int iDb; + int iMem = ++pParse->nMem; + int iAddr; + char *pKey; + + pKey = (char *)sqlite3IndexKeyinfo(pParse, pIdx); + iDb = sqlite3SchemaToIndex(db, pIdx->pSchema); + sqlite3VdbeUsesBtree(v, iDb); + + iAddr = sqlite3VdbeAddOp1(v, OP_If, iMem); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iMem); + + sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, pIdx->nColumn); + sqlite3VdbeAddOp4(v, OP_OpenRead, iTab, pIdx->tnum, iDb, + pKey,P4_KEYINFO_HANDOFF); + VdbeComment((v, "%s", pIdx->zName)); + eType = IN_INDEX_INDEX; + + sqlite3VdbeJumpHere(v, iAddr); + if( prNotFound && !pTab->aCol[iCol].notNull ){ + *prNotFound = ++pParse->nMem; + } + } + } + } } - pNC->pParse->nHeight += pExpr->nHeight; -#endif - savedHasAgg = pNC->hasAgg; - pNC->hasAgg = 0; - walkExprTree(pExpr, nameResolverStep, pNC); -#if SQLITE_MAX_EXPR_DEPTH>0 - pNC->pParse->nHeight -= pExpr->nHeight; -#endif - if( pNC->nErr>0 ){ - ExprSetProperty(pExpr, EP_Error); + + if( eType==0 ){ + int rMayHaveNull = 0; + eType = IN_INDEX_EPH; + if( prNotFound ){ + *prNotFound = rMayHaveNull = ++pParse->nMem; + }else if( pX->pLeft->iColumn<0 && pX->pSelect==0 ){ + eType = IN_INDEX_ROWID; + } + sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID); + }else{ + pX->iTable = iTab; } - if( pNC->hasAgg ){ - ExprSetProperty(pExpr, EP_Agg); - }else if( savedHasAgg ){ - pNC->hasAgg = 1; - } - return ExprHasProperty(pExpr, EP_Error); + return eType; } - -/* -** A pointer instance of this structure is used to pass information -** through walkExprTree into codeSubqueryStep(). -*/ -typedef struct QueryCoder QueryCoder; -struct QueryCoder { - Parse *pParse; /* The parsing context */ - NameContext *pNC; /* Namespace of first enclosing query */ -}; - +#endif /* ** Generate code for scalar subqueries used as an expression @@ -1505,9 +1245,20 @@ struct QueryCoder { ** ** The pExpr parameter describes the expression that contains the IN ** operator or subquery. +** +** If parameter isRowid is non-zero, then expression pExpr is guaranteed +** to be of the form " IN (?, ?, ?)", where is a reference +** to some integer key column of a table B-Tree. In this case, use an +** intkey B-Tree to store the set of IN(...) values instead of the usual +** (slower) variable length keys B-Tree. */ #ifndef SQLITE_OMIT_SUBQUERY -void sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ +void sqlite3CodeSubselect( + Parse *pParse, + Expr *pExpr, + int rMayHaveNull, + int isRowid +){ int testAddr = 0; /* One-time test address */ Vdbe *v = sqlite3GetVdbe(pParse); if( v==0 ) return; @@ -1524,11 +1275,10 @@ void sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ ** save the results, and reuse the same result on subsequent invocations. */ if( !ExprHasAnyProperty(pExpr, EP_VarSelect) && !pParse->trigStack ){ - int mem = pParse->nMem++; - sqlite3VdbeAddOp(v, OP_MemLoad, mem, 0); - testAddr = sqlite3VdbeAddOp(v, OP_If, 0, 0); - assert( testAddr>0 || sqlite3MallocFailed() ); - sqlite3VdbeAddOp(v, OP_MemInt, 1, mem); + int mem = ++pParse->nMem; + sqlite3VdbeAddOp1(v, OP_If, mem); + testAddr = sqlite3VdbeAddOp2(v, OP_Integer, 1, mem); + assert( testAddr>0 || pParse->db->mallocFailed ); } switch( pExpr->op ){ @@ -1536,8 +1286,13 @@ void sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ char affinity; KeyInfo keyInfo; int addr; /* Address of OP_OpenEphemeral instruction */ + Expr *pLeft = pExpr->pLeft; - affinity = sqlite3ExprAffinity(pExpr->pLeft); + if( rMayHaveNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, rMayHaveNull); + } + + affinity = sqlite3ExprAffinity(pLeft); /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' ** expression it is handled the same way. A virtual table is @@ -1553,10 +1308,9 @@ void sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ ** is used. */ pExpr->iTable = pParse->nTab++; - addr = sqlite3VdbeAddOp(v, OP_OpenEphemeral, pExpr->iTable, 0); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); memset(&keyInfo, 0, sizeof(keyInfo)); keyInfo.nField = 1; - sqlite3VdbeAddOp(v, OP_SetNumColumns, pExpr->iTable, 1); if( pExpr->pSelect ){ /* Case 1: expr IN (SELECT ...) @@ -1564,10 +1318,14 @@ void sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ ** Generate code to write the results of the select into the temporary ** table allocated and opened above. */ - int iParm = pExpr->iTable + (((int)affinity)<<16); + SelectDest dest; ExprList *pEList; + + assert( !isRowid ); + sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); + dest.affinity = (u8)affinity; assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); - if( sqlite3Select(pParse, pExpr->pSelect, SRT_Set, iParm, 0, 0, 0, 0) ){ + if( sqlite3Select(pParse, pExpr->pSelect, &dest) ){ return; } pEList = pExpr->pSelect->pEList; @@ -1586,13 +1344,17 @@ void sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ int i; ExprList *pList = pExpr->pList; struct ExprList_item *pItem; + int r1, r2, r3; if( !affinity ){ affinity = SQLITE_AFF_NONE; } - keyInfo.aColl[0] = pExpr->pLeft->pColl; + keyInfo.aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); /* Loop through each expression in . */ + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Null, 0, r2); for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ Expr *pE2 = pItem->pExpr; @@ -1601,18 +1363,32 @@ void sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ ** this code only executes once. Because for a non-constant ** expression we need to rerun this code each time. */ - if( testAddr>0 && !sqlite3ExprIsConstant(pE2) ){ - sqlite3VdbeChangeToNoop(v, testAddr-1, 3); + if( testAddr && !sqlite3ExprIsConstant(pE2) ){ + sqlite3VdbeChangeToNoop(v, testAddr-1, 2); testAddr = 0; } /* Evaluate the expression and insert it into the temp table */ - sqlite3ExprCode(pParse, pE2); - sqlite3VdbeOp3(v, OP_MakeRecord, 1, 0, &affinity, 1); - sqlite3VdbeAddOp(v, OP_IdxInsert, pExpr->iTable, 0); + pParse->disableColCache++; + r3 = sqlite3ExprCodeTarget(pParse, pE2, r1); + assert( pParse->disableColCache>0 ); + pParse->disableColCache--; + + if( isRowid ){ + sqlite3VdbeAddOp2(v, OP_MustBeInt, r3, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeAddOp3(v, OP_Insert, pExpr->iTable, r2, r3); + }else{ + sqlite3VdbeAddOp4(v, OP_MakeRecord, r3, 1, r2, &affinity, 1); + sqlite3ExprCacheAffinityChange(pParse, r3, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pExpr->iTable, r2); + } } + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempReg(pParse, r2); + } + if( !isRowid ){ + sqlite3VdbeChangeP4(v, addr, (void *)&keyInfo, P4_KEYINFO); } - sqlite3VdbeChangeP3(v, addr, (void *)&keyInfo, P3_KEYINFO); break; } @@ -1624,31 +1400,31 @@ void sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ */ static const Token one = { (u8*)"1", 0, 1 }; Select *pSel; - int iMem; - int sop; + SelectDest dest; - pExpr->iColumn = iMem = pParse->nMem++; pSel = pExpr->pSelect; + sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); if( pExpr->op==TK_SELECT ){ - sop = SRT_Mem; - sqlite3VdbeAddOp(v, OP_MemNull, iMem, 0); - VdbeComment((v, "# Init subquery result")); + dest.eDest = SRT_Mem; + sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iParm); + VdbeComment((v, "Init subquery result")); }else{ - sop = SRT_Exists; - sqlite3VdbeAddOp(v, OP_MemInt, 0, iMem); - VdbeComment((v, "# Init EXISTS result")); + dest.eDest = SRT_Exists; + sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iParm); + VdbeComment((v, "Init EXISTS result")); } - sqlite3ExprDelete(pSel->pLimit); - pSel->pLimit = sqlite3Expr(TK_INTEGER, 0, 0, &one); - if( sqlite3Select(pParse, pSel, sop, iMem, 0, 0, 0, 0) ){ + sqlite3ExprDelete(pParse->db, pSel->pLimit); + pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &one); + if( sqlite3Select(pParse, pSel, &dest) ){ return; } + pExpr->iColumn = dest.iParm; break; } } if( testAddr ){ - sqlite3VdbeJumpHere(v, testAddr); + sqlite3VdbeJumpHere(v, testAddr-1); } return; @@ -1656,19 +1432,73 @@ void sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ #endif /* SQLITE_OMIT_SUBQUERY */ /* -** Generate an instruction that will put the integer describe by -** text z[0..n-1] on the stack. +** Duplicate an 8-byte value */ -static void codeInteger(Vdbe *v, const char *z, int n){ - assert( z || sqlite3MallocFailed() ); +static char *dup8bytes(Vdbe *v, const char *in){ + char *out = sqlite3DbMallocRaw(sqlite3VdbeDb(v), 8); + if( out ){ + memcpy(out, in, 8); + } + return out; +} + +/* +** Generate an instruction that will put the floating point +** value described by z[0..n-1] into register iMem. +** +** The z[] string will probably not be zero-terminated. But the +** z[n] character is guaranteed to be something that does not look +** like the continuation of the number. +*/ +static void codeReal(Vdbe *v, const char *z, int n, int negateFlag, int iMem){ + assert( z || v==0 || sqlite3VdbeDb(v)->mallocFailed ); + assert( !z || !isdigit(z[n]) ); + UNUSED_PARAMETER(n); if( z ){ - int i; - if( sqlite3GetInt32(z, &i) ){ - sqlite3VdbeAddOp(v, OP_Integer, i, 0); - }else if( sqlite3FitsIn64Bits(z) ){ - sqlite3VdbeOp3(v, OP_Int64, 0, 0, z, n); + double value; + char *zV; + sqlite3AtoF(z, &value); + if( sqlite3IsNaN(value) ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, iMem); }else{ - sqlite3VdbeOp3(v, OP_Real, 0, 0, z, n); + if( negateFlag ) value = -value; + zV = dup8bytes(v, (char*)&value); + sqlite3VdbeAddOp4(v, OP_Real, 0, iMem, 0, zV, P4_REAL); + } + } +} + + +/* +** Generate an instruction that will put the integer describe by +** text z[0..n-1] into register iMem. +** +** The z[] string will probably not be zero-terminated. But the +** z[n] character is guaranteed to be something that does not look +** like the continuation of the number. +*/ +static void codeInteger(Vdbe *v, Expr *pExpr, int negFlag, int iMem){ + const char *z; + if( pExpr->flags & EP_IntValue ){ + int i = pExpr->iTable; + if( negFlag ) i = -i; + sqlite3VdbeAddOp2(v, OP_Integer, i, iMem); + }else if( (z = (char*)pExpr->token.z)!=0 ){ + int i; + int n = pExpr->token.n; + assert( !isdigit(z[n]) ); + if( sqlite3GetInt32(z, &i) ){ + if( negFlag ) i = -i; + sqlite3VdbeAddOp2(v, OP_Integer, i, iMem); + }else if( sqlite3FitsIn64Bits(z, negFlag) ){ + i64 value; + char *zV; + sqlite3Atoi64(z, &value); + if( negFlag ) value = -value; + zV = dup8bytes(v, (char*)&value); + sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64); + }else{ + codeReal(v, z, n, negFlag, iMem); } } } @@ -1676,59 +1506,270 @@ static void codeInteger(Vdbe *v, const char *z, int n){ /* ** Generate code that will extract the iColumn-th column from -** table pTab and push that column value on the stack. There -** is an open cursor to pTab in iTable. If iColumn<0 then -** code is generated that extracts the rowid. +** table pTab and store the column value in a register. An effort +** is made to store the column value in register iReg, but this is +** not guaranteed. The location of the column value is returned. +** +** There must be an open cursor to pTab in iTable when this routine +** is called. If iColumn<0 then code is generated that extracts the rowid. +** +** This routine might attempt to reuse the value of the column that +** has already been loaded into a register. The value will always +** be used if it has not undergone any affinity changes. But if +** an affinity change has occurred, then the cached value will only be +** used if allowAffChng is true. */ -void sqlite3ExprCodeGetColumn(Vdbe *v, Table *pTab, int iColumn, int iTable){ +int sqlite3ExprCodeGetColumn( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* Description of the table we are reading from */ + int iColumn, /* Index of the table column */ + int iTable, /* The cursor pointing to the table */ + int iReg, /* Store results here */ + int allowAffChng /* True if prior affinity changes are OK */ +){ + Vdbe *v = pParse->pVdbe; + int i; + struct yColCache *p; + + for(i=0, p=pParse->aColCache; inColCache; i++, p++){ + if( p->iTable==iTable && p->iColumn==iColumn + && (!p->affChange || allowAffChng) ){ +#if 0 + sqlite3VdbeAddOp0(v, OP_Noop); + VdbeComment((v, "OPT: tab%d.col%d -> r%d", iTable, iColumn, p->iReg)); +#endif + return p->iReg; + } + } + assert( v!=0 ); if( iColumn<0 ){ int op = (pTab && IsVirtual(pTab)) ? OP_VRowid : OP_Rowid; - sqlite3VdbeAddOp(v, op, iTable, 0); + sqlite3VdbeAddOp2(v, op, iTable, iReg); }else if( pTab==0 ){ - sqlite3VdbeAddOp(v, OP_Column, iTable, iColumn); + sqlite3VdbeAddOp3(v, OP_Column, iTable, iColumn, iReg); }else{ int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; - sqlite3VdbeAddOp(v, op, iTable, iColumn); + sqlite3VdbeAddOp3(v, op, iTable, iColumn, iReg); sqlite3ColumnDefault(v, pTab, iColumn); #ifndef SQLITE_OMIT_FLOATING_POINT if( pTab->aCol[iColumn].affinity==SQLITE_AFF_REAL ){ - sqlite3VdbeAddOp(v, OP_RealAffinity, 0, 0); + sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); } #endif } + if( pParse->disableColCache==0 ){ + i = pParse->iColCache; + p = &pParse->aColCache[i]; + p->iTable = iTable; + p->iColumn = iColumn; + p->iReg = iReg; + p->affChange = 0; + i++; + if( i>=ArraySize(pParse->aColCache) ) i = 0; + if( i>pParse->nColCache ) pParse->nColCache = i; + pParse->iColCache = i; + } + return iReg; +} + +/* +** Clear all column cache entries associated with the vdbe +** cursor with cursor number iTable. +*/ +void sqlite3ExprClearColumnCache(Parse *pParse, int iTable){ + if( iTable<0 ){ + pParse->nColCache = 0; + pParse->iColCache = 0; + }else{ + int i; + for(i=0; inColCache; i++){ + if( pParse->aColCache[i].iTable==iTable ){ + testcase( i==pParse->nColCache-1 ); + pParse->aColCache[i] = pParse->aColCache[--pParse->nColCache]; + pParse->iColCache = pParse->nColCache; + } + } + } } /* -** Generate code into the current Vdbe to evaluate the given -** expression and leave the result on the top of stack. -** -** This code depends on the fact that certain token values (ex: TK_EQ) -** are the same as opcode values (ex: OP_Eq) that implement the corresponding -** operation. Special comments in vdbe.c and the mkopcodeh.awk script in -** the make process cause these values to align. Assert()s in the code -** below verify that the numbers are aligned correctly. +** Record the fact that an affinity change has occurred on iCount +** registers starting with iStart. */ -void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ - Vdbe *v = pParse->pVdbe; - int op; - int stackChng = 1; /* Amount of change to stack depth */ - - if( v==0 ) return; - if( pExpr==0 ){ - sqlite3VdbeAddOp(v, OP_Null, 0, 0); - return; +void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, int iCount){ + int iEnd = iStart + iCount - 1; + int i; + for(i=0; inColCache; i++){ + int r = pParse->aColCache[i].iReg; + if( r>=iStart && r<=iEnd ){ + pParse->aColCache[i].affChange = 1; + } + } +} + +/* +** Generate code to move content from registers iFrom...iFrom+nReg-1 +** over to iTo..iTo+nReg-1. Keep the column cache up-to-date. +*/ +void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ + int i; + if( iFrom==iTo ) return; + sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); + for(i=0; inColCache; i++){ + int x = pParse->aColCache[i].iReg; + if( x>=iFrom && xaColCache[i].iReg += iTo-iFrom; + } + } +} + +/* +** Generate code to copy content from registers iFrom...iFrom+nReg-1 +** over to iTo..iTo+nReg-1. +*/ +void sqlite3ExprCodeCopy(Parse *pParse, int iFrom, int iTo, int nReg){ + int i; + if( iFrom==iTo ) return; + for(i=0; ipVdbe, OP_Copy, iFrom+i, iTo+i); + } +} + +/* +** Return true if any register in the range iFrom..iTo (inclusive) +** is used as part of the column cache. +*/ +static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ + int i; + for(i=0; inColCache; i++){ + int r = pParse->aColCache[i].iReg; + if( r>=iFrom && r<=iTo ) return 1; + } + return 0; +} + +/* +** There is a value in register iReg. +** +** We are going to modify the value, so we need to make sure it +** is not a cached register. If iReg is a cached register, +** then clear the corresponding cache line. +*/ +void sqlite3ExprWritableRegister(Parse *pParse, int iReg){ + int i; + if( usedAsColumnCache(pParse, iReg, iReg) ){ + for(i=0; inColCache; i++){ + if( pParse->aColCache[i].iReg==iReg ){ + pParse->aColCache[i] = pParse->aColCache[--pParse->nColCache]; + pParse->iColCache = pParse->nColCache; + } + } + } +} + +/* +** If the last instruction coded is an ephemeral copy of any of +** the registers in the nReg registers beginning with iReg, then +** convert the last instruction from OP_SCopy to OP_Copy. +*/ +void sqlite3ExprHardCopy(Parse *pParse, int iReg, int nReg){ + int addr; + VdbeOp *pOp; + Vdbe *v; + + v = pParse->pVdbe; + addr = sqlite3VdbeCurrentAddr(v); + pOp = sqlite3VdbeGetOp(v, addr-1); + assert( pOp || pParse->db->mallocFailed ); + if( pOp && pOp->opcode==OP_SCopy && pOp->p1>=iReg && pOp->p1opcode = OP_Copy; + } +} + +/* +** Generate code to store the value of the iAlias-th alias in register +** target. The first time this is called, pExpr is evaluated to compute +** the value of the alias. The value is stored in an auxiliary register +** and the number of that register is returned. On subsequent calls, +** the register number is returned without generating any code. +** +** Note that in order for this to work, code must be generated in the +** same order that it is executed. +** +** Aliases are numbered starting with 1. So iAlias is in the range +** of 1 to pParse->nAlias inclusive. +** +** pParse->aAlias[iAlias-1] records the register number where the value +** of the iAlias-th alias is stored. If zero, that means that the +** alias has not yet been computed. +*/ +static int codeAlias(Parse *pParse, int iAlias, Expr *pExpr, int target){ + sqlite3 *db = pParse->db; + int iReg; + if( pParse->nAliasAllocnAlias ){ + pParse->aAlias = sqlite3DbReallocOrFree(db, pParse->aAlias, + sizeof(pParse->aAlias[0])*pParse->nAlias ); + testcase( db->mallocFailed && pParse->nAliasAlloc>0 ); + if( db->mallocFailed ) return 0; + memset(&pParse->aAlias[pParse->nAliasAlloc], 0, + (pParse->nAlias-pParse->nAliasAlloc)*sizeof(pParse->aAlias[0])); + pParse->nAliasAlloc = pParse->nAlias; + } + assert( iAlias>0 && iAlias<=pParse->nAlias ); + iReg = pParse->aAlias[iAlias-1]; + if( iReg==0 ){ + if( pParse->disableColCache ){ + iReg = sqlite3ExprCodeTarget(pParse, pExpr, target); + }else{ + iReg = ++pParse->nMem; + sqlite3ExprCode(pParse, pExpr, iReg); + pParse->aAlias[iAlias-1] = iReg; + } + } + return iReg; +} + +/* +** Generate code into the current Vdbe to evaluate the given +** expression. Attempt to store the results in register "target". +** Return the register where results are stored. +** +** With this routine, there is no guarantee that results will +** be stored in target. The result might be stored in some other +** register if it is convenient to do so. The calling function +** must check the return code and move the results to the desired +** register. +*/ +int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ + Vdbe *v = pParse->pVdbe; /* The VM under construction */ + int op; /* The opcode being coded */ + int inReg = target; /* Results stored in register inReg */ + int regFree1 = 0; /* If non-zero free this temporary register */ + int regFree2 = 0; /* If non-zero free this temporary register */ + int r1, r2, r3, r4; /* Various register numbers */ + sqlite3 *db; + + db = pParse->db; + assert( v!=0 || db->mallocFailed ); + assert( target>0 && target<=pParse->nMem ); + if( v==0 ) return 0; + + if( pExpr==0 ){ + op = TK_NULL; + }else{ + op = pExpr->op; } - op = pExpr->op; switch( op ){ case TK_AGG_COLUMN: { AggInfo *pAggInfo = pExpr->pAggInfo; struct AggInfo_col *pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ - sqlite3VdbeAddOp(v, OP_MemLoad, pCol->iMem, 0); + assert( pCol->iMem>0 ); + inReg = pCol->iMem; break; }else if( pAggInfo->useSortingIdx ){ - sqlite3VdbeAddOp(v, OP_Column, pAggInfo->sortingIdx, - pCol->iSorterColumn); + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdx, + pCol->iSorterColumn, target); break; } /* Otherwise, fall thru into the TK_COLUMN case */ @@ -1736,60 +1777,70 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ case TK_COLUMN: { if( pExpr->iTable<0 ){ /* This only happens when coding check constraints */ - assert( pParse->ckOffset>0 ); - sqlite3VdbeAddOp(v, OP_Dup, pParse->ckOffset-pExpr->iColumn-1, 1); + assert( pParse->ckBase>0 ); + inReg = pExpr->iColumn + pParse->ckBase; }else{ - sqlite3ExprCodeGetColumn(v, pExpr->pTab, pExpr->iColumn, pExpr->iTable); + testcase( (pExpr->flags & EP_AnyAff)!=0 ); + inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, + pExpr->iColumn, pExpr->iTable, target, + pExpr->flags & EP_AnyAff); } break; } case TK_INTEGER: { - codeInteger(v, (char*)pExpr->token.z, pExpr->token.n); + codeInteger(v, pExpr, 0, target); + break; + } + case TK_FLOAT: { + codeReal(v, (char*)pExpr->token.z, pExpr->token.n, 0, target); break; } - case TK_FLOAT: case TK_STRING: { - assert( TK_FLOAT==OP_Real ); - assert( TK_STRING==OP_String8 ); - sqlite3DequoteExpr(pExpr); - sqlite3VdbeOp3(v, op, 0, 0, (char*)pExpr->token.z, pExpr->token.n); + sqlite3DequoteExpr(db, pExpr); + sqlite3VdbeAddOp4(v,OP_String8, 0, target, 0, + (char*)pExpr->token.z, pExpr->token.n); break; } case TK_NULL: { - sqlite3VdbeAddOp(v, OP_Null, 0, 0); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); break; } #ifndef SQLITE_OMIT_BLOB_LITERAL case TK_BLOB: { int n; const char *z; - assert( TK_BLOB==OP_HexBlob ); + char *zBlob; + assert( pExpr->token.n>=3 ); + assert( pExpr->token.z[0]=='x' || pExpr->token.z[0]=='X' ); + assert( pExpr->token.z[1]=='\'' ); + assert( pExpr->token.z[pExpr->token.n-1]=='\'' ); n = pExpr->token.n - 3; z = (char*)pExpr->token.z + 2; - assert( n>=0 ); - if( n==0 ){ - z = ""; - } - sqlite3VdbeOp3(v, op, 0, 0, z, n); + zBlob = sqlite3HexToBlob(sqlite3VdbeDb(v), z, n); + sqlite3VdbeAddOp4(v, OP_Blob, n/2, target, 0, zBlob, P4_DYNAMIC); break; } #endif case TK_VARIABLE: { - sqlite3VdbeAddOp(v, OP_Variable, pExpr->iTable, 0); + sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iTable, target); if( pExpr->token.n>1 ){ - sqlite3VdbeChangeP3(v, -1, (char*)pExpr->token.z, pExpr->token.n); + sqlite3VdbeChangeP4(v, -1, (char*)pExpr->token.z, pExpr->token.n); } break; } case TK_REGISTER: { - sqlite3VdbeAddOp(v, OP_MemLoad, pExpr->iTable, 0); + inReg = pExpr->iTable; + break; + } + case TK_AS: { + inReg = codeAlias(pParse, pExpr->iTable, pExpr->pLeft, target); break; } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ int aff, to_op; - sqlite3ExprCode(pParse, pExpr->pLeft); + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); aff = sqlite3AffinityType(&pExpr->token); to_op = aff - SQLITE_AFF_TEXT + OP_ToText; assert( to_op==OP_ToText || aff!=SQLITE_AFF_TEXT ); @@ -1797,8 +1848,18 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ assert( to_op==OP_ToNumeric || aff!=SQLITE_AFF_NUMERIC ); assert( to_op==OP_ToInt || aff!=SQLITE_AFF_INTEGER ); assert( to_op==OP_ToReal || aff!=SQLITE_AFF_REAL ); - sqlite3VdbeAddOp(v, to_op, 0, 0); - stackChng = 0; + testcase( to_op==OP_ToText ); + testcase( to_op==OP_ToBlob ); + testcase( to_op==OP_ToNumeric ); + testcase( to_op==OP_ToInt ); + testcase( to_op==OP_ToReal ); + if( inReg!=target ){ + sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); + inReg = target; + } + sqlite3VdbeAddOp1(v, to_op, inReg); + testcase( usedAsColumnCache(pParse, inReg, inReg) ); + sqlite3ExprCacheAffinityChange(pParse, inReg, 1); break; } #endif /* SQLITE_OMIT_CAST */ @@ -1814,10 +1875,18 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ assert( TK_GE==OP_Ge ); assert( TK_EQ==OP_Eq ); assert( TK_NE==OP_Ne ); - sqlite3ExprCode(pParse, pExpr->pLeft); - sqlite3ExprCode(pParse, pExpr->pRight); - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, 0, 0); - stackChng = -1; + testcase( op==TK_LT ); + testcase( op==TK_LE ); + testcase( op==TK_GT ); + testcase( op==TK_GE ); + testcase( op==TK_EQ ); + testcase( op==TK_NE ); + codeCompareOperands(pParse, pExpr->pLeft, &r1, ®Free1, + pExpr->pRight, &r2, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, inReg, SQLITE_STOREP2); + testcase( regFree1==0 ); + testcase( regFree2==0 ); break; } case TK_AND: @@ -1843,48 +1912,68 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ assert( TK_LSHIFT==OP_ShiftLeft ); assert( TK_RSHIFT==OP_ShiftRight ); assert( TK_CONCAT==OP_Concat ); - sqlite3ExprCode(pParse, pExpr->pLeft); - sqlite3ExprCode(pParse, pExpr->pRight); - sqlite3VdbeAddOp(v, op, 0, 0); - stackChng = -1; + testcase( op==TK_AND ); + testcase( op==TK_OR ); + testcase( op==TK_PLUS ); + testcase( op==TK_MINUS ); + testcase( op==TK_REM ); + testcase( op==TK_BITAND ); + testcase( op==TK_BITOR ); + testcase( op==TK_SLASH ); + testcase( op==TK_LSHIFT ); + testcase( op==TK_RSHIFT ); + testcase( op==TK_CONCAT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + sqlite3VdbeAddOp3(v, op, r2, r1, target); + testcase( regFree1==0 ); + testcase( regFree2==0 ); break; } case TK_UMINUS: { Expr *pLeft = pExpr->pLeft; assert( pLeft ); if( pLeft->op==TK_FLOAT || pLeft->op==TK_INTEGER ){ - Token *p = &pLeft->token; - char *z = sqlite3MPrintf("-%.*s", p->n, p->z); if( pLeft->op==TK_FLOAT ){ - sqlite3VdbeOp3(v, OP_Real, 0, 0, z, p->n+1); + codeReal(v, (char*)pLeft->token.z, pLeft->token.n, 1, target); }else{ - codeInteger(v, z, p->n+1); + codeInteger(v, pLeft, 1, target); } - sqliteFree(z); - break; + }else{ + regFree1 = r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Integer, 0, r1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free2); + sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); + testcase( regFree2==0 ); } - /* Fall through into TK_NOT */ + inReg = target; + break; } case TK_BITNOT: case TK_NOT: { assert( TK_BITNOT==OP_BitNot ); assert( TK_NOT==OP_Not ); - sqlite3ExprCode(pParse, pExpr->pLeft); - sqlite3VdbeAddOp(v, op, 0, 0); - stackChng = 0; + testcase( op==TK_BITNOT ); + testcase( op==TK_NOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + inReg = target; + sqlite3VdbeAddOp2(v, op, r1, inReg); break; } case TK_ISNULL: case TK_NOTNULL: { - int dest; + int addr; assert( TK_ISNULL==OP_IsNull ); assert( TK_NOTNULL==OP_NotNull ); - sqlite3VdbeAddOp(v, OP_Integer, 1, 0); - sqlite3ExprCode(pParse, pExpr->pLeft); - dest = sqlite3VdbeCurrentAddr(v) + 2; - sqlite3VdbeAddOp(v, op, 1, dest); - sqlite3VdbeAddOp(v, OP_AddImm, -1, 0); - stackChng = 0; + testcase( op==TK_ISNULL ); + testcase( op==TK_NOTNULL ); + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + addr = sqlite3VdbeAddOp1(v, op, r1); + sqlite3VdbeAddOp2(v, OP_AddImm, target, -1); + sqlite3VdbeJumpHere(v, addr); break; } case TK_AGG_FUNCTION: { @@ -1893,7 +1982,7 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ sqlite3ErrorMsg(pParse, "misuse of aggregate: %T", &pExpr->span); }else{ - sqlite3VdbeAddOp(v, OP_MemLoad, pInfo->aFunc[pExpr->iAgg].iMem, 0); + inReg = pInfo->aFunc[pExpr->iAgg].iMem; } break; } @@ -1906,13 +1995,22 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ const char *zId; int constMask = 0; int i; - u8 enc = ENC(pParse->db); + u8 enc = ENC(db); CollSeq *pColl = 0; + + testcase( op==TK_CONST_FUNC ); + testcase( op==TK_FUNCTION ); zId = (char*)pExpr->token.z; nId = pExpr->token.n; - pDef = sqlite3FindFunction(pParse->db, zId, nId, nExpr, enc, 0); + pDef = sqlite3FindFunction(db, zId, nId, nExpr, enc, 0); assert( pDef!=0 ); - nExpr = sqlite3ExprCodeExprList(pParse, pList); + if( pList ){ + nExpr = pList->nExpr; + r1 = sqlite3GetTempRange(pParse, nExpr); + sqlite3ExprCodeExprList(pParse, pList, r1, 1); + }else{ + nExpr = r1 = 0; + } #ifndef SQLITE_OMIT_VIRTUALTABLE /* Possibly overload the function if the first argument is ** a virtual table column. @@ -1927,96 +2025,205 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ ** for function overloading. But we use the B term in "glob(B,A)". */ if( nExpr>=2 && (pExpr->flags & EP_InfixFunc) ){ - pDef = sqlite3VtabOverloadFunction(pDef, nExpr, pList->a[1].pExpr); + pDef = sqlite3VtabOverloadFunction(db, pDef, nExpr, pList->a[1].pExpr); }else if( nExpr>0 ){ - pDef = sqlite3VtabOverloadFunction(pDef, nExpr, pList->a[0].pExpr); + pDef = sqlite3VtabOverloadFunction(db, pDef, nExpr, pList->a[0].pExpr); } #endif for(i=0; ia[i].pExpr) ){ constMask |= (1<needCollSeq && !pColl ){ + if( (pDef->flags & SQLITE_FUNC_NEEDCOLL)!=0 && !pColl ){ pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); } } - if( pDef->needCollSeq ){ - if( !pColl ) pColl = pParse->db->pDfltColl; - sqlite3VdbeOp3(v, OP_CollSeq, 0, 0, (char *)pColl, P3_COLLSEQ); + if( pDef->flags & SQLITE_FUNC_NEEDCOLL ){ + if( !pColl ) pColl = db->pDfltColl; + sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeOp3(v, OP_Function, constMask, nExpr, (char*)pDef, P3_FUNCDEF); - stackChng = 1-nExpr; + sqlite3VdbeAddOp4(v, OP_Function, constMask, r1, target, + (char*)pDef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nExpr); + if( nExpr ){ + sqlite3ReleaseTempRange(pParse, r1, nExpr); + } + sqlite3ExprCacheAffinityChange(pParse, r1, nExpr); break; } #ifndef SQLITE_OMIT_SUBQUERY case TK_EXISTS: case TK_SELECT: { + testcase( op==TK_EXISTS ); + testcase( op==TK_SELECT ); if( pExpr->iColumn==0 ){ - sqlite3CodeSubselect(pParse, pExpr); + sqlite3CodeSubselect(pParse, pExpr, 0, 0); } - sqlite3VdbeAddOp(v, OP_MemLoad, pExpr->iColumn, 0); - VdbeComment((v, "# load subquery result")); + inReg = pExpr->iColumn; break; } case TK_IN: { - int addr; + int rNotFound = 0; + int rMayHaveNull = 0; + int j2, j3, j4, j5; char affinity; - int ckOffset = pParse->ckOffset; - sqlite3CodeSubselect(pParse, pExpr); + int eType; + + VdbeNoopComment((v, "begin IN expr r%d", target)); + eType = sqlite3FindInIndex(pParse, pExpr, &rMayHaveNull); + if( rMayHaveNull ){ + rNotFound = ++pParse->nMem; + } /* Figure out the affinity to use to create a key from the results ** of the expression. affinityStr stores a static string suitable for - ** P3 of OP_MakeRecord. + ** P4 of OP_MakeRecord. */ affinity = comparisonAffinity(pExpr); - sqlite3VdbeAddOp(v, OP_Integer, 1, 0); - pParse->ckOffset = (ckOffset ? (ckOffset+1) : 0); /* Code the from " IN (...)". The temporary table ** pExpr->iTable contains the values that make up the (...) set. */ - sqlite3ExprCode(pParse, pExpr->pLeft); - addr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp(v, OP_NotNull, -1, addr+4); /* addr + 0 */ - sqlite3VdbeAddOp(v, OP_Pop, 2, 0); - sqlite3VdbeAddOp(v, OP_Null, 0, 0); - sqlite3VdbeAddOp(v, OP_Goto, 0, addr+7); - sqlite3VdbeOp3(v, OP_MakeRecord, 1, 0, &affinity, 1); /* addr + 4 */ - sqlite3VdbeAddOp(v, OP_Found, pExpr->iTable, addr+7); - sqlite3VdbeAddOp(v, OP_AddImm, -1, 0); /* addr + 6 */ + pParse->disableColCache++; + sqlite3ExprCode(pParse, pExpr->pLeft, target); + pParse->disableColCache--; + j2 = sqlite3VdbeAddOp1(v, OP_IsNull, target); + if( eType==IN_INDEX_ROWID ){ + j3 = sqlite3VdbeAddOp1(v, OP_MustBeInt, target); + j4 = sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, 0, target); + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + j5 = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, j3); + sqlite3VdbeJumpHere(v, j4); + sqlite3VdbeAddOp2(v, OP_Integer, 0, target); + }else{ + r2 = regFree2 = sqlite3GetTempReg(pParse); + /* Create a record and test for set membership. If the set contains + ** the value, then jump to the end of the test code. The target + ** register still contains the true (1) value written to it earlier. + */ + sqlite3VdbeAddOp4(v, OP_MakeRecord, target, 1, r2, &affinity, 1); + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + j5 = sqlite3VdbeAddOp3(v, OP_Found, pExpr->iTable, 0, r2); + + /* If the set membership test fails, then the result of the + ** "x IN (...)" expression must be either 0 or NULL. If the set + ** contains no NULL values, then the result is 0. If the set + ** contains one or more NULL values, then the result of the + ** expression is also NULL. + */ + if( rNotFound==0 ){ + /* This branch runs if it is known at compile time (now) that + ** the set contains no NULL values. This happens as the result + ** of a "NOT NULL" constraint in the database schema. No need + ** to test the data structure at runtime in this case. + */ + sqlite3VdbeAddOp2(v, OP_Integer, 0, target); + }else{ + /* This block populates the rNotFound register with either NULL + ** or 0 (an integer value). If the data structure contains one + ** or more NULLs, then set rNotFound to NULL. Otherwise, set it + ** to 0. If register rMayHaveNull is already set to some value + ** other than NULL, then the test has already been run and + ** rNotFound is already populated. + */ + static const char nullRecord[] = { 0x02, 0x00 }; + j3 = sqlite3VdbeAddOp1(v, OP_NotNull, rMayHaveNull); + sqlite3VdbeAddOp2(v, OP_Null, 0, rNotFound); + sqlite3VdbeAddOp4(v, OP_Blob, 2, rMayHaveNull, 0, + nullRecord, P4_STATIC); + j4 = sqlite3VdbeAddOp3(v, OP_Found, pExpr->iTable, 0, rMayHaveNull); + sqlite3VdbeAddOp2(v, OP_Integer, 0, rNotFound); + sqlite3VdbeJumpHere(v, j4); + sqlite3VdbeJumpHere(v, j3); + + /* Copy the value of register rNotFound (which is either NULL or 0) + ** into the target register. This will be the result of the + ** expression. + */ + sqlite3VdbeAddOp2(v, OP_Copy, rNotFound, target); + } + } + sqlite3VdbeJumpHere(v, j2); + sqlite3VdbeJumpHere(v, j5); + VdbeComment((v, "end IN expr r%d", target)); break; } #endif + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ case TK_BETWEEN: { Expr *pLeft = pExpr->pLeft; struct ExprList_item *pLItem = pExpr->pList->a; Expr *pRight = pLItem->pExpr; - sqlite3ExprCode(pParse, pLeft); - sqlite3VdbeAddOp(v, OP_Dup, 0, 0); - sqlite3ExprCode(pParse, pRight); - codeCompare(pParse, pLeft, pRight, OP_Ge, 0, 0); - sqlite3VdbeAddOp(v, OP_Pull, 1, 0); + + codeCompareOperands(pParse, pLeft, &r1, ®Free1, + pRight, &r2, ®Free2); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + r3 = sqlite3GetTempReg(pParse); + r4 = sqlite3GetTempReg(pParse); + codeCompare(pParse, pLeft, pRight, OP_Ge, + r1, r2, r3, SQLITE_STOREP2); pLItem++; pRight = pLItem->pExpr; - sqlite3ExprCode(pParse, pRight); - codeCompare(pParse, pLeft, pRight, OP_Le, 0, 0); - sqlite3VdbeAddOp(v, OP_And, 0, 0); + sqlite3ReleaseTempReg(pParse, regFree2); + r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); + testcase( regFree2==0 ); + codeCompare(pParse, pLeft, pRight, OP_Le, r1, r2, r4, SQLITE_STOREP2); + sqlite3VdbeAddOp3(v, OP_And, r3, r4, target); + sqlite3ReleaseTempReg(pParse, r3); + sqlite3ReleaseTempReg(pParse, r4); break; } case TK_UPLUS: { - sqlite3ExprCode(pParse, pExpr->pLeft); - stackChng = 0; + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); break; } + + /* + ** Form A: + ** CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END + ** + ** Form B: + ** CASE WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END + ** + ** Form A is can be transformed into the equivalent form B as follows: + ** CASE WHEN x=e1 THEN r1 WHEN x=e2 THEN r2 ... + ** WHEN x=eN THEN rN ELSE y END + ** + ** X (if it exists) is in pExpr->pLeft. + ** Y is in pExpr->pRight. The Y is also optional. If there is no + ** ELSE clause and no other term matches, then the result of the + ** exprssion is NULL. + ** Ei is in pExpr->pList->a[i*2] and Ri is pExpr->pList->a[i*2+1]. + ** + ** The result of the expression is the Ri for the first matching Ei, + ** or if there is no matching Ei, the ELSE term Y, or if there is + ** no ELSE term, NULL. + */ case TK_CASE: { - int expr_end_label; - int jumpInst; - int nExpr; - int i; - ExprList *pEList; - struct ExprList_item *aListelem; + int endLabel; /* GOTO label for end of CASE stmt */ + int nextCase; /* GOTO label for next WHEN clause */ + int nExpr; /* 2x number of WHEN terms */ + int i; /* Loop counter */ + ExprList *pEList; /* List of WHEN terms */ + struct ExprList_item *aListelem; /* Array of WHEN terms */ + Expr opCompare; /* The X==Ei expression */ + Expr cacheX; /* Cached expression X */ + Expr *pX; /* The X expression */ + Expr *pTest = 0; /* X==Ei (form A) or just Ei (form B) */ assert(pExpr->pList); assert((pExpr->pList->nExpr % 2) == 0); @@ -2024,33 +2231,42 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ pEList = pExpr->pList; aListelem = pEList->a; nExpr = pEList->nExpr; - expr_end_label = sqlite3VdbeMakeLabel(v); - if( pExpr->pLeft ){ - sqlite3ExprCode(pParse, pExpr->pLeft); + endLabel = sqlite3VdbeMakeLabel(v); + if( (pX = pExpr->pLeft)!=0 ){ + cacheX = *pX; + testcase( pX->op==TK_COLUMN || pX->op==TK_REGISTER ); + cacheX.iTable = sqlite3ExprCodeTemp(pParse, pX, ®Free1); + testcase( regFree1==0 ); + cacheX.op = TK_REGISTER; + opCompare.op = TK_EQ; + opCompare.pLeft = &cacheX; + pTest = &opCompare; } + pParse->disableColCache++; for(i=0; ipLeft ){ - sqlite3VdbeAddOp(v, OP_Dup, 1, 1); - jumpInst = codeCompare(pParse, pExpr->pLeft, aListelem[i].pExpr, - OP_Ne, 0, 1); - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); + if( pX ){ + assert( pTest!=0 ); + opCompare.pRight = aListelem[i].pExpr; }else{ - jumpInst = sqlite3VdbeAddOp(v, OP_IfNot, 1, 0); + pTest = aListelem[i].pExpr; } - sqlite3ExprCode(pParse, aListelem[i+1].pExpr); - sqlite3VdbeAddOp(v, OP_Goto, 0, expr_end_label); - sqlite3VdbeJumpHere(v, jumpInst); - } - if( pExpr->pLeft ){ - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); + nextCase = sqlite3VdbeMakeLabel(v); + testcase( pTest->op==TK_COLUMN || pTest->op==TK_REGISTER ); + sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); + testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); + testcase( aListelem[i+1].pExpr->op==TK_REGISTER ); + sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); + sqlite3VdbeAddOp2(v, OP_Goto, 0, endLabel); + sqlite3VdbeResolveLabel(v, nextCase); } if( pExpr->pRight ){ - sqlite3ExprCode(pParse, pExpr->pRight); + sqlite3ExprCode(pParse, pExpr->pRight, target); }else{ - sqlite3VdbeAddOp(v, OP_Null, 0, 0); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); } - sqlite3VdbeResolveLabel(v, expr_end_label); + sqlite3VdbeResolveLabel(v, endLabel); + assert( pParse->disableColCache>0 ); + pParse->disableColCache--; break; } #ifndef SQLITE_OMIT_TRIGGER @@ -2058,76 +2274,240 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr){ if( !pParse->trigStack ){ sqlite3ErrorMsg(pParse, "RAISE() may only be used within a trigger-program"); - return; + return 0; } if( pExpr->iColumn!=OE_Ignore ){ assert( pExpr->iColumn==OE_Rollback || pExpr->iColumn == OE_Abort || pExpr->iColumn == OE_Fail ); - sqlite3DequoteExpr(pExpr); - sqlite3VdbeOp3(v, OP_Halt, SQLITE_CONSTRAINT, pExpr->iColumn, + sqlite3DequoteExpr(db, pExpr); + sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, pExpr->iColumn, 0, (char*)pExpr->token.z, pExpr->token.n); } else { assert( pExpr->iColumn == OE_Ignore ); - sqlite3VdbeAddOp(v, OP_ContextPop, 0, 0); - sqlite3VdbeAddOp(v, OP_Goto, 0, pParse->trigStack->ignoreJump); - VdbeComment((v, "# raise(IGNORE)")); + sqlite3VdbeAddOp2(v, OP_ContextPop, 0, 0); + sqlite3VdbeAddOp2(v, OP_Goto, 0, pParse->trigStack->ignoreJump); + VdbeComment((v, "raise(IGNORE)")); } - stackChng = 0; break; } #endif } - - if( pParse->ckOffset ){ - pParse->ckOffset += stackChng; - assert( pParse->ckOffset ); - } + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); + return inReg; } -#ifndef SQLITE_OMIT_TRIGGER /* -** Generate code that evalutes the given expression and leaves the result -** on the stack. See also sqlite3ExprCode(). +** Generate code to evaluate an expression and store the results +** into a register. Return the register number where the results +** are stored. ** -** This routine might also cache the result and modify the pExpr tree -** so that it will make use of the cached result on subsequent evaluations -** rather than evaluate the whole expression again. Trivial expressions are -** not cached. If the expression is cached, its result is stored in a -** memory location. +** If the register is a temporary register that can be deallocated, +** then write its number into *pReg. If the result register is not +** a temporary, then set *pReg to zero. */ -void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr){ +int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ + int r1 = sqlite3GetTempReg(pParse); + int r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); + if( r2==r1 ){ + *pReg = r1; + }else{ + sqlite3ReleaseTempReg(pParse, r1); + *pReg = 0; + } + return r2; +} + +/* +** Generate code that will evaluate expression pExpr and store the +** results in register target. The results are guaranteed to appear +** in register target. +*/ +int sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ + int inReg; + + assert( target>0 && target<=pParse->nMem ); + inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); + assert( pParse->pVdbe || pParse->db->mallocFailed ); + if( inReg!=target && pParse->pVdbe ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); + } + return target; +} + +/* +** Generate code that evalutes the given expression and puts the result +** in register target. +** +** Also make a copy of the expression results into another "cache" register +** and modify the expression so that the next time it is evaluated, +** the result is a copy of the cache register. +** +** This routine is used for expressions that are used multiple +** times. They are evaluated once and the results of the expression +** are reused. +*/ +int sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ Vdbe *v = pParse->pVdbe; - int iMem; - int addr1, addr2; - if( v==0 ) return; - addr1 = sqlite3VdbeCurrentAddr(v); - sqlite3ExprCode(pParse, pExpr); - addr2 = sqlite3VdbeCurrentAddr(v); - if( addr2>addr1+1 || sqlite3VdbeGetOp(v, addr1)->opcode==OP_Function ){ - iMem = pExpr->iTable = pParse->nMem++; - sqlite3VdbeAddOp(v, OP_MemStore, iMem, 0); + int inReg; + inReg = sqlite3ExprCode(pParse, pExpr, target); + assert( target>0 ); + if( pExpr->op!=TK_REGISTER ){ + int iMem; + iMem = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Copy, inReg, iMem); + pExpr->iTable = iMem; pExpr->op = TK_REGISTER; } + return inReg; } + +/* +** Return TRUE if pExpr is an constant expression that is appropriate +** for factoring out of a loop. Appropriate expressions are: +** +** * Any expression that evaluates to two or more opcodes. +** +** * Any OP_Integer, OP_Real, OP_String, OP_Blob, OP_Null, +** or OP_Variable that does not need to be placed in a +** specific register. +** +** There is no point in factoring out single-instruction constant +** expressions that need to be placed in a particular register. +** We could factor them out, but then we would end up adding an +** OP_SCopy instruction to move the value into the correct register +** later. We might as well just use the original instruction and +** avoid the OP_SCopy. +*/ +static int isAppropriateForFactoring(Expr *p){ + if( !sqlite3ExprIsConstantNotJoin(p) ){ + return 0; /* Only constant expressions are appropriate for factoring */ + } + if( (p->flags & EP_FixedDest)==0 ){ + return 1; /* Any constant without a fixed destination is appropriate */ + } + while( p->op==TK_UPLUS ) p = p->pLeft; + switch( p->op ){ +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: #endif + case TK_VARIABLE: + case TK_INTEGER: + case TK_FLOAT: + case TK_NULL: + case TK_STRING: { + testcase( p->op==TK_BLOB ); + testcase( p->op==TK_VARIABLE ); + testcase( p->op==TK_INTEGER ); + testcase( p->op==TK_FLOAT ); + testcase( p->op==TK_NULL ); + testcase( p->op==TK_STRING ); + /* Single-instruction constants with a fixed destination are + ** better done in-line. If we factor them, they will just end + ** up generating an OP_SCopy to move the value to the destination + ** register. */ + return 0; + } + case TK_UMINUS: { + if( p->pLeft->op==TK_FLOAT || p->pLeft->op==TK_INTEGER ){ + return 0; + } + break; + } + default: { + break; + } + } + return 1; +} + +/* +** If pExpr is a constant expression that is appropriate for +** factoring out of a loop, then evaluate the expression +** into a register and convert the expression into a TK_REGISTER +** expression. +*/ +static int evalConstExpr(Walker *pWalker, Expr *pExpr){ + Parse *pParse = pWalker->pParse; + switch( pExpr->op ){ + case TK_REGISTER: { + return 1; + } + case TK_FUNCTION: + case TK_AGG_FUNCTION: + case TK_CONST_FUNC: { + /* The arguments to a function have a fixed destination. + ** Mark them this way to avoid generated unneeded OP_SCopy + ** instructions. + */ + ExprList *pList = pExpr->pList; + if( pList ){ + int i = pList->nExpr; + struct ExprList_item *pItem = pList->a; + for(; i>0; i--, pItem++){ + if( pItem->pExpr ) pItem->pExpr->flags |= EP_FixedDest; + } + } + break; + } + } + if( isAppropriateForFactoring(pExpr) ){ + int r1 = ++pParse->nMem; + int r2; + r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); + if( r1!=r2 ) sqlite3ReleaseTempReg(pParse, r1); + pExpr->op = TK_REGISTER; + pExpr->iTable = r2; + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Preevaluate constant subexpressions within pExpr and store the +** results in registers. Modify pExpr so that the constant subexpresions +** are TK_REGISTER opcodes that refer to the precomputed values. +*/ +void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ + Walker w; + w.xExprCallback = evalConstExpr; + w.xSelectCallback = 0; + w.pParse = pParse; + sqlite3WalkExpr(&w, pExpr); +} + /* ** Generate code that pushes the value of every element of the given -** expression list onto the stack. +** expression list into a sequence of registers beginning at target. ** -** Return the number of elements pushed onto the stack. +** Return the number of elements evaluated. */ int sqlite3ExprCodeExprList( Parse *pParse, /* Parsing context */ - ExprList *pList /* The expression list to be coded */ + ExprList *pList, /* The expression list to be coded */ + int target, /* Where to write results */ + int doHardCopy /* Make a hard copy of every element */ ){ struct ExprList_item *pItem; int i, n; - if( pList==0 ) return 0; + assert( pList!=0 ); + assert( target>0 ); n = pList->nExpr; - for(pItem=pList->a, i=n; i>0; i--, pItem++){ - sqlite3ExprCode(pParse, pItem->pExpr); + for(pItem=pList->a, i=0; iiAlias ){ + int iReg = codeAlias(pParse, pItem->iAlias, pItem->pExpr, target+i); + Vdbe *v = sqlite3GetVdbe(pParse); + if( iReg!=target+i ){ + sqlite3VdbeAddOp2(v, OP_SCopy, iReg, target+i); + } + }else{ + sqlite3ExprCode(pParse, pItem->pExpr, target+i); + } + if( doHardCopy ){ + sqlite3ExprHardCopy(pParse, target, n); + } } return n; } @@ -2138,7 +2518,7 @@ int sqlite3ExprCodeExprList( ** continues straight thru if the expression is false. ** ** If the expression evaluates to NULL (neither true nor false), then -** take the jump if the jumpIfNull flag is true. +** take the jump if the jumpIfNull flag is SQLITE_JUMPIFNULL. ** ** This code depends on the fact that certain token values (ex: TK_EQ) ** are the same as opcode values (ex: OP_Eq) that implement the corresponding @@ -2149,23 +2529,38 @@ int sqlite3ExprCodeExprList( void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ Vdbe *v = pParse->pVdbe; int op = 0; - int ckOffset = pParse->ckOffset; + int regFree1 = 0; + int regFree2 = 0; + int r1, r2; + + assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); if( v==0 || pExpr==0 ) return; op = pExpr->op; switch( op ){ case TK_AND: { int d2 = sqlite3VdbeMakeLabel(v); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2, !jumpIfNull); + testcase( jumpIfNull==0 ); + testcase( pParse->disableColCache==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2,jumpIfNull^SQLITE_JUMPIFNULL); + pParse->disableColCache++; sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + assert( pParse->disableColCache>0 ); + pParse->disableColCache--; sqlite3VdbeResolveLabel(v, d2); break; } case TK_OR: { + testcase( jumpIfNull==0 ); + testcase( pParse->disableColCache==0 ); sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + pParse->disableColCache++; sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + assert( pParse->disableColCache>0 ); + pParse->disableColCache--; break; } case TK_NOT: { + testcase( jumpIfNull==0 ); sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); break; } @@ -2181,50 +2576,74 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ assert( TK_GE==OP_Ge ); assert( TK_EQ==OP_Eq ); assert( TK_NE==OP_Ne ); - sqlite3ExprCode(pParse, pExpr->pLeft); - sqlite3ExprCode(pParse, pExpr->pRight); - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, dest, jumpIfNull); + testcase( op==TK_LT ); + testcase( op==TK_LE ); + testcase( op==TK_GT ); + testcase( op==TK_GE ); + testcase( op==TK_EQ ); + testcase( op==TK_NE ); + testcase( jumpIfNull==0 ); + codeCompareOperands(pParse, pExpr->pLeft, &r1, ®Free1, + pExpr->pRight, &r2, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, jumpIfNull); + testcase( regFree1==0 ); + testcase( regFree2==0 ); break; } case TK_ISNULL: case TK_NOTNULL: { assert( TK_ISNULL==OP_IsNull ); assert( TK_NOTNULL==OP_NotNull ); - sqlite3ExprCode(pParse, pExpr->pLeft); - sqlite3VdbeAddOp(v, op, 1, dest); + testcase( op==TK_ISNULL ); + testcase( op==TK_NOTNULL ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp2(v, op, r1, dest); + testcase( regFree1==0 ); break; } case TK_BETWEEN: { - /* The expression "x BETWEEN y AND z" is implemented as: + /* x BETWEEN y AND z ** - ** 1 IF (x < y) GOTO 3 - ** 2 IF (x <= z) GOTO - ** 3 ... + ** Is equivalent to + ** + ** x>=y AND x<=z + ** + ** Code it as such, taking care to do the common subexpression + ** elementation of x. */ - int addr; - Expr *pLeft = pExpr->pLeft; - Expr *pRight = pExpr->pList->a[0].pExpr; - sqlite3ExprCode(pParse, pLeft); - sqlite3VdbeAddOp(v, OP_Dup, 0, 0); - sqlite3ExprCode(pParse, pRight); - addr = codeCompare(pParse, pLeft, pRight, OP_Lt, 0, !jumpIfNull); + Expr exprAnd; + Expr compLeft; + Expr compRight; + Expr exprX; - pRight = pExpr->pList->a[1].pExpr; - sqlite3ExprCode(pParse, pRight); - codeCompare(pParse, pLeft, pRight, OP_Le, dest, jumpIfNull); - - sqlite3VdbeAddOp(v, OP_Integer, 0, 0); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); + exprX = *pExpr->pLeft; + exprAnd.op = TK_AND; + exprAnd.pLeft = &compLeft; + exprAnd.pRight = &compRight; + compLeft.op = TK_GE; + compLeft.pLeft = &exprX; + compLeft.pRight = pExpr->pList->a[0].pExpr; + compRight.op = TK_LE; + compRight.pLeft = &exprX; + compRight.pRight = pExpr->pList->a[1].pExpr; + exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1); + testcase( regFree1==0 ); + exprX.op = TK_REGISTER; + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull); break; } default: { - sqlite3ExprCode(pParse, pExpr); - sqlite3VdbeAddOp(v, OP_If, jumpIfNull, dest); + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); break; } } - pParse->ckOffset = ckOffset; + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); } /* @@ -2233,12 +2652,17 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ ** continues straight thru if the expression is true. ** ** If the expression evaluates to NULL (neither true nor false) then -** jump if jumpIfNull is true or fall through if jumpIfNull is false. +** jump if jumpIfNull is SQLITE_JUMPIFNULL or fall through if jumpIfNull +** is 0. */ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ Vdbe *v = pParse->pVdbe; int op = 0; - int ckOffset = pParse->ckOffset; + int regFree1 = 0; + int regFree2 = 0; + int r1, r2; + + assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); if( v==0 || pExpr==0 ) return; /* The value of pExpr->op and op are related as follows: @@ -2274,14 +2698,24 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ switch( pExpr->op ){ case TK_AND: { + testcase( jumpIfNull==0 ); + testcase( pParse->disableColCache==0 ); sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + pParse->disableColCache++; sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + assert( pParse->disableColCache>0 ); + pParse->disableColCache--; break; } case TK_OR: { int d2 = sqlite3VdbeMakeLabel(v); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, !jumpIfNull); + testcase( jumpIfNull==0 ); + testcase( pParse->disableColCache==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, jumpIfNull^SQLITE_JUMPIFNULL); + pParse->disableColCache++; sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + assert( pParse->disableColCache>0 ); + pParse->disableColCache--; sqlite3VdbeResolveLabel(v, d2); break; } @@ -2295,47 +2729,72 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_GE: case TK_NE: case TK_EQ: { - sqlite3ExprCode(pParse, pExpr->pLeft); - sqlite3ExprCode(pParse, pExpr->pRight); - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, dest, jumpIfNull); + testcase( op==TK_LT ); + testcase( op==TK_LE ); + testcase( op==TK_GT ); + testcase( op==TK_GE ); + testcase( op==TK_EQ ); + testcase( op==TK_NE ); + testcase( jumpIfNull==0 ); + codeCompareOperands(pParse, pExpr->pLeft, &r1, ®Free1, + pExpr->pRight, &r2, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, jumpIfNull); + testcase( regFree1==0 ); + testcase( regFree2==0 ); break; } case TK_ISNULL: case TK_NOTNULL: { - sqlite3ExprCode(pParse, pExpr->pLeft); - sqlite3VdbeAddOp(v, op, 1, dest); + testcase( op==TK_ISNULL ); + testcase( op==TK_NOTNULL ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp2(v, op, r1, dest); + testcase( regFree1==0 ); break; } case TK_BETWEEN: { - /* The expression is "x BETWEEN y AND z". It is implemented as: + /* x BETWEEN y AND z ** - ** 1 IF (x >= y) GOTO 3 - ** 2 GOTO - ** 3 IF (x > z) GOTO + ** Is equivalent to + ** + ** x>=y AND x<=z + ** + ** Code it as such, taking care to do the common subexpression + ** elementation of x. */ - int addr; - Expr *pLeft = pExpr->pLeft; - Expr *pRight = pExpr->pList->a[0].pExpr; - sqlite3ExprCode(pParse, pLeft); - sqlite3VdbeAddOp(v, OP_Dup, 0, 0); - sqlite3ExprCode(pParse, pRight); - addr = sqlite3VdbeCurrentAddr(v); - codeCompare(pParse, pLeft, pRight, OP_Ge, addr+3, !jumpIfNull); + Expr exprAnd; + Expr compLeft; + Expr compRight; + Expr exprX; - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); - sqlite3VdbeAddOp(v, OP_Goto, 0, dest); - pRight = pExpr->pList->a[1].pExpr; - sqlite3ExprCode(pParse, pRight); - codeCompare(pParse, pLeft, pRight, OP_Gt, dest, jumpIfNull); + exprX = *pExpr->pLeft; + exprAnd.op = TK_AND; + exprAnd.pLeft = &compLeft; + exprAnd.pRight = &compRight; + compLeft.op = TK_GE; + compLeft.pLeft = &exprX; + compLeft.pRight = pExpr->pList->a[0].pExpr; + compRight.op = TK_LE; + compRight.pLeft = &exprX; + compRight.pRight = pExpr->pList->a[1].pExpr; + exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1); + testcase( regFree1==0 ); + exprX.op = TK_REGISTER; + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull); break; } default: { - sqlite3ExprCode(pParse, pExpr); - sqlite3VdbeAddOp(v, OP_IfNot, jumpIfNull, dest); + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); break; } } - pParse->ckOffset = ckOffset; + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); } /* @@ -2389,9 +2848,10 @@ int sqlite3ExprCompare(Expr *pA, Expr *pB){ ** Add a new element to the pAggInfo->aCol[] array. Return the index of ** the new element. Return a negative number if malloc fails. */ -static int addAggInfoColumn(AggInfo *pInfo){ +static int addAggInfoColumn(sqlite3 *db, AggInfo *pInfo){ int i; pInfo->aCol = sqlite3ArrayAllocate( + db, pInfo->aCol, sizeof(pInfo->aCol[0]), 3, @@ -2406,9 +2866,10 @@ static int addAggInfoColumn(AggInfo *pInfo){ ** Add a new element to the pAggInfo->aFunc[] array. Return the index of ** the new element. Return a negative number if malloc fails. */ -static int addAggInfoFunc(AggInfo *pInfo){ +static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ int i; pInfo->aFunc = sqlite3ArrayAllocate( + db, pInfo->aFunc, sizeof(pInfo->aFunc[0]), 3, @@ -2420,23 +2881,22 @@ static int addAggInfoFunc(AggInfo *pInfo){ } /* -** This is an xFunc for walkExprTree() used to implement -** sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates +** This is the xExprCallback for a tree walker. It is used to +** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates ** for additional information. -** -** This routine analyzes the aggregate function at pExpr. */ -static int analyzeAggregate(void *pArg, Expr *pExpr){ +static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ int i; - NameContext *pNC = (NameContext *)pArg; + NameContext *pNC = pWalker->u.pNC; Parse *pParse = pNC->pParse; SrcList *pSrcList = pNC->pSrcList; AggInfo *pAggInfo = pNC->pAggInfo; - switch( pExpr->op ){ case TK_AGG_COLUMN: case TK_COLUMN: { + testcase( pExpr->op==TK_AGG_COLUMN ); + testcase( pExpr->op==TK_COLUMN ); /* Check to see if the column is in one of the tables in the FROM ** clause of the aggregate query */ if( pSrcList ){ @@ -2458,12 +2918,14 @@ static int analyzeAggregate(void *pArg, Expr *pExpr){ break; } } - if( k>=pAggInfo->nColumn && (k = addAggInfoColumn(pAggInfo))>=0 ){ + if( (k>=pAggInfo->nColumn) + && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 + ){ pCol = &pAggInfo->aCol[k]; pCol->pTab = pExpr->pTab; pCol->iTable = pExpr->iTable; pCol->iColumn = pExpr->iColumn; - pCol->iMem = pParse->nMem++; + pCol->iMem = ++pParse->nMem; pCol->iSorterColumn = -1; pCol->pExpr = pExpr; if( pAggInfo->pGroupBy ){ @@ -2496,7 +2958,7 @@ static int analyzeAggregate(void *pArg, Expr *pExpr){ } /* endif pExpr->iTable==pItem->iCursor */ } /* end loop over pSrcList */ } - return 1; + return WRC_Prune; } case TK_AGG_FUNCTION: { /* The pNC->nDepth==0 test causes aggregate functions in subqueries @@ -2515,11 +2977,11 @@ static int analyzeAggregate(void *pArg, Expr *pExpr){ /* pExpr is original. Make a new entry in pAggInfo->aFunc[] */ u8 enc = ENC(pParse->db); - i = addAggInfoFunc(pAggInfo); + i = addAggInfoFunc(pParse->db, pAggInfo); if( i>=0 ){ pItem = &pAggInfo->aFunc[i]; pItem->pExpr = pExpr; - pItem->iMem = pParse->nMem++; + pItem->iMem = ++pParse->nMem; pItem->pFunc = sqlite3FindFunction(pParse->db, (char*)pExpr->token.z, pExpr->token.n, pExpr->pList ? pExpr->pList->nExpr : 0, enc, 0); @@ -2534,21 +2996,22 @@ static int analyzeAggregate(void *pArg, Expr *pExpr){ */ pExpr->iAgg = i; pExpr->pAggInfo = pAggInfo; - return 1; + return WRC_Prune; } } } - - /* Recursively walk subqueries looking for TK_COLUMN nodes that need - ** to be changed to TK_AGG_COLUMN. But increment nDepth so that - ** TK_AGG_FUNCTION nodes in subqueries will be unchanged. - */ - if( pExpr->pSelect ){ + return WRC_Continue; +} +static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ + NameContext *pNC = pWalker->u.pNC; + if( pNC->nDepth==0 ){ pNC->nDepth++; - walkSelectExpr(pExpr->pSelect, analyzeAggregate, pNC); + sqlite3WalkSelect(pWalker, pSelect); pNC->nDepth--; + return WRC_Prune; + }else{ + return WRC_Continue; } - return 0; } /* @@ -2557,15 +3020,14 @@ static int analyzeAggregate(void *pArg, Expr *pExpr){ ** Make additional entries to the pParse->aAgg[] array as necessary. ** ** This routine should only be called after the expression has been -** analyzed by sqlite3ExprResolveNames(). -** -** If errors are seen, leave an error message in zErrMsg and return -** the number of errors. +** analyzed by sqlite3ResolveExprNames(). */ -int sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ - int nErr = pNC->pParse->nErr; - walkExprTree(pExpr, analyzeAggregate, pNC); - return pNC->pParse->nErr - nErr; +void sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ + Walker w; + w.xExprCallback = analyzeAggregate; + w.xSelectCallback = analyzeAggregatesInSelect; + w.u.pNC = pNC; + sqlite3WalkExpr(&w, pExpr); } /* @@ -2574,14 +3036,51 @@ int sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ ** ** If an error is found, the analysis is cut short. */ -int sqlite3ExprAnalyzeAggList(NameContext *pNC, ExprList *pList){ +void sqlite3ExprAnalyzeAggList(NameContext *pNC, ExprList *pList){ struct ExprList_item *pItem; int i; - int nErr = 0; if( pList ){ - for(pItem=pList->a, i=0; nErr==0 && inExpr; i++, pItem++){ - nErr += sqlite3ExprAnalyzeAggregates(pNC, pItem->pExpr); + for(pItem=pList->a, i=0; inExpr; i++, pItem++){ + sqlite3ExprAnalyzeAggregates(pNC, pItem->pExpr); } } - return nErr; +} + +/* +** Allocate or deallocate temporary use registers during code generation. +*/ +int sqlite3GetTempReg(Parse *pParse){ + if( pParse->nTempReg==0 ){ + return ++pParse->nMem; + } + return pParse->aTempReg[--pParse->nTempReg]; +} +void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ + if( iReg && pParse->nTempRegaTempReg) ){ + sqlite3ExprWritableRegister(pParse, iReg); + pParse->aTempReg[pParse->nTempReg++] = iReg; + } +} + +/* +** Allocate or deallocate a block of nReg consecutive registers +*/ +int sqlite3GetTempRange(Parse *pParse, int nReg){ + int i, n; + i = pParse->iRangeReg; + n = pParse->nRangeReg; + if( nReg<=n && !usedAsColumnCache(pParse, i, i+n-1) ){ + pParse->iRangeReg += nReg; + pParse->nRangeReg -= nReg; + }else{ + i = pParse->nMem+1; + pParse->nMem += nReg; + } + return i; +} +void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ + if( nReg>pParse->nRangeReg ){ + pParse->nRangeReg = nReg; + pParse->iRangeReg = iReg; + } } diff --git a/fault.c b/fault.c new file mode 100644 index 0000000..b36dd36 --- /dev/null +++ b/fault.c @@ -0,0 +1,91 @@ +/* +** 2008 Jan 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** $Id: fault.c,v 1.11 2008/09/02 00:52:52 drh Exp $ +*/ + +/* +** This file contains code to support the concept of "benign" +** malloc failures (when the xMalloc() or xRealloc() method of the +** sqlite3_mem_methods structure fails to allocate a block of memory +** and returns 0). +** +** Most malloc failures are non-benign. After they occur, SQLite +** abandons the current operation and returns an error code (usually +** SQLITE_NOMEM) to the user. However, sometimes a fault is not necessarily +** fatal. For example, if a malloc fails while resizing a hash table, this +** is completely recoverable simply by not carrying out the resize. The +** hash table will continue to function normally. So a malloc failure +** during a hash table resize is a benign fault. +*/ + +#include "sqliteInt.h" + +#ifndef SQLITE_OMIT_BUILTIN_TEST + +/* +** Global variables. +*/ +typedef struct BenignMallocHooks BenignMallocHooks; +static SQLITE_WSD struct BenignMallocHooks { + void (*xBenignBegin)(void); + void (*xBenignEnd)(void); +} sqlite3Hooks = { 0, 0 }; + +/* The "wsdHooks" macro will resolve to the appropriate BenignMallocHooks +** structure. If writable static data is unsupported on the target, +** we have to locate the state vector at run-time. In the more common +** case where writable static data is supported, wsdHooks can refer directly +** to the "sqlite3Hooks" state vector declared above. +*/ +#ifdef SQLITE_OMIT_WSD +# define wsdHooksInit \ + BenignMallocHooks *x = &GLOBAL(BenignMallocHooks,sqlite3Hooks) +# define wsdHooks x[0] +#else +# define wsdHooksInit +# define wsdHooks sqlite3Hooks +#endif + + +/* +** Register hooks to call when sqlite3BeginBenignMalloc() and +** sqlite3EndBenignMalloc() are called, respectively. +*/ +void sqlite3BenignMallocHooks( + void (*xBenignBegin)(void), + void (*xBenignEnd)(void) +){ + wsdHooksInit; + wsdHooks.xBenignBegin = xBenignBegin; + wsdHooks.xBenignEnd = xBenignEnd; +} + +/* +** This (sqlite3EndBenignMalloc()) is called by SQLite code to indicate that +** subsequent malloc failures are benign. A call to sqlite3EndBenignMalloc() +** indicates that subsequent malloc failures are non-benign. +*/ +void sqlite3BeginBenignMalloc(void){ + wsdHooksInit; + if( wsdHooks.xBenignBegin ){ + wsdHooks.xBenignBegin(); + } +} +void sqlite3EndBenignMalloc(void){ + wsdHooksInit; + if( wsdHooks.xBenignEnd ){ + wsdHooks.xBenignEnd(); + } +} + +#endif /* #ifndef SQLITE_OMIT_BUILTIN_TEST */ diff --git a/fts2.c b/fts3.c similarity index 70% rename from fts2.c rename to fts3.c index 0d958cf..260b17c 100644 --- a/fts2.c +++ b/fts3.c @@ -16,11 +16,11 @@ /* ** The code in this file is only compiled if: ** -** * The FTS2 module is being built as an extension +** * The FTS3 module is being built as an extension ** (in which case SQLITE_CORE is not defined), or ** -** * The FTS2 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS2 is defined). +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ /* TODO(shess) Consider exporting this comment to an HTML file or the @@ -208,8 +208,8 @@ ** ** **** Segment merging **** -** To amortize update costs, segments are groups into levels and -** merged in matches. Each increase in level represents exponentially +** To amortize update costs, segments are grouped into levels and +** merged in batches. Each increase in level represents exponentially ** more documents. ** ** New documents (actually, document updates) are tokenized and @@ -250,7 +250,7 @@ ** even with many segments. ** ** TODO(shess) That said, it would be nice to have a better query-side -** argument for MERGE_COUNT of 16. Also, it's possible/likely that +** argument for MERGE_COUNT of 16. Also, it is possible/likely that ** optimizations to things like doclist merging will swing the sweet ** spot around. ** @@ -272,9 +272,9 @@ ** into a single segment. */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#if defined(SQLITE_ENABLE_FTS2) && !defined(SQLITE_CORE) +#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) # define SQLITE_CORE 1 #endif @@ -284,12 +284,14 @@ #include #include -#include "fts2.h" -#include "fts2_hash.h" -#include "fts2_tokenizer.h" -#include "sqlite3.h" -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 +#include "fts3.h" +#include "fts3_expr.h" +#include "fts3_hash.h" +#include "fts3_tokenizer.h" +#ifndef SQLITE_CORE +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif /* TODO(shess) MAN, this thing needs some refactoring. At minimum, it @@ -306,9 +308,9 @@ SQLITE_EXTENSION_INIT1 */ #if 0 -# define TRACE(A) printf A; fflush(stdout) +# define FTSTRACE(A) printf A; fflush(stdout) #else -# define TRACE(A) +# define FTSTRACE(A) #endif /* It is not safe to call isspace(), tolower(), or isalnum() on @@ -384,7 +386,7 @@ enum { /* Write a 64-bit variable-length integer to memory starting at p[0]. * The length of data written will be between 1 and VARINT_MAX bytes. * The number of bytes written is returned. */ -static int putVarint(char *p, sqlite_int64 v){ +static int fts3PutVarint(char *p, sqlite_int64 v){ unsigned char *q = (unsigned char *) p; sqlite_uint64 vu = v; do{ @@ -399,7 +401,7 @@ static int putVarint(char *p, sqlite_int64 v){ /* Read a 64-bit variable-length integer from memory starting at p[0]. * Return the number of bytes read, or 0 on error. * The value is stored in *v. */ -static int getVarint(const char *p, sqlite_int64 *v){ +static int fts3GetVarint(const char *p, sqlite_int64 *v){ const unsigned char *q = (const unsigned char *) p; sqlite_uint64 x = 0, y = 1; while( (*q & 0x80) == 0x80 ){ @@ -415,9 +417,9 @@ static int getVarint(const char *p, sqlite_int64 *v){ return (int) (q - (unsigned char *)p); } -static int getVarint32(const char *p, int *pi){ +static int fts3GetVarint32(const char *p, int *pi){ sqlite_int64 i; - int ret = getVarint(p, &i); + int ret = fts3GetVarint(p, &i); *pi = (int) i; assert( *pi==i ); return ret; @@ -431,6 +433,7 @@ static int getVarint32(const char *p, int *pi){ ** dataBufferInit - create a buffer with given initial capacity. ** dataBufferReset - forget buffer's data, retaining capacity. ** dataBufferDestroy - free buffer's data. +** dataBufferSwap - swap contents of two buffers. ** dataBufferExpand - expand capacity without adding data. ** dataBufferAppend - append data. ** dataBufferAppend2 - append two pieces of data at once. @@ -446,15 +449,20 @@ static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){ assert( nCapacity>=0 ); pBuffer->nData = 0; pBuffer->nCapacity = nCapacity; - pBuffer->pData = nCapacity==0 ? NULL : malloc(nCapacity); + pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity); } static void dataBufferReset(DataBuffer *pBuffer){ pBuffer->nData = 0; } static void dataBufferDestroy(DataBuffer *pBuffer){ - if( pBuffer->pData!=NULL ) free(pBuffer->pData); + if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData); SCRAMBLE(pBuffer); } +static void dataBufferSwap(DataBuffer *pBuffer1, DataBuffer *pBuffer2){ + DataBuffer tmp = *pBuffer1; + *pBuffer1 = *pBuffer2; + *pBuffer2 = tmp; +} static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){ assert( nAddCapacity>0 ); /* TODO(shess) Consider expanding more aggressively. Note that the @@ -463,7 +471,7 @@ static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){ */ if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){ pBuffer->nCapacity = pBuffer->nData+nAddCapacity; - pBuffer->pData = realloc(pBuffer->pData, pBuffer->nCapacity); + pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity); } } static void dataBufferAppend(DataBuffer *pBuffer, @@ -609,13 +617,13 @@ static int dlrAllDataBytes(DLReader *pReader){ */ static const char *dlrPosData(DLReader *pReader){ sqlite_int64 iDummy; - int n = getVarint(pReader->pData, &iDummy); + int n = fts3GetVarint(pReader->pData, &iDummy); assert( !dlrAtEnd(pReader) ); return pReader->pData+n; } static int dlrPosDataLen(DLReader *pReader){ sqlite_int64 iDummy; - int n = getVarint(pReader->pData, &iDummy); + int n = fts3GetVarint(pReader->pData, &iDummy); assert( !dlrAtEnd(pReader) ); return pReader->nElement-n; } @@ -630,20 +638,20 @@ static void dlrStep(DLReader *pReader){ /* If there is more data, read the next doclist element. */ if( pReader->nData!=0 ){ sqlite_int64 iDocidDelta; - int iDummy, n = getVarint(pReader->pData, &iDocidDelta); + int iDummy, n = fts3GetVarint(pReader->pData, &iDocidDelta); pReader->iDocid += iDocidDelta; if( pReader->iType>=DL_POSITIONS ){ assert( nnData ); while( 1 ){ - n += getVarint32(pReader->pData+n, &iDummy); + n += fts3GetVarint32(pReader->pData+n, &iDummy); assert( n<=pReader->nData ); if( iDummy==POS_END ) break; if( iDummy==POS_COLUMN ){ - n += getVarint32(pReader->pData+n, &iDummy); + n += fts3GetVarint32(pReader->pData+n, &iDummy); assert( nnData ); }else if( pReader->iType==DL_POSITIONS_OFFSETS ){ - n += getVarint32(pReader->pData+n, &iDummy); - n += getVarint32(pReader->pData+n, &iDummy); + n += fts3GetVarint32(pReader->pData+n, &iDummy); + n += fts3GetVarint32(pReader->pData+n, &iDummy); assert( nnData ); } } @@ -670,7 +678,7 @@ static void dlrDestroy(DLReader *pReader){ #ifndef NDEBUG /* Verify that the doclist can be validly decoded. Also returns the -** last docid found because it's convenient in other assertions for +** last docid found because it is convenient in other assertions for ** DLWriter. */ static void docListValidate(DocListType iType, const char *pData, int nData, @@ -681,18 +689,18 @@ static void docListValidate(DocListType iType, const char *pData, int nData, assert( pData+nData>pData ); while( nData!=0 ){ sqlite_int64 iDocidDelta; - int n = getVarint(pData, &iDocidDelta); + int n = fts3GetVarint(pData, &iDocidDelta); iPrevDocid += iDocidDelta; if( iType>DL_DOCIDS ){ int iDummy; while( 1 ){ - n += getVarint32(pData+n, &iDummy); + n += fts3GetVarint32(pData+n, &iDummy); if( iDummy==POS_END ) break; if( iDummy==POS_COLUMN ){ - n += getVarint32(pData+n, &iDummy); + n += fts3GetVarint32(pData+n, &iDummy); }else if( iType>DL_POSITIONS ){ - n += getVarint32(pData+n, &iDummy); - n += getVarint32(pData+n, &iDummy); + n += fts3GetVarint32(pData+n, &iDummy); + n += fts3GetVarint32(pData+n, &iDummy); } assert( n<=nData ); } @@ -763,9 +771,9 @@ static void dlwAppend(DLWriter *pWriter, #endif /* Recode the initial docid as delta from iPrevDocid. */ - nFirstOld = getVarint(pData, &iDocid); + nFirstOld = fts3GetVarint(pData, &iDocid); assert( nFirstOldiType==DL_DOCIDS) ); - nFirstNew = putVarint(c, iFirstDocid-pWriter->iPrevDocid); + nFirstNew = fts3PutVarint(c, iFirstDocid-pWriter->iPrevDocid); /* Verify that the incoming doclist is valid AND that it ends with ** the expected docid. This is essential because we'll trust this @@ -791,7 +799,7 @@ static void dlwCopy(DLWriter *pWriter, DLReader *pReader){ } static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){ char c[VARINT_MAX]; - int n = putVarint(c, iDocid-pWriter->iPrevDocid); + int n = fts3PutVarint(c, iDocid-pWriter->iPrevDocid); /* Docids must ascend. */ assert( !pWriter->has_iPrevDocid || iDocid>pWriter->iPrevDocid ); @@ -858,12 +866,12 @@ static void plrStep(PLReader *pReader){ return; } - n = getVarint32(pReader->pData, &i); + n = fts3GetVarint32(pReader->pData, &i); if( i==POS_COLUMN ){ - n += getVarint32(pReader->pData+n, &pReader->iColumn); + n += fts3GetVarint32(pReader->pData+n, &pReader->iColumn); pReader->iPosition = 0; pReader->iStartOffset = 0; - n += getVarint32(pReader->pData+n, &i); + n += fts3GetVarint32(pReader->pData+n, &i); } /* Should never see adjacent column changes. */ assert( i!=POS_COLUMN ); @@ -876,9 +884,9 @@ static void plrStep(PLReader *pReader){ pReader->iPosition += i-POS_BASE; if( pReader->iType==DL_POSITIONS_OFFSETS ){ - n += getVarint32(pReader->pData+n, &i); + n += fts3GetVarint32(pReader->pData+n, &i); pReader->iStartOffset += i; - n += getVarint32(pReader->pData+n, &i); + n += fts3GetVarint32(pReader->pData+n, &i); pReader->iEndOffset = pReader->iStartOffset+i; } assert( n<=pReader->nData ); @@ -950,21 +958,21 @@ static void plwAdd(PLWriter *pWriter, int iColumn, int iPos, if( pWriter->dlw->iType==DL_DOCIDS ) return; if( iColumn!=pWriter->iColumn ){ - n += putVarint(c+n, POS_COLUMN); - n += putVarint(c+n, iColumn); + n += fts3PutVarint(c+n, POS_COLUMN); + n += fts3PutVarint(c+n, iColumn); pWriter->iColumn = iColumn; pWriter->iPos = 0; pWriter->iOffset = 0; } assert( iPos>=pWriter->iPos ); - n += putVarint(c+n, POS_BASE+(iPos-pWriter->iPos)); + n += fts3PutVarint(c+n, POS_BASE+(iPos-pWriter->iPos)); pWriter->iPos = iPos; if( pWriter->dlw->iType==DL_POSITIONS_OFFSETS ){ assert( iStartOffset>=pWriter->iOffset ); - n += putVarint(c+n, iStartOffset-pWriter->iOffset); + n += fts3PutVarint(c+n, iStartOffset-pWriter->iOffset); pWriter->iOffset = iStartOffset; assert( iEndOffset>=iStartOffset ); - n += putVarint(c+n, iEndOffset-iStartOffset); + n += fts3PutVarint(c+n, iEndOffset-iStartOffset); } dataBufferAppend(pWriter->dlw->b, c, n); } @@ -980,7 +988,7 @@ static void plwInit(PLWriter *pWriter, DLWriter *dlw, sqlite_int64 iDocid){ /* Docids must ascend. */ assert( !pWriter->dlw->has_iPrevDocid || iDocid>pWriter->dlw->iPrevDocid ); - n = putVarint(c, iDocid-pWriter->dlw->iPrevDocid); + n = fts3PutVarint(c, iDocid-pWriter->dlw->iPrevDocid); dataBufferAppend(pWriter->dlw->b, c, n); pWriter->dlw->iPrevDocid = iDocid; #ifndef NDEBUG @@ -1002,7 +1010,7 @@ static void plwInit(PLWriter *pWriter, DLWriter *dlw, sqlite_int64 iDocid){ static void plwTerminate(PLWriter *pWriter){ if( pWriter->dlw->iType>DL_DOCIDS ){ char c[VARINT_MAX]; - int n = putVarint(c, POS_END); + int n = fts3PutVarint(c, POS_END); dataBufferAppend(pWriter->dlw->b, c, n); } #ifndef NDEBUG @@ -1033,7 +1041,7 @@ typedef struct DLCollector { /* TODO(shess) This could also be done by calling plwTerminate() and ** dataBufferAppend(). I tried that, expecting nominal performance ** differences, but it seemed to pretty reliably be worth 1% to code -** it this way. I suspect it's the incremental malloc overhead (some +** it this way. I suspect it is the incremental malloc overhead (some ** percentage of the plwTerminate() calls will cause a realloc), so ** this might be worth revisiting if the DataBuffer implementation ** changes. @@ -1041,7 +1049,7 @@ typedef struct DLCollector { static void dlcAddDoclist(DLCollector *pCollector, DataBuffer *b){ if( pCollector->dlw.iType>DL_DOCIDS ){ char c[VARINT_MAX]; - int n = putVarint(c, POS_END); + int n = fts3PutVarint(c, POS_END); dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n); }else{ dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData); @@ -1058,7 +1066,7 @@ static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos, } static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){ - DLCollector *pCollector = malloc(sizeof(DLCollector)); + DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector)); dataBufferInit(&pCollector->b, 0); dlwInit(&pCollector->dlw, iType, &pCollector->b); plwInit(&pCollector->plw, &pCollector->dlw, iDocid); @@ -1069,7 +1077,7 @@ static void dlcDelete(DLCollector *pCollector){ dlwDestroy(&pCollector->dlw); dataBufferDestroy(&pCollector->b); SCRAMBLE(pCollector); - free(pCollector); + sqlite3_free(pCollector); } @@ -1328,7 +1336,7 @@ static void docListUnion( DLWriter writer; if( nLeft==0 ){ - dataBufferAppend(pOut, pRight, nRight); + if( nRight!=0) dataBufferAppend(pOut, pRight, nRight); return; } if( nRight==0 ){ @@ -1365,20 +1373,34 @@ static void docListUnion( dlwDestroy(&writer); } -/* pLeft and pRight are DLReaders positioned to the same docid. +/* +** This function is used as part of the implementation of phrase and +** NEAR matching. ** -** If there are no instances in pLeft or pRight where the position -** of pLeft is one less than the position of pRight, then this -** routine adds nothing to pOut. +** pLeft and pRight are DLReaders positioned to the same docid in +** lists of type DL_POSITION. This function writes an entry to the +** DLWriter pOut for each position in pRight that is less than +** (nNear+1) greater (but not equal to or smaller) than a position +** in pLeft. For example, if nNear is 0, and the positions contained +** by pLeft and pRight are: ** -** If there are one or more instances where positions from pLeft -** are exactly one less than positions from pRight, then add a new -** document record to pOut. If pOut wants to hold positions, then -** include the positions from pRight that are one more than a -** position in pLeft. In other words: pRight.iPos==pLeft.iPos+1. +** pLeft: 5 10 15 20 +** pRight: 6 9 17 21 +** +** then the docid is added to pOut. If pOut is of type DL_POSITIONS, +** then a positionids "6" and "21" are also added to pOut. +** +** If boolean argument isSaveLeft is true, then positionids are copied +** from pLeft instead of pRight. In the example above, the positions "5" +** and "20" would be added instead of "6" and "21". */ -static void posListPhraseMerge(DLReader *pLeft, DLReader *pRight, - DLWriter *pOut){ +static void posListPhraseMerge( + DLReader *pLeft, + DLReader *pRight, + int nNear, + int isSaveLeft, + DLWriter *pOut +){ PLReader left, right; PLWriter writer; int match = 0; @@ -1394,18 +1416,23 @@ static void posListPhraseMerge(DLReader *pLeft, DLReader *pRight, plrStep(&left); }else if( plrColumn(&left)>plrColumn(&right) ){ plrStep(&right); - }else if( plrPosition(&left)+1plrPosition(&right) ){ + }else if( plrPosition(&left)>=plrPosition(&right) ){ plrStep(&right); }else{ - if( !match ){ - plwInit(&writer, pOut, dlrDocid(pLeft)); - match = 1; + if( (plrPosition(&right)-plrPosition(&left))<=(nNear+1) ){ + if( !match ){ + plwInit(&writer, pOut, dlrDocid(pLeft)); + match = 1; + } + if( !isSaveLeft ){ + plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0); + }else{ + plwAdd(&writer, plrColumn(&left), plrPosition(&left), 0, 0); + } + plrStep(&right); + }else{ + plrStep(&left); } - plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0); - plrStep(&left); - plrStep(&right); } } @@ -1418,19 +1445,63 @@ static void posListPhraseMerge(DLReader *pLeft, DLReader *pRight, plrDestroy(&right); } -/* We have two doclists with positions: pLeft and pRight. -** Write the phrase intersection of these two doclists into pOut. +/* +** Compare the values pointed to by the PLReaders passed as arguments. +** Return -1 if the value pointed to by pLeft is considered less than +** the value pointed to by pRight, +1 if it is considered greater +** than it, or 0 if it is equal. i.e. +** +** (*pLeft - *pRight) +** +** A PLReader that is in the EOF condition is considered greater than +** any other. If neither argument is in EOF state, the return value of +** plrColumn() is used. If the plrColumn() values are equal, the +** comparison is on the basis of plrPosition(). +*/ +static int plrCompare(PLReader *pLeft, PLReader *pRight){ + assert(!plrAtEnd(pLeft) || !plrAtEnd(pRight)); + + if( plrAtEnd(pRight) || plrAtEnd(pLeft) ){ + return (plrAtEnd(pRight) ? -1 : 1); + } + if( plrColumn(pLeft)!=plrColumn(pRight) ){ + return ((plrColumn(pLeft)0) +** and write the results into pOut. ** ** A phrase intersection means that two documents only match ** if pLeft.iPos+1==pRight.iPos. ** +** A NEAR intersection means that two documents only match if +** (abs(pLeft.iPos-pRight.iPos) one AND (two OR three) - * [one OR two three] ==> (one OR two) AND three - * - * A "-" before a term matches all entries that lack that term. - * The "-" must occur immediately before the term with in intervening - * space. This is how the search engines do it. - * - * A NOT term cannot be the right-hand operand of an OR. If this - * occurs in the query string, the NOT is ignored: - * - * [one OR -two] ==> one OR two - * - */ -typedef struct Query { - fulltext_vtab *pFts; /* The full text index */ - int nTerms; /* Number of terms in the query */ - QueryTerm *pTerms; /* Array of terms. Space obtained from malloc() */ - int nextIsOr; /* Set the isOr flag on the next inserted term */ - int nextColumn; /* Next word parsed must be in this column */ - int dfltColumn; /* The default column */ -} Query; - - /* ** An instance of the following structure keeps track of generated ** matching-word offset information and snippets. @@ -1723,6 +1795,7 @@ typedef struct Snippet { char snStatus; /* Status flag for use while constructing snippets */ short int iCol; /* The column that contains the match */ short int iTerm; /* The index in Query.pTerms[] of the matching term */ + int iToken; /* The index of the matching document token */ short int nByte; /* Number of bytes in the term */ int iStart; /* The offset to the first character of the term */ } *aMatch; /* Points to space obtained from malloc */ @@ -1735,7 +1808,7 @@ typedef struct Snippet { typedef enum QueryType { QUERY_GENERIC, /* table scan */ - QUERY_ROWID, /* lookup by rowid */ + QUERY_DOCID, /* lookup by docid */ QUERY_FULLTEXT /* QUERY_FULLTEXT + [i] is a full-text search for column i*/ } QueryType; @@ -1744,17 +1817,22 @@ typedef enum fulltext_statement { CONTENT_SELECT_STMT, CONTENT_UPDATE_STMT, CONTENT_DELETE_STMT, + CONTENT_EXISTS_STMT, BLOCK_INSERT_STMT, BLOCK_SELECT_STMT, BLOCK_DELETE_STMT, + BLOCK_DELETE_ALL_STMT, SEGDIR_MAX_INDEX_STMT, SEGDIR_SET_STMT, - SEGDIR_SELECT_STMT, + SEGDIR_SELECT_LEVEL_STMT, SEGDIR_SPAN_STMT, SEGDIR_DELETE_STMT, + SEGDIR_SELECT_SEGMENT_STMT, SEGDIR_SELECT_ALL_STMT, + SEGDIR_DELETE_ALL_STMT, + SEGDIR_COUNT_STMT, MAX_STMT /* Always at end! */ } fulltext_statement; @@ -1766,25 +1844,38 @@ typedef enum fulltext_statement { */ static const char *const fulltext_zStatement[MAX_STMT] = { /* CONTENT_INSERT */ NULL, /* generated in contentInsertStatement() */ - /* CONTENT_SELECT */ "select * from %_content where rowid = ?", + /* CONTENT_SELECT */ NULL, /* generated in contentSelectStatement() */ /* CONTENT_UPDATE */ NULL, /* generated in contentUpdateStatement() */ - /* CONTENT_DELETE */ "delete from %_content where rowid = ?", + /* CONTENT_DELETE */ "delete from %_content where docid = ?", + /* CONTENT_EXISTS */ "select docid from %_content limit 1", - /* BLOCK_INSERT */ "insert into %_segments values (?)", - /* BLOCK_SELECT */ "select block from %_segments where rowid = ?", - /* BLOCK_DELETE */ "delete from %_segments where rowid between ? and ?", + /* BLOCK_INSERT */ + "insert into %_segments (blockid, block) values (null, ?)", + /* BLOCK_SELECT */ "select block from %_segments where blockid = ?", + /* BLOCK_DELETE */ "delete from %_segments where blockid between ? and ?", + /* BLOCK_DELETE_ALL */ "delete from %_segments", /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?", /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)", - /* SEGDIR_SELECT */ + /* SEGDIR_SELECT_LEVEL */ "select start_block, leaves_end_block, root from %_segdir " " where level = ? order by idx", /* SEGDIR_SPAN */ "select min(start_block), max(end_block) from %_segdir " " where level = ? and start_block <> 0", /* SEGDIR_DELETE */ "delete from %_segdir where level = ?", + + /* NOTE(shess): The first three results of the following two + ** statements must match. + */ + /* SEGDIR_SELECT_SEGMENT */ + "select start_block, leaves_end_block, root from %_segdir " + " where level = ? and idx = ?", /* SEGDIR_SELECT_ALL */ - "select root, leaves_end_block from %_segdir order by level desc, idx", + "select start_block, leaves_end_block, root from %_segdir " + " order by level desc, idx asc", + /* SEGDIR_DELETE_ALL */ "delete from %_segdir", + /* SEGDIR_COUNT */ "select count(*), ifnull(max(level),0) from %_segdir", }; /* @@ -1815,7 +1906,7 @@ struct fulltext_vtab { sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT]; /* The statement used to prepare pLeafSelectStmts. */ #define LEAF_SELECT \ - "select block from %_segments where rowid between ? and ? order by rowid" + "select block from %_segments where blockid between ? and ? order by blockid" /* These buffer pending index updates during transactions. ** nPendingData estimates the memory size of the pending data. It @@ -1830,7 +1921,7 @@ struct fulltext_vtab { int nPendingData; #define kPendingThreshold (1*1024*1024) sqlite_int64 iPrevDocid; - fts2Hash pendingTerms; + fts3Hash pendingTerms; }; /* @@ -1843,28 +1934,28 @@ typedef struct fulltext_cursor { QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ int eof; /* True if at End Of Results */ - Query q; /* Parsed query string */ + Fts3Expr *pExpr; /* Parsed MATCH query string */ Snippet snippet; /* Cached snippet for the current row */ int iColumn; /* Column being searched */ DataBuffer result; /* Doclist results from fulltextQuery */ DLReader reader; /* Result reader if result not empty */ } fulltext_cursor; -static struct fulltext_vtab *cursor_vtab(fulltext_cursor *c){ +static fulltext_vtab *cursor_vtab(fulltext_cursor *c){ return (fulltext_vtab *) c->base.pVtab; } -static const sqlite3_module fts2Module; /* forward declaration */ +static const sqlite3_module fts3Module; /* forward declaration */ /* Return a dynamically generated statement of the form - * insert into %_content (rowid, ...) values (?, ...) + * insert into %_content (docid, ...) values (?, ...) */ static const char *contentInsertStatement(fulltext_vtab *v){ StringBuffer sb; int i; initStringBuffer(&sb); - append(&sb, "insert into %_content (rowid, "); + append(&sb, "insert into %_content (docid, "); appendList(&sb, v->nColumn, v->azContentColumn); append(&sb, ") values (?"); for(i=0; inColumn; ++i) @@ -1873,9 +1964,21 @@ static const char *contentInsertStatement(fulltext_vtab *v){ return stringBufferData(&sb); } +/* Return a dynamically generated statement of the form + * select from %_content where docid = ? + */ +static const char *contentSelectStatement(fulltext_vtab *v){ + StringBuffer sb; + initStringBuffer(&sb); + append(&sb, "SELECT "); + appendList(&sb, v->nColumn, v->azContentColumn); + append(&sb, " FROM %_content WHERE docid = ?"); + return stringBufferData(&sb); +} + /* Return a dynamically generated statement of the form * update %_content set [col_0] = ?, [col_1] = ?, ... - * where rowid = ? + * where docid = ? */ static const char *contentUpdateStatement(fulltext_vtab *v){ StringBuffer sb; @@ -1890,7 +1993,7 @@ static const char *contentUpdateStatement(fulltext_vtab *v){ append(&sb, v->azContentColumn[i]); append(&sb, " = ?"); } - append(&sb, " where rowid = ?"); + append(&sb, " where docid = ?"); return stringBufferData(&sb); } @@ -1907,6 +2010,8 @@ static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt, switch( iStmt ){ case CONTENT_INSERT_STMT: zStmt = contentInsertStatement(v); break; + case CONTENT_SELECT_STMT: + zStmt = contentSelectStatement(v); break; case CONTENT_UPDATE_STMT: zStmt = contentUpdateStatement(v); break; default: @@ -1914,7 +2019,7 @@ static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt, } rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt], zStmt); - if( zStmt != fulltext_zStatement[iStmt]) free((void *) zStmt); + if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt); if( rc!=SQLITE_OK ) return rc; } else { int rc = sqlite3_reset(v->pFulltextStatements[iStmt]); @@ -1935,15 +2040,18 @@ static int sql_single_step(sqlite3_stmt *s){ } /* Like sql_get_statement(), but for special replicated LEAF_SELECT -** statements. +** statements. idx -1 is a special case for an uncached version of +** the statement (used in the optimize implementation). */ /* TODO(shess) Write version for generic statements and then share ** that between the cached-statement functions. */ static int sql_get_leaf_statement(fulltext_vtab *v, int idx, sqlite3_stmt **ppStmt){ - assert( idx>=0 && idxpLeafSelectStmts[idx]==NULL ){ + assert( idx>=-1 && idxdb, v->zDb, v->zName, ppStmt, LEAF_SELECT); + }else if( v->pLeafSelectStmts[idx]==NULL ){ int rc = sql_prepare(v->db, v->zDb, v->zName, &v->pLeafSelectStmts[idx], LEAF_SELECT); if( rc!=SQLITE_OK ) return rc; @@ -1956,15 +2064,18 @@ static int sql_get_leaf_statement(fulltext_vtab *v, int idx, return SQLITE_OK; } -/* insert into %_content (rowid, ...) values ([rowid], [pValues]) */ -static int content_insert(fulltext_vtab *v, sqlite3_value *rowid, +/* insert into %_content (docid, ...) values ([docid], [pValues]) +** If the docid contains SQL NULL, then a unique docid will be +** generated. +*/ +static int content_insert(fulltext_vtab *v, sqlite3_value *docid, sqlite3_value **pValues){ sqlite3_stmt *s; int i; int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s); if( rc!=SQLITE_OK ) return rc; - rc = sqlite3_bind_value(s, 1, rowid); + rc = sqlite3_bind_value(s, 1, docid); if( rc!=SQLITE_OK ) return rc; for(i=0; inColumn; ++i){ @@ -1976,9 +2087,9 @@ static int content_insert(fulltext_vtab *v, sqlite3_value *rowid, } /* update %_content set col0 = pValues[0], col1 = pValues[1], ... - * where rowid = [iRowid] */ + * where docid = [iDocid] */ static int content_update(fulltext_vtab *v, sqlite3_value **pValues, - sqlite_int64 iRowid){ + sqlite_int64 iDocid){ sqlite3_stmt *s; int i; int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s); @@ -1989,7 +2100,7 @@ static int content_update(fulltext_vtab *v, sqlite3_value **pValues, if( rc!=SQLITE_OK ) return rc; } - rc = sqlite3_bind_int64(s, 1+v->nColumn, iRowid); + rc = sqlite3_bind_int64(s, 1+v->nColumn, iDocid); if( rc!=SQLITE_OK ) return rc; return sql_single_step(s); @@ -1999,18 +2110,18 @@ static void freeStringArray(int nString, const char **pString){ int i; for (i=0 ; i < nString ; ++i) { - if( pString[i]!=NULL ) free((void *) pString[i]); + if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]); } - free((void *) pString); + sqlite3_free((void *) pString); } -/* select * from %_content where rowid = [iRow] +/* select * from %_content where docid = [iDocid] * The caller must delete the returned array and all strings in it. * null fields will be NULL in the returned array. * * TODO: Perhaps we should return pointer/length strings here for consistency * with other code which uses pointer/length. */ -static int content_select(fulltext_vtab *v, sqlite_int64 iRow, +static int content_select(fulltext_vtab *v, sqlite_int64 iDocid, const char ***pValues){ sqlite3_stmt *s; const char **values; @@ -2022,13 +2133,13 @@ static int content_select(fulltext_vtab *v, sqlite_int64 iRow, rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s); if( rc!=SQLITE_OK ) return rc; - rc = sqlite3_bind_int64(s, 1, iRow); + rc = sqlite3_bind_int64(s, 1, iDocid); if( rc!=SQLITE_OK ) return rc; rc = sqlite3_step(s); if( rc!=SQLITE_ROW ) return rc; - values = (const char **) malloc(v->nColumn * sizeof(const char *)); + values = (const char **) sqlite3_malloc(v->nColumn * sizeof(const char *)); for(i=0; inColumn; ++i){ if( sqlite3_column_type(s, i)==SQLITE_NULL ){ values[i] = NULL; @@ -2049,20 +2160,39 @@ static int content_select(fulltext_vtab *v, sqlite_int64 iRow, return rc; } -/* delete from %_content where rowid = [iRow ] */ -static int content_delete(fulltext_vtab *v, sqlite_int64 iRow){ +/* delete from %_content where docid = [iDocid ] */ +static int content_delete(fulltext_vtab *v, sqlite_int64 iDocid){ sqlite3_stmt *s; int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s); if( rc!=SQLITE_OK ) return rc; - rc = sqlite3_bind_int64(s, 1, iRow); + rc = sqlite3_bind_int64(s, 1, iDocid); if( rc!=SQLITE_OK ) return rc; return sql_single_step(s); } +/* Returns SQLITE_ROW if any rows exist in %_content, SQLITE_DONE if +** no rows exist, and any error in case of failure. +*/ +static int content_exists(fulltext_vtab *v){ + sqlite3_stmt *s; + int rc = sql_get_statement(v, CONTENT_EXISTS_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_step(s); + if( rc!=SQLITE_ROW ) return rc; + + /* We expect only one row. We must execute another sqlite3_step() + * to complete the iteration; otherwise the table will remain locked. */ + rc = sqlite3_step(s); + if( rc==SQLITE_DONE ) return SQLITE_ROW; + if( rc==SQLITE_ROW ) return SQLITE_ERROR; + return rc; +} + /* insert into %_segments values ([pData]) -** returns assigned rowid in *piBlockid +** returns assigned blockid in *piBlockid */ static int block_insert(fulltext_vtab *v, const char *pData, int nData, sqlite_int64 *piBlockid){ @@ -2077,12 +2207,13 @@ static int block_insert(fulltext_vtab *v, const char *pData, int nData, if( rc==SQLITE_ROW ) return SQLITE_ERROR; if( rc!=SQLITE_DONE ) return rc; + /* blockid column is an alias for rowid. */ *piBlockid = sqlite3_last_insert_rowid(v->db); return SQLITE_OK; } /* delete from %_segments -** where rowid between [iStartBlockid] and [iEndBlockid] +** where blockid between [iStartBlockid] and [iEndBlockid] ** ** Deletes the range of blocks, inclusive, used to delete the blocks ** which form a segment. @@ -2234,6 +2365,54 @@ static int segdir_delete(fulltext_vtab *v, int iLevel){ return sql_single_step(s); } +/* Delete entire fts index, SQLITE_OK on success, relevant error on +** failure. +*/ +static int segdir_delete_all(fulltext_vtab *v){ + sqlite3_stmt *s; + int rc = sql_get_statement(v, SEGDIR_DELETE_ALL_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + rc = sql_single_step(s); + if( rc!=SQLITE_OK ) return rc; + + rc = sql_get_statement(v, BLOCK_DELETE_ALL_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + return sql_single_step(s); +} + +/* Returns SQLITE_OK with *pnSegments set to the number of entries in +** %_segdir and *piMaxLevel set to the highest level which has a +** segment. Otherwise returns the SQLite error which caused failure. +*/ +static int segdir_count(fulltext_vtab *v, int *pnSegments, int *piMaxLevel){ + sqlite3_stmt *s; + int rc = sql_get_statement(v, SEGDIR_COUNT_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_step(s); + /* TODO(shess): This case should not be possible? Should stronger + ** measures be taken if it happens? + */ + if( rc==SQLITE_DONE ){ + *pnSegments = 0; + *piMaxLevel = 0; + return SQLITE_OK; + } + if( rc!=SQLITE_ROW ) return rc; + + *pnSegments = sqlite3_column_int(s, 0); + *piMaxLevel = sqlite3_column_int(s, 1); + + /* We expect only one row. We must execute another sqlite3_step() + * to complete the iteration; otherwise the table will remain locked. */ + rc = sqlite3_step(s); + if( rc==SQLITE_DONE ) return SQLITE_OK; + if( rc==SQLITE_ROW ) return SQLITE_ERROR; + return rc; +} + /* TODO(shess) clearPendingTerms() is far down the file because ** writeZeroSegment() is far down the file because LeafWriter is far ** down the file. Consider refactoring the code to move the non-vtab @@ -2248,7 +2427,7 @@ static int clearPendingTerms(fulltext_vtab *v); static void fulltext_vtab_destroy(fulltext_vtab *v){ int iStmt, i; - TRACE(("FTS2 Destroy %p\n", v)); + FTSTRACE(("FTS3 Destroy %p\n", v)); for( iStmt=0; iStmtpFulltextStatements[iStmt]!=NULL ){ sqlite3_finalize(v->pFulltextStatements[iStmt]); @@ -2270,12 +2449,12 @@ static void fulltext_vtab_destroy(fulltext_vtab *v){ clearPendingTerms(v); - free(v->azColumn); + sqlite3_free(v->azColumn); for(i = 0; i < v->nColumn; ++i) { sqlite3_free(v->azContentColumn[i]); } - free(v->azContentColumn); - free(v); + sqlite3_free(v->azContentColumn); + sqlite3_free(v); } /* @@ -2289,18 +2468,18 @@ static void fulltext_vtab_destroy(fulltext_vtab *v){ /* ** If X is a character that can be used in an identifier then -** IdChar(X) will be true. Otherwise it is false. +** ftsIdChar(X) will be true. Otherwise it is false. ** ** For ASCII, any character with the high-order bit set is ** allowed in an identifier. For 7-bit characters, -** sqlite3IsIdChar[X] must be 1. +** isFtsIdChar[X] must be 1. ** ** Ticket #1066. the SQL standard does not allow '$' in the ** middle of identfiers. But many SQL implementations do. ** SQLite will allow '$' in identifiers for compatibility. ** But the feature is undocumented. */ -static const char isIdChar[] = { +static const char isFtsIdChar[] = { /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ @@ -2309,14 +2488,14 @@ static const char isIdChar[] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ }; -#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && isIdChar[c-0x20])) +#define ftsIdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && isFtsIdChar[c-0x20])) /* ** Return the length of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ -static int getToken(const char *z, int *tokenType){ +static int ftsGetToken(const char *z, int *tokenType){ int i, c; switch( *z ){ case 0: { @@ -2350,10 +2529,10 @@ static int getToken(const char *z, int *tokenType){ return i; } default: { - if( !IdChar(*z) ){ + if( !ftsIdChar(*z) ){ break; } - for(i=1; IdChar(z[i]); i++){} + for(i=1; ftsIdChar(z[i]); i++){} *tokenType = TOKEN_ID; return i; } @@ -2366,10 +2545,10 @@ static int getToken(const char *z, int *tokenType){ ** A token extracted from a string is an instance of the following ** structure. */ -typedef struct Token { +typedef struct FtsToken { const char *z; /* Pointer to token text. Not '\000' terminated */ short int n; /* Length of the token text in bytes. */ -} Token; +} FtsToken; /* ** Given a input string (which is really one of the argv[] parameters @@ -2386,14 +2565,14 @@ typedef struct Token { */ static char **tokenizeString(const char *z, int *pnToken){ int nToken = 0; - Token *aToken = malloc( strlen(z) * sizeof(aToken[0]) ); + FtsToken *aToken = sqlite3_malloc( strlen(z) * sizeof(aToken[0]) ); int n = 1; int e, i; int totalSize = 0; char **azToken; char *zCopy; while( n>0 ){ - n = getToken(z, &e); + n = ftsGetToken(z, &e); if( e!=TOKEN_SPACE ){ aToken[nToken].z = z; aToken[nToken].n = n; @@ -2402,7 +2581,7 @@ static char **tokenizeString(const char *z, int *pnToken){ } z += n; } - azToken = (char**)malloc( nToken*sizeof(char*) + totalSize ); + azToken = (char**)sqlite3_malloc( nToken*sizeof(char*) + totalSize ); zCopy = (char*)&azToken[nToken]; nToken--; for(i=0; iazColumn); - free(p->azContentColumn); - free(p->azTokenizer); + sqlite3_free(p->azColumn); + sqlite3_free(p->azContentColumn); + sqlite3_free(p->azTokenizer); } /* Parse a CREATE VIRTUAL TABLE statement, which looks like this: * * CREATE VIRTUAL TABLE email - * USING fts2(subject, body, tokenize mytokenizer(myarg)) + * USING fts3(subject, body, tokenize mytokenizer(myarg)) * * We return parsed information in a TableSpec structure. * @@ -2588,7 +2767,7 @@ static int parseSpec(TableSpec *pSpec, int argc, const char *const*argv, for(i=n=0; iazContentColumn = malloc( pSpec->nColumn * sizeof(char *) ); + pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) ); if( pSpec->azContentColumn==0 ){ clearTableSpec(pSpec); return SQLITE_NOMEM; @@ -2677,7 +2856,10 @@ static char *fulltextSchema( zSchema = zNext; zSep = ","; } - zNext = sqlite3_mprintf("%s,%Q)", zSchema, zTableName); + zNext = sqlite3_mprintf("%s,%Q HIDDEN", zSchema, zTableName); + sqlite3_free(zSchema); + zSchema = zNext; + zNext = sqlite3_mprintf("%s,docid HIDDEN)", zSchema); sqlite3_free(zSchema); return zNext; } @@ -2688,7 +2870,7 @@ static char *fulltextSchema( */ static int constructVtab( sqlite3 *db, /* The SQLite database connection */ - fts2Hash *pHash, /* Hash table containing tokenizers */ + fts3Hash *pHash, /* Hash table containing tokenizers */ TableSpec *spec, /* Parsed spec information from parseSpec() */ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ char **pzErr /* Write any error message here */ @@ -2702,7 +2884,7 @@ static int constructVtab( char const *zTok; /* Name of tokenizer to use for this fts table */ int nTok; /* Length of zTok, including nul terminator */ - v = (fulltext_vtab *) malloc(sizeof(fulltext_vtab)); + v = (fulltext_vtab *) sqlite3_malloc(sizeof(fulltext_vtab)); if( v==0 ) return SQLITE_NOMEM; CLEAR(v); /* sqlite will initialize v->base */ @@ -2725,7 +2907,7 @@ static int constructVtab( } nTok = strlen(zTok)+1; - m = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zTok, nTok); + m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zTok, nTok); if( !m ){ *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]); rc = SQLITE_ERROR; @@ -2756,7 +2938,7 @@ static int constructVtab( v->nPendingData = -1; *ppVTab = &v->base; - TRACE(("FTS2 Connect %p\n", v)); + FTSTRACE(("FTS3 Connect %p\n", v)); return rc; @@ -2776,13 +2958,13 @@ static int fulltextConnect( int rc = parseSpec(&spec, argc, argv, pzErr); if( rc!=SQLITE_OK ) return rc; - rc = constructVtab(db, (fts2Hash *)pAux, &spec, ppVTab, pzErr); + rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr); clearTableSpec(&spec); return rc; } /* The %_content table holds the text of each document, with -** the rowid used as the docid. +** the docid column exposed as the SQLite rowid for the table. */ /* TODO(shess) This comment needs elaboration to match the updated ** code. Work it into the top-of-file comment at that time. @@ -2793,13 +2975,14 @@ static int fulltextCreate(sqlite3 *db, void *pAux, int rc; TableSpec spec; StringBuffer schema; - TRACE(("FTS2 Create\n")); + FTSTRACE(("FTS3 Create\n")); rc = parseSpec(&spec, argc, argv, pzErr); if( rc!=SQLITE_OK ) return rc; initStringBuffer(&schema); append(&schema, "CREATE TABLE %_content("); + append(&schema, " docid INTEGER PRIMARY KEY,"); appendList(&schema, spec.nColumn, spec.azContentColumn); append(&schema, ")"); rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema)); @@ -2807,7 +2990,11 @@ static int fulltextCreate(sqlite3 *db, void *pAux, if( rc!=SQLITE_OK ) goto out; rc = sql_exec(db, spec.zDb, spec.zName, - "create table %_segments(block blob);"); + "create table %_segments(" + " blockid INTEGER PRIMARY KEY," + " block blob" + ");" + ); if( rc!=SQLITE_OK ) goto out; rc = sql_exec(db, spec.zDb, spec.zName, @@ -2822,7 +3009,7 @@ static int fulltextCreate(sqlite3 *db, void *pAux, ");"); if( rc!=SQLITE_OK ) goto out; - rc = constructVtab(db, (fts2Hash *)pAux, &spec, ppVTab, pzErr); + rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr); out: clearTableSpec(&spec); @@ -2831,29 +3018,30 @@ out: /* Decide how to handle an SQL query. */ static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + fulltext_vtab *v = (fulltext_vtab *)pVTab; int i; - TRACE(("FTS2 BestIndex\n")); + FTSTRACE(("FTS3 BestIndex\n")); for(i=0; inConstraint; ++i){ const struct sqlite3_index_constraint *pConstraint; pConstraint = &pInfo->aConstraint[i]; if( pConstraint->usable ) { - if( pConstraint->iColumn==-1 && + if( (pConstraint->iColumn==-1 || pConstraint->iColumn==v->nColumn+1) && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - pInfo->idxNum = QUERY_ROWID; /* lookup by rowid */ - TRACE(("FTS2 QUERY_ROWID\n")); - } else if( pConstraint->iColumn>=0 && + pInfo->idxNum = QUERY_DOCID; /* lookup by docid */ + FTSTRACE(("FTS3 QUERY_DOCID\n")); + } else if( pConstraint->iColumn>=0 && pConstraint->iColumn<=v->nColumn && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ /* full-text search */ pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn; - TRACE(("FTS2 QUERY_FULLTEXT %d\n", pConstraint->iColumn)); + FTSTRACE(("FTS3 QUERY_FULLTEXT %d\n", pConstraint->iColumn)); } else continue; pInfo->aConstraintUsage[i].argvIndex = 1; pInfo->aConstraintUsage[i].omit = 1; /* An arbitrary value for now. - * TODO: Perhaps rowid matches should be considered cheaper than + * TODO: Perhaps docid matches should be considered cheaper than * full-text searches. */ pInfo->estimatedCost = 1.0; @@ -2865,7 +3053,7 @@ static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ } static int fulltextDisconnect(sqlite3_vtab *pVTab){ - TRACE(("FTS2 Disconnect %p\n", pVTab)); + FTSTRACE(("FTS3 Disconnect %p\n", pVTab)); fulltext_vtab_destroy((fulltext_vtab *)pVTab); return SQLITE_OK; } @@ -2874,7 +3062,7 @@ static int fulltextDestroy(sqlite3_vtab *pVTab){ fulltext_vtab *v = (fulltext_vtab *)pVTab; int rc; - TRACE(("FTS2 Destroy %p\n", pVTab)); + FTSTRACE(("FTS3 Destroy %p\n", pVTab)); rc = sql_exec(v->db, v->zDb, v->zName, "drop table if exists %_content;" "drop table if exists %_segments;" @@ -2889,48 +3077,42 @@ static int fulltextDestroy(sqlite3_vtab *pVTab){ static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ fulltext_cursor *c; - c = (fulltext_cursor *) calloc(sizeof(fulltext_cursor), 1); - /* sqlite will initialize c->base */ - *ppCursor = &c->base; - TRACE(("FTS2 Open %p: %p\n", pVTab, c)); - - return SQLITE_OK; -} - - -/* Free all of the dynamically allocated memory held by *q -*/ -static void queryClear(Query *q){ - int i; - for(i = 0; i < q->nTerms; ++i){ - free(q->pTerms[i].pTerm); + c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor)); + if( c ){ + memset(c, 0, sizeof(fulltext_cursor)); + /* sqlite will initialize c->base */ + *ppCursor = &c->base; + FTSTRACE(("FTS3 Open %p: %p\n", pVTab, c)); + return SQLITE_OK; + }else{ + return SQLITE_NOMEM; } - free(q->pTerms); - CLEAR(q); } /* Free all of the dynamically allocated memory held by the ** Snippet */ static void snippetClear(Snippet *p){ - free(p->aMatch); - free(p->zOffset); - free(p->zSnippet); + sqlite3_free(p->aMatch); + sqlite3_free(p->zOffset); + sqlite3_free(p->zSnippet); CLEAR(p); } + /* ** Append a single entry to the p->aMatch[] log. */ static void snippetAppendMatch( Snippet *p, /* Append the entry to this snippet */ int iCol, int iTerm, /* The column and query term */ + int iToken, /* Matching token in document */ int iStart, int nByte /* Offset and size of the match */ ){ int i; struct snippetMatch *pMatch; if( p->nMatch+1>=p->nAlloc ){ p->nAlloc = p->nAlloc*2 + 10; - p->aMatch = realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) ); + p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) ); if( p->aMatch==0 ){ p->nMatch = 0; p->nAlloc = 0; @@ -2941,6 +3123,7 @@ static void snippetAppendMatch( pMatch = &p->aMatch[i]; pMatch->iCol = iCol; pMatch->iTerm = iTerm; + pMatch->iToken = iToken; pMatch->iStart = iStart; pMatch->nByte = nByte; } @@ -2948,27 +3131,86 @@ static void snippetAppendMatch( /* ** Sizing information for the circular buffer used in snippetOffsetsOfColumn() */ -#define FTS2_ROTOR_SZ (32) -#define FTS2_ROTOR_MASK (FTS2_ROTOR_SZ-1) +#define FTS3_ROTOR_SZ (32) +#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1) + +/* +** Function to iterate through the tokens of a compiled expression. +** +** Except, skip all tokens on the right-hand side of a NOT operator. +** This function is used to find tokens as part of snippet and offset +** generation and we do nt want snippets and offsets to report matches +** for tokens on the RHS of a NOT. +*/ +static int fts3NextExprToken(Fts3Expr **ppExpr, int *piToken){ + Fts3Expr *p = *ppExpr; + int iToken = *piToken; + if( iToken<0 ){ + /* In this case the expression p is the root of an expression tree. + ** Move to the first token in the expression tree. + */ + while( p->pLeft ){ + p = p->pLeft; + } + iToken = 0; + }else{ + assert(p && p->eType==FTSQUERY_PHRASE ); + if( iToken<(p->pPhrase->nToken-1) ){ + iToken++; + }else{ + iToken = 0; + while( p->pParent && p->pParent->pLeft!=p ){ + assert( p->pParent->pRight==p ); + p = p->pParent; + } + p = p->pParent; + if( p ){ + assert( p->pRight!=0 ); + p = p->pRight; + while( p->pLeft ){ + p = p->pLeft; + } + } + } + } + + *ppExpr = p; + *piToken = iToken; + return p?1:0; +} + +/* +** Return TRUE if the expression node pExpr is located beneath the +** RHS of a NOT operator. +*/ +static int fts3ExprBeneathNot(Fts3Expr *p){ + Fts3Expr *pParent; + while( p ){ + pParent = p->pParent; + if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){ + return 1; + } + p = pParent; + } + return 0; +} /* ** Add entries to pSnippet->aMatch[] for every match that occurs against ** document zDoc[0..nDoc-1] which is stored in column iColumn. */ static void snippetOffsetsOfColumn( - Query *pQuery, - Snippet *pSnippet, - int iColumn, - const char *zDoc, - int nDoc + fulltext_cursor *pCur, /* The fulltest search cursor */ + Snippet *pSnippet, /* The Snippet object to be filled in */ + int iColumn, /* Index of fulltext table column */ + const char *zDoc, /* Text of the fulltext table column */ + int nDoc /* Length of zDoc in bytes */ ){ const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */ sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */ sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */ fulltext_vtab *pVtab; /* The full text index */ int nColumn; /* Number of columns in the index */ - const QueryTerm *aTerm; /* Query string terms */ - int nTerm; /* Number of query string terms */ int i, j; /* Loop counters */ int rc; /* Return code */ unsigned int match, prevMatch; /* Phrase search bitmasks */ @@ -2979,42 +3221,44 @@ static void snippetOffsetsOfColumn( /* The following variables keep a circular buffer of the last ** few tokens */ unsigned int iRotor = 0; /* Index of current token */ - int iRotorBegin[FTS2_ROTOR_SZ]; /* Beginning offset of token */ - int iRotorLen[FTS2_ROTOR_SZ]; /* Length of token */ + int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */ + int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */ - pVtab = pQuery->pFts; + pVtab = cursor_vtab(pCur); nColumn = pVtab->nColumn; pTokenizer = pVtab->pTokenizer; pTModule = pTokenizer->pModule; rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor); if( rc ) return; pTCursor->pTokenizer = pTokenizer; - aTerm = pQuery->pTerms; - nTerm = pQuery->nTerms; - if( nTerm>=FTS2_ROTOR_SZ ){ - nTerm = FTS2_ROTOR_SZ - 1; - } + prevMatch = 0; - while(1){ - rc = pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); - if( rc ) break; - iRotorBegin[iRotor&FTS2_ROTOR_MASK] = iBegin; - iRotorLen[iRotor&FTS2_ROTOR_MASK] = iEnd-iBegin; + while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){ + Fts3Expr *pIter = pCur->pExpr; + int iIter = -1; + iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin; + iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin; match = 0; - for(i=0; ipPhrase->nToken; + pToken = &pIter->pPhrase->aToken[iIter]; + iCol = pIter->pPhrase->iColumn; if( iCol>=0 && iColnToken ) continue; - if( !aTerm[i].isPrefix && aTerm[i].nTerm1 && (prevMatch & (1<n>nToken ) continue; + if( !pToken->isPrefix && pToken->nn<=nToken ); + if( memcmp(pToken->z, zToken, pToken->n) ) continue; + if( iIter>0 && (prevMatch & (1<=0; j--){ - int k = (iRotor-j) & FTS2_ROTOR_MASK; - snippetAppendMatch(pSnippet, iColumn, i-j, + if( i==(FTS3_ROTOR_SZ-2) || nPhrase==iIter+1 ){ + for(j=nPhrase-1; j>=0; j--){ + int k = (iRotor-j) & FTS3_ROTOR_MASK; + snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j, iRotorBegin[k], iRotorLen[k]); } } @@ -3025,6 +3269,133 @@ static void snippetOffsetsOfColumn( pTModule->xClose(pTCursor); } +/* +** Remove entries from the pSnippet structure to account for the NEAR +** operator. When this is called, pSnippet contains the list of token +** offsets produced by treating all NEAR operators as AND operators. +** This function removes any entries that should not be present after +** accounting for the NEAR restriction. For example, if the queried +** document is: +** +** "A B C D E A" +** +** and the query is: +** +** A NEAR/0 E +** +** then when this function is called the Snippet contains token offsets +** 0, 4 and 5. This function removes the "0" entry (because the first A +** is not near enough to an E). +** +** When this function is called, the value pointed to by parameter piLeft is +** the integer id of the left-most token in the expression tree headed by +** pExpr. This function increments *piLeft by the total number of tokens +** in the expression tree headed by pExpr. +** +** Return 1 if any trimming occurs. Return 0 if no trimming is required. +*/ +static int trimSnippetOffsets( + Fts3Expr *pExpr, /* The search expression */ + Snippet *pSnippet, /* The set of snippet offsets to be trimmed */ + int *piLeft /* Index of left-most token in pExpr */ +){ + if( pExpr ){ + if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){ + return 1; + } + + switch( pExpr->eType ){ + case FTSQUERY_PHRASE: + *piLeft += pExpr->pPhrase->nToken; + break; + case FTSQUERY_NEAR: { + /* The right-hand-side of a NEAR operator is always a phrase. The + ** left-hand-side is either a phrase or an expression tree that is + ** itself headed by a NEAR operator. The following initializations + ** set local variable iLeft to the token number of the left-most + ** token in the right-hand phrase, and iRight to the right most + ** token in the same phrase. For example, if we had: + ** + ** MATCH '"abc def" NEAR/2 "ghi jkl"' + ** + ** then iLeft will be set to 2 (token number of ghi) and nToken will + ** be set to 4. + */ + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + int iLeft = *piLeft; + int nNear = pExpr->nNear; + int nToken = pRight->pPhrase->nToken; + int jj, ii; + if( pLeft->eType==FTSQUERY_NEAR ){ + pLeft = pLeft->pRight; + } + assert( pRight->eType==FTSQUERY_PHRASE ); + assert( pLeft->eType==FTSQUERY_PHRASE ); + nToken += pLeft->pPhrase->nToken; + + for(ii=0; iinMatch; ii++){ + struct snippetMatch *p = &pSnippet->aMatch[ii]; + if( p->iTerm==iLeft ){ + int isOk = 0; + /* Snippet ii is an occurence of query term iLeft in the document. + ** It occurs at position (p->iToken) of the document. We now + ** search for an instance of token (iLeft-1) somewhere in the + ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within + ** the set of snippetMatch structures. If one is found, proceed. + ** If one cannot be found, then remove snippets ii..(ii+N-1) + ** from the matching snippets, where N is the number of tokens + ** in phrase pRight->pPhrase. + */ + for(jj=0; isOk==0 && jjnMatch; jj++){ + struct snippetMatch *p2 = &pSnippet->aMatch[jj]; + if( p2->iTerm==(iLeft-1) ){ + if( p2->iToken>=(p->iToken-nNear-1) + && p2->iToken<(p->iToken+nNear+nToken) + ){ + isOk = 1; + } + } + } + if( !isOk ){ + int kk; + for(kk=0; kkpPhrase->nToken; kk++){ + pSnippet->aMatch[kk+ii].iTerm = -2; + } + return 1; + } + } + if( p->iTerm==(iLeft-1) ){ + int isOk = 0; + for(jj=0; isOk==0 && jjnMatch; jj++){ + struct snippetMatch *p2 = &pSnippet->aMatch[jj]; + if( p2->iTerm==iLeft ){ + if( p2->iToken<=(p->iToken+nNear+1) + && p2->iToken>(p->iToken-nNear-nToken) + ){ + isOk = 1; + } + } + } + if( !isOk ){ + int kk; + for(kk=0; kkpPhrase->nToken; kk++){ + pSnippet->aMatch[ii-kk].iTerm = -2; + } + return 1; + } + } + } + break; + } + } + + if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){ + return 1; + } + } + return 0; +} /* ** Compute all offsets for the current row of the query. @@ -3034,17 +3405,20 @@ static void snippetAllOffsets(fulltext_cursor *p){ int nColumn; int iColumn, i; int iFirst, iLast; - fulltext_vtab *pFts; + int iTerm = 0; + fulltext_vtab *pFts = cursor_vtab(p); - if( p->snippet.nMatch ) return; - if( p->q.nTerms==0 ) return; - pFts = p->q.pFts; + if( p->snippet.nMatch || p->pExpr==0 ){ + return; + } nColumn = pFts->nColumn; iColumn = (p->iCursorType - QUERY_FULLTEXT); if( iColumn<0 || iColumn>=nColumn ){ + /* Look for matches over all columns of the full-text index */ iFirst = 0; iLast = nColumn-1; }else{ + /* Look for matches in the iColumn-th column of the index only */ iFirst = iColumn; iLast = iColumn; } @@ -3053,13 +3427,18 @@ static void snippetAllOffsets(fulltext_cursor *p){ int nDoc; zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1); nDoc = sqlite3_column_bytes(p->pStmt, i+1); - snippetOffsetsOfColumn(&p->q, &p->snippet, i, zDoc, nDoc); + snippetOffsetsOfColumn(p, &p->snippet, i, zDoc, nDoc); + } + + while( trimSnippetOffsets(p->pExpr, &p->snippet, &iTerm) ){ + iTerm = 0; } } /* ** Convert the information in the aMatch[] array of the snippet -** into the string zOffset[0..nOffset-1]. +** into the string zOffset[0..nOffset-1]. This string is used as +** the return of the SQL offsets() function. */ static void snippetOffsetText(Snippet *p){ int i; @@ -3070,11 +3449,18 @@ static void snippetOffsetText(Snippet *p){ initStringBuffer(&sb); for(i=0; inMatch; i++){ struct snippetMatch *pMatch = &p->aMatch[i]; - zBuf[0] = ' '; - sprintf(&zBuf[cnt>0], "%d %d %d %d", pMatch->iCol, - pMatch->iTerm, pMatch->iStart, pMatch->nByte); - append(&sb, zBuf); - cnt++; + if( pMatch->iTerm>=0 ){ + /* If snippetMatch.iTerm is less than 0, then the match was + ** discarded as part of processing the NEAR operator (see the + ** trimSnippetOffsetsForNear() function for details). Ignore + ** it in this case + */ + zBuf[0] = ' '; + sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d", + pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte); + append(&sb, zBuf); + cnt++; + } } p->zOffset = stringBufferData(&sb); p->nOffset = stringBufferLength(&sb); @@ -3157,7 +3543,7 @@ static void snippetText( int iMatch; - free(pCursor->snippet.zSnippet); + sqlite3_free(pCursor->snippet.zSnippet); pCursor->snippet.zSnippet = 0; aMatch = pCursor->snippet.aMatch; nMatch = pCursor->snippet.nMatch; @@ -3167,7 +3553,7 @@ static void snippetText( aMatch[i].snStatus = SNIPPET_IGNORE; } nDesired = 0; - for(i=0; iq.nTerms; i++){ + for(i=0; ipStmt); - queryClear(&c->q); + sqlite3Fts3ExprFree(c->pExpr); snippetClear(&c->snippet); - if( c->result.nData!=0 ) dlrDestroy(&c->reader); + if( c->result.nData!=0 ){ + dlrDestroy(&c->reader); + } dataBufferDestroy(&c->result); - free(c); + sqlite3_free(c); return SQLITE_OK; } @@ -3267,7 +3655,7 @@ static int fulltextNext(sqlite3_vtab_cursor *pCursor){ fulltext_cursor *c = (fulltext_cursor *) pCursor; int rc; - TRACE(("FTS2 Next %p\n", pCursor)); + FTSTRACE(("FTS3 Next %p\n", pCursor)); snippetClear(&c->snippet); if( c->iCursorType < QUERY_FULLTEXT ){ /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */ @@ -3314,202 +3702,127 @@ static int termSelect(fulltext_vtab *v, int iColumn, const char *pTerm, int nTerm, int isPrefix, DocListType iType, DataBuffer *out); -/* Return a DocList corresponding to the query term *pTerm. If *pTerm -** is the first term of a phrase query, go ahead and evaluate the phrase -** query and return the doclist for the entire phrase query. +/* +** Return a DocList corresponding to the phrase *pPhrase. ** ** The resulting DL_DOCIDS doclist is stored in pResult, which is ** overwritten. */ -static int docListOfTerm( - fulltext_vtab *v, /* The full text index */ - int iColumn, /* column to restrict to. No restriction if >=nColumn */ - QueryTerm *pQTerm, /* Term we are looking for, or 1st term of a phrase */ - DataBuffer *pResult /* Write the result here */ +static int docListOfPhrase( + fulltext_vtab *pTab, /* The full text index */ + Fts3Phrase *pPhrase, /* Phrase to return a doclist corresponding to */ + DocListType eListType, /* Either DL_DOCIDS or DL_POSITIONS */ + DataBuffer *pResult /* Write the result here */ ){ - DataBuffer left, right, new; - int i, rc; - - /* No phrase search if no position info. */ - assert( pQTerm->nPhrase==0 || DL_DEFAULT!=DL_DOCIDS ); + int ii; + int rc = SQLITE_OK; + int iCol = pPhrase->iColumn; + DocListType eType = eListType; + assert( eType==DL_POSITIONS || eType==DL_DOCIDS ); + if( pPhrase->nToken>1 ){ + eType = DL_POSITIONS; + } /* This code should never be called with buffered updates. */ - assert( v->nPendingData<0 ); + assert( pTab->nPendingData<0 ); - dataBufferInit(&left, 0); - rc = termSelect(v, iColumn, pQTerm->pTerm, pQTerm->nTerm, pQTerm->isPrefix, - 0nPhrase ? DL_POSITIONS : DL_DOCIDS, &left); - if( rc ) return rc; - for(i=1; i<=pQTerm->nPhrase && left.nData>0; i++){ - dataBufferInit(&right, 0); - rc = termSelect(v, iColumn, pQTerm[i].pTerm, pQTerm[i].nTerm, - pQTerm[i].isPrefix, DL_POSITIONS, &right); - if( rc ){ - dataBufferDestroy(&left); - return rc; + for(ii=0; rc==SQLITE_OK && iinToken; ii++){ + DataBuffer tmp; + struct PhraseToken *p = &pPhrase->aToken[ii]; + rc = termSelect(pTab, iCol, p->z, p->n, p->isPrefix, eType, &tmp); + if( rc==SQLITE_OK ){ + if( ii==0 ){ + *pResult = tmp; + }else{ + DataBuffer res = *pResult; + dataBufferInit(pResult, 0); + if( ii==(pPhrase->nToken-1) ){ + eType = eListType; + } + docListPhraseMerge( + res.pData, res.nData, tmp.pData, tmp.nData, 0, 0, eType, pResult + ); + dataBufferDestroy(&res); + dataBufferDestroy(&tmp); + } } - dataBufferInit(&new, 0); - docListPhraseMerge(left.pData, left.nData, right.pData, right.nData, - inPhrase ? DL_POSITIONS : DL_DOCIDS, &new); - dataBufferDestroy(&left); - dataBufferDestroy(&right); - left = new; } - *pResult = left; - return SQLITE_OK; -} -/* Add a new term pTerm[0..nTerm-1] to the query *q. -*/ -static void queryAdd(Query *q, const char *pTerm, int nTerm){ - QueryTerm *t; - ++q->nTerms; - q->pTerms = realloc(q->pTerms, q->nTerms * sizeof(q->pTerms[0])); - if( q->pTerms==0 ){ - q->nTerms = 0; - return; - } - t = &q->pTerms[q->nTerms - 1]; - CLEAR(t); - t->pTerm = malloc(nTerm+1); - memcpy(t->pTerm, pTerm, nTerm); - t->pTerm[nTerm] = 0; - t->nTerm = nTerm; - t->isOr = q->nextIsOr; - t->isPrefix = 0; - q->nextIsOr = 0; - t->iColumn = q->nextColumn; - q->nextColumn = q->dfltColumn; + return rc; } /* -** Check to see if the string zToken[0...nToken-1] matches any -** column name in the virtual table. If it does, -** return the zero-indexed column number. If not, return -1. +** Evaluate the full-text expression pExpr against fts3 table pTab. Write +** the results into pRes. */ -static int checkColumnSpecifier( - fulltext_vtab *pVtab, /* The virtual table */ - const char *zToken, /* Text of the token */ - int nToken /* Number of characters in the token */ +static int evalFts3Expr( + fulltext_vtab *pTab, /* Fts3 Virtual table object */ + Fts3Expr *pExpr, /* Parsed fts3 expression */ + DataBuffer *pRes /* OUT: Write results of the expression here */ ){ - int i; - for(i=0; inColumn; i++){ - if( memcmp(pVtab->azColumn[i], zToken, nToken)==0 - && pVtab->azColumn[i][nToken]==0 ){ - return i; - } - } - return -1; -} + int rc = SQLITE_OK; -/* -** Parse the text at pSegment[0..nSegment-1]. Add additional terms -** to the query being assemblied in pQuery. -** -** inPhrase is true if pSegment[0..nSegement-1] is contained within -** double-quotes. If inPhrase is true, then the first term -** is marked with the number of terms in the phrase less one and -** OR and "-" syntax is ignored. If inPhrase is false, then every -** term found is marked with nPhrase=0 and OR and "-" syntax is significant. -*/ -static int tokenizeSegment( - sqlite3_tokenizer *pTokenizer, /* The tokenizer to use */ - const char *pSegment, int nSegment, /* Query expression being parsed */ - int inPhrase, /* True if within "..." */ - Query *pQuery /* Append results here */ -){ - const sqlite3_tokenizer_module *pModule = pTokenizer->pModule; - sqlite3_tokenizer_cursor *pCursor; - int firstIndex = pQuery->nTerms; - int iCol; - int nTerm = 1; - - int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor); - if( rc!=SQLITE_OK ) return rc; - pCursor->pTokenizer = pTokenizer; + /* Initialize the output buffer. If this is an empty query (pExpr==0), + ** this is all that needs to be done. Empty queries produce empty + ** result sets. + */ + dataBufferInit(pRes, 0); - while( 1 ){ - const char *pToken; - int nToken, iBegin, iEnd, iPos; + if( pExpr ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + DocListType eType = DL_DOCIDS; + if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ + eType = DL_POSITIONS; + } + rc = docListOfPhrase(pTab, pExpr->pPhrase, eType, pRes); + }else{ + DataBuffer lhs; + DataBuffer rhs; - rc = pModule->xNext(pCursor, - &pToken, &nToken, - &iBegin, &iEnd, &iPos); - if( rc!=SQLITE_OK ) break; - if( !inPhrase && - pSegment[iEnd]==':' && - (iCol = checkColumnSpecifier(pQuery->pFts, pToken, nToken))>=0 ){ - pQuery->nextColumn = iCol; - continue; - } - if( !inPhrase && pQuery->nTerms>0 && nToken==2 - && pSegment[iBegin]=='O' && pSegment[iBegin+1]=='R' ){ - pQuery->nextIsOr = 1; - continue; - } - queryAdd(pQuery, pToken, nToken); - if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){ - pQuery->pTerms[pQuery->nTerms-1].isNot = 1; - } - if( iEndpTerms[pQuery->nTerms-1].isPrefix = 1; - } - pQuery->pTerms[pQuery->nTerms-1].iPhrase = nTerm; - if( inPhrase ){ - nTerm++; + dataBufferInit(&rhs, 0); + if( SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pLeft, &lhs)) + && SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pRight, &rhs)) + ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: { + int nToken; + Fts3Expr *pLeft; + DocListType eType = DL_DOCIDS; + if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ + eType = DL_POSITIONS; + } + pLeft = pExpr->pLeft; + while( pLeft->eType==FTSQUERY_NEAR ){ + pLeft=pLeft->pRight; + } + assert( pExpr->pRight->eType==FTSQUERY_PHRASE ); + assert( pLeft->eType==FTSQUERY_PHRASE ); + nToken = pLeft->pPhrase->nToken + pExpr->pRight->pPhrase->nToken; + docListPhraseMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, + pExpr->nNear+1, nToken, eType, pRes + ); + break; + } + case FTSQUERY_NOT: { + docListExceptMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,pRes); + break; + } + case FTSQUERY_AND: { + docListAndMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); + break; + } + case FTSQUERY_OR: { + docListOrMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); + break; + } + } + } + dataBufferDestroy(&lhs); + dataBufferDestroy(&rhs); } } - if( inPhrase && pQuery->nTerms>firstIndex ){ - pQuery->pTerms[firstIndex].nPhrase = pQuery->nTerms - firstIndex - 1; - } - - return pModule->xClose(pCursor); -} - -/* Parse a query string, yielding a Query object pQuery. -** -** The calling function will need to queryClear() to clean up -** the dynamically allocated memory held by pQuery. -*/ -static int parseQuery( - fulltext_vtab *v, /* The fulltext index */ - const char *zInput, /* Input text of the query string */ - int nInput, /* Size of the input text */ - int dfltColumn, /* Default column of the index to match against */ - Query *pQuery /* Write the parse results here. */ -){ - int iInput, inPhrase = 0; - - if( zInput==0 ) nInput = 0; - if( nInput<0 ) nInput = strlen(zInput); - pQuery->nTerms = 0; - pQuery->pTerms = NULL; - pQuery->nextIsOr = 0; - pQuery->nextColumn = dfltColumn; - pQuery->dfltColumn = dfltColumn; - pQuery->pFts = v; - - for(iInput=0; iInputiInput ){ - tokenizeSegment(v->pTokenizer, zInput+iInput, i-iInput, inPhrase, - pQuery); - } - iInput = i; - if( inTerms==0 ){ - dataBufferInit(pResult, 0); - return SQLITE_OK; + if( rc!=SQLITE_OK ){ + return rc; } - /* Merge AND terms. */ - /* TODO(shess) I think we can early-exit if( i>nNot && left.nData==0 ). */ - aTerm = pQuery->pTerms; - for(i = 0; inTerms; i=iNext){ - if( aTerm[i].isNot ){ - /* Handle all NOT terms in a separate pass */ - nNot++; - iNext = i + aTerm[i].nPhrase+1; - continue; - } - iNext = i + aTerm[i].nPhrase + 1; - rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &right); - if( rc ){ - if( i!=nNot ) dataBufferDestroy(&left); - queryClear(pQuery); - return rc; - } - while( iNextnTerms && aTerm[iNext].isOr ){ - rc = docListOfTerm(v, aTerm[iNext].iColumn, &aTerm[iNext], &or); - iNext += aTerm[iNext].nPhrase + 1; - if( rc ){ - if( i!=nNot ) dataBufferDestroy(&left); - dataBufferDestroy(&right); - queryClear(pQuery); - return rc; - } - dataBufferInit(&new, 0); - docListOrMerge(right.pData, right.nData, or.pData, or.nData, &new); - dataBufferDestroy(&right); - dataBufferDestroy(&or); - right = new; - } - if( i==nNot ){ /* first term processed. */ - left = right; - }else{ - dataBufferInit(&new, 0); - docListAndMerge(left.pData, left.nData, right.pData, right.nData, &new); - dataBufferDestroy(&right); - dataBufferDestroy(&left); - left = new; - } + /* Parse the query passed to the MATCH operator. */ + rc = sqlite3Fts3ExprParse(v->pTokenizer, + v->azColumn, v->nColumn, iColumn, zInput, nInput, ppExpr + ); + if( rc!=SQLITE_OK ){ + assert( 0==(*ppExpr) ); + return rc; } - if( nNot==pQuery->nTerms ){ - /* We do not yet know how to handle a query of only NOT terms */ - return SQLITE_ERROR; - } - - /* Do the EXCEPT terms */ - for(i=0; inTerms; i += aTerm[i].nPhrase + 1){ - if( !aTerm[i].isNot ) continue; - rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &right); - if( rc ){ - queryClear(pQuery); - dataBufferDestroy(&left); - return rc; - } - dataBufferInit(&new, 0); - docListExceptMerge(left.pData, left.nData, right.pData, right.nData, &new); - dataBufferDestroy(&right); - dataBufferDestroy(&left); - left = new; - } - - *pResult = left; - return rc; + return evalFts3Expr(v, *ppExpr, pResult); } /* @@ -3635,7 +3879,7 @@ static int fulltextQuery( ** If idxNum==QUERY_GENERIC then do a full table scan against ** the %_content table. ** -** If idxNum==QUERY_ROWID then do a rowid lookup for a single entry +** If idxNum==QUERY_DOCID then do a docid lookup for a single entry ** in the %_content table. ** ** If idxNum>=QUERY_FULLTEXT then use the full text index. The @@ -3646,7 +3890,7 @@ static int fulltextQuery( /* TODO(shess) Upgrade the cursor initialization and destruction to ** account for fulltextFilter() being called multiple times on the ** same cursor. The current solution is very fragile. Apply fix to -** fts2 as appropriate. +** fts3 as appropriate. */ static int fulltextFilter( sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ @@ -3656,33 +3900,58 @@ static int fulltextFilter( fulltext_cursor *c = (fulltext_cursor *) pCursor; fulltext_vtab *v = cursor_vtab(c); int rc; - char *zSql; - TRACE(("FTS2 Filter %p\n",pCursor)); + FTSTRACE(("FTS3 Filter %p\n",pCursor)); - zSql = sqlite3_mprintf("select rowid, * from %%_content %s", - idxNum==QUERY_GENERIC ? "" : "where rowid=?"); - sqlite3_finalize(c->pStmt); - rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, zSql); - sqlite3_free(zSql); - if( rc!=SQLITE_OK ) return rc; + /* If the cursor has a statement that was not prepared according to + ** idxNum, clear it. I believe all calls to fulltextFilter with a + ** given cursor will have the same idxNum , but in this case it's + ** easy to be safe. + */ + if( c->pStmt && c->iCursorType!=idxNum ){ + sqlite3_finalize(c->pStmt); + c->pStmt = NULL; + } + + /* Get a fresh statement appropriate to idxNum. */ + /* TODO(shess): Add a prepared-statement cache in the vt structure. + ** The cache must handle multiple open cursors. Easier to cache the + ** statement variants at the vt to reduce malloc/realloc/free here. + ** Or we could have a StringBuffer variant which allowed stack + ** construction for small values. + */ + if( !c->pStmt ){ + StringBuffer sb; + initStringBuffer(&sb); + append(&sb, "SELECT docid, "); + appendList(&sb, v->nColumn, v->azContentColumn); + append(&sb, " FROM %_content"); + if( idxNum!=QUERY_GENERIC ) append(&sb, " WHERE docid = ?"); + rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, + stringBufferData(&sb)); + stringBufferDestroy(&sb); + if( rc!=SQLITE_OK ) return rc; + c->iCursorType = idxNum; + }else{ + sqlite3_reset(c->pStmt); + assert( c->iCursorType==idxNum ); + } - c->iCursorType = idxNum; switch( idxNum ){ case QUERY_GENERIC: break; - case QUERY_ROWID: + case QUERY_DOCID: rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0])); if( rc!=SQLITE_OK ) return rc; break; default: /* full-text search */ { + int iCol = idxNum-QUERY_FULLTEXT; const char *zQuery = (const char *)sqlite3_value_text(argv[0]); assert( idxNum<=QUERY_FULLTEXT+v->nColumn); assert( argc==1 ); - queryClear(&c->q); if( c->result.nData!=0 ){ /* This case happens if the same cursor is used repeatedly. */ dlrDestroy(&c->reader); @@ -3690,7 +3959,7 @@ static int fulltextFilter( }else{ dataBufferInit(&c->result, 0); } - rc = fulltextQuery(v, idxNum-QUERY_FULLTEXT, zQuery, -1, &c->result, &c->q); + rc = fulltextQuery(v, iCol, zQuery, -1, &c->result, &c->pExpr); if( rc!=SQLITE_OK ) return rc; if( c->result.nData!=0 ){ dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData); @@ -3730,12 +3999,17 @@ static int fulltextColumn(sqlite3_vtab_cursor *pCursor, ** Return a blob which is a pointer to the cursor */ sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT); + }else if( idxCol==v->nColumn+1 ){ + /* The docid column, which is an alias for rowid. */ + sqlite3_value *pVal = sqlite3_column_value(c->pStmt, 0); + sqlite3_result_value(pContext, pVal); } return SQLITE_OK; } /* This is the xRowid method. The SQLite core calls this routine to -** retrive the rowid for the current row of the result set. The +** retrieve the rowid for the current row of the result set. fts3 +** exposes %_content.docid as the rowid for the virtual table. The ** rowid should be written to *pRowid. */ static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ @@ -3762,27 +4036,28 @@ static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid, if( rc!=SQLITE_OK ) return rc; pCursor->pTokenizer = pTokenizer; - while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor, - &pToken, &nTokenBytes, - &iStartOffset, &iEndOffset, - &iPosition) ){ + while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor, + &pToken, &nTokenBytes, + &iStartOffset, &iEndOffset, + &iPosition)) ){ DLCollector *p; int nData; /* Size of doclist before our update. */ - /* Positions can't be negative; we use -1 as a terminator internally. */ - if( iPosition<0 ){ - pTokenizer->pModule->xClose(pCursor); - return SQLITE_ERROR; + /* Positions can't be negative; we use -1 as a terminator + * internally. Token can't be NULL or empty. */ + if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){ + rc = SQLITE_ERROR; + break; } - p = fts2HashFind(&v->pendingTerms, pToken, nTokenBytes); + p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes); if( p==NULL ){ nData = 0; p = dlcNew(iDocid, DL_DEFAULT); - fts2HashInsert(&v->pendingTerms, pToken, nTokenBytes, p); + fts3HashInsert(&v->pendingTerms, pToken, nTokenBytes, p); /* Overhead for our hash table entry, the key, and the value. */ - v->nPendingData += sizeof(struct fts2HashElem)+sizeof(*p)+nTokenBytes; + v->nPendingData += sizeof(struct fts3HashElem)+sizeof(*p)+nTokenBytes; }else{ nData = p->b.nData; if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid); @@ -3801,16 +4076,17 @@ static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid, ** not durable. *ponder* */ pTokenizer->pModule->xClose(pCursor); + if( SQLITE_DONE == rc ) return SQLITE_OK; return rc; } /* Add doclists for all terms in [pValues] to pendingTerms table. */ -static int insertTerms(fulltext_vtab *v, sqlite_int64 iRowid, +static int insertTerms(fulltext_vtab *v, sqlite_int64 iDocid, sqlite3_value **pValues){ int i; for(i = 0; i < v->nColumn ; ++i){ char *zText = (char*)sqlite3_value_text(pValues[i]); - int rc = buildTerms(v, iRowid, zText, i); + int rc = buildTerms(v, iDocid, zText, i); if( rc!=SQLITE_OK ) return rc; } return SQLITE_OK; @@ -3819,18 +4095,18 @@ static int insertTerms(fulltext_vtab *v, sqlite_int64 iRowid, /* Add empty doclists for all terms in the given row's content to ** pendingTerms. */ -static int deleteTerms(fulltext_vtab *v, sqlite_int64 iRowid){ +static int deleteTerms(fulltext_vtab *v, sqlite_int64 iDocid){ const char **pValues; int i, rc; /* TODO(shess) Should we allow such tables at all? */ if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR; - rc = content_select(v, iRowid, &pValues); + rc = content_select(v, iDocid, &pValues); if( rc!=SQLITE_OK ) return rc; for(i = 0 ; i < v->nColumn; ++i) { - rc = buildTerms(v, iRowid, pValues[i], -1); + rc = buildTerms(v, iDocid, pValues[i], -1); if( rc!=SQLITE_OK ) break; } @@ -3841,21 +4117,22 @@ static int deleteTerms(fulltext_vtab *v, sqlite_int64 iRowid){ /* TODO(shess) Refactor the code to remove this forward decl. */ static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid); -/* Insert a row into the %_content table; set *piRowid to be the ID of the +/* Insert a row into the %_content table; set *piDocid to be the ID of the ** new row. Add doclists for terms to pendingTerms. */ -static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestRowid, - sqlite3_value **pValues, sqlite_int64 *piRowid){ +static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestDocid, + sqlite3_value **pValues, sqlite_int64 *piDocid){ int rc; - rc = content_insert(v, pRequestRowid, pValues); /* execute an SQL INSERT */ + rc = content_insert(v, pRequestDocid, pValues); /* execute an SQL INSERT */ if( rc!=SQLITE_OK ) return rc; - *piRowid = sqlite3_last_insert_rowid(v->db); - rc = initPendingTerms(v, *piRowid); + /* docid column is an alias for rowid. */ + *piDocid = sqlite3_last_insert_rowid(v->db); + rc = initPendingTerms(v, *piDocid); if( rc!=SQLITE_OK ) return rc; - return insertTerms(v, *piRowid, pValues); + return insertTerms(v, *piDocid, pValues); } /* Delete a row from the %_content table; add empty doclists for terms @@ -3935,18 +4212,20 @@ typedef struct InteriorBlock { static InteriorBlock *interiorBlockNew(int iHeight, sqlite_int64 iChildBlock, const char *pTerm, int nTerm){ - InteriorBlock *block = calloc(1, sizeof(InteriorBlock)); + InteriorBlock *block = sqlite3_malloc(sizeof(InteriorBlock)); char c[VARINT_MAX+VARINT_MAX]; int n; - dataBufferInit(&block->term, 0); - dataBufferReplace(&block->term, pTerm, nTerm); - - n = putVarint(c, iHeight); - n += putVarint(c+n, iChildBlock); - dataBufferInit(&block->data, INTERIOR_MAX); - dataBufferReplace(&block->data, c, n); + if( block ){ + memset(block, 0, sizeof(*block)); + dataBufferInit(&block->term, 0); + dataBufferReplace(&block->term, pTerm, nTerm); + n = fts3PutVarint(c, iHeight); + n += fts3PutVarint(c+n, iChildBlock); + dataBufferInit(&block->data, INTERIOR_MAX); + dataBufferReplace(&block->data, c, n); + } return block; } @@ -3963,7 +4242,7 @@ static void interiorBlockValidate(InteriorBlock *pBlock){ assert( pData+nData>pData ); /* Must lead with height of node as a varint(n), n>0 */ - n = getVarint32(pData, &iDummy); + n = fts3GetVarint32(pData, &iDummy); assert( n>0 ); assert( iDummy>0 ); assert( n0 ); assert( n<=nData ); pData += n; @@ -3980,7 +4259,7 @@ static void interiorBlockValidate(InteriorBlock *pBlock){ /* Zero or more terms of positive length */ if( nData!=0 ){ /* First term is not delta-encoded. */ - n = getVarint32(pData, &iDummy); + n = fts3GetVarint32(pData, &iDummy); assert( n>0 ); assert( iDummy>0 ); assert( n+iDummy>0); @@ -3991,7 +4270,7 @@ static void interiorBlockValidate(InteriorBlock *pBlock){ /* Following terms delta-encoded. */ while( nData!=0 ){ /* Length of shared prefix. */ - n = getVarint32(pData, &iDummy); + n = fts3GetVarint32(pData, &iDummy); assert( n>0 ); assert( iDummy>=0 ); assert( n0 ); assert( iDummy>0 ); assert( n+iDummy>0); @@ -4066,15 +4345,15 @@ static void interiorWriterAppend(InteriorWriter *pWriter, ** at 0. */ if( pWriter->term.nData==0 ){ - n = putVarint(c, nTerm); + n = fts3PutVarint(c, nTerm); }else{ while( nPrefixterm.nData && pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){ nPrefix++; } - n = putVarint(c, nPrefix); - n += putVarint(c+n, nTerm-nPrefix); + n = fts3PutVarint(c, nPrefix); + n += fts3PutVarint(c+n, nTerm-nPrefix); } #ifndef NDEBUG @@ -4111,11 +4390,11 @@ static int interiorWriterDestroy(InteriorWriter *pWriter){ block = block->next; dataBufferDestroy(&b->term); dataBufferDestroy(&b->data); - free(b); + sqlite3_free(b); } if( pWriter->parentWriter!=NULL ){ interiorWriterDestroy(pWriter->parentWriter); - free(pWriter->parentWriter); + sqlite3_free(pWriter->parentWriter); } dataBufferDestroy(&pWriter->term); SCRAMBLE(pWriter); @@ -4149,7 +4428,7 @@ static int interiorWriterRootInfo(fulltext_vtab *v, InteriorWriter *pWriter, if( rc!=SQLITE_OK ) return rc; *piEndBlockid = iBlockid; - pWriter->parentWriter = malloc(sizeof(*pWriter->parentWriter)); + pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter)); interiorWriterInit(pWriter->iHeight+1, block->term.pData, block->term.nData, iBlockid, pWriter->parentWriter); @@ -4186,6 +4465,7 @@ typedef struct InteriorReader { } InteriorReader; static void interiorReaderDestroy(InteriorReader *pReader){ + dataBufferDestroy(&pReader->term); SCRAMBLE(pReader); } @@ -4203,7 +4483,7 @@ static void interiorReaderInit(const char *pData, int nData, CLEAR(pReader); /* Decode the base blockid, and set the cursor to the first term. */ - n = getVarint(pData+1, &pReader->iBlockid); + n = fts3GetVarint(pData+1, &pReader->iBlockid); assert( 1+n<=nData ); pReader->pData = pData+1+n; pReader->nData = nData-(1+n); @@ -4215,7 +4495,7 @@ static void interiorReaderInit(const char *pData, int nData, if( pReader->nData==0 ){ dataBufferInit(&pReader->term, 0); }else{ - n = getVarint32(pReader->pData, &nTerm); + n = fts3GetVarint32(pReader->pData, &nTerm); dataBufferInit(&pReader->term, nTerm); dataBufferReplace(&pReader->term, pReader->pData+n, nTerm); assert( n+nTerm<=pReader->nData ); @@ -4253,8 +4533,8 @@ static void interiorReaderStep(InteriorReader *pReader){ }else{ int n, nPrefix, nSuffix; - n = getVarint32(pReader->pData, &nPrefix); - n += getVarint32(pReader->pData+n, &nSuffix); + n = fts3GetVarint32(pReader->pData, &nPrefix); + n += fts3GetVarint32(pReader->pData+n, &nSuffix); /* Truncate the current term and append suffix data. */ pReader->term.nData = nPrefix; @@ -4359,7 +4639,7 @@ static void leafNodeValidate(const char *pData, int nData){ assert( pData+nData>pData ); /* Must lead with a varint(0) */ - n = getVarint32(pData, &iDummy); + n = fts3GetVarint32(pData, &iDummy); assert( iDummy==0 ); assert( n>0 ); assert( n0 ); assert( iDummy>0 ); assert( n+iDummy>0 ); @@ -4376,7 +4656,7 @@ static void leafNodeValidate(const char *pData, int nData){ nData -= n+iDummy; /* Leading term's doclist length and data must fit. */ - n = getVarint32(pData, &iDummy); + n = fts3GetVarint32(pData, &iDummy); assert( n>0 ); assert( iDummy>0 ); assert( n+iDummy>0 ); @@ -4387,13 +4667,13 @@ static void leafNodeValidate(const char *pData, int nData){ /* Verify that trailing terms and doclists also are readable. */ while( nData!=0 ){ - n = getVarint32(pData, &iDummy); + n = fts3GetVarint32(pData, &iDummy); assert( n>0 ); assert( iDummy>=0 ); assert( n0 ); assert( iDummy>0 ); assert( n+iDummy>0 ); @@ -4401,7 +4681,7 @@ static void leafNodeValidate(const char *pData, int nData){ pData += n+iDummy; nData -= n+iDummy; - n = getVarint32(pData, &iDummy); + n = fts3GetVarint32(pData, &iDummy); assert( n>0 ); assert( iDummy>0 ); assert( n+iDummy>0 ); @@ -4441,7 +4721,7 @@ static int leafWriterInternalFlush(fulltext_vtab *v, LeafWriter *pWriter, /* Reconstruct the first term in the leaf for purposes of building ** the interior node. */ - n = getVarint32(pWriter->data.pData+iData+1, &nStartingTerm); + n = fts3GetVarint32(pWriter->data.pData+iData+1, &nStartingTerm); pStartingTerm = pWriter->data.pData+iData+1+n; assert( pWriter->data.nData>iData+1+n+nStartingTerm ); assert( pWriter->nTermDistinct>0 ); @@ -4565,8 +4845,8 @@ static int leafWriterEncodeTerm(LeafWriter *pWriter, ** varint(nTerm) ** char pTerm[nTerm] */ - n = putVarint(c, '\0'); - n += putVarint(c+n, nTerm); + n = fts3PutVarint(c, '\0'); + n += fts3PutVarint(c+n, nTerm); dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm); }else{ /* Delta-encode the term as: @@ -4574,8 +4854,8 @@ static int leafWriterEncodeTerm(LeafWriter *pWriter, ** varint(nSuffix) ** char pTermSuffix[nSuffix] */ - n = putVarint(c, nPrefix); - n += putVarint(c+n, nTerm-nPrefix); + n = fts3PutVarint(c, nPrefix); + n += fts3PutVarint(c+n, nTerm-nPrefix); dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix); } dataBufferReplace(&pWriter->term, pTerm, nTerm); @@ -4592,8 +4872,8 @@ static int leafWriterInlineFlush(fulltext_vtab *v, LeafWriter *pWriter, const char *pTerm, int nTerm, int iDoclistData){ char c[VARINT_MAX+VARINT_MAX]; - int iData, n = putVarint(c, 0); - n += putVarint(c+n, nTerm); + int iData, n = fts3PutVarint(c, 0); + n += fts3PutVarint(c+n, nTerm); /* There should always be room for the header. Even if pTerm shared ** a substantial prefix with the previous term, the entire prefix @@ -4633,7 +4913,7 @@ static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter, for(i=0, nData=0; idata, c, n); docListMerge(&pWriter->data, pReaders, nReaders); @@ -4647,7 +4927,7 @@ static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter, ** not a big deal, we can just use memmove() to adjust things. */ nActualData = pWriter->data.nData-(iDoclistData+n); - nActual = putVarint(c, nActualData); + nActual = fts3PutVarint(c, nActualData); assert( nActualData<=nData ); assert( nActual<=n ); @@ -4710,8 +4990,8 @@ static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter, pWriter->nTermDistinct = nTermDistinct; /* Rebuild header using the current term */ - n = putVarint(pWriter->data.pData, 0); - n += putVarint(pWriter->data.pData+n, nTerm); + n = fts3PutVarint(pWriter->data.pData, 0); + n += fts3PutVarint(pWriter->data.pData+n, nTerm); memcpy(pWriter->data.pData+n, pTerm, nTerm); n += nTerm; @@ -4785,13 +5065,13 @@ static const char *leafReaderTerm(LeafReader *pReader){ static int leafReaderDataBytes(LeafReader *pReader){ int nData; assert( pReader->term.nData>0 ); - getVarint32(pReader->pData, &nData); + fts3GetVarint32(pReader->pData, &nData); return nData; } static const char *leafReaderData(LeafReader *pReader){ int n, nData; assert( pReader->term.nData>0 ); - n = getVarint32(pReader->pData, &nData); + n = fts3GetVarint32(pReader->pData, &nData); return pReader->pData+n; } @@ -4805,7 +5085,7 @@ static void leafReaderInit(const char *pData, int nData, CLEAR(pReader); /* Read the first term, skipping the header byte. */ - n = getVarint32(pData+1, &nTerm); + n = fts3GetVarint32(pData+1, &nTerm); dataBufferInit(&pReader->term, nTerm); dataBufferReplace(&pReader->term, pData+1+n, nTerm); @@ -4821,7 +5101,7 @@ static void leafReaderStep(LeafReader *pReader){ assert( !leafReaderAtEnd(pReader) ); /* Skip previous entry's data block. */ - n = getVarint32(pReader->pData, &nData); + n = fts3GetVarint32(pReader->pData, &nData); assert( n+nData<=pReader->nData ); pReader->pData += n+nData; pReader->nData -= n+nData; @@ -4830,8 +5110,8 @@ static void leafReaderStep(LeafReader *pReader){ /* Construct the new term using a prefix from the old term plus a ** suffix from the leaf data. */ - n = getVarint32(pReader->pData, &nPrefix); - n += getVarint32(pReader->pData+n, &nSuffix); + n = fts3GetVarint32(pReader->pData, &nPrefix); + n += fts3GetVarint32(pReader->pData+n, &nSuffix); assert( n+nSuffixnData ); pReader->term.nData = nPrefix; dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix); @@ -4916,6 +5196,12 @@ static int leavesReaderReset(LeavesReader *pReader){ } static void leavesReaderDestroy(LeavesReader *pReader){ + /* If idx is -1, that means we're using a non-cached statement + ** handle in the optimize() case, so we need to release it. + */ + if( pReader->pStmt!=NULL && pReader->idx==-1 ){ + sqlite3_finalize(pReader->pStmt); + } leafReaderDestroy(&pReader->leafReader); dataBufferDestroy(&pReader->rootData); SCRAMBLE(pReader); @@ -5036,7 +5322,7 @@ static void leavesReaderReorder(LeavesReader *pLr, int nLr){ static int leavesReadersInit(fulltext_vtab *v, int iLevel, LeavesReader *pReaders, int *piReaders){ sqlite3_stmt *s; - int i, rc = sql_get_statement(v, SEGDIR_SELECT_STMT, &s); + int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s); if( rc!=SQLITE_OK ) return rc; rc = sqlite3_bind_int(s, 1, iLevel); @@ -5188,6 +5474,26 @@ static int segmentMerge(fulltext_vtab *v, int iLevel){ return rc; } +/* Accumulate the union of *acc and *pData into *acc. */ +static void docListAccumulateUnion(DataBuffer *acc, + const char *pData, int nData) { + DataBuffer tmp = *acc; + dataBufferInit(acc, tmp.nData+nData); + docListUnion(tmp.pData, tmp.nData, pData, nData, acc); + dataBufferDestroy(&tmp); +} + +/* TODO(shess) It might be interesting to explore different merge +** strategies, here. For instance, since this is a sorted merge, we +** could easily merge many doclists in parallel. With some +** comprehension of the storage format, we could merge all of the +** doclists within a leaf node directly from the leaf node's storage. +** It may be worthwhile to merge smaller doclists before larger +** doclists, since they can be traversed more quickly - but the +** results may have less overlap, making them more expensive in a +** different way. +*/ + /* Scan pReader for pTerm/nTerm, and merge the term's doclist over ** *out (any doclists with duplicate docids overwrite those in *out). ** Internal function for loadSegmentLeaf(). @@ -5195,39 +5501,116 @@ static int segmentMerge(fulltext_vtab *v, int iLevel){ static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader, const char *pTerm, int nTerm, int isPrefix, DataBuffer *out){ + /* doclist data is accumulated into pBuffers similar to how one does + ** increment in binary arithmetic. If index 0 is empty, the data is + ** stored there. If there is data there, it is merged and the + ** results carried into position 1, with further merge-and-carry + ** until an empty position is found. + */ + DataBuffer *pBuffers = NULL; + int nBuffers = 0, nMaxBuffers = 0, rc; + assert( nTerm>0 ); - /* Process while the prefix matches. */ - while( !leavesReaderAtEnd(pReader) ){ + for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader); + rc=leavesReaderStep(v, pReader)){ /* TODO(shess) Really want leavesReaderTermCmp(), but that name is ** already taken to compare the terms of two LeavesReaders. Think ** on a better name. [Meanwhile, break encapsulation rather than ** use a confusing name.] */ - int rc; int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix); + if( c>0 ) break; /* Past any possible matches. */ if( c==0 ){ const char *pData = leavesReaderData(pReader); - int nData = leavesReaderDataBytes(pReader); - if( out->nData==0 ){ - dataBufferReplace(out, pData, nData); + int iBuffer, nData = leavesReaderDataBytes(pReader); + + /* Find the first empty buffer. */ + for(iBuffer=0; iBuffer0 ){ + assert(pBuffers!=NULL); + memcpy(p, pBuffers, nBuffers*sizeof(*pBuffers)); + sqlite3_free(pBuffers); + } + pBuffers = p; + } + dataBufferInit(&(pBuffers[nBuffers]), 0); + nBuffers++; + } + + /* At this point, must have an empty at iBuffer. */ + assert(iBuffernData+nData); - docListUnion(out->pData, out->nData, pData, nData, &result); - dataBufferDestroy(out); - *out = result; - /* TODO(shess) Rather than destroy out, we could retain it for - ** later reuse. + /* pAcc is the empty buffer the merged data will end up in. */ + DataBuffer *pAcc = &(pBuffers[iBuffer]); + DataBuffer *p = &(pBuffers[0]); + + /* Handle position 0 specially to avoid need to prime pAcc + ** with pData/nData. */ + dataBufferSwap(p, pAcc); + docListAccumulateUnion(pAcc, pData, nData); + + /* Accumulate remaining doclists into pAcc. */ + for(++p; ppData, p->nData); + + /* dataBufferReset() could allow a large doclist to blow up + ** our memory requirements. + */ + if( p->nCapacity<1024 ){ + dataBufferReset(p); + }else{ + dataBufferDestroy(p); + dataBufferInit(p, 0); + } + } } } - if( c>0 ) break; /* Past any possible matches. */ - - rc = leavesReaderStep(v, pReader); - if( rc!=SQLITE_OK ) return rc; } - return SQLITE_OK; + + /* Union all the doclists together into *out. */ + /* TODO(shess) What if *out is big? Sigh. */ + if( rc==SQLITE_OK && nBuffers>0 ){ + int iBuffer; + for(iBuffer=0; iBuffer0 ){ + if( out->nData==0 ){ + dataBufferSwap(out, &(pBuffers[iBuffer])); + }else{ + docListAccumulateUnion(out, pBuffers[iBuffer].pData, + pBuffers[iBuffer].nData); + } + } + } + } + + while( nBuffers-- ){ + dataBufferDestroy(&(pBuffers[nBuffers])); + } + if( pBuffers!=NULL ) sqlite3_free(pBuffers); + + return rc; } /* Call loadSegmentLeavesInt() with pData/nData as input. */ @@ -5278,7 +5661,7 @@ static int loadSegmentLeaves(fulltext_vtab *v, /* TODO(shess) The calling code may already know that the end child is ** not worth calculating, because the end may be in a later sibling ** node. Consider whether breaking symmetry is worthwhile. I suspect -** it's not worthwhile. +** it is not worthwhile. */ static void getChildrenContaining(const char *pData, int nData, const char *pTerm, int nTerm, int isPrefix, @@ -5460,9 +5843,14 @@ static int loadSegment(fulltext_vtab *v, const char *pData, int nData, /* Scan the database and merge together the posting lists for the term ** into *out. */ -static int termSelect(fulltext_vtab *v, int iColumn, - const char *pTerm, int nTerm, int isPrefix, - DocListType iType, DataBuffer *out){ +static int termSelect( + fulltext_vtab *v, + int iColumn, + const char *pTerm, int nTerm, /* Term to query for */ + int isPrefix, /* True for a prefix search */ + DocListType iType, + DataBuffer *out /* Write results here */ +){ DataBuffer doclist; sqlite3_stmt *s; int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s); @@ -5472,13 +5860,14 @@ static int termSelect(fulltext_vtab *v, int iColumn, assert( v->nPendingData<0 ); dataBufferInit(&doclist, 0); + dataBufferInit(out, 0); /* Traverse the segments from oldest to newest so that newer doclist ** elements for given docids overwrite older elements. */ while( (rc = sqlite3_step(s))==SQLITE_ROW ){ - const char *pData = sqlite3_column_blob(s, 0); - const int nData = sqlite3_column_bytes(s, 0); + const char *pData = sqlite3_column_blob(s, 2); + const int nData = sqlite3_column_bytes(s, 2); const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix, &doclist); @@ -5526,8 +5915,8 @@ static int termDataCmp(const void *av, const void *bv){ /* Order pTerms data by term, then write a new level 0 segment using ** LeafWriter. */ -static int writeZeroSegment(fulltext_vtab *v, fts2Hash *pTerms){ - fts2HashElem *e; +static int writeZeroSegment(fulltext_vtab *v, fts3Hash *pTerms){ + fts3HashElem *e; int idx, rc, i, n; TermData *pData; LeafWriter writer; @@ -5537,14 +5926,14 @@ static int writeZeroSegment(fulltext_vtab *v, fts2Hash *pTerms){ rc = segdirNextIndex(v, 0, &idx); if( rc!=SQLITE_OK ) return rc; - n = fts2HashCount(pTerms); - pData = malloc(n*sizeof(TermData)); + n = fts3HashCount(pTerms); + pData = sqlite3_malloc(n*sizeof(TermData)); - for(i = 0, e = fts2HashFirst(pTerms); e; i++, e = fts2HashNext(e)){ + for(i = 0, e = fts3HashFirst(pTerms); e; i++, e = fts3HashNext(e)){ assert( inPendingData>=0 ){ - fts2HashElem *e; - for(e=fts2HashFirst(&v->pendingTerms); e; e=fts2HashNext(e)){ - dlcDelete(fts2HashData(e)); + fts3HashElem *e; + for(e=fts3HashFirst(&v->pendingTerms); e; e=fts3HashNext(e)){ + dlcDelete(fts3HashData(e)); } - fts2HashClear(&v->pendingTerms); + fts3HashClear(&v->pendingTerms); v->nPendingData = -1; } return SQLITE_OK; @@ -5614,37 +6003,58 @@ static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){ if( rc!=SQLITE_OK ) return rc; } if( v->nPendingData<0 ){ - fts2HashInit(&v->pendingTerms, FTS2_HASH_STRING, 1); + fts3HashInit(&v->pendingTerms, FTS3_HASH_STRING, 1); v->nPendingData = 0; } v->iPrevDocid = iDocid; return SQLITE_OK; } -/* This function implements the xUpdate callback; it's the top-level entry +/* This function implements the xUpdate callback; it is the top-level entry * point for inserting, deleting or updating a row in a full-text table. */ static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg, - sqlite_int64 *pRowid){ + sqlite_int64 *pRowid){ fulltext_vtab *v = (fulltext_vtab *) pVtab; int rc; - TRACE(("FTS2 Update %p\n", pVtab)); + FTSTRACE(("FTS3 Update %p\n", pVtab)); if( nArg<2 ){ rc = index_delete(v, sqlite3_value_int64(ppArg[0])); + if( rc==SQLITE_OK ){ + /* If we just deleted the last row in the table, clear out the + ** index data. + */ + rc = content_exists(v); + if( rc==SQLITE_ROW ){ + rc = SQLITE_OK; + }else if( rc==SQLITE_DONE ){ + /* Clear the pending terms so we don't flush a useless level-0 + ** segment when the transaction closes. + */ + rc = clearPendingTerms(v); + if( rc==SQLITE_OK ){ + rc = segdir_delete_all(v); + } + } + } } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){ /* An update: * ppArg[0] = old rowid * ppArg[1] = new rowid * ppArg[2..2+v->nColumn-1] = values * ppArg[2+v->nColumn] = value for magic column (we ignore this) + * ppArg[2+v->nColumn+1] = value for docid */ sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]); if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER || - sqlite3_value_int64(ppArg[1]) != rowid ){ + sqlite3_value_int64(ppArg[1]) != rowid ){ rc = SQLITE_ERROR; /* we don't allow changing the rowid */ - } else { - assert( nArg==2+v->nColumn+1); + }else if( sqlite3_value_type(ppArg[2+v->nColumn+1]) != SQLITE_INTEGER || + sqlite3_value_int64(ppArg[2+v->nColumn+1]) != rowid ){ + rc = SQLITE_ERROR; /* we don't allow changing the docid */ + }else{ + assert( nArg==2+v->nColumn+2); rc = index_update(v, rowid, &ppArg[2]); } } else { @@ -5652,22 +6062,39 @@ static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg, * ppArg[1] = requested rowid * ppArg[2..2+v->nColumn-1] = values * ppArg[2+v->nColumn] = value for magic column (we ignore this) + * ppArg[2+v->nColumn+1] = value for docid */ - assert( nArg==2+v->nColumn+1); - rc = index_insert(v, ppArg[1], &ppArg[2], pRowid); + sqlite3_value *pRequestDocid = ppArg[2+v->nColumn+1]; + assert( nArg==2+v->nColumn+2); + if( SQLITE_NULL != sqlite3_value_type(pRequestDocid) && + SQLITE_NULL != sqlite3_value_type(ppArg[1]) ){ + /* TODO(shess) Consider allowing this to work if the values are + ** identical. I'm inclined to discourage that usage, though, + ** given that both rowid and docid are special columns. Better + ** would be to define one or the other as the default winner, + ** but should it be fts3-centric (docid) or SQLite-centric + ** (rowid)? + */ + rc = SQLITE_ERROR; + }else{ + if( SQLITE_NULL == sqlite3_value_type(pRequestDocid) ){ + pRequestDocid = ppArg[1]; + } + rc = index_insert(v, pRequestDocid, &ppArg[2], pRowid); + } } return rc; } static int fulltextSync(sqlite3_vtab *pVtab){ - TRACE(("FTS2 xSync()\n")); + FTSTRACE(("FTS3 xSync()\n")); return flushPendingTerms((fulltext_vtab *)pVtab); } static int fulltextBegin(sqlite3_vtab *pVtab){ fulltext_vtab *v = (fulltext_vtab *) pVtab; - TRACE(("FTS2 xBegin()\n")); + FTSTRACE(("FTS3 xBegin()\n")); /* Any buffered updates should have been cleared by the previous ** transaction. @@ -5678,7 +6105,7 @@ static int fulltextBegin(sqlite3_vtab *pVtab){ static int fulltextCommit(sqlite3_vtab *pVtab){ fulltext_vtab *v = (fulltext_vtab *) pVtab; - TRACE(("FTS2 xCommit()\n")); + FTSTRACE(("FTS3 xCommit()\n")); /* Buffered updates should have been cleared by fulltextSync(). */ assert( v->nPendingData<0 ); @@ -5686,12 +6113,12 @@ static int fulltextCommit(sqlite3_vtab *pVtab){ } static int fulltextRollback(sqlite3_vtab *pVtab){ - TRACE(("FTS2 xRollback()\n")); + FTSTRACE(("FTS3 xRollback()\n")); return clearPendingTerms((fulltext_vtab *)pVtab); } /* -** Implementation of the snippet() function for FTS2 +** Implementation of the snippet() function for FTS3 */ static void snippetFunc( sqlite3_context *pContext, @@ -5725,7 +6152,7 @@ static void snippetFunc( } /* -** Implementation of the offsets() function for FTS2 +** Implementation of the offsets() function for FTS3 */ static void snippetOffsetsFunc( sqlite3_context *pContext, @@ -5747,8 +6174,667 @@ static void snippetOffsetsFunc( } } +/* OptLeavesReader is nearly identical to LeavesReader, except that +** where LeavesReader is geared towards the merging of complete +** segment levels (with exactly MERGE_COUNT segments), OptLeavesReader +** is geared towards implementation of the optimize() function, and +** can merge all segments simultaneously. This version may be +** somewhat less efficient than LeavesReader because it merges into an +** accumulator rather than doing an N-way merge, but since segment +** size grows exponentially (so segment count logrithmically) this is +** probably not an immediate problem. +*/ +/* TODO(shess): Prove that assertion, or extend the merge code to +** merge tree fashion (like the prefix-searching code does). +*/ +/* TODO(shess): OptLeavesReader and LeavesReader could probably be +** merged with little or no loss of performance for LeavesReader. The +** merged code would need to handle >MERGE_COUNT segments, and would +** also need to be able to optionally optimize away deletes. +*/ +typedef struct OptLeavesReader { + /* Segment number, to order readers by age. */ + int segment; + LeavesReader reader; +} OptLeavesReader; + +static int optLeavesReaderAtEnd(OptLeavesReader *pReader){ + return leavesReaderAtEnd(&pReader->reader); +} +static int optLeavesReaderTermBytes(OptLeavesReader *pReader){ + return leavesReaderTermBytes(&pReader->reader); +} +static const char *optLeavesReaderData(OptLeavesReader *pReader){ + return leavesReaderData(&pReader->reader); +} +static int optLeavesReaderDataBytes(OptLeavesReader *pReader){ + return leavesReaderDataBytes(&pReader->reader); +} +static const char *optLeavesReaderTerm(OptLeavesReader *pReader){ + return leavesReaderTerm(&pReader->reader); +} +static int optLeavesReaderStep(fulltext_vtab *v, OptLeavesReader *pReader){ + return leavesReaderStep(v, &pReader->reader); +} +static int optLeavesReaderTermCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){ + return leavesReaderTermCmp(&lr1->reader, &lr2->reader); +} +/* Order by term ascending, segment ascending (oldest to newest), with +** exhausted readers to the end. +*/ +static int optLeavesReaderCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){ + int c = optLeavesReaderTermCmp(lr1, lr2); + if( c!=0 ) return c; + return lr1->segment-lr2->segment; +} +/* Bubble pLr[0] to appropriate place in pLr[1..nLr-1]. Assumes that +** pLr[1..nLr-1] is already sorted. +*/ +static void optLeavesReaderReorder(OptLeavesReader *pLr, int nLr){ + while( nLr>1 && optLeavesReaderCmp(pLr, pLr+1)>0 ){ + OptLeavesReader tmp = pLr[0]; + pLr[0] = pLr[1]; + pLr[1] = tmp; + nLr--; + pLr++; + } +} + +/* optimize() helper function. Put the readers in order and iterate +** through them, merging doclists for matching terms into pWriter. +** Returns SQLITE_OK on success, or the SQLite error code which +** prevented success. +*/ +static int optimizeInternal(fulltext_vtab *v, + OptLeavesReader *readers, int nReaders, + LeafWriter *pWriter){ + int i, rc = SQLITE_OK; + DataBuffer doclist, merged, tmp; + + /* Order the readers. */ + i = nReaders; + while( i-- > 0 ){ + optLeavesReaderReorder(&readers[i], nReaders-i); + } + + dataBufferInit(&doclist, LEAF_MAX); + dataBufferInit(&merged, LEAF_MAX); + + /* Exhausted readers bubble to the end, so when the first reader is + ** at eof, all are at eof. + */ + while( !optLeavesReaderAtEnd(&readers[0]) ){ + + /* Figure out how many readers share the next term. */ + for(i=1; i 0 ){ + dlrDestroy(&dlReaders[nReaders]); + } + + /* Accumulated doclist to reader 0 for next pass. */ + dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData); + } + + /* Destroy reader that was left in the pipeline. */ + dlrDestroy(&dlReaders[0]); + + /* Trim deletions from the doclist. */ + dataBufferReset(&merged); + docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, + -1, DL_DEFAULT, &merged); + } + + /* Only pass doclists with hits (skip if all hits deleted). */ + if( merged.nData>0 ){ + rc = leafWriterStep(v, pWriter, + optLeavesReaderTerm(&readers[0]), + optLeavesReaderTermBytes(&readers[0]), + merged.pData, merged.nData); + if( rc!=SQLITE_OK ) goto err; + } + + /* Step merged readers to next term and reorder. */ + while( i-- > 0 ){ + rc = optLeavesReaderStep(v, &readers[i]); + if( rc!=SQLITE_OK ) goto err; + + optLeavesReaderReorder(&readers[i], nReaders-i); + } + } + + err: + dataBufferDestroy(&doclist); + dataBufferDestroy(&merged); + return rc; +} + +/* Implement optimize() function for FTS3. optimize(t) merges all +** segments in the fts index into a single segment. 't' is the magic +** table-named column. +*/ +static void optimizeFunc(sqlite3_context *pContext, + int argc, sqlite3_value **argv){ + fulltext_cursor *pCursor; + if( argc>1 ){ + sqlite3_result_error(pContext, "excess arguments to optimize()",-1); + }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || + sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ + sqlite3_result_error(pContext, "illegal first argument to optimize",-1); + }else{ + fulltext_vtab *v; + int i, rc, iMaxLevel; + OptLeavesReader *readers; + int nReaders; + LeafWriter writer; + sqlite3_stmt *s; + + memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); + v = cursor_vtab(pCursor); + + /* Flush any buffered updates before optimizing. */ + rc = flushPendingTerms(v); + if( rc!=SQLITE_OK ) goto err; + + rc = segdir_count(v, &nReaders, &iMaxLevel); + if( rc!=SQLITE_OK ) goto err; + if( nReaders==0 || nReaders==1 ){ + sqlite3_result_text(pContext, "Index already optimal", -1, + SQLITE_STATIC); + return; + } + + rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s); + if( rc!=SQLITE_OK ) goto err; + + readers = sqlite3_malloc(nReaders*sizeof(readers[0])); + if( readers==NULL ) goto err; + + /* Note that there will already be a segment at this position + ** until we call segdir_delete() on iMaxLevel. + */ + leafWriterInit(iMaxLevel, 0, &writer); + + i = 0; + while( (rc = sqlite3_step(s))==SQLITE_ROW ){ + sqlite_int64 iStart = sqlite3_column_int64(s, 0); + sqlite_int64 iEnd = sqlite3_column_int64(s, 1); + const char *pRootData = sqlite3_column_blob(s, 2); + int nRootData = sqlite3_column_bytes(s, 2); + + assert( i 0 ){ + leavesReaderDestroy(&readers[i].reader); + } + sqlite3_free(readers); + + /* If we've successfully gotten to here, delete the old segments + ** and flush the interior structure of the new segment. + */ + if( rc==SQLITE_OK ){ + for( i=0; i<=iMaxLevel; i++ ){ + rc = segdir_delete(v, i); + if( rc!=SQLITE_OK ) break; + } + + if( rc==SQLITE_OK ) rc = leafWriterFinalize(v, &writer); + } + + leafWriterDestroy(&writer); + + if( rc!=SQLITE_OK ) goto err; + + sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); + return; + + /* TODO(shess): Error-handling needs to be improved along the + ** lines of the dump_ functions. + */ + err: + { + char buf[512]; + sqlite3_snprintf(sizeof(buf), buf, "Error in optimize: %s", + sqlite3_errmsg(sqlite3_context_db_handle(pContext))); + sqlite3_result_error(pContext, buf, -1); + } + } +} + +#ifdef SQLITE_TEST +/* Generate an error of the form ": ". If msg is NULL, +** pull the error from the context's db handle. +*/ +static void generateError(sqlite3_context *pContext, + const char *prefix, const char *msg){ + char buf[512]; + if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext)); + sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg); + sqlite3_result_error(pContext, buf, -1); +} + +/* Helper function to collect the set of terms in the segment into +** pTerms. The segment is defined by the leaf nodes between +** iStartBlockid and iEndBlockid, inclusive, or by the contents of +** pRootData if iStartBlockid is 0 (in which case the entire segment +** fit in a leaf). +*/ +static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s, + fts3Hash *pTerms){ + const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0); + const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1); + const char *pRootData = sqlite3_column_blob(s, 2); + const int nRootData = sqlite3_column_bytes(s, 2); + LeavesReader reader; + int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid, + pRootData, nRootData, &reader); + if( rc!=SQLITE_OK ) return rc; + + while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){ + const char *pTerm = leavesReaderTerm(&reader); + const int nTerm = leavesReaderTermBytes(&reader); + void *oldValue = sqlite3Fts3HashFind(pTerms, pTerm, nTerm); + void *newValue = (void *)((char *)oldValue+1); + + /* From the comment before sqlite3Fts3HashInsert in fts3_hash.c, + ** the data value passed is returned in case of malloc failure. + */ + if( newValue==sqlite3Fts3HashInsert(pTerms, pTerm, nTerm, newValue) ){ + rc = SQLITE_NOMEM; + }else{ + rc = leavesReaderStep(v, &reader); + } + } + + leavesReaderDestroy(&reader); + return rc; +} + +/* Helper function to build the result string for dump_terms(). */ +static int generateTermsResult(sqlite3_context *pContext, fts3Hash *pTerms){ + int iTerm, nTerms, nResultBytes, iByte; + char *result; + TermData *pData; + fts3HashElem *e; + + /* Iterate pTerms to generate an array of terms in pData for + ** sorting. + */ + nTerms = fts3HashCount(pTerms); + assert( nTerms>0 ); + pData = sqlite3_malloc(nTerms*sizeof(TermData)); + if( pData==NULL ) return SQLITE_NOMEM; + + nResultBytes = 0; + for(iTerm = 0, e = fts3HashFirst(pTerms); e; iTerm++, e = fts3HashNext(e)){ + nResultBytes += fts3HashKeysize(e)+1; /* Term plus trailing space */ + assert( iTerm0 ); /* nTerms>0, nResultsBytes must be, too. */ + result = sqlite3_malloc(nResultBytes); + if( result==NULL ){ + sqlite3_free(pData); + return SQLITE_NOMEM; + } + + if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp); + + /* Read the terms in order to build the result. */ + iByte = 0; + for(iTerm=0; iTerm0 ){ + rc = generateTermsResult(pContext, &terms); + if( rc==SQLITE_NOMEM ){ + generateError(pContext, "dump_terms", "out of memory"); + }else{ + assert( rc==SQLITE_OK ); + } + }else if( argc==3 ){ + /* The specific segment asked for could not be found. */ + generateError(pContext, "dump_terms", "segment not found"); + }else{ + /* No segments found. */ + /* TODO(shess): It should be impossible to reach this. This + ** case can only happen for an empty table, in which case + ** SQLite has no rows to call this function on. + */ + sqlite3_result_null(pContext); + } + } + sqlite3Fts3HashClear(&terms); + } +} + +/* Expand the DL_DEFAULT doclist in pData into a text result in +** pContext. +*/ +static void createDoclistResult(sqlite3_context *pContext, + const char *pData, int nData){ + DataBuffer dump; + DLReader dlReader; + + assert( pData!=NULL && nData>0 ); + + dataBufferInit(&dump, 0); + dlrInit(&dlReader, DL_DEFAULT, pData, nData); + for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){ + char buf[256]; + PLReader plReader; + + plrInit(&plReader, &dlReader); + if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){ + sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader)); + dataBufferAppend(&dump, buf, strlen(buf)); + }else{ + int iColumn = plrColumn(&plReader); + + sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[", + dlrDocid(&dlReader), iColumn); + dataBufferAppend(&dump, buf, strlen(buf)); + + for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){ + if( plrColumn(&plReader)!=iColumn ){ + iColumn = plrColumn(&plReader); + sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn); + assert( dump.nData>0 ); + dump.nData--; /* Overwrite trailing space. */ + assert( dump.pData[dump.nData]==' '); + dataBufferAppend(&dump, buf, strlen(buf)); + } + if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){ + sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ", + plrPosition(&plReader), + plrStartOffset(&plReader), plrEndOffset(&plReader)); + }else if( DL_DEFAULT==DL_POSITIONS ){ + sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader)); + }else{ + assert( NULL=="Unhandled DL_DEFAULT value"); + } + dataBufferAppend(&dump, buf, strlen(buf)); + } + plrDestroy(&plReader); + + assert( dump.nData>0 ); + dump.nData--; /* Overwrite trailing space. */ + assert( dump.pData[dump.nData]==' '); + dataBufferAppend(&dump, "]] ", 3); + } + } + dlrDestroy(&dlReader); + + assert( dump.nData>0 ); + dump.nData--; /* Overwrite trailing space. */ + assert( dump.pData[dump.nData]==' '); + dump.pData[dump.nData] = '\0'; + assert( dump.nData>0 ); + + /* Passes ownership of dump's buffer to pContext. */ + sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free); + dump.pData = NULL; + dump.nData = dump.nCapacity = 0; +} + +/* Implements dump_doclist() for use in inspecting the fts3 index from +** tests. TEXT result containing a string representation of the +** doclist for the indicated term. dump_doclist(t, term, level, idx) +** dumps the doclist for term from the segment specified by level, idx +** (in %_segdir), while dump_doclist(t, term) dumps the logical +** doclist for the term across all segments. The per-segment doclist +** can contain deletions, while the full-index doclist will not +** (deletions are omitted). +** +** Result formats differ with the setting of DL_DEFAULTS. Examples: +** +** DL_DOCIDS: [1] [3] [7] +** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]] +** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]] +** +** In each case the number after the outer '[' is the docid. In the +** latter two cases, the number before the inner '[' is the column +** associated with the values within. For DL_POSITIONS the numbers +** within are the positions, for DL_POSITIONS_OFFSETS they are the +** position, the start offset, and the end offset. +*/ +static void dumpDoclistFunc( + sqlite3_context *pContext, + int argc, sqlite3_value **argv +){ + fulltext_cursor *pCursor; + if( argc!=2 && argc!=4 ){ + generateError(pContext, "dump_doclist", "incorrect arguments"); + }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || + sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ + generateError(pContext, "dump_doclist", "illegal first argument"); + }else if( sqlite3_value_text(argv[1])==NULL || + sqlite3_value_text(argv[1])[0]=='\0' ){ + generateError(pContext, "dump_doclist", "empty second argument"); + }else{ + const char *pTerm = (const char *)sqlite3_value_text(argv[1]); + const int nTerm = strlen(pTerm); + fulltext_vtab *v; + int rc; + DataBuffer doclist; + + memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); + v = cursor_vtab(pCursor); + + dataBufferInit(&doclist, 0); + + /* termSelect() yields the same logical doclist that queries are + ** run against. + */ + if( argc==2 ){ + rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist); + }else{ + sqlite3_stmt *s = NULL; + + /* Get our specific segment's information. */ + rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2])); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3])); + } + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_step(s); + + if( rc==SQLITE_DONE ){ + dataBufferDestroy(&doclist); + generateError(pContext, "dump_doclist", "segment not found"); + return; + } + + /* Found a segment, load it into doclist. */ + if( rc==SQLITE_ROW ){ + const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); + const char *pData = sqlite3_column_blob(s, 2); + const int nData = sqlite3_column_bytes(s, 2); + + /* loadSegment() is used by termSelect() to load each + ** segment's data. + */ + rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0, + &doclist); + if( rc==SQLITE_OK ){ + rc = sqlite3_step(s); + + /* Should not have more than one matching segment. */ + if( rc!=SQLITE_DONE ){ + sqlite3_reset(s); + dataBufferDestroy(&doclist); + generateError(pContext, "dump_doclist", "invalid segdir"); + return; + } + rc = SQLITE_OK; + } + } + } + + sqlite3_reset(s); + } + + if( rc==SQLITE_OK ){ + if( doclist.nData>0 ){ + createDoclistResult(pContext, doclist.pData, doclist.nData); + }else{ + /* TODO(shess): This can happen if the term is not present, or + ** if all instances of the term have been deleted and this is + ** an all-index dump. It may be interesting to distinguish + ** these cases. + */ + sqlite3_result_text(pContext, "", 0, SQLITE_STATIC); + } + }else if( rc==SQLITE_NOMEM ){ + /* Handle out-of-memory cases specially because if they are + ** generated in fts3 code they may not be reflected in the db + ** handle. + */ + /* TODO(shess): Handle this more comprehensively. + ** sqlite3ErrStr() has what I need, but is internal. + */ + generateError(pContext, "dump_doclist", "out of memory"); + }else{ + generateError(pContext, "dump_doclist", NULL); + } + + dataBufferDestroy(&doclist); + } +} +#endif + /* -** This routine implements the xFindFunction method for the FTS2 +** This routine implements the xFindFunction method for the FTS3 ** virtual table. */ static int fulltextFindFunction( @@ -5764,12 +6850,29 @@ static int fulltextFindFunction( }else if( strcmp(zName,"offsets")==0 ){ *pxFunc = snippetOffsetsFunc; return 1; + }else if( strcmp(zName,"optimize")==0 ){ + *pxFunc = optimizeFunc; + return 1; +#ifdef SQLITE_TEST + /* NOTE(shess): These functions are present only for testing + ** purposes. No particular effort is made to optimize their + ** execution or how they build their results. + */ + }else if( strcmp(zName,"dump_terms")==0 ){ + /* fprintf(stderr, "Found dump_terms\n"); */ + *pxFunc = dumpTermsFunc; + return 1; + }else if( strcmp(zName,"dump_doclist")==0 ){ + /* fprintf(stderr, "Found dump_doclist\n"); */ + *pxFunc = dumpDoclistFunc; + return 1; +#endif } return 0; } /* -** Rename an fts2 table. +** Rename an fts3 table. */ static int fulltextRename( sqlite3_vtab *pVtab, @@ -5792,7 +6895,7 @@ static int fulltextRename( return rc; } -static const sqlite3_module fts2Module = { +static const sqlite3_module fts3Module = { /* iVersion */ 0, /* xCreate */ fulltextCreate, /* xConnect */ fulltextConnect, @@ -5816,83 +6919,92 @@ static const sqlite3_module fts2Module = { }; static void hashDestroy(void *p){ - fts2Hash *pHash = (fts2Hash *)p; - sqlite3Fts2HashClear(pHash); + fts3Hash *pHash = (fts3Hash *)p; + sqlite3Fts3HashClear(pHash); sqlite3_free(pHash); } /* -** The fts2 built-in tokenizers - "simple" and "porter" - are implemented -** in files fts2_tokenizer1.c and fts2_porter.c respectively. The following +** The fts3 built-in tokenizers - "simple" and "porter" - are implemented +** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following ** two forward declarations are for functions declared in these files ** used to retrieve the respective implementations. ** -** Calling sqlite3Fts2SimpleTokenizerModule() sets the value pointed +** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed ** to by the argument to point a the "simple" tokenizer implementation. ** Function ...PorterTokenizerModule() sets *pModule to point to the ** porter tokenizer/stemmer implementation. */ -void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); -void sqlite3Fts2PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); -void sqlite3Fts2IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); +void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); +void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); -int sqlite3Fts2InitHashTable(sqlite3 *, fts2Hash *, const char *); +int sqlite3Fts3InitHashTable(sqlite3 *, fts3Hash *, const char *); /* -** Initialise the fts2 extension. If this extension is built as part +** Initialise the fts3 extension. If this extension is built as part ** of the sqlite library, then this function is called directly by -** SQLite. If fts2 is built as a dynamically loadable extension, this +** SQLite. If fts3 is built as a dynamically loadable extension, this ** function is called by the sqlite3_extension_init() entry point. */ -int sqlite3Fts2Init(sqlite3 *db){ +int sqlite3Fts3Init(sqlite3 *db){ int rc = SQLITE_OK; - fts2Hash *pHash = 0; + fts3Hash *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; const sqlite3_tokenizer_module *pIcu = 0; - sqlite3Fts2SimpleTokenizerModule(&pSimple); - sqlite3Fts2PorterTokenizerModule(&pPorter); + sqlite3Fts3SimpleTokenizerModule(&pSimple); + sqlite3Fts3PorterTokenizerModule(&pPorter); #ifdef SQLITE_ENABLE_ICU - sqlite3Fts2IcuTokenizerModule(&pIcu); + sqlite3Fts3IcuTokenizerModule(&pIcu); #endif /* Allocate and initialise the hash-table used to store tokenizers. */ - pHash = sqlite3_malloc(sizeof(fts2Hash)); + pHash = sqlite3_malloc(sizeof(fts3Hash)); if( !pHash ){ rc = SQLITE_NOMEM; }else{ - sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1); + sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); } /* Load the built-in tokenizers into the hash table */ if( rc==SQLITE_OK ){ - if( sqlite3Fts2HashInsert(pHash, "simple", 7, (void *)pSimple) - || sqlite3Fts2HashInsert(pHash, "porter", 7, (void *)pPorter) - || (pIcu && sqlite3Fts2HashInsert(pHash, "icu", 4, (void *)pIcu)) + if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) + || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) + || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) ){ rc = SQLITE_NOMEM; } } +#ifdef SQLITE_TEST + sqlite3Fts3ExprInitTestInterface(db); +#endif + /* Create the virtual table wrapper around the hash-table and overload ** the two scalar functions. If this is successful, register the ** module with sqlite. */ if( SQLITE_OK==rc - && SQLITE_OK==(rc = sqlite3Fts2InitHashTable(db, pHash, "fts2_tokenizer")) + && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1)) +#ifdef SQLITE_TEST + && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1)) +#endif ){ return sqlite3_create_module_v2( - db, "fts2", &fts2Module, (void *)pHash, hashDestroy + db, "fts3", &fts3Module, (void *)pHash, hashDestroy ); } /* An error has occured. Delete the hash table and return the error code. */ assert( rc!=SQLITE_OK ); if( pHash ){ - sqlite3Fts2HashClear(pHash); + sqlite3Fts3HashClear(pHash); sqlite3_free(pHash); } return rc; @@ -5905,8 +7017,8 @@ int sqlite3_extension_init( const sqlite3_api_routines *pApi ){ SQLITE_EXTENSION_INIT2(pApi) - return sqlite3Fts2Init(db); + return sqlite3Fts3Init(db); } #endif -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/fts2.h b/fts3.h similarity index 86% rename from fts2.h rename to fts3.h index 4da4c38..c1aa8ca 100644 --- a/fts2.h +++ b/fts3.h @@ -11,7 +11,7 @@ ****************************************************************************** ** ** This header file is used by programs that want to link against the -** FTS2 library. All it does is declare the sqlite3Fts2Init() interface. +** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. */ #include "sqlite3.h" @@ -19,7 +19,7 @@ extern "C" { #endif /* __cplusplus */ -int sqlite3Fts2Init(sqlite3 *db); +int sqlite3Fts3Init(sqlite3 *db); #ifdef __cplusplus } /* extern "C" */ diff --git a/fts3_expr.c b/fts3_expr.c new file mode 100644 index 0000000..eb4d4f8 --- /dev/null +++ b/fts3_expr.c @@ -0,0 +1,890 @@ +/* +** 2008 Nov 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This module contains code that implements a parser for fts3 query strings +** (the right-hand argument to the MATCH operator). Because the supported +** syntax is relatively simple, the whole tokenizer/parser system is +** hand-coded. The public interface to this module is declared in source +** code file "fts3_expr.h". +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* +** By default, this module parses the legacy syntax that has been +** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined, then it uses the new syntax. The differences between +** the new and the old syntaxes are: +** +** a) The new syntax supports parenthesis. The old does not. +** +** b) The new syntax supports the AND and NOT operators. The old does not. +** +** c) The old syntax supports the "-" token qualifier. This is not +** supported by the new syntax (it is replaced by the NOT operator). +** +** d) When using the old syntax, the OR operator has a greater precedence +** than an implicit AND. When using the new, both implicity and explicit +** AND operators have a higher precedence than OR. +** +** If compiled with SQLITE_TEST defined, then this module exports the +** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable +** to zero causes the module to use the old syntax. If it is set to +** non-zero the new syntax is activated. This is so both syntaxes can +** be tested using a single build of testfixture. +*/ +#ifdef SQLITE_TEST +int sqlite3_fts3_enable_parentheses = 0; +#else +# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS +# define sqlite3_fts3_enable_parentheses 1 +# else +# define sqlite3_fts3_enable_parentheses 0 +# endif +#endif + +/* +** Default span for NEAR operators. +*/ +#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 + +#include "fts3_expr.h" +#include "sqlite3.h" +#include +#include +#include + +typedef struct ParseContext ParseContext; +struct ParseContext { + sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ + const char **azCol; /* Array of column names for fts3 table */ + int nCol; /* Number of entries in azCol[] */ + int iDefaultCol; /* Default column to query */ + sqlite3_context *pCtx; /* Write error message here */ + int nNest; /* Number of nested brackets */ +}; + +/* +** This function is equivalent to the standard isspace() function. +** +** The standard isspace() can be awkward to use safely, because although it +** is defined to accept an argument of type int, its behaviour when passed +** an integer that falls outside of the range of the unsigned char type +** is undefined (and sometimes, "undefined" means segfault). This wrapper +** is defined to accept an argument of type char, and always returns 0 for +** any values that fall outside of the range of the unsigned char type (i.e. +** negative values). +*/ +static int fts3isspace(char c){ + return (c&0x80)==0 ? isspace(c) : 0; +} + +/* +** Extract the next token from buffer z (length n) using the tokenizer +** and other information (column names etc.) in pParse. Create an Fts3Expr +** structure of type FTSQUERY_PHRASE containing a phrase consisting of this +** single token and set *ppExpr to point to it. If the end of the buffer is +** reached before a token is found, set *ppExpr to zero. It is the +** responsibility of the caller to eventually deallocate the allocated +** Fts3Expr structure (if any) by passing it to sqlite3_free(). +** +** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation +** fails. +*/ +static int getNextToken( + ParseContext *pParse, /* fts3 query parse context */ + int iCol, /* Value for Fts3Phrase.iColumn */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + sqlite3_tokenizer_cursor *pCursor; + Fts3Expr *pRet = 0; + int nConsumed = 0; + + rc = pModule->xOpen(pTokenizer, z, n, &pCursor); + if( rc==SQLITE_OK ){ + const char *zToken; + int nToken, iStart, iEnd, iPosition; + int nByte; /* total space to allocate */ + + pCursor->pTokenizer = pTokenizer; + rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); + + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; + pRet = (Fts3Expr *)sqlite3_malloc(nByte); + if( !pRet ){ + rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + pRet->eType = FTSQUERY_PHRASE; + pRet->pPhrase = (Fts3Phrase *)&pRet[1]; + pRet->pPhrase->nToken = 1; + pRet->pPhrase->iColumn = iCol; + pRet->pPhrase->aToken[0].n = nToken; + pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; + memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); + + if( iEndpPhrase->aToken[0].isPrefix = 1; + iEnd++; + } + if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ + pRet->pPhrase->isNot = 1; + } + } + } + nConsumed = iEnd; + + pModule->xClose(pCursor); + } + + *pnConsumed = nConsumed; + *ppExpr = pRet; + return rc; +} + + +/* +** Enlarge a memory allocation. If an out-of-memory allocation occurs, +** then free the old allocation. +*/ +void *fts3ReallocOrFree(void *pOrig, int nNew){ + void *pRet = sqlite3_realloc(pOrig, nNew); + if( !pRet ){ + sqlite3_free(pOrig); + } + return pRet; +} + +/* +** Buffer zInput, length nInput, contains the contents of a quoted string +** that appeared as part of an fts3 query expression. Neither quote character +** is included in the buffer. This function attempts to tokenize the entire +** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE +** containing the results. +** +** If successful, SQLITE_OK is returned and *ppExpr set to point at the +** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory +** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set +** to 0. +*/ +static int getNextString( + ParseContext *pParse, /* fts3 query parse context */ + const char *zInput, int nInput, /* Input string */ + Fts3Expr **ppExpr /* OUT: expression */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + Fts3Expr *p = 0; + sqlite3_tokenizer_cursor *pCursor = 0; + char *zTemp = 0; + int nTemp = 0; + + rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor); + if( rc==SQLITE_OK ){ + int ii; + pCursor->pTokenizer = pTokenizer; + for(ii=0; rc==SQLITE_OK; ii++){ + const char *zToken; + int nToken, iBegin, iEnd, iPos; + rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); + if( rc==SQLITE_OK ){ + int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + p = fts3ReallocOrFree(p, nByte+ii*sizeof(struct PhraseToken)); + zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken); + if( !p || !zTemp ){ + goto no_mem; + } + if( ii==0 ){ + memset(p, 0, nByte); + p->pPhrase = (Fts3Phrase *)&p[1]; + p->eType = FTSQUERY_PHRASE; + p->pPhrase->iColumn = pParse->iDefaultCol; + } + p->pPhrase = (Fts3Phrase *)&p[1]; + p->pPhrase->nToken = ii+1; + p->pPhrase->aToken[ii].n = nToken; + memcpy(&zTemp[nTemp], zToken, nToken); + nTemp += nToken; + if( iEndpPhrase->aToken[ii].isPrefix = 1; + }else{ + p->pPhrase->aToken[ii].isPrefix = 0; + } + } + } + + pModule->xClose(pCursor); + pCursor = 0; + } + + if( rc==SQLITE_DONE ){ + int jj; + char *zNew; + int nNew = 0; + int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + nByte += (p->pPhrase->nToken-1) * sizeof(struct PhraseToken); + p = fts3ReallocOrFree(p, nByte + nTemp); + if( !p ){ + goto no_mem; + } + p->pPhrase = (Fts3Phrase *)&p[1]; + zNew = &(((char *)p)[nByte]); + memcpy(zNew, zTemp, nTemp); + for(jj=0; jjpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = &zNew[nNew]; + nNew += p->pPhrase->aToken[jj].n; + } + sqlite3_free(zTemp); + rc = SQLITE_OK; + } + + *ppExpr = p; + return rc; +no_mem: + + if( pCursor ){ + pModule->xClose(pCursor); + } + sqlite3_free(zTemp); + sqlite3_free(p); + *ppExpr = 0; + return SQLITE_NOMEM; +} + +/* +** Function getNextNode(), which is called by fts3ExprParse(), may itself +** call fts3ExprParse(). So this forward declaration is required. +*/ +static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); + +/* +** The output variable *ppExpr is populated with an allocated Fts3Expr +** structure, or set to 0 if the end of the input buffer is reached. +** +** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM +** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered. +** If SQLITE_ERROR is returned, pContext is populated with an error message. +*/ +static int getNextNode( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + static const struct Fts3Keyword { + char z[4]; /* Keyword text */ + unsigned char n; /* Length of the keyword */ + unsigned char parenOnly; /* Only valid in paren mode */ + unsigned char eType; /* Keyword code */ + } aKeyword[] = { + { "OR" , 2, 0, FTSQUERY_OR }, + { "AND", 3, 1, FTSQUERY_AND }, + { "NOT", 3, 1, FTSQUERY_NOT }, + { "NEAR", 4, 0, FTSQUERY_NEAR } + }; + int ii; + int iCol; + int iColLen; + int rc; + Fts3Expr *pRet = 0; + + const char *zInput = z; + int nInput = n; + + /* Skip over any whitespace before checking for a keyword, an open or + ** close bracket, or a quoted string. + */ + while( nInput>0 && fts3isspace(*zInput) ){ + nInput--; + zInput++; + } + if( nInput==0 ){ + return SQLITE_DONE; + } + + /* See if we are dealing with a keyword. */ + for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){ + const struct Fts3Keyword *pKey = &aKeyword[ii]; + + if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){ + continue; + } + + if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){ + int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM; + int nKey = pKey->n; + char cNext; + + /* If this is a "NEAR" keyword, check for an explicit nearness. */ + if( pKey->eType==FTSQUERY_NEAR ){ + assert( nKey==4 ); + if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ + nNear = 0; + for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){ + nNear = nNear * 10 + (zInput[nKey] - '0'); + } + } + } + + /* At this point this is probably a keyword. But for that to be true, + ** the next byte must contain either whitespace, an open or close + ** parenthesis, a quote character, or EOF. + */ + cNext = zInput[nKey]; + if( fts3isspace(cNext) + || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 + ){ + pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr)); + memset(pRet, 0, sizeof(Fts3Expr)); + pRet->eType = pKey->eType; + pRet->nNear = nNear; + *ppExpr = pRet; + *pnConsumed = (zInput - z) + nKey; + return SQLITE_OK; + } + + /* Turns out that wasn't a keyword after all. This happens if the + ** user has supplied a token such as "ORacle". Continue. + */ + } + } + + /* Check for an open bracket. */ + if( sqlite3_fts3_enable_parentheses ){ + if( *zInput=='(' ){ + int nConsumed; + int rc; + pParse->nNest++; + rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed); + if( rc==SQLITE_OK && !*ppExpr ){ + rc = SQLITE_DONE; + } + *pnConsumed = (zInput - z) + 1 + nConsumed; + return rc; + } + + /* Check for a close bracket. */ + if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (zInput - z) + 1; + return SQLITE_DONE; + } + } + + /* See if we are dealing with a quoted phrase. If this is the case, then + ** search for the closing quote and pass the whole string to getNextString() + ** for processing. This is easy to do, as fts3 has no syntax for escaping + ** a quote character embedded in a string. + */ + if( *zInput=='"' ){ + for(ii=1; iiiDefaultCol; + iColLen = 0; + for(ii=0; iinCol; ii++){ + const char *zStr = pParse->azCol[ii]; + int nStr = strlen(zStr); + if( nInput>nStr && zInput[nStr]==':' && memcmp(zStr, zInput, nStr)==0 ){ + iCol = ii; + iColLen = ((zInput - z) + nStr + 1); + break; + } + } + rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed); + *pnConsumed += iColLen; + return rc; +} + +/* +** The argument is an Fts3Expr structure for a binary operator (any type +** except an FTSQUERY_PHRASE). Return an integer value representing the +** precedence of the operator. Lower values have a higher precedence (i.e. +** group more tightly). For example, in the C language, the == operator +** groups more tightly than ||, and would therefore have a higher precedence. +** +** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined), the order of the operators in precedence from highest to +** lowest is: +** +** NEAR +** NOT +** AND (including implicit ANDs) +** OR +** +** Note that when using the old query syntax, the OR operator has a higher +** precedence than the AND operator. +*/ +static int opPrecedence(Fts3Expr *p){ + assert( p->eType!=FTSQUERY_PHRASE ); + if( sqlite3_fts3_enable_parentheses ){ + return p->eType; + }else if( p->eType==FTSQUERY_NEAR ){ + return 1; + }else if( p->eType==FTSQUERY_OR ){ + return 2; + } + assert( p->eType==FTSQUERY_AND ); + return 3; +} + +/* +** Argument ppHead contains a pointer to the current head of a query +** expression tree being parsed. pPrev is the expression node most recently +** inserted into the tree. This function adds pNew, which is always a binary +** operator node, into the expression tree based on the relative precedence +** of pNew and the existing nodes of the tree. This may result in the head +** of the tree changing, in which case *ppHead is set to the new root node. +*/ +static void insertBinaryOperator( + Fts3Expr **ppHead, /* Pointer to the root node of a tree */ + Fts3Expr *pPrev, /* Node most recently inserted into the tree */ + Fts3Expr *pNew /* New binary node to insert into expression tree */ +){ + Fts3Expr *pSplit = pPrev; + while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){ + pSplit = pSplit->pParent; + } + + if( pSplit->pParent ){ + assert( pSplit->pParent->pRight==pSplit ); + pSplit->pParent->pRight = pNew; + pNew->pParent = pSplit->pParent; + }else{ + *ppHead = pNew; + } + pNew->pLeft = pSplit; + pSplit->pParent = pNew; +} + +/* +** Parse the fts3 query expression found in buffer z, length n. This function +** returns either when the end of the buffer is reached or an unmatched +** closing bracket - ')' - is encountered. +** +** If successful, SQLITE_OK is returned, *ppExpr is set to point to the +** parsed form of the expression and *pnConsumed is set to the number of +** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM +** (out of memory error) or SQLITE_ERROR (parse error) is returned. +*/ +static int fts3ExprParse( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + Fts3Expr *pRet = 0; + Fts3Expr *pPrev = 0; + Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */ + int nIn = n; + const char *zIn = z; + int rc = SQLITE_OK; + int isRequirePhrase = 1; + + while( rc==SQLITE_OK ){ + Fts3Expr *p = 0; + int nByte = 0; + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + if( rc==SQLITE_OK ){ + int isPhrase; + + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = sqlite3_malloc(sizeof(Fts3Expr)); + if( !pNot ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + memset(pNot, 0, sizeof(Fts3Expr)); + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + if( pNotBranch ){ + pNotBranch->pLeft = p; + pNot->pRight = pNotBranch; + } + pNotBranch = pNot; + }else{ + int eType = p->eType; + assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot ); + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); + + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = sqlite3_malloc(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + memset(pAnd, 0, sizeof(Fts3Expr)); + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; + } + + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( + (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) + || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } + }else{ + insertBinaryOperator(&pRet, pPrev, p); + } + isRequirePhrase = !isPhrase; + } + assert( nByte>0 ); + } + assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); + nIn -= nByte; + zIn += nByte; + pPrev = p; + } + + if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + if( !sqlite3_fts3_enable_parentheses && pNotBranch ){ + if( !pRet ){ + rc = SQLITE_ERROR; + }else{ + pNotBranch->pLeft = pRet; + pRet = pNotBranch; + } + } + } + *pnConsumed = n - nIn; + +exprparse_out: + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRet); + sqlite3Fts3ExprFree(pNotBranch); + pRet = 0; + } + *ppExpr = pRet; + return rc; +} + +/* +** Parameters z and n contain a pointer to and length of a buffer containing +** an fts3 query expression, respectively. This function attempts to parse the +** query expression and create a tree of Fts3Expr structures representing the +** parsed expression. If successful, *ppExpr is set to point to the head +** of the parsed expression tree and SQLITE_OK is returned. If an error +** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse +** error) is returned and *ppExpr is set to 0. +** +** If parameter n is a negative number, then z is assumed to point to a +** nul-terminated string and the length is determined using strlen(). +** +** The first parameter, pTokenizer, is passed the fts3 tokenizer module to +** use to normalize query tokens while parsing the expression. The azCol[] +** array, which is assumed to contain nCol entries, should contain the names +** of each column in the target fts3 table, in order from left to right. +** Column names must be nul-terminated strings. +** +** The iDefaultCol parameter should be passed the index of the table column +** that appears on the left-hand-side of the MATCH operator (the default +** column to match against for tokens for which a column name is not explicitly +** specified as part of the query string), or -1 if tokens may by default +** match any table column. +*/ +int sqlite3Fts3ExprParse( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + char **azCol, /* Array of column names for fts3 table */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr /* OUT: Parsed query structure */ +){ + int nParsed; + int rc; + ParseContext sParse; + sParse.pTokenizer = pTokenizer; + sParse.azCol = (const char **)azCol; + sParse.nCol = nCol; + sParse.iDefaultCol = iDefaultCol; + sParse.nNest = 0; + if( z==0 ){ + *ppExpr = 0; + return SQLITE_OK; + } + if( n<0 ){ + n = strlen(z); + } + rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); + + /* Check for mismatched parenthesis */ + if( rc==SQLITE_OK && sParse.nNest ){ + rc = SQLITE_ERROR; + sqlite3Fts3ExprFree(*ppExpr); + *ppExpr = 0; + } + + return rc; +} + +/* +** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +*/ +void sqlite3Fts3ExprFree(Fts3Expr *p){ + if( p ){ + sqlite3Fts3ExprFree(p->pLeft); + sqlite3Fts3ExprFree(p->pRight); + sqlite3_free(p); + } +} + +/**************************************************************************** +***************************************************************************** +** Everything after this point is just test code. +*/ + +#ifdef SQLITE_TEST + +#include + +/* +** Function to query the hash-table of tokenizers (see README.tokenizers). +*/ +static int queryTestTokenizer( + sqlite3 *db, + const char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ + memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); +} + +/* +** This function is part of the test interface for the query parser. It +** writes a text representation of the query expression pExpr into the +** buffer pointed to by argument zBuf. It is assumed that zBuf is large +** enough to store the required text representation. +*/ +static void exprToString(Fts3Expr *pExpr, char *zBuf){ + switch( pExpr->eType ){ + case FTSQUERY_PHRASE: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + zBuf += sprintf(zBuf, "PHRASE %d %d", pPhrase->iColumn, pPhrase->isNot); + for(i=0; inToken; i++){ + zBuf += sprintf(zBuf," %.*s",pPhrase->aToken[i].n,pPhrase->aToken[i].z); + zBuf += sprintf(zBuf,"%s", (pPhrase->aToken[i].isPrefix?"+":"")); + } + return; + } + + case FTSQUERY_NEAR: + zBuf += sprintf(zBuf, "NEAR/%d ", pExpr->nNear); + break; + case FTSQUERY_NOT: + zBuf += sprintf(zBuf, "NOT "); + break; + case FTSQUERY_AND: + zBuf += sprintf(zBuf, "AND "); + break; + case FTSQUERY_OR: + zBuf += sprintf(zBuf, "OR "); + break; + } + + zBuf += sprintf(zBuf, "{"); + exprToString(pExpr->pLeft, zBuf); + zBuf += strlen(zBuf); + zBuf += sprintf(zBuf, "} "); + + zBuf += sprintf(zBuf, "{"); + exprToString(pExpr->pRight, zBuf); + zBuf += strlen(zBuf); + zBuf += sprintf(zBuf, "}"); +} + +/* +** This is the implementation of a scalar SQL function used to test the +** expression parser. It should be called as follows: +** +** fts3_exprtest(, , , ...); +** +** The first argument, , is the name of the fts3 tokenizer used +** to parse the query expression (see README.tokenizers). The second argument +** is the query expression to parse. Each subsequent argument is the name +** of a column of the fts3 table that the query expression may refer to. +** For example: +** +** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); +*/ +static void fts3ExprTest( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_tokenizer_module const *pModule = 0; + sqlite3_tokenizer *pTokenizer = 0; + int rc; + char **azCol = 0; + const char *zExpr; + int nExpr; + int nCol; + int ii; + Fts3Expr *pExpr; + sqlite3 *db = sqlite3_context_db_handle(context); + + if( argc<3 ){ + sqlite3_result_error(context, + "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 + ); + return; + } + + rc = queryTestTokenizer(db, + (const char *)sqlite3_value_text(argv[0]), &pModule); + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + }else if( !pModule ){ + sqlite3_result_error(context, "No such tokenizer module", -1); + goto exprtest_out; + } + + rc = pModule->xCreate(0, 0, &pTokenizer); + assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + } + pTokenizer->pModule = pModule; + + zExpr = (const char *)sqlite3_value_text(argv[1]); + nExpr = sqlite3_value_bytes(argv[1]); + nCol = argc-2; + azCol = (char **)sqlite3_malloc(nCol*sizeof(char *)); + if( !azCol ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + } + for(ii=0; iixDestroy(pTokenizer); + } + sqlite3_free(azCol); +} + +/* +** Register the query expression parser test function fts3_exprtest() +** with database connection db. +*/ +void sqlite3Fts3ExprInitTestInterface(sqlite3* db){ + sqlite3_create_function( + db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 + ); +} + +#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/fts3_expr.h b/fts3_expr.h new file mode 100644 index 0000000..a48dee6 --- /dev/null +++ b/fts3_expr.h @@ -0,0 +1,96 @@ +/* +** 2008 Nov 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + +#include "fts3_tokenizer.h" +#include "sqlite3.h" + +/* +** The following describes the syntax supported by the fts3 MATCH +** operator in a similar format to that used by the lemon parser +** generator. This module does not use actually lemon, it uses a +** custom parser. +** +** query ::= andexpr (OR andexpr)*. +** +** andexpr ::= notexpr (AND? notexpr)*. +** +** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. +** notexpr ::= LP query RP. +** +** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. +** +** distance_opt ::= . +** distance_opt ::= / INTEGER. +** +** phrase ::= TOKEN. +** phrase ::= COLUMN:TOKEN. +** phrase ::= "TOKEN TOKEN TOKEN...". +*/ + +typedef struct Fts3Expr Fts3Expr; +typedef struct Fts3Phrase Fts3Phrase; + +/* +** A "phrase" is a sequence of one or more tokens that must match in +** sequence. A single token is the base case and the most common case. +** For a sequence of tokens contained in "...", nToken will be the number +** of tokens in the string. +*/ +struct Fts3Phrase { + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ + int isNot; /* Phrase prefixed by unary not (-) operator */ + struct PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer pointed to by z */ + int isPrefix; /* True if token ends in with a "*" character */ + } aToken[1]; /* One entry for each token in the phrase */ +}; + +/* +** A tree of these objects forms the RHS of a MATCH operator. +*/ +struct Fts3Expr { + int eType; /* One of the FTSQUERY_XXX values defined below */ + int nNear; /* Valid if eType==FTSQUERY_NEAR */ + Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ + Fts3Expr *pLeft; /* Left operand */ + Fts3Expr *pRight; /* Right operand */ + Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ +}; + +int sqlite3Fts3ExprParse(sqlite3_tokenizer *, char **, int, int, + const char *, int, Fts3Expr **); +void sqlite3Fts3ExprFree(Fts3Expr *); + +/* +** Candidate values for Fts3Query.eType. Note that the order of the first +** four values is in order of precedence when parsing expressions. For +** example, the following: +** +** "a OR b AND c NOT d NEAR e" +** +** is equivalent to: +** +** "a OR (b AND (c NOT (d NEAR e)))" +*/ +#define FTSQUERY_NEAR 1 +#define FTSQUERY_NOT 2 +#define FTSQUERY_AND 3 +#define FTSQUERY_OR 4 +#define FTSQUERY_PHRASE 5 + +#ifdef SQLITE_TEST +void sqlite3Fts3ExprInitTestInterface(sqlite3 *db); +#endif diff --git a/fts2_hash.c b/fts3_hash.c similarity index 65% rename from fts2_hash.c rename to fts3_hash.c index fcd5cc2..ee30117 100644 --- a/fts2_hash.c +++ b/fts3_hash.c @@ -17,79 +17,84 @@ /* ** The code in this file is only compiled if: ** -** * The FTS2 module is being built as an extension +** * The FTS3 module is being built as an extension ** (in which case SQLITE_CORE is not defined), or ** -** * The FTS2 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS2 is defined). +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include #include #include -#include "fts2_hash.h" +#include "sqlite3.h" +#include "fts3_hash.h" -static void *malloc_and_zero(int n){ - void *p = malloc(n); +/* +** Malloc and Free functions +*/ +static void *fts3HashMalloc(int n){ + void *p = sqlite3_malloc(n); if( p ){ memset(p, 0, n); } return p; } +static void fts3HashFree(void *p){ + sqlite3_free(p); +} /* Turn bulk memory into a hash table object by initializing the ** fields of the Hash structure. ** ** "pNew" is a pointer to the hash table that is to be initialized. ** keyClass is one of the constants -** FTS2_HASH_BINARY or FTS2_HASH_STRING. The value of keyClass +** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass ** determines what kind of key the hash table will use. "copyKey" is ** true if the hash table should make its own private copy of keys and ** false if it should just use the supplied pointer. */ -void sqlite3Fts2HashInit(fts2Hash *pNew, int keyClass, int copyKey){ +void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){ assert( pNew!=0 ); - assert( keyClass>=FTS2_HASH_STRING && keyClass<=FTS2_HASH_BINARY ); + assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); pNew->keyClass = keyClass; pNew->copyKey = copyKey; pNew->first = 0; pNew->count = 0; pNew->htsize = 0; pNew->ht = 0; - pNew->xMalloc = malloc_and_zero; - pNew->xFree = free; } /* Remove all entries from a hash table. Reclaim all memory. ** Call this routine to delete a hash table or to reset a hash table ** to the empty state. */ -void sqlite3Fts2HashClear(fts2Hash *pH){ - fts2HashElem *elem; /* For looping over all elements of the table */ +void sqlite3Fts3HashClear(fts3Hash *pH){ + fts3HashElem *elem; /* For looping over all elements of the table */ assert( pH!=0 ); elem = pH->first; pH->first = 0; - if( pH->ht ) pH->xFree(pH->ht); + fts3HashFree(pH->ht); pH->ht = 0; pH->htsize = 0; while( elem ){ - fts2HashElem *next_elem = elem->next; + fts3HashElem *next_elem = elem->next; if( pH->copyKey && elem->pKey ){ - pH->xFree(elem->pKey); + fts3HashFree(elem->pKey); } - pH->xFree(elem); + fts3HashFree(elem); elem = next_elem; } pH->count = 0; } /* -** Hash and comparison functions when the mode is FTS2_HASH_STRING +** Hash and comparison functions when the mode is FTS3_HASH_STRING */ -static int strHash(const void *pKey, int nKey){ +static int fts3StrHash(const void *pKey, int nKey){ const char *z = (const char *)pKey; int h = 0; if( nKey<=0 ) nKey = (int) strlen(z); @@ -99,15 +104,15 @@ static int strHash(const void *pKey, int nKey){ } return h & 0x7fffffff; } -static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){ +static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ if( n1!=n2 ) return 1; return strncmp((const char*)pKey1,(const char*)pKey2,n1); } /* -** Hash and comparison functions when the mode is FTS2_HASH_BINARY +** Hash and comparison functions when the mode is FTS3_HASH_BINARY */ -static int binHash(const void *pKey, int nKey){ +static int fts3BinHash(const void *pKey, int nKey){ int h = 0; const char *z = (const char *)pKey; while( nKey-- > 0 ){ @@ -115,7 +120,7 @@ static int binHash(const void *pKey, int nKey){ } return h & 0x7fffffff; } -static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){ +static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ if( n1!=n2 ) return 1; return memcmp(pKey1,pKey2,n1); } @@ -126,18 +131,18 @@ static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){ ** The C syntax in this function definition may be unfamilar to some ** programmers, so we provide the following additional explanation: ** -** The name of the function is "hashFunction". The function takes a -** single parameter "keyClass". The return value of hashFunction() +** The name of the function is "ftsHashFunction". The function takes a +** single parameter "keyClass". The return value of ftsHashFunction() ** is a pointer to another function. Specifically, the return value -** of hashFunction() is a pointer to a function that takes two parameters +** of ftsHashFunction() is a pointer to a function that takes two parameters ** with types "const void*" and "int" and returns an "int". */ -static int (*hashFunction(int keyClass))(const void*,int){ - if( keyClass==FTS2_HASH_STRING ){ - return &strHash; +static int (*ftsHashFunction(int keyClass))(const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrHash; }else{ - assert( keyClass==FTS2_HASH_BINARY ); - return &binHash; + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinHash; } } @@ -147,23 +152,23 @@ static int (*hashFunction(int keyClass))(const void*,int){ ** For help in interpreted the obscure C code in the function definition, ** see the header comment on the previous function. */ -static int (*compareFunction(int keyClass))(const void*,int,const void*,int){ - if( keyClass==FTS2_HASH_STRING ){ - return &strCompare; +static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrCompare; }else{ - assert( keyClass==FTS2_HASH_BINARY ); - return &binCompare; + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinCompare; } } /* Link an element into the hash table */ -static void insertElement( - fts2Hash *pH, /* The complete hash table */ - struct _fts2ht *pEntry, /* The entry into which pNew is inserted */ - fts2HashElem *pNew /* The element to be inserted */ +static void fts3HashInsertElement( + fts3Hash *pH, /* The complete hash table */ + struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ + fts3HashElem *pNew /* The element to be inserted */ ){ - fts2HashElem *pHead; /* First element already in pEntry */ + fts3HashElem *pHead; /* First element already in pEntry */ pHead = pEntry->chain; if( pHead ){ pNew->next = pHead; @@ -186,22 +191,22 @@ static void insertElement( ** "new_size" must be a power of 2. The hash table might fail ** to resize if sqliteMalloc() fails. */ -static void rehash(fts2Hash *pH, int new_size){ - struct _fts2ht *new_ht; /* The new hash table */ - fts2HashElem *elem, *next_elem; /* For looping over existing elements */ +static void fts3Rehash(fts3Hash *pH, int new_size){ + struct _fts3ht *new_ht; /* The new hash table */ + fts3HashElem *elem, *next_elem; /* For looping over existing elements */ int (*xHash)(const void*,int); /* The hash function */ assert( (new_size & (new_size-1))==0 ); - new_ht = (struct _fts2ht *)pH->xMalloc( new_size*sizeof(struct _fts2ht) ); + new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); if( new_ht==0 ) return; - if( pH->ht ) pH->xFree(pH->ht); + fts3HashFree(pH->ht); pH->ht = new_ht; pH->htsize = new_size; - xHash = hashFunction(pH->keyClass); + xHash = ftsHashFunction(pH->keyClass); for(elem=pH->first, pH->first=0; elem; elem = next_elem){ int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); next_elem = elem->next; - insertElement(pH, &new_ht[h], elem); + fts3HashInsertElement(pH, &new_ht[h], elem); } } @@ -209,21 +214,21 @@ static void rehash(fts2Hash *pH, int new_size){ ** hash table that matches the given key. The hash for this key has ** already been computed and is passed as the 4th parameter. */ -static fts2HashElem *findElementGivenHash( - const fts2Hash *pH, /* The pH to be searched */ +static fts3HashElem *fts3FindElementByHash( + const fts3Hash *pH, /* The pH to be searched */ const void *pKey, /* The key we are searching for */ int nKey, int h /* The hash for this key. */ ){ - fts2HashElem *elem; /* Used to loop thru the element list */ + fts3HashElem *elem; /* Used to loop thru the element list */ int count; /* Number of elements left to test */ int (*xCompare)(const void*,int,const void*,int); /* comparison function */ if( pH->ht ){ - struct _fts2ht *pEntry = &pH->ht[h]; + struct _fts3ht *pEntry = &pH->ht[h]; elem = pEntry->chain; count = pEntry->count; - xCompare = compareFunction(pH->keyClass); + xCompare = ftsCompareFunction(pH->keyClass); while( count-- && elem ){ if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ return elem; @@ -237,12 +242,12 @@ static fts2HashElem *findElementGivenHash( /* Remove a single entry from the hash table given a pointer to that ** element and a hash on the element's key. */ -static void removeElementGivenHash( - fts2Hash *pH, /* The pH containing "elem" */ - fts2HashElem* elem, /* The element to be removed from the pH */ +static void fts3RemoveElementByHash( + fts3Hash *pH, /* The pH containing "elem" */ + fts3HashElem* elem, /* The element to be removed from the pH */ int h /* Hash value for the element */ ){ - struct _fts2ht *pEntry; + struct _fts3ht *pEntry; if( elem->prev ){ elem->prev->next = elem->next; }else{ @@ -260,14 +265,14 @@ static void removeElementGivenHash( pEntry->chain = 0; } if( pH->copyKey && elem->pKey ){ - pH->xFree(elem->pKey); + fts3HashFree(elem->pKey); } - pH->xFree( elem ); + fts3HashFree( elem ); pH->count--; if( pH->count<=0 ){ assert( pH->first==0 ); assert( pH->count==0 ); - fts2HashClear(pH); + fts3HashClear(pH); } } @@ -275,17 +280,17 @@ static void removeElementGivenHash( ** that matches pKey,nKey. Return the data for this element if it is ** found, or NULL if there is no match. */ -void *sqlite3Fts2HashFind(const fts2Hash *pH, const void *pKey, int nKey){ +void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){ int h; /* A hash on key */ - fts2HashElem *elem; /* The element that matches key */ + fts3HashElem *elem; /* The element that matches key */ int (*xHash)(const void*,int); /* The hash function */ if( pH==0 || pH->ht==0 ) return 0; - xHash = hashFunction(pH->keyClass); + xHash = ftsHashFunction(pH->keyClass); assert( xHash!=0 ); h = (*xHash)(pKey,nKey); assert( (pH->htsize & (pH->htsize-1))==0 ); - elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1)); + elem = fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); return elem ? elem->data : 0; } @@ -304,41 +309,48 @@ void *sqlite3Fts2HashFind(const fts2Hash *pH, const void *pKey, int nKey){ ** If the "data" parameter to this function is NULL, then the ** element corresponding to "key" is removed from the hash table. */ -void *sqlite3Fts2HashInsert( - fts2Hash *pH, /* The hash table to insert into */ +void *sqlite3Fts3HashInsert( + fts3Hash *pH, /* The hash table to insert into */ const void *pKey, /* The key */ int nKey, /* Number of bytes in the key */ void *data /* The data */ ){ int hraw; /* Raw hash value of the key */ int h; /* the hash of the key modulo hash table size */ - fts2HashElem *elem; /* Used to loop thru the element list */ - fts2HashElem *new_elem; /* New element added to the pH */ + fts3HashElem *elem; /* Used to loop thru the element list */ + fts3HashElem *new_elem; /* New element added to the pH */ int (*xHash)(const void*,int); /* The hash function */ assert( pH!=0 ); - xHash = hashFunction(pH->keyClass); + xHash = ftsHashFunction(pH->keyClass); assert( xHash!=0 ); hraw = (*xHash)(pKey, nKey); assert( (pH->htsize & (pH->htsize-1))==0 ); h = hraw & (pH->htsize-1); - elem = findElementGivenHash(pH,pKey,nKey,h); + elem = fts3FindElementByHash(pH,pKey,nKey,h); if( elem ){ void *old_data = elem->data; if( data==0 ){ - removeElementGivenHash(pH,elem,h); + fts3RemoveElementByHash(pH,elem,h); }else{ elem->data = data; } return old_data; } if( data==0 ) return 0; - new_elem = (fts2HashElem*)pH->xMalloc( sizeof(fts2HashElem) ); + if( pH->htsize==0 ){ + fts3Rehash(pH,8); + if( pH->htsize==0 ){ + pH->count = 0; + return data; + } + } + new_elem = (fts3HashElem*)fts3HashMalloc( sizeof(fts3HashElem) ); if( new_elem==0 ) return data; if( pH->copyKey && pKey!=0 ){ - new_elem->pKey = pH->xMalloc( nKey ); + new_elem->pKey = fts3HashMalloc( nKey ); if( new_elem->pKey==0 ){ - pH->xFree(new_elem); + fts3HashFree(new_elem); return data; } memcpy((void*)new_elem->pKey, pKey, nKey); @@ -347,23 +359,15 @@ void *sqlite3Fts2HashInsert( } new_elem->nKey = nKey; pH->count++; - if( pH->htsize==0 ){ - rehash(pH,8); - if( pH->htsize==0 ){ - pH->count = 0; - pH->xFree(new_elem); - return data; - } - } if( pH->count > pH->htsize ){ - rehash(pH,pH->htsize*2); + fts3Rehash(pH,pH->htsize*2); } assert( pH->htsize>0 ); assert( (pH->htsize & (pH->htsize-1))==0 ); h = hraw & (pH->htsize-1); - insertElement(pH, &pH->ht[h], new_elem); + fts3HashInsertElement(pH, &pH->ht[h], new_elem); new_elem->data = data; return 0; } -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/fts2_hash.h b/fts3_hash.h similarity index 61% rename from fts2_hash.h rename to fts3_hash.h index 97f3529..e01954e 100644 --- a/fts2_hash.h +++ b/fts3_hash.h @@ -14,12 +14,12 @@ ** hash table implementation for the full-text indexing module. ** */ -#ifndef _FTS2_HASH_H_ -#define _FTS2_HASH_H_ +#ifndef _FTS3_HASH_H_ +#define _FTS3_HASH_H_ /* Forward declarations of structures. */ -typedef struct fts2Hash fts2Hash; -typedef struct fts2HashElem fts2HashElem; +typedef struct fts3Hash fts3Hash; +typedef struct fts3HashElem fts3HashElem; /* A complete hash table is an instance of the following structure. ** The internals of this structure are intended to be opaque -- client @@ -29,17 +29,15 @@ typedef struct fts2HashElem fts2HashElem; ** accessing this structure are really macros, so we can't really make ** this structure opaque. */ -struct fts2Hash { +struct fts3Hash { char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ char copyKey; /* True if copy of key made on insert */ int count; /* Number of entries in this table */ - fts2HashElem *first; /* The first element of the array */ - void *(*xMalloc)(int); /* malloc() function to use */ - void (*xFree)(void *); /* free() function to use */ + fts3HashElem *first; /* The first element of the array */ int htsize; /* Number of buckets in the hash table */ - struct _fts2ht { /* the hash table */ + struct _fts3ht { /* the hash table */ int count; /* Number of entries with this hash */ - fts2HashElem *chain; /* Pointer to first entry with this hash */ + fts3HashElem *chain; /* Pointer to first entry with this hash */ } *ht; }; @@ -49,8 +47,8 @@ struct fts2Hash { ** Again, this structure is intended to be opaque, but it can't really ** be opaque because it is used by macros. */ -struct fts2HashElem { - fts2HashElem *next, *prev; /* Next and previous elements in the table */ +struct fts3HashElem { + fts3HashElem *next, *prev; /* Next and previous elements in the table */ void *data; /* Data associated with this element */ void *pKey; int nKey; /* Key associated with this element */ }; @@ -58,55 +56,55 @@ struct fts2HashElem { /* ** There are 2 different modes of operation for a hash table: ** -** FTS2_HASH_STRING pKey points to a string that is nKey bytes long +** FTS3_HASH_STRING pKey points to a string that is nKey bytes long ** (including the null-terminator, if any). Case ** is respected in comparisons. ** -** FTS2_HASH_BINARY pKey points to binary data nKey bytes long. +** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. ** memcmp() is used to compare keys. ** -** A copy of the key is made if the copyKey parameter to fts2HashInit is 1. +** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. */ -#define FTS2_HASH_STRING 1 -#define FTS2_HASH_BINARY 2 +#define FTS3_HASH_STRING 1 +#define FTS3_HASH_BINARY 2 /* ** Access routines. To delete, insert a NULL pointer. */ -void sqlite3Fts2HashInit(fts2Hash*, int keytype, int copyKey); -void *sqlite3Fts2HashInsert(fts2Hash*, const void *pKey, int nKey, void *pData); -void *sqlite3Fts2HashFind(const fts2Hash*, const void *pKey, int nKey); -void sqlite3Fts2HashClear(fts2Hash*); +void sqlite3Fts3HashInit(fts3Hash*, int keytype, int copyKey); +void *sqlite3Fts3HashInsert(fts3Hash*, const void *pKey, int nKey, void *pData); +void *sqlite3Fts3HashFind(const fts3Hash*, const void *pKey, int nKey); +void sqlite3Fts3HashClear(fts3Hash*); /* ** Shorthand for the functions above */ -#define fts2HashInit sqlite3Fts2HashInit -#define fts2HashInsert sqlite3Fts2HashInsert -#define fts2HashFind sqlite3Fts2HashFind -#define fts2HashClear sqlite3Fts2HashClear +#define fts3HashInit sqlite3Fts3HashInit +#define fts3HashInsert sqlite3Fts3HashInsert +#define fts3HashFind sqlite3Fts3HashFind +#define fts3HashClear sqlite3Fts3HashClear /* ** Macros for looping over all elements of a hash table. The idiom is ** like this: ** -** fts2Hash h; -** fts2HashElem *p; +** fts3Hash h; +** fts3HashElem *p; ** ... -** for(p=fts2HashFirst(&h); p; p=fts2HashNext(p)){ -** SomeStructure *pData = fts2HashData(p); +** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ +** SomeStructure *pData = fts3HashData(p); ** // do something with pData ** } */ -#define fts2HashFirst(H) ((H)->first) -#define fts2HashNext(E) ((E)->next) -#define fts2HashData(E) ((E)->data) -#define fts2HashKey(E) ((E)->pKey) -#define fts2HashKeysize(E) ((E)->nKey) +#define fts3HashFirst(H) ((H)->first) +#define fts3HashNext(E) ((E)->next) +#define fts3HashData(E) ((E)->data) +#define fts3HashKey(E) ((E)->pKey) +#define fts3HashKeysize(E) ((E)->nKey) /* ** Number of entries in a hash table */ -#define fts2HashCount(H) ((H)->count) +#define fts3HashCount(H) ((H)->count) -#endif /* _FTS2_HASH_H_ */ +#endif /* _FTS3_HASH_H_ */ diff --git a/fts2_icu.c b/fts3_icu.c similarity index 94% rename from fts2_icu.c rename to fts3_icu.c index ed15f33..85390d3 100644 --- a/fts2_icu.c +++ b/fts3_icu.c @@ -9,17 +9,17 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file implements a tokenizer for fts2 based on the ICU library. +** This file implements a tokenizer for fts3 based on the ICU library. ** -** $Id: fts2_icu.c,v 1.1 2007/06/22 15:21:16 danielk1977 Exp $ +** $Id: fts3_icu.c,v 1.3 2008/09/01 18:34:20 danielk1977 Exp $ */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #ifdef SQLITE_ENABLE_ICU #include #include -#include "fts2_tokenizer.h" +#include "fts3_tokenizer.h" #include #include @@ -112,6 +112,9 @@ static int icuOpen( *ppCursor = 0; + if( nInput<0 ){ + nInput = strlen(zInput); + } nChar = nInput+1; pCsr = (IcuCursor *)sqlite3_malloc( sizeof(IcuCursor) + /* IcuCursor */ @@ -247,11 +250,11 @@ static const sqlite3_tokenizer_module icuTokenizerModule = { /* ** Set *ppModule to point at the implementation of the ICU tokenizer. */ -void sqlite3Fts2IcuTokenizerModule( +void sqlite3Fts3IcuTokenizerModule( sqlite3_tokenizer_module const**ppModule ){ *ppModule = &icuTokenizerModule; } #endif /* defined(SQLITE_ENABLE_ICU) */ -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/fts2_porter.c b/fts3_porter.c similarity index 96% rename from fts2_porter.c rename to fts3_porter.c index dab1849..6ff67a9 100644 --- a/fts2_porter.c +++ b/fts3_porter.c @@ -16,13 +16,13 @@ /* ** The code in this file is only compiled if: ** -** * The FTS2 module is being built as an extension +** * The FTS3 module is being built as an extension ** (in which case SQLITE_CORE is not defined), or ** -** * The FTS2 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS2 is defined). +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include @@ -31,7 +31,7 @@ #include #include -#include "fts2_tokenizer.h" +#include "fts3_tokenizer.h" /* ** Class derived from sqlite3_tokenizer @@ -66,9 +66,9 @@ static int porterCreate( sqlite3_tokenizer **ppTokenizer ){ porter_tokenizer *t; - t = (porter_tokenizer *) calloc(sizeof(*t), 1); + t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); if( t==NULL ) return SQLITE_NOMEM; - + memset(t, 0, sizeof(*t)); *ppTokenizer = &t->base; return SQLITE_OK; } @@ -77,7 +77,7 @@ static int porterCreate( ** Destroy a tokenizer */ static int porterDestroy(sqlite3_tokenizer *pTokenizer){ - free(pTokenizer); + sqlite3_free(pTokenizer); return SQLITE_OK; } @@ -94,7 +94,7 @@ static int porterOpen( ){ porter_tokenizer_cursor *c; - c = (porter_tokenizer_cursor *) malloc(sizeof(*c)); + c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); if( c==NULL ) return SQLITE_NOMEM; c->zInput = zInput; @@ -120,8 +120,8 @@ static int porterOpen( */ static int porterClose(sqlite3_tokenizer_cursor *pCursor){ porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; - free(c->zToken); - free(c); + sqlite3_free(c->zToken); + sqlite3_free(c); return SQLITE_OK; } /* @@ -603,7 +603,7 @@ static int porterNext( int n = c->iOffset-iStartOffset; if( n>c->nAllocated ){ c->nAllocated = n+20; - c->zToken = realloc(c->zToken, c->nAllocated); + c->zToken = sqlite3_realloc(c->zToken, c->nAllocated); if( c->zToken==NULL ) return SQLITE_NOMEM; } porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); @@ -633,10 +633,10 @@ static const sqlite3_tokenizer_module porterTokenizerModule = { ** Allocate a new porter tokenizer. Return a pointer to the new ** tokenizer in *ppModule */ -void sqlite3Fts2PorterTokenizerModule( +void sqlite3Fts3PorterTokenizerModule( sqlite3_tokenizer_module const**ppModule ){ *ppModule = &porterTokenizerModule; } -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/fts2_tokenizer.c b/fts3_tokenizer.c similarity index 86% rename from fts2_tokenizer.c rename to fts3_tokenizer.c index 66b9dbe..ef19995 100644 --- a/fts2_tokenizer.c +++ b/fts3_tokenizer.c @@ -17,20 +17,21 @@ /* ** The code in this file is only compiled if: ** -** * The FTS2 module is being built as an extension +** * The FTS3 module is being built as an extension ** (in which case SQLITE_CORE is not defined), or ** -** * The FTS2 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS2 is defined). +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -#include "sqlite3.h" #include "sqlite3ext.h" +#ifndef SQLITE_CORE + SQLITE_EXTENSION_INIT1 +#endif -#include "fts2_hash.h" -#include "fts2_tokenizer.h" +#include "fts3_hash.h" +#include "fts3_tokenizer.h" #include /* @@ -41,7 +42,7 @@ ** SELECT (, ); ** ** where is the name passed as the second argument -** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer'). +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). ** ** If the argument is specified, it must be a blob value ** containing a pointer to be stored as the hash data corresponding @@ -58,14 +59,14 @@ static void scalarFunc( int argc, sqlite3_value **argv ){ - fts2Hash *pHash; + fts3Hash *pHash; void *pPtr = 0; const unsigned char *zName; int nName; assert( argc==1 || argc==2 ); - pHash = (fts2Hash *)sqlite3_user_data(context); + pHash = (fts3Hash *)sqlite3_user_data(context); zName = sqlite3_value_text(argv[0]); nName = sqlite3_value_bytes(argv[0])+1; @@ -78,13 +79,13 @@ static void scalarFunc( return; } pPtr = *(void **)sqlite3_value_blob(argv[1]); - pOld = sqlite3Fts2HashInsert(pHash, (void *)zName, nName, pPtr); + pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); if( pOld==pPtr ){ sqlite3_result_error(context, "out of memory", -1); return; } }else{ - pPtr = sqlite3Fts2HashFind(pHash, zName, nName); + pPtr = sqlite3Fts3HashFind(pHash, zName, nName); if( !pPtr ){ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); sqlite3_result_error(context, zErr, -1); @@ -110,8 +111,8 @@ static void scalarFunc( ** SELECT (, ); ** ** where is the name passed as the second argument -** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer') -** concatenated with the string '_test' (e.g. 'fts2_tokenizer_test'). +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') +** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). ** ** The return value is a string that may be interpreted as a Tcl ** list. For each token in the , three elements are @@ -132,7 +133,7 @@ static void testFunc( int argc, sqlite3_value **argv ){ - fts2Hash *pHash; + fts3Hash *pHash; sqlite3_tokenizer_module *p; sqlite3_tokenizer *pTokenizer = 0; sqlite3_tokenizer_cursor *pCsr = 0; @@ -165,8 +166,8 @@ static void testFunc( zArg = (const char *)sqlite3_value_text(argv[1]); } - pHash = (fts2Hash *)sqlite3_user_data(context); - p = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zName, nName+1); + pHash = (fts3Hash *)sqlite3_user_data(context); + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); if( !p ){ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); @@ -223,7 +224,7 @@ int registerTokenizer( ){ int rc; sqlite3_stmt *pStmt; - const char zSql[] = "SELECT fts2_tokenizer(?, ?)"; + const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( rc!=SQLITE_OK ){ @@ -245,7 +246,7 @@ int queryTokenizer( ){ int rc; sqlite3_stmt *pStmt; - const char zSql[] = "SELECT fts2_tokenizer(?)"; + const char zSql[] = "SELECT fts3_tokenizer(?)"; *pp = 0; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); @@ -263,24 +264,24 @@ int queryTokenizer( return sqlite3_finalize(pStmt); } -void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); +void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); /* -** Implementation of the scalar function fts2_tokenizer_internal_test(). +** Implementation of the scalar function fts3_tokenizer_internal_test(). ** This function is used for testing only, it is not included in the ** build unless SQLITE_TEST is defined. ** -** The purpose of this is to test that the fts2_tokenizer() function +** The purpose of this is to test that the fts3_tokenizer() function ** can be used as designed by the C-code in the queryTokenizer and ** registerTokenizer() functions above. These two functions are repeated ** in the README.tokenizer file as an example, so it is important to ** test them. ** -** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar +** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar ** function with no arguments. An assert() will fail if a problem is ** detected. i.e.: ** -** SELECT fts2_tokenizer_internal_test(); +** SELECT fts3_tokenizer_internal_test(); ** */ static void intTestFunc( @@ -294,7 +295,7 @@ static void intTestFunc( sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); /* Test the query function */ - sqlite3Fts2SimpleTokenizerModule(&p1); + sqlite3Fts3SimpleTokenizerModule(&p1); rc = queryTokenizer(db, "simple", &p2); assert( rc==SQLITE_OK ); assert( p1==p2 ); @@ -321,7 +322,7 @@ static void intTestFunc( ** been initialised to use string keys, and to take a private copy ** of the key when a value is inserted. i.e. by a call similar to: ** -** sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1); +** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); ** ** This function adds a scalar function (see header comment above ** scalarFunc() in this file for details) and, if ENABLE_TABLE is @@ -332,9 +333,9 @@ static void intTestFunc( ** The third argument to this function, zName, is used as the name ** of both the scalar and, if created, the virtual table. */ -int sqlite3Fts2InitHashTable( +int sqlite3Fts3InitHashTable( sqlite3 *db, - fts2Hash *pHash, + fts3Hash *pHash, const char *zName ){ int rc = SQLITE_OK; @@ -367,4 +368,4 @@ int sqlite3Fts2InitHashTable( return rc; } -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/fts2_tokenizer.h b/fts3_tokenizer.h similarity index 91% rename from fts2_tokenizer.h rename to fts3_tokenizer.h index 8c256b2..4faef56 100644 --- a/fts2_tokenizer.h +++ b/fts3_tokenizer.h @@ -17,8 +17,8 @@ ** sqlite3_tokenizer_cursor is generated by a tokenizer to generate ** tokens from a particular input. */ -#ifndef _FTS2_TOKENIZER_H_ -#define _FTS2_TOKENIZER_H_ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ /* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. ** If tokenizers are to be allowed to call sqlite3_*() functions, then @@ -32,12 +32,12 @@ ** an sqlite3_tokenizer_module containing pointers to the callback ** functions that make up an implementation. ** -** When an fts2 table is created, it passes any arguments passed to +** When an fts3 table is created, it passes any arguments passed to ** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the ** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer ** implementation. The xCreate() function in turn returns an ** sqlite3_tokenizer structure representing the specific tokenizer to -** be used for the fts2 table (customized by the tokenizer clause arguments). +** be used for the fts3 table (customized by the tokenizer clause arguments). ** ** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() ** method is called. It returns an sqlite3_tokenizer_cursor object @@ -59,10 +59,10 @@ struct sqlite3_tokenizer_module { /* ** Create a new tokenizer. The values in the argv[] array are the ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL - ** TABLE statement that created the fts2 table. For example, if + ** TABLE statement that created the fts3 table. For example, if ** the following SQL is executed: ** - ** CREATE .. USING fts2( ... , tokenizer arg1 arg2) + ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) ** ** then argc is set to 2, and the argv[] array contains pointers ** to the strings "arg1" and "arg2". @@ -80,7 +80,7 @@ struct sqlite3_tokenizer_module { ); /* - ** Destroy an existing tokenizer. The fts2 module calls this method + ** Destroy an existing tokenizer. The fts3 module calls this method ** exactly once for each successful call to xCreate(). */ int (*xDestroy)(sqlite3_tokenizer *pTokenizer); @@ -97,7 +97,7 @@ struct sqlite3_tokenizer_module { ); /* - ** Destroy an existing tokenizer cursor. The fts2 module calls this + ** Destroy an existing tokenizer cursor. The fts3 module calls this ** method exactly once for each successful call to xOpen(). */ int (*xClose)(sqlite3_tokenizer_cursor *pCursor); @@ -142,4 +142,4 @@ struct sqlite3_tokenizer_cursor { /* Tokenizer implementations will typically add additional fields */ }; -#endif /* _FTS2_TOKENIZER_H_ */ +#endif /* _FTS3_TOKENIZER_H_ */ diff --git a/fts2_tokenizer1.c b/fts3_tokenizer1.c similarity index 89% rename from fts2_tokenizer1.c rename to fts3_tokenizer1.c index 540ba27..da255d9 100644 --- a/fts2_tokenizer1.c +++ b/fts3_tokenizer1.c @@ -16,13 +16,13 @@ /* ** The code in this file is only compiled if: ** -** * The FTS2 module is being built as an extension +** * The FTS3 module is being built as an extension ** (in which case SQLITE_CORE is not defined), or ** -** * The FTS2 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS2 is defined). +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include @@ -31,7 +31,7 @@ #include #include -#include "fts2_tokenizer.h" +#include "fts3_tokenizer.h" typedef struct simple_tokenizer { sqlite3_tokenizer base; @@ -65,8 +65,9 @@ static int simpleCreate( ){ simple_tokenizer *t; - t = (simple_tokenizer *) calloc(sizeof(*t), 1); + t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); /* TODO(shess) Delimiters need to remain the same from run to run, ** else we need to reindex. One solution would be a meta-table to @@ -79,7 +80,7 @@ static int simpleCreate( unsigned char ch = argv[1][i]; /* We explicitly don't support UTF-8 delimiters for now. */ if( ch>=0x80 ){ - free(t); + sqlite3_free(t); return SQLITE_ERROR; } t->delim[ch] = 1; @@ -100,7 +101,7 @@ static int simpleCreate( ** Destroy a tokenizer */ static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ - free(pTokenizer); + sqlite3_free(pTokenizer); return SQLITE_OK; } @@ -117,7 +118,7 @@ static int simpleOpen( ){ simple_tokenizer_cursor *c; - c = (simple_tokenizer_cursor *) malloc(sizeof(*c)); + c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); if( c==NULL ) return SQLITE_NOMEM; c->pInput = pInput; @@ -143,8 +144,8 @@ static int simpleOpen( */ static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; - free(c->pToken); - free(c); + sqlite3_free(c->pToken); + sqlite3_free(c); return SQLITE_OK; } @@ -182,7 +183,7 @@ static int simpleNext( int i, n = c->iOffset-iStartOffset; if( n>c->nTokenAllocated ){ c->nTokenAllocated = n+20; - c->pToken = realloc(c->pToken, c->nTokenAllocated); + c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated); if( c->pToken==NULL ) return SQLITE_NOMEM; } for(i=0; i -/* #include */ #include #include #include "vdbeInt.h" -#include "os.h" - /* ** Return the collating function associated with a function. @@ -68,10 +65,11 @@ static void minmaxFunc( */ static void typeofFunc( sqlite3_context *context, - int argc, + int NotUsed, sqlite3_value **argv ){ const char *z = 0; + UNUSED_PARAMETER(NotUsed); switch( sqlite3_value_type(argv[0]) ){ case SQLITE_NULL: z = "null"; break; case SQLITE_INTEGER: z = "integer"; break; @@ -94,6 +92,7 @@ static void lengthFunc( int len; assert( argc==1 ); + UNUSED_PARAMETER(argc); switch( sqlite3_value_type(argv[0]) ){ case SQLITE_BLOB: case SQLITE_INTEGER: @@ -124,6 +123,7 @@ static void lengthFunc( */ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ assert( argc==1 ); + UNUSED_PARAMETER(argc); switch( sqlite3_value_type(argv[0]) ){ case SQLITE_INTEGER: { i64 iVal = sqlite3_value_int64(argv[0]); @@ -171,7 +171,7 @@ static void substrFunc( int p0type; i64 p1, p2; - assert( argc==3 ); + assert( argc==3 || argc==2 ); p0type = sqlite3_value_type(argv[0]); if( p0type==SQLITE_BLOB ){ len = sqlite3_value_bytes(argv[0]); @@ -187,7 +187,11 @@ static void substrFunc( } } p1 = sqlite3_value_int(argv[1]); - p2 = sqlite3_value_int(argv[2]); + if( argc==3 ){ + p2 = sqlite3_value_int(argv[2]); + }else{ + p2 = sqlite3_context_db_handle(context)->aLimit[SQLITE_LIMIT_LENGTH]; + } if( p1<0 ){ p1 += len; if( p1<0 ){ @@ -208,10 +212,10 @@ static void substrFunc( for(z2=z; *z2 && p2; p2--){ SQLITE_SKIP_UTF8(z2); } - sqlite3_result_text(context, (char*)z, z2-z, SQLITE_TRANSIENT); + sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT); }else{ if( p2<0 ) p2 = 0; - sqlite3_result_blob(context, (char*)&z[p1], p2, SQLITE_TRANSIENT); + sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT); } } @@ -236,6 +240,25 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3_result_double(context, r); } +/* +** Allocate nByte bytes of space using sqlite3_malloc(). If the +** allocation fails, call sqlite3_result_error_nomem() to notify +** the database handle that malloc() has failed. +*/ +static void *contextMalloc(sqlite3_context *context, i64 nByte){ + char *z; + if( nByte>sqlite3_context_db_handle(context)->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + z = 0; + }else{ + z = sqlite3Malloc((int)nByte); + if( !z && nByte>0 ){ + sqlite3_result_error_nomem(context); + } + } + return z; +} + /* ** Implementation of the upper() and lower() SQL functions. */ @@ -249,11 +272,11 @@ static void upperFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ /* Verify that the call to _bytes() does not invalidate the _text() pointer */ assert( z2==(char*)sqlite3_value_text(argv[0]) ); if( z2 ){ - z1 = sqlite3_malloc(n+1); + z1 = contextMalloc(context, ((i64)n)+1); if( z1 ){ memcpy(z1, z2, n+1); for(i=0; z1[i]; i++){ - z1[i] = toupper(z1[i]); + z1[i] = (char)toupper(z1[i]); } sqlite3_result_text(context, z1, -1, sqlite3_free); } @@ -269,11 +292,11 @@ static void lowerFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ /* Verify that the call to _bytes() does not invalidate the _text() pointer */ assert( z2==(char*)sqlite3_value_text(argv[0]) ); if( z2 ){ - z1 = sqlite3_malloc(n+1); + z1 = contextMalloc(context, ((i64)n)+1); if( z1 ){ memcpy(z1, z2, n+1); for(i=0; z1[i]; i++){ - z1[i] = tolower(z1[i]); + z1[i] = (char)tolower(z1[i]); } sqlite3_result_text(context, z1, -1, sqlite3_free); } @@ -304,11 +327,12 @@ static void ifnullFunc( */ static void randomFunc( sqlite3_context *context, - int argc, - sqlite3_value **argv + int NotUsed, + sqlite3_value **NotUsed2 ){ sqlite_int64 r; - sqlite3Randomness(sizeof(r), &r); + UNUSED_PARAMETER2(NotUsed, NotUsed2); + sqlite3_randomness(sizeof(r), &r); if( (r<<1)==0 ) r = 0; /* Prevent 0x8000.... as the result so that we */ /* can always do abs() of the result */ sqlite3_result_int64(context, r); @@ -326,18 +350,15 @@ static void randomBlob( int n; unsigned char *p; assert( argc==1 ); + UNUSED_PARAMETER(argc); n = sqlite3_value_int(argv[0]); if( n<1 ){ n = 1; } - if( n>SQLITE_MAX_LENGTH ){ - sqlite3_result_error_toobig(context); - return; - } - p = sqliteMalloc(n); + p = contextMalloc(context, n); if( p ){ - sqlite3Randomness(n, p); - sqlite3_result_blob(context, (char*)p, n, sqlite3FreeX); + sqlite3_randomness(n, p); + sqlite3_result_blob(context, (char*)p, n, sqlite3_free); } } @@ -347,10 +368,11 @@ static void randomBlob( */ static void last_insert_rowid( sqlite3_context *context, - int arg, - sqlite3_value **argv + int NotUsed, + sqlite3_value **NotUsed2 ){ - sqlite3 *db = sqlite3_user_data(context); + sqlite3 *db = sqlite3_context_db_handle(context); + UNUSED_PARAMETER2(NotUsed, NotUsed2); sqlite3_result_int64(context, sqlite3_last_insert_rowid(db)); } @@ -360,10 +382,11 @@ static void last_insert_rowid( */ static void changes( sqlite3_context *context, - int arg, - sqlite3_value **argv + int NotUsed, + sqlite3_value **NotUsed2 ){ - sqlite3 *db = sqlite3_user_data(context); + sqlite3 *db = sqlite3_context_db_handle(context); + UNUSED_PARAMETER2(NotUsed, NotUsed2); sqlite3_result_int(context, sqlite3_changes(db)); } @@ -373,10 +396,11 @@ static void changes( */ static void total_changes( sqlite3_context *context, - int arg, - sqlite3_value **argv + int NotUsed, + sqlite3_value **NotUsed2 ){ - sqlite3 *db = sqlite3_user_data(context); + sqlite3 *db = sqlite3_context_db_handle(context); + UNUSED_PARAMETER2(NotUsed, NotUsed2); sqlite3_result_int(context, sqlite3_total_changes(db)); } @@ -390,6 +414,19 @@ struct compareInfo { u8 noCase; }; +/* +** For LIKE and GLOB matching on EBCDIC machines, assume that every +** character is exactly one byte in size. Also, all characters are +** able to participate in upper-case-to-lower-case mappings in EBCDIC +** whereas only characters less than 0x80 do in ASCII. +*/ +#if defined(SQLITE_EBCDIC) +# define sqlite3Utf8Read(A,B,C) (*(A++)) +# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] +#else +# define GlogUpperToLower(A) if( A<0x80 ){ A = sqlite3UpperToLower[A]; } +#endif + static const struct compareInfo globInfo = { '*', '?', '[', 0 }; /* The correct SQL-92 behavior is for the LIKE operator to ignore ** case. Thus 'a' LIKE 'A' would be true. */ @@ -466,11 +503,11 @@ static int patternCompare( } while( (c2 = sqlite3Utf8Read(zString,0,&zString))!=0 ){ if( noCase ){ - c2 = c2<0x80 ? sqlite3UpperToLower[c2] : c2; - c = c<0x80 ? sqlite3UpperToLower[c] : c; + GlogUpperToLower(c2); + GlogUpperToLower(c); while( c2 != 0 && c2 != c ){ c2 = sqlite3Utf8Read(zString, 0, &zString); - if( c2<0x80 ) c2 = sqlite3UpperToLower[c2]; + GlogUpperToLower(c2); } }else{ while( c2 != 0 && c2 != c ){ @@ -522,8 +559,8 @@ static int patternCompare( }else{ c2 = sqlite3Utf8Read(zString, 0, &zString); if( noCase ){ - c = c<0x80 ? sqlite3UpperToLower[c] : c; - c2 = c2<0x80 ? sqlite3UpperToLower[c2] : c2; + GlogUpperToLower(c); + GlogUpperToLower(c2); } if( c!=c2 ){ return 0; @@ -563,6 +600,7 @@ static void likeFunc( ){ const unsigned char *zA, *zB; int escape = 0; + sqlite3 *db = sqlite3_context_db_handle(context); zB = sqlite3_value_text(argv[0]); zA = sqlite3_value_text(argv[1]); @@ -570,7 +608,8 @@ static void likeFunc( /* Limit the length of the LIKE or GLOB pattern to avoid problems ** of deep recursion and N*N behavior in patternCompare(). */ - if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ + if( sqlite3_value_bytes(argv[0]) > + db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH] ){ sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); return; } @@ -606,10 +645,11 @@ static void likeFunc( */ static void nullifFunc( sqlite3_context *context, - int argc, + int NotUsed, sqlite3_value **argv ){ CollSeq *pColl = sqlite3GetFuncCollSeq(context); + UNUSED_PARAMETER(NotUsed); if( sqlite3MemCompare(argv[0], argv[1], pColl)!=0 ){ sqlite3_result_value(context, argv[0]); } @@ -621,9 +661,10 @@ static void nullifFunc( */ static void versionFunc( sqlite3_context *context, - int argc, - sqlite3_value **argv + int NotUsed, + sqlite3_value **NotUsed2 ){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); sqlite3_result_text(context, sqlite3_version, -1, SQLITE_STATIC); } @@ -662,15 +703,8 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ char const *zBlob = sqlite3_value_blob(argv[0]); int nBlob = sqlite3_value_bytes(argv[0]); assert( zBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ - - if( 2*nBlob+4>SQLITE_MAX_LENGTH ){ - sqlite3_result_error_toobig(context); - return; - } - zText = (char *)sqliteMalloc((2*nBlob)+4); - if( !zText ){ - sqlite3_result_error(context, "out of memory", -1); - }else{ + zText = (char *)contextMalloc(context, (2*(i64)nBlob)+4); + if( zText ){ int i; for(i=0; i>4)&0x0F]; @@ -681,7 +715,7 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ zText[0] = 'X'; zText[1] = '\''; sqlite3_result_text(context, zText, -1, SQLITE_TRANSIENT); - sqliteFree(zText); + sqlite3_free(zText); } break; } @@ -693,23 +727,19 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ if( zArg==0 ) return; for(i=0, n=0; zArg[i]; i++){ if( zArg[i]=='\'' ) n++; } - if( i+n+3>SQLITE_MAX_LENGTH ){ - sqlite3_result_error_toobig(context); - return; - } - z = sqliteMalloc( i+n+3 ); - if( z==0 ) return; - z[0] = '\''; - for(i=0, j=1; zArg[i]; i++){ - z[j++] = zArg[i]; - if( zArg[i]=='\'' ){ - z[j++] = '\''; + z = contextMalloc(context, ((i64)i)+((i64)n)+3); + if( z ){ + z[0] = '\''; + for(i=0, j=1; zArg[i]; i++){ + z[j++] = zArg[i]; + if( zArg[i]=='\'' ){ + z[j++] = '\''; + } } + z[j++] = '\''; + z[j] = 0; + sqlite3_result_text(context, z, j, sqlite3_free); } - z[j++] = '\''; - z[j] = 0; - sqlite3_result_text(context, z, j, SQLITE_TRANSIENT); - sqliteFree(z); } } } @@ -727,22 +757,20 @@ static void hexFunc( const unsigned char *pBlob; char *zHex, *z; assert( argc==1 ); + UNUSED_PARAMETER(argc); pBlob = sqlite3_value_blob(argv[0]); n = sqlite3_value_bytes(argv[0]); - if( n*2+1>SQLITE_MAX_LENGTH ){ - sqlite3_result_error_toobig(context); - return; - } assert( pBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ - z = zHex = sqlite3_malloc(n*2 + 1); - if( zHex==0 ) return; - for(i=0; i>4)&0xf]; - *(z++) = hexdigits[c&0xf]; + z = zHex = contextMalloc(context, ((i64)n)*2 + 1); + if( zHex ){ + for(i=0; i>4)&0xf]; + *(z++) = hexdigits[c&0xf]; + } + *z = 0; + sqlite3_result_text(context, zHex, n*2, sqlite3_free); } - *z = 0; - sqlite3_result_text(context, zHex, n*2, sqlite3_free); } /* @@ -755,11 +783,12 @@ static void zeroblobFunc( ){ i64 n; assert( argc==1 ); + UNUSED_PARAMETER(argc); n = sqlite3_value_int64(argv[0]); if( n>SQLITE_MAX_LENGTH ){ sqlite3_result_error_toobig(context); }else{ - sqlite3_result_zeroblob(context, n); + sqlite3_result_zeroblob(context, (int)n); } } @@ -786,6 +815,7 @@ static void replaceFunc( int i, j; /* Loop counters */ assert( argc==3 ); + UNUSED_PARAMETER(argc); zStr = sqlite3_value_text(argv[0]); if( zStr==0 ) return; nStr = sqlite3_value_bytes(argv[0]); @@ -800,7 +830,7 @@ static void replaceFunc( assert( zRep==sqlite3_value_text(argv[2]) ); nOut = nStr + 1; assert( nOut=SQLITE_MAX_LENGTH ){ + if( nOut>=db->aLimit[SQLITE_LIMIT_LENGTH] ){ sqlite3_result_error_toobig(context); - sqlite3_free(zOut); + sqlite3DbFree(db, zOut); return; } + zOld = zOut; zOut = sqlite3_realloc(zOut, (int)nOut); if( zOut==0 ){ + sqlite3_result_error_nomem(context); + sqlite3DbFree(db, zOld); return; } memcpy(&zOut[j], zRep, nRep); @@ -846,8 +881,8 @@ static void trimFunc( int nIn; /* Number of bytes in input */ int flags; /* 1: trimleft 2: trimright 3: trim */ int i; /* Loop counter */ - unsigned char *aLen; /* Length of each character in zCharSet */ - const unsigned char **azChar; /* Individual characters in zCharSet */ + unsigned char *aLen = 0; /* Length of each character in zCharSet */ + unsigned char **azChar = 0; /* Individual characters in zCharSet */ int nChar; /* Number of characters in zCharSet */ if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ @@ -859,10 +894,10 @@ static void trimFunc( assert( zIn==sqlite3_value_text(argv[0]) ); if( argc==1 ){ static const unsigned char lenOne[] = { 1 }; - static const unsigned char *azOne[] = { (u8*)" " }; + static unsigned char * const azOne[] = { (u8*)" " }; nChar = 1; aLen = (u8*)lenOne; - azChar = azOne; + azChar = (unsigned char **)azOne; zCharSet = 0; }else if( (zCharSet = sqlite3_value_text(argv[1]))==0 ){ return; @@ -872,23 +907,23 @@ static void trimFunc( SQLITE_SKIP_UTF8(z); } if( nChar>0 ){ - azChar = sqlite3_malloc( nChar*(sizeof(char*)+1) ); + azChar = contextMalloc(context, ((i64)nChar)*(sizeof(char*)+1)); if( azChar==0 ){ return; } aLen = (unsigned char*)&azChar[nChar]; for(z=zCharSet, nChar=0; *z; nChar++){ - azChar[nChar] = z; + azChar[nChar] = (unsigned char *)z; SQLITE_SKIP_UTF8(z); - aLen[nChar] = z - azChar[nChar]; + aLen[nChar] = (u8)(z - azChar[nChar]); } } } if( nChar>0 ){ - flags = (int)sqlite3_user_data(context); + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(context)); if( flags & 1 ){ while( nIn>0 ){ - int len; + int len = 0; for(i=0; i0 ){ - int len; + int len = 0; for(i=0; i "; - int iMin, iMax, n, r, i; - unsigned char zBuf[1000]; - if( argc>=1 ){ - iMin = sqlite3_value_int(argv[0]); - if( iMin<0 ) iMin = 0; - if( iMin>=sizeof(zBuf) ) iMin = sizeof(zBuf)-1; - }else{ - iMin = 1; - } - if( argc>=2 ){ - iMax = sqlite3_value_int(argv[1]); - if( iMax=sizeof(zBuf) ) iMax = sizeof(zBuf)-1; - }else{ - iMax = 50; - } - n = iMin; - if( iMax>iMin ){ - sqlite3Randomness(sizeof(r), &r); - r &= 0x7fffffff; - n += r%(iMax + 1 - iMin); - } - assert( nrSum += v; if( (p->approx|p->overflow)==0 ){ i64 iNewSum = p->iSum + v; - int s1 = p->iSum >> (sizeof(i64)*8-1); - int s2 = v >> (sizeof(i64)*8-1); - int s3 = iNewSum >> (sizeof(i64)*8-1); - p->overflow = (s1&s2&~s3) | (~s1&~s2&s3); + int s1 = (int)(p->iSum >> (sizeof(i64)*8-1)); + int s2 = (int)(v >> (sizeof(i64)*8-1)); + int s3 = (int)(iNewSum >> (sizeof(i64)*8-1)); + p->overflow = ((s1&s2&~s3) | (~s1&~s2&s3))?1:0; p->iSum = iNewSum; } }else{ @@ -1250,9 +1129,14 @@ static void countFinalize(sqlite3_context *context){ /* ** Routines to implement min() and max() aggregate functions. */ -static void minmaxStep(sqlite3_context *context, int argc, sqlite3_value **argv){ +static void minmaxStep( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ Mem *pArg = (Mem *)argv[0]; Mem *pBest; + UNUSED_PARAMETER(NotUsed); if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; pBest = (Mem *)sqlite3_aggregate_context(context, sizeof(*pBest)); @@ -1290,6 +1174,58 @@ static void minMaxFinalize(sqlite3_context *context){ } } +/* +** group_concat(EXPR, ?SEPARATOR?) +*/ +static void groupConcatStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zVal; + StrAccum *pAccum; + const char *zSep; + int nVal, nSep, i; + if( argc==0 || sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pAccum = (StrAccum*)sqlite3_aggregate_context(context, sizeof(*pAccum)); + + if( pAccum ){ + sqlite3 *db = sqlite3_context_db_handle(context); + pAccum->useMalloc = 1; + pAccum->mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; + if( pAccum->nChar ){ + if( argc>1 ){ + zSep = (char*)sqlite3_value_text(argv[argc-1]); + nSep = sqlite3_value_bytes(argv[argc-1]); + }else{ + zSep = ","; + nSep = 1; + } + sqlite3StrAccumAppend(pAccum, zSep, nSep); + } + i = 0; + do{ + zVal = (char*)sqlite3_value_text(argv[i]); + nVal = sqlite3_value_bytes(argv[i]); + sqlite3StrAccumAppend(pAccum, zVal, nVal); + i++; + }while( itooBig ){ + sqlite3_result_error_toobig(context); + }else if( pAccum->mallocFailed ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_text(context, sqlite3StrAccumFinish(pAccum), -1, + sqlite3_free); + } + } +} /* ** This function registered all of the above C functions as SQL @@ -1297,140 +1233,28 @@ static void minMaxFinalize(sqlite3_context *context){ ** external linkage. */ void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ - static const struct { - char *zName; - signed char nArg; - u8 argType; /* ff: db 1: 0, 2: 1, 3: 2,... N: N-1. */ - u8 eTextRep; /* 1: UTF-16. 0: UTF-8 */ - u8 needCollSeq; - void (*xFunc)(sqlite3_context*,int,sqlite3_value **); - } aFuncs[] = { - { "min", -1, 0, SQLITE_UTF8, 1, minmaxFunc }, - { "min", 0, 0, SQLITE_UTF8, 1, 0 }, - { "max", -1, 1, SQLITE_UTF8, 1, minmaxFunc }, - { "max", 0, 1, SQLITE_UTF8, 1, 0 }, - { "typeof", 1, 0, SQLITE_UTF8, 0, typeofFunc }, - { "length", 1, 0, SQLITE_UTF8, 0, lengthFunc }, - { "substr", 3, 0, SQLITE_UTF8, 0, substrFunc }, - { "abs", 1, 0, SQLITE_UTF8, 0, absFunc }, - { "round", 1, 0, SQLITE_UTF8, 0, roundFunc }, - { "round", 2, 0, SQLITE_UTF8, 0, roundFunc }, - { "upper", 1, 0, SQLITE_UTF8, 0, upperFunc }, - { "lower", 1, 0, SQLITE_UTF8, 0, lowerFunc }, - { "coalesce", -1, 0, SQLITE_UTF8, 0, ifnullFunc }, - { "coalesce", 0, 0, SQLITE_UTF8, 0, 0 }, - { "coalesce", 1, 0, SQLITE_UTF8, 0, 0 }, - { "hex", 1, 0, SQLITE_UTF8, 0, hexFunc }, - { "ifnull", 2, 0, SQLITE_UTF8, 1, ifnullFunc }, - { "random", -1, 0, SQLITE_UTF8, 0, randomFunc }, - { "randomblob", 1, 0, SQLITE_UTF8, 0, randomBlob }, - { "nullif", 2, 0, SQLITE_UTF8, 1, nullifFunc }, - { "sqlite_version", 0, 0, SQLITE_UTF8, 0, versionFunc}, - { "quote", 1, 0, SQLITE_UTF8, 0, quoteFunc }, - { "last_insert_rowid", 0, 0xff, SQLITE_UTF8, 0, last_insert_rowid }, - { "changes", 0, 0xff, SQLITE_UTF8, 0, changes }, - { "total_changes", 0, 0xff, SQLITE_UTF8, 0, total_changes }, - { "replace", 3, 0, SQLITE_UTF8, 0, replaceFunc }, - { "ltrim", 1, 1, SQLITE_UTF8, 0, trimFunc }, - { "ltrim", 2, 1, SQLITE_UTF8, 0, trimFunc }, - { "rtrim", 1, 2, SQLITE_UTF8, 0, trimFunc }, - { "rtrim", 2, 2, SQLITE_UTF8, 0, trimFunc }, - { "trim", 1, 3, SQLITE_UTF8, 0, trimFunc }, - { "trim", 2, 3, SQLITE_UTF8, 0, trimFunc }, - { "zeroblob", 1, 0, SQLITE_UTF8, 0, zeroblobFunc }, -#ifdef SQLITE_SOUNDEX - { "soundex", 1, 0, SQLITE_UTF8, 0, soundexFunc}, -#endif -#ifndef SQLITE_OMIT_LOAD_EXTENSION - { "load_extension", 1, 0xff, SQLITE_UTF8, 0, loadExt }, - { "load_extension", 2, 0xff, SQLITE_UTF8, 0, loadExt }, -#endif -#ifdef SQLITE_TEST - { "randstr", 2, 0, SQLITE_UTF8, 0, randStr }, - { "test_destructor", 1, 0xff, SQLITE_UTF8, 0, test_destructor}, - { "test_destructor_count", 0, 0, SQLITE_UTF8, 0, test_destructor_count}, - { "test_auxdata", -1, 0, SQLITE_UTF8, 0, test_auxdata}, - { "test_error", 1, 0, SQLITE_UTF8, 0, test_error}, -#endif - }; - static const struct { - char *zName; - signed char nArg; - u8 argType; - u8 needCollSeq; - void (*xStep)(sqlite3_context*,int,sqlite3_value**); - void (*xFinalize)(sqlite3_context*); - } aAggs[] = { - { "min", 1, 0, 1, minmaxStep, minMaxFinalize }, - { "max", 1, 1, 1, minmaxStep, minMaxFinalize }, - { "sum", 1, 0, 0, sumStep, sumFinalize }, - { "total", 1, 0, 0, sumStep, totalFinalize }, - { "avg", 1, 0, 0, sumStep, avgFinalize }, - { "count", 0, 0, 0, countStep, countFinalize }, - { "count", 1, 0, 0, countStep, countFinalize }, - }; - int i; - - for(i=0; ineedCollSeq = 1; - } - } - } #ifndef SQLITE_OMIT_ALTERTABLE sqlite3AlterFunctions(db); #endif -#ifndef SQLITE_OMIT_PARSER - sqlite3AttachFunctions(db); -#endif - for(i=0; ineedCollSeq = 1; - } - } - } - sqlite3RegisterDateTimeFunctions(db); - if( !sqlite3MallocFailed() ){ + if( !db->mallocFailed ){ int rc = sqlite3_overload_function(db, "MATCH", 2); assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); if( rc==SQLITE_NOMEM ){ - sqlite3FailedMalloc(); + db->mallocFailed = 1; } } #ifdef SQLITE_SSE (void)sqlite3SseFunctions(db); #endif -#ifdef SQLITE_CASE_SENSITIVE_LIKE - sqlite3RegisterLikeFunctions(db, 1); -#else - sqlite3RegisterLikeFunctions(db, 0); -#endif } /* ** Set the LIKEOPT flag on the 2-argument function with the given name. */ -static void setLikeOptFlag(sqlite3 *db, const char *zName, int flagVal){ +static void setLikeOptFlag(sqlite3 *db, const char *zName, u8 flagVal){ FuncDef *pDef; - pDef = sqlite3FindFunction(db, zName, strlen(zName), 2, SQLITE_UTF8, 0); + pDef = sqlite3FindFunction(db, zName, sqlite3Strlen30(zName), + 2, SQLITE_UTF8, 0); if( pDef ){ pDef->flags = flagVal; } @@ -1489,3 +1313,90 @@ int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ *pIsNocase = (pDef->flags & SQLITE_FUNC_CASE)==0; return 1; } + +/* +** All all of the FuncDef structures in the aBuiltinFunc[] array above +** to the global function hash table. This occurs at start-time (as +** a consequence of calling sqlite3_initialize()). +** +** After this routine runs +*/ +void sqlite3RegisterGlobalFunctions(void){ + /* + ** The following array holds FuncDef structures for all of the functions + ** defined in this file. + ** + ** The array cannot be constant since changes are made to the + ** FuncDef.pHash elements at start-time. The elements of this array + ** are read-only after initialization is complete. + */ + static SQLITE_WSD FuncDef aBuiltinFunc[] = { + FUNCTION(ltrim, 1, 1, 0, trimFunc ), + FUNCTION(ltrim, 2, 1, 0, trimFunc ), + FUNCTION(rtrim, 1, 2, 0, trimFunc ), + FUNCTION(rtrim, 2, 2, 0, trimFunc ), + FUNCTION(trim, 1, 3, 0, trimFunc ), + FUNCTION(trim, 2, 3, 0, trimFunc ), + FUNCTION(min, -1, 0, 1, minmaxFunc ), + FUNCTION(min, 0, 0, 1, 0 ), + AGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize ), + FUNCTION(max, -1, 1, 1, minmaxFunc ), + FUNCTION(max, 0, 1, 1, 0 ), + AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), + FUNCTION(typeof, 1, 0, 0, typeofFunc ), + FUNCTION(length, 1, 0, 0, lengthFunc ), + FUNCTION(substr, 2, 0, 0, substrFunc ), + FUNCTION(substr, 3, 0, 0, substrFunc ), + FUNCTION(abs, 1, 0, 0, absFunc ), + FUNCTION(round, 1, 0, 0, roundFunc ), + FUNCTION(round, 2, 0, 0, roundFunc ), + FUNCTION(upper, 1, 0, 0, upperFunc ), + FUNCTION(lower, 1, 0, 0, lowerFunc ), + FUNCTION(coalesce, 1, 0, 0, 0 ), + FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), + FUNCTION(coalesce, 0, 0, 0, 0 ), + FUNCTION(hex, 1, 0, 0, hexFunc ), + FUNCTION(ifnull, 2, 0, 1, ifnullFunc ), + FUNCTION(random, -1, 0, 0, randomFunc ), + FUNCTION(randomblob, 1, 0, 0, randomBlob ), + FUNCTION(nullif, 2, 0, 1, nullifFunc ), + FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + FUNCTION(quote, 1, 0, 0, quoteFunc ), + FUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), + FUNCTION(changes, 0, 0, 0, changes ), + FUNCTION(total_changes, 0, 0, 0, total_changes ), + FUNCTION(replace, 3, 0, 0, replaceFunc ), + FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), + #ifdef SQLITE_SOUNDEX + FUNCTION(soundex, 1, 0, 0, soundexFunc ), + #endif + #ifndef SQLITE_OMIT_LOAD_EXTENSION + FUNCTION(load_extension, 1, 0, 0, loadExt ), + FUNCTION(load_extension, 2, 0, 0, loadExt ), + #endif + AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), + AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), + AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), + AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), + AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), + AGGREGATE(group_concat, -1, 0, 0, groupConcatStep, groupConcatFinalize), + + LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), + #ifdef SQLITE_CASE_SENSITIVE_LIKE + LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), + LIKEFUNC(like, 3, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), + #else + LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE), + LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE), + #endif + }; + + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aBuiltinFunc); + + for(i=0; i @@ -21,28 +21,16 @@ ** fields of the Hash structure. ** ** "pNew" is a pointer to the hash table that is to be initialized. -** keyClass is one of the constants SQLITE_HASH_INT, SQLITE_HASH_POINTER, -** SQLITE_HASH_BINARY, or SQLITE_HASH_STRING. The value of keyClass -** determines what kind of key the hash table will use. "copyKey" is -** true if the hash table should make its own private copy of keys and -** false if it should just use the supplied pointer. CopyKey only makes -** sense for SQLITE_HASH_STRING and SQLITE_HASH_BINARY and is ignored -** for other key classes. +** "copyKey" is true if the hash table should make its own private +** copy of keys and false if it should just use the supplied pointer. */ -void sqlite3HashInit(Hash *pNew, int keyClass, int copyKey){ +void sqlite3HashInit(Hash *pNew, int copyKey){ assert( pNew!=0 ); - assert( keyClass>=SQLITE_HASH_STRING && keyClass<=SQLITE_HASH_BINARY ); - pNew->keyClass = keyClass; -#if 0 - if( keyClass==SQLITE_HASH_POINTER || keyClass==SQLITE_HASH_INT ) copyKey = 0; -#endif - pNew->copyKey = copyKey; + pNew->copyKey = copyKey!=0; pNew->first = 0; pNew->count = 0; pNew->htsize = 0; pNew->ht = 0; - pNew->xMalloc = sqlite3MallocX; - pNew->xFree = sqlite3FreeX; } /* Remove all entries from a hash table. Reclaim all memory. @@ -55,54 +43,27 @@ void sqlite3HashClear(Hash *pH){ assert( pH!=0 ); elem = pH->first; pH->first = 0; - if( pH->ht ) pH->xFree(pH->ht); + sqlite3_free(pH->ht); pH->ht = 0; pH->htsize = 0; while( elem ){ HashElem *next_elem = elem->next; - if( pH->copyKey && elem->pKey ){ - pH->xFree(elem->pKey); + if( pH->copyKey ){ + sqlite3_free(elem->pKey); } - pH->xFree(elem); + sqlite3_free(elem); elem = next_elem; } pH->count = 0; } -#if 0 /* NOT USED */ -/* -** Hash and comparison functions when the mode is SQLITE_HASH_INT -*/ -static int intHash(const void *pKey, int nKey){ - return nKey ^ (nKey<<8) ^ (nKey>>8); -} -static int intCompare(const void *pKey1, int n1, const void *pKey2, int n2){ - return n2 - n1; -} -#endif - -#if 0 /* NOT USED */ -/* -** Hash and comparison functions when the mode is SQLITE_HASH_POINTER -*/ -static int ptrHash(const void *pKey, int nKey){ - uptr x = Addr(pKey); - return x ^ (x<<8) ^ (x>>8); -} -static int ptrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ - if( pKey1==pKey2 ) return 0; - if( pKey1 0 ){ h = (h<<3) ^ h ^ sqlite3UpperToLower[(unsigned char)*z++]; nKey--; @@ -114,79 +75,6 @@ static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){ return sqlite3StrNICmp((const char*)pKey1,(const char*)pKey2,n1); } -/* -** Hash and comparison functions when the mode is SQLITE_HASH_BINARY -*/ -static int binHash(const void *pKey, int nKey){ - int h = 0; - const char *z = (const char *)pKey; - while( nKey-- > 0 ){ - h = (h<<3) ^ h ^ *(z++); - } - return h & 0x7fffffff; -} -static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){ - if( n1!=n2 ) return 1; - return memcmp(pKey1,pKey2,n1); -} - -/* -** Return a pointer to the appropriate hash function given the key class. -** -** The C syntax in this function definition may be unfamilar to some -** programmers, so we provide the following additional explanation: -** -** The name of the function is "hashFunction". The function takes a -** single parameter "keyClass". The return value of hashFunction() -** is a pointer to another function. Specifically, the return value -** of hashFunction() is a pointer to a function that takes two parameters -** with types "const void*" and "int" and returns an "int". -*/ -static int (*hashFunction(int keyClass))(const void*,int){ -#if 0 /* HASH_INT and HASH_POINTER are never used */ - switch( keyClass ){ - case SQLITE_HASH_INT: return &intHash; - case SQLITE_HASH_POINTER: return &ptrHash; - case SQLITE_HASH_STRING: return &strHash; - case SQLITE_HASH_BINARY: return &binHash;; - default: break; - } - return 0; -#else - if( keyClass==SQLITE_HASH_STRING ){ - return &strHash; - }else{ - assert( keyClass==SQLITE_HASH_BINARY ); - return &binHash; - } -#endif -} - -/* -** Return a pointer to the appropriate hash function given the key class. -** -** For help in interpreted the obscure C code in the function definition, -** see the header comment on the previous function. -*/ -static int (*compareFunction(int keyClass))(const void*,int,const void*,int){ -#if 0 /* HASH_INT and HASH_POINTER are never used */ - switch( keyClass ){ - case SQLITE_HASH_INT: return &intCompare; - case SQLITE_HASH_POINTER: return &ptrCompare; - case SQLITE_HASH_STRING: return &strCompare; - case SQLITE_HASH_BINARY: return &binCompare; - default: break; - } - return 0; -#else - if( keyClass==SQLITE_HASH_STRING ){ - return &strCompare; - }else{ - assert( keyClass==SQLITE_HASH_BINARY ); - return &binCompare; - } -#endif -} /* Link an element into the hash table */ @@ -216,22 +104,34 @@ static void insertElement( /* Resize the hash table so that it cantains "new_size" buckets. ** "new_size" must be a power of 2. The hash table might fail -** to resize if sqliteMalloc() fails. +** to resize if sqlite3_malloc() fails. */ static void rehash(Hash *pH, int new_size){ struct _ht *new_ht; /* The new hash table */ HashElem *elem, *next_elem; /* For looping over existing elements */ - int (*xHash)(const void*,int); /* The hash function */ - assert( (new_size & (new_size-1))==0 ); - new_ht = (struct _ht *)pH->xMalloc( new_size*sizeof(struct _ht) ); +#ifdef SQLITE_MALLOC_SOFT_LIMIT + if( new_size*sizeof(struct _ht)>SQLITE_MALLOC_SOFT_LIMIT ){ + new_size = SQLITE_MALLOC_SOFT_LIMIT/sizeof(struct _ht); + } + if( new_size==pH->htsize ) return; +#endif + + /* There is a call to sqlite3_malloc() inside rehash(). If there is + ** already an allocation at pH->ht, then if this malloc() fails it + ** is benign (since failing to resize a hash table is a performance + ** hit only, not a fatal error). + */ + if( pH->htsize>0 ) sqlite3BeginBenignMalloc(); + new_ht = (struct _ht *)sqlite3MallocZero( new_size*sizeof(struct _ht) ); + if( pH->htsize>0 ) sqlite3EndBenignMalloc(); + if( new_ht==0 ) return; - if( pH->ht ) pH->xFree(pH->ht); + sqlite3_free(pH->ht); pH->ht = new_ht; pH->htsize = new_size; - xHash = hashFunction(pH->keyClass); for(elem=pH->first, pH->first=0; elem; elem = next_elem){ - int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); + int h = strHash(elem->pKey, elem->nKey) & (new_size-1); next_elem = elem->next; insertElement(pH, &new_ht[h], elem); } @@ -249,15 +149,13 @@ static HashElem *findElementGivenHash( ){ HashElem *elem; /* Used to loop thru the element list */ int count; /* Number of elements left to test */ - int (*xCompare)(const void*,int,const void*,int); /* comparison function */ if( pH->ht ){ struct _ht *pEntry = &pH->ht[h]; elem = pEntry->chain; count = pEntry->count; - xCompare = compareFunction(pH->keyClass); while( count-- && elem ){ - if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ + if( strCompare(elem->pKey,elem->nKey,pKey,nKey)==0 ){ return elem; } elem = elem->next; @@ -292,9 +190,9 @@ static void removeElementGivenHash( pEntry->chain = 0; } if( pH->copyKey ){ - pH->xFree(elem->pKey); + sqlite3_free(elem->pKey); } - pH->xFree( elem ); + sqlite3_free( elem ); pH->count--; if( pH->count<=0 ){ assert( pH->first==0 ); @@ -303,21 +201,28 @@ static void removeElementGivenHash( } } +/* Attempt to locate an element of the hash table pH with a key +** that matches pKey,nKey. Return a pointer to the corresponding +** HashElem structure for this element if it is found, or NULL +** otherwise. +*/ +HashElem *sqlite3HashFindElem(const Hash *pH, const void *pKey, int nKey){ + int h; /* A hash on key */ + HashElem *elem; /* The element that matches key */ + + if( pH==0 || pH->ht==0 ) return 0; + h = strHash(pKey,nKey); + elem = findElementGivenHash(pH,pKey,nKey, h % pH->htsize); + return elem; +} + /* Attempt to locate an element of the hash table pH with a key ** that matches pKey,nKey. Return the data for this element if it is ** found, or NULL if there is no match. */ void *sqlite3HashFind(const Hash *pH, const void *pKey, int nKey){ - int h; /* A hash on key */ HashElem *elem; /* The element that matches key */ - int (*xHash)(const void*,int); /* The hash function */ - - if( pH==0 || pH->ht==0 ) return 0; - xHash = hashFunction(pH->keyClass); - assert( xHash!=0 ); - h = (*xHash)(pKey,nKey); - assert( (pH->htsize & (pH->htsize-1))==0 ); - elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1)); + elem = sqlite3HashFindElem(pH, pKey, nKey); return elem ? elem->data : 0; } @@ -341,31 +246,33 @@ void *sqlite3HashInsert(Hash *pH, const void *pKey, int nKey, void *data){ int h; /* the hash of the key modulo hash table size */ HashElem *elem; /* Used to loop thru the element list */ HashElem *new_elem; /* New element added to the pH */ - int (*xHash)(const void*,int); /* The hash function */ assert( pH!=0 ); - xHash = hashFunction(pH->keyClass); - assert( xHash!=0 ); - hraw = (*xHash)(pKey, nKey); - assert( (pH->htsize & (pH->htsize-1))==0 ); - h = hraw & (pH->htsize-1); - elem = findElementGivenHash(pH,pKey,nKey,h); - if( elem ){ - void *old_data = elem->data; - if( data==0 ){ - removeElementGivenHash(pH,elem,h); - }else{ - elem->data = data; + hraw = strHash(pKey, nKey); + if( pH->htsize ){ + h = hraw % pH->htsize; + elem = findElementGivenHash(pH,pKey,nKey,h); + if( elem ){ + void *old_data = elem->data; + if( data==0 ){ + removeElementGivenHash(pH,elem,h); + }else{ + elem->data = data; + if( !pH->copyKey ){ + elem->pKey = (void *)pKey; + } + assert(nKey==elem->nKey); + } + return old_data; } - return old_data; } if( data==0 ) return 0; - new_elem = (HashElem*)pH->xMalloc( sizeof(HashElem) ); + new_elem = (HashElem*)sqlite3Malloc( sizeof(HashElem) ); if( new_elem==0 ) return data; if( pH->copyKey && pKey!=0 ){ - new_elem->pKey = pH->xMalloc( nKey ); + new_elem->pKey = sqlite3Malloc( nKey ); if( new_elem->pKey==0 ){ - pH->xFree(new_elem); + sqlite3_free(new_elem); return data; } memcpy((void*)new_elem->pKey, pKey, nKey); @@ -375,13 +282,13 @@ void *sqlite3HashInsert(Hash *pH, const void *pKey, int nKey, void *data){ new_elem->nKey = nKey; pH->count++; if( pH->htsize==0 ){ - rehash(pH,8); + rehash(pH, 128/sizeof(pH->ht[0])); if( pH->htsize==0 ){ pH->count = 0; if( pH->copyKey ){ - pH->xFree(new_elem->pKey); + sqlite3_free(new_elem->pKey); } - pH->xFree(new_elem); + sqlite3_free(new_elem); return data; } } @@ -389,8 +296,7 @@ void *sqlite3HashInsert(Hash *pH, const void *pKey, int nKey, void *data){ rehash(pH,pH->htsize*2); } assert( pH->htsize>0 ); - assert( (pH->htsize & (pH->htsize-1))==0 ); - h = hraw & (pH->htsize-1); + h = hraw % pH->htsize; insertElement(pH, &pH->ht[h], new_elem); new_elem->data = data; return 0; diff --git a/hash.h b/hash.h index 78f60b4..16b391f 100644 --- a/hash.h +++ b/hash.h @@ -12,7 +12,7 @@ ** This is the header file for the generic hash-table implemenation ** used in SQLite. ** -** $Id: hash.h,v 1.9 2006/02/14 10:48:39 danielk1977 Exp $ +** $Id: hash.h,v 1.12 2008/10/10 17:41:29 drh Exp $ */ #ifndef _SQLITE_HASH_H_ #define _SQLITE_HASH_H_ @@ -30,16 +30,13 @@ typedef struct HashElem HashElem; ** this structure opaque. */ struct Hash { - char keyClass; /* SQLITE_HASH_INT, _POINTER, _STRING, _BINARY */ - char copyKey; /* True if copy of key made on insert */ - int count; /* Number of entries in this table */ - HashElem *first; /* The first element of the array */ - void *(*xMalloc)(int); /* malloc() function to use */ - void (*xFree)(void *); /* free() function to use */ - int htsize; /* Number of buckets in the hash table */ - struct _ht { /* the hash table */ - int count; /* Number of entries with this hash */ - HashElem *chain; /* Pointer to first entry with this hash */ + unsigned int copyKey: 1; /* True if copy of key made on insert */ + unsigned int htsize : 31; /* Number of buckets in the hash table */ + unsigned int count; /* Number of entries in this table */ + HashElem *first; /* The first element of the array */ + struct _ht { /* the hash table */ + int count; /* Number of entries with this hash */ + HashElem *chain; /* Pointer to first entry with this hash */ } *ht; }; @@ -55,34 +52,13 @@ struct HashElem { void *pKey; int nKey; /* Key associated with this element */ }; -/* -** There are 4 different modes of operation for a hash table: -** -** SQLITE_HASH_INT nKey is used as the key and pKey is ignored. -** -** SQLITE_HASH_POINTER pKey is used as the key and nKey is ignored. -** -** SQLITE_HASH_STRING pKey points to a string that is nKey bytes long -** (including the null-terminator, if any). Case -** is ignored in comparisons. -** -** SQLITE_HASH_BINARY pKey points to binary data nKey bytes long. -** memcmp() is used to compare keys. -** -** A copy of the key is made for SQLITE_HASH_STRING and SQLITE_HASH_BINARY -** if the copyKey parameter to HashInit is 1. -*/ -/* #define SQLITE_HASH_INT 1 // NOT USED */ -/* #define SQLITE_HASH_POINTER 2 // NOT USED */ -#define SQLITE_HASH_STRING 3 -#define SQLITE_HASH_BINARY 4 - /* ** Access routines. To delete, insert a NULL pointer. */ -void sqlite3HashInit(Hash*, int keytype, int copyKey); +void sqlite3HashInit(Hash*, int copyKey); void *sqlite3HashInsert(Hash*, const void *pKey, int nKey, void *pData); void *sqlite3HashFind(const Hash*, const void *pKey, int nKey); +HashElem *sqlite3HashFindElem(const Hash*, const void *pKey, int nKey); void sqlite3HashClear(Hash*); /* diff --git a/hwtime.h b/hwtime.h new file mode 100644 index 0000000..896041e --- /dev/null +++ b/hwtime.h @@ -0,0 +1,87 @@ +/* +** 2008 May 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 class CPUs. +** +** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $ +*/ +#ifndef _HWTIME_H_ +#define _HWTIME_H_ + +/* +** The following routine only works on pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + + #elif defined(_MSC_VER) + + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } + + #endif + +#elif (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; + __asm__ __volatile__ ("rdtsc" : "=A" (val)); + return val; + } + +#elif (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; + } + +#else + + #error Need implementation of sqlite3Hwtime() for your platform. + + /* + ** To compile without implementing sqlite3Hwtime() for your platform, + ** you can remove the above #error and use the following + ** stub function. You will lose timing support for many + ** of the debugging and testing utilities, but it should at + ** least compile and run. + */ + sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +#endif + +#endif /* !defined(_HWTIME_H_) */ diff --git a/insert.c b/insert.c index ba9426d..0ab647e 100644 --- a/insert.c +++ b/insert.c @@ -12,12 +12,12 @@ ** This file contains C code routines that are called by the parser ** to handle INSERT statements in SQLite. ** -** $Id: insert.c,v 1.188 2007/07/23 19:39:47 drh Exp $ +** $Id: insert.c,v 1.256 2008/12/10 21:19:57 drh Exp $ */ #include "sqliteInt.h" /* -** Set P3 of the most recently inserted opcode to a column affinity +** Set P4 of the most recently inserted opcode to a column affinity ** string for index pIdx. A column affinity string has one character ** for each column in the table, according to the affinity of the column: ** @@ -28,6 +28,9 @@ ** 'c' NUMERIC ** 'd' INTEGER ** 'e' REAL +** +** An extra 'b' is appended to the end of the string to cover the +** rowid that appears as the last column in every index. */ void sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ if( !pIdx->zColAff ){ @@ -41,21 +44,24 @@ void sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ */ int n; Table *pTab = pIdx->pTable; - pIdx->zColAff = (char *)sqliteMalloc(pIdx->nColumn+1); + sqlite3 *db = sqlite3VdbeDb(v); + pIdx->zColAff = (char *)sqlite3Malloc(pIdx->nColumn+2); if( !pIdx->zColAff ){ + db->mallocFailed = 1; return; } for(n=0; nnColumn; n++){ pIdx->zColAff[n] = pTab->aCol[pIdx->aiColumn[n]].affinity; } - pIdx->zColAff[pIdx->nColumn] = '\0'; + pIdx->zColAff[n++] = SQLITE_AFF_NONE; + pIdx->zColAff[n] = 0; } - sqlite3VdbeChangeP3(v, -1, pIdx->zColAff, 0); + sqlite3VdbeChangeP4(v, -1, pIdx->zColAff, 0); } /* -** Set P3 of the most recently inserted opcode to a column affinity +** Set P4 of the most recently inserted opcode to a column affinity ** string for table pTab. A column affinity string has one character ** for each column indexed by the index, according to the affinity of the ** column: @@ -79,9 +85,11 @@ void sqlite3TableAffinityStr(Vdbe *v, Table *pTab){ if( !pTab->zColAff ){ char *zColAff; int i; + sqlite3 *db = sqlite3VdbeDb(v); - zColAff = (char *)sqliteMalloc(pTab->nCol+1); + zColAff = (char *)sqlite3Malloc(pTab->nCol+1); if( !zColAff ){ + db->mallocFailed = 1; return; } @@ -93,27 +101,41 @@ void sqlite3TableAffinityStr(Vdbe *v, Table *pTab){ pTab->zColAff = zColAff; } - sqlite3VdbeChangeP3(v, -1, pTab->zColAff, 0); + sqlite3VdbeChangeP4(v, -1, pTab->zColAff, 0); } /* -** Return non-zero if SELECT statement p opens the table with rootpage -** iTab in database iDb. This is used to see if a statement of the form -** "INSERT INTO SELECT ..." can run without using temporary -** table for the results of the SELECT. -** -** No checking is done for sub-selects that are part of expressions. +** Return non-zero if the table pTab in database iDb or any of its indices +** have been opened at any point in the VDBE program beginning at location +** iStartAddr throught the end of the program. This is used to see if +** a statement of the form "INSERT INTO SELECT ..." can +** run without using temporary table for the results of the SELECT. */ -static int selectReadsTable(Select *p, Schema *pSchema, int iTab){ +static int readsTable(Vdbe *v, int iStartAddr, int iDb, Table *pTab){ int i; - struct SrcList_item *pItem; - if( p->pSrc==0 ) return 0; - for(i=0, pItem=p->pSrc->a; ipSrc->nSrc; i++, pItem++){ - if( pItem->pSelect ){ - if( selectReadsTable(pItem->pSelect, pSchema, iTab) ) return 1; - }else{ - if( pItem->pTab->pSchema==pSchema && pItem->pTab->tnum==iTab ) return 1; + int iEnd = sqlite3VdbeCurrentAddr(v); + for(i=iStartAddr; iopcode==OP_OpenRead && pOp->p3==iDb ){ + Index *pIndex; + int tnum = pOp->p2; + if( tnum==pTab->tnum ){ + return 1; + } + for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ + if( tnum==pIndex->tnum ){ + return 1; + } + } } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pOp->opcode==OP_VOpen && pOp->p4.pVtab==pTab->pVtab ){ + assert( pOp->p4.pVtab!=0 ); + assert( pOp->p4type==P4_VTAB ); + return 1; + } +#endif } return 0; } @@ -122,45 +144,49 @@ static int selectReadsTable(Select *p, Schema *pSchema, int iTab){ /* ** Write out code to initialize the autoincrement logic. This code ** looks up the current autoincrement value in the sqlite_sequence -** table and stores that value in a memory cell. Code generated by -** autoIncStep() will keep that memory cell holding the largest +** table and stores that value in a register. Code generated by +** autoIncStep() will keep that register holding the largest ** rowid value. Code generated by autoIncEnd() will write the new ** largest value of the counter back into the sqlite_sequence table. ** ** This routine returns the index of the mem[] cell that contains ** the maximum rowid counter. ** -** Two memory cells are allocated. The next memory cell after the -** one returned holds the rowid in sqlite_sequence where we will -** write back the revised maximum rowid. +** Three consecutive registers are allocated by this routine. The +** first two hold the name of the target table and the maximum rowid +** inserted into the target table, respectively. +** The third holds the rowid in sqlite_sequence where we will +** write back the revised maximum rowid. This routine returns the +** index of the second of these three registers. */ static int autoIncBegin( Parse *pParse, /* Parsing context */ int iDb, /* Index of the database holding pTab */ Table *pTab /* The table we are writing to */ ){ - int memId = 0; - if( pTab->autoInc ){ + int memId = 0; /* Register holding maximum rowid */ + if( pTab->tabFlags & TF_Autoincrement ){ Vdbe *v = pParse->pVdbe; Db *pDb = &pParse->db->aDb[iDb]; int iCur = pParse->nTab; - int addr; + int addr; /* Address of the top of the loop */ assert( v ); - addr = sqlite3VdbeCurrentAddr(v); - memId = pParse->nMem+1; - pParse->nMem += 2; + pParse->nMem++; /* Holds name of table */ + memId = ++pParse->nMem; + pParse->nMem++; sqlite3OpenTable(pParse, iCur, iDb, pDb->pSchema->pSeqTab, OP_OpenRead); - sqlite3VdbeAddOp(v, OP_Rewind, iCur, addr+13); - sqlite3VdbeAddOp(v, OP_Column, iCur, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, pTab->zName, 0); - sqlite3VdbeAddOp(v, OP_Ne, 0x100, addr+12); - sqlite3VdbeAddOp(v, OP_Rowid, iCur, 0); - sqlite3VdbeAddOp(v, OP_MemStore, memId-1, 1); - sqlite3VdbeAddOp(v, OP_Column, iCur, 1); - sqlite3VdbeAddOp(v, OP_MemStore, memId, 1); - sqlite3VdbeAddOp(v, OP_Goto, 0, addr+13); - sqlite3VdbeAddOp(v, OP_Next, iCur, addr+4); - sqlite3VdbeAddOp(v, OP_Close, iCur, 0); + addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp4(v, OP_String8, 0, memId-1, 0, pTab->zName, 0); + sqlite3VdbeAddOp2(v, OP_Rewind, iCur, addr+9); + sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, memId); + sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, memId+1); + sqlite3VdbeAddOp3(v, OP_Column, iCur, 1, memId); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr+9); + sqlite3VdbeAddOp2(v, OP_Next, iCur, addr+2); + sqlite3VdbeAddOp2(v, OP_Integer, 0, memId); + sqlite3VdbeAddOp2(v, OP_Close, iCur, 0); } return memId; } @@ -173,15 +199,15 @@ static int autoIncBegin( ** larger than the maximum rowid in the memId memory cell, then the ** memory cell is updated. The stack is unchanged. */ -static void autoIncStep(Parse *pParse, int memId){ +static void autoIncStep(Parse *pParse, int memId, int regRowid){ if( memId>0 ){ - sqlite3VdbeAddOp(pParse->pVdbe, OP_MemMax, memId, 0); + sqlite3VdbeAddOp2(pParse->pVdbe, OP_MemMax, memId, regRowid); } } /* ** After doing one or more inserts, the maximum rowid is stored -** in mem[memId]. Generate code to write this value back into the +** in reg[memId]. Generate code to write this value back into the ** the sqlite_sequence table. */ static void autoIncEnd( @@ -190,23 +216,22 @@ static void autoIncEnd( Table *pTab, /* Table we are inserting into */ int memId /* Memory cell holding the maximum rowid */ ){ - if( pTab->autoInc ){ + if( pTab->tabFlags & TF_Autoincrement ){ int iCur = pParse->nTab; Vdbe *v = pParse->pVdbe; Db *pDb = &pParse->db->aDb[iDb]; - int addr; + int j1; + int iRec = ++pParse->nMem; /* Memory cell used for record */ + assert( v ); - addr = sqlite3VdbeCurrentAddr(v); sqlite3OpenTable(pParse, iCur, iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); - sqlite3VdbeAddOp(v, OP_MemLoad, memId-1, 0); - sqlite3VdbeAddOp(v, OP_NotNull, -1, addr+7); - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); - sqlite3VdbeAddOp(v, OP_NewRowid, iCur, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, pTab->zName, 0); - sqlite3VdbeAddOp(v, OP_MemLoad, memId, 0); - sqlite3VdbeAddOp(v, OP_MakeRecord, 2, 0); - sqlite3VdbeAddOp(v, OP_Insert, iCur, OPFLAG_APPEND); - sqlite3VdbeAddOp(v, OP_Close, iCur, 0); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); + sqlite3VdbeAddOp2(v, OP_NewRowid, iCur, memId+1); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); + sqlite3VdbeAddOp3(v, OP_Insert, iCur, iRec, memId+1); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3VdbeAddOp1(v, OP_Close, iCur); } } #else @@ -215,7 +240,7 @@ static void autoIncEnd( ** above are all no-ops */ # define autoIncBegin(A,B,C) (0) -# define autoIncStep(A,B) +# define autoIncStep(A,B,C) # define autoIncEnd(A,B,C,D) #endif /* SQLITE_OMIT_AUTOINCREMENT */ @@ -246,7 +271,8 @@ static int xferOptimization( ** ** The code generated follows one of four templates. For a simple ** select with data coming from a VALUES clause, the code executes -** once straight down through. The template looks like this: +** once straight down through. Pseudo-code follows (we call this +** the "1st template"): ** ** open write cursor to
and its indices ** puts VALUES clause expressions onto the stack @@ -264,7 +290,7 @@ static int xferOptimization( ** schemas, including all the same indices, then a special optimization ** is invoked that copies raw records from over to . ** See the xferOptimization() function for the implementation of this -** template. This is the second template. +** template. This is the 2nd template. ** ** open a write cursor to
** open read cursor on @@ -277,45 +303,58 @@ static int xferOptimization( ** close cursors ** end foreach ** -** The third template is for when the second template does not apply +** The 3rd template is for when the second template does not apply ** and the SELECT clause does not read from
at any time. ** The generated code follows this template: ** +** EOF <- 0 +** X <- A ** goto B ** A: setup for the SELECT ** loop over the rows in the SELECT -** gosub C +** load values into registers R..R+n +** yield X ** end loop ** cleanup after the SELECT -** goto D -** B: open write cursor to
and its indices +** EOF <- 1 +** yield X ** goto A -** C: insert the select result into
-** return +** B: open write cursor to
and its indices +** C: yield X +** if EOF goto D +** insert the select result into
from R..R+n +** goto C ** D: cleanup ** -** The fourth template is used if the insert statement takes its +** The 4th template is used if the insert statement takes its ** values from a SELECT but the data is being inserted into a table ** that is also read as part of the SELECT. In the third form, ** we have to use a intermediate table to store the results of ** the select. The template is like this: ** +** EOF <- 0 +** X <- A ** goto B ** A: setup for the SELECT ** loop over the tables in the SELECT -** gosub C +** load value into register R..R+n +** yield X ** end loop ** cleanup after the SELECT -** goto D -** C: insert the select result into the intermediate table -** return -** B: open a cursor to an intermediate table -** goto A -** D: open write cursor to
and its indices -** loop over the intermediate table +** EOF <- 1 +** yield X +** halt-error +** B: open temp table +** L: yield X +** if EOF goto M +** insert row from R..R+n into temp table +** goto L +** M: open write cursor to
and its indices +** rewind temp table +** C: loop over rows of intermediate table ** transfer values form intermediate table into
-** end the loop -** cleanup +** end loop +** D: cleanup */ void sqlite3Insert( Parse *pParse, /* Parser context */ @@ -325,41 +364,51 @@ void sqlite3Insert( IdList *pColumn, /* Column names corresponding to IDLIST. */ int onError /* How to handle constraint errors */ ){ - Table *pTab; /* The table to insert into */ + sqlite3 *db; /* The main database structure */ + Table *pTab; /* The table to insert into. aka TABLE */ char *zTab; /* Name of the table into which we are inserting */ const char *zDb; /* Name of the database holding this table */ int i, j, idx; /* Loop counters */ Vdbe *v; /* Generate code into this virtual machine */ Index *pIdx; /* For looping over indices of the table */ int nColumn; /* Number of columns in the data */ - int base = 0; /* VDBE Cursor number for pTab */ - int iCont=0,iBreak=0; /* Beginning and end of the loop over srcTab */ - sqlite3 *db; /* The main database structure */ + int nHidden = 0; /* Number of hidden columns if TABLE is virtual */ + int baseCur = 0; /* VDBE Cursor number for pTab */ int keyColumn = -1; /* Column that is the INTEGER PRIMARY KEY */ int endOfLoop; /* Label for the end of the insertion loop */ int useTempTable = 0; /* Store SELECT results in intermediate table */ int srcTab = 0; /* Data comes from this temporary cursor if >=0 */ - int iSelectLoop = 0; /* Address of code that implements the SELECT */ - int iCleanup = 0; /* Address of the cleanup code */ - int iInsertBlock = 0; /* Address of the subroutine used to insert data */ - int iCntMem = 0; /* Memory cell used for the row counter */ - int newIdx = -1; /* Cursor for the NEW table */ + int addrInsTop = 0; /* Jump to label "D" */ + int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ + int addrSelect = 0; /* Address of coroutine that implements the SELECT */ + SelectDest dest; /* Destination for SELECT on rhs of INSERT */ + int newIdx = -1; /* Cursor for the NEW pseudo-table */ + int iDb; /* Index of database holding TABLE */ Db *pDb; /* The database containing table being inserted into */ - int counterMem = 0; /* Memory cell holding AUTOINCREMENT counter */ int appendFlag = 0; /* True if the insert is likely to be an append */ - int iDb; - int nHidden = 0; + /* Register allocations */ + int regFromSelect = 0;/* Base register for data coming from SELECT */ + int regAutoinc = 0; /* Register holding the AUTOINCREMENT counter */ + int regRowCount = 0; /* Memory cell used for the row counter */ + int regIns; /* Block of regs holding rowid+data being inserted */ + int regRowid; /* registers holding insert rowid */ + int regData; /* register holding first column to insert */ + int regRecord; /* Holds the assemblied row record */ + int regEof = 0; /* Register recording end of SELECT data */ + int *aRegIdx = 0; /* One register allocated to each index */ + #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to insert into a view */ int triggers_exist = 0; /* True if there are FOR EACH ROW triggers */ #endif - if( pParse->nErr || sqlite3MallocFailed() ){ + db = pParse->db; + memset(&dest, 0, sizeof(dest)); + if( pParse->nErr || db->mallocFailed ){ goto insert_cleanup; } - db = pParse->db; /* Locate the table into which we will be inserting new information. */ @@ -382,7 +431,7 @@ void sqlite3Insert( ** inserted into is a view */ #ifndef SQLITE_OMIT_TRIGGER - triggers_exist = sqlite3TriggersExist(pParse, pTab, TK_INSERT, 0); + triggers_exist = sqlite3TriggersExist(pTab, TK_INSERT, 0); isView = pTab->pSelect!=0; #else # define triggers_exist 0 @@ -429,6 +478,8 @@ void sqlite3Insert( ** ** Then special optimizations can be applied that make the transfer ** very fast and which reduce fragmentation of indices. + ** + ** This is the 2nd template. */ if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ assert( !triggers_exist ); @@ -438,74 +489,108 @@ void sqlite3Insert( #endif /* SQLITE_OMIT_XFER_OPT */ /* If this is an AUTOINCREMENT table, look up the sequence number in the - ** sqlite_sequence table and store it in memory cell counterMem. Also - ** remember the rowid of the sqlite_sequence table entry in memory cell - ** counterRowid. + ** sqlite_sequence table and store it in memory cell regAutoinc. */ - counterMem = autoIncBegin(pParse, iDb, pTab); + regAutoinc = autoIncBegin(pParse, iDb, pTab); /* Figure out how many columns of data are supplied. If the data - ** is coming from a SELECT statement, then this step also generates - ** all the code to implement the SELECT statement and invoke a subroutine - ** to process each row of the result. (Template 2.) If the SELECT - ** statement uses the the table that is being inserted into, then the - ** subroutine is also coded here. That subroutine stores the SELECT - ** results in a temporary table. (Template 3.) + ** is coming from a SELECT statement, then generate a co-routine that + ** produces a single row of the SELECT on each invocation. The + ** co-routine is the common header to the 3rd and 4th templates. */ if( pSelect ){ /* Data is coming from a SELECT. Generate code to implement that SELECT + ** as a co-routine. The code is common to both the 3rd and 4th + ** templates: + ** + ** EOF <- 0 + ** X <- A + ** goto B + ** A: setup for the SELECT + ** loop over the tables in the SELECT + ** load value into register R..R+n + ** yield X + ** end loop + ** cleanup after the SELECT + ** EOF <- 1 + ** yield X + ** halt-error + ** + ** On each invocation of the co-routine, it puts a single row of the + ** SELECT result into registers dest.iMem...dest.iMem+dest.nMem-1. + ** (These output registers are allocated by sqlite3Select().) When + ** the SELECT completes, it sets the EOF flag stored in regEof. */ - int rc, iInitCode; - iInitCode = sqlite3VdbeAddOp(v, OP_Goto, 0, 0); - iSelectLoop = sqlite3VdbeCurrentAddr(v); - iInsertBlock = sqlite3VdbeMakeLabel(v); + int rc, j1; + + regEof = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regEof); /* EOF <- 0 */ + VdbeComment((v, "SELECT eof flag")); + sqlite3SelectDestInit(&dest, SRT_Coroutine, ++pParse->nMem); + addrSelect = sqlite3VdbeCurrentAddr(v)+2; + sqlite3VdbeAddOp2(v, OP_Integer, addrSelect-1, dest.iParm); + j1 = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + VdbeComment((v, "Jump over SELECT coroutine")); /* Resolve the expressions in the SELECT statement and execute it. */ - rc = sqlite3Select(pParse, pSelect, SRT_Subroutine, iInsertBlock,0,0,0,0); - if( rc || pParse->nErr || sqlite3MallocFailed() ){ + rc = sqlite3Select(pParse, pSelect, &dest); + if( rc || pParse->nErr || db->mallocFailed ){ goto insert_cleanup; } + sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); /* EOF <- 1 */ + sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); /* yield X */ + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_INTERNAL, OE_Abort); + VdbeComment((v, "End of SELECT coroutine")); + sqlite3VdbeJumpHere(v, j1); /* label B: */ - iCleanup = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp(v, OP_Goto, 0, iCleanup); + regFromSelect = dest.iMem; assert( pSelect->pEList ); nColumn = pSelect->pEList->nExpr; + assert( dest.nMem==nColumn ); /* Set useTempTable to TRUE if the result of the SELECT statement - ** should be written into a temporary table. Set to FALSE if each - ** row of the SELECT can be written directly into the result table. + ** should be written into a temporary table (template 4). Set to + ** FALSE if each* row of the SELECT can be written directly into + ** the destination table (template 3). ** ** A temp table must be used if the table being updated is also one ** of the tables being read by the SELECT statement. Also use a ** temp table in the case of row triggers. */ - if( triggers_exist || selectReadsTable(pSelect,pTab->pSchema,pTab->tnum) ){ + if( triggers_exist || readsTable(v, addrSelect, iDb, pTab) ){ useTempTable = 1; } if( useTempTable ){ - /* Generate the subroutine that SELECT calls to process each row of - ** the result. Store the result in a temporary table + /* Invoke the coroutine to extract information from the SELECT + ** and add it to a transient table srcTab. The code generated + ** here is from the 4th template: + ** + ** B: open temp table + ** L: yield X + ** if EOF goto M + ** insert row from R..R+n into temp table + ** goto L + ** M: ... */ - srcTab = pParse->nTab++; - sqlite3VdbeResolveLabel(v, iInsertBlock); - sqlite3VdbeAddOp(v, OP_MakeRecord, nColumn, 0); - sqlite3VdbeAddOp(v, OP_NewRowid, srcTab, 0); - sqlite3VdbeAddOp(v, OP_Pull, 1, 0); - sqlite3VdbeAddOp(v, OP_Insert, srcTab, OPFLAG_APPEND); - sqlite3VdbeAddOp(v, OP_Return, 0, 0); + int regRec; /* Register to hold packed record */ + int regTempRowid; /* Register to hold temp table ROWID */ + int addrTop; /* Label "L" */ + int addrIf; /* Address of jump to M */ - /* The following code runs first because the GOTO at the very top - ** of the program jumps to it. Create the temporary table, then jump - ** back up and execute the SELECT code above. - */ - sqlite3VdbeJumpHere(v, iInitCode); - sqlite3VdbeAddOp(v, OP_OpenEphemeral, srcTab, 0); - sqlite3VdbeAddOp(v, OP_SetNumColumns, srcTab, nColumn); - sqlite3VdbeAddOp(v, OP_Goto, 0, iSelectLoop); - sqlite3VdbeResolveLabel(v, iCleanup); - }else{ - sqlite3VdbeJumpHere(v, iInitCode); + srcTab = pParse->nTab++; + regRec = sqlite3GetTempReg(pParse); + regTempRowid = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, srcTab, nColumn); + addrTop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); + addrIf = sqlite3VdbeAddOp1(v, OP_If, regEof); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); + sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); + sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeJumpHere(v, addrIf); + sqlite3ReleaseTempReg(pParse, regRec); + sqlite3ReleaseTempReg(pParse, regTempRowid); } }else{ /* This is the case if the data for the INSERT is coming from a VALUES @@ -515,10 +600,10 @@ void sqlite3Insert( memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; srcTab = -1; - useTempTable = 0; + assert( useTempTable==0 ); nColumn = pList ? pList->nExpr : 0; for(i=0; ia[i].pExpr) ){ + if( sqlite3ResolveExprNames(&sNC, pList->a[i].pExpr) ){ goto insert_cleanup; } } @@ -592,41 +677,78 @@ void sqlite3Insert( /* Open the temp table for FOR EACH ROW triggers */ if( triggers_exist ){ - sqlite3VdbeAddOp(v, OP_OpenPseudo, newIdx, 0); - sqlite3VdbeAddOp(v, OP_SetNumColumns, newIdx, pTab->nCol); + sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, pTab->nCol); + sqlite3VdbeAddOp2(v, OP_OpenPseudo, newIdx, 0); } /* Initialize the count of rows to be inserted */ if( db->flags & SQLITE_CountRows ){ - iCntMem = pParse->nMem++; - sqlite3VdbeAddOp(v, OP_MemInt, 0, iCntMem); + regRowCount = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); } - /* Open tables and indices if there are no row triggers */ - if( !triggers_exist ){ - base = pParse->nTab; - sqlite3OpenTableAndIndices(pParse, pTab, base, OP_OpenWrite); + /* If this is not a view, open the table and and all indices */ + if( !isView ){ + int nIdx; + + baseCur = pParse->nTab; + nIdx = sqlite3OpenTableAndIndices(pParse, pTab, baseCur, OP_OpenWrite); + aRegIdx = sqlite3DbMallocRaw(db, sizeof(int)*(nIdx+1)); + if( aRegIdx==0 ){ + goto insert_cleanup; + } + for(i=0; inMem; + } } - /* If the data source is a temporary table, then we have to create - ** a loop because there might be multiple rows of data. If the data - ** source is a subroutine call from the SELECT statement, then we need - ** to launch the SELECT statement processing. - */ + /* This is the top of the main insertion loop */ if( useTempTable ){ - iBreak = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp(v, OP_Rewind, srcTab, iBreak); - iCont = sqlite3VdbeCurrentAddr(v); + /* This block codes the top of loop only. The complete loop is the + ** following pseudocode (template 4): + ** + ** rewind temp table + ** C: loop over rows of intermediate table + ** transfer values form intermediate table into
+ ** end loop + ** D: ... + */ + addrInsTop = sqlite3VdbeAddOp1(v, OP_Rewind, srcTab); + addrCont = sqlite3VdbeCurrentAddr(v); }else if( pSelect ){ - sqlite3VdbeAddOp(v, OP_Goto, 0, iSelectLoop); - sqlite3VdbeResolveLabel(v, iInsertBlock); + /* This block codes the top of loop only. The complete loop is the + ** following pseudocode (template 3): + ** + ** C: yield X + ** if EOF goto D + ** insert the select result into
from R..R+n + ** goto C + ** D: ... + */ + addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); + addrInsTop = sqlite3VdbeAddOp1(v, OP_If, regEof); } + /* Allocate registers for holding the rowid of the new row, + ** the content of the new row, and the assemblied row record. + */ + regRecord = ++pParse->nMem; + regRowid = regIns = pParse->nMem+1; + pParse->nMem += pTab->nCol + 1; + if( IsVirtual(pTab) ){ + regRowid++; + pParse->nMem++; + } + regData = regRowid+1; + /* Run the BEFORE and INSTEAD OF triggers, if there are any */ endOfLoop = sqlite3VdbeMakeLabel(v); if( triggers_exist & TRIGGER_BEFORE ){ + int regTrigRowid; + int regCols; + int regRec; /* build the NEW.* reference row. Note that if there is an INTEGER ** PRIMARY KEY into which a NULL is being inserted, that NULL will be @@ -634,17 +756,19 @@ void sqlite3Insert( ** we do not know what the unique ID will be (because the insert has ** not happened yet) so we substitute a rowid of -1 */ + regTrigRowid = sqlite3GetTempReg(pParse); if( keyColumn<0 ){ - sqlite3VdbeAddOp(v, OP_Integer, -1, 0); + sqlite3VdbeAddOp2(v, OP_Integer, -1, regTrigRowid); }else if( useTempTable ){ - sqlite3VdbeAddOp(v, OP_Column, srcTab, keyColumn); + sqlite3VdbeAddOp3(v, OP_Column, srcTab, keyColumn, regTrigRowid); }else{ + int j1; assert( pSelect==0 ); /* Otherwise useTempTable is true */ - sqlite3ExprCode(pParse, pList->a[keyColumn].pExpr); - sqlite3VdbeAddOp(v, OP_NotNull, -1, sqlite3VdbeCurrentAddr(v)+3); - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); - sqlite3VdbeAddOp(v, OP_Integer, -1, 0); - sqlite3VdbeAddOp(v, OP_MustBeInt, 0, 0); + sqlite3ExprCode(pParse, pList->a[keyColumn].pExpr, regTrigRowid); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regTrigRowid); + sqlite3VdbeAddOp2(v, OP_Integer, -1, regTrigRowid); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regTrigRowid); } /* Cannot have triggers on a virtual table. If it were possible, @@ -654,6 +778,7 @@ void sqlite3Insert( /* Create the new column data */ + regCols = sqlite3GetTempRange(pParse, pTab->nCol); for(i=0; inCol; i++){ if( pColumn==0 ){ j = i; @@ -663,15 +788,16 @@ void sqlite3Insert( } } if( pColumn && j>=pColumn->nId ){ - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt); + sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regCols+i); }else if( useTempTable ){ - sqlite3VdbeAddOp(v, OP_Column, srcTab, j); + sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, regCols+i); }else{ assert( pSelect==0 ); /* Otherwise useTempTable is true */ - sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr); + sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr, regCols+i); } } - sqlite3VdbeAddOp(v, OP_MakeRecord, pTab->nCol, 0); + regRec = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regCols, pTab->nCol, regRec); /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger, ** do not attempt any conversions before assembling the record. @@ -681,23 +807,18 @@ void sqlite3Insert( if( !isView ){ sqlite3TableAffinityStr(v, pTab); } - sqlite3VdbeAddOp(v, OP_Insert, newIdx, 0); + sqlite3VdbeAddOp3(v, OP_Insert, newIdx, regRec, regTrigRowid); + sqlite3ReleaseTempReg(pParse, regRec); + sqlite3ReleaseTempReg(pParse, regTrigRowid); + sqlite3ReleaseTempRange(pParse, regCols, pTab->nCol); /* Fire BEFORE or INSTEAD OF triggers */ if( sqlite3CodeRowTrigger(pParse, TK_INSERT, 0, TRIGGER_BEFORE, pTab, - newIdx, -1, onError, endOfLoop) ){ + newIdx, -1, onError, endOfLoop, 0, 0) ){ goto insert_cleanup; } } - /* If any triggers exists, the opening of tables and indices is deferred - ** until now. - */ - if( triggers_exist && !isView ){ - base = pParse->nTab; - sqlite3OpenTableAndIndices(pParse, pTab, base, OP_OpenWrite); - } - /* Push the record number for the new entry onto the stack. The ** record number is a randomly generate integer created by NewRowid ** except when the table has an INTEGER PRIMARY KEY column, in which @@ -705,53 +826,61 @@ void sqlite3Insert( */ if( !isView ){ if( IsVirtual(pTab) ){ - /* The row that the VUpdate opcode will delete: none */ - sqlite3VdbeAddOp(v, OP_Null, 0, 0); + /* The row that the VUpdate opcode will delete: none */ + sqlite3VdbeAddOp2(v, OP_Null, 0, regIns); } if( keyColumn>=0 ){ if( useTempTable ){ - sqlite3VdbeAddOp(v, OP_Column, srcTab, keyColumn); + sqlite3VdbeAddOp3(v, OP_Column, srcTab, keyColumn, regRowid); }else if( pSelect ){ - sqlite3VdbeAddOp(v, OP_Dup, nColumn - keyColumn - 1, 1); + sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+keyColumn, regRowid); }else{ VdbeOp *pOp; - sqlite3ExprCode(pParse, pList->a[keyColumn].pExpr); + sqlite3ExprCode(pParse, pList->a[keyColumn].pExpr, regRowid); pOp = sqlite3VdbeGetOp(v, sqlite3VdbeCurrentAddr(v) - 1); - if( pOp && pOp->opcode==OP_Null ){ + if( pOp && pOp->opcode==OP_Null && !IsVirtual(pTab) ){ appendFlag = 1; pOp->opcode = OP_NewRowid; - pOp->p1 = base; - pOp->p2 = counterMem; + pOp->p1 = baseCur; + pOp->p2 = regRowid; + pOp->p3 = regAutoinc; } } /* If the PRIMARY KEY expression is NULL, then use OP_NewRowid ** to generate a unique primary key value. */ if( !appendFlag ){ - sqlite3VdbeAddOp(v, OP_NotNull, -1, sqlite3VdbeCurrentAddr(v)+3); - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); - sqlite3VdbeAddOp(v, OP_NewRowid, base, counterMem); - sqlite3VdbeAddOp(v, OP_MustBeInt, 0, 0); + int j1; + if( !IsVirtual(pTab) ){ + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); + sqlite3VdbeAddOp3(v, OP_NewRowid, baseCur, regRowid, regAutoinc); + sqlite3VdbeJumpHere(v, j1); + }else{ + j1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); + } + sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); } }else if( IsVirtual(pTab) ){ - sqlite3VdbeAddOp(v, OP_Null, 0, 0); + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowid); }else{ - sqlite3VdbeAddOp(v, OP_NewRowid, base, counterMem); + sqlite3VdbeAddOp3(v, OP_NewRowid, baseCur, regRowid, regAutoinc); appendFlag = 1; } - autoIncStep(pParse, counterMem); + autoIncStep(pParse, regAutoinc, regRowid); /* Push onto the stack, data for all columns of the new entry, beginning ** with the first column. */ nHidden = 0; for(i=0; inCol; i++){ + int iRegStore = regRowid+1+i; if( i==pTab->iPKey ){ /* The value of the INTEGER PRIMARY KEY column is always a NULL. ** Whenever this column is read, the record number will be substituted ** in its place. So will fill this column with a NULL to avoid ** taking up data space with information that will never be used. */ - sqlite3VdbeAddOp(v, OP_Null, 0, 0); + sqlite3VdbeAddOp2(v, OP_Null, 0, iRegStore); continue; } if( pColumn==0 ){ @@ -768,13 +897,13 @@ void sqlite3Insert( } } if( j<0 || nColumn==0 || (pColumn && j>=pColumn->nId) ){ - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt); + sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, iRegStore); }else if( useTempTable ){ - sqlite3VdbeAddOp(v, OP_Column, srcTab, j); + sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, iRegStore); }else if( pSelect ){ - sqlite3VdbeAddOp(v, OP_Dup, i+nColumn-j+IsVirtual(pTab), 1); + sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+j, iRegStore); }else{ - sqlite3ExprCode(pParse, pList->a[j].pExpr); + sqlite3ExprCode(pParse, pList->a[j].pExpr, iRegStore); } } @@ -783,68 +912,76 @@ void sqlite3Insert( */ #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pTab) ){ - pParse->pVirtualLock = pTab; - sqlite3VdbeOp3(v, OP_VUpdate, 1, pTab->nCol+2, - (const char*)pTab->pVtab, P3_VTAB); + sqlite3VtabMakeWritable(pParse, pTab); + sqlite3VdbeAddOp4(v, OP_VUpdate, 1, pTab->nCol+2, regIns, + (const char*)pTab->pVtab, P4_VTAB); }else #endif { - sqlite3GenerateConstraintChecks(pParse, pTab, base, 0, keyColumn>=0, - 0, onError, endOfLoop); - sqlite3CompleteInsertion(pParse, pTab, base, 0,0,0, - (triggers_exist & TRIGGER_AFTER)!=0 ? newIdx : -1, - appendFlag); + sqlite3GenerateConstraintChecks( + pParse, + pTab, + baseCur, + regIns, + aRegIdx, + keyColumn>=0, + 0, + onError, + endOfLoop + ); + sqlite3CompleteInsertion( + pParse, + pTab, + baseCur, + regIns, + aRegIdx, + 0, + (triggers_exist & TRIGGER_AFTER)!=0 ? newIdx : -1, + appendFlag + ); } } /* Update the count of rows that are inserted */ if( (db->flags & SQLITE_CountRows)!=0 ){ - sqlite3VdbeAddOp(v, OP_MemIncr, 1, iCntMem); + sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); } if( triggers_exist ){ - /* Close all tables opened */ - if( !isView ){ - sqlite3VdbeAddOp(v, OP_Close, base, 0); - for(idx=1, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, idx++){ - sqlite3VdbeAddOp(v, OP_Close, idx+base, 0); - } - } - /* Code AFTER triggers */ if( sqlite3CodeRowTrigger(pParse, TK_INSERT, 0, TRIGGER_AFTER, pTab, - newIdx, -1, onError, endOfLoop) ){ + newIdx, -1, onError, endOfLoop, 0, 0) ){ goto insert_cleanup; } } - /* The bottom of the loop, if the data source is a SELECT statement + /* The bottom of the main insertion loop, if the data source + ** is a SELECT statement. */ sqlite3VdbeResolveLabel(v, endOfLoop); if( useTempTable ){ - sqlite3VdbeAddOp(v, OP_Next, srcTab, iCont); - sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp(v, OP_Close, srcTab, 0); + sqlite3VdbeAddOp2(v, OP_Next, srcTab, addrCont); + sqlite3VdbeJumpHere(v, addrInsTop); + sqlite3VdbeAddOp1(v, OP_Close, srcTab); }else if( pSelect ){ - sqlite3VdbeAddOp(v, OP_Pop, nColumn, 0); - sqlite3VdbeAddOp(v, OP_Return, 0, 0); - sqlite3VdbeResolveLabel(v, iCleanup); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrCont); + sqlite3VdbeJumpHere(v, addrInsTop); } - if( !triggers_exist && !IsVirtual(pTab) ){ + if( !IsVirtual(pTab) && !isView ){ /* Close all tables opened */ - sqlite3VdbeAddOp(v, OP_Close, base, 0); + sqlite3VdbeAddOp1(v, OP_Close, baseCur); for(idx=1, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, idx++){ - sqlite3VdbeAddOp(v, OP_Close, idx+base, 0); + sqlite3VdbeAddOp1(v, OP_Close, idx+baseCur); } } /* Update the sqlite_sequence table by storing the content of the - ** counter value in memory counterMem back into the sqlite_sequence + ** counter value in memory regAutoinc back into the sqlite_sequence ** table. */ - autoIncEnd(pParse, iDb, pTab, counterMem); + autoIncEnd(pParse, iDb, pTab, regAutoinc); /* ** Return the number of rows inserted. If this routine is @@ -852,28 +989,27 @@ void sqlite3Insert( ** invoke the callback function. */ if( db->flags & SQLITE_CountRows && pParse->nested==0 && !pParse->trigStack ){ - sqlite3VdbeAddOp(v, OP_MemLoad, iCntMem, 0); - sqlite3VdbeAddOp(v, OP_Callback, 1, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1); sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows inserted", P3_STATIC); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows inserted", SQLITE_STATIC); } insert_cleanup: - sqlite3SrcListDelete(pTabList); - sqlite3ExprListDelete(pList); - sqlite3SelectDelete(pSelect); - sqlite3IdListDelete(pColumn); + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprListDelete(db, pList); + sqlite3SelectDelete(db, pSelect); + sqlite3IdListDelete(db, pColumn); + sqlite3DbFree(db, aRegIdx); } /* -** Generate code to do a constraint check prior to an INSERT or an UPDATE. +** Generate code to do constraint checks prior to an INSERT or an UPDATE. ** -** When this routine is called, the stack contains (from bottom to top) -** the following values: +** The input is a range of consecutive registers as follows: ** ** 1. The rowid of the row to be updated before the update. This ** value is omitted unless we are doing an UPDATE that involves a -** change to the record number. +** change to the record number or writing to a virtual table. ** ** 2. The rowid of the row after the update. ** @@ -883,15 +1019,20 @@ insert_cleanup: ** ** N. The data in the last column of the entry after the update. ** +** The regRowid parameter is the index of the register containing (2). +** ** The old rowid shown as entry (1) above is omitted unless both isUpdate ** and rowidChng are 1. isUpdate is true for UPDATEs and false for -** INSERTs and rowidChng is true if the record number is being changed. +** INSERTs. RowidChng means that the new rowid is explicitly specified by +** the update or insert statement. If rowidChng is false, it means that +** the rowid is computed automatically in an insert or that the rowid value +** is not modified by the update. ** -** The code generated by this routine pushes additional entries onto -** the stack which are the keys for new index entries for the new record. -** The order of index keys is the same as the order of the indices on -** the pTable->pIndex list. A key is only created for index i if -** aIdxUsed!=0 and aIdxUsed[i]!=0. +** The code generated by this routine store new index entries into +** registers identified by aRegIdx[]. No index entry is created for +** indices where aRegIdx[i]==0. The order of indices in aRegIdx[] is +** the same as the order of indices on the linked list of indices +** attached to the table. ** ** This routine also generates code to check constraints. NOT NULL, ** CHECK, and UNIQUE constraints are all checked. If a constraint fails, @@ -933,23 +1074,18 @@ insert_cleanup: ** for the constraint is used. ** ** The calling routine must open a read/write cursor for pTab with -** cursor number "base". All indices of pTab must also have open -** read/write cursors with cursor number base+i for the i-th cursor. +** cursor number "baseCur". All indices of pTab must also have open +** read/write cursors with cursor number baseCur+i for the i-th cursor. ** Except, if there is no possibility of a REPLACE action then -** cursors do not need to be open for indices where aIdxUsed[i]==0. -** -** If the isUpdate flag is true, it means that the "base" cursor is -** initially pointing to an entry that is being updated. The isUpdate -** flag causes extra code to be generated so that the "base" cursor -** is still pointing at the same entry after the routine returns. -** Without the isUpdate flag, the "base" cursor might be moved. +** cursors do not need to be open for indices where aRegIdx[i]==0. */ void sqlite3GenerateConstraintChecks( Parse *pParse, /* The parser context */ Table *pTab, /* the table into which we are inserting */ - int base, /* Index of a read/write cursor pointing at pTab */ - char *aIdxUsed, /* Which indices are used. NULL means all are used */ - int rowidChng, /* True if the record number will change */ + int baseCur, /* Index of a read/write cursor pointing at pTab */ + int regRowid, /* Index of the range of input registers */ + int *aRegIdx, /* Register used by each index. 0 for unused indices */ + int rowidChng, /* True if the rowid might collide with existing entry */ int isUpdate, /* True for UPDATE, False for INSERT */ int overrideError, /* Override onError to this if not OE_Default */ int ignoreDest /* Jump to this label on an OE_Ignore resolution */ @@ -958,18 +1094,20 @@ void sqlite3GenerateConstraintChecks( Vdbe *v; int nCol; int onError; - int addr; - int extra; + int j1; /* Addresss of jump instruction */ + int j2 = 0, j3; /* Addresses of jump instructions */ + int regData; /* Register containing first data column */ int iCur; Index *pIdx; int seenReplace = 0; - int jumpInst1=0, jumpInst2; int hasTwoRowids = (isUpdate && rowidChng); v = sqlite3GetVdbe(pParse); assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ nCol = pTab->nCol; + regData = regRowid + 1; + /* Test all NOT NULL constraints. */ @@ -987,33 +1125,30 @@ void sqlite3GenerateConstraintChecks( if( onError==OE_Replace && pTab->aCol[i].pDflt==0 ){ onError = OE_Abort; } - sqlite3VdbeAddOp(v, OP_Dup, nCol-1-i, 1); - addr = sqlite3VdbeAddOp(v, OP_NotNull, 1, 0); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regData+i); assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail || onError==OE_Ignore || onError==OE_Replace ); switch( onError ){ case OE_Rollback: case OE_Abort: case OE_Fail: { - char *zMsg = 0; - sqlite3VdbeAddOp(v, OP_Halt, SQLITE_CONSTRAINT, onError); - sqlite3SetString(&zMsg, pTab->zName, ".", pTab->aCol[i].zName, - " may not be NULL", (char*)0); - sqlite3VdbeChangeP3(v, -1, zMsg, P3_DYNAMIC); + char *zMsg; + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_CONSTRAINT, onError); + zMsg = sqlite3MPrintf(pParse->db, "%s.%s may not be NULL", + pTab->zName, pTab->aCol[i].zName); + sqlite3VdbeChangeP4(v, -1, zMsg, P4_DYNAMIC); break; } case OE_Ignore: { - sqlite3VdbeAddOp(v, OP_Pop, nCol+1+hasTwoRowids, 0); - sqlite3VdbeAddOp(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); break; } case OE_Replace: { - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt); - sqlite3VdbeAddOp(v, OP_Push, nCol-i, 0); + sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regData+i); break; } } - sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeJumpHere(v, j1); } /* Test all CHECK constraints @@ -1021,17 +1156,13 @@ void sqlite3GenerateConstraintChecks( #ifndef SQLITE_OMIT_CHECK if( pTab->pCheck && (pParse->db->flags & SQLITE_IgnoreChecks)==0 ){ int allOk = sqlite3VdbeMakeLabel(v); - assert( pParse->ckOffset==0 ); - pParse->ckOffset = nCol; - sqlite3ExprIfTrue(pParse, pTab->pCheck, allOk, 1); - assert( pParse->ckOffset==nCol ); - pParse->ckOffset = 0; + pParse->ckBase = regData; + sqlite3ExprIfTrue(pParse, pTab->pCheck, allOk, SQLITE_JUMPIFNULL); onError = overrideError!=OE_Default ? overrideError : OE_Abort; if( onError==OE_Ignore ){ - sqlite3VdbeAddOp(v, OP_Pop, nCol+1+hasTwoRowids, 0); - sqlite3VdbeAddOp(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); }else{ - sqlite3VdbeAddOp(v, OP_Halt, SQLITE_CONSTRAINT, onError); + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_CONSTRAINT, onError); } sqlite3VdbeResolveLabel(v, allOk); } @@ -1049,70 +1180,66 @@ void sqlite3GenerateConstraintChecks( onError = OE_Abort; } - if( isUpdate ){ - sqlite3VdbeAddOp(v, OP_Dup, nCol+1, 1); - sqlite3VdbeAddOp(v, OP_Dup, nCol+1, 1); - jumpInst1 = sqlite3VdbeAddOp(v, OP_Eq, 0, 0); - } - sqlite3VdbeAddOp(v, OP_Dup, nCol, 1); - jumpInst2 = sqlite3VdbeAddOp(v, OP_NotExists, base, 0); - switch( onError ){ - default: { - onError = OE_Abort; - /* Fall thru into the next case */ + if( onError!=OE_Replace || pTab->pIndex ){ + if( isUpdate ){ + j2 = sqlite3VdbeAddOp3(v, OP_Eq, regRowid, 0, regRowid-1); } - case OE_Rollback: - case OE_Abort: - case OE_Fail: { - sqlite3VdbeOp3(v, OP_Halt, SQLITE_CONSTRAINT, onError, - "PRIMARY KEY must be unique", P3_STATIC); - break; - } - case OE_Replace: { - sqlite3GenerateRowIndexDelete(v, pTab, base, 0); - if( isUpdate ){ - sqlite3VdbeAddOp(v, OP_Dup, nCol+hasTwoRowids, 1); - sqlite3VdbeAddOp(v, OP_MoveGe, base, 0); + j3 = sqlite3VdbeAddOp3(v, OP_NotExists, baseCur, 0, regRowid); + switch( onError ){ + default: { + onError = OE_Abort; + /* Fall thru into the next case */ + } + case OE_Rollback: + case OE_Abort: + case OE_Fail: { + sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, onError, 0, + "PRIMARY KEY must be unique", P4_STATIC); + break; + } + case OE_Replace: { + sqlite3GenerateRowIndexDelete(pParse, pTab, baseCur, 0); + seenReplace = 1; + break; + } + case OE_Ignore: { + assert( seenReplace==0 ); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); + break; } - seenReplace = 1; - break; } - case OE_Ignore: { - assert( seenReplace==0 ); - sqlite3VdbeAddOp(v, OP_Pop, nCol+1+hasTwoRowids, 0); - sqlite3VdbeAddOp(v, OP_Goto, 0, ignoreDest); - break; + sqlite3VdbeJumpHere(v, j3); + if( isUpdate ){ + sqlite3VdbeJumpHere(v, j2); } } - sqlite3VdbeJumpHere(v, jumpInst2); - if( isUpdate ){ - sqlite3VdbeJumpHere(v, jumpInst1); - sqlite3VdbeAddOp(v, OP_Dup, nCol+1, 1); - sqlite3VdbeAddOp(v, OP_MoveGe, base, 0); - } } /* Test all UNIQUE constraints by creating entries for each UNIQUE ** index and making sure that duplicate entries do not already exist. ** Add the new records to the indices as we go. */ - extra = -1; for(iCur=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, iCur++){ - if( aIdxUsed && aIdxUsed[iCur]==0 ) continue; /* Skip unused indices */ - extra++; + int regIdx; + int regR; + + if( aRegIdx[iCur]==0 ) continue; /* Skip unused indices */ /* Create a key for accessing the index entry */ - sqlite3VdbeAddOp(v, OP_Dup, nCol+extra, 1); + regIdx = sqlite3GetTempRange(pParse, pIdx->nColumn+1); for(i=0; inColumn; i++){ int idx = pIdx->aiColumn[i]; if( idx==pTab->iPKey ){ - sqlite3VdbeAddOp(v, OP_Dup, i+extra+nCol+1, 1); + sqlite3VdbeAddOp2(v, OP_SCopy, regRowid, regIdx+i); }else{ - sqlite3VdbeAddOp(v, OP_Dup, i+extra+nCol-idx, 1); + sqlite3VdbeAddOp2(v, OP_SCopy, regData+idx, regIdx+i); } } - jumpInst1 = sqlite3VdbeAddOp(v, OP_MakeIdxRec, pIdx->nColumn, 0); + sqlite3VdbeAddOp2(v, OP_SCopy, regRowid, regIdx+i); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn+1, aRegIdx[iCur]); sqlite3IndexAffinityStr(v, pIdx); + sqlite3ExprCacheAffinityChange(pParse, regIdx, pIdx->nColumn+1); + sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn+1); /* Find out what action to take in case there is an indexing conflict */ onError = pIdx->onError; @@ -1129,8 +1256,12 @@ void sqlite3GenerateConstraintChecks( /* Check to see if the new index entry will be unique */ - sqlite3VdbeAddOp(v, OP_Dup, extra+nCol+1+hasTwoRowids, 1); - jumpInst2 = sqlite3VdbeAddOp(v, OP_IsUnique, base+iCur+1, 0); + j2 = sqlite3VdbeAddOp3(v, OP_IsNull, regIdx, 0, pIdx->nColumn); + regR = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_SCopy, regRowid-hasTwoRowids, regR); + j3 = sqlite3VdbeAddOp4(v, OP_IsUnique, baseCur+iCur+1, 0, + regR, SQLITE_INT_TO_PTR(aRegIdx[iCur]), + P4_INT32); /* Generate code that executes if the new index entry is not unique */ assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail @@ -1141,59 +1272,52 @@ void sqlite3GenerateConstraintChecks( case OE_Fail: { int j, n1, n2; char zErrMsg[200]; - sqlite3_snprintf(sizeof(zErrMsg), zErrMsg, + sqlite3_snprintf(ArraySize(zErrMsg), zErrMsg, pIdx->nColumn>1 ? "columns " : "column "); - n1 = strlen(zErrMsg); - for(j=0; jnColumn && n1nColumn && n1aCol[pIdx->aiColumn[j]].zName; - n2 = strlen(zCol); + n2 = sqlite3Strlen30(zCol); if( j>0 ){ - sqlite3_snprintf(sizeof(zErrMsg)-n1, &zErrMsg[n1], ", "); + sqlite3_snprintf(ArraySize(zErrMsg)-n1, &zErrMsg[n1], ", "); n1 += 2; } - if( n1+n2>sizeof(zErrMsg)-30 ){ - sqlite3_snprintf(sizeof(zErrMsg)-n1, &zErrMsg[n1], "..."); + if( n1+n2>ArraySize(zErrMsg)-30 ){ + sqlite3_snprintf(ArraySize(zErrMsg)-n1, &zErrMsg[n1], "..."); n1 += 3; break; }else{ - sqlite3_snprintf(sizeof(zErrMsg)-n1, &zErrMsg[n1], "%s", zCol); + sqlite3_snprintf(ArraySize(zErrMsg)-n1, &zErrMsg[n1], "%s", zCol); n1 += n2; } } - sqlite3_snprintf(sizeof(zErrMsg)-n1, &zErrMsg[n1], + sqlite3_snprintf(ArraySize(zErrMsg)-n1, &zErrMsg[n1], pIdx->nColumn>1 ? " are not unique" : " is not unique"); - sqlite3VdbeOp3(v, OP_Halt, SQLITE_CONSTRAINT, onError, zErrMsg, 0); + sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, onError, 0, zErrMsg,0); break; } case OE_Ignore: { assert( seenReplace==0 ); - sqlite3VdbeAddOp(v, OP_Pop, nCol+extra+3+hasTwoRowids, 0); - sqlite3VdbeAddOp(v, OP_Goto, 0, ignoreDest); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); break; } case OE_Replace: { - sqlite3GenerateRowDelete(pParse->db, v, pTab, base, 0); - if( isUpdate ){ - sqlite3VdbeAddOp(v, OP_Dup, nCol+extra+1+hasTwoRowids, 1); - sqlite3VdbeAddOp(v, OP_MoveGe, base, 0); - } + sqlite3GenerateRowDelete(pParse, pTab, baseCur, regR, 0); seenReplace = 1; break; } } -#if NULL_DISTINCT_FOR_UNIQUE - sqlite3VdbeJumpHere(v, jumpInst1); -#endif - sqlite3VdbeJumpHere(v, jumpInst2); + sqlite3VdbeJumpHere(v, j2); + sqlite3VdbeJumpHere(v, j3); + sqlite3ReleaseTempReg(pParse, regR); } } /* ** This routine generates code to finish the INSERT or UPDATE operation ** that was started by a prior call to sqlite3GenerateConstraintChecks. -** The stack must contain keys for all active indices followed by data -** and the rowid for the new entry. This routine creates the new -** entries in all indices and in the main table. +** A consecutive range of registers starting at regRowid contains the +** rowid and the content to be inserted. ** ** The arguments to this routine should be the same as the first six ** arguments to sqlite3GenerateConstraintChecks. @@ -1201,9 +1325,9 @@ void sqlite3GenerateConstraintChecks( void sqlite3CompleteInsertion( Parse *pParse, /* The parser context */ Table *pTab, /* the table into which we are inserting */ - int base, /* Index of a read/write cursor pointing at pTab */ - char *aIdxUsed, /* Which indices are used. NULL means all are used */ - int rowidChng, /* True if the record number will change */ + int baseCur, /* Index of a read/write cursor pointing at pTab */ + int regRowid, /* Range of content */ + int *aRegIdx, /* Register used by each index. 0 for unused indices */ int isUpdate, /* True for UPDATE, False for INSERT */ int newIdx, /* Index of NEW table for triggers. -1 if none */ int appendBias /* True if this is likely to be an append */ @@ -1212,23 +1336,26 @@ void sqlite3CompleteInsertion( Vdbe *v; int nIdx; Index *pIdx; - int pik_flags; + u8 pik_flags; + int regData; + int regRec; v = sqlite3GetVdbe(pParse); assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){} for(i=nIdx-1; i>=0; i--){ - if( aIdxUsed && aIdxUsed[i]==0 ) continue; - sqlite3VdbeAddOp(v, OP_IdxInsert, base+i+1, 0); + if( aRegIdx[i]==0 ) continue; + sqlite3VdbeAddOp2(v, OP_IdxInsert, baseCur+i+1, aRegIdx[i]); } - sqlite3VdbeAddOp(v, OP_MakeRecord, pTab->nCol, 0); + regData = regRowid + 1; + regRec = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); sqlite3TableAffinityStr(v, pTab); + sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); #ifndef SQLITE_OMIT_TRIGGER if( newIdx>=0 ){ - sqlite3VdbeAddOp(v, OP_Dup, 1, 0); - sqlite3VdbeAddOp(v, OP_Dup, 1, 0); - sqlite3VdbeAddOp(v, OP_Insert, newIdx, 0); + sqlite3VdbeAddOp3(v, OP_Insert, newIdx, regRec, regRowid); } #endif if( pParse->nested ){ @@ -1240,25 +1367,24 @@ void sqlite3CompleteInsertion( if( appendBias ){ pik_flags |= OPFLAG_APPEND; } - sqlite3VdbeAddOp(v, OP_Insert, base, pik_flags); + sqlite3VdbeAddOp3(v, OP_Insert, baseCur, regRec, regRowid); if( !pParse->nested ){ - sqlite3VdbeChangeP3(v, -1, pTab->zName, P3_STATIC); - } - - if( isUpdate && rowidChng ){ - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); + sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_STATIC); } + sqlite3VdbeChangeP5(v, pik_flags); } /* ** Generate code that will open cursors for a table and for all -** indices of that table. The "base" parameter is the cursor number used +** indices of that table. The "baseCur" parameter is the cursor number used ** for the table. Indices are opened on subsequent cursors. +** +** Return the number of indices on the table. */ -void sqlite3OpenTableAndIndices( +int sqlite3OpenTableAndIndices( Parse *pParse, /* Parsing context */ Table *pTab, /* Table to be opened */ - int base, /* Cursor number assigned to the table */ + int baseCur, /* Cursor number assigned to the table */ int op /* OP_OpenRead or OP_OpenWrite */ ){ int i; @@ -1266,21 +1392,22 @@ void sqlite3OpenTableAndIndices( Index *pIdx; Vdbe *v; - if( IsVirtual(pTab) ) return; + if( IsVirtual(pTab) ) return 0; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); v = sqlite3GetVdbe(pParse); assert( v!=0 ); - sqlite3OpenTable(pParse, base, iDb, pTab, op); + sqlite3OpenTable(pParse, baseCur, iDb, pTab, op); for(i=1, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); assert( pIdx->pSchema==pTab->pSchema ); - sqlite3VdbeAddOp(v, OP_Integer, iDb, 0); - VdbeComment((v, "# %s", pIdx->zName)); - sqlite3VdbeOp3(v, op, i+base, pIdx->tnum, (char*)pKey, P3_KEYINFO_HANDOFF); + sqlite3VdbeAddOp4(v, op, i+baseCur, pIdx->tnum, iDb, + (char*)pKey, P4_KEYINFO_HANDOFF); + VdbeComment((v, "%s", pIdx->zName)); } - if( pParse->nTab<=base+i ){ - pParse->nTab = base+i; + if( pParse->nTab<=baseCur+i ){ + pParse->nTab = baseCur+i; } + return i-1; } @@ -1338,7 +1465,7 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ return 0; /* Different sort orders */ } if( pSrc->azColl[i]!=pDest->azColl[i] ){ - return 0; /* Different sort orders */ + return 0; /* Different collating sequences */ } } @@ -1396,8 +1523,9 @@ static int xferOptimization( int emptySrcTest; /* Address of test for empty pSrc */ Vdbe *v; /* The VDBE we are building */ KeyInfo *pKey; /* Key information for an index */ - int counterMem; /* Memory register used by AUTOINC */ + int regAutoinc; /* Memory register used by AUTOINC */ int destHasUniqueIdx = 0; /* True if pDest has a UNIQUE index */ + int regData, regRowid; /* Registers holding data and rowid */ if( pSelect==0 ){ return 0; /* Must be of the form INSERT INTO ... SELECT ... */ @@ -1406,7 +1534,7 @@ static int xferOptimization( return 0; /* tab1 must not have triggers */ } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pDest->isVirtual ){ + if( pDest->tabFlags & TF_Virtual ){ return 0; /* tab1 must not be a virtual table */ } #endif @@ -1416,9 +1544,7 @@ static int xferOptimization( if( onError!=OE_Abort && onError!=OE_Rollback ){ return 0; /* Cannot do OR REPLACE or OR IGNORE or OR FAIL */ } - if( pSelect->pSrc==0 ){ - return 0; /* SELECT must have a FROM clause */ - } + assert(pSelect->pSrc); /* allocated even if there is no FROM clause */ if( pSelect->pSrc->nSrc!=1 ){ return 0; /* FROM clause must have exactly one term */ } @@ -1443,7 +1569,7 @@ static int xferOptimization( if( pSelect->pPrior ){ return 0; /* SELECT may not be a compound query */ } - if( pSelect->isDistinct ){ + if( pSelect->selFlags & SF_Distinct ){ return 0; /* SELECT may not be DISTINCT */ } pEList = pSelect->pEList; @@ -1461,7 +1587,7 @@ static int xferOptimization( ** we have to check the semantics. */ pItem = pSelect->pSrc->a; - pSrc = sqlite3LocateTable(pParse, pItem->zName, pItem->zDatabase); + pSrc = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); if( pSrc==0 ){ return 0; /* FROM clause does not contain a real table */ } @@ -1469,7 +1595,7 @@ static int xferOptimization( return 0; /* tab1 and tab2 may not be the same table */ } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pSrc->isVirtual ){ + if( pSrc->tabFlags & TF_Virtual ){ return 0; /* tab2 must not be a virtual table */ } #endif @@ -1523,9 +1649,10 @@ static int xferOptimization( #endif iDbSrc = sqlite3SchemaToIndex(pParse->db, pSrc->pSchema); v = sqlite3GetVdbe(pParse); + sqlite3CodeVerifySchema(pParse, iDbSrc); iSrc = pParse->nTab++; iDest = pParse->nTab++; - counterMem = autoIncBegin(pParse, iDbDest, pDest); + regAutoinc = autoIncBegin(pParse, iDbDest, pDest); sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); if( (pDest->iPKey<0 && pDest->pIndex!=0) || destHasUniqueIdx ){ /* If tables do not have an INTEGER PRIMARY KEY and there @@ -1538,67 +1665,73 @@ static int xferOptimization( ** insure that all entries in the union of DEST and SRC will be ** unique. */ - addr1 = sqlite3VdbeAddOp(v, OP_Rewind, iDest, 0); - emptyDestTest = sqlite3VdbeAddOp(v, OP_Goto, 0, 0); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); + emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); sqlite3VdbeJumpHere(v, addr1); }else{ emptyDestTest = 0; } sqlite3OpenTable(pParse, iSrc, iDbSrc, pSrc, OP_OpenRead); - emptySrcTest = sqlite3VdbeAddOp(v, OP_Rewind, iSrc, 0); + emptySrcTest = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); + regData = sqlite3GetTempReg(pParse); + regRowid = sqlite3GetTempReg(pParse); if( pDest->iPKey>=0 ){ - addr1 = sqlite3VdbeAddOp(v, OP_Rowid, iSrc, 0); - sqlite3VdbeAddOp(v, OP_Dup, 0, 0); - addr2 = sqlite3VdbeAddOp(v, OP_NotExists, iDest, 0); - sqlite3VdbeOp3(v, OP_Halt, SQLITE_CONSTRAINT, onError, - "PRIMARY KEY must be unique", P3_STATIC); + addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); + addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); + sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CONSTRAINT, onError, 0, + "PRIMARY KEY must be unique", P4_STATIC); sqlite3VdbeJumpHere(v, addr2); - autoIncStep(pParse, counterMem); + autoIncStep(pParse, regAutoinc, regRowid); }else if( pDest->pIndex==0 ){ - addr1 = sqlite3VdbeAddOp(v, OP_NewRowid, iDest, 0); + addr1 = sqlite3VdbeAddOp2(v, OP_NewRowid, iDest, regRowid); }else{ - addr1 = sqlite3VdbeAddOp(v, OP_Rowid, iSrc, 0); - assert( pDest->autoInc==0 ); + addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); + assert( (pDest->tabFlags & TF_Autoincrement)==0 ); } - sqlite3VdbeAddOp(v, OP_RowData, iSrc, 0); - sqlite3VdbeOp3(v, OP_Insert, iDest, - OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND, - pDest->zName, 0); - sqlite3VdbeAddOp(v, OP_Next, iSrc, addr1); - autoIncEnd(pParse, iDbDest, pDest, counterMem); + sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); + sqlite3VdbeChangeP4(v, -1, pDest->zName, 0); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); + autoIncEnd(pParse, iDbDest, pDest, regAutoinc); for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ for(pSrcIdx=pSrc->pIndex; pSrcIdx; pSrcIdx=pSrcIdx->pNext){ if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; } assert( pSrcIdx ); - sqlite3VdbeAddOp(v, OP_Close, iSrc, 0); - sqlite3VdbeAddOp(v, OP_Close, iDest, 0); - sqlite3VdbeAddOp(v, OP_Integer, iDbSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); pKey = sqlite3IndexKeyinfo(pParse, pSrcIdx); - VdbeComment((v, "# %s", pSrcIdx->zName)); - sqlite3VdbeOp3(v, OP_OpenRead, iSrc, pSrcIdx->tnum, - (char*)pKey, P3_KEYINFO_HANDOFF); - sqlite3VdbeAddOp(v, OP_Integer, iDbDest, 0); + sqlite3VdbeAddOp4(v, OP_OpenRead, iSrc, pSrcIdx->tnum, iDbSrc, + (char*)pKey, P4_KEYINFO_HANDOFF); + VdbeComment((v, "%s", pSrcIdx->zName)); pKey = sqlite3IndexKeyinfo(pParse, pDestIdx); - VdbeComment((v, "# %s", pDestIdx->zName)); - sqlite3VdbeOp3(v, OP_OpenWrite, iDest, pDestIdx->tnum, - (char*)pKey, P3_KEYINFO_HANDOFF); - addr1 = sqlite3VdbeAddOp(v, OP_Rewind, iSrc, 0); - sqlite3VdbeAddOp(v, OP_RowKey, iSrc, 0); - sqlite3VdbeAddOp(v, OP_IdxInsert, iDest, 1); - sqlite3VdbeAddOp(v, OP_Next, iSrc, addr1+1); + sqlite3VdbeAddOp4(v, OP_OpenWrite, iDest, pDestIdx->tnum, iDbDest, + (char*)pKey, P4_KEYINFO_HANDOFF); + VdbeComment((v, "%s", pDestIdx->zName)); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_RowKey, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); sqlite3VdbeJumpHere(v, addr1); } sqlite3VdbeJumpHere(v, emptySrcTest); - sqlite3VdbeAddOp(v, OP_Close, iSrc, 0); - sqlite3VdbeAddOp(v, OP_Close, iDest, 0); + sqlite3ReleaseTempReg(pParse, regRowid); + sqlite3ReleaseTempReg(pParse, regData); + sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); if( emptyDestTest ){ - sqlite3VdbeAddOp(v, OP_Halt, SQLITE_OK, 0); + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, 0); sqlite3VdbeJumpHere(v, emptyDestTest); - sqlite3VdbeAddOp(v, OP_Close, iDest, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); return 0; }else{ return 1; } } #endif /* SQLITE_OMIT_XFER_OPT */ + +/* Make sure "isView" gets undefined in case this file becomes part of +** the amalgamation - so that subsequent files do not see isView as a +** macro. */ +#undef isView diff --git a/journal.c b/journal.c new file mode 100644 index 0000000..bdc36bc --- /dev/null +++ b/journal.c @@ -0,0 +1,239 @@ +/* +** 2007 August 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** @(#) $Id: journal.c,v 1.8 2008/05/01 18:01:47 drh Exp $ +*/ + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + +/* +** This file implements a special kind of sqlite3_file object used +** by SQLite to create journal files if the atomic-write optimization +** is enabled. +** +** The distinctive characteristic of this sqlite3_file is that the +** actual on disk file is created lazily. When the file is created, +** the caller specifies a buffer size for an in-memory buffer to +** be used to service read() and write() requests. The actual file +** on disk is not created or populated until either: +** +** 1) The in-memory representation grows too large for the allocated +** buffer, or +** 2) The xSync() method is called. +*/ + +#include "sqliteInt.h" + + +/* +** A JournalFile object is a subclass of sqlite3_file used by +** as an open file handle for journal files. +*/ +struct JournalFile { + sqlite3_io_methods *pMethod; /* I/O methods on journal files */ + int nBuf; /* Size of zBuf[] in bytes */ + char *zBuf; /* Space to buffer journal writes */ + int iSize; /* Amount of zBuf[] currently used */ + int flags; /* xOpen flags */ + sqlite3_vfs *pVfs; /* The "real" underlying VFS */ + sqlite3_file *pReal; /* The "real" underlying file descriptor */ + const char *zJournal; /* Name of the journal file */ +}; +typedef struct JournalFile JournalFile; + +/* +** If it does not already exists, create and populate the on-disk file +** for JournalFile p. +*/ +static int createFile(JournalFile *p){ + int rc = SQLITE_OK; + if( !p->pReal ){ + sqlite3_file *pReal = (sqlite3_file *)&p[1]; + rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0); + if( rc==SQLITE_OK ){ + p->pReal = pReal; + if( p->iSize>0 ){ + assert(p->iSize<=p->nBuf); + rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); + } + } + } + return rc; +} + +/* +** Close the file. +*/ +static int jrnlClose(sqlite3_file *pJfd){ + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + sqlite3OsClose(p->pReal); + } + sqlite3_free(p->zBuf); + return SQLITE_OK; +} + +/* +** Read data from the file. +*/ +static int jrnlRead( + sqlite3_file *pJfd, /* The journal file from which to read */ + void *zBuf, /* Put the results here */ + int iAmt, /* Number of bytes to read */ + sqlite_int64 iOfst /* Begin reading at this offset */ +){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst); + }else{ + assert( iAmt+iOfst<=p->iSize ); + memcpy(zBuf, &p->zBuf[iOfst], iAmt); + } + return rc; +} + +/* +** Write data to the file. +*/ +static int jrnlWrite( + sqlite3_file *pJfd, /* The journal file into which to write */ + const void *zBuf, /* Take data to be written from here */ + int iAmt, /* Number of bytes to write */ + sqlite_int64 iOfst /* Begin writing at this offset into the file */ +){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( !p->pReal && (iOfst+iAmt)>p->nBuf ){ + rc = createFile(p); + } + if( rc==SQLITE_OK ){ + if( p->pReal ){ + rc = sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); + }else{ + memcpy(&p->zBuf[iOfst], zBuf, iAmt); + if( p->iSize<(iOfst+iAmt) ){ + p->iSize = (iOfst+iAmt); + } + } + } + return rc; +} + +/* +** Truncate the file. +*/ +static int jrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsTruncate(p->pReal, size); + }else if( sizeiSize ){ + p->iSize = size; + } + return rc; +} + +/* +** Sync the file. +*/ +static int jrnlSync(sqlite3_file *pJfd, int flags){ + int rc; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsSync(p->pReal, flags); + }else{ + rc = SQLITE_OK; + } + return rc; +} + +/* +** Query the size of the file in bytes. +*/ +static int jrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ + int rc = SQLITE_OK; + JournalFile *p = (JournalFile *)pJfd; + if( p->pReal ){ + rc = sqlite3OsFileSize(p->pReal, pSize); + }else{ + *pSize = (sqlite_int64) p->iSize; + } + return rc; +} + +/* +** Table of methods for JournalFile sqlite3_file object. +*/ +static struct sqlite3_io_methods JournalFileMethods = { + 1, /* iVersion */ + jrnlClose, /* xClose */ + jrnlRead, /* xRead */ + jrnlWrite, /* xWrite */ + jrnlTruncate, /* xTruncate */ + jrnlSync, /* xSync */ + jrnlFileSize, /* xFileSize */ + 0, /* xLock */ + 0, /* xUnlock */ + 0, /* xCheckReservedLock */ + 0, /* xFileControl */ + 0, /* xSectorSize */ + 0 /* xDeviceCharacteristics */ +}; + +/* +** Open a journal file. +*/ +int sqlite3JournalOpen( + sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ + const char *zName, /* Name of the journal file */ + sqlite3_file *pJfd, /* Preallocated, blank file handle */ + int flags, /* Opening flags */ + int nBuf /* Bytes buffered before opening the file */ +){ + JournalFile *p = (JournalFile *)pJfd; + memset(p, 0, sqlite3JournalSize(pVfs)); + if( nBuf>0 ){ + p->zBuf = sqlite3MallocZero(nBuf); + if( !p->zBuf ){ + return SQLITE_NOMEM; + } + }else{ + return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); + } + p->pMethod = &JournalFileMethods; + p->nBuf = nBuf; + p->flags = flags; + p->zJournal = zName; + p->pVfs = pVfs; + return SQLITE_OK; +} + +/* +** If the argument p points to a JournalFile structure, and the underlying +** file has not yet been created, create it now. +*/ +int sqlite3JournalCreate(sqlite3_file *p){ + if( p->pMethods!=&JournalFileMethods ){ + return SQLITE_OK; + } + return createFile((JournalFile *)p); +} + +/* +** Return the number of bytes required to store a JournalFile that uses vfs +** pVfs to create the underlying on-disk files. +*/ +int sqlite3JournalSize(sqlite3_vfs *pVfs){ + return (pVfs->szOsFile+sizeof(JournalFile)); +} +#endif diff --git a/keywordhash.h b/keywordhash.h index 9d127b0..ce29fa9 100644 --- a/keywordhash.h +++ b/keywordhash.h @@ -2,7 +2,7 @@ ** ** The code in this file has been automatically generated by ** -** $Header: /sqlite/sqlite/tool/mkkeywordhash.c,v 1.31 2007/07/30 18:26:20 rse Exp $ +** $Header: /sqlite/sqlite/tool/mkkeywordhash.c,v 1.36 2008/12/31 21:52:41 drh Exp $ ** ** The code in this file implements a function that determines whether ** or not a given identifier is really an SQL keyword. The same thing @@ -11,89 +11,124 @@ ** is substantially reduced. This is important for embedded applications ** on platforms with limited memory. */ -/* Hash score: 165 */ +/* Hash score: 171 */ static int keywordCode(const char *z, int n){ - /* zText[] encodes 775 bytes of keywords in 526 bytes */ - static const char zText[526] = - "BEFOREIGNOREGEXPLAINSTEADDESCAPEACHECKEYCONSTRAINTERSECTABLEFT" - "HENDATABASELECTRANSACTIONATURALTERAISELSEXCEPTRIGGEREFERENCES" - "UNIQUERYATTACHAVINGROUPDATEMPORARYBEGINNEREINDEXCLUSIVEXISTSBETWEEN" - "OTNULLIKECASCADEFERRABLECASECOLLATECREATECURRENT_DATEDELETEDETACH" - "IMMEDIATEJOINSERTMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMITWHEN" - "WHERENAMEAFTEREPLACEANDEFAULTAUTOINCREMENTCASTCOLUMNCOMMITCONFLICT" - "CROSSCURRENT_TIMESTAMPRIMARYDEFERREDISTINCTDROPFAILFROMFULLGLOB" - "YIFINTOFFSETISNULLORDERESTRICTOUTERIGHTROLLBACKROWUNIONUSINGVACUUM" - "VIEWINITIALLY"; + /* zText[] encodes 801 bytes of keywords in 541 bytes */ + /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ + /* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ + /* XISTSAVEPOINTERSECTRIGGEREFERENCESCONSTRAINTOFFSETEMPORARY */ + /* UNIQUERYATTACHAVINGROUPDATEBEGINNERELEASEBETWEENOTNULLIKE */ + /* CASCADELETECASECOLLATECREATECURRENT_DATEDETACHIMMEDIATEJOIN */ + /* SERTMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMITWHENWHERENAME */ + /* AFTEREPLACEANDEFAULTAUTOINCREMENTCASTCOLUMNCOMMITCONFLICTCROSS */ + /* CURRENT_TIMESTAMPRIMARYDEFERREDISTINCTDROPFAILFROMFULLGLOBYIF */ + /* ISNULLORDERESTRICTOUTERIGHTROLLBACKROWUNIONUSINGVACUUMVIEW */ + /* INITIALLY */ + static const char zText[540] = { + 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', + 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', + 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', + 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', + 'E','R','R','A','B','L','E','L','S','E','X','C','E','P','T','R','A','N', + 'S','A','C','T','I','O','N','A','T','U','R','A','L','T','E','R','A','I', + 'S','E','X','C','L','U','S','I','V','E','X','I','S','T','S','A','V','E', + 'P','O','I','N','T','E','R','S','E','C','T','R','I','G','G','E','R','E', + 'F','E','R','E','N','C','E','S','C','O','N','S','T','R','A','I','N','T', + 'O','F','F','S','E','T','E','M','P','O','R','A','R','Y','U','N','I','Q', + 'U','E','R','Y','A','T','T','A','C','H','A','V','I','N','G','R','O','U', + 'P','D','A','T','E','B','E','G','I','N','N','E','R','E','L','E','A','S', + 'E','B','E','T','W','E','E','N','O','T','N','U','L','L','I','K','E','C', + 'A','S','C','A','D','E','L','E','T','E','C','A','S','E','C','O','L','L', + 'A','T','E','C','R','E','A','T','E','C','U','R','R','E','N','T','_','D', + 'A','T','E','D','E','T','A','C','H','I','M','M','E','D','I','A','T','E', + 'J','O','I','N','S','E','R','T','M','A','T','C','H','P','L','A','N','A', + 'L','Y','Z','E','P','R','A','G','M','A','B','O','R','T','V','A','L','U', + 'E','S','V','I','R','T','U','A','L','I','M','I','T','W','H','E','N','W', + 'H','E','R','E','N','A','M','E','A','F','T','E','R','E','P','L','A','C', + 'E','A','N','D','E','F','A','U','L','T','A','U','T','O','I','N','C','R', + 'E','M','E','N','T','C','A','S','T','C','O','L','U','M','N','C','O','M', + 'M','I','T','C','O','N','F','L','I','C','T','C','R','O','S','S','C','U', + 'R','R','E','N','T','_','T','I','M','E','S','T','A','M','P','R','I','M', + 'A','R','Y','D','E','F','E','R','R','E','D','I','S','T','I','N','C','T', + 'D','R','O','P','F','A','I','L','F','R','O','M','F','U','L','L','G','L', + 'O','B','Y','I','F','I','S','N','U','L','L','O','R','D','E','R','E','S', + 'T','R','I','C','T','O','U','T','E','R','I','G','H','T','R','O','L','L', + 'B','A','C','K','R','O','W','U','N','I','O','N','U','S','I','N','G','V', + 'A','C','U','U','M','V','I','E','W','I','N','I','T','I','A','L','L','Y', + }; static const unsigned char aHash[127] = { - 63, 92, 109, 61, 0, 39, 0, 0, 69, 0, 64, 0, 0, - 101, 4, 65, 7, 0, 108, 72, 103, 99, 0, 22, 0, 0, - 113, 0, 111, 106, 0, 18, 80, 0, 1, 0, 0, 56, 57, - 0, 55, 11, 0, 33, 77, 89, 0, 110, 88, 0, 0, 45, - 0, 90, 54, 0, 20, 0, 114, 34, 19, 0, 10, 97, 28, - 83, 0, 0, 116, 93, 47, 115, 41, 12, 44, 0, 78, 0, - 87, 29, 0, 86, 0, 0, 0, 82, 79, 84, 75, 96, 6, - 14, 95, 0, 68, 0, 21, 76, 98, 27, 0, 112, 67, 104, - 49, 40, 71, 0, 0, 81, 100, 0, 107, 0, 15, 0, 0, - 24, 0, 73, 42, 50, 0, 16, 48, 0, 37, + 70, 99, 112, 68, 0, 43, 0, 0, 76, 0, 71, 0, 0, + 41, 12, 72, 15, 0, 111, 79, 49, 106, 0, 19, 0, 0, + 116, 0, 114, 109, 0, 22, 87, 0, 9, 0, 0, 64, 65, + 0, 63, 6, 0, 47, 84, 96, 0, 113, 95, 0, 0, 44, + 0, 97, 24, 0, 17, 0, 117, 48, 23, 0, 5, 104, 25, + 90, 0, 0, 119, 100, 55, 118, 52, 7, 50, 0, 85, 0, + 94, 26, 0, 93, 0, 0, 0, 89, 86, 91, 82, 103, 14, + 38, 102, 0, 75, 0, 18, 83, 105, 31, 0, 115, 74, 107, + 56, 45, 78, 0, 0, 88, 39, 0, 110, 0, 35, 0, 0, + 28, 0, 80, 53, 58, 0, 20, 57, 0, 51, }; - static const unsigned char aNext[116] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, - 17, 0, 0, 0, 36, 38, 0, 0, 25, 0, 0, 31, 0, - 0, 0, 43, 52, 0, 0, 0, 53, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 51, 0, 0, 0, 0, 26, 0, 8, 46, - 3, 0, 0, 0, 0, 0, 0, 0, 2, 58, 66, 0, 13, - 0, 91, 85, 0, 94, 0, 74, 0, 0, 0, 62, 35, 102, - 0, 0, 105, 23, 30, 60, 70, 0, 0, 59, 0, 0, + static const unsigned char aNext[119] = { + 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 3, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 32, 21, 0, 0, 0, 42, 2, 46, 0, + 0, 0, 0, 0, 29, 0, 37, 0, 0, 0, 1, 60, 0, + 0, 61, 0, 40, 0, 0, 0, 0, 0, 0, 0, 59, 0, + 0, 0, 0, 30, 54, 16, 33, 11, 0, 0, 0, 0, 0, + 0, 0, 10, 66, 73, 0, 8, 0, 98, 92, 0, 101, 0, + 81, 0, 69, 0, 0, 108, 27, 36, 67, 77, 0, 34, 62, + 0, 0, }; - static const unsigned char aLen[116] = { - 6, 3, 7, 6, 6, 7, 7, 3, 4, 6, 4, 5, 3, - 10, 9, 5, 4, 4, 3, 8, 2, 6, 11, 2, 7, 5, - 5, 4, 6, 7, 10, 6, 5, 6, 6, 5, 6, 9, 4, - 2, 5, 5, 7, 5, 9, 6, 7, 7, 3, 4, 4, 7, - 3, 10, 4, 7, 6, 12, 6, 6, 9, 4, 6, 5, 4, - 7, 6, 5, 6, 7, 5, 4, 5, 6, 5, 7, 3, 7, - 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, 7, 8, 8, - 2, 4, 4, 4, 4, 4, 2, 2, 4, 2, 6, 3, 6, - 5, 8, 5, 5, 8, 3, 5, 5, 6, 4, 9, 3, + static const unsigned char aLen[119] = { + 7, 5, 7, 4, 6, 4, 5, 3, 6, 3, 7, 6, 6, + 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 6, + 11, 2, 7, 5, 5, 9, 6, 9, 9, 7, 10, 10, 4, + 6, 2, 3, 4, 9, 2, 6, 5, 6, 6, 5, 6, 5, + 5, 7, 7, 3, 7, 4, 4, 7, 3, 6, 4, 7, 6, + 12, 6, 9, 4, 6, 5, 4, 7, 6, 5, 6, 7, 5, + 4, 5, 6, 5, 7, 3, 7, 13, 2, 2, 4, 6, 6, + 8, 5, 17, 12, 7, 8, 8, 2, 4, 4, 4, 4, 4, + 2, 2, 6, 5, 8, 5, 5, 8, 3, 5, 5, 6, 4, + 9, 3, }; - static const unsigned short int aOffset[116] = { - 0, 2, 2, 6, 10, 13, 18, 23, 25, 26, 31, 33, 37, - 40, 47, 55, 58, 61, 63, 65, 70, 71, 76, 85, 86, 91, - 95, 99, 102, 107, 113, 123, 126, 131, 136, 141, 144, 148, 148, - 152, 157, 160, 164, 166, 169, 177, 183, 189, 189, 192, 195, 199, - 200, 204, 214, 218, 225, 231, 243, 249, 255, 264, 266, 272, 277, - 279, 286, 291, 296, 302, 308, 313, 317, 320, 326, 330, 337, 339, - 346, 348, 350, 359, 363, 369, 375, 383, 388, 388, 404, 411, 418, - 419, 426, 430, 434, 438, 442, 445, 447, 449, 452, 452, 455, 458, - 464, 468, 476, 480, 485, 493, 496, 501, 506, 512, 516, 521, + static const unsigned short int aOffset[119] = { + 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, + 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, + 86, 95, 96, 101, 105, 109, 117, 122, 128, 136, 142, 152, 159, + 162, 162, 165, 167, 167, 171, 176, 179, 184, 189, 194, 197, 203, + 206, 210, 217, 223, 223, 226, 229, 233, 234, 238, 244, 248, 255, + 261, 273, 279, 288, 290, 296, 301, 303, 310, 315, 320, 326, 332, + 337, 341, 344, 350, 354, 361, 363, 370, 372, 374, 383, 387, 393, + 399, 407, 412, 412, 428, 435, 442, 443, 450, 454, 458, 462, 466, + 469, 471, 473, 479, 483, 491, 495, 500, 508, 511, 516, 521, 527, + 531, 536, }; - static const unsigned char aCode[116] = { - TK_BEFORE, TK_FOR, TK_FOREIGN, TK_IGNORE, TK_LIKE_KW, - TK_EXPLAIN, TK_INSTEAD, TK_ADD, TK_DESC, TK_ESCAPE, - TK_EACH, TK_CHECK, TK_KEY, TK_CONSTRAINT, TK_INTERSECT, - TK_TABLE, TK_JOIN_KW, TK_THEN, TK_END, TK_DATABASE, - TK_AS, TK_SELECT, TK_TRANSACTION,TK_ON, TK_JOIN_KW, - TK_ALTER, TK_RAISE, TK_ELSE, TK_EXCEPT, TK_TRIGGER, - TK_REFERENCES, TK_UNIQUE, TK_QUERY, TK_ATTACH, TK_HAVING, - TK_GROUP, TK_UPDATE, TK_TEMP, TK_TEMP, TK_OR, - TK_BEGIN, TK_JOIN_KW, TK_REINDEX, TK_INDEX, TK_EXCLUSIVE, - TK_EXISTS, TK_BETWEEN, TK_NOTNULL, TK_NOT, TK_NULL, - TK_LIKE_KW, TK_CASCADE, TK_ASC, TK_DEFERRABLE, TK_CASE, - TK_COLLATE, TK_CREATE, TK_CTIME_KW, TK_DELETE, TK_DETACH, - TK_IMMEDIATE, TK_JOIN, TK_INSERT, TK_MATCH, TK_PLAN, - TK_ANALYZE, TK_PRAGMA, TK_ABORT, TK_VALUES, TK_VIRTUAL, - TK_LIMIT, TK_WHEN, TK_WHERE, TK_RENAME, TK_AFTER, - TK_REPLACE, TK_AND, TK_DEFAULT, TK_AUTOINCR, TK_TO, - TK_IN, TK_CAST, TK_COLUMNKW, TK_COMMIT, TK_CONFLICT, - TK_JOIN_KW, TK_CTIME_KW, TK_CTIME_KW, TK_PRIMARY, TK_DEFERRED, - TK_DISTINCT, TK_IS, TK_DROP, TK_FAIL, TK_FROM, - TK_JOIN_KW, TK_LIKE_KW, TK_BY, TK_IF, TK_INTO, - TK_OF, TK_OFFSET, TK_SET, TK_ISNULL, TK_ORDER, - TK_RESTRICT, TK_JOIN_KW, TK_JOIN_KW, TK_ROLLBACK, TK_ROW, - TK_UNION, TK_USING, TK_VACUUM, TK_VIEW, TK_INITIALLY, - TK_ALL, + static const unsigned char aCode[119] = { + TK_REINDEX, TK_INDEX, TK_INDEXED, TK_DESC, TK_ESCAPE, + TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOR, + TK_FOREIGN, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, + TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE, + TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE, + TK_EXCEPT, TK_TRANSACTION,TK_ON, TK_JOIN_KW, TK_ALTER, + TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_SAVEPOINT, TK_INTERSECT, + TK_TRIGGER, TK_REFERENCES, TK_CONSTRAINT, TK_INTO, TK_OFFSET, + TK_OF, TK_SET, TK_TEMP, TK_TEMP, TK_OR, + TK_UNIQUE, TK_QUERY, TK_ATTACH, TK_HAVING, TK_GROUP, + TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RELEASE, TK_BETWEEN, + TK_NOT, TK_NOTNULL, TK_NULL, TK_LIKE_KW, TK_CASCADE, + TK_ASC, TK_DELETE, TK_CASE, TK_COLLATE, TK_CREATE, + TK_CTIME_KW, TK_DETACH, TK_IMMEDIATE, TK_JOIN, TK_INSERT, + TK_MATCH, TK_PLAN, TK_ANALYZE, TK_PRAGMA, TK_ABORT, + TK_VALUES, TK_VIRTUAL, TK_LIMIT, TK_WHEN, TK_WHERE, + TK_RENAME, TK_AFTER, TK_REPLACE, TK_AND, TK_DEFAULT, + TK_AUTOINCR, TK_TO, TK_IN, TK_CAST, TK_COLUMNKW, + TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW, TK_CTIME_KW, + TK_PRIMARY, TK_DEFERRED, TK_DISTINCT, TK_IS, TK_DROP, + TK_FAIL, TK_FROM, TK_JOIN_KW, TK_LIKE_KW, TK_BY, + TK_IF, TK_ISNULL, TK_ORDER, TK_RESTRICT, TK_JOIN_KW, + TK_JOIN_KW, TK_ROLLBACK, TK_ROW, TK_UNION, TK_USING, + TK_VACUUM, TK_VIEW, TK_INITIALLY, TK_ALL, }; int h, i; if( n<2 ) return TK_ID; @@ -102,6 +137,125 @@ static int keywordCode(const char *z, int n){ n) % 127; for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){ if( aLen[i]==n && sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){ + testcase( i==0 ); /* TK_REINDEX */ + testcase( i==1 ); /* TK_INDEX */ + testcase( i==2 ); /* TK_INDEXED */ + testcase( i==3 ); /* TK_DESC */ + testcase( i==4 ); /* TK_ESCAPE */ + testcase( i==5 ); /* TK_EACH */ + testcase( i==6 ); /* TK_CHECK */ + testcase( i==7 ); /* TK_KEY */ + testcase( i==8 ); /* TK_BEFORE */ + testcase( i==9 ); /* TK_FOR */ + testcase( i==10 ); /* TK_FOREIGN */ + testcase( i==11 ); /* TK_IGNORE */ + testcase( i==12 ); /* TK_LIKE_KW */ + testcase( i==13 ); /* TK_EXPLAIN */ + testcase( i==14 ); /* TK_INSTEAD */ + testcase( i==15 ); /* TK_ADD */ + testcase( i==16 ); /* TK_DATABASE */ + testcase( i==17 ); /* TK_AS */ + testcase( i==18 ); /* TK_SELECT */ + testcase( i==19 ); /* TK_TABLE */ + testcase( i==20 ); /* TK_JOIN_KW */ + testcase( i==21 ); /* TK_THEN */ + testcase( i==22 ); /* TK_END */ + testcase( i==23 ); /* TK_DEFERRABLE */ + testcase( i==24 ); /* TK_ELSE */ + testcase( i==25 ); /* TK_EXCEPT */ + testcase( i==26 ); /* TK_TRANSACTION */ + testcase( i==27 ); /* TK_ON */ + testcase( i==28 ); /* TK_JOIN_KW */ + testcase( i==29 ); /* TK_ALTER */ + testcase( i==30 ); /* TK_RAISE */ + testcase( i==31 ); /* TK_EXCLUSIVE */ + testcase( i==32 ); /* TK_EXISTS */ + testcase( i==33 ); /* TK_SAVEPOINT */ + testcase( i==34 ); /* TK_INTERSECT */ + testcase( i==35 ); /* TK_TRIGGER */ + testcase( i==36 ); /* TK_REFERENCES */ + testcase( i==37 ); /* TK_CONSTRAINT */ + testcase( i==38 ); /* TK_INTO */ + testcase( i==39 ); /* TK_OFFSET */ + testcase( i==40 ); /* TK_OF */ + testcase( i==41 ); /* TK_SET */ + testcase( i==42 ); /* TK_TEMP */ + testcase( i==43 ); /* TK_TEMP */ + testcase( i==44 ); /* TK_OR */ + testcase( i==45 ); /* TK_UNIQUE */ + testcase( i==46 ); /* TK_QUERY */ + testcase( i==47 ); /* TK_ATTACH */ + testcase( i==48 ); /* TK_HAVING */ + testcase( i==49 ); /* TK_GROUP */ + testcase( i==50 ); /* TK_UPDATE */ + testcase( i==51 ); /* TK_BEGIN */ + testcase( i==52 ); /* TK_JOIN_KW */ + testcase( i==53 ); /* TK_RELEASE */ + testcase( i==54 ); /* TK_BETWEEN */ + testcase( i==55 ); /* TK_NOT */ + testcase( i==56 ); /* TK_NOTNULL */ + testcase( i==57 ); /* TK_NULL */ + testcase( i==58 ); /* TK_LIKE_KW */ + testcase( i==59 ); /* TK_CASCADE */ + testcase( i==60 ); /* TK_ASC */ + testcase( i==61 ); /* TK_DELETE */ + testcase( i==62 ); /* TK_CASE */ + testcase( i==63 ); /* TK_COLLATE */ + testcase( i==64 ); /* TK_CREATE */ + testcase( i==65 ); /* TK_CTIME_KW */ + testcase( i==66 ); /* TK_DETACH */ + testcase( i==67 ); /* TK_IMMEDIATE */ + testcase( i==68 ); /* TK_JOIN */ + testcase( i==69 ); /* TK_INSERT */ + testcase( i==70 ); /* TK_MATCH */ + testcase( i==71 ); /* TK_PLAN */ + testcase( i==72 ); /* TK_ANALYZE */ + testcase( i==73 ); /* TK_PRAGMA */ + testcase( i==74 ); /* TK_ABORT */ + testcase( i==75 ); /* TK_VALUES */ + testcase( i==76 ); /* TK_VIRTUAL */ + testcase( i==77 ); /* TK_LIMIT */ + testcase( i==78 ); /* TK_WHEN */ + testcase( i==79 ); /* TK_WHERE */ + testcase( i==80 ); /* TK_RENAME */ + testcase( i==81 ); /* TK_AFTER */ + testcase( i==82 ); /* TK_REPLACE */ + testcase( i==83 ); /* TK_AND */ + testcase( i==84 ); /* TK_DEFAULT */ + testcase( i==85 ); /* TK_AUTOINCR */ + testcase( i==86 ); /* TK_TO */ + testcase( i==87 ); /* TK_IN */ + testcase( i==88 ); /* TK_CAST */ + testcase( i==89 ); /* TK_COLUMNKW */ + testcase( i==90 ); /* TK_COMMIT */ + testcase( i==91 ); /* TK_CONFLICT */ + testcase( i==92 ); /* TK_JOIN_KW */ + testcase( i==93 ); /* TK_CTIME_KW */ + testcase( i==94 ); /* TK_CTIME_KW */ + testcase( i==95 ); /* TK_PRIMARY */ + testcase( i==96 ); /* TK_DEFERRED */ + testcase( i==97 ); /* TK_DISTINCT */ + testcase( i==98 ); /* TK_IS */ + testcase( i==99 ); /* TK_DROP */ + testcase( i==100 ); /* TK_FAIL */ + testcase( i==101 ); /* TK_FROM */ + testcase( i==102 ); /* TK_JOIN_KW */ + testcase( i==103 ); /* TK_LIKE_KW */ + testcase( i==104 ); /* TK_BY */ + testcase( i==105 ); /* TK_IF */ + testcase( i==106 ); /* TK_ISNULL */ + testcase( i==107 ); /* TK_ORDER */ + testcase( i==108 ); /* TK_RESTRICT */ + testcase( i==109 ); /* TK_JOIN_KW */ + testcase( i==110 ); /* TK_JOIN_KW */ + testcase( i==111 ); /* TK_ROLLBACK */ + testcase( i==112 ); /* TK_ROW */ + testcase( i==113 ); /* TK_UNION */ + testcase( i==114 ); /* TK_USING */ + testcase( i==115 ); /* TK_VACUUM */ + testcase( i==116 ); /* TK_VIEW */ + testcase( i==117 ); /* TK_INITIALLY */ + testcase( i==118 ); /* TK_ALL */ return aCode[i]; } } diff --git a/legacy.c b/legacy.c index 8d96b1e..5b037f0 100644 --- a/legacy.c +++ b/legacy.c @@ -14,11 +14,10 @@ ** other files are for internal use by SQLite and should not be ** accessed by users of the library. ** -** $Id: legacy.c,v 1.18 2007/05/04 13:15:56 drh Exp $ +** $Id: legacy.c,v 1.30 2008/12/10 19:26:24 drh Exp $ */ #include "sqliteInt.h" -#include "os.h" #include /* @@ -46,7 +45,10 @@ int sqlite3_exec( int nRetry = 0; int nCallback; - if( zSql==0 ) return SQLITE_OK; + if( zSql==0 ) zSql = ""; + + sqlite3_mutex_enter(db->mutex); + sqlite3Error(db, SQLITE_OK, 0); while( (rc==SQLITE_OK || (rc==SQLITE_SCHEMA && (++nRetry)<2)) && zSql[0] ){ int nCol; char **azVals = 0; @@ -64,12 +66,7 @@ int sqlite3_exec( } nCallback = 0; - nCol = sqlite3_column_count(pStmt); - azCols = sqliteMalloc(2*nCol*sizeof(const char *) + 1); - if( azCols==0 ){ - goto exec_out; - } while( 1 ){ int i; @@ -79,8 +76,17 @@ int sqlite3_exec( if( xCallback && (SQLITE_ROW==rc || (SQLITE_DONE==rc && !nCallback && db->flags&SQLITE_NullCallback)) ){ if( 0==nCallback ){ + if( azCols==0 ){ + azCols = sqlite3DbMallocZero(db, 2*nCol*sizeof(const char*) + 1); + if( azCols==0 ){ + goto exec_out; + } + } for(i=0; imallocFailed = 1; + goto exec_out; + } } } if( xCallback(pArg, nCol, azVals, azCols) ){ rc = SQLITE_ABORT; + sqlite3_finalize(pStmt); + pStmt = 0; + sqlite3Error(db, SQLITE_ABORT, 0); goto exec_out; } } @@ -108,18 +121,18 @@ int sqlite3_exec( } } - sqliteFree(azCols); + sqlite3DbFree(db, azCols); azCols = 0; } exec_out: if( pStmt ) sqlite3_finalize(pStmt); - if( azCols ) sqliteFree(azCols); + sqlite3DbFree(db, azCols); - rc = sqlite3ApiExit(0, rc); + rc = sqlite3ApiExit(db, rc); if( rc!=SQLITE_OK && rc==sqlite3_errcode(db) && pzErrMsg ){ - int nErrMsg = 1 + strlen(sqlite3_errmsg(db)); - *pzErrMsg = sqlite3_malloc(nErrMsg); + int nErrMsg = 1 + sqlite3Strlen30(sqlite3_errmsg(db)); + *pzErrMsg = sqlite3Malloc(nErrMsg); if( *pzErrMsg ){ memcpy(*pzErrMsg, sqlite3_errmsg(db), nErrMsg); } @@ -128,5 +141,6 @@ exec_out: } assert( (rc&db->errMask)==rc ); + sqlite3_mutex_leave(db->mutex); return rc; } diff --git a/loadext.c b/loadext.c index b2f4726..c59cd9c 100644 --- a/loadext.c +++ b/loadext.c @@ -11,16 +11,20 @@ ************************************************************************* ** This file contains code used to dynamically load extensions into ** the SQLite library. +** +** $Id: loadext.c,v 1.57 2008/12/08 18:19:18 drh Exp $ */ -#ifndef SQLITE_OMIT_LOAD_EXTENSION -#define SQLITE_CORE 1 /* Disable the API redefinition in sqlite3ext.h */ +#ifndef SQLITE_CORE + #define SQLITE_CORE 1 /* Disable the API redefinition in sqlite3ext.h */ +#endif #include "sqlite3ext.h" #include "sqliteInt.h" -#include "os.h" #include #include +#ifndef SQLITE_OMIT_LOAD_EXTENSION + /* ** Some API routines are omitted when various features are ** excluded from a build of SQLite. Substitute a NULL pointer @@ -94,6 +98,15 @@ # define sqlite3_get_table 0 #endif +#ifdef SQLITE_OMIT_INCRBLOB +#define sqlite3_bind_zeroblob 0 +#define sqlite3_blob_bytes 0 +#define sqlite3_blob_close 0 +#define sqlite3_blob_open 0 +#define sqlite3_blob_read 0 +#define sqlite3_blob_write 0 +#endif + /* ** The following structure contains pointers to all SQLite API routines. ** A pointer to this structure is passed into extensions when they are @@ -109,9 +122,13 @@ ** also check to make sure that the pointer to the function is ** not NULL before calling it. */ -const sqlite3_api_routines sqlite3_apis = { +static const sqlite3_api_routines sqlite3Apis = { sqlite3_aggregate_context, +#ifndef SQLITE_OMIT_DEPRECATED sqlite3_aggregate_count, +#else + 0, +#endif sqlite3_bind_blob, sqlite3_bind_double, sqlite3_bind_int, @@ -166,7 +183,11 @@ const sqlite3_api_routines sqlite3_apis = { sqlite3_errmsg, sqlite3_errmsg16, sqlite3_exec, +#ifndef SQLITE_OMIT_DEPRECATED sqlite3_expired, +#else + 0, +#endif sqlite3_finalize, sqlite3_free, sqlite3_free_table, @@ -206,10 +227,18 @@ const sqlite3_api_routines sqlite3_apis = { sqlite3_snprintf, sqlite3_step, sqlite3_table_column_metadata, +#ifndef SQLITE_OMIT_DEPRECATED sqlite3_thread_cleanup, +#else + 0, +#endif sqlite3_total_changes, sqlite3_trace, +#ifndef SQLITE_OMIT_DEPRECATED sqlite3_transfer_bindings, +#else + 0, +#endif sqlite3_update_hook, sqlite3_user_data, sqlite3_value_blob, @@ -247,6 +276,60 @@ const sqlite3_api_routines sqlite3_apis = { */ sqlite3_create_module_v2, + /* + ** Added for 3.5.0 + */ + sqlite3_bind_zeroblob, + sqlite3_blob_bytes, + sqlite3_blob_close, + sqlite3_blob_open, + sqlite3_blob_read, + sqlite3_blob_write, + sqlite3_create_collation_v2, + sqlite3_file_control, + sqlite3_memory_highwater, + sqlite3_memory_used, +#ifdef SQLITE_MUTEX_OMIT + 0, + 0, + 0, + 0, + 0, +#else + sqlite3_mutex_alloc, + sqlite3_mutex_enter, + sqlite3_mutex_free, + sqlite3_mutex_leave, + sqlite3_mutex_try, +#endif + sqlite3_open_v2, + sqlite3_release_memory, + sqlite3_result_error_nomem, + sqlite3_result_error_toobig, + sqlite3_sleep, + sqlite3_soft_heap_limit, + sqlite3_vfs_find, + sqlite3_vfs_register, + sqlite3_vfs_unregister, + + /* + ** Added for 3.5.8 + */ + sqlite3_threadsafe, + sqlite3_result_zeroblob, + sqlite3_result_error_code, + sqlite3_test_control, + sqlite3_randomness, + sqlite3_context_db_handle, + + /* + ** Added for 3.6.0 + */ + sqlite3_extended_result_codes, + sqlite3_limit, + sqlite3_next_stmt, + sqlite3_sql, + sqlite3_status, }; /* @@ -259,14 +342,15 @@ const sqlite3_api_routines sqlite3_apis = { ** ** If an error occurs and pzErrMsg is not 0, then fill *pzErrMsg with ** error message text. The calling function should free this memory -** by calling sqlite3_free(). +** by calling sqlite3DbFree(db, ). */ -int sqlite3_load_extension( +static int sqlite3LoadExtension( sqlite3 *db, /* Load the extension into this database connection */ const char *zFile, /* Name of the shared library containing extension */ const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ char **pzErrMsg /* Put error message here if not 0 */ ){ + sqlite3_vfs *pVfs = db->pVfs; void *handle; int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); char *zErrmsg = 0; @@ -289,46 +373,66 @@ int sqlite3_load_extension( zProc = "sqlite3_extension_init"; } - handle = sqlite3OsDlopen(zFile); + handle = sqlite3OsDlOpen(pVfs, zFile); if( handle==0 ){ if( pzErrMsg ){ - *pzErrMsg = sqlite3_mprintf("unable to open shared library [%s]", zFile); + char zErr[256]; + zErr[sizeof(zErr)-1] = '\0'; + sqlite3_snprintf(sizeof(zErr)-1, zErr, + "unable to open shared library [%s]", zFile); + sqlite3OsDlError(pVfs, sizeof(zErr)-1, zErr); + *pzErrMsg = sqlite3DbStrDup(0, zErr); } return SQLITE_ERROR; } xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - sqlite3OsDlsym(handle, zProc); + sqlite3OsDlSym(pVfs, handle, zProc); if( xInit==0 ){ if( pzErrMsg ){ - *pzErrMsg = sqlite3_mprintf("no entry point [%s] in shared library [%s]", - zProc, zFile); + char zErr[256]; + zErr[sizeof(zErr)-1] = '\0'; + sqlite3_snprintf(sizeof(zErr)-1, zErr, + "no entry point [%s] in shared library [%s]", zProc,zFile); + sqlite3OsDlError(pVfs, sizeof(zErr)-1, zErr); + *pzErrMsg = sqlite3DbStrDup(0, zErr); + sqlite3OsDlClose(pVfs, handle); } - sqlite3OsDlclose(handle); return SQLITE_ERROR; - }else if( xInit(db, &zErrmsg, &sqlite3_apis) ){ + }else if( xInit(db, &zErrmsg, &sqlite3Apis) ){ if( pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } sqlite3_free(zErrmsg); - sqlite3OsDlclose(handle); + sqlite3OsDlClose(pVfs, handle); return SQLITE_ERROR; } /* Append the new shared library handle to the db->aExtension array. */ - db->nExtension++; - aHandle = sqliteMalloc(sizeof(handle)*db->nExtension); + aHandle = sqlite3DbMallocZero(db, sizeof(handle)*(db->nExtension+1)); if( aHandle==0 ){ return SQLITE_NOMEM; } if( db->nExtension>0 ){ - memcpy(aHandle, db->aExtension, sizeof(handle)*(db->nExtension-1)); + memcpy(aHandle, db->aExtension, sizeof(handle)*db->nExtension); } - sqliteFree(db->aExtension); + sqlite3DbFree(db, db->aExtension); db->aExtension = aHandle; - db->aExtension[db->nExtension-1] = handle; + db->aExtension[db->nExtension++] = handle; return SQLITE_OK; } +int sqlite3_load_extension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ +){ + int rc; + sqlite3_mutex_enter(db->mutex); + rc = sqlite3LoadExtension(db, zFile, zProc, pzErrMsg); + sqlite3_mutex_leave(db->mutex); + return rc; +} /* ** Call this routine when the database connection is closing in order @@ -336,10 +440,11 @@ int sqlite3_load_extension( */ void sqlite3CloseExtensions(sqlite3 *db){ int i; + assert( sqlite3_mutex_held(db->mutex) ); for(i=0; inExtension; i++){ - sqlite3OsDlclose(db->aExtension[i]); + sqlite3OsDlClose(db->pVfs, db->aExtension[i]); } - sqliteFree(db->aExtension); + sqlite3DbFree(db, db->aExtension); } /* @@ -347,60 +452,116 @@ void sqlite3CloseExtensions(sqlite3 *db){ ** default so as not to open security holes in older applications. */ int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ + sqlite3_mutex_enter(db->mutex); if( onoff ){ db->flags |= SQLITE_LoadExtension; }else{ db->flags &= ~SQLITE_LoadExtension; } + sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ + /* -** A list of automatically loaded extensions. -** -** This list is shared across threads, so be sure to hold the -** mutex while accessing or changing it. +** The auto-extension code added regardless of whether or not extension +** loading is supported. We need a dummy sqlite3Apis pointer for that +** code if regular extension loading is not available. This is that +** dummy pointer. */ -static int nAutoExtension = 0; -static void **aAutoExtension = 0; +#ifdef SQLITE_OMIT_LOAD_EXTENSION +static const sqlite3_api_routines sqlite3Apis = { 0 }; +#endif + + +/* +** The following object holds the list of automatically loaded +** extensions. +** +** This list is shared across threads. The SQLITE_MUTEX_STATIC_MASTER +** mutex must be held while accessing this list. +*/ +typedef struct sqlite3AutoExtList sqlite3AutoExtList; +static SQLITE_WSD struct sqlite3AutoExtList { + int nExt; /* Number of entries in aExt[] */ + void (**aExt)(void); /* Pointers to the extension init functions */ +} sqlite3Autoext = { 0, 0 }; + +/* The "wsdAutoext" macro will resolve to the autoextension +** state vector. If writable static data is unsupported on the target, +** we have to locate the state vector at run-time. In the more common +** case where writable static data is supported, wsdStat can refer directly +** to the "sqlite3Autoext" state vector declared above. +*/ +#ifdef SQLITE_OMIT_WSD +# define wsdAutoextInit \ + sqlite3AutoExtList *x = &GLOBAL(sqlite3AutoExtList,sqlite3Autoext) +# define wsdAutoext x[0] +#else +# define wsdAutoextInit +# define wsdAutoext sqlite3Autoext +#endif /* ** Register a statically linked extension that is automatically ** loaded by every new database connection. */ -int sqlite3_auto_extension(void *xInit){ - int i; +int sqlite3_auto_extension(void (*xInit)(void)){ int rc = SQLITE_OK; - sqlite3OsEnterMutex(); - for(i=0; i=nAutoExtension ){ +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + sqlite3_mutex_enter(mutex); + if( i>=wsdAutoext.nExt ){ xInit = 0; go = 0; }else{ xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - aAutoExtension[i]; + wsdAutoext.aExt[i]; } - sqlite3OsLeaveMutex(); - if( xInit && xInit(db, &zErrmsg, &sqlite3_apis) ){ + sqlite3_mutex_leave(mutex); + if( xInit && xInit(db, &zErrmsg, &sqlite3Apis) ){ sqlite3Error(db, SQLITE_ERROR, "automatic extension loading failed: %s", zErrmsg); go = 0; rc = SQLITE_ERROR; + sqlite3_free(zErrmsg); } } return rc; } - -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ diff --git a/main.c b/main.c index a8c69cb..35f3666 100644 --- a/main.c +++ b/main.c @@ -14,26 +14,40 @@ ** other files are for internal use by SQLite and should not be ** accessed by users of the library. ** -** $Id: main.c,v 1.378 2007/08/13 15:28:34 danielk1977 Exp $ +** $Id: main.c,v 1.521 2009/01/10 16:15:22 drh Exp $ */ #include "sqliteInt.h" -#include "os.h" #include +#ifdef SQLITE_ENABLE_FTS3 +# include "fts3.h" +#endif +#ifdef SQLITE_ENABLE_RTREE +# include "rtree.h" +#endif +#ifdef SQLITE_ENABLE_ICU +# include "sqliteicu.h" +#endif + /* ** The version of the library */ +#ifndef SQLITE_AMALGAMATION const char sqlite3_version[] = SQLITE_VERSION; +#endif const char *sqlite3_libversion(void){ return sqlite3_version; } int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } +int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } +#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) /* ** If the following function pointer is not NULL and if ** SQLITE_ENABLE_IOTRACE is enabled, then messages describing ** I/O active are written using this function. These messages ** are intended for debugging activity only. */ -void (*sqlite3_io_trace)(const char*, ...) = 0; +void (*sqlite3IoTrace)(const char*, ...) = 0; +#endif /* ** If the following global variable points to a string which is the @@ -44,13 +58,420 @@ void (*sqlite3_io_trace)(const char*, ...) = 0; */ char *sqlite3_temp_directory = 0; +/* +** Initialize SQLite. +** +** This routine must be called to initialize the memory allocation, +** VFS, and mutex subsystems prior to doing any serious work with +** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT +** this routine will be called automatically by key routines such as +** sqlite3_open(). +** +** This routine is a no-op except on its very first call for the process, +** or for the first call after a call to sqlite3_shutdown. +** +** The first thread to call this routine runs the initialization to +** completion. If subsequent threads call this routine before the first +** thread has finished the initialization process, then the subsequent +** threads must block until the first thread finishes with the initialization. +** +** The first thread might call this routine recursively. Recursive +** calls to this routine should not block, of course. Otherwise the +** initialization process would never complete. +** +** Let X be the first thread to enter this routine. Let Y be some other +** thread. Then while the initial invocation of this routine by X is +** incomplete, it is required that: +** +** * Calls to this routine from Y must block until the outer-most +** call by X completes. +** +** * Recursive calls to this routine from thread X return immediately +** without blocking. +*/ +int sqlite3_initialize(void){ + sqlite3_mutex *pMaster; /* The main static mutex */ + int rc; /* Result code */ + +#ifdef SQLITE_OMIT_WSD + rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + + /* If SQLite is already completely initialized, then this call + ** to sqlite3_initialize() should be a no-op. But the initialization + ** must be complete. So isInit must not be set until the very end + ** of this routine. + */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; + + /* Make sure the mutex subsystem is initialized. If unable to + ** initialize the mutex subsystem, return early with the error. + ** If the system is so sick that we are unable to allocate a mutex, + ** there is not much SQLite is going to be able to do. + ** + ** The mutex subsystem must take care of serializing its own + ** initialization. + */ + rc = sqlite3MutexInit(); + if( rc ) return rc; + + /* Initialize the malloc() system and the recursive pInitMutex mutex. + ** This operation is protected by the STATIC_MASTER mutex. Note that + ** MutexAlloc() is called for a static mutex prior to initializing the + ** malloc subsystem - this implies that the allocation of a static + ** mutex must not require support from the malloc subsystem. + */ + pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(pMaster); + if( !sqlite3GlobalConfig.isMallocInit ){ + rc = sqlite3MallocInit(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isMallocInit = 1; + if( !sqlite3GlobalConfig.pInitMutex ){ + sqlite3GlobalConfig.pInitMutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ + rc = SQLITE_NOMEM; + } + } + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.nRefInitMutex++; + } + sqlite3_mutex_leave(pMaster); + + /* If unable to initialize the malloc subsystem, then return early. + ** There is little hope of getting SQLite to run if the malloc + ** subsystem cannot be initialized. + */ + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Do the rest of the initialization under the recursive mutex so + ** that we will be able to handle recursive calls into + ** sqlite3_initialize(). The recursive calls normally come through + ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other + ** recursive calls might also be possible. + */ + sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); + if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + sqlite3GlobalConfig.inProgress = 1; + memset(pHash, 0, sizeof(sqlite3GlobalFunctions)); + sqlite3RegisterGlobalFunctions(); + rc = sqlite3_os_init(); + if( rc==SQLITE_OK ){ + rc = sqlite3PcacheInitialize(); + sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, + sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); + } + sqlite3GlobalConfig.inProgress = 0; + sqlite3GlobalConfig.isInit = (rc==SQLITE_OK ? 1 : 0); + } + sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); + + /* Go back under the static mutex and clean up the recursive + ** mutex to prevent a resource leak. + */ + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.nRefInitMutex--; + if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ + assert( sqlite3GlobalConfig.nRefInitMutex==0 ); + sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); + sqlite3GlobalConfig.pInitMutex = 0; + } + sqlite3_mutex_leave(pMaster); + + /* The following is just a sanity check to make sure SQLite has + ** been compiled correctly. It is important to run this code, but + ** we don't want to run it too often and soak up CPU cycles for no + ** reason. So we run it once during initialization. + */ +#ifndef NDEBUG + /* This section of code's only "output" is via assert() statements. */ + if ( rc==SQLITE_OK ){ + u64 x = (((u64)1)<<63)-1; + double y; + assert(sizeof(x)==8); + assert(sizeof(x)==sizeof(y)); + memcpy(&y, &x, 8); + assert( sqlite3IsNaN(y) ); + } +#endif + + return rc; +} + +/* +** Undo the effects of sqlite3_initialize(). Must not be called while +** there are outstanding database connections or memory allocations or +** while any part of SQLite is otherwise in use in any thread. This +** routine is not threadsafe. Not by a long shot. +*/ +int sqlite3_shutdown(void){ + sqlite3GlobalConfig.isMallocInit = 0; + sqlite3PcacheShutdown(); + if( sqlite3GlobalConfig.isInit ){ + sqlite3_os_end(); + } + sqlite3MallocEnd(); + sqlite3MutexEnd(); + sqlite3GlobalConfig.isInit = 0; + return SQLITE_OK; +} + +/* +** This API allows applications to modify the global configuration of +** the SQLite library at run-time. +** +** This routine should only be called when there are no outstanding +** database connections or memory allocations. This routine is not +** threadsafe. Failure to heed these warnings can lead to unpredictable +** behavior. +*/ +int sqlite3_config(int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE; + + va_start(ap, op); + switch( op ){ + + /* Mutex configuration options are only available in a threadsafe + ** compile. + */ +#if SQLITE_THREADSAFE + case SQLITE_CONFIG_SINGLETHREAD: { + /* Disable all mutexing */ + sqlite3GlobalConfig.bCoreMutex = 0; + sqlite3GlobalConfig.bFullMutex = 0; + break; + } + case SQLITE_CONFIG_MULTITHREAD: { + /* Disable mutexing of database connections */ + /* Enable mutexing of core data structures */ + sqlite3GlobalConfig.bCoreMutex = 1; + sqlite3GlobalConfig.bFullMutex = 0; + break; + } + case SQLITE_CONFIG_SERIALIZED: { + /* Enable all mutexing */ + sqlite3GlobalConfig.bCoreMutex = 1; + sqlite3GlobalConfig.bFullMutex = 1; + break; + } + case SQLITE_CONFIG_MUTEX: { + /* Specify an alternative mutex implementation */ + sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); + break; + } + case SQLITE_CONFIG_GETMUTEX: { + /* Retrieve the current mutex implementation */ + *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; + break; + } +#endif + + + case SQLITE_CONFIG_MALLOC: { + /* Specify an alternative malloc implementation */ + sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); + break; + } + case SQLITE_CONFIG_GETMALLOC: { + /* Retrieve the current malloc() implementation */ + if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); + *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; + break; + } + case SQLITE_CONFIG_MEMSTATUS: { + /* Enable or disable the malloc status collection */ + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_SCRATCH: { + /* Designate a buffer for scratch memory space */ + sqlite3GlobalConfig.pScratch = va_arg(ap, void*); + sqlite3GlobalConfig.szScratch = va_arg(ap, int); + sqlite3GlobalConfig.nScratch = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PAGECACHE: { + /* Designate a buffer for scratch memory space */ + sqlite3GlobalConfig.pPage = va_arg(ap, void*); + sqlite3GlobalConfig.szPage = va_arg(ap, int); + sqlite3GlobalConfig.nPage = va_arg(ap, int); + break; + } + + case SQLITE_CONFIG_PCACHE: { + /* Specify an alternative malloc implementation */ + sqlite3GlobalConfig.pcache = *va_arg(ap, sqlite3_pcache_methods*); + break; + } + + case SQLITE_CONFIG_GETPCACHE: { + if( sqlite3GlobalConfig.pcache.xInit==0 ){ + sqlite3PCacheSetDefault(); + } + *va_arg(ap, sqlite3_pcache_methods*) = sqlite3GlobalConfig.pcache; + break; + } + +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + case SQLITE_CONFIG_HEAP: { + /* Designate a buffer for heap memory space */ + sqlite3GlobalConfig.pHeap = va_arg(ap, void*); + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + sqlite3GlobalConfig.mnReq = va_arg(ap, int); + + if( sqlite3GlobalConfig.pHeap==0 ){ + /* If the heap pointer is NULL, then restore the malloc implementation + ** back to NULL pointers too. This will cause the malloc to go + ** back to its default implementation when sqlite3_initialize() is + ** run. + */ + memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); + }else{ + /* The heap pointer is not NULL, then install one of the + ** mem5.c/mem3.c methods. If neither ENABLE_MEMSYS3 nor + ** ENABLE_MEMSYS5 is defined, return an error. + ** the default case and return an error. + */ +#ifdef SQLITE_ENABLE_MEMSYS3 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); +#endif + } + break; + } +#endif + + case SQLITE_CONFIG_LOOKASIDE: { + sqlite3GlobalConfig.szLookaside = va_arg(ap, int); + sqlite3GlobalConfig.nLookaside = va_arg(ap, int); + break; + } + + default: { + rc = SQLITE_ERROR; + break; + } + } + va_end(ap); + return rc; +} + +/* +** Set up the lookaside buffers for a database connection. +** Return SQLITE_OK on success. +** If lookaside is already active, return SQLITE_BUSY. +** +** The sz parameter is the number of bytes in each lookaside slot. +** The cnt parameter is the number of slots. If pStart is NULL the +** space for the lookaside memory is obtained from sqlite3_malloc(). +** If pStart is not NULL then it is sz*cnt bytes of memory to use for +** the lookaside memory. +*/ +static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ + void *pStart; + if( db->lookaside.nOut ){ + return SQLITE_BUSY; + } + if( sz<0 ) sz = 0; + if( cnt<0 ) cnt = 0; + if( pBuf==0 ){ + sz = (sz + 7)&~7; + sqlite3BeginBenignMalloc(); + pStart = sqlite3Malloc( sz*cnt ); + sqlite3EndBenignMalloc(); + }else{ + sz = sz&~7; + pStart = pBuf; + } + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + db->lookaside.pStart = pStart; + db->lookaside.pFree = 0; + db->lookaside.sz = (u16)sz; + db->lookaside.bMalloced = pBuf==0 ?1:0; + if( pStart ){ + int i; + LookasideSlot *p; + p = (LookasideSlot*)pStart; + for(i=cnt-1; i>=0; i--){ + p->pNext = db->lookaside.pFree; + db->lookaside.pFree = p; + p = (LookasideSlot*)&((u8*)p)[sz]; + } + db->lookaside.pEnd = p; + db->lookaside.bEnabled = 1; + }else{ + db->lookaside.pEnd = 0; + db->lookaside.bEnabled = 0; + } + return SQLITE_OK; +} + +/* +** Return the mutex associated with a database connection. +*/ +sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ + return db->mutex; +} + +/* +** Configuration settings for an individual database connection +*/ +int sqlite3_db_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc; + va_start(ap, op); + switch( op ){ + case SQLITE_DBCONFIG_LOOKASIDE: { + void *pBuf = va_arg(ap, void*); + int sz = va_arg(ap, int); + int cnt = va_arg(ap, int); + rc = setupLookaside(db, pBuf, sz, cnt); + break; + } + default: { + rc = SQLITE_ERROR; + break; + } + } + va_end(ap); + return rc; +} + + +/* +** Return true if the buffer z[0..n-1] contains all spaces. +*/ +static int allSpaces(const char *z, int n){ + while( n>0 && z[n-1]==' ' ){ n--; } + return n==0; +} /* ** This is the default collating function named "BINARY" which is always ** available. +** +** If the padFlag argument is not NULL then space padding at the end +** of strings is ignored. This implements the RTRIM collation. */ static int binCollFunc( - void *NotUsed, + void *padFlag, int nKey1, const void *pKey1, int nKey2, const void *pKey2 ){ @@ -58,7 +479,14 @@ static int binCollFunc( n = nKey1nTotalChange; } +/* +** Close all open savepoints. This function only manipulates fields of the +** database handle object, it does not close any savepoints that may be open +** at the b-tree/pager level. +*/ +void sqlite3CloseSavepoints(sqlite3 *db){ + while( db->pSavepoint ){ + Savepoint *pTmp = db->pSavepoint; + db->pSavepoint = pTmp->pNext; + sqlite3DbFree(db, pTmp); + } + db->nSavepoint = 0; + db->isTransactionSavepoint = 0; +} + /* ** Close an existing SQLite database */ @@ -116,9 +560,10 @@ int sqlite3_close(sqlite3 *db){ if( !db ){ return SQLITE_OK; } - if( sqlite3SafetyCheck(db) ){ + if( !sqlite3SafetyCheckSickOrOk(db) ){ return SQLITE_MISUSE; } + sqlite3_mutex_enter(db->mutex); #ifdef SQLITE_SSE { @@ -142,22 +587,13 @@ int sqlite3_close(sqlite3 *db){ if( db->pVdbe ){ sqlite3Error(db, SQLITE_BUSY, "Unable to close due to unfinalised statements"); + sqlite3_mutex_leave(db->mutex); return SQLITE_BUSY; } - assert( !sqlite3SafetyCheck(db) ); + assert( sqlite3SafetyCheckSickOrOk(db) ); - /* FIX ME: db->magic may be set to SQLITE_MAGIC_CLOSED if the database - ** cannot be opened for some reason. So this routine needs to run in - ** that case. But maybe there should be an extra magic value for the - ** "failed to open" state. - ** - ** TODO: Coverage tests do not test the case where this condition is - ** true. It's hard to see how to cause it without messing with threads. - */ - if( db->magic!=SQLITE_MAGIC_CLOSED && sqlite3SafetyOn(db) ){ - /* printf("DID NOT CLOSE\n"); fflush(stdout); */ - return SQLITE_ERROR; - } + /* Free any outstanding Savepoint structures. */ + sqlite3CloseSavepoints(db); for(j=0; jnDb; j++){ struct Db *pDb = &db->aDb[j]; @@ -172,14 +608,17 @@ int sqlite3_close(sqlite3 *db){ sqlite3ResetInternalSchema(db, 0); assert( db->nDb<=2 ); assert( db->aDb==db->aDbStatic ); - for(i=sqliteHashFirst(&db->aFunc); i; i=sqliteHashNext(i)){ - FuncDef *pFunc, *pNext; - for(pFunc = (FuncDef*)sqliteHashData(i); pFunc; pFunc=pNext){ - pNext = pFunc->pNext; - sqliteFree(pFunc); + for(j=0; jaFunc.a); j++){ + FuncDef *pNext, *pHash, *p; + for(p=db->aFunc.a[j]; p; p=pHash){ + pHash = p->pHash; + while( p ){ + pNext = p->pNext; + sqlite3DbFree(db, p); + p = pNext; + } } } - for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ CollSeq *pColl = (CollSeq *)sqliteHashData(i); /* Invoke any destructors registered for collation sequence user data. */ @@ -188,7 +627,7 @@ int sqlite3_close(sqlite3 *db){ pColl[j].xDel(pColl[j].pUser); } } - sqliteFree(pColl); + sqlite3DbFree(db, pColl); } sqlite3HashClear(&db->aCollSeq); #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -197,12 +636,11 @@ int sqlite3_close(sqlite3 *db){ if( pMod->xDestroy ){ pMod->xDestroy(pMod->pAux); } - sqliteFree(pMod); + sqlite3DbFree(db, pMod); } sqlite3HashClear(&db->aModule); #endif - sqlite3HashClear(&db->aFunc); sqlite3Error(db, SQLITE_OK, 0); /* Deallocates any cached error strings. */ if( db->pErr ){ sqlite3ValueFree(db->pErr); @@ -217,9 +655,15 @@ int sqlite3_close(sqlite3 *db){ ** the same sqliteMalloc() as the one that allocates the database ** structure? */ - sqliteFree(db->aDb[1].pSchema); - sqliteFree(db); - sqlite3ReleaseThreadData(); + sqlite3DbFree(db, db->aDb[1].pSchema); + sqlite3_mutex_leave(db->mutex); + db->magic = SQLITE_MAGIC_CLOSED; + sqlite3_mutex_free(db->mutex); + assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + sqlite3_free(db); return SQLITE_OK; } @@ -229,6 +673,8 @@ int sqlite3_close(sqlite3 *db){ void sqlite3RollbackAll(sqlite3 *db){ int i; int inTrans = 0; + assert( sqlite3_mutex_held(db->mutex) ); + sqlite3BeginBenignMalloc(); for(i=0; inDb; i++){ if( db->aDb[i].pBt ){ if( sqlite3BtreeIsInTrans(db->aDb[i].pBt) ){ @@ -239,6 +685,8 @@ void sqlite3RollbackAll(sqlite3 *db){ } } sqlite3VtabRollback(db); + sqlite3EndBenignMalloc(); + if( db->flags&SQLITE_InternChanges ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetInternalSchema(db, 0); @@ -278,7 +726,7 @@ const char *sqlite3ErrStr(int rc){ case SQLITE_CONSTRAINT: z = "constraint failed"; break; case SQLITE_MISMATCH: z = "datatype mismatch"; break; case SQLITE_MISUSE: z = "library routine called out of sequence";break; - case SQLITE_NOLFS: z = "kernel lacks large file support"; break; + case SQLITE_NOLFS: z = "large file support is disabled"; break; case SQLITE_AUTH: z = "authorization denied"; break; case SQLITE_FORMAT: z = "auxiliary database format error"; break; case SQLITE_RANGE: z = "bind or column index out of range"; break; @@ -298,13 +746,14 @@ static int sqliteDefaultBusyCallback( void *ptr, /* Database connection */ int count /* Number of times table has been busy */ ){ -#if OS_WIN || (defined(HAVE_USLEEP) && HAVE_USLEEP) +#if SQLITE_OS_WIN || (defined(HAVE_USLEEP) && HAVE_USLEEP) static const u8 delays[] = { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 }; static const u8 totals[] = { 0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228 }; # define NDELAY (sizeof(delays)/sizeof(delays[0])) - int timeout = ((sqlite3 *)ptr)->busyTimeout; + sqlite3 *db = (sqlite3 *)ptr; + int timeout = db->busyTimeout; int delay, prior; assert( count>=0 ); @@ -319,14 +768,15 @@ static int sqliteDefaultBusyCallback( delay = timeout - prior; if( delay<=0 ) return 0; } - sqlite3OsSleep(delay); + sqlite3OsSleep(db->pVfs, delay*1000); return 1; #else + sqlite3 *db = (sqlite3 *)ptr; int timeout = ((sqlite3 *)ptr)->busyTimeout; if( (count+1)*1000 > timeout ){ return 0; } - sqlite3OsSleep(1000); + sqlite3OsSleep(db->pVfs, 1000000); return 1; #endif } @@ -340,7 +790,7 @@ static int sqliteDefaultBusyCallback( */ int sqlite3InvokeBusyHandler(BusyHandler *p){ int rc; - if( p==0 || p->xFunc==0 || p->nBusy<0 ) return 0; + if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0; rc = p->xFunc(p->pArg, p->nBusy); if( rc==0 ){ p->nBusy = -1; @@ -359,12 +809,11 @@ int sqlite3_busy_handler( int (*xBusy)(void*,int), void *pArg ){ - if( sqlite3SafetyCheck(db) ){ - return SQLITE_MISUSE; - } + sqlite3_mutex_enter(db->mutex); db->busyHandler.xFunc = xBusy; db->busyHandler.pArg = pArg; db->busyHandler.nBusy = 0; + sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -380,17 +829,17 @@ void sqlite3_progress_handler( int (*xProgress)(void*), void *pArg ){ - if( !sqlite3SafetyCheck(db) ){ - if( nOps>0 ){ - db->xProgress = xProgress; - db->nProgressOps = nOps; - db->pProgressArg = pArg; - }else{ - db->xProgress = 0; - db->nProgressOps = 0; - db->pProgressArg = 0; - } + sqlite3_mutex_enter(db->mutex); + if( nOps>0 ){ + db->xProgress = xProgress; + db->nProgressOps = nOps; + db->pProgressArg = pArg; + }else{ + db->xProgress = 0; + db->nProgressOps = 0; + db->pProgressArg = 0; } + sqlite3_mutex_leave(db->mutex); } #endif @@ -400,9 +849,6 @@ void sqlite3_progress_handler( ** specified number of milliseconds before returning 0. */ int sqlite3_busy_timeout(sqlite3 *db, int ms){ - if( sqlite3SafetyCheck(db) ){ - return SQLITE_MISUSE; - } if( ms>0 ){ db->busyTimeout = ms; sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); @@ -416,35 +862,9 @@ int sqlite3_busy_timeout(sqlite3 *db, int ms){ ** Cause any pending operation to stop at its earliest opportunity. */ void sqlite3_interrupt(sqlite3 *db){ - if( db && (db->magic==SQLITE_MAGIC_OPEN || db->magic==SQLITE_MAGIC_BUSY) ){ - db->u1.isInterrupted = 1; - } + db->u1.isInterrupted = 1; } -/* -** Memory allocation routines that use SQLites internal memory -** memory allocator. Depending on how SQLite is compiled, the -** internal memory allocator might be just an alias for the -** system default malloc/realloc/free. Or the built-in allocator -** might do extra stuff like put sentinals around buffers to -** check for overruns or look for memory leaks. -** -** Use sqlite3_free() to free memory returned by sqlite3_mprintf(). -*/ -void sqlite3_free(void *p){ if( p ) sqlite3OsFree(p); } -void *sqlite3_malloc(int nByte){ return nByte>0 ? sqlite3OsMalloc(nByte) : 0; } -void *sqlite3_realloc(void *pOld, int nByte){ - if( pOld ){ - if( nByte>0 ){ - return sqlite3OsRealloc(pOld, nByte); - }else{ - sqlite3OsFree(pOld); - return 0; - } - }else{ - return sqlite3_malloc(nByte); - } -} /* ** This function is exactly the same as sqlite3_create_function(), except @@ -465,15 +885,13 @@ int sqlite3CreateFunc( FuncDef *p; int nName; - if( sqlite3SafetyCheck(db) ){ - return SQLITE_MISUSE; - } + assert( sqlite3_mutex_held(db->mutex) ); if( zFunctionName==0 || (xFunc && (xFinal || xStep)) || (!xFunc && (xFinal && !xStep)) || (!xFunc && (!xFinal && xStep)) || - (nArg<-1 || nArg>127) || - (255<(nName = strlen(zFunctionName))) ){ + (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || + (255<(nName = sqlite3Strlen(db, zFunctionName))) ){ sqlite3Error(db, SQLITE_ERROR, "bad parameters"); return SQLITE_ERROR; } @@ -492,10 +910,13 @@ int sqlite3CreateFunc( int rc; rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8, pUserData, xFunc, xStep, xFinal); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE, - pUserData, xFunc, xStep, xFinal); - if( rc!=SQLITE_OK ) return rc; + if( rc==SQLITE_OK ){ + rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE, + pUserData, xFunc, xStep, xFinal); + } + if( rc!=SQLITE_OK ){ + return rc; + } enc = SQLITE_UTF16BE; } #else @@ -507,27 +928,29 @@ int sqlite3CreateFunc( ** is being overridden/deleted but there are no active VMs, allow the ** operation to continue but invalidate all precompiled statements. */ - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, enc, 0); + p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); if( p && p->iPrefEnc==enc && p->nArg==nArg ){ if( db->activeVdbeCnt ){ sqlite3Error(db, SQLITE_BUSY, "Unable to delete/modify user-function due to active statements"); - assert( !sqlite3MallocFailed() ); + assert( !db->mallocFailed ); return SQLITE_BUSY; }else{ sqlite3ExpirePreparedStatements(db); } } - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, enc, 1); - if( p ){ - p->flags = 0; - p->xFunc = xFunc; - p->xStep = xStep; - p->xFinalize = xFinal; - p->pUserData = pUserData; - p->nArg = nArg; + p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); + assert(p || db->mallocFailed); + if( !p ){ + return SQLITE_NOMEM; } + p->flags = 0; + p->xFunc = xFunc; + p->xStep = xStep; + p->xFinalize = xFinal; + p->pUserData = pUserData; + p->nArg = (u16)nArg; return SQLITE_OK; } @@ -545,10 +968,11 @@ int sqlite3_create_function( void (*xFinal)(sqlite3_context*) ){ int rc; - assert( !sqlite3MallocFailed() ); + sqlite3_mutex_enter(db->mutex); rc = sqlite3CreateFunc(db, zFunctionName, nArg, enc, p, xFunc, xStep, xFinal); - - return sqlite3ApiExit(db, rc); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } #ifndef SQLITE_OMIT_UTF16 @@ -564,13 +988,14 @@ int sqlite3_create_function16( ){ int rc; char *zFunc8; - assert( !sqlite3MallocFailed() ); - - zFunc8 = sqlite3Utf16to8(zFunctionName, -1); + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1); rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal); - sqliteFree(zFunc8); - - return sqlite3ApiExit(db, rc); + sqlite3DbFree(db, zFunc8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } #endif @@ -592,12 +1017,16 @@ int sqlite3_overload_function( const char *zName, int nArg ){ - int nName = strlen(zName); + int nName = sqlite3Strlen(db, zName); + int rc; + sqlite3_mutex_enter(db->mutex); if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, 0, sqlite3InvalidFunction, 0, 0); } - return sqlite3ApiExit(db, SQLITE_OK); + rc = sqlite3ApiExit(db, SQLITE_OK); + sqlite3_mutex_leave(db->mutex); + return rc; } #ifndef SQLITE_OMIT_TRACE @@ -610,9 +1039,12 @@ int sqlite3_overload_function( ** SQL statement. */ void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ - void *pOld = db->pTraceArg; + void *pOld; + sqlite3_mutex_enter(db->mutex); + pOld = db->pTraceArg; db->xTrace = xTrace; db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); return pOld; } /* @@ -628,9 +1060,12 @@ void *sqlite3_profile( void (*xProfile)(void*,const char*,sqlite_uint64), void *pArg ){ - void *pOld = db->pProfileArg; + void *pOld; + sqlite3_mutex_enter(db->mutex); + pOld = db->pProfileArg; db->xProfile = xProfile; db->pProfileArg = pArg; + sqlite3_mutex_leave(db->mutex); return pOld; } #endif /* SQLITE_OMIT_TRACE */ @@ -646,9 +1081,12 @@ void *sqlite3_commit_hook( int (*xCallback)(void*), /* Function to invoke on each commit */ void *pArg /* Argument to the function */ ){ - void *pOld = db->pCommitArg; + void *pOld; + sqlite3_mutex_enter(db->mutex); + pOld = db->pCommitArg; db->xCommitCallback = xCallback; db->pCommitArg = pArg; + sqlite3_mutex_leave(db->mutex); return pOld; } @@ -661,9 +1099,12 @@ void *sqlite3_update_hook( void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), void *pArg /* Argument to the function */ ){ - void *pRet = db->pUpdateArg; + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pUpdateArg; db->xUpdateCallback = xCallback; db->pUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); return pRet; } @@ -676,9 +1117,12 @@ void *sqlite3_rollback_hook( void (*xCallback)(void*), /* Callback function */ void *pArg /* Argument to the function */ ){ - void *pRet = db->pRollbackArg; + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pRollbackArg; db->xRollbackCallback = xCallback; db->pRollbackArg = pArg; + sqlite3_mutex_leave(db->mutex); return pRet; } @@ -693,57 +1137,67 @@ void *sqlite3_rollback_hook( ** ** A virtual database can be either a disk file (that is automatically ** deleted when the file is closed) or it an be held entirely in memory, -** depending on the values of the TEMP_STORE compile-time macro and the +** depending on the values of the SQLITE_TEMP_STORE compile-time macro and the ** db->temp_store variable, according to the following chart: ** -** TEMP_STORE db->temp_store Location of temporary database -** ---------- -------------- ------------------------------ -** 0 any file -** 1 1 file -** 1 2 memory -** 1 0 file -** 2 1 file -** 2 2 memory -** 2 0 memory -** 3 any memory +** SQLITE_TEMP_STORE db->temp_store Location of temporary database +** ----------------- -------------- ------------------------------ +** 0 any file +** 1 1 file +** 1 2 memory +** 1 0 file +** 2 1 file +** 2 2 memory +** 2 0 memory +** 3 any memory */ int sqlite3BtreeFactory( const sqlite3 *db, /* Main database when opening aux otherwise 0 */ const char *zFilename, /* Name of the file containing the BTree database */ int omitJournal, /* if TRUE then do not journal this file */ int nCache, /* How many pages in the page cache */ + int vfsFlags, /* Flags passed through to vfsOpen */ Btree **ppBtree /* Pointer to new Btree object written here */ ){ - int btree_flags = 0; + int btFlags = 0; int rc; + assert( sqlite3_mutex_held(db->mutex) ); assert( ppBtree != 0); if( omitJournal ){ - btree_flags |= BTREE_OMIT_JOURNAL; + btFlags |= BTREE_OMIT_JOURNAL; } if( db->flags & SQLITE_NoReadlock ){ - btree_flags |= BTREE_NO_READLOCK; + btFlags |= BTREE_NO_READLOCK; } if( zFilename==0 ){ -#if TEMP_STORE==0 +#if SQLITE_TEMP_STORE==0 /* Do nothing */ #endif #ifndef SQLITE_OMIT_MEMORYDB -#if TEMP_STORE==1 +#if SQLITE_TEMP_STORE==1 if( db->temp_store==2 ) zFilename = ":memory:"; #endif -#if TEMP_STORE==2 +#if SQLITE_TEMP_STORE==2 if( db->temp_store!=1 ) zFilename = ":memory:"; #endif -#if TEMP_STORE==3 +#if SQLITE_TEMP_STORE==3 zFilename = ":memory:"; #endif #endif /* SQLITE_OMIT_MEMORYDB */ } - rc = sqlite3BtreeOpen(zFilename, (sqlite3 *)db, ppBtree, btree_flags); - if( rc==SQLITE_OK ){ - sqlite3BtreeSetBusyHandler(*ppBtree, (void*)&db->busyHandler); + if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (zFilename==0 || *zFilename==0) ){ + vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; + } + rc = sqlite3BtreeOpen(zFilename, (sqlite3 *)db, ppBtree, btFlags, vfsFlags); + + /* If the B-Tree was successfully opened, set the pager-cache size to the + ** default value. Except, if the call to BtreeOpen() returned a handle + ** open on an existing shared pager-cache, do not change the pager-cache + ** size. + */ + if( rc==SQLITE_OK && 0==sqlite3BtreeSchema(*ppBtree, 0, 0) ){ sqlite3BtreeSetCacheSize(*ppBtree, nCache); } return rc; @@ -755,17 +1209,23 @@ int sqlite3BtreeFactory( */ const char *sqlite3_errmsg(sqlite3 *db){ const char *z; - assert( !sqlite3MallocFailed() ); if( !db ){ return sqlite3ErrStr(SQLITE_NOMEM); } - if( sqlite3SafetyCheck(db) || db->errCode==SQLITE_MISUSE ){ + if( !sqlite3SafetyCheckSickOrOk(db) ){ return sqlite3ErrStr(SQLITE_MISUSE); } + if( db->mallocFailed ){ + return sqlite3ErrStr(SQLITE_NOMEM); + } + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); z = (char*)sqlite3_value_text(db->pErr); + assert( !db->mallocFailed ); if( z==0 ){ z = sqlite3ErrStr(db->errCode); } + sqlite3_mutex_leave(db->mutex); return z; } @@ -795,20 +1255,27 @@ const void *sqlite3_errmsg16(sqlite3 *db){ }; const void *z; - assert( !sqlite3MallocFailed() ); if( !db ){ return (void *)(&outOfMemBe[SQLITE_UTF16NATIVE==SQLITE_UTF16LE?1:0]); } - if( sqlite3SafetyCheck(db) || db->errCode==SQLITE_MISUSE ){ + if( !sqlite3SafetyCheckSickOrOk(db) ){ return (void *)(&misuseBe[SQLITE_UTF16NATIVE==SQLITE_UTF16LE?1:0]); } + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); z = sqlite3_value_text16(db->pErr); if( z==0 ){ sqlite3ValueSetStr(db->pErr, -1, sqlite3ErrStr(db->errCode), SQLITE_UTF8, SQLITE_STATIC); z = sqlite3_value_text16(db->pErr); } - sqlite3ApiExit(0, 0); + /* A malloc() may have failed within the call to sqlite3_value_text16() + ** above. If this is the case, then the db->mallocFailed flag needs to + ** be cleared before returning. Do this directly, instead of via + ** sqlite3ApiExit(), to avoid setting the database handle error message. + */ + db->mallocFailed = 0; + sqlite3_mutex_leave(db->mutex); return z; } #endif /* SQLITE_OMIT_UTF16 */ @@ -818,14 +1285,23 @@ const void *sqlite3_errmsg16(sqlite3 *db){ ** passed to this function, we assume a malloc() failed during sqlite3_open(). */ int sqlite3_errcode(sqlite3 *db){ - if( !db || sqlite3MallocFailed() ){ - return SQLITE_NOMEM; - } - if( sqlite3SafetyCheck(db) ){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ return SQLITE_MISUSE; } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM; + } return db->errCode & db->errMask; } +int sqlite3_extended_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM; + } + return db->errCode; +} /* ** Create a new collating function for database "db". The name is zName @@ -841,10 +1317,9 @@ static int createCollation( ){ CollSeq *pColl; int enc2; + int nName; - if( sqlite3SafetyCheck(db) ){ - return SQLITE_MISUSE; - } + assert( sqlite3_mutex_held(db->mutex) ); /* If SQLITE_UTF16 is specified as the encoding type, transform this ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the @@ -854,17 +1329,16 @@ static int createCollation( if( enc2==SQLITE_UTF16 ){ enc2 = SQLITE_UTF16NATIVE; } - if( (enc2&~3)!=0 ){ - sqlite3Error(db, SQLITE_ERROR, "unknown encoding"); - return SQLITE_ERROR; + return SQLITE_MISUSE; } /* Check if this call is removing or replacing an existing collation ** sequence. If so, and there are active VMs, return busy. If there ** are no active VMs, invalidate any pre-compiled statements. */ - pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, strlen(zName), 0); + nName = sqlite3Strlen(db, zName); + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, nName, 0); if( pColl && pColl->xCmp ){ if( db->activeVdbeCnt ){ sqlite3Error(db, SQLITE_BUSY, @@ -880,7 +1354,7 @@ static int createCollation( ** to be called. */ if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ - CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName, strlen(zName)); + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName, nName); int j; for(j=0; j<3; j++){ CollSeq *p = &aColl[j]; @@ -894,18 +1368,96 @@ static int createCollation( } } - pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, strlen(zName), 1); + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, nName, 1); if( pColl ){ pColl->xCmp = xCompare; pColl->pUser = pCtx; pColl->xDel = xDel; - pColl->enc = enc2 | (enc & SQLITE_UTF16_ALIGNED); + pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); } sqlite3Error(db, SQLITE_OK, 0); return SQLITE_OK; } +/* +** This array defines hard upper bounds on limit values. The +** initializer must be kept in sync with the SQLITE_LIMIT_* +** #defines in sqlite3.h. +*/ +static const int aHardLimit[] = { + SQLITE_MAX_LENGTH, + SQLITE_MAX_SQL_LENGTH, + SQLITE_MAX_COLUMN, + SQLITE_MAX_EXPR_DEPTH, + SQLITE_MAX_COMPOUND_SELECT, + SQLITE_MAX_VDBE_OP, + SQLITE_MAX_FUNCTION_ARG, + SQLITE_MAX_ATTACHED, + SQLITE_MAX_LIKE_PATTERN_LENGTH, + SQLITE_MAX_VARIABLE_NUMBER, +}; + +/* +** Make sure the hard limits are set to reasonable values +*/ +#if SQLITE_MAX_LENGTH<100 +# error SQLITE_MAX_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH<100 +# error SQLITE_MAX_SQL_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH +# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH +#endif +#if SQLITE_MAX_COMPOUND_SELECT<2 +# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 +#endif +#if SQLITE_MAX_VDBE_OP<40 +# error SQLITE_MAX_VDBE_OP must be at least 40 +#endif +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 +#endif +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>30 +# error SQLITE_MAX_ATTACHED must be between 0 and 30 +#endif +#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 +# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 +#endif +#if SQLITE_MAX_VARIABLE_NUMBER<1 +# error SQLITE_MAX_VARIABLE_NUMBER must be at least 1 +#endif +#if SQLITE_MAX_COLUMN>32767 +# error SQLITE_MAX_COLUMN must not exceed 32767 +#endif + + +/* +** Change the value of a limit. Report the old value. +** If an invalid limit index is supplied, report -1. +** Make no changes but still report the old value if the +** new limit is negative. +** +** A new lower limit does not shrink existing constructs. +** It merely prevents new constructs that exceed the limit +** from forming. +*/ +int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ + int oldLimit; + if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ + return -1; + } + oldLimit = db->aLimit[limitId]; + if( newLimit>=0 ){ + if( newLimit>aHardLimit[limitId] ){ + newLimit = aHardLimit[limitId]; + } + db->aLimit[limitId] = newLimit; + } + return oldLimit; +} + /* ** This routine does the work of opening a database on behalf of ** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" @@ -913,23 +1465,66 @@ static int createCollation( */ static int openDatabase( const char *zFilename, /* Database filename UTF-8 encoded */ - sqlite3 **ppDb /* OUT: Returned database handle */ + sqlite3 **ppDb, /* OUT: Returned database handle */ + unsigned flags, /* Operational flags */ + const char *zVfs /* Name of the VFS to use */ ){ sqlite3 *db; int rc; CollSeq *pColl; + int isThreadsafe; - assert( !sqlite3MallocFailed() ); +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + + if( sqlite3GlobalConfig.bCoreMutex==0 ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_NOMUTEX ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_FULLMUTEX ){ + isThreadsafe = 1; + }else{ + isThreadsafe = sqlite3GlobalConfig.bFullMutex; + } + + /* Remove harmful bits from the flags parameter */ + flags &= ~( SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_MAIN_DB | + SQLITE_OPEN_TEMP_DB | + SQLITE_OPEN_TRANSIENT_DB | + SQLITE_OPEN_MAIN_JOURNAL | + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_SUBJOURNAL | + SQLITE_OPEN_MASTER_JOURNAL | + SQLITE_OPEN_NOMUTEX | + SQLITE_OPEN_FULLMUTEX + ); /* Allocate the sqlite data structure */ - db = sqliteMalloc( sizeof(sqlite3) ); + db = sqlite3MallocZero( sizeof(sqlite3) ); if( db==0 ) goto opendb_out; + if( isThreadsafe ){ + db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( db->mutex==0 ){ + sqlite3_free(db); + db = 0; + goto opendb_out; + } + } + sqlite3_mutex_enter(db->mutex); db->errMask = 0xff; db->priorNewRowid = 0; - db->magic = SQLITE_MAGIC_BUSY; db->nDb = 2; + db->magic = SQLITE_MAGIC_BUSY; db->aDb = db->aDbStatic; + + assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); + memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->autoCommit = 1; + db->nextAutovac = -1; + db->nextPagesize = 0; db->flags |= SQLITE_ShortColNames #if SQLITE_DEFAULT_FILE_FORMAT<4 | SQLITE_LegacyFileFmt @@ -938,25 +1533,31 @@ static int openDatabase( | SQLITE_LoadExtension #endif ; - sqlite3HashInit(&db->aFunc, SQLITE_HASH_STRING, 0); - sqlite3HashInit(&db->aCollSeq, SQLITE_HASH_STRING, 0); + sqlite3HashInit(&db->aCollSeq, 0); #ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3HashInit(&db->aModule, SQLITE_HASH_STRING, 0); + sqlite3HashInit(&db->aModule, 0); #endif + db->pVfs = sqlite3_vfs_find(zVfs); + if( !db->pVfs ){ + rc = SQLITE_ERROR; + sqlite3Error(db, rc, "no such vfs: %s", zVfs); + goto opendb_out; + } + /* Add the default collation sequence BINARY. BINARY works for both UTF-8 ** and UTF-16, so add a version for each to avoid any unnecessary ** conversions. The only error that can occur here is a malloc() failure. */ - if( createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0) || - createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0) || - createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0) || - (db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 6, 0))==0 - ){ - assert( sqlite3MallocFailed() ); - db->magic = SQLITE_MAGIC_CLOSED; + createCollation(db, "BINARY", SQLITE_UTF8, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16BE, 0, binCollFunc, 0); + createCollation(db, "BINARY", SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); + if( db->mallocFailed ){ goto opendb_out; } + db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 6, 0); + assert( db->pDfltColl!=0 ); /* Also add a UTF-8 case-insensitive collation sequence. */ createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); @@ -969,15 +1570,19 @@ static int openDatabase( } /* Open the backend database driver */ - rc = sqlite3BtreeFactory(db, zFilename, 0, SQLITE_DEFAULT_CACHE_SIZE, + db->openFlags = flags; + rc = sqlite3BtreeFactory(db, zFilename, 0, SQLITE_DEFAULT_CACHE_SIZE, + flags | SQLITE_OPEN_MAIN_DB, &db->aDb[0].pBt); if( rc!=SQLITE_OK ){ + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM; + } sqlite3Error(db, rc, 0); - db->magic = SQLITE_MAGIC_CLOSED; goto opendb_out; } - db->aDb[0].pSchema = sqlite3SchemaGet(db->aDb[0].pBt); - db->aDb[1].pSchema = sqlite3SchemaGet(0); + db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); /* The default safety_level for the main database is 'full'; for the temp @@ -991,7 +1596,7 @@ static int openDatabase( #endif db->magic = SQLITE_MAGIC_OPEN; - if( sqlite3MallocFailed() ){ + if( db->mallocFailed ){ goto opendb_out; } @@ -1011,25 +1616,37 @@ static int openDatabase( } #ifdef SQLITE_ENABLE_FTS1 - if( !sqlite3MallocFailed() ){ + if( !db->mallocFailed ){ extern int sqlite3Fts1Init(sqlite3*); rc = sqlite3Fts1Init(db); } #endif #ifdef SQLITE_ENABLE_FTS2 - if( !sqlite3MallocFailed() && rc==SQLITE_OK ){ + if( !db->mallocFailed && rc==SQLITE_OK ){ extern int sqlite3Fts2Init(sqlite3*); rc = sqlite3Fts2Init(db); } #endif +#ifdef SQLITE_ENABLE_FTS3 + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts3Init(db); + } +#endif + #ifdef SQLITE_ENABLE_ICU - if( !sqlite3MallocFailed() && rc==SQLITE_OK ){ - extern int sqlite3IcuInit(sqlite3*); + if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3IcuInit(db); } #endif + +#ifdef SQLITE_ENABLE_RTREE + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3RtreeInit(db); + } +#endif + sqlite3Error(db, rc, 0); /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking @@ -1042,10 +1659,21 @@ static int openDatabase( SQLITE_DEFAULT_LOCKING_MODE); #endif + /* Enable the lookaside-malloc subsystem */ + setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, + sqlite3GlobalConfig.nLookaside); + opendb_out: - if( SQLITE_NOMEM==(rc = sqlite3_errcode(db)) ){ + if( db ){ + assert( db->mutex!=0 || isThreadsafe==0 || sqlite3GlobalConfig.bFullMutex==0 ); + sqlite3_mutex_leave(db->mutex); + } + rc = sqlite3_errcode(db); + if( rc==SQLITE_NOMEM ){ sqlite3_close(db); db = 0; + }else if( rc!=SQLITE_OK ){ + db->magic = SQLITE_MAGIC_SICK; } *ppDb = db; return sqlite3ApiExit(0, rc); @@ -1058,7 +1686,16 @@ int sqlite3_open( const char *zFilename, sqlite3 **ppDb ){ - return openDatabase(zFilename, ppDb); + return openDatabase(zFilename, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); +} +int sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +){ + return openDatabase(filename, ppDb, flags, zVfs); } #ifndef SQLITE_OMIT_UTF16 @@ -1070,24 +1707,28 @@ int sqlite3_open16( sqlite3 **ppDb ){ char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */ - int rc = SQLITE_OK; sqlite3_value *pVal; + int rc; assert( zFilename ); assert( ppDb ); *ppDb = 0; - pVal = sqlite3ValueNew(); +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + pVal = sqlite3ValueNew(0); sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); if( zFilename8 ){ - rc = openDatabase(zFilename8, ppDb); - if( rc==SQLITE_OK && *ppDb ){ - rc = sqlite3_exec(*ppDb, "PRAGMA encoding = 'UTF-16'", 0, 0, 0); - if( rc!=SQLITE_OK ){ - sqlite3_close(*ppDb); - *ppDb = 0; - } + rc = openDatabase(zFilename8, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); + assert( *ppDb || rc==SQLITE_NOMEM ); + if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ + ENC(*ppDb) = SQLITE_UTF16NATIVE; } + }else{ + rc = SQLITE_NOMEM; } sqlite3ValueFree(pVal); @@ -1095,45 +1736,6 @@ int sqlite3_open16( } #endif /* SQLITE_OMIT_UTF16 */ -/* -** The following routine destroys a virtual machine that is created by -** the sqlite3_compile() routine. The integer returned is an SQLITE_ -** success/failure code that describes the result of executing the virtual -** machine. -** -** This routine sets the error code and string returned by -** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). -*/ -int sqlite3_finalize(sqlite3_stmt *pStmt){ - int rc; - if( pStmt==0 ){ - rc = SQLITE_OK; - }else{ - rc = sqlite3VdbeFinalize((Vdbe*)pStmt); - } - return rc; -} - -/* -** Terminate the current execution of an SQL statement and reset it -** back to its starting state so that it can be reused. A success code from -** the prior execution is returned. -** -** This routine sets the error code and string returned by -** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). -*/ -int sqlite3_reset(sqlite3_stmt *pStmt){ - int rc; - if( pStmt==0 ){ - rc = SQLITE_OK; - }else{ - rc = sqlite3VdbeReset((Vdbe*)pStmt); - sqlite3VdbeMakeReady((Vdbe*)pStmt, -1, 0, 0, 0); - assert( (rc & (sqlite3_db_handle(pStmt)->errMask))==rc ); - } - return rc; -} - /* ** Register a new collation sequence with the database handle db. */ @@ -1145,9 +1747,12 @@ int sqlite3_create_collation( int(*xCompare)(void*,int,const void*,int,const void*) ){ int rc; - assert( !sqlite3MallocFailed() ); + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); rc = createCollation(db, zName, enc, pCtx, xCompare, 0); - return sqlite3ApiExit(db, rc); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } /* @@ -1162,9 +1767,12 @@ int sqlite3_create_collation_v2( void(*xDel)(void*) ){ int rc; - assert( !sqlite3MallocFailed() ); + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); rc = createCollation(db, zName, enc, pCtx, xCompare, xDel); - return sqlite3ApiExit(db, rc); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } #ifndef SQLITE_OMIT_UTF16 @@ -1173,20 +1781,23 @@ int sqlite3_create_collation_v2( */ int sqlite3_create_collation16( sqlite3* db, - const char *zName, + const void *zName, int enc, void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*) ){ int rc = SQLITE_OK; - char *zName8; - assert( !sqlite3MallocFailed() ); - zName8 = sqlite3Utf16to8(zName, -1); + char *zName8; + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zName8 = sqlite3Utf16to8(db, zName, -1); if( zName8 ){ rc = createCollation(db, zName8, enc, pCtx, xCompare, 0); - sqliteFree(zName8); + sqlite3DbFree(db, zName8); } - return sqlite3ApiExit(db, rc); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } #endif /* SQLITE_OMIT_UTF16 */ @@ -1199,12 +1810,11 @@ int sqlite3_collation_needed( void *pCollNeededArg, void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) ){ - if( sqlite3SafetyCheck(db) ){ - return SQLITE_MISUSE; - } + sqlite3_mutex_enter(db->mutex); db->xCollNeeded = xCollNeeded; db->xCollNeeded16 = 0; db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -1218,25 +1828,26 @@ int sqlite3_collation_needed16( void *pCollNeededArg, void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) ){ - if( sqlite3SafetyCheck(db) ){ - return SQLITE_MISUSE; - } + sqlite3_mutex_enter(db->mutex); db->xCollNeeded = 0; db->xCollNeeded16 = xCollNeeded16; db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } #endif /* SQLITE_OMIT_UTF16 */ #ifndef SQLITE_OMIT_GLOBALRECOVER +#ifndef SQLITE_OMIT_DEPRECATED /* ** This function is now an anachronism. It used to be used to recover from a ** malloc() failure, but SQLite now does this automatically. */ -int sqlite3_global_recover(){ +int sqlite3_global_recover(void){ return SQLITE_OK; } #endif +#endif /* ** Test to see whether or not the database connection is in autocommit @@ -1261,46 +1872,17 @@ int sqlite3Corrupt(void){ } #endif - -#ifndef SQLITE_OMIT_SHARED_CACHE -/* -** Enable or disable the shared pager and schema features for the -** current thread. -** -** This routine should only be called when there are no open -** database connections. -*/ -int sqlite3_enable_shared_cache(int enable){ - ThreadData *pTd = sqlite3ThreadData(); - if( pTd ){ - /* It is only legal to call sqlite3_enable_shared_cache() when there - ** are no currently open b-trees that were opened by the calling thread. - ** This condition is only easy to detect if the shared-cache were - ** previously enabled (and is being disabled). - */ - if( pTd->pBtree && !enable ){ - assert( pTd->useSharedData ); - return SQLITE_MISUSE; - } - - pTd->useSharedData = enable; - sqlite3ReleaseThreadData(); - } - return sqlite3ApiExit(0, SQLITE_OK); -} -#endif - +#ifndef SQLITE_OMIT_DEPRECATED /* ** This is a convenience routine that makes sure that all thread-specific ** data for this thread has been deallocated. +** +** SQLite no longer uses thread-specific data so this routine is now a +** no-op. It is retained for historical compatibility. */ void sqlite3_thread_cleanup(void){ - ThreadData *pTd = sqlite3OsThreadSpecificData(0); - if( pTd ){ - memset(pTd, 0, sizeof(*pTd)); - sqlite3OsThreadSpecificData(-1); - } } +#endif /* ** Return meta information about a specific column of a database table. @@ -1316,7 +1898,7 @@ int sqlite3_table_column_metadata( char const **pzCollSeq, /* OUTPUT: Collation sequence name */ int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ int *pPrimaryKey, /* OUTPUT: True if column part of PK */ - int *pAutoinc /* OUTPUT: True if colums is auto-increment */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ ){ int rc; char *zErrMsg = 0; @@ -1331,10 +1913,11 @@ int sqlite3_table_column_metadata( int autoinc = 0; /* Ensure the database schema has been loaded */ - if( sqlite3SafetyOn(db) ){ - return SQLITE_MISUSE; - } + sqlite3_mutex_enter(db->mutex); + (void)sqlite3SafetyOn(db); + sqlite3BtreeEnterAll(db); rc = sqlite3Init(db, &zErrMsg); + sqlite3BtreeLeaveAll(db); if( SQLITE_OK!=rc ){ goto error_out; } @@ -1378,9 +1961,9 @@ int sqlite3_table_column_metadata( if( pCol ){ zDataType = pCol->zType; zCollSeq = pCol->zColl; - notnull = (pCol->notNull?1:0); - primarykey = (pCol->isPrimKey?1:0); - autoinc = ((pTab->iPKey==iCol && pTab->autoInc)?1:0); + notnull = pCol->notNull!=0; + primarykey = pCol->isPrimKey!=0; + autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; }else{ zDataType = "INTEGER"; primarykey = 1; @@ -1390,9 +1973,7 @@ int sqlite3_table_column_metadata( } error_out: - if( sqlite3SafetyOff(db) ){ - rc = SQLITE_MISUSE; - } + (void)sqlite3SafetyOff(db); /* Whether the function call succeeded or failed, set the output parameters ** to whatever their local counterparts contain. If an error did occur, @@ -1405,39 +1986,149 @@ error_out: if( pAutoinc ) *pAutoinc = autoinc; if( SQLITE_OK==rc && !pTab ){ - sqlite3SetString(&zErrMsg, "no such table column: ", zTableName, ".", - zColumnName, 0); + sqlite3DbFree(db, zErrMsg); + zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, + zColumnName); rc = SQLITE_ERROR; } sqlite3Error(db, rc, (zErrMsg?"%s":0), zErrMsg); - sqliteFree(zErrMsg); - return sqlite3ApiExit(db, rc); -} -#endif - -/* -** Set all the parameters in the compiled SQL statement to NULL. -*/ -int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ - int i; - int rc = SQLITE_OK; - for(i=1; rc==SQLITE_OK && i<=sqlite3_bind_parameter_count(pStmt); i++){ - rc = sqlite3_bind_null(pStmt, i); - } + sqlite3DbFree(db, zErrMsg); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); return rc; } +#endif /* ** Sleep for a little while. Return the amount of time slept. */ int sqlite3_sleep(int ms){ - return sqlite3OsSleep(ms); + sqlite3_vfs *pVfs; + int rc; + pVfs = sqlite3_vfs_find(0); + if( pVfs==0 ) return 0; + + /* This function works in milliseconds, but the underlying OsSleep() + ** API uses microseconds. Hence the 1000's. + */ + rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); + return rc; } /* ** Enable or disable the extended result codes. */ int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ + sqlite3_mutex_enter(db->mutex); db->errMask = onoff ? 0xffffffff : 0xff; + sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } + +/* +** Invoke the xFileControl method on a particular database. +*/ +int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ + int rc = SQLITE_ERROR; + int iDb; + sqlite3_mutex_enter(db->mutex); + if( zDbName==0 ){ + iDb = 0; + }else{ + for(iDb=0; iDbnDb; iDb++){ + if( strcmp(db->aDb[iDb].zName, zDbName)==0 ) break; + } + } + if( iDbnDb ){ + Btree *pBtree = db->aDb[iDb].pBt; + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + assert( fd!=0 ); + if( fd->pMethods ){ + rc = sqlite3OsFileControl(fd, op, pArg); + } + sqlite3BtreeLeave(pBtree); + } + } + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Interface to the testing logic. +*/ +int sqlite3_test_control(int op, ...){ + int rc = 0; +#ifndef SQLITE_OMIT_BUILTIN_TEST + va_list ap; + va_start(ap, op); + switch( op ){ + + /* + ** Save the current state of the PRNG. + */ + case SQLITE_TESTCTRL_PRNG_SAVE: { + sqlite3PrngSaveState(); + break; + } + + /* + ** Restore the state of the PRNG to the last state saved using + ** PRNG_SAVE. If PRNG_SAVE has never before been called, then + ** this verb acts like PRNG_RESET. + */ + case SQLITE_TESTCTRL_PRNG_RESTORE: { + sqlite3PrngRestoreState(); + break; + } + + /* + ** Reset the PRNG back to its uninitialized state. The next call + ** to sqlite3_randomness() will reseed the PRNG using a single call + ** to the xRandomness method of the default VFS. + */ + case SQLITE_TESTCTRL_PRNG_RESET: { + sqlite3PrngResetState(); + break; + } + + /* + ** sqlite3_test_control(BITVEC_TEST, size, program) + ** + ** Run a test against a Bitvec object of size. The program argument + ** is an array of integers that defines the test. Return -1 on a + ** memory allocation error, 0 on success, or non-zero for an error. + ** See the sqlite3BitvecBuiltinTest() for additional information. + */ + case SQLITE_TESTCTRL_BITVEC_TEST: { + int sz = va_arg(ap, int); + int *aProg = va_arg(ap, int*); + rc = sqlite3BitvecBuiltinTest(sz, aProg); + break; + } + + /* + ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) + ** + ** Register hooks to call to indicate which malloc() failures + ** are benign. + */ + case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { + typedef void (*void_function)(void); + void_function xBenignBegin; + void_function xBenignEnd; + xBenignBegin = va_arg(ap, void_function); + xBenignEnd = va_arg(ap, void_function); + sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd); + break; + } + } + va_end(ap); +#endif /* SQLITE_OMIT_BUILTIN_TEST */ + return rc; +} diff --git a/malloc.c b/malloc.c index 8a99332..f4fc0d3 100644 --- a/malloc.c +++ b/malloc.c @@ -9,711 +9,708 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* +** ** Memory allocation functions used throughout sqlite. ** -** -** $Id: malloc.c,v 1.4 2007/08/08 01:04:52 drh Exp $ +** $Id: malloc.c,v 1.53 2008/12/16 17:20:38 shane Exp $ */ #include "sqliteInt.h" -#include "os.h" #include #include /* -** MALLOC WRAPPER ARCHITECTURE -** -** The sqlite code accesses dynamic memory allocation/deallocation by invoking -** the following six APIs (which may be implemented as macros). -** -** sqlite3Malloc() -** sqlite3MallocRaw() -** sqlite3Realloc() -** sqlite3ReallocOrFree() -** sqlite3Free() -** sqlite3AllocSize() -** -** The function sqlite3FreeX performs the same task as sqlite3Free and is -** guaranteed to be a real function. The same holds for sqlite3MallocX -** -** The above APIs are implemented in terms of the functions provided in the -** operating-system interface. The OS interface is never accessed directly -** by code outside of this file. -** -** sqlite3OsMalloc() -** sqlite3OsRealloc() -** sqlite3OsFree() -** sqlite3OsAllocationSize() -** -** Functions sqlite3MallocRaw() and sqlite3Realloc() may invoke -** sqlite3_release_memory() if a call to sqlite3OsMalloc() or -** sqlite3OsRealloc() fails (or if the soft-heap-limit for the thread is -** exceeded). Function sqlite3Malloc() usually invokes -** sqlite3MallocRaw(). -** -** MALLOC TEST WRAPPER ARCHITECTURE -** -** The test wrapper provides extra test facilities to ensure the library -** does not leak memory and handles the failure of the underlying OS level -** allocation system correctly. It is only present if the library is -** compiled with the SQLITE_MEMDEBUG macro set. -** -** * Guardposts to detect overwrites. -** * Ability to cause a specific Malloc() or Realloc() to fail. -** * Audit outstanding memory allocations (i.e check for leaks). +** This routine runs when the memory allocator sees that the +** total memory allocation is about to exceed the soft heap +** limit. */ +static void softHeapLimitEnforcer( + void *NotUsed, + sqlite3_int64 NotUsed2, + int allocSize +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + sqlite3_release_memory(allocSize); +} -#define MAX(x,y) ((x)>(y)?(x):(y)) - -#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) && !defined(SQLITE_OMIT_DISKIO) /* -** Set the soft heap-size limit for the current thread. Passing a negative -** value indicates no limit. +** Set the soft heap-size limit for the library. Passing a zero or +** negative value indicates no limit. */ void sqlite3_soft_heap_limit(int n){ - ThreadData *pTd = sqlite3ThreadData(); - if( pTd ){ - pTd->nSoftHeapLimit = n; + sqlite3_uint64 iLimit; + int overage; + if( n<0 ){ + iLimit = 0; + }else{ + iLimit = n; + } + sqlite3_initialize(); + if( iLimit>0 ){ + sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, iLimit); + }else{ + sqlite3MemoryAlarm(0, 0, 0); + } + overage = (int)(sqlite3_memory_used() - (i64)n); + if( overage>0 ){ + sqlite3_release_memory(overage); } - sqlite3ReleaseThreadData(); } /* -** Release memory held by SQLite instances created by the current thread. +** Attempt to release up to n bytes of non-essential memory currently +** held by SQLite. An example of non-essential memory is memory used to +** cache database pages that are not currently in use. */ int sqlite3_release_memory(int n){ - return sqlite3PagerReleaseMemory(n); -} +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + int nRet = 0; +#if 0 + nRet += sqlite3VdbeReleaseMemory(n); +#endif + nRet += sqlite3PcacheReleaseMemory(n-nRet); + return nRet; #else -/* If SQLITE_ENABLE_MEMORY_MANAGEMENT is not defined, then define a version -** of sqlite3_release_memory() to be used by other code in this file. -** This is done for no better reason than to reduce the number of -** pre-processor #ifndef statements. -*/ -#define sqlite3_release_memory(x) 0 /* 0 == no memory freed */ + UNUSED_PARAMETER(n); + return SQLITE_OK; #endif - -#ifdef SQLITE_MEMDEBUG -/*-------------------------------------------------------------------------- -** Begin code for memory allocation system test layer. -** -** Memory debugging is turned on by defining the SQLITE_MEMDEBUG macro. -** -** SQLITE_MEMDEBUG==1 -> Fence-posting only (thread safe) -** SQLITE_MEMDEBUG==2 -> Fence-posting + linked list of allocations (not ts) -** SQLITE_MEMDEBUG==3 -> Above + backtraces (not thread safe, req. glibc) -*/ - -/* Figure out whether or not to store backtrace() information for each malloc. -** The backtrace() function is only used if SQLITE_MEMDEBUG is set to 2 or -** greater and glibc is in use. If we don't want to use backtrace(), then just -** define it as an empty macro and set the amount of space reserved to 0. -*/ -#if defined(__GLIBC__) && SQLITE_MEMDEBUG>2 - extern int backtrace(void **, int); - #define TESTALLOC_STACKSIZE 128 - #define TESTALLOC_STACKFRAMES ((TESTALLOC_STACKSIZE-8)/sizeof(void*)) -#else - #define backtrace(x, y) - #define TESTALLOC_STACKSIZE 0 - #define TESTALLOC_STACKFRAMES 0 -#endif - -/* -** Number of 32-bit guard words. This should probably be a multiple of -** 2 since on 64-bit machines we want the value returned by sqliteMalloc() -** to be 8-byte aligned. -*/ -#ifndef TESTALLOC_NGUARD -# define TESTALLOC_NGUARD 2 -#endif - -/* -** Size reserved for storing file-name along with each malloc()ed blob. -*/ -#define TESTALLOC_FILESIZE 64 - -/* -** Size reserved for storing the user string. Each time a Malloc() or Realloc() -** call succeeds, up to TESTALLOC_USERSIZE bytes of the string pointed to by -** sqlite3_malloc_id are stored along with the other test system metadata. -*/ -#define TESTALLOC_USERSIZE 64 -const char *sqlite3_malloc_id = 0; - -/* -** Blocks used by the test layer have the following format: -** -** -** -** -** -** -** <32-bit line number> -** -** -*/ - -#define TESTALLOC_OFFSET_GUARD1(p) (sizeof(void *) * 2) -#define TESTALLOC_OFFSET_DATA(p) ( \ - TESTALLOC_OFFSET_GUARD1(p) + sizeof(u32) * TESTALLOC_NGUARD \ -) -#define TESTALLOC_OFFSET_GUARD2(p) ( \ - TESTALLOC_OFFSET_DATA(p) + sqlite3OsAllocationSize(p) - TESTALLOC_OVERHEAD \ -) -#define TESTALLOC_OFFSET_LINENUMBER(p) ( \ - TESTALLOC_OFFSET_GUARD2(p) + sizeof(u32) * TESTALLOC_NGUARD \ -) -#define TESTALLOC_OFFSET_FILENAME(p) ( \ - TESTALLOC_OFFSET_LINENUMBER(p) + sizeof(u32) \ -) -#define TESTALLOC_OFFSET_USER(p) ( \ - TESTALLOC_OFFSET_FILENAME(p) + TESTALLOC_FILESIZE \ -) -#define TESTALLOC_OFFSET_STACK(p) ( \ - TESTALLOC_OFFSET_USER(p) + TESTALLOC_USERSIZE + 8 - \ - (TESTALLOC_OFFSET_USER(p) % 8) \ -) - -#define TESTALLOC_OVERHEAD ( \ - sizeof(void *)*2 + /* pPrev and pNext pointers */ \ - TESTALLOC_NGUARD*sizeof(u32)*2 + /* Guard words */ \ - sizeof(u32) + TESTALLOC_FILESIZE + /* File and line number */ \ - TESTALLOC_USERSIZE + /* User string */ \ - TESTALLOC_STACKSIZE /* backtrace() stack */ \ -) - - -/* -** For keeping track of the number of mallocs and frees. This -** is used to check for memory leaks. The iMallocFail and iMallocReset -** values are used to simulate malloc() failures during testing in -** order to verify that the library correctly handles an out-of-memory -** condition. -*/ -int sqlite3_nMalloc; /* Number of sqliteMalloc() calls */ -int sqlite3_nFree; /* Number of sqliteFree() calls */ -int sqlite3_memUsed; /* TODO Total memory obtained from malloc */ -int sqlite3_memMax; /* TODO Mem usage high-water mark */ -int sqlite3_iMallocFail; /* Fail sqliteMalloc() after this many calls */ -int sqlite3_iMallocReset = -1; /* When iMallocFail reaches 0, set to this */ - -void *sqlite3_pFirst = 0; /* Pointer to linked list of allocations */ -int sqlite3_nMaxAlloc = 0; /* High water mark of ThreadData.nAlloc */ -int sqlite3_mallocDisallowed = 0; /* assert() in sqlite3Malloc() if set */ -int sqlite3_isFail = 0; /* True if all malloc calls should fail */ -const char *sqlite3_zFile = 0; /* Filename to associate debug info with */ -int sqlite3_iLine = 0; /* Line number for debug info */ -int sqlite3_mallocfail_trace = 0; /* Print a msg on malloc fail if true */ - -/* -** Check for a simulated memory allocation failure. Return true if -** the failure should be simulated. Return false to proceed as normal. -*/ -int sqlite3TestMallocFail(){ - if( sqlite3_isFail ){ - return 1; - } - if( sqlite3_iMallocFail>=0 ){ - sqlite3_iMallocFail--; - if( sqlite3_iMallocFail==0 ){ - sqlite3_iMallocFail = sqlite3_iMallocReset; - sqlite3_isFail = 1; - if( sqlite3_mallocfail_trace ){ - sqlite3DebugPrintf("###_malloc_fails_###\n"); - } - return 1; - } - } - return 0; } /* -** The argument is a pointer returned by sqlite3OsMalloc() or xRealloc(). -** assert() that the first and last (TESTALLOC_NGUARD*4) bytes are set to the -** values set by the applyGuards() function. +** State information local to the memory allocation subsystem. */ -static void checkGuards(u32 *p) -{ - int i; - char *zAlloc = (char *)p; - char *z; +static SQLITE_WSD struct Mem0Global { + /* Number of free pages for scratch and page-cache memory */ + u32 nScratchFree; + u32 nPageFree; - /* First set of guard words */ - z = &zAlloc[TESTALLOC_OFFSET_GUARD1(p)]; - for(i=0; i1 -/* -** The argument points to an Os level allocation. Link it into the threads list -** of allocations. -*/ -static void linkAlloc(void *p){ - void **pp = (void **)p; - pp[0] = 0; - pp[1] = sqlite3_pFirst; - if( sqlite3_pFirst ){ - ((void **)sqlite3_pFirst)[0] = p; - } - sqlite3_pFirst = p; -} - -/* -** The argument points to an Os level allocation. Unlinke it from the threads -** list of allocations. -*/ -static void unlinkAlloc(void *p) -{ - void **pp = (void **)p; - if( p==sqlite3_pFirst ){ - assert(!pp[0]); - assert(!pp[1] || ((void **)(pp[1]))[0]==p); - sqlite3_pFirst = pp[1]; - if( sqlite3_pFirst ){ - ((void **)sqlite3_pFirst)[0] = 0; - } - }else{ - void **pprev = pp[0]; - void **pnext = pp[1]; - assert(pprev); - assert(pprev[1]==p); - pprev[1] = (void *)pnext; - if( pnext ){ - assert(pnext[0]==p); - pnext[0] = (void *)pprev; - } - } -} - -/* -** Pointer p is a pointer to an OS level allocation that has just been -** realloc()ed. Set the list pointers that point to this entry to it's new -** location. -*/ -static void relinkAlloc(void *p) -{ - void **pp = (void **)p; - if( pp[0] ){ - ((void **)(pp[0]))[1] = p; - }else{ - sqlite3_pFirst = p; - } - if( pp[1] ){ - ((void **)(pp[1]))[0] = p; - } -} -#else -#define linkAlloc(x) -#define relinkAlloc(x) -#define unlinkAlloc(x) -#endif - -/* -** This function sets the result of the Tcl interpreter passed as an argument -** to a list containing an entry for each currently outstanding call made to -** sqliteMalloc and friends by the current thread. Each list entry is itself a -** list, consisting of the following (in order): -** -** * The number of bytes allocated -** * The __FILE__ macro at the time of the sqliteMalloc() call. -** * The __LINE__ macro ... -** * The value of the sqlite3_malloc_id variable ... -** * The output of backtrace() (if available) ... -** -** Todo: We could have a version of this function that outputs to stdout, -** to debug memory leaks when Tcl is not available. -*/ -#if defined(TCLSH) && defined(SQLITE_DEBUG) && SQLITE_MEMDEBUG>1 -#include -int sqlite3OutstandingMallocs(Tcl_Interp *interp){ - void *p; - Tcl_Obj *pRes = Tcl_NewObj(); - Tcl_IncrRefCount(pRes); - - - for(p=sqlite3_pFirst; p; p=((void **)p)[1]){ - Tcl_Obj *pEntry = Tcl_NewObj(); - Tcl_Obj *pStack = Tcl_NewObj(); - char *z; - u32 iLine; - int nBytes = sqlite3OsAllocationSize(p) - TESTALLOC_OVERHEAD; - char *zAlloc = (char *)p; + if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100 + && sqlite3GlobalConfig.nScratch>=0 ){ int i; - - Tcl_ListObjAppendElement(0, pEntry, Tcl_NewIntObj(nBytes)); - - z = &zAlloc[TESTALLOC_OFFSET_FILENAME(p)]; - Tcl_ListObjAppendElement(0, pEntry, Tcl_NewStringObj(z, -1)); - - z = &zAlloc[TESTALLOC_OFFSET_LINENUMBER(p)]; - memcpy(&iLine, z, sizeof(u32)); - Tcl_ListObjAppendElement(0, pEntry, Tcl_NewIntObj(iLine)); - - z = &zAlloc[TESTALLOC_OFFSET_USER(p)]; - Tcl_ListObjAppendElement(0, pEntry, Tcl_NewStringObj(z, -1)); - - z = &zAlloc[TESTALLOC_OFFSET_STACK(p)]; - for(i=0; i=512 + && sqlite3GlobalConfig.nPage>=1 ){ + int i; + int overhead; + int sz = sqlite3GlobalConfig.szPage & ~7; + int n = sqlite3GlobalConfig.nPage; + overhead = (4*n + sz - 1)/sz; + sqlite3GlobalConfig.nPage -= overhead; + mem0.aPageFree = (u32*)&((char*)sqlite3GlobalConfig.pPage) + [sqlite3GlobalConfig.szPage*sqlite3GlobalConfig.nPage]; + for(i=0; inAlloc); -#endif - assert( !sqlite3_mallocDisallowed ); - if( !sqlite3TestMallocFail() ){ - u32 *p; - p = (u32 *)sqlite3OsMalloc(n + TESTALLOC_OVERHEAD); - assert(p); - sqlite3_nMalloc++; - applyGuards(p); - linkAlloc(p); - sqlite3OsLeaveMutex(); - return (void *)(&p[TESTALLOC_NGUARD + 2*sizeof(void *)/sizeof(u32)]); - } - sqlite3OsLeaveMutex(); - return 0; +static void sqlite3MallocAlarm(int nByte){ + void (*xCallback)(void*,sqlite3_int64,int); + sqlite3_int64 nowUsed; + void *pArg; + if( mem0.alarmCallback==0 || mem0.alarmBusy ) return; + mem0.alarmBusy = 1; + xCallback = mem0.alarmCallback; + nowUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + pArg = mem0.alarmArg; + sqlite3_mutex_leave(mem0.mutex); + xCallback(pArg, nowUsed, nByte); + sqlite3_mutex_enter(mem0.mutex); + mem0.alarmBusy = 0; } -static int OSSIZEOF(void *p){ +/* +** Do a memory allocation with statistics and alarms. Assume the +** lock is already held. +*/ +static int mallocWithAlarm(int n, void **pp){ + int nFull; + void *p; + assert( sqlite3_mutex_held(mem0.mutex) ); + nFull = sqlite3GlobalConfig.m.xRoundup(n); + sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); + if( mem0.alarmCallback!=0 ){ + int nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( nUsed+nFull >= mem0.alarmThreshold ){ + sqlite3MallocAlarm(nFull); + } + } + p = sqlite3GlobalConfig.m.xMalloc(nFull); + if( p==0 && mem0.alarmCallback ){ + sqlite3MallocAlarm(nFull); + p = sqlite3GlobalConfig.m.xMalloc(nFull); + } if( p ){ - u32 *pOs = (u32 *)getOsPointer(p); - return sqlite3OsAllocationSize(pOs) - TESTALLOC_OVERHEAD; + nFull = sqlite3MallocSize(p); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nFull); } - return 0; + *pp = p; + return nFull; } /* -** This is the test layer's wrapper around sqlite3OsFree(). The argument is a -** pointer to the space allocated for the application to use. +** Allocate memory. This routine is like sqlite3_malloc() except that it +** assumes the memory subsystem has already been initialized. */ -static void OSFREE(void *pFree){ - u32 *p; /* Pointer to the OS-layer allocation */ - sqlite3OsEnterMutex(); - p = (u32 *)getOsPointer(pFree); - checkGuards(p); - unlinkAlloc(p); - memset(pFree, 0x55, OSSIZEOF(pFree)); - sqlite3OsFree(p); - sqlite3_nFree++; - sqlite3OsLeaveMutex(); -} - -/* -** This is the test layer's wrapper around sqlite3OsRealloc(). -*/ -static void * OSREALLOC(void *pRealloc, int n){ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - sqlite3_nMaxAlloc = - MAX(sqlite3_nMaxAlloc, sqlite3ThreadDataReadOnly()->nAlloc); -#endif - assert( !sqlite3_mallocDisallowed ); - if( !sqlite3TestMallocFail() ){ - u32 *p = (u32 *)getOsPointer(pRealloc); - checkGuards(p); - p = sqlite3OsRealloc(p, n + TESTALLOC_OVERHEAD); - applyGuards(p); - relinkAlloc(p); - return (void *)(&p[TESTALLOC_NGUARD + 2*sizeof(void *)/sizeof(u32)]); - } - return 0; -} - -static void OSMALLOC_FAILED(){ - sqlite3_isFail = 0; -} - -#else -/* Define macros to call the sqlite3OsXXX interface directly if -** the SQLITE_MEMDEBUG macro is not defined. -*/ -#define OSMALLOC(x) sqlite3OsMalloc(x) -#define OSREALLOC(x,y) sqlite3OsRealloc(x,y) -#define OSFREE(x) sqlite3OsFree(x) -#define OSSIZEOF(x) sqlite3OsAllocationSize(x) -#define OSMALLOC_FAILED() - -#endif /* SQLITE_MEMDEBUG */ -/* -** End code for memory allocation system test layer. -**--------------------------------------------------------------------------*/ - -/* -** This routine is called when we are about to allocate n additional bytes -** of memory. If the new allocation will put is over the soft allocation -** limit, then invoke sqlite3_release_memory() to try to release some -** memory before continuing with the allocation. -** -** This routine also makes sure that the thread-specific-data (TSD) has -** be allocated. If it has not and can not be allocated, then return -** false. The updateMemoryUsedCount() routine below will deallocate -** the TSD if it ought to be. -** -** If SQLITE_ENABLE_MEMORY_MANAGEMENT is not defined, this routine is -** a no-op -*/ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT -static int enforceSoftLimit(int n){ - ThreadData *pTsd = sqlite3ThreadData(); - if( pTsd==0 ){ - return 0; - } - assert( pTsd->nAlloc>=0 ); - if( n>0 && pTsd->nSoftHeapLimit>0 ){ - while( pTsd->nAlloc+n>pTsd->nSoftHeapLimit && sqlite3_release_memory(n) ){} - } - return 1; -} -#else -# define enforceSoftLimit(X) 1 -#endif - -/* -** Update the count of total outstanding memory that is held in -** thread-specific-data (TSD). If after this update the TSD is -** no longer being used, then deallocate it. -** -** If SQLITE_ENABLE_MEMORY_MANAGEMENT is not defined, this routine is -** a no-op -*/ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT -static void updateMemoryUsedCount(int n){ - ThreadData *pTsd = sqlite3ThreadData(); - if( pTsd ){ - pTsd->nAlloc += n; - assert( pTsd->nAlloc>=0 ); - if( pTsd->nAlloc==0 && pTsd->nSoftHeapLimit==0 ){ - sqlite3ReleaseThreadData(); - } - } -} -#else -#define updateMemoryUsedCount(x) /* no-op */ -#endif - -/* -** Allocate and return N bytes of uninitialised memory by calling -** sqlite3OsMalloc(). If the Malloc() call fails, attempt to free memory -** by calling sqlite3_release_memory(). -*/ -void *sqlite3MallocRaw(int n, int doMemManage){ - void *p = 0; - if( n>0 && !sqlite3MallocFailed() && (!doMemManage || enforceSoftLimit(n)) ){ - while( (p = OSMALLOC(n))==0 && sqlite3_release_memory(n) ){} - if( !p ){ - sqlite3FailedMalloc(); - OSMALLOC_FAILED(); - }else if( doMemManage ){ - updateMemoryUsedCount(OSSIZEOF(p)); - } +void *sqlite3Malloc(int n){ + void *p; + if( n<=0 ){ + p = 0; + }else if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_mutex_enter(mem0.mutex); + mallocWithAlarm(n, &p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + p = sqlite3GlobalConfig.m.xMalloc(n); } return p; } /* -** Resize the allocation at p to n bytes by calling sqlite3OsRealloc(). The -** pointer to the new allocation is returned. If the Realloc() call fails, -** attempt to free memory by calling sqlite3_release_memory(). +** This version of the memory allocation is for use by the application. +** First make sure the memory subsystem is initialized, then do the +** allocation. */ -void *sqlite3Realloc(void *p, int n){ - if( sqlite3MallocFailed() ){ +void *sqlite3_malloc(int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return sqlite3Malloc(n); +} + +/* +** Each thread may only have a single outstanding allocation from +** xScratchMalloc(). We verify this constraint in the single-threaded +** case by setting scratchAllocOut to 1 when an allocation +** is outstanding clearing it when the allocation is freed. +*/ +#if SQLITE_THREADSAFE==0 && !defined(NDEBUG) +static int scratchAllocOut = 0; +#endif + + +/* +** Allocate memory that is to be used and released right away. +** This routine is similar to alloca() in that it is not intended +** for situations where the memory might be held long-term. This +** routine is intended to get memory to old large transient data +** structures that would not normally fit on the stack of an +** embedded processor. +*/ +void *sqlite3ScratchMalloc(int n){ + void *p; + assert( n>0 ); + +#if SQLITE_THREADSAFE==0 && !defined(NDEBUG) + /* Verify that no more than one scratch allocation per thread + ** is outstanding at one time. (This is only checked in the + ** single-threaded case since checking in the multi-threaded case + ** would be much more complicated.) */ + assert( scratchAllocOut==0 ); +#endif + + if( sqlite3GlobalConfig.szScratch=(void*)mem0.aScratchFree ){ + if( sqlite3GlobalConfig.bMemstat ){ + int iSize = sqlite3MallocSize(p); + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, -iSize); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -iSize); + sqlite3GlobalConfig.m.xFree(p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3GlobalConfig.m.xFree(p); + } + }else{ + int i; + i = (int)((u8*)p - (u8*)sqlite3GlobalConfig.pScratch); + i /= sqlite3GlobalConfig.szScratch; + assert( i>=0 && i0 ); + assert( (n & (n-1))==0 ); + assert( n>=512 && n<=32768 ); + + if( sqlite3GlobalConfig.szPage=(void*)mem0.aPageFree ){ + /* In this case, the page allocation was obtained from a regular + ** call to sqlite3_mem_methods.xMalloc() (a page-cache-memory + ** "overflow"). Free the block with sqlite3_mem_methods.xFree(). + */ + if( sqlite3GlobalConfig.bMemstat ){ + int iSize = sqlite3MallocSize(p); + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -iSize); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -iSize); + sqlite3GlobalConfig.m.xFree(p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3GlobalConfig.m.xFree(p); + } + }else{ + /* The page allocation was allocated from the sqlite3GlobalConfig.pPage + ** buffer. In this case all that is add the index of the page in + ** the sqlite3GlobalConfig.pPage array to the set of free indexes stored + ** in the mem0.aPageFree[] array. + */ + int i; + i = (u8 *)p - (u8 *)sqlite3GlobalConfig.pPage; + i /= sqlite3GlobalConfig.szPage; + assert( i>=0 && i=db->lookaside.pStart && plookaside.pEnd; +} +#else +#define isLookaside(A,B) 0 +#endif + +/* +** Return the size of a memory allocation previously obtained from +** sqlite3Malloc() or sqlite3_malloc(). +*/ +int sqlite3MallocSize(void *p){ + return sqlite3GlobalConfig.m.xSize(p); +} +int sqlite3DbMallocSize(sqlite3 *db, void *p){ + if( p==0 ){ + return 0; + }else if( isLookaside(db, p) ){ + return db->lookaside.sz; + }else{ + return sqlite3GlobalConfig.m.xSize(p); + } +} + +/* +** Free memory previously obtained from sqlite3Malloc(). +*/ +void sqlite3_free(void *p){ + if( p==0 ) return; + if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -sqlite3MallocSize(p)); + sqlite3GlobalConfig.m.xFree(p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3GlobalConfig.m.xFree(p); + } +} + +/* +** Free memory that might be associated with a particular database +** connection. +*/ +void sqlite3DbFree(sqlite3 *db, void *p){ + if( isLookaside(db, p) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + db->lookaside.nOut--; + }else{ + sqlite3_free(p); + } +} + +/* +** Change the size of an existing memory allocation +*/ +void *sqlite3Realloc(void *pOld, int nBytes){ + int nOld, nNew; + void *pNew; + if( pOld==0 ){ + return sqlite3Malloc(nBytes); + } + if( nBytes<=0 ){ + sqlite3_free(pOld); return 0; } - - if( !p ){ - return sqlite3Malloc(n, 1); - }else{ - void *np = 0; -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - int origSize = OSSIZEOF(p); -#endif - if( enforceSoftLimit(n - origSize) ){ - while( (np = OSREALLOC(p, n))==0 && sqlite3_release_memory(n) ){} - if( !np ){ - sqlite3FailedMalloc(); - OSMALLOC_FAILED(); - }else{ - updateMemoryUsedCount(OSSIZEOF(np) - origSize); + nOld = sqlite3MallocSize(pOld); + if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, nBytes); + nNew = sqlite3GlobalConfig.m.xRoundup(nBytes); + if( nOld==nNew ){ + pNew = pOld; + }else{ + if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nNew-nOld >= + mem0.alarmThreshold ){ + sqlite3MallocAlarm(nNew-nOld); + } + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); + if( pNew==0 && mem0.alarmCallback ){ + sqlite3MallocAlarm(nBytes); + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); + } + if( pNew ){ + nNew = sqlite3MallocSize(pNew); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nNew-nOld); } } - return np; - } -} - -/* -** Free the memory pointed to by p. p must be either a NULL pointer or a -** value returned by a previous call to sqlite3Malloc() or sqlite3Realloc(). -*/ -void sqlite3FreeX(void *p){ - if( p ){ - updateMemoryUsedCount(0 - OSSIZEOF(p)); - OSFREE(p); - } -} - -/* -** A version of sqliteMalloc() that is always a function, not a macro. -** Currently, this is used only to alloc to allocate the parser engine. -*/ -void *sqlite3MallocX(int n){ - return sqliteMalloc(n); -} - -/* -** sqlite3Malloc -** sqlite3ReallocOrFree -** -** These two are implemented as wrappers around sqlite3MallocRaw(), -** sqlite3Realloc() and sqlite3Free(). -*/ -void *sqlite3Malloc(int n, int doMemManage){ - void *p = sqlite3MallocRaw(n, doMemManage); - if( p ){ - memset(p, 0, n); - } - return p; -} -void *sqlite3ReallocOrFree(void *p, int n){ - void *pNew; - pNew = sqlite3Realloc(p, n); - if( !pNew ){ - sqlite3FreeX(p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nBytes); } return pNew; } /* -** sqlite3ThreadSafeMalloc() and sqlite3ThreadSafeFree() are used in those -** rare scenarios where sqlite may allocate memory in one thread and free -** it in another. They are exactly the same as sqlite3Malloc() and -** sqlite3Free() except that: -** -** * The allocated memory is not included in any calculations with -** respect to the soft-heap-limit, and -** -** * sqlite3ThreadSafeMalloc() must be matched with ThreadSafeFree(), -** not sqlite3Free(). Calling sqlite3Free() on memory obtained from -** ThreadSafeMalloc() will cause an error somewhere down the line. +** The public interface to sqlite3Realloc. Make sure that the memory +** subsystem is initialized prior to invoking sqliteRealloc. */ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT -void *sqlite3ThreadSafeMalloc(int n){ - (void)ENTER_MALLOC; - return sqlite3Malloc(n, 0); -} -void sqlite3ThreadSafeFree(void *p){ - (void)ENTER_MALLOC; - if( p ){ - OSFREE(p); - } -} +void *sqlite3_realloc(void *pOld, int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; #endif + return sqlite3Realloc(pOld, n); +} /* -** Return the number of bytes allocated at location p. p must be either -** a NULL pointer (in which case 0 is returned) or a pointer returned by -** sqlite3Malloc(), sqlite3Realloc() or sqlite3ReallocOrFree(). -** -** The number of bytes allocated does not include any overhead inserted by -** any malloc() wrapper functions that may be called. So the value returned -** is the number of bytes that were available to SQLite using pointer p, -** regardless of how much memory was actually allocated. -*/ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT -int sqlite3AllocSize(void *p){ - return OSSIZEOF(p); +** Allocate and zero memory. +*/ +void *sqlite3MallocZero(int n){ + void *p = sqlite3Malloc(n); + if( p ){ + memset(p, 0, n); + } + return p; } + +/* +** Allocate and zero memory. If the allocation fails, make +** the mallocFailed flag in the connection pointer. +*/ +void *sqlite3DbMallocZero(sqlite3 *db, int n){ + void *p = sqlite3DbMallocRaw(db, n); + if( p ){ + memset(p, 0, n); + } + return p; +} + +/* +** Allocate and zero memory. If the allocation fails, make +** the mallocFailed flag in the connection pointer. +** +** If db!=0 and db->mallocFailed is true (indicating a prior malloc +** failure on the same database connection) then always return 0. +** Hence for a particular database connection, once malloc starts +** failing, it fails consistently until mallocFailed is reset. +** This is an important assumption. There are many places in the +** code that do things like this: +** +** int *a = (int*)sqlite3DbMallocRaw(db, 100); +** int *b = (int*)sqlite3DbMallocRaw(db, 200); +** if( b ) a[10] = 9; +** +** In other words, if a subsequent malloc (ex: "b") worked, it is assumed +** that all prior mallocs (ex: "a") worked too. +*/ +void *sqlite3DbMallocRaw(sqlite3 *db, int n){ + void *p; +#ifndef SQLITE_OMIT_LOOKASIDE + if( db ){ + LookasideSlot *pBuf; + if( db->mallocFailed ){ + return 0; + } + if( db->lookaside.bEnabled && n<=db->lookaside.sz + && (pBuf = db->lookaside.pFree)!=0 ){ + db->lookaside.pFree = pBuf->pNext; + db->lookaside.nOut++; + if( db->lookaside.nOut>db->lookaside.mxOut ){ + db->lookaside.mxOut = db->lookaside.nOut; + } + return (void*)pBuf; + } + } +#else + if( db && db->mallocFailed ){ + return 0; + } #endif + p = sqlite3Malloc(n); + if( !p && db ){ + db->mallocFailed = 1; + } + return p; +} + +/* +** Resize the block of memory pointed to by p to n bytes. If the +** resize fails, set the mallocFailed flag in the connection object. +*/ +void *sqlite3DbRealloc(sqlite3 *db, void *p, int n){ + void *pNew = 0; + if( db->mallocFailed==0 ){ + if( p==0 ){ + return sqlite3DbMallocRaw(db, n); + } + if( isLookaside(db, p) ){ + if( n<=db->lookaside.sz ){ + return p; + } + pNew = sqlite3DbMallocRaw(db, n); + if( pNew ){ + memcpy(pNew, p, db->lookaside.sz); + sqlite3DbFree(db, p); + } + }else{ + pNew = sqlite3_realloc(p, n); + if( !pNew ){ + db->mallocFailed = 1; + } + } + } + return pNew; +} + +/* +** Attempt to reallocate p. If the reallocation fails, then free p +** and set the mallocFailed flag in the database connection. +*/ +void *sqlite3DbReallocOrFree(sqlite3 *db, void *p, int n){ + void *pNew; + pNew = sqlite3DbRealloc(db, p, n); + if( !pNew ){ + sqlite3DbFree(db, p); + } + return pNew; +} /* ** Make a copy of a string in memory obtained from sqliteMalloc(). These @@ -722,19 +719,27 @@ int sqlite3AllocSize(void *p){ ** called via macros that record the current file and line number in the ** ThreadData structure. */ -char *sqlite3StrDup(const char *z){ +char *sqlite3DbStrDup(sqlite3 *db, const char *z){ char *zNew; - int n; - if( z==0 ) return 0; - n = strlen(z)+1; - zNew = sqlite3MallocRaw(n, 1); - if( zNew ) memcpy(zNew, z, n); + size_t n; + if( z==0 ){ + return 0; + } + n = (db ? sqlite3Strlen(db, z) : sqlite3Strlen30(z))+1; + assert( (n&0x7fffffff)==n ); + zNew = sqlite3DbMallocRaw(db, (int)n); + if( zNew ){ + memcpy(zNew, z, n); + } return zNew; } -char *sqlite3StrNDup(const char *z, int n){ +char *sqlite3DbStrNDup(sqlite3 *db, const char *z, int n){ char *zNew; - if( z==0 ) return 0; - zNew = sqlite3MallocRaw(n+1, 1); + if( z==0 ){ + return 0; + } + assert( (n&0x7fffffff)==n ); + zNew = sqlite3DbMallocRaw(db, n+1); if( zNew ){ memcpy(zNew, z, n); zNew[n] = 0; @@ -743,46 +748,26 @@ char *sqlite3StrNDup(const char *z, int n){ } /* -** Create a string from the 2nd and subsequent arguments (up to the -** first NULL argument), store the string in memory obtained from -** sqliteMalloc() and make the pointer indicated by the 1st argument -** point to that string. The 1st argument must either be NULL or -** point to memory obtained from sqliteMalloc(). +** Create a string from the zFromat argument and the va_list that follows. +** Store the string in memory obtained from sqliteMalloc() and make *pz +** point to that string. */ -void sqlite3SetString(char **pz, ...){ +void sqlite3SetString(char **pz, sqlite3 *db, const char *zFormat, ...){ va_list ap; - int nByte; - const char *z; - char *zResult; + char *z; - assert( pz!=0 ); - nByte = 1; - va_start(ap, pz); - while( (z = va_arg(ap, const char*))!=0 ){ - nByte += strlen(z); - } - va_end(ap); - sqliteFree(*pz); - *pz = zResult = sqliteMallocRaw( nByte ); - if( zResult==0 ){ - return; - } - *zResult = 0; - va_start(ap, pz); - while( (z = va_arg(ap, const char*))!=0 ){ - int n = strlen(z); - memcpy(zResult, z, n); - zResult += n; - } - zResult[0] = 0; + va_start(ap, zFormat); + z = sqlite3VMPrintf(db, zFormat, ap); va_end(ap); + sqlite3DbFree(db, *pz); + *pz = z; } /* ** This function must be called before exiting any API function (i.e. -** returning control to the user) that has called sqlite3Malloc or -** sqlite3Realloc. +** returning control to the user) that has called sqlite3_malloc or +** sqlite3_realloc. ** ** The returned value is normally a copy of the second argument to this ** function. However, if a malloc() failure has occured since the previous @@ -792,44 +777,16 @@ void sqlite3SetString(char **pz, ...){ ** then the connection error-code (the value returned by sqlite3_errcode()) ** is set to SQLITE_NOMEM. */ -int sqlite3MallocHasFailed = 0; int sqlite3ApiExit(sqlite3* db, int rc){ - if( sqlite3MallocFailed() ){ - sqlite3MallocHasFailed = 0; - sqlite3OsLeaveMutex(); + /* If the db handle is not NULL, then we must hold the connection handle + ** mutex here. Otherwise the read (and possible write) of db->mallocFailed + ** is unsafe, as is the call to sqlite3Error(). + */ + assert( !db || sqlite3_mutex_held(db->mutex) ); + if( db && (db->mallocFailed || rc==SQLITE_IOERR_NOMEM) ){ sqlite3Error(db, SQLITE_NOMEM, 0); + db->mallocFailed = 0; rc = SQLITE_NOMEM; } return rc & (db ? db->errMask : 0xff); } - -/* -** Set the "malloc has failed" condition to true for this thread. -*/ -void sqlite3FailedMalloc(){ - if( !sqlite3MallocFailed() ){ - sqlite3OsEnterMutex(); - assert( sqlite3MallocHasFailed==0 ); - sqlite3MallocHasFailed = 1; - } -} - -#ifdef SQLITE_MEMDEBUG -/* -** This function sets a flag in the thread-specific-data structure that will -** cause an assert to fail if sqliteMalloc() or sqliteRealloc() is called. -*/ -void sqlite3MallocDisallow(){ - assert( sqlite3_mallocDisallowed>=0 ); - sqlite3_mallocDisallowed++; -} - -/* -** This function clears the flag set in the thread-specific-data structure set -** by sqlite3MallocDisallow(). -*/ -void sqlite3MallocAllow(){ - assert( sqlite3_mallocDisallowed>0 ); - sqlite3_mallocDisallowed--; -} -#endif diff --git a/mem0.c b/mem0.c new file mode 100644 index 0000000..0cac07b --- /dev/null +++ b/mem0.c @@ -0,0 +1,61 @@ +/* +** 2008 October 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains a no-op memory allocation drivers for use when +** SQLITE_ZERO_MALLOC is defined. The allocation drivers implemented +** here always fail. SQLite will not operate with these drivers. These +** are merely placeholders. Real drivers must be substituted using +** sqlite3_config() before SQLite will operate. +** +** $Id: mem0.c,v 1.1 2008/10/28 18:58:20 drh Exp $ +*/ +#include "sqliteInt.h" + +/* +** This version of the memory allocator is the default. It is +** used when no other memory allocator is specified using compile-time +** macros. +*/ +#ifdef SQLITE_ZERO_MALLOC + +/* +** No-op versions of all memory allocation routines +*/ +static void *sqlite3MemMalloc(int nByte){ return 0; } +static void sqlite3MemFree(void *pPrior){ return; } +static void *sqlite3MemRealloc(void *pPrior, int nByte){ return 0; } +static int sqlite3MemSize(void *pPrior){ return 0; } +static int sqlite3MemRoundup(int n){ return n; } +static int sqlite3MemInit(void *NotUsed){ return SQLITE_OK; } +static void sqlite3MemShutdown(void *NotUsed){ return; } + +/* +** This routine is the only routine in this file with external linkage. +** +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. +*/ +void sqlite3MemSetDefault(void){ + static const sqlite3_mem_methods defaultMethods = { + sqlite3MemMalloc, + sqlite3MemFree, + sqlite3MemRealloc, + sqlite3MemSize, + sqlite3MemRoundup, + sqlite3MemInit, + sqlite3MemShutdown, + 0 + }; + sqlite3_config(SQLITE_CONFIG_MALLOC, &defaultMethods); +} + +#endif /* SQLITE_ZERO_MALLOC */ diff --git a/mem1.c b/mem1.c new file mode 100644 index 0000000..ca1e6ae --- /dev/null +++ b/mem1.c @@ -0,0 +1,145 @@ +/* +** 2007 August 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains low-level memory allocation drivers for when +** SQLite will use the standard C-library malloc/realloc/free interface +** to obtain the memory it needs. +** +** This file contains implementations of the low-level memory allocation +** routines specified in the sqlite3_mem_methods object. +** +** $Id: mem1.c,v 1.29 2008/12/10 21:19:57 drh Exp $ +*/ +#include "sqliteInt.h" + +/* +** This version of the memory allocator is the default. It is +** used when no other memory allocator is specified using compile-time +** macros. +*/ +#ifdef SQLITE_SYSTEM_MALLOC + +/* +** Like malloc(), but remember the size of the allocation +** so that we can find it later using sqlite3MemSize(). +** +** For this low-level routine, we are guaranteed that nByte>0 because +** cases of nByte<=0 will be intercepted and dealt with by higher level +** routines. +*/ +static void *sqlite3MemMalloc(int nByte){ + sqlite3_int64 *p; + assert( nByte>0 ); + nByte = (nByte+7)&~7; + p = malloc( nByte+8 ); + if( p ){ + p[0] = nByte; + p++; + } + return (void *)p; +} + +/* +** Like free() but works for allocations obtained from sqlite3MemMalloc() +** or sqlite3MemRealloc(). +** +** For this low-level routine, we already know that pPrior!=0 since +** cases where pPrior==0 will have been intecepted and dealt with +** by higher-level routines. +*/ +static void sqlite3MemFree(void *pPrior){ + sqlite3_int64 *p = (sqlite3_int64*)pPrior; + assert( pPrior!=0 ); + p--; + free(p); +} + +/* +** Like realloc(). Resize an allocation previously obtained from +** sqlite3MemMalloc(). +** +** For this low-level interface, we know that pPrior!=0. Cases where +** pPrior==0 while have been intercepted by higher-level routine and +** redirected to xMalloc. Similarly, we know that nByte>0 becauses +** cases where nByte<=0 will have been intercepted by higher-level +** routines and redirected to xFree. +*/ +static void *sqlite3MemRealloc(void *pPrior, int nByte){ + sqlite3_int64 *p = (sqlite3_int64*)pPrior; + assert( pPrior!=0 && nByte>0 ); + nByte = (nByte+7)&~7; + p = (sqlite3_int64*)pPrior; + p--; + p = realloc(p, nByte+8 ); + if( p ){ + p[0] = nByte; + p++; + } + return (void*)p; +} + +/* +** Report the allocated size of a prior return from xMalloc() +** or xRealloc(). +*/ +static int sqlite3MemSize(void *pPrior){ + sqlite3_int64 *p; + if( pPrior==0 ) return 0; + p = (sqlite3_int64*)pPrior; + p--; + return (int)p[0]; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int sqlite3MemRoundup(int n){ + return (n+7) & ~7; +} + +/* +** Initialize this module. +*/ +static int sqlite3MemInit(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + return SQLITE_OK; +} + +/* +** Deinitialize this module. +*/ +static void sqlite3MemShutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + return; +} + +/* +** This routine is the only routine in this file with external linkage. +** +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. +*/ +void sqlite3MemSetDefault(void){ + static const sqlite3_mem_methods defaultMethods = { + sqlite3MemMalloc, + sqlite3MemFree, + sqlite3MemRealloc, + sqlite3MemSize, + sqlite3MemRoundup, + sqlite3MemInit, + sqlite3MemShutdown, + 0 + }; + sqlite3_config(SQLITE_CONFIG_MALLOC, &defaultMethods); +} + +#endif /* SQLITE_SYSTEM_MALLOC */ diff --git a/mem2.c b/mem2.c new file mode 100644 index 0000000..25a6a56 --- /dev/null +++ b/mem2.c @@ -0,0 +1,440 @@ +/* +** 2007 August 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains low-level memory allocation drivers for when +** SQLite will use the standard C-library malloc/realloc/free interface +** to obtain the memory it needs while adding lots of additional debugging +** information to each allocation in order to help detect and fix memory +** leaks and memory usage errors. +** +** This file contains implementations of the low-level memory allocation +** routines specified in the sqlite3_mem_methods object. +** +** $Id: mem2.c,v 1.42 2008/12/10 19:26:24 drh Exp $ +*/ +#include "sqliteInt.h" + +/* +** This version of the memory allocator is used only if the +** SQLITE_MEMDEBUG macro is defined +*/ +#ifdef SQLITE_MEMDEBUG + +/* +** The backtrace functionality is only available with GLIBC +*/ +#ifdef __GLIBC__ + extern int backtrace(void**,int); + extern void backtrace_symbols_fd(void*const*,int,int); +#else +# define backtrace(A,B) 1 +# define backtrace_symbols_fd(A,B,C) +#endif +#include + +/* +** Each memory allocation looks like this: +** +** ------------------------------------------------------------------------ +** | Title | backtrace pointers | MemBlockHdr | allocation | EndGuard | +** ------------------------------------------------------------------------ +** +** The application code sees only a pointer to the allocation. We have +** to back up from the allocation pointer to find the MemBlockHdr. The +** MemBlockHdr tells us the size of the allocation and the number of +** backtrace pointers. There is also a guard word at the end of the +** MemBlockHdr. +*/ +struct MemBlockHdr { + i64 iSize; /* Size of this allocation */ + struct MemBlockHdr *pNext, *pPrev; /* Linked list of all unfreed memory */ + char nBacktrace; /* Number of backtraces on this alloc */ + char nBacktraceSlots; /* Available backtrace slots */ + short nTitle; /* Bytes of title; includes '\0' */ + int iForeGuard; /* Guard word for sanity */ +}; + +/* +** Guard words +*/ +#define FOREGUARD 0x80F5E153 +#define REARGUARD 0xE4676B53 + +/* +** Number of malloc size increments to track. +*/ +#define NCSIZE 1000 + +/* +** All of the static variables used by this module are collected +** into a single structure named "mem". This is to keep the +** static variables organized and to reduce namespace pollution +** when this module is combined with other in the amalgamation. +*/ +static struct { + + /* + ** Mutex to control access to the memory allocation subsystem. + */ + sqlite3_mutex *mutex; + + /* + ** Head and tail of a linked list of all outstanding allocations + */ + struct MemBlockHdr *pFirst; + struct MemBlockHdr *pLast; + + /* + ** The number of levels of backtrace to save in new allocations. + */ + int nBacktrace; + void (*xBacktrace)(int, int, void **); + + /* + ** Title text to insert in front of each block + */ + int nTitle; /* Bytes of zTitle to save. Includes '\0' and padding */ + char zTitle[100]; /* The title text */ + + /* + ** sqlite3MallocDisallow() increments the following counter. + ** sqlite3MallocAllow() decrements it. + */ + int disallow; /* Do not allow memory allocation */ + + /* + ** Gather statistics on the sizes of memory allocations. + ** nAlloc[i] is the number of allocation attempts of i*8 + ** bytes. i==NCSIZE is the number of allocation attempts for + ** sizes more than NCSIZE*8 bytes. + */ + int nAlloc[NCSIZE]; /* Total number of allocations */ + int nCurrent[NCSIZE]; /* Current number of allocations */ + int mxCurrent[NCSIZE]; /* Highwater mark for nCurrent */ + +} mem; + + +/* +** Adjust memory usage statistics +*/ +static void adjustStats(int iSize, int increment){ + int i = ((iSize+7)&~7)/8; + if( i>NCSIZE-1 ){ + i = NCSIZE - 1; + } + if( increment>0 ){ + mem.nAlloc[i]++; + mem.nCurrent[i]++; + if( mem.nCurrent[i]>mem.mxCurrent[i] ){ + mem.mxCurrent[i] = mem.nCurrent[i]; + } + }else{ + mem.nCurrent[i]--; + assert( mem.nCurrent[i]>=0 ); + } +} + +/* +** Given an allocation, find the MemBlockHdr for that allocation. +** +** This routine checks the guards at either end of the allocation and +** if they are incorrect it asserts. +*/ +static struct MemBlockHdr *sqlite3MemsysGetHeader(void *pAllocation){ + struct MemBlockHdr *p; + int *pInt; + u8 *pU8; + int nReserve; + + p = (struct MemBlockHdr*)pAllocation; + p--; + assert( p->iForeGuard==(int)FOREGUARD ); + nReserve = (p->iSize+7)&~7; + pInt = (int*)pAllocation; + pU8 = (u8*)pAllocation; + assert( pInt[nReserve/sizeof(int)]==(int)REARGUARD ); + assert( (nReserve-0)<=p->iSize || pU8[nReserve-1]==0x65 ); + assert( (nReserve-1)<=p->iSize || pU8[nReserve-2]==0x65 ); + assert( (nReserve-2)<=p->iSize || pU8[nReserve-3]==0x65 ); + return p; +} + +/* +** Return the number of bytes currently allocated at address p. +*/ +static int sqlite3MemSize(void *p){ + struct MemBlockHdr *pHdr; + if( !p ){ + return 0; + } + pHdr = sqlite3MemsysGetHeader(p); + return pHdr->iSize; +} + +/* +** Initialize the memory allocation subsystem. +*/ +static int sqlite3MemInit(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + if( !sqlite3GlobalConfig.bMemstat ){ + /* If memory status is enabled, then the malloc.c wrapper will already + ** hold the STATIC_MEM mutex when the routines here are invoked. */ + mem.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + return SQLITE_OK; +} + +/* +** Deinitialize the memory allocation subsystem. +*/ +static void sqlite3MemShutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + mem.mutex = 0; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int sqlite3MemRoundup(int n){ + return (n+7) & ~7; +} + +/* +** Allocate nByte bytes of memory. +*/ +static void *sqlite3MemMalloc(int nByte){ + struct MemBlockHdr *pHdr; + void **pBt; + char *z; + int *pInt; + void *p = 0; + int totalSize; + int nReserve; + sqlite3_mutex_enter(mem.mutex); + assert( mem.disallow==0 ); + nReserve = (nByte+7)&~7; + totalSize = nReserve + sizeof(*pHdr) + sizeof(int) + + mem.nBacktrace*sizeof(void*) + mem.nTitle; + p = malloc(totalSize); + if( p ){ + z = p; + pBt = (void**)&z[mem.nTitle]; + pHdr = (struct MemBlockHdr*)&pBt[mem.nBacktrace]; + pHdr->pNext = 0; + pHdr->pPrev = mem.pLast; + if( mem.pLast ){ + mem.pLast->pNext = pHdr; + }else{ + mem.pFirst = pHdr; + } + mem.pLast = pHdr; + pHdr->iForeGuard = FOREGUARD; + pHdr->nBacktraceSlots = mem.nBacktrace; + pHdr->nTitle = mem.nTitle; + if( mem.nBacktrace ){ + void *aAddr[40]; + pHdr->nBacktrace = backtrace(aAddr, mem.nBacktrace+1)-1; + memcpy(pBt, &aAddr[1], pHdr->nBacktrace*sizeof(void*)); + if( mem.xBacktrace ){ + mem.xBacktrace(nByte, pHdr->nBacktrace-1, &aAddr[1]); + } + }else{ + pHdr->nBacktrace = 0; + } + if( mem.nTitle ){ + memcpy(z, mem.zTitle, mem.nTitle); + } + pHdr->iSize = nByte; + adjustStats(nByte, +1); + pInt = (int*)&pHdr[1]; + pInt[nReserve/sizeof(int)] = REARGUARD; + memset(pInt, 0x65, nReserve); + p = (void*)pInt; + } + sqlite3_mutex_leave(mem.mutex); + return p; +} + +/* +** Free memory. +*/ +static void sqlite3MemFree(void *pPrior){ + struct MemBlockHdr *pHdr; + void **pBt; + char *z; + assert( sqlite3GlobalConfig.bMemstat || mem.mutex!=0 ); + pHdr = sqlite3MemsysGetHeader(pPrior); + pBt = (void**)pHdr; + pBt -= pHdr->nBacktraceSlots; + sqlite3_mutex_enter(mem.mutex); + if( pHdr->pPrev ){ + assert( pHdr->pPrev->pNext==pHdr ); + pHdr->pPrev->pNext = pHdr->pNext; + }else{ + assert( mem.pFirst==pHdr ); + mem.pFirst = pHdr->pNext; + } + if( pHdr->pNext ){ + assert( pHdr->pNext->pPrev==pHdr ); + pHdr->pNext->pPrev = pHdr->pPrev; + }else{ + assert( mem.pLast==pHdr ); + mem.pLast = pHdr->pPrev; + } + z = (char*)pBt; + z -= pHdr->nTitle; + adjustStats(pHdr->iSize, -1); + memset(z, 0x2b, sizeof(void*)*pHdr->nBacktraceSlots + sizeof(*pHdr) + + pHdr->iSize + sizeof(int) + pHdr->nTitle); + free(z); + sqlite3_mutex_leave(mem.mutex); +} + +/* +** Change the size of an existing memory allocation. +** +** For this debugging implementation, we *always* make a copy of the +** allocation into a new place in memory. In this way, if the +** higher level code is using pointer to the old allocation, it is +** much more likely to break and we are much more liking to find +** the error. +*/ +static void *sqlite3MemRealloc(void *pPrior, int nByte){ + struct MemBlockHdr *pOldHdr; + void *pNew; + assert( mem.disallow==0 ); + pOldHdr = sqlite3MemsysGetHeader(pPrior); + pNew = sqlite3MemMalloc(nByte); + if( pNew ){ + memcpy(pNew, pPrior, nByteiSize ? nByte : pOldHdr->iSize); + if( nByte>pOldHdr->iSize ){ + memset(&((char*)pNew)[pOldHdr->iSize], 0x2b, nByte - pOldHdr->iSize); + } + sqlite3MemFree(pPrior); + } + return pNew; +} + +/* +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. +*/ +void sqlite3MemSetDefault(void){ + static const sqlite3_mem_methods defaultMethods = { + sqlite3MemMalloc, + sqlite3MemFree, + sqlite3MemRealloc, + sqlite3MemSize, + sqlite3MemRoundup, + sqlite3MemInit, + sqlite3MemShutdown, + 0 + }; + sqlite3_config(SQLITE_CONFIG_MALLOC, &defaultMethods); +} + +/* +** Set the number of backtrace levels kept for each allocation. +** A value of zero turns off backtracing. The number is always rounded +** up to a multiple of 2. +*/ +void sqlite3MemdebugBacktrace(int depth){ + if( depth<0 ){ depth = 0; } + if( depth>20 ){ depth = 20; } + depth = (depth+1)&0xfe; + mem.nBacktrace = depth; +} + +void sqlite3MemdebugBacktraceCallback(void (*xBacktrace)(int, int, void **)){ + mem.xBacktrace = xBacktrace; +} + +/* +** Set the title string for subsequent allocations. +*/ +void sqlite3MemdebugSettitle(const char *zTitle){ + unsigned int n = sqlite3Strlen30(zTitle) + 1; + sqlite3_mutex_enter(mem.mutex); + if( n>=sizeof(mem.zTitle) ) n = sizeof(mem.zTitle)-1; + memcpy(mem.zTitle, zTitle, n); + mem.zTitle[n] = 0; + mem.nTitle = (n+7)&~7; + sqlite3_mutex_leave(mem.mutex); +} + +void sqlite3MemdebugSync(){ + struct MemBlockHdr *pHdr; + for(pHdr=mem.pFirst; pHdr; pHdr=pHdr->pNext){ + void **pBt = (void**)pHdr; + pBt -= pHdr->nBacktraceSlots; + mem.xBacktrace(pHdr->iSize, pHdr->nBacktrace-1, &pBt[1]); + } +} + +/* +** Open the file indicated and write a log of all unfreed memory +** allocations into that log. +*/ +void sqlite3MemdebugDump(const char *zFilename){ + FILE *out; + struct MemBlockHdr *pHdr; + void **pBt; + int i; + out = fopen(zFilename, "w"); + if( out==0 ){ + fprintf(stderr, "** Unable to output memory debug output log: %s **\n", + zFilename); + return; + } + for(pHdr=mem.pFirst; pHdr; pHdr=pHdr->pNext){ + char *z = (char*)pHdr; + z -= pHdr->nBacktraceSlots*sizeof(void*) + pHdr->nTitle; + fprintf(out, "**** %lld bytes at %p from %s ****\n", + pHdr->iSize, &pHdr[1], pHdr->nTitle ? z : "???"); + if( pHdr->nBacktrace ){ + fflush(out); + pBt = (void**)pHdr; + pBt -= pHdr->nBacktraceSlots; + backtrace_symbols_fd(pBt, pHdr->nBacktrace, fileno(out)); + fprintf(out, "\n"); + } + } + fprintf(out, "COUNTS:\n"); + for(i=0; i=1 ); + size = mem3.aPool[i-1].u.hdr.size4x/4; + assert( size==mem3.aPool[i+size-1].u.hdr.prevSize ); + assert( size>=2 ); + if( size <= MX_SMALL ){ + memsys3UnlinkFromList(i, &mem3.aiSmall[size-2]); + }else{ + hash = size % N_HASH; + memsys3UnlinkFromList(i, &mem3.aiHash[hash]); + } +} + +/* +** Link the chunk at mem3.aPool[i] so that is on the list rooted +** at *pRoot. +*/ +static void memsys3LinkIntoList(u32 i, u32 *pRoot){ + assert( sqlite3_mutex_held(mem3.mutex) ); + mem3.aPool[i].u.list.next = *pRoot; + mem3.aPool[i].u.list.prev = 0; + if( *pRoot ){ + mem3.aPool[*pRoot].u.list.prev = i; + } + *pRoot = i; +} + +/* +** Link the chunk at index i into either the appropriate +** small chunk list, or into the large chunk hash table. +*/ +static void memsys3Link(u32 i){ + u32 size, hash; + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( i>=1 ); + assert( (mem3.aPool[i-1].u.hdr.size4x & 1)==0 ); + size = mem3.aPool[i-1].u.hdr.size4x/4; + assert( size==mem3.aPool[i+size-1].u.hdr.prevSize ); + assert( size>=2 ); + if( size <= MX_SMALL ){ + memsys3LinkIntoList(i, &mem3.aiSmall[size-2]); + }else{ + hash = size % N_HASH; + memsys3LinkIntoList(i, &mem3.aiHash[hash]); + } +} + +/* +** If the STATIC_MEM mutex is not already held, obtain it now. The mutex +** will already be held (obtained by code in malloc.c) if +** sqlite3GlobalConfig.bMemStat is true. +*/ +static void memsys3Enter(void){ + if( sqlite3GlobalConfig.bMemstat==0 && mem3.mutex==0 ){ + mem3.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + sqlite3_mutex_enter(mem3.mutex); +} +static void memsys3Leave(void){ + sqlite3_mutex_leave(mem3.mutex); +} + +/* +** Called when we are unable to satisfy an allocation of nBytes. +*/ +static void memsys3OutOfMemory(int nByte){ + if( !mem3.alarmBusy ){ + mem3.alarmBusy = 1; + assert( sqlite3_mutex_held(mem3.mutex) ); + sqlite3_mutex_leave(mem3.mutex); + sqlite3_release_memory(nByte); + sqlite3_mutex_enter(mem3.mutex); + mem3.alarmBusy = 0; + } +} + + +/* +** Chunk i is a free chunk that has been unlinked. Adjust its +** size parameters for check-out and return a pointer to the +** user portion of the chunk. +*/ +static void *memsys3Checkout(u32 i, u32 nBlock){ + u32 x; + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( i>=1 ); + assert( mem3.aPool[i-1].u.hdr.size4x/4==nBlock ); + assert( mem3.aPool[i+nBlock-1].u.hdr.prevSize==nBlock ); + x = mem3.aPool[i-1].u.hdr.size4x; + mem3.aPool[i-1].u.hdr.size4x = nBlock*4 | 1 | (x&2); + mem3.aPool[i+nBlock-1].u.hdr.prevSize = nBlock; + mem3.aPool[i+nBlock-1].u.hdr.size4x |= 2; + return &mem3.aPool[i]; +} + +/* +** Carve a piece off of the end of the mem3.iMaster free chunk. +** Return a pointer to the new allocation. Or, if the master chunk +** is not large enough, return 0. +*/ +static void *memsys3FromMaster(u32 nBlock){ + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( mem3.szMaster>=nBlock ); + if( nBlock>=mem3.szMaster-1 ){ + /* Use the entire master */ + void *p = memsys3Checkout(mem3.iMaster, mem3.szMaster); + mem3.iMaster = 0; + mem3.szMaster = 0; + mem3.mnMaster = 0; + return p; + }else{ + /* Split the master block. Return the tail. */ + u32 newi, x; + newi = mem3.iMaster + mem3.szMaster - nBlock; + assert( newi > mem3.iMaster+1 ); + mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.prevSize = nBlock; + mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.size4x |= 2; + mem3.aPool[newi-1].u.hdr.size4x = nBlock*4 + 1; + mem3.szMaster -= nBlock; + mem3.aPool[newi-1].u.hdr.prevSize = mem3.szMaster; + x = mem3.aPool[mem3.iMaster-1].u.hdr.size4x & 2; + mem3.aPool[mem3.iMaster-1].u.hdr.size4x = mem3.szMaster*4 | x; + if( mem3.szMaster < mem3.mnMaster ){ + mem3.mnMaster = mem3.szMaster; + } + return (void*)&mem3.aPool[newi]; + } +} + +/* +** *pRoot is the head of a list of free chunks of the same size +** or same size hash. In other words, *pRoot is an entry in either +** mem3.aiSmall[] or mem3.aiHash[]. +** +** This routine examines all entries on the given list and tries +** to coalesce each entries with adjacent free chunks. +** +** If it sees a chunk that is larger than mem3.iMaster, it replaces +** the current mem3.iMaster with the new larger chunk. In order for +** this mem3.iMaster replacement to work, the master chunk must be +** linked into the hash tables. That is not the normal state of +** affairs, of course. The calling routine must link the master +** chunk before invoking this routine, then must unlink the (possibly +** changed) master chunk once this routine has finished. +*/ +static void memsys3Merge(u32 *pRoot){ + u32 iNext, prev, size, i, x; + + assert( sqlite3_mutex_held(mem3.mutex) ); + for(i=*pRoot; i>0; i=iNext){ + iNext = mem3.aPool[i].u.list.next; + size = mem3.aPool[i-1].u.hdr.size4x; + assert( (size&1)==0 ); + if( (size&2)==0 ){ + memsys3UnlinkFromList(i, pRoot); + assert( i > mem3.aPool[i-1].u.hdr.prevSize ); + prev = i - mem3.aPool[i-1].u.hdr.prevSize; + if( prev==iNext ){ + iNext = mem3.aPool[prev].u.list.next; + } + memsys3Unlink(prev); + size = i + size/4 - prev; + x = mem3.aPool[prev-1].u.hdr.size4x & 2; + mem3.aPool[prev-1].u.hdr.size4x = size*4 | x; + mem3.aPool[prev+size-1].u.hdr.prevSize = size; + memsys3Link(prev); + i = prev; + }else{ + size /= 4; + } + if( size>mem3.szMaster ){ + mem3.iMaster = i; + mem3.szMaster = size; + } + } +} + +/* +** Return a block of memory of at least nBytes in size. +** Return NULL if unable. +** +** This function assumes that the necessary mutexes, if any, are +** already held by the caller. Hence "Unsafe". +*/ +static void *memsys3MallocUnsafe(int nByte){ + u32 i; + u32 nBlock; + u32 toFree; + + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( sizeof(Mem3Block)==8 ); + if( nByte<=12 ){ + nBlock = 2; + }else{ + nBlock = (nByte + 11)/8; + } + assert( nBlock>=2 ); + + /* STEP 1: + ** Look for an entry of the correct size in either the small + ** chunk table or in the large chunk hash table. This is + ** successful most of the time (about 9 times out of 10). + */ + if( nBlock <= MX_SMALL ){ + i = mem3.aiSmall[nBlock-2]; + if( i>0 ){ + memsys3UnlinkFromList(i, &mem3.aiSmall[nBlock-2]); + return memsys3Checkout(i, nBlock); + } + }else{ + int hash = nBlock % N_HASH; + for(i=mem3.aiHash[hash]; i>0; i=mem3.aPool[i].u.list.next){ + if( mem3.aPool[i-1].u.hdr.size4x/4==nBlock ){ + memsys3UnlinkFromList(i, &mem3.aiHash[hash]); + return memsys3Checkout(i, nBlock); + } + } + } + + /* STEP 2: + ** Try to satisfy the allocation by carving a piece off of the end + ** of the master chunk. This step usually works if step 1 fails. + */ + if( mem3.szMaster>=nBlock ){ + return memsys3FromMaster(nBlock); + } + + + /* STEP 3: + ** Loop through the entire memory pool. Coalesce adjacent free + ** chunks. Recompute the master chunk as the largest free chunk. + ** Then try again to satisfy the allocation by carving a piece off + ** of the end of the master chunk. This step happens very + ** rarely (we hope!) + */ + for(toFree=nBlock*16; toFree<(mem3.nPool*16); toFree *= 2){ + memsys3OutOfMemory(toFree); + if( mem3.iMaster ){ + memsys3Link(mem3.iMaster); + mem3.iMaster = 0; + mem3.szMaster = 0; + } + for(i=0; i=nBlock ){ + return memsys3FromMaster(nBlock); + } + } + } + + /* If none of the above worked, then we fail. */ + return 0; +} + +/* +** Free an outstanding memory allocation. +** +** This function assumes that the necessary mutexes, if any, are +** already held by the caller. Hence "Unsafe". +*/ +void memsys3FreeUnsafe(void *pOld){ + Mem3Block *p = (Mem3Block*)pOld; + int i; + u32 size, x; + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( p>mem3.aPool && p<&mem3.aPool[mem3.nPool] ); + i = p - mem3.aPool; + assert( (mem3.aPool[i-1].u.hdr.size4x&1)==1 ); + size = mem3.aPool[i-1].u.hdr.size4x/4; + assert( i+size<=mem3.nPool+1 ); + mem3.aPool[i-1].u.hdr.size4x &= ~1; + mem3.aPool[i+size-1].u.hdr.prevSize = size; + mem3.aPool[i+size-1].u.hdr.size4x &= ~2; + memsys3Link(i); + + /* Try to expand the master using the newly freed chunk */ + if( mem3.iMaster ){ + while( (mem3.aPool[mem3.iMaster-1].u.hdr.size4x&2)==0 ){ + size = mem3.aPool[mem3.iMaster-1].u.hdr.prevSize; + mem3.iMaster -= size; + mem3.szMaster += size; + memsys3Unlink(mem3.iMaster); + x = mem3.aPool[mem3.iMaster-1].u.hdr.size4x & 2; + mem3.aPool[mem3.iMaster-1].u.hdr.size4x = mem3.szMaster*4 | x; + mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.prevSize = mem3.szMaster; + } + x = mem3.aPool[mem3.iMaster-1].u.hdr.size4x & 2; + while( (mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.size4x&1)==0 ){ + memsys3Unlink(mem3.iMaster+mem3.szMaster); + mem3.szMaster += mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.size4x/4; + mem3.aPool[mem3.iMaster-1].u.hdr.size4x = mem3.szMaster*4 | x; + mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.prevSize = mem3.szMaster; + } + } +} + +/* +** Return the size of an outstanding allocation, in bytes. The +** size returned omits the 8-byte header overhead. This only +** works for chunks that are currently checked out. +*/ +static int memsys3Size(void *p){ + Mem3Block *pBlock; + if( p==0 ) return 0; + pBlock = (Mem3Block*)p; + assert( (pBlock[-1].u.hdr.size4x&1)!=0 ); + return (pBlock[-1].u.hdr.size4x&~3)*2 - 4; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int memsys3Roundup(int n){ + if( n<=12 ){ + return 12; + }else{ + return ((n+11)&~7) - 4; + } +} + +/* +** Allocate nBytes of memory. +*/ +static void *memsys3Malloc(int nBytes){ + sqlite3_int64 *p; + assert( nBytes>0 ); /* malloc.c filters out 0 byte requests */ + memsys3Enter(); + p = memsys3MallocUnsafe(nBytes); + memsys3Leave(); + return (void*)p; +} + +/* +** Free memory. +*/ +void memsys3Free(void *pPrior){ + assert( pPrior ); + memsys3Enter(); + memsys3FreeUnsafe(pPrior); + memsys3Leave(); +} + +/* +** Change the size of an existing memory allocation +*/ +void *memsys3Realloc(void *pPrior, int nBytes){ + int nOld; + void *p; + if( pPrior==0 ){ + return sqlite3_malloc(nBytes); + } + if( nBytes<=0 ){ + sqlite3_free(pPrior); + return 0; + } + nOld = memsys3Size(pPrior); + if( nBytes<=nOld && nBytes>=nOld-128 ){ + return pPrior; + } + memsys3Enter(); + p = memsys3MallocUnsafe(nBytes); + if( p ){ + if( nOld>1)!=(size&1) ){ + fprintf(out, "%p tail checkout bit is incorrect\n", &mem3.aPool[i]); + assert( 0 ); + break; + } + if( size&1 ){ + fprintf(out, "%p %6d bytes checked out\n", &mem3.aPool[i], (size/4)*8-8); + }else{ + fprintf(out, "%p %6d bytes free%s\n", &mem3.aPool[i], (size/4)*8-8, + i==mem3.iMaster ? " **master**" : ""); + } + } + for(i=0; i0; j=mem3.aPool[j].u.list.next){ + fprintf(out, " %p(%d)", &mem3.aPool[j], + (mem3.aPool[j-1].u.hdr.size4x/4)*8-8); + } + fprintf(out, "\n"); + } + for(i=0; i0; j=mem3.aPool[j].u.list.next){ + fprintf(out, " %p(%d)", &mem3.aPool[j], + (mem3.aPool[j-1].u.hdr.size4x/4)*8-8); + } + fprintf(out, "\n"); + } + fprintf(out, "master=%d\n", mem3.iMaster); + fprintf(out, "nowUsed=%d\n", mem3.nPool*8 - mem3.szMaster*8); + fprintf(out, "mxUsed=%d\n", mem3.nPool*8 - mem3.mnMaster*8); + sqlite3_mutex_leave(mem3.mutex); + if( out==stdout ){ + fflush(stdout); + }else{ + fclose(out); + } +#else + UNUSED_PARAMETER(zFilename); +#endif +} + +/* +** This routine is the only routine in this file with external +** linkage. +** +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. The +** arguments specify the block of memory to manage. +** +** This routine is only called by sqlite3_config(), and therefore +** is not required to be threadsafe (it is not). +*/ +const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){ + static const sqlite3_mem_methods mempoolMethods = { + memsys3Malloc, + memsys3Free, + memsys3Realloc, + memsys3Size, + memsys3Roundup, + memsys3Init, + memsys3Shutdown, + 0 + }; + return &mempoolMethods; +} + +#endif /* SQLITE_ENABLE_MEMSYS3 */ diff --git a/mem5.c b/mem5.c new file mode 100644 index 0000000..f9389e9 --- /dev/null +++ b/mem5.c @@ -0,0 +1,488 @@ +/* +** 2007 October 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement a memory +** allocation subsystem for use by SQLite. +** +** This version of the memory allocation subsystem omits all +** use of malloc(). The SQLite user supplies a block of memory +** before calling sqlite3_initialize() from which allocations +** are made and returned by the xMalloc() and xRealloc() +** implementations. Once sqlite3_initialize() has been called, +** the amount of memory available to SQLite is fixed and cannot +** be changed. +** +** This version of the memory allocation subsystem is included +** in the build only if SQLITE_ENABLE_MEMSYS5 is defined. +** +** $Id: mem5.c,v 1.19 2008/11/19 16:52:44 danielk1977 Exp $ +*/ +#include "sqliteInt.h" + +/* +** This version of the memory allocator is used only when +** SQLITE_ENABLE_MEMSYS5 is defined. +*/ +#ifdef SQLITE_ENABLE_MEMSYS5 + +/* +** A minimum allocation is an instance of the following structure. +** Larger allocations are an array of these structures where the +** size of the array is a power of 2. +*/ +typedef struct Mem5Link Mem5Link; +struct Mem5Link { + int next; /* Index of next free chunk */ + int prev; /* Index of previous free chunk */ +}; + +/* +** Maximum size of any allocation is ((1<=0 && i=0 && iLogsize<=LOGMAX ); + assert( (mem5.aCtrl[i] & CTRL_LOGSIZE)==iLogsize ); + + next = MEM5LINK(i)->next; + prev = MEM5LINK(i)->prev; + if( prev<0 ){ + mem5.aiFreelist[iLogsize] = next; + }else{ + MEM5LINK(prev)->next = next; + } + if( next>=0 ){ + MEM5LINK(next)->prev = prev; + } +} + +/* +** Link the chunk at mem5.aPool[i] so that is on the iLogsize +** free list. +*/ +static void memsys5Link(int i, int iLogsize){ + int x; + assert( sqlite3_mutex_held(mem5.mutex) ); + assert( i>=0 && i=0 && iLogsize<=LOGMAX ); + assert( (mem5.aCtrl[i] & CTRL_LOGSIZE)==iLogsize ); + + x = MEM5LINK(i)->next = mem5.aiFreelist[iLogsize]; + MEM5LINK(i)->prev = -1; + if( x>=0 ){ + assert( xprev = i; + } + mem5.aiFreelist[iLogsize] = i; +} + +/* +** If the STATIC_MEM mutex is not already held, obtain it now. The mutex +** will already be held (obtained by code in malloc.c) if +** sqlite3GlobalConfig.bMemStat is true. +*/ +static void memsys5Enter(void){ + if( sqlite3GlobalConfig.bMemstat==0 && mem5.mutex==0 ){ + mem5.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + sqlite3_mutex_enter(mem5.mutex); +} +static void memsys5Leave(void){ + sqlite3_mutex_leave(mem5.mutex); +} + +/* +** Return the size of an outstanding allocation, in bytes. The +** size returned omits the 8-byte header overhead. This only +** works for chunks that are currently checked out. +*/ +static int memsys5Size(void *p){ + int iSize = 0; + if( p ){ + int i = ((u8 *)p-mem5.zPool)/mem5.nAtom; + assert( i>=0 && i=0 && iLogsize<=LOGMAX ); + i = iFirst = mem5.aiFreelist[iLogsize]; + assert( iFirst>=0 ); + while( i>0 ){ + if( inext; + } + memsys5Unlink(iFirst, iLogsize); + return iFirst; +} + +/* +** Return a block of memory of at least nBytes in size. +** Return NULL if unable. +*/ +static void *memsys5MallocUnsafe(int nByte){ + int i; /* Index of a mem5.aPool[] slot */ + int iBin; /* Index into mem5.aiFreelist[] */ + int iFullSz; /* Size of allocation rounded up to power of 2 */ + int iLogsize; /* Log2 of iFullSz/POW2_MIN */ + + /* Keep track of the maximum allocation request. Even unfulfilled + ** requests are counted */ + if( (u32)nByte>mem5.maxRequest ){ + mem5.maxRequest = nByte; + } + + /* Round nByte up to the next valid power of two */ + for(iFullSz=mem5.nAtom, iLogsize=0; iFullSzLOGMAX ) return 0; + i = memsys5UnlinkFirst(iBin); + while( iBin>iLogsize ){ + int newSize; + + iBin--; + newSize = 1 << iBin; + mem5.aCtrl[i+newSize] = CTRL_FREE | iBin; + memsys5Link(i+newSize, iBin); + } + mem5.aCtrl[i] = iLogsize; + + /* Update allocator performance statistics. */ + mem5.nAlloc++; + mem5.totalAlloc += iFullSz; + mem5.totalExcess += iFullSz - nByte; + mem5.currentCount++; + mem5.currentOut += iFullSz; + if( mem5.maxCount=0 && iBlock0 ); + assert( mem5.currentOut>=(size*mem5.nAtom) ); + mem5.currentCount--; + mem5.currentOut -= size*mem5.nAtom; + assert( mem5.currentOut>0 || mem5.currentCount==0 ); + assert( mem5.currentCount>0 || mem5.currentOut==0 ); + + mem5.aCtrl[iBlock] = CTRL_FREE | iLogsize; + while( iLogsize>iLogsize) & 1 ){ + iBuddy = iBlock - size; + }else{ + iBuddy = iBlock + size; + } + assert( iBuddy>=0 ); + if( (iBuddy+(1<mem5.nBlock ) break; + if( mem5.aCtrl[iBuddy]!=(CTRL_FREE | iLogsize) ) break; + memsys5Unlink(iBuddy, iLogsize); + iLogsize++; + if( iBuddy0 ){ + memsys5Enter(); + p = memsys5MallocUnsafe(nBytes); + memsys5Leave(); + } + return (void*)p; +} + +/* +** Free memory. +*/ +static void memsys5Free(void *pPrior){ + if( pPrior==0 ){ +assert(0); + return; + } + memsys5Enter(); + memsys5FreeUnsafe(pPrior); + memsys5Leave(); +} + +/* +** Change the size of an existing memory allocation +*/ +static void *memsys5Realloc(void *pPrior, int nBytes){ + int nOld; + void *p; + if( pPrior==0 ){ + return memsys5Malloc(nBytes); + } + if( nBytes<=0 ){ + memsys5Free(pPrior); + return 0; + } + nOld = memsys5Size(pPrior); + if( nBytes<=nOld ){ + return pPrior; + } + memsys5Enter(); + p = memsys5MallocUnsafe(nBytes); + if( p ){ + memcpy(p, pPrior, nOld); + memsys5FreeUnsafe(pPrior); + } + memsys5Leave(); + return p; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int memsys5Roundup(int n){ + int iFullSz; + for(iFullSz=mem5.nAtom; iFullSzmem5.nAtom ){ + mem5.nAtom = mem5.nAtom << 1; + } + + mem5.nBlock = (nByte / (mem5.nAtom+sizeof(u8))); + mem5.zPool = zByte; + mem5.aCtrl = (u8 *)&mem5.zPool[mem5.nBlock*mem5.nAtom]; + + for(ii=0; ii<=LOGMAX; ii++){ + mem5.aiFreelist[ii] = -1; + } + + iOffset = 0; + for(ii=LOGMAX; ii>=0; ii--){ + int nAlloc = (1<mem5.nBlock); + } + + return SQLITE_OK; +} + +/* +** Deinitialize this module. +*/ +static void memsys5Shutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + return; +} + +/* +** Open the file indicated and write a log of all unfreed memory +** allocations into that log. +*/ +void sqlite3Memsys5Dump(const char *zFilename){ +#ifdef SQLITE_DEBUG + FILE *out; + int i, j, n; + int nMinLog; + + if( zFilename==0 || zFilename[0]==0 ){ + out = stdout; + }else{ + out = fopen(zFilename, "w"); + if( out==0 ){ + fprintf(stderr, "** Unable to output memory debug output log: %s **\n", + zFilename); + return; + } + } + memsys5Enter(); + nMinLog = memsys5Log(mem5.nAtom); + for(i=0; i<=LOGMAX && i+nMinLog<32; i++){ + for(n=0, j=mem5.aiFreelist[i]; j>=0; j = MEM5LINK(j)->next, n++){} + fprintf(out, "freelist items of size %d: %d\n", mem5.nAtom << i, n); + } + fprintf(out, "mem5.nAlloc = %llu\n", mem5.nAlloc); + fprintf(out, "mem5.totalAlloc = %llu\n", mem5.totalAlloc); + fprintf(out, "mem5.totalExcess = %llu\n", mem5.totalExcess); + fprintf(out, "mem5.currentOut = %u\n", mem5.currentOut); + fprintf(out, "mem5.currentCount = %u\n", mem5.currentCount); + fprintf(out, "mem5.maxOut = %u\n", mem5.maxOut); + fprintf(out, "mem5.maxCount = %u\n", mem5.maxCount); + fprintf(out, "mem5.maxRequest = %u\n", mem5.maxRequest); + memsys5Leave(); + if( out==stdout ){ + fflush(stdout); + }else{ + fclose(out); + } +#else + UNUSED_PARAMETER(zFilename); +#endif +} + +/* +** This routine is the only routine in this file with external +** linkage. It returns a pointer to a static sqlite3_mem_methods +** struct populated with the memsys5 methods. +*/ +const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ + static const sqlite3_mem_methods memsys5Methods = { + memsys5Malloc, + memsys5Free, + memsys5Realloc, + memsys5Size, + memsys5Roundup, + memsys5Init, + memsys5Shutdown, + 0 + }; + return &memsys5Methods; +} + +#endif /* SQLITE_ENABLE_MEMSYS5 */ diff --git a/memjournal.c b/memjournal.c new file mode 100644 index 0000000..945b5da --- /dev/null +++ b/memjournal.c @@ -0,0 +1,245 @@ +/* +** 2008 October 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code use to implement an in-memory rollback journal. +** The in-memory rollback journal is used to journal transactions for +** ":memory:" databases and when the journal_mode=MEMORY pragma is used. +** +** @(#) $Id: memjournal.c,v 1.8 2008/12/20 02:14:40 drh Exp $ +*/ +#include "sqliteInt.h" + +/* Forward references to internal structures */ +typedef struct MemJournal MemJournal; +typedef struct FilePoint FilePoint; +typedef struct FileChunk FileChunk; + +/* Space to hold the rollback journal is allocated in increments of +** this many bytes. +*/ +#define JOURNAL_CHUNKSIZE 1024 + +/* Macro to find the minimum of two numeric values. +*/ +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif + +/* +** The rollback journal is composed of a linked list of these structures. +*/ +struct FileChunk { + FileChunk *pNext; /* Next chunk in the journal */ + u8 zChunk[JOURNAL_CHUNKSIZE]; /* Content of this chunk */ +}; + +/* +** An instance of this object serves as a cursor into the rollback journal. +** The cursor can be either for reading or writing. +*/ +struct FilePoint { + sqlite3_int64 iOffset; /* Offset from the beginning of the file */ + FileChunk *pChunk; /* Specific chunk into which cursor points */ +}; + +/* +** This subclass is a subclass of sqlite3_file. Each open memory-journal +** is an instance of this class. +*/ +struct MemJournal { + sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ + FileChunk *pFirst; /* Head of in-memory chunk-list */ + FilePoint endpoint; /* Pointer to the end of the file */ + FilePoint readpoint; /* Pointer to the end of the last xRead() */ +}; + +/* +** Read data from the file. +*/ +static int memjrnlRead( + sqlite3_file *pJfd, /* The journal file from which to read */ + void *zBuf, /* Put the results here */ + int iAmt, /* Number of bytes to read */ + sqlite_int64 iOfst /* Begin reading at this offset */ +){ + MemJournal *p = (MemJournal *)pJfd; + u8 *zOut = zBuf; + int nRead = iAmt; + int iChunkOffset; + FileChunk *pChunk; + + assert( iOfst+iAmt<=p->endpoint.iOffset ); + + if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ + sqlite3_int64 iOff = 0; + for(pChunk=p->pFirst; + pChunk && (iOff+JOURNAL_CHUNKSIZE)<=iOfst; + pChunk=pChunk->pNext + ){ + iOff += JOURNAL_CHUNKSIZE; + } + }else{ + pChunk = p->readpoint.pChunk; + } + + iChunkOffset = (int)(iOfst%JOURNAL_CHUNKSIZE); + do { + int iSpace = JOURNAL_CHUNKSIZE - iChunkOffset; + int nCopy = MIN(nRead, (JOURNAL_CHUNKSIZE - iChunkOffset)); + memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy); + zOut += nCopy; + nRead -= iSpace; + iChunkOffset = 0; + } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); + p->readpoint.iOffset = iOfst+iAmt; + p->readpoint.pChunk = pChunk; + + return SQLITE_OK; +} + +/* +** Write data to the file. +*/ +static int memjrnlWrite( + sqlite3_file *pJfd, /* The journal file into which to write */ + const void *zBuf, /* Take data to be written from here */ + int iAmt, /* Number of bytes to write */ + sqlite_int64 iOfst /* Begin writing at this offset into the file */ +){ + MemJournal *p = (MemJournal *)pJfd; + int nWrite = iAmt; + u8 *zWrite = (u8 *)zBuf; + + /* An in-memory journal file should only ever be appended to. Random + ** access writes are not required by sqlite. + */ + assert(iOfst==p->endpoint.iOffset); + UNUSED_PARAMETER(iOfst); + + while( nWrite>0 ){ + FileChunk *pChunk = p->endpoint.pChunk; + int iChunkOffset = (int)(p->endpoint.iOffset%JOURNAL_CHUNKSIZE); + int iSpace = MIN(nWrite, JOURNAL_CHUNKSIZE - iChunkOffset); + + if( iChunkOffset==0 ){ + /* New chunk is required to extend the file. */ + FileChunk *pNew = sqlite3_malloc(sizeof(FileChunk)); + if( !pNew ){ + return SQLITE_IOERR_NOMEM; + } + pNew->pNext = 0; + if( pChunk ){ + assert( p->pFirst ); + pChunk->pNext = pNew; + }else{ + assert( !p->pFirst ); + p->pFirst = pNew; + } + p->endpoint.pChunk = pNew; + } + + memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace); + zWrite += iSpace; + nWrite -= iSpace; + p->endpoint.iOffset += iSpace; + } + + return SQLITE_OK; +} + +/* +** Truncate the file. +*/ +static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ + MemJournal *p = (MemJournal *)pJfd; + FileChunk *pChunk; + assert(size==0); + UNUSED_PARAMETER(size); + pChunk = p->pFirst; + while( pChunk ){ + FileChunk *pTmp = pChunk; + pChunk = pChunk->pNext; + sqlite3_free(pTmp); + } + sqlite3MemJournalOpen(pJfd); + return SQLITE_OK; +} + +/* +** Close the file. +*/ +static int memjrnlClose(sqlite3_file *pJfd){ + memjrnlTruncate(pJfd, 0); + return SQLITE_OK; +} + + +/* +** Sync the file. +*/ +static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return SQLITE_OK; +} + +/* +** Query the size of the file in bytes. +*/ +static int memjrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ + MemJournal *p = (MemJournal *)pJfd; + *pSize = (sqlite_int64) p->endpoint.iOffset; + return SQLITE_OK; +} + +/* +** Table of methods for MemJournal sqlite3_file object. +*/ +static struct sqlite3_io_methods MemJournalMethods = { + 1, /* iVersion */ + memjrnlClose, /* xClose */ + memjrnlRead, /* xRead */ + memjrnlWrite, /* xWrite */ + memjrnlTruncate, /* xTruncate */ + memjrnlSync, /* xSync */ + memjrnlFileSize, /* xFileSize */ + 0, /* xLock */ + 0, /* xUnlock */ + 0, /* xCheckReservedLock */ + 0, /* xFileControl */ + 0, /* xSectorSize */ + 0 /* xDeviceCharacteristics */ +}; + +/* +** Open a journal file. +*/ +void sqlite3MemJournalOpen(sqlite3_file *pJfd){ + MemJournal *p = (MemJournal *)pJfd; + memset(p, 0, sqlite3MemJournalSize()); + p->pMethod = &MemJournalMethods; +} + +/* +** Return true if the file-handle passed as an argument is +** an in-memory journal +*/ +int sqlite3IsMemJournal(sqlite3_file *pJfd){ + return pJfd->pMethods==&MemJournalMethods; +} + +/* +** Return the number of bytes required to store a MemJournal that uses vfs +** pVfs to create the underlying on-disk files. +*/ +int sqlite3MemJournalSize(void){ + return sizeof(MemJournal); +} diff --git a/mutex.c b/mutex.c new file mode 100644 index 0000000..03cae57 --- /dev/null +++ b/mutex.c @@ -0,0 +1,147 @@ +/* +** 2007 August 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes. +** +** This file contains code that is common across all mutex implementations. + +** +** $Id: mutex.c,v 1.29 2008/10/07 15:25:48 drh Exp $ +*/ +#include "sqliteInt.h" + +#ifndef SQLITE_MUTEX_OMIT +/* +** Initialize the mutex system. +*/ +int sqlite3MutexInit(void){ + int rc = SQLITE_OK; + if( sqlite3GlobalConfig.bCoreMutex ){ + if( !sqlite3GlobalConfig.mutex.xMutexAlloc ){ + /* If the xMutexAlloc method has not been set, then the user did not + ** install a mutex implementation via sqlite3_config() prior to + ** sqlite3_initialize() being called. This block copies pointers to + ** the default implementation into the sqlite3GlobalConfig structure. + ** + ** The danger is that although sqlite3_config() is not a threadsafe + ** API, sqlite3_initialize() is, and so multiple threads may be + ** attempting to run this function simultaneously. To guard write + ** access to the sqlite3GlobalConfig structure, the 'MASTER' static mutex + ** is obtained before modifying it. + */ + sqlite3_mutex_methods *p = sqlite3DefaultMutex(); + sqlite3_mutex *pMaster = 0; + + rc = p->xMutexInit(); + if( rc==SQLITE_OK ){ + pMaster = p->xMutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + assert(pMaster); + p->xMutexEnter(pMaster); + assert( sqlite3GlobalConfig.mutex.xMutexAlloc==0 + || sqlite3GlobalConfig.mutex.xMutexAlloc==p->xMutexAlloc + ); + if( !sqlite3GlobalConfig.mutex.xMutexAlloc ){ + sqlite3GlobalConfig.mutex = *p; + } + p->xMutexLeave(pMaster); + } + }else{ + rc = sqlite3GlobalConfig.mutex.xMutexInit(); + } + } + + return rc; +} + +/* +** Shutdown the mutex system. This call frees resources allocated by +** sqlite3MutexInit(). +*/ +int sqlite3MutexEnd(void){ + int rc = SQLITE_OK; + rc = sqlite3GlobalConfig.mutex.xMutexEnd(); + return rc; +} + +/* +** Retrieve a pointer to a static mutex or allocate a new dynamic one. +*/ +sqlite3_mutex *sqlite3_mutex_alloc(int id){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return sqlite3GlobalConfig.mutex.xMutexAlloc(id); +} + +sqlite3_mutex *sqlite3MutexAlloc(int id){ + if( !sqlite3GlobalConfig.bCoreMutex ){ + return 0; + } + return sqlite3GlobalConfig.mutex.xMutexAlloc(id); +} + +/* +** Free a dynamic mutex. +*/ +void sqlite3_mutex_free(sqlite3_mutex *p){ + if( p ){ + sqlite3GlobalConfig.mutex.xMutexFree(p); + } +} + +/* +** Obtain the mutex p. If some other thread already has the mutex, block +** until it can be obtained. +*/ +void sqlite3_mutex_enter(sqlite3_mutex *p){ + if( p ){ + sqlite3GlobalConfig.mutex.xMutexEnter(p); + } +} + +/* +** Obtain the mutex p. If successful, return SQLITE_OK. Otherwise, if another +** thread holds the mutex and it cannot be obtained, return SQLITE_BUSY. +*/ +int sqlite3_mutex_try(sqlite3_mutex *p){ + int rc = SQLITE_OK; + if( p ){ + return sqlite3GlobalConfig.mutex.xMutexTry(p); + } + return rc; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was previously +** entered by the same thread. The behavior is undefined if the mutex +** is not currently entered. If a NULL pointer is passed as an argument +** this function is a no-op. +*/ +void sqlite3_mutex_leave(sqlite3_mutex *p){ + if( p ){ + sqlite3GlobalConfig.mutex.xMutexLeave(p); + } +} + +#ifndef NDEBUG +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use inside assert() statements. +*/ +int sqlite3_mutex_held(sqlite3_mutex *p){ + return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); +} +int sqlite3_mutex_notheld(sqlite3_mutex *p){ + return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); +} +#endif + +#endif /* SQLITE_OMIT_MUTEX */ diff --git a/mutex.h b/mutex.h new file mode 100644 index 0000000..9457cb4 --- /dev/null +++ b/mutex.h @@ -0,0 +1,73 @@ +/* +** 2007 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the common header for all mutex implementations. +** The sqliteInt.h header #includes this file so that it is available +** to all source files. We break it out in an effort to keep the code +** better organized. +** +** NOTE: source files should *not* #include this header file directly. +** Source files should #include the sqliteInt.h file and let that file +** include this one indirectly. +** +** $Id: mutex.h,v 1.9 2008/10/07 15:25:48 drh Exp $ +*/ + + +/* +** Figure out what version of the code to use. The choices are +** +** SQLITE_MUTEX_OMIT No mutex logic. Not even stubs. The +** mutexes implemention cannot be overridden +** at start-time. +** +** SQLITE_MUTEX_NOOP For single-threaded applications. No +** mutual exclusion is provided. But this +** implementation can be overridden at +** start-time. +** +** SQLITE_MUTEX_PTHREADS For multi-threaded applications on Unix. +** +** SQLITE_MUTEX_W32 For multi-threaded applications on Win32. +** +** SQLITE_MUTEX_OS2 For multi-threaded applications on OS/2. +*/ +#if !SQLITE_THREADSAFE +# define SQLITE_MUTEX_OMIT +#endif +#if SQLITE_THREADSAFE && !defined(SQLITE_MUTEX_NOOP) +# if SQLITE_OS_UNIX +# define SQLITE_MUTEX_PTHREADS +# elif SQLITE_OS_WIN +# define SQLITE_MUTEX_W32 +# elif SQLITE_OS_OS2 +# define SQLITE_MUTEX_OS2 +# else +# define SQLITE_MUTEX_NOOP +# endif +#endif + +#ifdef SQLITE_MUTEX_OMIT +/* +** If this is a no-op implementation, implement everything as macros. +*/ +#define sqlite3_mutex_alloc(X) ((sqlite3_mutex*)8) +#define sqlite3_mutex_free(X) +#define sqlite3_mutex_enter(X) +#define sqlite3_mutex_try(X) SQLITE_OK +#define sqlite3_mutex_leave(X) +#define sqlite3_mutex_held(X) 1 +#define sqlite3_mutex_notheld(X) 1 +#define sqlite3MutexAlloc(X) ((sqlite3_mutex*)8) +#define sqlite3MutexInit() SQLITE_OK +#define sqlite3MutexEnd() +#endif /* defined(SQLITE_OMIT_MUTEX) */ diff --git a/mutex_noop.c b/mutex_noop.c new file mode 100644 index 0000000..7c13244 --- /dev/null +++ b/mutex_noop.c @@ -0,0 +1,186 @@ +/* +** 2008 October 07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes. +** +** This implementation in this file does not provide any mutual +** exclusion and is thus suitable for use only in applications +** that use SQLite in a single thread. The routines defined +** here are place-holders. Applications can substitute working +** mutex routines at start-time using the +** +** sqlite3_config(SQLITE_CONFIG_MUTEX,...) +** +** interface. +** +** If compiled with SQLITE_DEBUG, then additional logic is inserted +** that does error checking on mutexes to make sure they are being +** called correctly. +** +** $Id: mutex_noop.c,v 1.3 2008/12/05 17:17:08 drh Exp $ +*/ +#include "sqliteInt.h" + + +#if defined(SQLITE_MUTEX_NOOP) && !defined(SQLITE_DEBUG) +/* +** Stub routines for all mutex methods. +** +** This routines provide no mutual exclusion or error checking. +*/ +static int noopMutexHeld(sqlite3_mutex *p){ return 1; } +static int noopMutexNotheld(sqlite3_mutex *p){ return 1; } +static int noopMutexInit(void){ return SQLITE_OK; } +static int noopMutexEnd(void){ return SQLITE_OK; } +static sqlite3_mutex *noopMutexAlloc(int id){ return (sqlite3_mutex*)8; } +static void noopMutexFree(sqlite3_mutex *p){ return; } +static void noopMutexEnter(sqlite3_mutex *p){ return; } +static int noopMutexTry(sqlite3_mutex *p){ return SQLITE_OK; } +static void noopMutexLeave(sqlite3_mutex *p){ return; } + +sqlite3_mutex_methods *sqlite3DefaultMutex(void){ + static sqlite3_mutex_methods sMutex = { + noopMutexInit, + noopMutexEnd, + noopMutexAlloc, + noopMutexFree, + noopMutexEnter, + noopMutexTry, + noopMutexLeave, + + noopMutexHeld, + noopMutexNotheld + }; + + return &sMutex; +} +#endif /* defined(SQLITE_MUTEX_NOOP) && !defined(SQLITE_DEBUG) */ + +#if defined(SQLITE_MUTEX_NOOP) && defined(SQLITE_DEBUG) +/* +** In this implementation, error checking is provided for testing +** and debugging purposes. The mutexes still do not provide any +** mutual exclusion. +*/ + +/* +** The mutex object +*/ +struct sqlite3_mutex { + int id; /* The mutex type */ + int cnt; /* Number of entries without a matching leave */ +}; + +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use inside assert() statements. +*/ +static int debugMutexHeld(sqlite3_mutex *p){ + return p==0 || p->cnt>0; +} +static int debugMutexNotheld(sqlite3_mutex *p){ + return p==0 || p->cnt==0; +} + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int debugMutexInit(void){ return SQLITE_OK; } +static int debugMutexEnd(void){ return SQLITE_OK; } + +/* +** The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. If it returns NULL +** that means that a mutex could not be allocated. +*/ +static sqlite3_mutex *debugMutexAlloc(int id){ + static sqlite3_mutex aStatic[6]; + sqlite3_mutex *pNew = 0; + switch( id ){ + case SQLITE_MUTEX_FAST: + case SQLITE_MUTEX_RECURSIVE: { + pNew = sqlite3Malloc(sizeof(*pNew)); + if( pNew ){ + pNew->id = id; + pNew->cnt = 0; + } + break; + } + default: { + assert( id-2 >= 0 ); + assert( id-2 < (int)(sizeof(aStatic)/sizeof(aStatic[0])) ); + pNew = &aStatic[id-2]; + pNew->id = id; + break; + } + } + return pNew; +} + +/* +** This routine deallocates a previously allocated mutex. +*/ +static void debugMutexFree(sqlite3_mutex *p){ + assert( p->cnt==0 ); + assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); + sqlite3_free(p); +} + +/* +** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK +** upon successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can +** be entered multiple times by the same thread. In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter. If the same thread tries to enter any other kind of mutex +** more than once, the behavior is undefined. +*/ +static void debugMutexEnter(sqlite3_mutex *p){ + assert( p->id==SQLITE_MUTEX_RECURSIVE || debugMutexNotheld(p) ); + p->cnt++; +} +static int debugMutexTry(sqlite3_mutex *p){ + assert( p->id==SQLITE_MUTEX_RECURSIVE || debugMutexNotheld(p) ); + p->cnt++; + return SQLITE_OK; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered or +** is not currently allocated. SQLite will never do either. +*/ +static void debugMutexLeave(sqlite3_mutex *p){ + assert( debugMutexHeld(p) ); + p->cnt--; + assert( p->id==SQLITE_MUTEX_RECURSIVE || debugMutexNotheld(p) ); +} + +sqlite3_mutex_methods *sqlite3DefaultMutex(void){ + static sqlite3_mutex_methods sMutex = { + debugMutexInit, + debugMutexEnd, + debugMutexAlloc, + debugMutexFree, + debugMutexEnter, + debugMutexTry, + debugMutexLeave, + + debugMutexHeld, + debugMutexNotheld + }; + + return &sMutex; +} +#endif /* defined(SQLITE_MUTEX_NOOP) && defined(SQLITE_DEBUG) */ diff --git a/mutex_os2.c b/mutex_os2.c new file mode 100644 index 0000000..15ffd2c --- /dev/null +++ b/mutex_os2.c @@ -0,0 +1,273 @@ +/* +** 2007 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes for OS/2 +** +** $Id: mutex_os2.c,v 1.11 2008/11/22 19:50:54 pweilbacher Exp $ +*/ +#include "sqliteInt.h" + +/* +** The code in this file is only used if SQLITE_MUTEX_OS2 is defined. +** See the mutex.h file for details. +*/ +#ifdef SQLITE_MUTEX_OS2 + +/********************** OS/2 Mutex Implementation ********************** +** +** This implementation of mutexes is built using the OS/2 API. +*/ + +/* +** The mutex object +** Each recursive mutex is an instance of the following structure. +*/ +struct sqlite3_mutex { + HMTX mutex; /* Mutex controlling the lock */ + int id; /* Mutex type */ + int nRef; /* Number of references */ + TID owner; /* Thread holding this mutex */ +}; + +#define OS2_MUTEX_INITIALIZER 0,0,0,0 + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int os2MutexInit(void){ return SQLITE_OK; } +static int os2MutexEnd(void){ return SQLITE_OK; } + +/* +** The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. If it returns NULL +** that means that a mutex could not be allocated. +** SQLite will unwind its stack and return an error. The argument +** to sqlite3_mutex_alloc() is one of these integer constants: +** +**
    +**
  • SQLITE_MUTEX_FAST 0 +**
  • SQLITE_MUTEX_RECURSIVE 1 +**
  • SQLITE_MUTEX_STATIC_MASTER 2 +**
  • SQLITE_MUTEX_STATIC_MEM 3 +**
  • SQLITE_MUTEX_STATIC_PRNG 4 +**
+** +** The first two constants cause sqlite3_mutex_alloc() to create +** a new mutex. The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. But SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** The other allowed parameters to sqlite3_mutex_alloc() each return +** a pointer to a static preexisting mutex. Three static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. But for the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +*/ +static sqlite3_mutex *os2MutexAlloc(int iType){ + sqlite3_mutex *p = NULL; + switch( iType ){ + case SQLITE_MUTEX_FAST: + case SQLITE_MUTEX_RECURSIVE: { + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ + p->id = iType; + if( DosCreateMutexSem( 0, &p->mutex, 0, FALSE ) != NO_ERROR ){ + sqlite3_free( p ); + p = NULL; + } + } + break; + } + default: { + static volatile int isInit = 0; + static sqlite3_mutex staticMutexes[] = { + { OS2_MUTEX_INITIALIZER, }, + { OS2_MUTEX_INITIALIZER, }, + { OS2_MUTEX_INITIALIZER, }, + { OS2_MUTEX_INITIALIZER, }, + { OS2_MUTEX_INITIALIZER, }, + { OS2_MUTEX_INITIALIZER, }, + }; + if ( !isInit ){ + APIRET rc; + PTIB ptib; + PPIB ppib; + HMTX mutex; + char name[32]; + DosGetInfoBlocks( &ptib, &ppib ); + sqlite3_snprintf( sizeof(name), name, "\\SEM32\\SQLITE%04x", + ppib->pib_ulpid ); + while( !isInit ){ + mutex = 0; + rc = DosCreateMutexSem( name, &mutex, 0, FALSE); + if( rc == NO_ERROR ){ + unsigned int i; + if( !isInit ){ + for( i = 0; i < sizeof(staticMutexes)/sizeof(staticMutexes[0]); i++ ){ + DosCreateMutexSem( 0, &staticMutexes[i].mutex, 0, FALSE ); + } + isInit = 1; + } + DosCloseMutexSem( mutex ); + }else if( rc == ERROR_DUPLICATE_NAME ){ + DosSleep( 1 ); + }else{ + return p; + } + } + } + assert( iType-2 >= 0 ); + assert( iType-2 < sizeof(staticMutexes)/sizeof(staticMutexes[0]) ); + p = &staticMutexes[iType-2]; + p->id = iType; + break; + } + } + return p; +} + + +/* +** This routine deallocates a previously allocated mutex. +** SQLite is careful to deallocate every mutex that it allocates. +*/ +static void os2MutexFree(sqlite3_mutex *p){ + if( p==0 ) return; + assert( p->nRef==0 ); + assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); + DosCloseMutexSem( p->mutex ); + sqlite3_free( p ); +} + +/* +** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK +** upon successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can +** be entered multiple times by the same thread. In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter. If the same thread tries to enter any other kind of mutex +** more than once, the behavior is undefined. +*/ +static void os2MutexEnter(sqlite3_mutex *p){ + TID tid; + PID holder1; + ULONG holder2; + if( p==0 ) return; + assert( p->id==SQLITE_MUTEX_RECURSIVE || os2MutexNotheld(p) ); + DosRequestMutexSem(p->mutex, SEM_INDEFINITE_WAIT); + DosQueryMutexSem(p->mutex, &holder1, &tid, &holder2); + p->owner = tid; + p->nRef++; +} +static int os2MutexTry(sqlite3_mutex *p){ + int rc; + TID tid; + PID holder1; + ULONG holder2; + if( p==0 ) return SQLITE_OK; + assert( p->id==SQLITE_MUTEX_RECURSIVE || os2MutexNotheld(p) ); + if( DosRequestMutexSem(p->mutex, SEM_IMMEDIATE_RETURN) == NO_ERROR) { + DosQueryMutexSem(p->mutex, &holder1, &tid, &holder2); + p->owner = tid; + p->nRef++; + rc = SQLITE_OK; + } else { + rc = SQLITE_BUSY; + } + + return rc; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered or +** is not currently allocated. SQLite will never do either. +*/ +static void os2MutexLeave(sqlite3_mutex *p){ + TID tid; + PID holder1; + ULONG holder2; + if( p==0 ) return; + assert( p->nRef>0 ); + DosQueryMutexSem(p->mutex, &holder1, &tid, &holder2); + assert( p->owner==tid ); + p->nRef--; + assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); + DosReleaseMutexSem(p->mutex); +} + +#ifdef SQLITE_DEBUG +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use inside assert() statements. +*/ +static int os2MutexHeld(sqlite3_mutex *p){ + TID tid; + PID pid; + ULONG ulCount; + PTIB ptib; + if( p!=0 ) { + DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount); + } else { + DosGetInfoBlocks(&ptib, NULL); + tid = ptib->tib_ptib2->tib2_ultid; + } + return p==0 || (p->nRef!=0 && p->owner==tid); +} +static int os2MutexNotheld(sqlite3_mutex *p){ + TID tid; + PID pid; + ULONG ulCount; + PTIB ptib; + if( p!= 0 ) { + DosQueryMutexSem(p->mutex, &pid, &tid, &ulCount); + } else { + DosGetInfoBlocks(&ptib, NULL); + tid = ptib->tib_ptib2->tib2_ultid; + } + return p==0 || p->nRef==0 || p->owner!=tid; +} +#endif + +sqlite3_mutex_methods *sqlite3DefaultMutex(void){ + static sqlite3_mutex_methods sMutex = { + os2MutexInit, + os2MutexEnd, + os2MutexAlloc, + os2MutexFree, + os2MutexEnter, + os2MutexTry, + os2MutexLeave, +#ifdef SQLITE_DEBUG + os2MutexHeld, + os2MutexNotheld +#endif + }; + + return &sMutex; +} +#endif /* SQLITE_MUTEX_OS2 */ diff --git a/mutex_unix.c b/mutex_unix.c new file mode 100644 index 0000000..36b0682 --- /dev/null +++ b/mutex_unix.c @@ -0,0 +1,328 @@ +/* +** 2007 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes for pthreads +** +** $Id: mutex_unix.c,v 1.16 2008/12/08 18:19:18 drh Exp $ +*/ +#include "sqliteInt.h" + +/* +** The code in this file is only used if we are compiling threadsafe +** under unix with pthreads. +** +** Note that this implementation requires a version of pthreads that +** supports recursive mutexes. +*/ +#ifdef SQLITE_MUTEX_PTHREADS + +#include + + +/* +** Each recursive mutex is an instance of the following structure. +*/ +struct sqlite3_mutex { + pthread_mutex_t mutex; /* Mutex controlling the lock */ + int id; /* Mutex type */ + int nRef; /* Number of entrances */ + pthread_t owner; /* Thread that is within this mutex */ +#ifdef SQLITE_DEBUG + int trace; /* True to trace changes */ +#endif +}; +#ifdef SQLITE_DEBUG +#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0, 0, (pthread_t)0, 0 } +#else +#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0, 0, (pthread_t)0 } +#endif + +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use only inside assert() statements. On some platforms, +** there might be race conditions that can cause these routines to +** deliver incorrect results. In particular, if pthread_equal() is +** not an atomic operation, then these routines might delivery +** incorrect results. On most platforms, pthread_equal() is a +** comparison of two integers and is therefore atomic. But we are +** told that HPUX is not such a platform. If so, then these routines +** will not always work correctly on HPUX. +** +** On those platforms where pthread_equal() is not atomic, SQLite +** should be compiled without -DSQLITE_DEBUG and with -DNDEBUG to +** make sure no assert() statements are evaluated and hence these +** routines are never called. +*/ +#if !defined(NDEBUG) || defined(SQLITE_DEBUG) +static int pthreadMutexHeld(sqlite3_mutex *p){ + return (p->nRef!=0 && pthread_equal(p->owner, pthread_self())); +} +static int pthreadMutexNotheld(sqlite3_mutex *p){ + return p->nRef==0 || pthread_equal(p->owner, pthread_self())==0; +} +#endif + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int pthreadMutexInit(void){ return SQLITE_OK; } +static int pthreadMutexEnd(void){ return SQLITE_OK; } + +/* +** The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. If it returns NULL +** that means that a mutex could not be allocated. SQLite +** will unwind its stack and return an error. The argument +** to sqlite3_mutex_alloc() is one of these integer constants: +** +**
    +**
  • SQLITE_MUTEX_FAST +**
  • SQLITE_MUTEX_RECURSIVE +**
  • SQLITE_MUTEX_STATIC_MASTER +**
  • SQLITE_MUTEX_STATIC_MEM +**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_PRNG +**
  • SQLITE_MUTEX_STATIC_LRU +**
+** +** The first two constants cause sqlite3_mutex_alloc() to create +** a new mutex. The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. But SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** The other allowed parameters to sqlite3_mutex_alloc() each return +** a pointer to a static preexisting mutex. Three static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. But for the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +*/ +static sqlite3_mutex *pthreadMutexAlloc(int iType){ + static sqlite3_mutex staticMutexes[] = { + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER + }; + sqlite3_mutex *p; + switch( iType ){ + case SQLITE_MUTEX_RECURSIVE: { + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + /* If recursive mutexes are not available, we will have to + ** build our own. See below. */ + pthread_mutex_init(&p->mutex, 0); +#else + /* Use a recursive mutex if it is available */ + pthread_mutexattr_t recursiveAttr; + pthread_mutexattr_init(&recursiveAttr); + pthread_mutexattr_settype(&recursiveAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&p->mutex, &recursiveAttr); + pthread_mutexattr_destroy(&recursiveAttr); +#endif + p->id = iType; + } + break; + } + case SQLITE_MUTEX_FAST: { + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ + p->id = iType; + pthread_mutex_init(&p->mutex, 0); + } + break; + } + default: { + assert( iType-2 >= 0 ); + assert( iType-2 < ArraySize(staticMutexes) ); + p = &staticMutexes[iType-2]; + p->id = iType; + break; + } + } + return p; +} + + +/* +** This routine deallocates a previously +** allocated mutex. SQLite is careful to deallocate every +** mutex that it allocates. +*/ +static void pthreadMutexFree(sqlite3_mutex *p){ + assert( p->nRef==0 ); + assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); + pthread_mutex_destroy(&p->mutex); + sqlite3_free(p); +} + +/* +** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK +** upon successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can +** be entered multiple times by the same thread. In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter. If the same thread tries to enter any other kind of mutex +** more than once, the behavior is undefined. +*/ +static void pthreadMutexEnter(sqlite3_mutex *p){ + assert( p->id==SQLITE_MUTEX_RECURSIVE || pthreadMutexNotheld(p) ); + +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + /* If recursive mutexes are not available, then we have to grow + ** our own. This implementation assumes that pthread_equal() + ** is atomic - that it cannot be deceived into thinking self + ** and p->owner are equal if p->owner changes between two values + ** that are not equal to self while the comparison is taking place. + ** This implementation also assumes a coherent cache - that + ** separate processes cannot read different values from the same + ** address at the same time. If either of these two conditions + ** are not met, then the mutexes will fail and problems will result. + */ + { + pthread_t self = pthread_self(); + if( p->nRef>0 && pthread_equal(p->owner, self) ){ + p->nRef++; + }else{ + pthread_mutex_lock(&p->mutex); + assert( p->nRef==0 ); + p->owner = self; + p->nRef = 1; + } + } +#else + /* Use the built-in recursive mutexes if they are available. + */ + pthread_mutex_lock(&p->mutex); + p->owner = pthread_self(); + p->nRef++; +#endif + +#ifdef SQLITE_DEBUG + if( p->trace ){ + printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif +} +static int pthreadMutexTry(sqlite3_mutex *p){ + int rc; + assert( p->id==SQLITE_MUTEX_RECURSIVE || pthreadMutexNotheld(p) ); + +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + /* If recursive mutexes are not available, then we have to grow + ** our own. This implementation assumes that pthread_equal() + ** is atomic - that it cannot be deceived into thinking self + ** and p->owner are equal if p->owner changes between two values + ** that are not equal to self while the comparison is taking place. + ** This implementation also assumes a coherent cache - that + ** separate processes cannot read different values from the same + ** address at the same time. If either of these two conditions + ** are not met, then the mutexes will fail and problems will result. + */ + { + pthread_t self = pthread_self(); + if( p->nRef>0 && pthread_equal(p->owner, self) ){ + p->nRef++; + rc = SQLITE_OK; + }else if( pthread_mutex_trylock(&p->mutex)==0 ){ + assert( p->nRef==0 ); + p->owner = self; + p->nRef = 1; + rc = SQLITE_OK; + }else{ + rc = SQLITE_BUSY; + } + } +#else + /* Use the built-in recursive mutexes if they are available. + */ + if( pthread_mutex_trylock(&p->mutex)==0 ){ + p->owner = pthread_self(); + p->nRef++; + rc = SQLITE_OK; + }else{ + rc = SQLITE_BUSY; + } +#endif + +#ifdef SQLITE_DEBUG + if( rc==SQLITE_OK && p->trace ){ + printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif + return rc; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered or +** is not currently allocated. SQLite will never do either. +*/ +static void pthreadMutexLeave(sqlite3_mutex *p){ + assert( pthreadMutexHeld(p) ); + p->nRef--; + assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); + +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + if( p->nRef==0 ){ + pthread_mutex_unlock(&p->mutex); + } +#else + pthread_mutex_unlock(&p->mutex); +#endif + +#ifdef SQLITE_DEBUG + if( p->trace ){ + printf("leave mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif +} + +sqlite3_mutex_methods *sqlite3DefaultMutex(void){ + static sqlite3_mutex_methods sMutex = { + pthreadMutexInit, + pthreadMutexEnd, + pthreadMutexAlloc, + pthreadMutexFree, + pthreadMutexEnter, + pthreadMutexTry, + pthreadMutexLeave, +#ifdef SQLITE_DEBUG + pthreadMutexHeld, + pthreadMutexNotheld +#else + 0, + 0 +#endif + }; + + return &sMutex; +} + +#endif /* SQLITE_MUTEX_PTHREAD */ diff --git a/mutex_w32.c b/mutex_w32.c new file mode 100644 index 0000000..320c448 --- /dev/null +++ b/mutex_w32.c @@ -0,0 +1,254 @@ +/* +** 2007 August 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes for win32 +** +** $Id: mutex_w32.c,v 1.13 2008/12/08 18:19:18 drh Exp $ +*/ +#include "sqliteInt.h" + +/* +** The code in this file is only used if we are compiling multithreaded +** on a win32 system. +*/ +#ifdef SQLITE_MUTEX_W32 + +/* +** Each recursive mutex is an instance of the following structure. +*/ +struct sqlite3_mutex { + CRITICAL_SECTION mutex; /* Mutex controlling the lock */ + int id; /* Mutex type */ + int nRef; /* Number of enterances */ + DWORD owner; /* Thread holding this mutex */ +}; + +/* +** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, +** or WinCE. Return false (zero) for Win95, Win98, or WinME. +** +** Here is an interesting observation: Win95, Win98, and WinME lack +** the LockFileEx() API. But we can still statically link against that +** API as long as we don't call it win running Win95/98/ME. A call to +** this routine is used to determine if the host is Win95/98/ME or +** WinNT/2K/XP so that we will know whether or not we can safely call +** the LockFileEx() API. +** +** mutexIsNT() is only used for the TryEnterCriticalSection() API call, +** which is only available if your application was compiled with +** _WIN32_WINNT defined to a value >= 0x0400. Currently, the only +** call to TryEnterCriticalSection() is #ifdef'ed out, so #ifdef +** this out as well. +*/ +#if 0 +#if SQLITE_OS_WINCE +# define mutexIsNT() (1) +#else + static int mutexIsNT(void){ + static int osType = 0; + if( osType==0 ){ + OSVERSIONINFO sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + GetVersionEx(&sInfo); + osType = sInfo.dwPlatformId==VER_PLATFORM_WIN32_NT ? 2 : 1; + } + return osType==2; + } +#endif /* SQLITE_OS_WINCE */ +#endif + +#ifdef SQLITE_DEBUG +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use only inside assert() statements. +*/ +static int winMutexHeld(sqlite3_mutex *p){ + return p->nRef!=0 && p->owner==GetCurrentThreadId(); +} +static int winMutexNotheld(sqlite3_mutex *p){ + return p->nRef==0 || p->owner!=GetCurrentThreadId(); +} +#endif + + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int winMutexInit(void){ return SQLITE_OK; } +static int winMutexEnd(void){ return SQLITE_OK; } + +/* +** The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. If it returns NULL +** that means that a mutex could not be allocated. SQLite +** will unwind its stack and return an error. The argument +** to sqlite3_mutex_alloc() is one of these integer constants: +** +**
    +**
  • SQLITE_MUTEX_FAST 0 +**
  • SQLITE_MUTEX_RECURSIVE 1 +**
  • SQLITE_MUTEX_STATIC_MASTER 2 +**
  • SQLITE_MUTEX_STATIC_MEM 3 +**
  • SQLITE_MUTEX_STATIC_PRNG 4 +**
+** +** The first two constants cause sqlite3_mutex_alloc() to create +** a new mutex. The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. But SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** The other allowed parameters to sqlite3_mutex_alloc() each return +** a pointer to a static preexisting mutex. Three static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. But for the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +*/ +static sqlite3_mutex *winMutexAlloc(int iType){ + sqlite3_mutex *p; + + switch( iType ){ + case SQLITE_MUTEX_FAST: + case SQLITE_MUTEX_RECURSIVE: { + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ + p->id = iType; + InitializeCriticalSection(&p->mutex); + } + break; + } + default: { + static sqlite3_mutex staticMutexes[6]; + static int isInit = 0; + while( !isInit ){ + static long lock = 0; + if( InterlockedIncrement(&lock)==1 ){ + int i; + for(i=0; i= 0 ); + assert( iType-2 < sizeof(staticMutexes)/sizeof(staticMutexes[0]) ); + p = &staticMutexes[iType-2]; + p->id = iType; + break; + } + } + return p; +} + + +/* +** This routine deallocates a previously +** allocated mutex. SQLite is careful to deallocate every +** mutex that it allocates. +*/ +static void winMutexFree(sqlite3_mutex *p){ + assert( p ); + assert( p->nRef==0 ); + assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); + DeleteCriticalSection(&p->mutex); + sqlite3_free(p); +} + +/* +** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK +** upon successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can +** be entered multiple times by the same thread. In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter. If the same thread tries to enter any other kind of mutex +** more than once, the behavior is undefined. +*/ +static void winMutexEnter(sqlite3_mutex *p){ + assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld(p) ); + EnterCriticalSection(&p->mutex); + p->owner = GetCurrentThreadId(); + p->nRef++; +} +static int winMutexTry(sqlite3_mutex *p){ + int rc = SQLITE_BUSY; + assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld(p) ); + /* + ** The sqlite3_mutex_try() routine is very rarely used, and when it + ** is used it is merely an optimization. So it is OK for it to always + ** fail. + ** + ** The TryEnterCriticalSection() interface is only available on WinNT. + ** And some windows compilers complain if you try to use it without + ** first doing some #defines that prevent SQLite from building on Win98. + ** For that reason, we will omit this optimization for now. See + ** ticket #2685. + */ +#if 0 + if( mutexIsNT() && TryEnterCriticalSection(&p->mutex) ){ + p->owner = GetCurrentThreadId(); + p->nRef++; + rc = SQLITE_OK; + } +#endif + return rc; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered or +** is not currently allocated. SQLite will never do either. +*/ +static void winMutexLeave(sqlite3_mutex *p){ + assert( p->nRef>0 ); + assert( p->owner==GetCurrentThreadId() ); + p->nRef--; + assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); + LeaveCriticalSection(&p->mutex); +} + +sqlite3_mutex_methods *sqlite3DefaultMutex(void){ + static sqlite3_mutex_methods sMutex = { + winMutexInit, + winMutexEnd, + winMutexAlloc, + winMutexFree, + winMutexEnter, + winMutexTry, + winMutexLeave, +#ifdef SQLITE_DEBUG + winMutexHeld, + winMutexNotheld +#else + 0, + 0 +#endif + }; + + return &sMutex; +} +#endif /* SQLITE_MUTEX_W32 */ diff --git a/opcodes.c b/opcodes.c index 131b8fe..e949275 100644 --- a/opcodes.c +++ b/opcodes.c @@ -1,148 +1,9 @@ /* Automatically generated. Do not edit */ /* See the mkopcodec.awk script for details. */ #if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) -const char *const sqlite3OpcodeNames[] = { "?", - /* 1 */ "ReadCookie", - /* 2 */ "AutoCommit", - /* 3 */ "Found", - /* 4 */ "NullRow", - /* 5 */ "MoveLe", - /* 6 */ "Variable", - /* 7 */ "Pull", - /* 8 */ "RealAffinity", - /* 9 */ "Sort", - /* 10 */ "IfNot", - /* 11 */ "Gosub", - /* 12 */ "NotFound", - /* 13 */ "MoveLt", - /* 14 */ "Rowid", - /* 15 */ "CreateIndex", - /* 16 */ "Not", - /* 17 */ "Push", - /* 18 */ "Explain", - /* 19 */ "Statement", - /* 20 */ "Callback", - /* 21 */ "MemLoad", - /* 22 */ "DropIndex", - /* 23 */ "Null", - /* 24 */ "Int64", - /* 25 */ "LoadAnalysis", - /* 26 */ "IdxInsert", - /* 27 */ "VUpdate", - /* 28 */ "Next", - /* 29 */ "SetNumColumns", - /* 30 */ "MemInt", - /* 31 */ "Dup", - /* 32 */ "Rewind", - /* 33 */ "Last", - /* 34 */ "MustBeInt", - /* 35 */ "MoveGe", - /* 36 */ "IncrVacuum", - /* 37 */ "String", - /* 38 */ "VFilter", - /* 39 */ "ForceInt", - /* 40 */ "Close", - /* 41 */ "AggFinal", - /* 42 */ "AbsValue", - /* 43 */ "RowData", - /* 44 */ "IdxRowid", - /* 45 */ "MoveGt", - /* 46 */ "OpenPseudo", - /* 47 */ "Halt", - /* 48 */ "MemMove", - /* 49 */ "NewRowid", - /* 50 */ "IdxLT", - /* 51 */ "Distinct", - /* 52 */ "MemMax", - /* 53 */ "Function", - /* 54 */ "IntegrityCk", - /* 55 */ "FifoWrite", - /* 56 */ "NotExists", - /* 57 */ "VDestroy", - /* 58 */ "MemStore", - /* 59 */ "IdxDelete", - /* 60 */ "Or", - /* 61 */ "And", - /* 62 */ "Vacuum", - /* 63 */ "If", - /* 64 */ "Destroy", - /* 65 */ "IsNull", - /* 66 */ "NotNull", - /* 67 */ "Ne", - /* 68 */ "Eq", - /* 69 */ "Gt", - /* 70 */ "Le", - /* 71 */ "Lt", - /* 72 */ "Ge", - /* 73 */ "AggStep", - /* 74 */ "BitAnd", - /* 75 */ "BitOr", - /* 76 */ "ShiftLeft", - /* 77 */ "ShiftRight", - /* 78 */ "Add", - /* 79 */ "Subtract", - /* 80 */ "Multiply", - /* 81 */ "Divide", - /* 82 */ "Remainder", - /* 83 */ "Concat", - /* 84 */ "Clear", - /* 85 */ "Negative", - /* 86 */ "Insert", - /* 87 */ "BitNot", - /* 88 */ "String8", - /* 89 */ "VBegin", - /* 90 */ "IdxGE", - /* 91 */ "OpenEphemeral", - /* 92 */ "IfMemZero", - /* 93 */ "VRowid", - /* 94 */ "MakeRecord", - /* 95 */ "SetCookie", - /* 96 */ "Prev", - /* 97 */ "ContextPush", - /* 98 */ "DropTrigger", - /* 99 */ "IdxGT", - /* 100 */ "MemNull", - /* 101 */ "IfMemNeg", - /* 102 */ "VColumn", - /* 103 */ "Return", - /* 104 */ "OpenWrite", - /* 105 */ "Integer", - /* 106 */ "Transaction", - /* 107 */ "CollSeq", - /* 108 */ "VRename", - /* 109 */ "Sequence", - /* 110 */ "ContextPop", - /* 111 */ "VCreate", - /* 112 */ "CreateTable", - /* 113 */ "AddImm", - /* 114 */ "DropTable", - /* 115 */ "IsUnique", - /* 116 */ "VOpen", - /* 117 */ "Noop", - /* 118 */ "RowKey", - /* 119 */ "Expire", - /* 120 */ "FifoRead", - /* 121 */ "Delete", - /* 122 */ "IfMemPos", - /* 123 */ "MemIncr", - /* 124 */ "Blob", - /* 125 */ "Real", - /* 126 */ "HexBlob", - /* 127 */ "MakeIdxRec", - /* 128 */ "Goto", - /* 129 */ "ParseSchema", - /* 130 */ "VNext", - /* 131 */ "Pop", - /* 132 */ "TableLock", - /* 133 */ "VerifyCookie", - /* 134 */ "Column", - /* 135 */ "OpenRead", - /* 136 */ "ResetCount", - /* 137 */ "NotUsed_137", - /* 138 */ "ToText", - /* 139 */ "ToBlob", - /* 140 */ "ToNumeric", - /* 141 */ "ToInt", - /* 142 */ "ToReal", -}; +const char *sqlite3OpcodeName(int i){ + static const char *const azName[] = { "?", + }; + return azName[i]; +} #endif diff --git a/opcodes.h b/opcodes.h index eee36a3..208a0ac 100644 --- a/opcodes.h +++ b/opcodes.h @@ -1,160 +1,181 @@ /* Automatically generated. Do not edit */ /* See the mkopcodeh.awk script for details */ -#define OP_ReadCookie 1 -#define OP_AutoCommit 2 -#define OP_Found 3 -#define OP_NullRow 4 -#define OP_Lt 71 /* same as TK_LT */ -#define OP_MoveLe 5 -#define OP_Variable 6 -#define OP_Pull 7 -#define OP_RealAffinity 8 -#define OP_Sort 9 -#define OP_IfNot 10 -#define OP_Gosub 11 -#define OP_Add 78 /* same as TK_PLUS */ -#define OP_NotFound 12 -#define OP_IsNull 65 /* same as TK_ISNULL */ -#define OP_MoveLt 13 -#define OP_Rowid 14 -#define OP_CreateIndex 15 -#define OP_Push 17 -#define OP_Explain 18 -#define OP_Statement 19 -#define OP_Callback 20 -#define OP_MemLoad 21 -#define OP_DropIndex 22 -#define OP_Null 23 -#define OP_ToInt 141 /* same as TK_TO_INT */ -#define OP_Int64 24 -#define OP_LoadAnalysis 25 -#define OP_IdxInsert 26 -#define OP_VUpdate 27 -#define OP_Next 28 -#define OP_SetNumColumns 29 -#define OP_ToNumeric 140 /* same as TK_TO_NUMERIC*/ -#define OP_Ge 72 /* same as TK_GE */ -#define OP_BitNot 87 /* same as TK_BITNOT */ -#define OP_MemInt 30 -#define OP_Dup 31 -#define OP_Rewind 32 -#define OP_Multiply 80 /* same as TK_STAR */ -#define OP_ToReal 142 /* same as TK_TO_REAL */ -#define OP_Gt 69 /* same as TK_GT */ -#define OP_Last 33 -#define OP_MustBeInt 34 -#define OP_Ne 67 /* same as TK_NE */ -#define OP_MoveGe 35 -#define OP_IncrVacuum 36 -#define OP_String 37 -#define OP_VFilter 38 -#define OP_ForceInt 39 -#define OP_Close 40 -#define OP_AggFinal 41 -#define OP_AbsValue 42 -#define OP_RowData 43 -#define OP_IdxRowid 44 -#define OP_BitOr 75 /* same as TK_BITOR */ -#define OP_NotNull 66 /* same as TK_NOTNULL */ -#define OP_MoveGt 45 -#define OP_Not 16 /* same as TK_NOT */ -#define OP_OpenPseudo 46 -#define OP_Halt 47 -#define OP_MemMove 48 +#define OP_VRowid 1 +#define OP_VFilter 2 +#define OP_IfNeg 3 +#define OP_ContextPop 4 +#define OP_IntegrityCk 5 +#define OP_DropTrigger 6 +#define OP_DropIndex 7 +#define OP_IdxInsert 8 +#define OP_Delete 9 +#define OP_SeekLt 10 +#define OP_OpenEphemeral 11 +#define OP_VerifyCookie 12 +#define OP_Blob 13 +#define OP_RowKey 14 +#define OP_IsUnique 15 +#define OP_SetNumColumns 16 +#define OP_Eq 71 /* same as TK_EQ */ +#define OP_VUpdate 17 +#define OP_Expire 18 +#define OP_NullRow 20 +#define OP_OpenPseudo 21 +#define OP_OpenWrite 22 +#define OP_OpenRead 23 +#define OP_Transaction 24 +#define OP_AutoCommit 25 +#define OP_Copy 26 +#define OP_Halt 27 +#define OP_VRename 28 +#define OP_Vacuum 29 +#define OP_RowData 30 +#define OP_NotExists 31 +#define OP_SetCookie 32 +#define OP_Move 33 +#define OP_Variable 34 +#define OP_Pagecount 35 +#define OP_VNext 36 +#define OP_VDestroy 37 +#define OP_TableLock 38 +#define OP_RowSetAdd 39 +#define OP_LoadAnalysis 40 +#define OP_IdxDelete 41 +#define OP_Sort 42 +#define OP_ResetCount 43 +#define OP_NotNull 69 /* same as TK_NOTNULL */ +#define OP_Ge 75 /* same as TK_GE */ +#define OP_Remainder 85 /* same as TK_REM */ +#define OP_Divide 84 /* same as TK_SLASH */ +#define OP_Integer 44 +#define OP_Explain 45 +#define OP_IncrVacuum 46 +#define OP_AggStep 47 +#define OP_CreateIndex 48 #define OP_NewRowid 49 -#define OP_Real 125 /* same as TK_FLOAT */ -#define OP_IdxLT 50 -#define OP_Distinct 51 -#define OP_MemMax 52 -#define OP_Function 53 -#define OP_IntegrityCk 54 -#define OP_Remainder 82 /* same as TK_REM */ -#define OP_HexBlob 126 /* same as TK_BLOB */ -#define OP_ShiftLeft 76 /* same as TK_LSHIFT */ -#define OP_FifoWrite 55 -#define OP_BitAnd 74 /* same as TK_BITAND */ -#define OP_Or 60 /* same as TK_OR */ -#define OP_NotExists 56 -#define OP_VDestroy 57 -#define OP_MemStore 58 -#define OP_IdxDelete 59 -#define OP_Vacuum 62 -#define OP_If 63 -#define OP_Destroy 64 -#define OP_AggStep 73 -#define OP_Clear 84 -#define OP_Insert 86 -#define OP_VBegin 89 -#define OP_IdxGE 90 -#define OP_OpenEphemeral 91 -#define OP_Divide 81 /* same as TK_SLASH */ -#define OP_String8 88 /* same as TK_STRING */ -#define OP_IfMemZero 92 -#define OP_Concat 83 /* same as TK_CONCAT */ -#define OP_VRowid 93 +#define OP_And 64 /* same as TK_AND */ +#define OP_ShiftLeft 79 /* same as TK_LSHIFT */ +#define OP_Real 129 /* same as TK_FLOAT */ +#define OP_Return 50 +#define OP_Trace 51 +#define OP_IfPos 52 +#define OP_IdxLT 53 +#define OP_Rewind 54 +#define OP_SeekGe 55 +#define OP_Affinity 56 +#define OP_Gt 72 /* same as TK_GT */ +#define OP_AddImm 57 +#define OP_Subtract 82 /* same as TK_MINUS */ +#define OP_Null 58 +#define OP_VColumn 59 +#define OP_Clear 60 +#define OP_IsNull 68 /* same as TK_ISNULL */ +#define OP_If 61 +#define OP_Permutation 62 +#define OP_ToBlob 142 /* same as TK_TO_BLOB */ +#define OP_RealAffinity 65 +#define OP_Yield 66 +#define OP_AggFinal 67 +#define OP_IfZero 76 +#define OP_Last 87 +#define OP_Rowid 88 +#define OP_Sequence 89 +#define OP_NotFound 92 +#define OP_SeekGt 93 #define OP_MakeRecord 94 -#define OP_SetCookie 95 -#define OP_Prev 96 -#define OP_ContextPush 97 -#define OP_DropTrigger 98 -#define OP_IdxGT 99 -#define OP_MemNull 100 -#define OP_IfMemNeg 101 -#define OP_And 61 /* same as TK_AND */ -#define OP_VColumn 102 -#define OP_Return 103 -#define OP_OpenWrite 104 -#define OP_Integer 105 -#define OP_Transaction 106 -#define OP_CollSeq 107 -#define OP_VRename 108 -#define OP_ToBlob 139 /* same as TK_TO_BLOB */ -#define OP_Sequence 109 -#define OP_ContextPop 110 -#define OP_ShiftRight 77 /* same as TK_RSHIFT */ -#define OP_VCreate 111 -#define OP_CreateTable 112 -#define OP_AddImm 113 -#define OP_ToText 138 /* same as TK_TO_TEXT */ -#define OP_DropTable 114 -#define OP_IsUnique 115 -#define OP_VOpen 116 -#define OP_Noop 117 -#define OP_RowKey 118 -#define OP_Expire 119 -#define OP_FifoRead 120 -#define OP_Delete 121 -#define OP_IfMemPos 122 -#define OP_Subtract 79 /* same as TK_MINUS */ -#define OP_MemIncr 123 -#define OP_Blob 124 -#define OP_MakeIdxRec 127 -#define OP_Goto 128 -#define OP_Negative 85 /* same as TK_UMINUS */ -#define OP_ParseSchema 129 -#define OP_Eq 68 /* same as TK_EQ */ -#define OP_VNext 130 -#define OP_Pop 131 -#define OP_Le 70 /* same as TK_LE */ -#define OP_TableLock 132 -#define OP_VerifyCookie 133 -#define OP_Column 134 -#define OP_OpenRead 135 -#define OP_ResetCount 136 +#define OP_ToText 141 /* same as TK_TO_TEXT */ +#define OP_BitAnd 77 /* same as TK_BITAND */ +#define OP_Add 81 /* same as TK_PLUS */ +#define OP_ResultRow 95 +#define OP_String 96 +#define OP_Goto 97 +#define OP_Noop 98 +#define OP_VCreate 99 +#define OP_RowSetRead 100 +#define OP_DropTable 101 +#define OP_IdxRowid 102 +#define OP_Insert 103 +#define OP_Column 104 +#define OP_Not 19 /* same as TK_NOT */ +#define OP_Compare 105 +#define OP_Le 73 /* same as TK_LE */ +#define OP_BitOr 78 /* same as TK_BITOR */ +#define OP_Multiply 83 /* same as TK_STAR */ +#define OP_String8 91 /* same as TK_STRING */ +#define OP_VOpen 106 +#define OP_CreateTable 107 +#define OP_Found 108 +#define OP_Seek 109 +#define OP_Close 110 +#define OP_Savepoint 111 +#define OP_Statement 112 +#define OP_IfNot 113 +#define OP_ToInt 144 /* same as TK_TO_INT */ +#define OP_VBegin 114 +#define OP_MemMax 115 +#define OP_Next 116 +#define OP_Prev 117 +#define OP_SeekLe 118 +#define OP_Lt 74 /* same as TK_LT */ +#define OP_Ne 70 /* same as TK_NE */ +#define OP_MustBeInt 119 +#define OP_ShiftRight 80 /* same as TK_RSHIFT */ +#define OP_CollSeq 120 +#define OP_Gosub 121 +#define OP_ContextPush 122 +#define OP_ParseSchema 123 +#define OP_Destroy 124 +#define OP_IdxGE 125 +#define OP_ReadCookie 126 +#define OP_BitNot 90 /* same as TK_BITNOT */ +#define OP_Or 63 /* same as TK_OR */ +#define OP_Jump 127 +#define OP_ToReal 145 /* same as TK_TO_REAL */ +#define OP_ToNumeric 143 /* same as TK_TO_NUMERIC*/ +#define OP_Function 128 +#define OP_Concat 86 /* same as TK_CONCAT */ +#define OP_SCopy 130 +#define OP_Int64 131 /* The following opcode values are never used */ +#define OP_NotUsed_132 132 +#define OP_NotUsed_133 133 +#define OP_NotUsed_134 134 +#define OP_NotUsed_135 135 +#define OP_NotUsed_136 136 #define OP_NotUsed_137 137 +#define OP_NotUsed_138 138 +#define OP_NotUsed_139 139 +#define OP_NotUsed_140 140 -/* Opcodes that are guaranteed to never push a value onto the stack -** contain a 1 their corresponding position of the following mask -** set. See the opcodeNoPush() function in vdbeaux.c */ -#define NOPUSH_MASK_0 0x3fbc -#define NOPUSH_MASK_1 0x3e5b -#define NOPUSH_MASK_2 0xe3df -#define NOPUSH_MASK_3 0xff9c -#define NOPUSH_MASK_4 0xfffe -#define NOPUSH_MASK_5 0x9ef7 -#define NOPUSH_MASK_6 0xddaf -#define NOPUSH_MASK_7 0x0ebe -#define NOPUSH_MASK_8 0x7dbf -#define NOPUSH_MASK_9 0x0000 + +/* Properties such as "out2" or "jump" that are specified in +** comments following the "case" for each opcode in the vdbe.c +** are encoded into bitvectors as follows: +*/ +#define OPFLG_JUMP 0x0001 /* jump: P2 holds jmp target */ +#define OPFLG_OUT2_PRERELEASE 0x0002 /* out2-prerelease: */ +#define OPFLG_IN1 0x0004 /* in1: P1 is an input */ +#define OPFLG_IN2 0x0008 /* in2: P2 is an input */ +#define OPFLG_IN3 0x0010 /* in3: P3 is an input */ +#define OPFLG_OUT3 0x0020 /* out3: P3 is an output */ +#define OPFLG_INITIALIZER {\ +/* 0 */ 0x00, 0x02, 0x01, 0x05, 0x00, 0x00, 0x00, 0x00,\ +/* 8 */ 0x08, 0x00, 0x11, 0x00, 0x00, 0x02, 0x00, 0x11,\ +/* 16 */ 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00,\ +/* 24 */ 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x11,\ +/* 32 */ 0x10, 0x00, 0x02, 0x02, 0x01, 0x00, 0x00, 0x08,\ +/* 40 */ 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x01, 0x00,\ +/* 48 */ 0x02, 0x02, 0x04, 0x00, 0x05, 0x11, 0x01, 0x11,\ +/* 56 */ 0x00, 0x04, 0x02, 0x00, 0x00, 0x05, 0x00, 0x2c,\ +/* 64 */ 0x2c, 0x04, 0x04, 0x00, 0x05, 0x05, 0x15, 0x15,\ +/* 72 */ 0x15, 0x15, 0x15, 0x15, 0x05, 0x2c, 0x2c, 0x2c,\ +/* 80 */ 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x01,\ +/* 88 */ 0x02, 0x02, 0x04, 0x02, 0x11, 0x11, 0x00, 0x00,\ +/* 96 */ 0x02, 0x01, 0x00, 0x00, 0x21, 0x00, 0x02, 0x00,\ +/* 104 */ 0x00, 0x00, 0x00, 0x02, 0x11, 0x08, 0x00, 0x00,\ +/* 112 */ 0x00, 0x05, 0x00, 0x0c, 0x01, 0x01, 0x11, 0x05,\ +/* 120 */ 0x00, 0x01, 0x00, 0x00, 0x02, 0x11, 0x02, 0x01,\ +/* 128 */ 0x00, 0x02, 0x04, 0x02, 0x00, 0x00, 0x00, 0x00,\ +/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04,\ +/* 144 */ 0x04, 0x04,} diff --git a/os.c b/os.c index 7cd1ab9..c353388 100644 --- a/os.c +++ b/os.c @@ -12,85 +12,267 @@ ** ** This file contains OS interface code that is common to all ** architectures. +** +** $Id: os.c,v 1.125 2008/12/08 18:19:18 drh Exp $ */ #define _SQLITE_OS_C_ 1 #include "sqliteInt.h" -#include "os.h" #undef _SQLITE_OS_C_ +/* +** The default SQLite sqlite3_vfs implementations do not allocate +** memory (actually, os_unix.c allocates a small amount of memory +** from within OsOpen()), but some third-party implementations may. +** So we test the effects of a malloc() failing and the sqlite3OsXXX() +** function returning SQLITE_IOERR_NOMEM using the DO_OS_MALLOC_TEST macro. +** +** The following functions are instrumented for malloc() failure +** testing: +** +** sqlite3OsOpen() +** sqlite3OsRead() +** sqlite3OsWrite() +** sqlite3OsSync() +** sqlite3OsLock() +** +*/ +#if defined(SQLITE_TEST) && (SQLITE_OS_WIN==0) + #define DO_OS_MALLOC_TEST if (1) { \ + void *pTstAlloc = sqlite3Malloc(10); \ + if (!pTstAlloc) return SQLITE_IOERR_NOMEM; \ + sqlite3_free(pTstAlloc); \ + } +#else + #define DO_OS_MALLOC_TEST +#endif + /* ** The following routines are convenience wrappers around methods -** of the OsFile object. This is mostly just syntactic sugar. All +** of the sqlite3_file object. This is mostly just syntactic sugar. All ** of this would be completely automatic if SQLite were coded using ** C++ instead of plain old C. */ -int sqlite3OsClose(OsFile **pId){ - OsFile *id; - if( pId!=0 && (id = *pId)!=0 ){ - return id->pMethod->xClose(pId); - }else{ - return SQLITE_OK; +int sqlite3OsClose(sqlite3_file *pId){ + int rc = SQLITE_OK; + if( pId->pMethods ){ + rc = pId->pMethods->xClose(pId); + pId->pMethods = 0; } + return rc; } -int sqlite3OsOpenDirectory(OsFile *id, const char *zName){ - return id->pMethod->xOpenDirectory(id, zName); +int sqlite3OsRead(sqlite3_file *id, void *pBuf, int amt, i64 offset){ + DO_OS_MALLOC_TEST; + return id->pMethods->xRead(id, pBuf, amt, offset); } -int sqlite3OsRead(OsFile *id, void *pBuf, int amt){ - return id->pMethod->xRead(id, pBuf, amt); +int sqlite3OsWrite(sqlite3_file *id, const void *pBuf, int amt, i64 offset){ + DO_OS_MALLOC_TEST; + return id->pMethods->xWrite(id, pBuf, amt, offset); } -int sqlite3OsWrite(OsFile *id, const void *pBuf, int amt){ - return id->pMethod->xWrite(id, pBuf, amt); +int sqlite3OsTruncate(sqlite3_file *id, i64 size){ + return id->pMethods->xTruncate(id, size); } -int sqlite3OsSeek(OsFile *id, i64 offset){ - return id->pMethod->xSeek(id, offset); +int sqlite3OsSync(sqlite3_file *id, int flags){ + DO_OS_MALLOC_TEST; + return id->pMethods->xSync(id, flags); } -int sqlite3OsTruncate(OsFile *id, i64 size){ - return id->pMethod->xTruncate(id, size); +int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ + DO_OS_MALLOC_TEST; + return id->pMethods->xFileSize(id, pSize); } -int sqlite3OsSync(OsFile *id, int fullsync){ - return id->pMethod->xSync(id, fullsync); +int sqlite3OsLock(sqlite3_file *id, int lockType){ + DO_OS_MALLOC_TEST; + return id->pMethods->xLock(id, lockType); } -void sqlite3OsSetFullSync(OsFile *id, int value){ - id->pMethod->xSetFullSync(id, value); +int sqlite3OsUnlock(sqlite3_file *id, int lockType){ + return id->pMethods->xUnlock(id, lockType); } -int sqlite3OsFileSize(OsFile *id, i64 *pSize){ - return id->pMethod->xFileSize(id, pSize); +int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut){ + DO_OS_MALLOC_TEST; + return id->pMethods->xCheckReservedLock(id, pResOut); } -int sqlite3OsLock(OsFile *id, int lockType){ - return id->pMethod->xLock(id, lockType); +int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){ + return id->pMethods->xFileControl(id, op, pArg); } -int sqlite3OsUnlock(OsFile *id, int lockType){ - return id->pMethod->xUnlock(id, lockType); +int sqlite3OsSectorSize(sqlite3_file *id){ + int (*xSectorSize)(sqlite3_file*) = id->pMethods->xSectorSize; + return (xSectorSize ? xSectorSize(id) : SQLITE_DEFAULT_SECTOR_SIZE); } -int sqlite3OsCheckReservedLock(OsFile *id){ - return id->pMethod->xCheckReservedLock(id); -} -int sqlite3OsSectorSize(OsFile *id){ - int (*xSectorSize)(OsFile*) = id->pMethod->xSectorSize; - return xSectorSize ? xSectorSize(id) : SQLITE_DEFAULT_SECTOR_SIZE; +int sqlite3OsDeviceCharacteristics(sqlite3_file *id){ + return id->pMethods->xDeviceCharacteristics(id); } -#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) - /* These methods are currently only used for testing and debugging. */ - int sqlite3OsFileHandle(OsFile *id){ - return id->pMethod->xFileHandle(id); - } - int sqlite3OsLockState(OsFile *id){ - return id->pMethod->xLockState(id); - } -#endif - -#ifdef SQLITE_ENABLE_REDEF_IO /* -** A function to return a pointer to the virtual function table. -** This routine really does not accomplish very much since the -** virtual function table is a global variable and anybody who -** can call this function can just as easily access the variable -** for themselves. Nevertheless, we include this routine for -** backwards compatibility with an earlier redefinable I/O -** interface design. +** The next group of routines are convenience wrappers around the +** VFS methods. */ -struct sqlite3OsVtbl *sqlite3_os_switch(void){ - return &sqlite3Os; +int sqlite3OsOpen( + sqlite3_vfs *pVfs, + const char *zPath, + sqlite3_file *pFile, + int flags, + int *pFlagsOut +){ + DO_OS_MALLOC_TEST; + return pVfs->xOpen(pVfs, zPath, pFile, flags, pFlagsOut); } +int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return pVfs->xDelete(pVfs, zPath, dirSync); +} +int sqlite3OsAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + DO_OS_MALLOC_TEST; + return pVfs->xAccess(pVfs, zPath, flags, pResOut); +} +int sqlite3OsFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nPathOut, + char *zPathOut +){ + return pVfs->xFullPathname(pVfs, zPath, nPathOut, zPathOut); +} +#ifndef SQLITE_OMIT_LOAD_EXTENSION +void *sqlite3OsDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return pVfs->xDlOpen(pVfs, zPath); +} +void sqlite3OsDlError(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + pVfs->xDlError(pVfs, nByte, zBufOut); +} +void (*sqlite3OsDlSym(sqlite3_vfs *pVfs, void *pHdle, const char *zSym))(void){ + return pVfs->xDlSym(pVfs, pHdle, zSym); +} +void sqlite3OsDlClose(sqlite3_vfs *pVfs, void *pHandle){ + pVfs->xDlClose(pVfs, pHandle); +} +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return pVfs->xRandomness(pVfs, nByte, zBufOut); +} +int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ + return pVfs->xSleep(pVfs, nMicro); +} +int sqlite3OsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return pVfs->xCurrentTime(pVfs, pTimeOut); +} + +int sqlite3OsOpenMalloc( + sqlite3_vfs *pVfs, + const char *zFile, + sqlite3_file **ppFile, + int flags, + int *pOutFlags +){ + int rc = SQLITE_NOMEM; + sqlite3_file *pFile; + pFile = (sqlite3_file *)sqlite3Malloc(pVfs->szOsFile); + if( pFile ){ + rc = sqlite3OsOpen(pVfs, zFile, pFile, flags, pOutFlags); + if( rc!=SQLITE_OK ){ + sqlite3_free(pFile); + }else{ + *ppFile = pFile; + } + } + return rc; +} +int sqlite3OsCloseFree(sqlite3_file *pFile){ + int rc = SQLITE_OK; + assert( pFile ); + rc = sqlite3OsClose(pFile); + sqlite3_free(pFile); + return rc; +} + +/* +** The list of all registered VFS implementations. +*/ +static sqlite3_vfs * SQLITE_WSD vfsList = 0; +#define vfsList GLOBAL(sqlite3_vfs *, vfsList) + +/* +** Locate a VFS by name. If no name is given, simply return the +** first VFS on the list. +*/ +sqlite3_vfs *sqlite3_vfs_find(const char *zVfs){ + sqlite3_vfs *pVfs = 0; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex; #endif +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return 0; +#endif +#if SQLITE_THREADSAFE + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + sqlite3_mutex_enter(mutex); + for(pVfs = vfsList; pVfs; pVfs=pVfs->pNext){ + if( zVfs==0 ) break; + if( strcmp(zVfs, pVfs->zName)==0 ) break; + } + sqlite3_mutex_leave(mutex); + return pVfs; +} + +/* +** Unlink a VFS from the linked list +*/ +static void vfsUnlink(sqlite3_vfs *pVfs){ + assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ); + if( pVfs==0 ){ + /* No-op */ + }else if( vfsList==pVfs ){ + vfsList = pVfs->pNext; + }else if( vfsList ){ + sqlite3_vfs *p = vfsList; + while( p->pNext && p->pNext!=pVfs ){ + p = p->pNext; + } + if( p->pNext==pVfs ){ + p->pNext = pVfs->pNext; + } + } +} + +/* +** Register a VFS with the system. It is harmless to register the same +** VFS multiple times. The new VFS becomes the default if makeDflt is +** true. +*/ +int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ + sqlite3_mutex *mutex = 0; +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(mutex); + vfsUnlink(pVfs); + if( makeDflt || vfsList==0 ){ + pVfs->pNext = vfsList; + vfsList = pVfs; + }else{ + pVfs->pNext = vfsList->pNext; + vfsList->pNext = pVfs; + } + assert(vfsList); + sqlite3_mutex_leave(mutex); + return SQLITE_OK; +} + +/* +** Unregister a VFS so that it is no longer accessible. +*/ +int sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif + sqlite3_mutex_enter(mutex); + vfsUnlink(pVfs); + sqlite3_mutex_leave(mutex); + return SQLITE_OK; +} diff --git a/os.h b/os.h index 76a9f2c..d1ac6b7 100644 --- a/os.h +++ b/os.h @@ -13,60 +13,78 @@ ** This header file (together with is companion C source-code file ** "os.c") attempt to abstract the underlying operating system so that ** the SQLite library will work on both POSIX and windows systems. +** +** This header file is #include-ed by sqliteInt.h and thus ends up +** being included by every source file. +** +** $Id: os.h,v 1.107 2009/01/14 23:03:41 drh Exp $ */ #ifndef _SQLITE_OS_H_ #define _SQLITE_OS_H_ /* ** Figure out if we are dealing with Unix, Windows, or some other -** operating system. +** operating system. After the following block of preprocess macros, +** all of SQLITE_OS_UNIX, SQLITE_OS_WIN, SQLITE_OS_OS2, and SQLITE_OS_OTHER +** will defined to either 1 or 0. One of the four will be 1. The other +** three will be 0. */ -#if defined(OS_OTHER) -# if OS_OTHER==1 -# undef OS_UNIX -# define OS_UNIX 0 -# undef OS_WIN -# define OS_WIN 0 -# undef OS_OS2 -# define OS_OS2 0 +#if defined(SQLITE_OS_OTHER) +# if SQLITE_OS_OTHER==1 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# undef SQLITE_OS_OS2 +# define SQLITE_OS_OS2 0 # else -# undef OS_OTHER +# undef SQLITE_OS_OTHER # endif #endif -#if !defined(OS_UNIX) && !defined(OS_OTHER) -# define OS_OTHER 0 -# ifndef OS_WIN +#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) +# define SQLITE_OS_OTHER 0 +# ifndef SQLITE_OS_WIN # if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) || defined(__BORLANDC__) -# define OS_WIN 1 -# define OS_UNIX 0 -# define OS_OS2 0 +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# define SQLITE_OS_OS2 0 # elif defined(__EMX__) || defined(_OS2) || defined(OS2) || defined(_OS2_) || defined(__OS2__) -# define OS_WIN 0 -# define OS_UNIX 0 -# define OS_OS2 1 +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 0 +# define SQLITE_OS_OS2 1 # else -# define OS_WIN 0 -# define OS_UNIX 1 -# define OS_OS2 0 +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# define SQLITE_OS_OS2 0 # endif # else -# define OS_UNIX 0 -# define OS_OS2 0 +# define SQLITE_OS_UNIX 0 +# define SQLITE_OS_OS2 0 # endif #else -# ifndef OS_WIN -# define OS_WIN 0 +# ifndef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 # endif #endif +/* +** Determine if we are dealing with WindowsCE - which has a much +** reduced API. +*/ +#if defined(_WIN32_WCE) +# define SQLITE_OS_WINCE 1 +#else +# define SQLITE_OS_WINCE 0 +#endif + /* ** Define the maximum size of a temporary filename */ -#if OS_WIN +#if SQLITE_OS_WIN # include # define SQLITE_TEMPNAME_SIZE (MAX_PATH+50) -#elif OS_OS2 +#elif SQLITE_OS_OS2 # if (__GNUC__ > 3 || __GNUC__ == 3 && __GNUC_MINOR__ >= 3) && defined(OS2_HIGH_MEMORY) # include /* has to be included before os2.h for linking to work */ # endif @@ -76,7 +94,9 @@ # define INCL_DOSMISC # define INCL_DOSPROCESS # define INCL_DOSMODULEMGR +# define INCL_DOSSEMAPHORES # include +# include # define SQLITE_TEMPNAME_SIZE (CCHMAXPATHCOMP) #else # define SQLITE_TEMPNAME_SIZE 200 @@ -103,7 +123,7 @@ ** If sqlite is being embedded in another program, you may wish to change the ** prefix to reflect your program's name, so that if your program exits ** prematurely, old temporary files can be easily identified. This can be done -** using -DTEMP_FILE_PREFIX=myprefix_ on the compiler command line. +** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line. ** ** 2006-10-31: The default prefix used to be "sqlite_". But then ** Mcafee started using SQLite in their anti-virus product and it @@ -117,143 +137,10 @@ ** enough to know that calling the developer will not help get rid ** of the file. */ -#ifndef TEMP_FILE_PREFIX -# define TEMP_FILE_PREFIX "etilqs_" +#ifndef SQLITE_TEMP_FILE_PREFIX +# define SQLITE_TEMP_FILE_PREFIX "etilqs_" #endif -/* -** Define the interfaces for Unix, Windows, and OS/2. -*/ -#if OS_UNIX -#define sqlite3OsOpenReadWrite sqlite3UnixOpenReadWrite -#define sqlite3OsOpenExclusive sqlite3UnixOpenExclusive -#define sqlite3OsOpenReadOnly sqlite3UnixOpenReadOnly -#define sqlite3OsDelete sqlite3UnixDelete -#define sqlite3OsFileExists sqlite3UnixFileExists -#define sqlite3OsFullPathname sqlite3UnixFullPathname -#define sqlite3OsIsDirWritable sqlite3UnixIsDirWritable -#define sqlite3OsSyncDirectory sqlite3UnixSyncDirectory -#define sqlite3OsTempFileName sqlite3UnixTempFileName -#define sqlite3OsRandomSeed sqlite3UnixRandomSeed -#define sqlite3OsSleep sqlite3UnixSleep -#define sqlite3OsCurrentTime sqlite3UnixCurrentTime -#define sqlite3OsEnterMutex sqlite3UnixEnterMutex -#define sqlite3OsLeaveMutex sqlite3UnixLeaveMutex -#define sqlite3OsInMutex sqlite3UnixInMutex -#define sqlite3OsThreadSpecificData sqlite3UnixThreadSpecificData -#define sqlite3OsMalloc sqlite3GenericMalloc -#define sqlite3OsRealloc sqlite3GenericRealloc -#define sqlite3OsFree sqlite3GenericFree -#define sqlite3OsAllocationSize sqlite3GenericAllocationSize -#define sqlite3OsDlopen sqlite3UnixDlopen -#define sqlite3OsDlsym sqlite3UnixDlsym -#define sqlite3OsDlclose sqlite3UnixDlclose -#endif -#if OS_WIN -#define sqlite3OsOpenReadWrite sqlite3WinOpenReadWrite -#define sqlite3OsOpenExclusive sqlite3WinOpenExclusive -#define sqlite3OsOpenReadOnly sqlite3WinOpenReadOnly -#define sqlite3OsDelete sqlite3WinDelete -#define sqlite3OsFileExists sqlite3WinFileExists -#define sqlite3OsFullPathname sqlite3WinFullPathname -#define sqlite3OsIsDirWritable sqlite3WinIsDirWritable -#define sqlite3OsSyncDirectory sqlite3WinSyncDirectory -#define sqlite3OsTempFileName sqlite3WinTempFileName -#define sqlite3OsRandomSeed sqlite3WinRandomSeed -#define sqlite3OsSleep sqlite3WinSleep -#define sqlite3OsCurrentTime sqlite3WinCurrentTime -#define sqlite3OsEnterMutex sqlite3WinEnterMutex -#define sqlite3OsLeaveMutex sqlite3WinLeaveMutex -#define sqlite3OsInMutex sqlite3WinInMutex -#define sqlite3OsThreadSpecificData sqlite3WinThreadSpecificData -#define sqlite3OsMalloc sqlite3GenericMalloc -#define sqlite3OsRealloc sqlite3GenericRealloc -#define sqlite3OsFree sqlite3GenericFree -#define sqlite3OsAllocationSize sqlite3GenericAllocationSize -#define sqlite3OsDlopen sqlite3WinDlopen -#define sqlite3OsDlsym sqlite3WinDlsym -#define sqlite3OsDlclose sqlite3WinDlclose -#endif -#if OS_OS2 -#define sqlite3OsOpenReadWrite sqlite3Os2OpenReadWrite -#define sqlite3OsOpenExclusive sqlite3Os2OpenExclusive -#define sqlite3OsOpenReadOnly sqlite3Os2OpenReadOnly -#define sqlite3OsDelete sqlite3Os2Delete -#define sqlite3OsFileExists sqlite3Os2FileExists -#define sqlite3OsFullPathname sqlite3Os2FullPathname -#define sqlite3OsIsDirWritable sqlite3Os2IsDirWritable -#define sqlite3OsSyncDirectory sqlite3Os2SyncDirectory -#define sqlite3OsTempFileName sqlite3Os2TempFileName -#define sqlite3OsRandomSeed sqlite3Os2RandomSeed -#define sqlite3OsSleep sqlite3Os2Sleep -#define sqlite3OsCurrentTime sqlite3Os2CurrentTime -#define sqlite3OsEnterMutex sqlite3Os2EnterMutex -#define sqlite3OsLeaveMutex sqlite3Os2LeaveMutex -#define sqlite3OsInMutex sqlite3Os2InMutex -#define sqlite3OsThreadSpecificData sqlite3Os2ThreadSpecificData -#define sqlite3OsMalloc sqlite3GenericMalloc -#define sqlite3OsRealloc sqlite3GenericRealloc -#define sqlite3OsFree sqlite3GenericFree -#define sqlite3OsAllocationSize sqlite3GenericAllocationSize -#define sqlite3OsDlopen sqlite3Os2Dlopen -#define sqlite3OsDlsym sqlite3Os2Dlsym -#define sqlite3OsDlclose sqlite3Os2Dlclose -#endif - - - - -/* -** If using an alternative OS interface, then we must have an "os_other.h" -** header file available for that interface. Presumably the "os_other.h" -** header file contains #defines similar to those above. -*/ -#if OS_OTHER -# include "os_other.h" -#endif - - - -/* -** Forward declarations -*/ -typedef struct OsFile OsFile; -typedef struct IoMethod IoMethod; - -/* -** An instance of the following structure contains pointers to all -** methods on an OsFile object. -*/ -struct IoMethod { - int (*xClose)(OsFile**); - int (*xOpenDirectory)(OsFile*, const char*); - int (*xRead)(OsFile*, void*, int amt); - int (*xWrite)(OsFile*, const void*, int amt); - int (*xSeek)(OsFile*, i64 offset); - int (*xTruncate)(OsFile*, i64 size); - int (*xSync)(OsFile*, int); - void (*xSetFullSync)(OsFile *id, int setting); - int (*xFileHandle)(OsFile *id); - int (*xFileSize)(OsFile*, i64 *pSize); - int (*xLock)(OsFile*, int); - int (*xUnlock)(OsFile*, int); - int (*xLockState)(OsFile *id); - int (*xCheckReservedLock)(OsFile *id); - int (*xSectorSize)(OsFile *id); -}; - -/* -** The OsFile object describes an open disk file in an OS-dependent way. -** The version of OsFile defined here is a generic version. Each OS -** implementation defines its own subclass of this structure that contains -** additional information needed to handle file I/O. But the pMethod -** entry (pointing to the virtual function table) always occurs first -** so that we can always find the appropriate methods. -*/ -struct OsFile { - IoMethod const *pMethod; -}; - /* ** The following values may be passed as the second argument to ** sqlite3OsLock(). The various locks exhibit the following semantics: @@ -345,204 +232,45 @@ extern unsigned int sqlite3_pending_byte; #define SHARED_FIRST (PENDING_BYTE+2) #define SHARED_SIZE 510 -/* -** Prototypes for operating system interface routines. +/* +** Functions for accessing sqlite3_file methods */ -int sqlite3OsClose(OsFile**); -int sqlite3OsOpenDirectory(OsFile*, const char*); -int sqlite3OsRead(OsFile*, void*, int amt); -int sqlite3OsWrite(OsFile*, const void*, int amt); -int sqlite3OsSeek(OsFile*, i64 offset); -int sqlite3OsTruncate(OsFile*, i64 size); -int sqlite3OsSync(OsFile*, int); -void sqlite3OsSetFullSync(OsFile *id, int setting); -int sqlite3OsFileSize(OsFile*, i64 *pSize); -int sqlite3OsLock(OsFile*, int); -int sqlite3OsUnlock(OsFile*, int); -int sqlite3OsCheckReservedLock(OsFile *id); -int sqlite3OsOpenReadWrite(const char*, OsFile**, int*); -int sqlite3OsOpenExclusive(const char*, OsFile**, int); -int sqlite3OsOpenReadOnly(const char*, OsFile**); -int sqlite3OsDelete(const char*); -int sqlite3OsFileExists(const char*); -char *sqlite3OsFullPathname(const char*); -int sqlite3OsIsDirWritable(char*); -int sqlite3OsSyncDirectory(const char*); -int sqlite3OsSectorSize(OsFile *id); -int sqlite3OsTempFileName(char*); -int sqlite3OsRandomSeed(char*); -int sqlite3OsSleep(int ms); -int sqlite3OsCurrentTime(double*); -void sqlite3OsEnterMutex(void); -void sqlite3OsLeaveMutex(void); -int sqlite3OsInMutex(int); -ThreadData *sqlite3OsThreadSpecificData(int); -void *sqlite3OsMalloc(int); -void *sqlite3OsRealloc(void *, int); -void sqlite3OsFree(void *); -int sqlite3OsAllocationSize(void *); -void *sqlite3OsDlopen(const char*); -void *sqlite3OsDlsym(void*, const char*); -int sqlite3OsDlclose(void*); +int sqlite3OsClose(sqlite3_file*); +int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); +int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); +int sqlite3OsTruncate(sqlite3_file*, i64 size); +int sqlite3OsSync(sqlite3_file*, int); +int sqlite3OsFileSize(sqlite3_file*, i64 *pSize); +int sqlite3OsLock(sqlite3_file*, int); +int sqlite3OsUnlock(sqlite3_file*, int); +int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); +int sqlite3OsFileControl(sqlite3_file*,int,void*); +#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 +int sqlite3OsSectorSize(sqlite3_file *id); +int sqlite3OsDeviceCharacteristics(sqlite3_file *id); -#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) - int sqlite3OsFileHandle(OsFile *id); - int sqlite3OsLockState(OsFile *id); -#endif +/* +** Functions for accessing sqlite3_vfs methods +*/ +int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); +int sqlite3OsDelete(sqlite3_vfs *, const char *, int); +int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut); +int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *); +#ifndef SQLITE_OMIT_LOAD_EXTENSION +void *sqlite3OsDlOpen(sqlite3_vfs *, const char *); +void sqlite3OsDlError(sqlite3_vfs *, int, char *); +void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void); +void sqlite3OsDlClose(sqlite3_vfs *, void *); +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +int sqlite3OsRandomness(sqlite3_vfs *, int, char *); +int sqlite3OsSleep(sqlite3_vfs *, int); +int sqlite3OsCurrentTime(sqlite3_vfs *, double*); /* -** If the SQLITE_ENABLE_REDEF_IO macro is defined, then the OS-layer -** interface routines are not called directly but are invoked using -** pointers to functions. This allows the implementation of various -** OS-layer interface routines to be modified at run-time. There are -** obscure but legitimate reasons for wanting to do this. But for -** most users, a direct call to the underlying interface is preferable -** so the the redefinable I/O interface is turned off by default. +** Convenience functions for opening and closing files using +** sqlite3_malloc() to obtain space for the file-handle structure. */ -#ifdef SQLITE_ENABLE_REDEF_IO - -/* -** When redefinable I/O is enabled, a single global instance of the -** following structure holds pointers to the routines that SQLite -** uses to talk with the underlying operating system. Modify this -** structure (before using any SQLite API!) to accomodate perculiar -** operating system interfaces or behaviors. -*/ -struct sqlite3OsVtbl { - int (*xOpenReadWrite)(const char*, OsFile**, int*); - int (*xOpenExclusive)(const char*, OsFile**, int); - int (*xOpenReadOnly)(const char*, OsFile**); - - int (*xDelete)(const char*); - int (*xFileExists)(const char*); - char *(*xFullPathname)(const char*); - int (*xIsDirWritable)(char*); - int (*xSyncDirectory)(const char*); - int (*xTempFileName)(char*); - - int (*xRandomSeed)(char*); - int (*xSleep)(int ms); - int (*xCurrentTime)(double*); - - void (*xEnterMutex)(void); - void (*xLeaveMutex)(void); - int (*xInMutex)(int); - ThreadData *(*xThreadSpecificData)(int); - - void *(*xMalloc)(int); - void *(*xRealloc)(void *, int); - void (*xFree)(void *); - int (*xAllocationSize)(void *); - - void *(*xDlopen)(const char*); - void *(*xDlsym)(void*, const char*); - int (*xDlclose)(void*); -}; - -/* Macro used to comment out routines that do not exists when there is -** no disk I/O or extension loading -*/ -#ifdef SQLITE_OMIT_DISKIO -# define IF_DISKIO(X) 0 -#else -# define IF_DISKIO(X) X -#endif -#ifdef SQLITE_OMIT_LOAD_EXTENSION -# define IF_DLOPEN(X) 0 -#else -# define IF_DLOPEN(X) X -#endif - - -#if defined(_SQLITE_OS_C_) || defined(SQLITE_AMALGAMATION) - /* - ** The os.c file implements the global virtual function table. - ** We have to put this file here because the initializers - ** (ex: sqlite3OsRandomSeed) are macros that are about to be - ** redefined. - */ - struct sqlite3OsVtbl sqlite3Os = { - IF_DISKIO( sqlite3OsOpenReadWrite ), - IF_DISKIO( sqlite3OsOpenExclusive ), - IF_DISKIO( sqlite3OsOpenReadOnly ), - IF_DISKIO( sqlite3OsDelete ), - IF_DISKIO( sqlite3OsFileExists ), - IF_DISKIO( sqlite3OsFullPathname ), - IF_DISKIO( sqlite3OsIsDirWritable ), - IF_DISKIO( sqlite3OsSyncDirectory ), - IF_DISKIO( sqlite3OsTempFileName ), - sqlite3OsRandomSeed, - sqlite3OsSleep, - sqlite3OsCurrentTime, - sqlite3OsEnterMutex, - sqlite3OsLeaveMutex, - sqlite3OsInMutex, - sqlite3OsThreadSpecificData, - sqlite3OsMalloc, - sqlite3OsRealloc, - sqlite3OsFree, - sqlite3OsAllocationSize, - IF_DLOPEN( sqlite3OsDlopen ), - IF_DLOPEN( sqlite3OsDlsym ), - IF_DLOPEN( sqlite3OsDlclose ), - }; -#else - /* - ** Files other than os.c just reference the global virtual function table. - */ - extern struct sqlite3OsVtbl sqlite3Os; -#endif /* _SQLITE_OS_C_ */ - - -/* This additional API routine is available with redefinable I/O */ -struct sqlite3OsVtbl *sqlite3_os_switch(void); - - -/* -** Redefine the OS interface to go through the virtual function table -** rather than calling routines directly. -*/ -#undef sqlite3OsOpenReadWrite -#undef sqlite3OsOpenExclusive -#undef sqlite3OsOpenReadOnly -#undef sqlite3OsDelete -#undef sqlite3OsFileExists -#undef sqlite3OsFullPathname -#undef sqlite3OsIsDirWritable -#undef sqlite3OsSyncDirectory -#undef sqlite3OsTempFileName -#undef sqlite3OsRandomSeed -#undef sqlite3OsSleep -#undef sqlite3OsCurrentTime -#undef sqlite3OsEnterMutex -#undef sqlite3OsLeaveMutex -#undef sqlite3OsInMutex -#undef sqlite3OsThreadSpecificData -#undef sqlite3OsMalloc -#undef sqlite3OsRealloc -#undef sqlite3OsFree -#undef sqlite3OsAllocationSize -#define sqlite3OsOpenReadWrite sqlite3Os.xOpenReadWrite -#define sqlite3OsOpenExclusive sqlite3Os.xOpenExclusive -#define sqlite3OsOpenReadOnly sqlite3Os.xOpenReadOnly -#define sqlite3OsDelete sqlite3Os.xDelete -#define sqlite3OsFileExists sqlite3Os.xFileExists -#define sqlite3OsFullPathname sqlite3Os.xFullPathname -#define sqlite3OsIsDirWritable sqlite3Os.xIsDirWritable -#define sqlite3OsSyncDirectory sqlite3Os.xSyncDirectory -#define sqlite3OsTempFileName sqlite3Os.xTempFileName -#define sqlite3OsRandomSeed sqlite3Os.xRandomSeed -#define sqlite3OsSleep sqlite3Os.xSleep -#define sqlite3OsCurrentTime sqlite3Os.xCurrentTime -#define sqlite3OsEnterMutex sqlite3Os.xEnterMutex -#define sqlite3OsLeaveMutex sqlite3Os.xLeaveMutex -#define sqlite3OsInMutex sqlite3Os.xInMutex -#define sqlite3OsThreadSpecificData sqlite3Os.xThreadSpecificData -#define sqlite3OsMalloc sqlite3Os.xMalloc -#define sqlite3OsRealloc sqlite3Os.xRealloc -#define sqlite3OsFree sqlite3Os.xFree -#define sqlite3OsAllocationSize sqlite3Os.xAllocationSize - -#endif /* SQLITE_ENABLE_REDEF_IO */ +int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); +int sqlite3OsCloseFree(sqlite3_file *); #endif /* _SQLITE_OS_H_ */ diff --git a/os_common.h b/os_common.h index 0c54405..5fab7a4 100644 --- a/os_common.h +++ b/os_common.h @@ -16,7 +16,11 @@ ** ** This file should be #included by the os_*.c files only. It is not a ** general purpose header file. +** +** $Id: os_common.h,v 1.37 2008/05/29 20:22:37 shane Exp $ */ +#ifndef _OS_COMMON_H_ +#define _OS_COMMON_H_ /* ** At least two bugs have slipped in because we changed the MEMORY_DEBUG @@ -37,16 +41,16 @@ unsigned int sqlite3_pending_byte = 0x40000000; #endif #ifdef SQLITE_DEBUG -int sqlite3_os_trace = 0; -#define OSTRACE1(X) if( sqlite3_os_trace ) sqlite3DebugPrintf(X) -#define OSTRACE2(X,Y) if( sqlite3_os_trace ) sqlite3DebugPrintf(X,Y) -#define OSTRACE3(X,Y,Z) if( sqlite3_os_trace ) sqlite3DebugPrintf(X,Y,Z) -#define OSTRACE4(X,Y,Z,A) if( sqlite3_os_trace ) sqlite3DebugPrintf(X,Y,Z,A) -#define OSTRACE5(X,Y,Z,A,B) if( sqlite3_os_trace ) sqlite3DebugPrintf(X,Y,Z,A,B) +int sqlite3OSTrace = 0; +#define OSTRACE1(X) if( sqlite3OSTrace ) sqlite3DebugPrintf(X) +#define OSTRACE2(X,Y) if( sqlite3OSTrace ) sqlite3DebugPrintf(X,Y) +#define OSTRACE3(X,Y,Z) if( sqlite3OSTrace ) sqlite3DebugPrintf(X,Y,Z) +#define OSTRACE4(X,Y,Z,A) if( sqlite3OSTrace ) sqlite3DebugPrintf(X,Y,Z,A) +#define OSTRACE5(X,Y,Z,A,B) if( sqlite3OSTrace ) sqlite3DebugPrintf(X,Y,Z,A,B) #define OSTRACE6(X,Y,Z,A,B,C) \ - if(sqlite3_os_trace) sqlite3DebugPrintf(X,Y,Z,A,B,C) + if(sqlite3OSTrace) sqlite3DebugPrintf(X,Y,Z,A,B,C) #define OSTRACE7(X,Y,Z,A,B,C,D) \ - if(sqlite3_os_trace) sqlite3DebugPrintf(X,Y,Z,A,B,C,D) + if(sqlite3OSTrace) sqlite3DebugPrintf(X,Y,Z,A,B,C,D) #else #define OSTRACE1(X) #define OSTRACE2(X,Y) @@ -62,22 +66,22 @@ int sqlite3_os_trace = 0; ** on i486 hardware. */ #ifdef SQLITE_PERFORMANCE_TRACE -__inline__ unsigned long long int hwtime(void){ - unsigned long long int x; - __asm__("rdtsc\n\t" - "mov %%edx, %%ecx\n\t" - :"=A" (x)); - return x; -} -static unsigned long long int g_start; -static unsigned int elapse; -#define TIMER_START g_start=hwtime() -#define TIMER_END elapse=hwtime()-g_start -#define TIMER_ELAPSED elapse + +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. +*/ +#include "hwtime.h" + +static sqlite_uint64 g_start; +static sqlite_uint64 g_elapsed; +#define TIMER_START g_start=sqlite3Hwtime() +#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start +#define TIMER_ELAPSED g_elapsed #else #define TIMER_START #define TIMER_END -#define TIMER_ELAPSED 0 +#define TIMER_ELAPSED ((sqlite_uint64)0) #endif /* @@ -86,19 +90,22 @@ static unsigned int elapse; ** is used for testing the I/O recovery logic. */ #ifdef SQLITE_TEST -int sqlite3_io_error_hit = 0; -int sqlite3_io_error_pending = 0; -int sqlite3_io_error_persist = 0; +int sqlite3_io_error_hit = 0; /* Total number of I/O Errors */ +int sqlite3_io_error_hardhit = 0; /* Number of non-benign errors */ +int sqlite3_io_error_pending = 0; /* Count down to first I/O error */ +int sqlite3_io_error_persist = 0; /* True if I/O errors persist */ +int sqlite3_io_error_benign = 0; /* True if errors are benign */ int sqlite3_diskfull_pending = 0; int sqlite3_diskfull = 0; +#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) #define SimulateIOError(CODE) \ - if( sqlite3_io_error_pending || sqlite3_io_error_hit ) \ - if( sqlite3_io_error_pending-- == 1 \ - || (sqlite3_io_error_persist && sqlite3_io_error_hit) ) \ - { local_ioerr(); CODE; } + if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ + || sqlite3_io_error_pending-- == 1 ) \ + { local_ioerr(); CODE; } static void local_ioerr(){ IOTRACE(("IOERR\n")); - sqlite3_io_error_hit = 1; + sqlite3_io_error_hit++; + if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; } #define SimulateDiskfullError(CODE) \ if( sqlite3_diskfull_pending ){ \ @@ -112,6 +119,7 @@ static void local_ioerr(){ } \ } #else +#define SimulateIOErrorBenign(X) #define SimulateIOError(A) #define SimulateDiskfullError(A) #endif @@ -126,73 +134,4 @@ int sqlite3_open_file_count = 0; #define OpenCounter(X) #endif -/* -** sqlite3GenericMalloc -** sqlite3GenericRealloc -** sqlite3GenericOsFree -** sqlite3GenericAllocationSize -** -** Implementation of the os level dynamic memory allocation interface in terms -** of the standard malloc(), realloc() and free() found in many operating -** systems. No rocket science here. -** -** There are two versions of these four functions here. The version -** implemented here is only used if memory-management or memory-debugging is -** enabled. This version allocates an extra 8-bytes at the beginning of each -** block and stores the size of the allocation there. -** -** If neither memory-management or debugging is enabled, the second -** set of implementations is used instead. -*/ -#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || defined (SQLITE_MEMDEBUG) -void *sqlite3GenericMalloc(int n){ - char *p = (char *)malloc(n+8); - assert(n>0); - assert(sizeof(int)<=8); - if( p ){ - *(int *)p = n; - p += 8; - } - return (void *)p; -} -void *sqlite3GenericRealloc(void *p, int n){ - char *p2 = ((char *)p - 8); - assert(n>0); - p2 = (char*)realloc(p2, n+8); - if( p2 ){ - *(int *)p2 = n; - p2 += 8; - } - return (void *)p2; -} -void sqlite3GenericFree(void *p){ - assert(p); - free((void *)((char *)p - 8)); -} -int sqlite3GenericAllocationSize(void *p){ - return p ? *(int *)((char *)p - 8) : 0; -} -#else -void *sqlite3GenericMalloc(int n){ - char *p = (char *)malloc(n); - return (void *)p; -} -void *sqlite3GenericRealloc(void *p, int n){ - assert(n>0); - p = realloc(p, n); - return p; -} -void sqlite3GenericFree(void *p){ - assert(p); - free(p); -} -/* Never actually used, but needed for the linker */ -int sqlite3GenericAllocationSize(void *p){ return 0; } -#endif - -/* -** The default size of a disk sector -*/ -#ifndef PAGER_SECTOR_SIZE -# define PAGER_SECTOR_SIZE 512 -#endif +#endif /* !defined(_OS_COMMON_H_) */ diff --git a/os_os2.c b/os_os2.c index 45d796b..07e9e6d 100644 --- a/os_os2.c +++ b/os_os2.c @@ -11,17 +11,44 @@ ****************************************************************************** ** ** This file contains code that is specific to OS/2. +** +** $Id: os_os2.c,v 1.63 2008/12/10 19:26:24 drh Exp $ */ #include "sqliteInt.h" -#include "os.h" -#if OS_OS2 +#if SQLITE_OS_OS2 + +/* +** A Note About Memory Allocation: +** +** This driver uses malloc()/free() directly rather than going through +** the SQLite-wrappers sqlite3_malloc()/sqlite3_free(). Those wrappers +** are designed for use on embedded systems where memory is scarce and +** malloc failures happen frequently. OS/2 does not typically run on +** embedded systems, and when it does the developers normally have bigger +** problems to worry about than running out of memory. So there is not +** a compelling need to use the wrappers. +** +** But there is a good reason to not use the wrappers. If we use the +** wrappers then we will get simulated malloc() failures within this +** driver. And that causes all kinds of problems for our tests. We +** could enhance SQLite to deal with simulated malloc failures within +** the OS driver, but the code to deal with those failure would not +** be exercised on Linux (which does not need to malloc() in the driver) +** and so we would have difficulty writing coverage tests for that +** code. Better to leave the code out, we think. +** +** The point of this discussion is as follows: When creating a new +** OS layer for an embedded system, if you use this file as an example, +** avoid the use of malloc()/free(). Those routines work ok on OS/2 +** desktops but not so well in embedded systems. +*/ /* ** Macros used to determine whether or not to use threads. */ -#if defined(THREADSAFE) && THREADSAFE +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE # define SQLITE_OS2_THREADS 1 #endif @@ -31,242 +58,40 @@ #include "os_common.h" /* -** The os2File structure is subclass of OsFile specific for the OS/2 +** The os2File structure is subclass of sqlite3_file specific for the OS/2 ** protability layer. */ typedef struct os2File os2File; struct os2File { - IoMethod const *pMethod; /* Always the first entry */ + const sqlite3_io_methods *pMethod; /* Always the first entry */ HFILE h; /* Handle for accessing the file */ - int delOnClose; /* True if file is to be deleted on close */ - char* pathToDel; /* Name of file to delete on close */ + char* pathToDel; /* Name of file to delete on close, NULL if not */ unsigned char locktype; /* Type of lock currently held on this file */ }; -/* -** Do not include any of the File I/O interface procedures if the -** SQLITE_OMIT_DISKIO macro is defined (indicating that there database -** will be in-memory only) -*/ -#ifndef SQLITE_OMIT_DISKIO +#define LOCK_TIMEOUT 10L /* the default locking timeout */ -/* -** Delete the named file -*/ -int sqlite3Os2Delete( const char *zFilename ){ - APIRET rc = NO_ERROR; - - rc = DosDelete( (PSZ)zFilename ); - OSTRACE2( "DELETE \"%s\"\n", zFilename ); - return rc == NO_ERROR ? SQLITE_OK : SQLITE_IOERR; -} - -/* -** Return TRUE if the named file exists. -*/ -int sqlite3Os2FileExists( const char *zFilename ){ - FILESTATUS3 fsts3ConfigInfo; - memset(&fsts3ConfigInfo, 0, sizeof(fsts3ConfigInfo)); - return DosQueryPathInfo( (PSZ)zFilename, FIL_STANDARD, - &fsts3ConfigInfo, sizeof(FILESTATUS3) ) == NO_ERROR; -} - -/* Forward declaration */ -int allocateOs2File( os2File *pInit, OsFile **pld ); - -/* -** Attempt to open a file for both reading and writing. If that -** fails, try opening it read-only. If the file does not exist, -** try to create it. -** -** On success, a handle for the open file is written to *id -** and *pReadonly is set to 0 if the file was opened for reading and -** writing or 1 if the file was opened read-only. The function returns -** SQLITE_OK. -** -** On failure, the function returns SQLITE_CANTOPEN and leaves -** *id and *pReadonly unchanged. -*/ -int sqlite3Os2OpenReadWrite( - const char *zFilename, - OsFile **pld, - int *pReadonly -){ - os2File f; - HFILE hf; - ULONG ulAction; - APIRET rc = NO_ERROR; - - assert( *pld == 0 ); - rc = DosOpen( (PSZ)zFilename, &hf, &ulAction, 0L, - FILE_ARCHIVED | FILE_NORMAL, - OPEN_ACTION_CREATE_IF_NEW | OPEN_ACTION_OPEN_IF_EXISTS, - OPEN_FLAGS_FAIL_ON_ERROR | OPEN_FLAGS_RANDOM | - OPEN_SHARE_DENYNONE | OPEN_ACCESS_READWRITE, (PEAOP2)NULL ); - if( rc != NO_ERROR ){ - rc = DosOpen( (PSZ)zFilename, &hf, &ulAction, 0L, - FILE_ARCHIVED | FILE_NORMAL, - OPEN_ACTION_CREATE_IF_NEW | OPEN_ACTION_OPEN_IF_EXISTS, - OPEN_FLAGS_FAIL_ON_ERROR | OPEN_FLAGS_RANDOM | - OPEN_SHARE_DENYWRITE | OPEN_ACCESS_READONLY, (PEAOP2)NULL ); - if( rc != NO_ERROR ){ - return SQLITE_CANTOPEN; - } - *pReadonly = 1; - } - else{ - *pReadonly = 0; - } - f.h = hf; - f.locktype = NO_LOCK; - f.delOnClose = 0; - f.pathToDel = NULL; - OpenCounter(+1); - OSTRACE3( "OPEN R/W %d \"%s\"\n", hf, zFilename ); - return allocateOs2File( &f, pld ); -} - - -/* -** Attempt to open a new file for exclusive access by this process. -** The file will be opened for both reading and writing. To avoid -** a potential security problem, we do not allow the file to have -** previously existed. Nor do we allow the file to be a symbolic -** link. -** -** If delFlag is true, then make arrangements to automatically delete -** the file when it is closed. -** -** On success, write the file handle into *id and return SQLITE_OK. -** -** On failure, return SQLITE_CANTOPEN. -*/ -int sqlite3Os2OpenExclusive( const char *zFilename, OsFile **pld, int delFlag ){ - os2File f; - HFILE hf; - ULONG ulAction; - APIRET rc = NO_ERROR; - - assert( *pld == 0 ); - rc = DosOpen( (PSZ)zFilename, &hf, &ulAction, 0L, FILE_NORMAL, - OPEN_ACTION_CREATE_IF_NEW | OPEN_ACTION_REPLACE_IF_EXISTS, - OPEN_FLAGS_FAIL_ON_ERROR | OPEN_FLAGS_RANDOM | - OPEN_SHARE_DENYREADWRITE | OPEN_ACCESS_READWRITE, (PEAOP2)NULL ); - if( rc != NO_ERROR ){ - return SQLITE_CANTOPEN; - } - - f.h = hf; - f.locktype = NO_LOCK; - f.delOnClose = delFlag ? 1 : 0; - f.pathToDel = delFlag ? sqlite3OsFullPathname( zFilename ) : NULL; - OpenCounter( +1 ); - if( delFlag ) DosForceDelete( (PSZ)sqlite3OsFullPathname( zFilename ) ); - OSTRACE3( "OPEN EX %d \"%s\"\n", hf, sqlite3OsFullPathname ( zFilename ) ); - return allocateOs2File( &f, pld ); -} - -/* -** Attempt to open a new file for read-only access. -** -** On success, write the file handle into *id and return SQLITE_OK. -** -** On failure, return SQLITE_CANTOPEN. -*/ -int sqlite3Os2OpenReadOnly( const char *zFilename, OsFile **pld ){ - os2File f; - HFILE hf; - ULONG ulAction; - APIRET rc = NO_ERROR; - - assert( *pld == 0 ); - rc = DosOpen( (PSZ)zFilename, &hf, &ulAction, 0L, - FILE_NORMAL, OPEN_ACTION_OPEN_IF_EXISTS, - OPEN_FLAGS_FAIL_ON_ERROR | OPEN_FLAGS_RANDOM | - OPEN_SHARE_DENYWRITE | OPEN_ACCESS_READONLY, (PEAOP2)NULL ); - if( rc != NO_ERROR ){ - return SQLITE_CANTOPEN; - } - f.h = hf; - f.locktype = NO_LOCK; - f.delOnClose = 0; - f.pathToDel = NULL; - OpenCounter( +1 ); - OSTRACE3( "OPEN RO %d \"%s\"\n", hf, zFilename ); - return allocateOs2File( &f, pld ); -} - -/* -** Attempt to open a file descriptor for the directory that contains a -** file. This file descriptor can be used to fsync() the directory -** in order to make sure the creation of a new file is actually written -** to disk. -** -** This routine is only meaningful for Unix. It is a no-op under -** OS/2 since OS/2 does not support hard links. -** -** On success, a handle for a previously open file is at *id is -** updated with the new directory file descriptor and SQLITE_OK is -** returned. -** -** On failure, the function returns SQLITE_CANTOPEN and leaves -** *id unchanged. -*/ -int os2OpenDirectory( - OsFile *id, - const char *zDirname -){ - return SQLITE_OK; -} - -/* -** Create a temporary file name in zBuf. zBuf must be big enough to -** hold at least SQLITE_TEMPNAME_SIZE characters. -*/ -int sqlite3Os2TempFileName( char *zBuf ){ - static const unsigned char zChars[] = - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "0123456789"; - int i, j; - PSZ zTempPath = 0; - if( DosScanEnv( (PSZ)"TEMP", &zTempPath ) ){ - if( DosScanEnv( (PSZ)"TMP", &zTempPath ) ){ - if( DosScanEnv( (PSZ)"TMPDIR", &zTempPath ) ){ - ULONG ulDriveNum = 0, ulDriveMap = 0; - DosQueryCurrentDisk( &ulDriveNum, &ulDriveMap ); - sprintf( (char*)zTempPath, "%c:", (char)( 'A' + ulDriveNum - 1 ) ); - } - } - } - for(;;){ - sprintf( zBuf, "%s\\"TEMP_FILE_PREFIX, zTempPath ); - j = strlen( zBuf ); - sqlite3Randomness( 15, &zBuf[j] ); - for( i = 0; i < 15; i++, j++ ){ - zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; - } - zBuf[j] = 0; - if( !sqlite3OsFileExists( zBuf ) ) break; - } - OSTRACE2( "TEMP FILENAME: %s\n", zBuf ); - return SQLITE_OK; -} +/***************************************************************************** +** The next group of routines implement the I/O methods specified +** by the sqlite3_io_methods object. +******************************************************************************/ /* ** Close a file. */ -int os2Close( OsFile **pld ){ - os2File *pFile; +static int os2Close( sqlite3_file *id ){ APIRET rc = NO_ERROR; - if( pld && (pFile = (os2File*)*pld) != 0 ){ + os2File *pFile; + if( id && (pFile = (os2File*)id) != 0 ){ OSTRACE2( "CLOSE %d\n", pFile->h ); rc = DosClose( pFile->h ); pFile->locktype = NO_LOCK; - if( pFile->delOnClose != 0 ){ - rc = DosForceDelete( (PSZ)pFile->pathToDel ); + if( pFile->pathToDel != NULL ){ + rc = DosForceDelete( (PSZ)pFile->pathToDel ); + free( pFile->pathToDel ); + pFile->pathToDel = NULL; } - *pld = 0; + id = 0; OpenCounter( -1 ); } @@ -278,17 +103,28 @@ int os2Close( OsFile **pld ){ ** bytes were read successfully and SQLITE_IOERR if anything goes ** wrong. */ -int os2Read( OsFile *id, void *pBuf, int amt ){ +static int os2Read( + sqlite3_file *id, /* File to read from */ + void *pBuf, /* Write content into this buffer */ + int amt, /* Number of bytes to read */ + sqlite3_int64 offset /* Begin reading at this offset */ +){ + ULONG fileLocation = 0L; ULONG got; + os2File *pFile = (os2File*)id; assert( id!=0 ); - SimulateIOError( return SQLITE_IOERR ); - OSTRACE3( "READ %d lock=%d\n", ((os2File*)id)->h, ((os2File*)id)->locktype ); - DosRead( ((os2File*)id)->h, pBuf, amt, &got ); - if (got == (ULONG)amt) - return SQLITE_OK; - else if (got == 0) + SimulateIOError( return SQLITE_IOERR_READ ); + OSTRACE3( "READ %d lock=%d\n", pFile->h, pFile->locktype ); + if( DosSetFilePtr(pFile->h, offset, FILE_BEGIN, &fileLocation) != NO_ERROR ){ + return SQLITE_IOERR; + } + if( DosRead( pFile->h, pBuf, amt, &got ) != NO_ERROR ){ return SQLITE_IOERR_READ; + } + if( got == (ULONG)amt ) + return SQLITE_OK; else { + /* Unread portions of the input buffer must be zero-filled */ memset(&((char*)pBuf)[got], 0, amt-got); return SQLITE_IOERR_SHORT_READ; } @@ -298,101 +134,113 @@ int os2Read( OsFile *id, void *pBuf, int amt ){ ** Write data from a buffer into a file. Return SQLITE_OK on success ** or some other error code on failure. */ -int os2Write( OsFile *id, const void *pBuf, int amt ){ +static int os2Write( + sqlite3_file *id, /* File to write into */ + const void *pBuf, /* The bytes to be written */ + int amt, /* Number of bytes to write */ + sqlite3_int64 offset /* Offset into the file to begin writing at */ +){ + ULONG fileLocation = 0L; APIRET rc = NO_ERROR; ULONG wrote; + os2File *pFile = (os2File*)id; assert( id!=0 ); - SimulateIOError( return SQLITE_IOERR ); + SimulateIOError( return SQLITE_IOERR_WRITE ); SimulateDiskfullError( return SQLITE_FULL ); - OSTRACE3( "WRITE %d lock=%d\n", ((os2File*)id)->h, ((os2File*)id)->locktype ); + OSTRACE3( "WRITE %d lock=%d\n", pFile->h, pFile->locktype ); + if( DosSetFilePtr(pFile->h, offset, FILE_BEGIN, &fileLocation) != NO_ERROR ){ + return SQLITE_IOERR; + } + assert( amt>0 ); while( amt > 0 && - (rc = DosWrite( ((os2File*)id)->h, (PVOID)pBuf, amt, &wrote )) && wrote > 0 ){ - amt -= wrote; - pBuf = &((char*)pBuf)[wrote]; + ( rc = DosWrite( pFile->h, (PVOID)pBuf, amt, &wrote ) ) == NO_ERROR && + wrote > 0 + ){ + amt -= wrote; + pBuf = &((char*)pBuf)[wrote]; } return ( rc != NO_ERROR || amt > (int)wrote ) ? SQLITE_FULL : SQLITE_OK; } /* -** Move the read/write pointer in a file. +** Truncate an open file to a specified size */ -int os2Seek( OsFile *id, i64 offset ){ +static int os2Truncate( sqlite3_file *id, i64 nByte ){ APIRET rc = NO_ERROR; - ULONG filePointer = 0L; - assert( id!=0 ); - rc = DosSetFilePtr( ((os2File*)id)->h, offset, FILE_BEGIN, &filePointer ); - OSTRACE3( "SEEK %d %lld\n", ((os2File*)id)->h, offset ); - return rc == NO_ERROR ? SQLITE_OK : SQLITE_IOERR; + os2File *pFile = (os2File*)id; + OSTRACE3( "TRUNCATE %d %lld\n", pFile->h, nByte ); + SimulateIOError( return SQLITE_IOERR_TRUNCATE ); + rc = DosSetFileSize( pFile->h, nByte ); + return rc == NO_ERROR ? SQLITE_OK : SQLITE_IOERR_TRUNCATE; } +#ifdef SQLITE_TEST +/* +** Count the number of fullsyncs and normal syncs. This is used to test +** that syncs and fullsyncs are occuring at the right times. +*/ +int sqlite3_sync_count = 0; +int sqlite3_fullsync_count = 0; +#endif + /* ** Make sure all writes to a particular file are committed to disk. */ -int os2Sync( OsFile *id, int dataOnly ){ - assert( id!=0 ); - OSTRACE3( "SYNC %d lock=%d\n", ((os2File*)id)->h, ((os2File*)id)->locktype ); - return DosResetBuffer( ((os2File*)id)->h ) == NO_ERROR ? SQLITE_OK : SQLITE_IOERR; -} - -/* -** Sync the directory zDirname. This is a no-op on operating systems other -** than UNIX. -*/ -int sqlite3Os2SyncDirectory( const char *zDirname ){ - SimulateIOError( return SQLITE_IOERR ); - return SQLITE_OK; -} - -/* -** Truncate an open file to a specified size -*/ -int os2Truncate( OsFile *id, i64 nByte ){ - APIRET rc = NO_ERROR; - ULONG upperBits = nByte>>32; - assert( id!=0 ); - OSTRACE3( "TRUNCATE %d %lld\n", ((os2File*)id)->h, nByte ); - SimulateIOError( return SQLITE_IOERR ); - rc = DosSetFilePtr( ((os2File*)id)->h, nByte, FILE_BEGIN, &upperBits ); - if( rc != NO_ERROR ){ - return SQLITE_IOERR; +static int os2Sync( sqlite3_file *id, int flags ){ + os2File *pFile = (os2File*)id; + OSTRACE3( "SYNC %d lock=%d\n", pFile->h, pFile->locktype ); +#ifdef SQLITE_TEST + if( flags & SQLITE_SYNC_FULL){ + sqlite3_fullsync_count++; } - rc = DosSetFilePtr( ((os2File*)id)->h, 0L, FILE_END, &upperBits ); - return rc == NO_ERROR ? SQLITE_OK : SQLITE_IOERR; + sqlite3_sync_count++; +#endif + /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a + ** no-op + */ +#ifdef SQLITE_NO_SYNC + UNUSED_PARAMETER(pFile); + return SQLITE_OK; +#else + return DosResetBuffer( pFile->h ) == NO_ERROR ? SQLITE_OK : SQLITE_IOERR; +#endif } /* ** Determine the current size of a file in bytes */ -int os2FileSize( OsFile *id, i64 *pSize ){ +static int os2FileSize( sqlite3_file *id, sqlite3_int64 *pSize ){ APIRET rc = NO_ERROR; FILESTATUS3 fsts3FileInfo; memset(&fsts3FileInfo, 0, sizeof(fsts3FileInfo)); assert( id!=0 ); - SimulateIOError( return SQLITE_IOERR ); + SimulateIOError( return SQLITE_IOERR_FSTAT ); rc = DosQueryFileInfo( ((os2File*)id)->h, FIL_STANDARD, &fsts3FileInfo, sizeof(FILESTATUS3) ); if( rc == NO_ERROR ){ *pSize = fsts3FileInfo.cbFile; return SQLITE_OK; - } - else{ - return SQLITE_IOERR; + }else{ + return SQLITE_IOERR_FSTAT; } } /* ** Acquire a reader lock. */ -static int getReadLock( os2File *id ){ +static int getReadLock( os2File *pFile ){ FILELOCK LockArea, UnlockArea; + APIRET res; memset(&LockArea, 0, sizeof(LockArea)); memset(&UnlockArea, 0, sizeof(UnlockArea)); LockArea.lOffset = SHARED_FIRST; LockArea.lRange = SHARED_SIZE; UnlockArea.lOffset = 0L; UnlockArea.lRange = 0L; - return DosSetFileLocks( id->h, &UnlockArea, &LockArea, 2000L, 1L ); + res = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, LOCK_TIMEOUT, 1L ); + OSTRACE3( "GETREADLOCK %d res=%d\n", pFile->h, res ); + return res; } /* @@ -401,34 +249,18 @@ static int getReadLock( os2File *id ){ static int unlockReadLock( os2File *id ){ FILELOCK LockArea, UnlockArea; + APIRET res; memset(&LockArea, 0, sizeof(LockArea)); memset(&UnlockArea, 0, sizeof(UnlockArea)); LockArea.lOffset = 0L; LockArea.lRange = 0L; UnlockArea.lOffset = SHARED_FIRST; UnlockArea.lRange = SHARED_SIZE; - return DosSetFileLocks( id->h, &UnlockArea, &LockArea, 2000L, 1L ); + res = DosSetFileLocks( id->h, &UnlockArea, &LockArea, LOCK_TIMEOUT, 1L ); + OSTRACE3( "UNLOCK-READLOCK file handle=%d res=%d?\n", id->h, res ); + return res; } -#ifndef SQLITE_OMIT_PAGER_PRAGMAS -/* -** Check that a given pathname is a directory and is writable -** -*/ -int sqlite3Os2IsDirWritable( char *zDirname ){ - FILESTATUS3 fsts3ConfigInfo; - APIRET rc = NO_ERROR; - memset(&fsts3ConfigInfo, 0, sizeof(fsts3ConfigInfo)); - if( zDirname==0 ) return 0; - if( strlen(zDirname)>CCHMAXPATH ) return 0; - rc = DosQueryPathInfo( (PSZ)zDirname, FIL_STANDARD, &fsts3ConfigInfo, sizeof(FILESTATUS3) ); - if( rc != NO_ERROR ) return 0; - if( (fsts3ConfigInfo.attrFile & FILE_DIRECTORY) != FILE_DIRECTORY ) return 0; - - return 1; -} -#endif /* SQLITE_OMIT_PAGER_PRAGMAS */ - /* ** Lock the file with the lock specified by parameter locktype - one ** of the following: @@ -455,10 +287,10 @@ int sqlite3Os2IsDirWritable( char *zDirname ){ ** It is not possible to lower the locking level one step at a time. You ** must go straight to locking level 0. */ -int os2Lock( OsFile *id, int locktype ){ - APIRET rc = SQLITE_OK; /* Return code from subroutines */ +static int os2Lock( sqlite3_file *id, int locktype ){ + int rc = SQLITE_OK; /* Return code from subroutines */ APIRET res = NO_ERROR; /* Result of an OS/2 lock call */ - int newLocktype; /* Set id->locktype to this value before exiting */ + int newLocktype; /* Set pFile->locktype to this value before exiting */ int gotPendingLock = 0;/* True if we acquired a PENDING lock this time */ FILELOCK LockArea, UnlockArea; @@ -469,10 +301,11 @@ int os2Lock( OsFile *id, int locktype ){ OSTRACE4( "LOCK %d %d was %d\n", pFile->h, locktype, pFile->locktype ); /* If there is already a lock of this type or more restrictive on the - ** OsFile, do nothing. Don't use the end_lock: exit path, as - ** sqlite3OsEnterMutex() hasn't been called yet. + ** os2File, do nothing. Don't use the end_lock: exit path, as + ** sqlite3_mutex_enter() hasn't been called yet. */ if( pFile->locktype>=locktype ){ + OSTRACE3( "LOCK %d %d ok (already held)\n", pFile->h, locktype ); return SQLITE_OK; } @@ -488,59 +321,58 @@ int os2Lock( OsFile *id, int locktype ){ */ newLocktype = pFile->locktype; if( pFile->locktype==NO_LOCK - || (locktype==EXCLUSIVE_LOCK && pFile->locktype==RESERVED_LOCK) + || (locktype==EXCLUSIVE_LOCK && pFile->locktype==RESERVED_LOCK) ){ - int cnt = 3; - LockArea.lOffset = PENDING_BYTE; LockArea.lRange = 1L; UnlockArea.lOffset = 0L; UnlockArea.lRange = 0L; - while( cnt-->0 && (res = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, 2000L, 1L) )!=NO_ERROR ){ - /* Try 3 times to get the pending lock. The pending lock might be - ** held by another reader process who will release it momentarily. - */ - OSTRACE2( "could not get a PENDING lock. cnt=%d\n", cnt ); - DosSleep(1); + /* wait longer than LOCK_TIMEOUT here not to have to try multiple times */ + res = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, 100L, 0L ); + if( res == NO_ERROR ){ + gotPendingLock = 1; + OSTRACE3( "LOCK %d pending lock boolean set. res=%d\n", pFile->h, res ); } - gotPendingLock = res; } /* Acquire a shared lock */ - if( locktype==SHARED_LOCK && res ){ + if( locktype==SHARED_LOCK && res == NO_ERROR ){ assert( pFile->locktype==NO_LOCK ); res = getReadLock(pFile); if( res == NO_ERROR ){ newLocktype = SHARED_LOCK; } + OSTRACE3( "LOCK %d acquire shared lock. res=%d\n", pFile->h, res ); } /* Acquire a RESERVED lock */ - if( locktype==RESERVED_LOCK && res ){ + if( locktype==RESERVED_LOCK && res == NO_ERROR ){ assert( pFile->locktype==SHARED_LOCK ); LockArea.lOffset = RESERVED_BYTE; LockArea.lRange = 1L; UnlockArea.lOffset = 0L; UnlockArea.lRange = 0L; - res = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, 2000L, 1L ); + res = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, LOCK_TIMEOUT, 0L ); if( res == NO_ERROR ){ newLocktype = RESERVED_LOCK; } + OSTRACE3( "LOCK %d acquire reserved lock. res=%d\n", pFile->h, res ); } /* Acquire a PENDING lock */ - if( locktype==EXCLUSIVE_LOCK && res ){ + if( locktype==EXCLUSIVE_LOCK && res == NO_ERROR ){ newLocktype = PENDING_LOCK; gotPendingLock = 0; + OSTRACE2( "LOCK %d acquire pending lock. pending lock boolean unset.\n", pFile->h ); } /* Acquire an EXCLUSIVE lock */ - if( locktype==EXCLUSIVE_LOCK && res ){ + if( locktype==EXCLUSIVE_LOCK && res == NO_ERROR ){ assert( pFile->locktype>=SHARED_LOCK ); res = unlockReadLock(pFile); OSTRACE2( "unreadlock = %d\n", res ); @@ -548,23 +380,27 @@ int os2Lock( OsFile *id, int locktype ){ LockArea.lRange = SHARED_SIZE; UnlockArea.lOffset = 0L; UnlockArea.lRange = 0L; - res = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, 2000L, 1L ); + res = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, LOCK_TIMEOUT, 0L ); if( res == NO_ERROR ){ newLocktype = EXCLUSIVE_LOCK; }else{ - OSTRACE2( "error-code = %d\n", res ); + OSTRACE2( "OS/2 error-code = %d\n", res ); + getReadLock(pFile); } + OSTRACE3( "LOCK %d acquire exclusive lock. res=%d\n", pFile->h, res ); } /* If we are holding a PENDING lock that ought to be released, then ** release it now. */ if( gotPendingLock && locktype==SHARED_LOCK ){ + int r; LockArea.lOffset = 0L; LockArea.lRange = 0L; UnlockArea.lOffset = PENDING_BYTE; UnlockArea.lRange = 1L; - DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, 2000L, 1L ); + r = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, LOCK_TIMEOUT, 0L ); + OSTRACE3( "LOCK %d unlocking pending/is shared. r=%d\n", pFile->h, r ); } /* Update the state of the lock has held in the file descriptor then @@ -574,10 +410,11 @@ int os2Lock( OsFile *id, int locktype ){ rc = SQLITE_OK; }else{ OSTRACE4( "LOCK FAILED %d trying for %d but got %d\n", pFile->h, - locktype, newLocktype ); + locktype, newLocktype ); rc = SQLITE_BUSY; } pFile->locktype = newLocktype; + OSTRACE3( "LOCK %d now %d\n", pFile->h, pFile->locktype ); return rc; } @@ -586,33 +423,39 @@ int os2Lock( OsFile *id, int locktype ){ ** file by this or any other process. If such a lock is held, return ** non-zero, otherwise zero. */ -int os2CheckReservedLock( OsFile *id ){ - APIRET rc = NO_ERROR; +static int os2CheckReservedLock( sqlite3_file *id, int *pOut ){ + int r = 0; os2File *pFile = (os2File*)id; assert( pFile!=0 ); if( pFile->locktype>=RESERVED_LOCK ){ - rc = 1; - OSTRACE3( "TEST WR-LOCK %d %d (local)\n", pFile->h, rc ); + r = 1; + OSTRACE3( "TEST WR-LOCK %d %d (local)\n", pFile->h, r ); }else{ FILELOCK LockArea, UnlockArea; + APIRET rc = NO_ERROR; memset(&LockArea, 0, sizeof(LockArea)); memset(&UnlockArea, 0, sizeof(UnlockArea)); LockArea.lOffset = RESERVED_BYTE; LockArea.lRange = 1L; UnlockArea.lOffset = 0L; UnlockArea.lRange = 0L; - rc = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, 2000L, 1L ); + rc = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, LOCK_TIMEOUT, 0L ); + OSTRACE3( "TEST WR-LOCK %d lock reserved byte rc=%d\n", pFile->h, rc ); if( rc == NO_ERROR ){ + APIRET rcu = NO_ERROR; /* return code for unlocking */ LockArea.lOffset = 0L; LockArea.lRange = 0L; UnlockArea.lOffset = RESERVED_BYTE; UnlockArea.lRange = 1L; - rc = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, 2000L, 1L ); + rcu = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, LOCK_TIMEOUT, 0L ); + OSTRACE3( "TEST WR-LOCK %d unlock reserved byte r=%d\n", pFile->h, rcu ); } - OSTRACE3( "TEST WR-LOCK %d %d (remote)\n", pFile->h, rc ); + r = !(rc == NO_ERROR); + OSTRACE3( "TEST WR-LOCK %d %d (remote)\n", pFile->h, r ); } - return rc; + *pOut = r; + return SQLITE_OK; } /* @@ -626,10 +469,11 @@ int os2CheckReservedLock( OsFile *id ){ ** is NO_LOCK. If the second argument is SHARED_LOCK then this routine ** might return SQLITE_IOERR; */ -int os2Unlock( OsFile *id, int locktype ){ +static int os2Unlock( sqlite3_file *id, int locktype ){ int type; - APIRET rc = SQLITE_OK; os2File *pFile = (os2File*)id; + APIRET rc = SQLITE_OK; + APIRET res = NO_ERROR; FILELOCK LockArea, UnlockArea; memset(&LockArea, 0, sizeof(LockArea)); @@ -643,11 +487,13 @@ int os2Unlock( OsFile *id, int locktype ){ LockArea.lRange = 0L; UnlockArea.lOffset = SHARED_FIRST; UnlockArea.lRange = SHARED_SIZE; - DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, 2000L, 1L ); + res = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, LOCK_TIMEOUT, 0L ); + OSTRACE3( "UNLOCK %d exclusive lock res=%d\n", pFile->h, res ); if( locktype==SHARED_LOCK && getReadLock(pFile) != NO_ERROR ){ /* This should never happen. We should always be able to ** reacquire the read lock */ - rc = SQLITE_IOERR; + OSTRACE3( "UNLOCK %d to %d getReadLock() failed\n", pFile->h, locktype ); + rc = SQLITE_IOERR_UNLOCK; } } if( type>=RESERVED_LOCK ){ @@ -655,75 +501,38 @@ int os2Unlock( OsFile *id, int locktype ){ LockArea.lRange = 0L; UnlockArea.lOffset = RESERVED_BYTE; UnlockArea.lRange = 1L; - DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, 2000L, 1L ); + res = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, LOCK_TIMEOUT, 0L ); + OSTRACE3( "UNLOCK %d reserved res=%d\n", pFile->h, res ); } if( locktype==NO_LOCK && type>=SHARED_LOCK ){ - unlockReadLock(pFile); + res = unlockReadLock(pFile); + OSTRACE5( "UNLOCK %d is %d want %d res=%d\n", pFile->h, type, locktype, res ); } if( type>=PENDING_LOCK ){ LockArea.lOffset = 0L; LockArea.lRange = 0L; UnlockArea.lOffset = PENDING_BYTE; UnlockArea.lRange = 1L; - DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, 2000L, 1L ); + res = DosSetFileLocks( pFile->h, &UnlockArea, &LockArea, LOCK_TIMEOUT, 0L ); + OSTRACE3( "UNLOCK %d pending res=%d\n", pFile->h, res ); } pFile->locktype = locktype; + OSTRACE3( "UNLOCK %d now %d\n", pFile->h, pFile->locktype ); return rc; } /* -** Turn a relative pathname into a full pathname. Return a pointer -** to the full pathname stored in space obtained from sqliteMalloc(). -** The calling function is responsible for freeing this space once it -** is no longer needed. +** Control and query of the open file handle. */ -char *sqlite3Os2FullPathname( const char *zRelative ){ - char *zFull = 0; - if( strchr(zRelative, ':') ){ - sqlite3SetString( &zFull, zRelative, (char*)0 ); - }else{ - ULONG ulDriveNum = 0; - ULONG ulDriveMap = 0; - ULONG cbzBufLen = SQLITE_TEMPNAME_SIZE; - char zDrive[2]; - char *zBuff; - - zBuff = sqliteMalloc( cbzBufLen ); - if( zBuff != 0 ){ - DosQueryCurrentDisk( &ulDriveNum, &ulDriveMap ); - if( DosQueryCurrentDir( ulDriveNum, (PBYTE)zBuff, &cbzBufLen ) == NO_ERROR ){ - sprintf( zDrive, "%c", (char)('A' + ulDriveNum - 1) ); - sqlite3SetString( &zFull, zDrive, ":\\", zBuff, - "\\", zRelative, (char*)0 ); - } - sqliteFree( zBuff ); +static int os2FileControl(sqlite3_file *id, int op, void *pArg){ + switch( op ){ + case SQLITE_FCNTL_LOCKSTATE: { + *(int*)pArg = ((os2File*)id)->locktype; + OSTRACE3( "FCNTL_LOCKSTATE %d lock=%d\n", ((os2File*)id)->h, ((os2File*)id)->locktype ); + return SQLITE_OK; } } - return zFull; -} - -/* -** The fullSync option is meaningless on os2, or correct me if I'm wrong. This is a no-op. -** From os_unix.c: Change the value of the fullsync flag in the given file descriptor. -** From os_unix.c: ((unixFile*)id)->fullSync = v; -*/ -static void os2SetFullSync( OsFile *id, int v ){ - return; -} - -/* -** Return the underlying file handle for an OsFile -*/ -static int os2FileHandle( OsFile *id ){ - return (int)((os2File*)id)->h; -} - -/* -** Return an integer that indices the type of lock currently held -** by this handle. (Used for testing and analysis only.) -*/ -static int os2LockState( OsFile *id ){ - return ((os2File*)id)->locktype; + return SQLITE_ERROR; } /* @@ -733,81 +542,407 @@ static int os2LockState( OsFile *id ){ ** ** SQLite code assumes this function cannot fail. It also assumes that ** if two files are created in the same file-system directory (i.e. -** a database and it's journal file) that the sector size will be the +** a database and its journal file) that the sector size will be the ** same for both. */ -static int os2SectorSize(OsFile *id){ +static int os2SectorSize(sqlite3_file *id){ return SQLITE_DEFAULT_SECTOR_SIZE; } /* -** This vector defines all the methods that can operate on an OsFile -** for os2. +** Return a vector of device characteristics. */ -static const IoMethod sqlite3Os2IoMethod = { +static int os2DeviceCharacteristics(sqlite3_file *id){ + return 0; +} + + +/* +** Character set conversion objects used by conversion routines. +*/ +static UconvObject ucUtf8 = NULL; /* convert between UTF-8 and UCS-2 */ +static UconvObject uclCp = NULL; /* convert between local codepage and UCS-2 */ + +/* +** Helper function to initialize the conversion objects from and to UTF-8. +*/ +static void initUconvObjects( void ){ + if( UniCreateUconvObject( UTF_8, &ucUtf8 ) != ULS_SUCCESS ) + ucUtf8 = NULL; + if ( UniCreateUconvObject( (UniChar *)L"@path=yes", &uclCp ) != ULS_SUCCESS ) + uclCp = NULL; +} + +/* +** Helper function to free the conversion objects from and to UTF-8. +*/ +static void freeUconvObjects( void ){ + if ( ucUtf8 ) + UniFreeUconvObject( ucUtf8 ); + if ( uclCp ) + UniFreeUconvObject( uclCp ); + ucUtf8 = NULL; + uclCp = NULL; +} + +/* +** Helper function to convert UTF-8 filenames to local OS/2 codepage. +** The two-step process: first convert the incoming UTF-8 string +** into UCS-2 and then from UCS-2 to the current codepage. +** The returned char pointer has to be freed. +*/ +static char *convertUtf8PathToCp( const char *in ){ + UniChar tempPath[CCHMAXPATH]; + char *out = (char *)calloc( CCHMAXPATH, 1 ); + + if( !out ) + return NULL; + + if( !ucUtf8 || !uclCp ) + initUconvObjects(); + + /* determine string for the conversion of UTF-8 which is CP1208 */ + if( UniStrToUcs( ucUtf8, tempPath, (char *)in, CCHMAXPATH ) != ULS_SUCCESS ) + return out; /* if conversion fails, return the empty string */ + + /* conversion for current codepage which can be used for paths */ + UniStrFromUcs( uclCp, out, tempPath, CCHMAXPATH ); + + return out; +} + +/* +** Helper function to convert filenames from local codepage to UTF-8. +** The two-step process: first convert the incoming codepage-specific +** string into UCS-2 and then from UCS-2 to the codepage of UTF-8. +** The returned char pointer has to be freed. +** +** This function is non-static to be able to use this in shell.c and +** similar applications that take command line arguments. +*/ +char *convertCpPathToUtf8( const char *in ){ + UniChar tempPath[CCHMAXPATH]; + char *out = (char *)calloc( CCHMAXPATH, 1 ); + + if( !out ) + return NULL; + + if( !ucUtf8 || !uclCp ) + initUconvObjects(); + + /* conversion for current codepage which can be used for paths */ + if( UniStrToUcs( uclCp, tempPath, (char *)in, CCHMAXPATH ) != ULS_SUCCESS ) + return out; /* if conversion fails, return the empty string */ + + /* determine string for the conversion of UTF-8 which is CP1208 */ + UniStrFromUcs( ucUtf8, out, tempPath, CCHMAXPATH ); + + return out; +} + +/* +** This vector defines all the methods that can operate on an +** sqlite3_file for os2. +*/ +static const sqlite3_io_methods os2IoMethod = { + 1, /* iVersion */ os2Close, - os2OpenDirectory, os2Read, os2Write, - os2Seek, os2Truncate, os2Sync, - os2SetFullSync, - os2FileHandle, os2FileSize, os2Lock, os2Unlock, - os2LockState, os2CheckReservedLock, + os2FileControl, os2SectorSize, + os2DeviceCharacteristics }; +/*************************************************************************** +** Here ends the I/O methods that form the sqlite3_io_methods object. +** +** The next block of code implements the VFS methods. +****************************************************************************/ + /* -** Allocate memory for an OsFile. Initialize the new OsFile -** to the value given in pInit and return a pointer to the new -** OsFile. If we run out of memory, close the file and return NULL. +** Create a temporary file name in zBuf. zBuf must be big enough to +** hold at pVfs->mxPathname characters. */ -int allocateOs2File( os2File *pInit, OsFile **pld ){ - os2File *pNew; - pNew = sqliteMalloc( sizeof(*pNew) ); - if( pNew==0 ){ - DosClose( pInit->h ); - *pld = 0; - return SQLITE_NOMEM; +static int getTempname(int nBuf, char *zBuf ){ + static const unsigned char zChars[] = + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789"; + int i, j; + char zTempPathBuf[3]; + PSZ zTempPath = (PSZ)&zTempPathBuf; + if( sqlite3_temp_directory ){ + zTempPath = sqlite3_temp_directory; }else{ - *pNew = *pInit; - pNew->pMethod = &sqlite3Os2IoMethod; - pNew->locktype = NO_LOCK; - *pld = (OsFile*)pNew; - OpenCounter(+1); - return SQLITE_OK; + if( DosScanEnv( (PSZ)"TEMP", &zTempPath ) ){ + if( DosScanEnv( (PSZ)"TMP", &zTempPath ) ){ + if( DosScanEnv( (PSZ)"TMPDIR", &zTempPath ) ){ + ULONG ulDriveNum = 0, ulDriveMap = 0; + DosQueryCurrentDisk( &ulDriveNum, &ulDriveMap ); + sprintf( (char*)zTempPath, "%c:", (char)( 'A' + ulDriveNum - 1 ) ); + } + } + } } + /* Strip off a trailing slashes or backslashes, otherwise we would get * + * multiple (back)slashes which causes DosOpen() to fail. * + * Trailing spaces are not allowed, either. */ + j = sqlite3Strlen30(zTempPath); + while( j > 0 && ( zTempPath[j-1] == '\\' || zTempPath[j-1] == '/' + || zTempPath[j-1] == ' ' ) ){ + j--; + } + zTempPath[j] = '\0'; + if( !sqlite3_temp_directory ){ + char *zTempPathUTF = convertCpPathToUtf8( zTempPath ); + sqlite3_snprintf( nBuf-30, zBuf, + "%s\\"SQLITE_TEMP_FILE_PREFIX, zTempPathUTF ); + free( zTempPathUTF ); + }else{ + sqlite3_snprintf( nBuf-30, zBuf, + "%s\\"SQLITE_TEMP_FILE_PREFIX, zTempPath ); + } + j = sqlite3Strlen30( zBuf ); + sqlite3_randomness( 20, &zBuf[j] ); + for( i = 0; i < 20; i++, j++ ){ + zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; + } + zBuf[j] = 0; + OSTRACE2( "TEMP FILENAME: %s\n", zBuf ); + return SQLITE_OK; +} + + +/* +** Turn a relative pathname into a full pathname. Write the full +** pathname into zFull[]. zFull[] will be at least pVfs->mxPathname +** bytes in size. +*/ +static int os2FullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zRelative, /* Possibly relative input path */ + int nFull, /* Size of output buffer in bytes */ + char *zFull /* Output buffer */ +){ + char *zRelativeCp = convertUtf8PathToCp( zRelative ); + char zFullCp[CCHMAXPATH] = "\0"; + char *zFullUTF; + APIRET rc = DosQueryPathInfo( zRelativeCp, FIL_QUERYFULLNAME, zFullCp, + CCHMAXPATH ); + free( zRelativeCp ); + zFullUTF = convertCpPathToUtf8( zFullCp ); + sqlite3_snprintf( nFull, zFull, zFullUTF ); + free( zFullUTF ); + return rc == NO_ERROR ? SQLITE_OK : SQLITE_IOERR; +} + + +/* +** Open a file. +*/ +static int os2Open( + sqlite3_vfs *pVfs, /* Not used */ + const char *zName, /* Name of the file */ + sqlite3_file *id, /* Write the SQLite file handle here */ + int flags, /* Open mode flags */ + int *pOutFlags /* Status return flags */ +){ + HFILE h; + ULONG ulFileAttribute = FILE_NORMAL; + ULONG ulOpenFlags = 0; + ULONG ulOpenMode = 0; + os2File *pFile = (os2File*)id; + APIRET rc = NO_ERROR; + ULONG ulAction; + char *zNameCp; + char zTmpname[CCHMAXPATH+1]; /* Buffer to hold name of temp file */ + + /* If the second argument to this function is NULL, generate a + ** temporary file name to use + */ + if( !zName ){ + int rc = getTempname(CCHMAXPATH+1, zTmpname); + if( rc!=SQLITE_OK ){ + return rc; + } + zName = zTmpname; + } + + + memset( pFile, 0, sizeof(*pFile) ); + + OSTRACE2( "OPEN want %d\n", flags ); + + if( flags & SQLITE_OPEN_READWRITE ){ + ulOpenMode |= OPEN_ACCESS_READWRITE; + OSTRACE1( "OPEN read/write\n" ); + }else{ + ulOpenMode |= OPEN_ACCESS_READONLY; + OSTRACE1( "OPEN read only\n" ); + } + + if( flags & SQLITE_OPEN_CREATE ){ + ulOpenFlags |= OPEN_ACTION_OPEN_IF_EXISTS | OPEN_ACTION_CREATE_IF_NEW; + OSTRACE1( "OPEN open new/create\n" ); + }else{ + ulOpenFlags |= OPEN_ACTION_OPEN_IF_EXISTS | OPEN_ACTION_FAIL_IF_NEW; + OSTRACE1( "OPEN open existing\n" ); + } + + if( flags & SQLITE_OPEN_MAIN_DB ){ + ulOpenMode |= OPEN_SHARE_DENYNONE; + OSTRACE1( "OPEN share read/write\n" ); + }else{ + ulOpenMode |= OPEN_SHARE_DENYWRITE; + OSTRACE1( "OPEN share read only\n" ); + } + + if( flags & SQLITE_OPEN_DELETEONCLOSE ){ + char pathUtf8[CCHMAXPATH]; +#ifdef NDEBUG /* when debugging we want to make sure it is deleted */ + ulFileAttribute = FILE_HIDDEN; +#endif + os2FullPathname( pVfs, zName, CCHMAXPATH, pathUtf8 ); + pFile->pathToDel = convertUtf8PathToCp( pathUtf8 ); + OSTRACE1( "OPEN hidden/delete on close file attributes\n" ); + }else{ + pFile->pathToDel = NULL; + OSTRACE1( "OPEN normal file attribute\n" ); + } + + /* always open in random access mode for possibly better speed */ + ulOpenMode |= OPEN_FLAGS_RANDOM; + ulOpenMode |= OPEN_FLAGS_FAIL_ON_ERROR; + ulOpenMode |= OPEN_FLAGS_NOINHERIT; + + zNameCp = convertUtf8PathToCp( zName ); + rc = DosOpen( (PSZ)zNameCp, + &h, + &ulAction, + 0L, + ulFileAttribute, + ulOpenFlags, + ulOpenMode, + (PEAOP2)NULL ); + free( zNameCp ); + if( rc != NO_ERROR ){ + OSTRACE7( "OPEN Invalid handle rc=%d: zName=%s, ulAction=%#lx, ulAttr=%#lx, ulFlags=%#lx, ulMode=%#lx\n", + rc, zName, ulAction, ulFileAttribute, ulOpenFlags, ulOpenMode ); + if( pFile->pathToDel ) + free( pFile->pathToDel ); + pFile->pathToDel = NULL; + if( flags & SQLITE_OPEN_READWRITE ){ + OSTRACE2( "OPEN %d Invalid handle\n", ((flags | SQLITE_OPEN_READONLY) & ~SQLITE_OPEN_READWRITE) ); + return os2Open( pVfs, zName, id, + ((flags | SQLITE_OPEN_READONLY) & ~SQLITE_OPEN_READWRITE), + pOutFlags ); + }else{ + return SQLITE_CANTOPEN; + } + } + + if( pOutFlags ){ + *pOutFlags = flags & SQLITE_OPEN_READWRITE ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READONLY; + } + + pFile->pMethod = &os2IoMethod; + pFile->h = h; + OpenCounter(+1); + OSTRACE3( "OPEN %d pOutFlags=%d\n", pFile->h, pOutFlags ); + return SQLITE_OK; +} + +/* +** Delete the named file. +*/ +static int os2Delete( + sqlite3_vfs *pVfs, /* Not used on os2 */ + const char *zFilename, /* Name of file to delete */ + int syncDir /* Not used on os2 */ +){ + APIRET rc = NO_ERROR; + char *zFilenameCp = convertUtf8PathToCp( zFilename ); + SimulateIOError( return SQLITE_IOERR_DELETE ); + rc = DosDelete( (PSZ)zFilenameCp ); + free( zFilenameCp ); + OSTRACE2( "DELETE \"%s\"\n", zFilename ); + return rc == NO_ERROR ? SQLITE_OK : SQLITE_IOERR_DELETE; +} + +/* +** Check the existance and status of a file. +*/ +static int os2Access( + sqlite3_vfs *pVfs, /* Not used on os2 */ + const char *zFilename, /* Name of file to check */ + int flags, /* Type of test to make on this file */ + int *pOut /* Write results here */ +){ + FILESTATUS3 fsts3ConfigInfo; + APIRET rc = NO_ERROR; + char *zFilenameCp = convertUtf8PathToCp( zFilename ); + + memset( &fsts3ConfigInfo, 0, sizeof(fsts3ConfigInfo) ); + rc = DosQueryPathInfo( (PSZ)zFilenameCp, FIL_STANDARD, + &fsts3ConfigInfo, sizeof(FILESTATUS3) ); + free( zFilenameCp ); + OSTRACE4( "ACCESS fsts3ConfigInfo.attrFile=%d flags=%d rc=%d\n", + fsts3ConfigInfo.attrFile, flags, rc ); + switch( flags ){ + case SQLITE_ACCESS_READ: + case SQLITE_ACCESS_EXISTS: + rc = (rc == NO_ERROR); + OSTRACE3( "ACCESS %s access of read and exists rc=%d\n", zFilename, rc ); + break; + case SQLITE_ACCESS_READWRITE: + rc = (rc == NO_ERROR) && ( (fsts3ConfigInfo.attrFile & FILE_READONLY) == 0 ); + OSTRACE3( "ACCESS %s access of read/write rc=%d\n", zFilename, rc ); + break; + default: + assert( !"Invalid flags argument" ); + } + *pOut = rc; + return SQLITE_OK; } -#endif /* SQLITE_OMIT_DISKIO */ -/*************************************************************************** -** Everything above deals with file I/O. Everything that follows deals -** with other miscellanous aspects of the operating system interface -****************************************************************************/ #ifndef SQLITE_OMIT_LOAD_EXTENSION /* ** Interfaces for opening a shared library, finding entry points ** within the shared library, and closing the shared library. */ -void *sqlite3Os2Dlopen(const char *zFilename){ +/* +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. +*/ +static void *os2DlOpen(sqlite3_vfs *pVfs, const char *zFilename){ UCHAR loadErr[256]; HMODULE hmod; APIRET rc; - rc = DosLoadModule((PSZ)loadErr, sizeof(loadErr), zFilename, &hmod); - if (rc != NO_ERROR) return 0; - return (void*)hmod; + char *zFilenameCp = convertUtf8PathToCp(zFilename); + rc = DosLoadModule((PSZ)loadErr, sizeof(loadErr), zFilenameCp, &hmod); + free(zFilenameCp); + return rc != NO_ERROR ? 0 : (void*)hmod; } -void *sqlite3Os2Dlsym(void *pHandle, const char *zSymbol){ +/* +** A no-op since the error code is returned on the DosLoadModule call. +** os2Dlopen returns zero if DosLoadModule is not successful. +*/ +static void os2DlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ +/* no-op */ +} +static void *os2DlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol){ PFN pfn; APIRET rc; rc = DosQueryProcAddr((HMODULE)pHandle, 0L, zSymbol, &pfn); - if (rc != NO_ERROR) { + if( rc != NO_ERROR ){ /* if the symbol itself was not found, search again for the same * symbol with an extra underscore, that might be needed depending * on the calling convention */ @@ -815,100 +950,92 @@ void *sqlite3Os2Dlsym(void *pHandle, const char *zSymbol){ strncat(_zSymbol, zSymbol, 255); rc = DosQueryProcAddr((HMODULE)pHandle, 0L, _zSymbol, &pfn); } - if (rc != NO_ERROR) return 0; - return (void *)pfn; + return rc != NO_ERROR ? 0 : (void*)pfn; } -int sqlite3Os2Dlclose(void *pHandle){ - return DosFreeModule((HMODULE)pHandle); +static void os2DlClose(sqlite3_vfs *pVfs, void *pHandle){ + DosFreeModule((HMODULE)pHandle); } -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ + #define os2DlOpen 0 + #define os2DlError 0 + #define os2DlSym 0 + #define os2DlClose 0 +#endif /* -** Get information to seed the random number generator. The seed -** is written into the buffer zBuf[256]. The calling function must -** supply a sufficiently large buffer. +** Write up to nBuf bytes of randomness into zBuf. */ -int sqlite3Os2RandomSeed( char *zBuf ){ - /* We have to initialize zBuf to prevent valgrind from reporting - ** errors. The reports issued by valgrind are incorrect - we would - ** prefer that the randomness be increased by making use of the - ** uninitialized space in zBuf - but valgrind errors tend to worry - ** some users. Rather than argue, it seems easier just to initialize - ** the whole array and silence valgrind, even if that means less randomness - ** in the random seed. - ** - ** When testing, initializing zBuf[] to zero is all we do. That means - ** that we always use the same random number sequence. This makes the - ** tests repeatable. - */ - memset( zBuf, 0, 256 ); - DosGetDateTime( (PDATETIME)zBuf ); - return SQLITE_OK; +static int os2Randomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf ){ + int n = 0; +#if defined(SQLITE_TEST) + n = nBuf; + memset(zBuf, 0, nBuf); +#else + int sizeofULong = sizeof(ULONG); + if( (int)sizeof(DATETIME) <= nBuf - n ){ + DATETIME x; + DosGetDateTime(&x); + memcpy(&zBuf[n], &x, sizeof(x)); + n += sizeof(x); + } + + if( sizeofULong <= nBuf - n ){ + PPIB ppib; + DosGetInfoBlocks(NULL, &ppib); + memcpy(&zBuf[n], &ppib->pib_ulpid, sizeofULong); + n += sizeofULong; + } + + if( sizeofULong <= nBuf - n ){ + PTIB ptib; + DosGetInfoBlocks(&ptib, NULL); + memcpy(&zBuf[n], &ptib->tib_ptib2->tib2_ultid, sizeofULong); + n += sizeofULong; + } + + /* if we still haven't filled the buffer yet the following will */ + /* grab everything once instead of making several calls for a single item */ + if( sizeofULong <= nBuf - n ){ + ULONG ulSysInfo[QSV_MAX]; + DosQuerySysInfo(1L, QSV_MAX, ulSysInfo, sizeofULong * QSV_MAX); + + memcpy(&zBuf[n], &ulSysInfo[QSV_MS_COUNT - 1], sizeofULong); + n += sizeofULong; + + if( sizeofULong <= nBuf - n ){ + memcpy(&zBuf[n], &ulSysInfo[QSV_TIMER_INTERVAL - 1], sizeofULong); + n += sizeofULong; + } + if( sizeofULong <= nBuf - n ){ + memcpy(&zBuf[n], &ulSysInfo[QSV_TIME_LOW - 1], sizeofULong); + n += sizeofULong; + } + if( sizeofULong <= nBuf - n ){ + memcpy(&zBuf[n], &ulSysInfo[QSV_TIME_HIGH - 1], sizeofULong); + n += sizeofULong; + } + if( sizeofULong <= nBuf - n ){ + memcpy(&zBuf[n], &ulSysInfo[QSV_TOTAVAILMEM - 1], sizeofULong); + n += sizeofULong; + } + } +#endif + + return n; } /* ** Sleep for a little while. Return the amount of time slept. +** The argument is the number of microseconds we want to sleep. +** The return value is the number of microseconds of sleep actually +** requested from the underlying operating system, a number which +** might be greater than or equal to the argument, but not less +** than the argument. */ -int sqlite3Os2Sleep( int ms ){ - DosSleep( ms ); - return ms; -} - -/* -** Static variables used for thread synchronization -*/ -static int inMutex = 0; -#ifdef SQLITE_OS2_THREADS -static ULONG mutexOwner; -#endif - -/* -** The following pair of routines implement mutual exclusion for -** multi-threaded processes. Only a single thread is allowed to -** executed code that is surrounded by EnterMutex() and LeaveMutex(). -** -** SQLite uses only a single Mutex. There is not much critical -** code and what little there is executes quickly and without blocking. -*/ -void sqlite3Os2EnterMutex(){ -#ifdef SQLITE_OS2_THREADS - PTIB ptib; - DosEnterCritSec(); - DosGetInfoBlocks( &ptib, NULL ); - mutexOwner = ptib->tib_ptib2->tib2_ultid; -#endif - assert( !inMutex ); - inMutex = 1; -} -void sqlite3Os2LeaveMutex(){ -#ifdef SQLITE_OS2_THREADS - PTIB ptib; -#endif - assert( inMutex ); - inMutex = 0; -#ifdef SQLITE_OS2_THREADS - DosGetInfoBlocks( &ptib, NULL ); - assert( mutexOwner == ptib->tib_ptib2->tib2_ultid ); - DosExitCritSec(); -#endif -} - -/* -** Return TRUE if the mutex is currently held. -** -** If the thisThreadOnly parameter is true, return true if and only if the -** calling thread holds the mutex. If the parameter is false, return -** true if any thread holds the mutex. -*/ -int sqlite3Os2InMutex( int thisThreadOnly ){ -#ifdef SQLITE_OS2_THREADS - PTIB ptib; - DosGetInfoBlocks( &ptib, NULL ); - return inMutex>0 && (thisThreadOnly==0 || mutexOwner==ptib->tib_ptib2->tib2_ultid); -#else - return inMutex>0; -#endif +static int os2Sleep( sqlite3_vfs *pVfs, int microsec ){ + DosSleep( (microsec/1000) ); + return microsec; } /* @@ -924,14 +1051,15 @@ int sqlite3_current_time = 0; ** current time and date as a Julian Day number into *prNow and ** return 0. Return 1 if the time and date cannot be found. */ -int sqlite3Os2CurrentTime( double *prNow ){ +int os2CurrentTime( sqlite3_vfs *pVfs, double *prNow ){ double now; - USHORT second, minute, hour, + SHORT minute; /* needs to be able to cope with negative timezone offset */ + USHORT second, hour, day, month, year; DATETIME dt; DosGetDateTime( &dt ); second = (USHORT)dt.seconds; - minute = (USHORT)dt.minutes + dt.timezone; + minute = (SHORT)dt.minutes + dt.timezone; hour = (USHORT)dt.hours; day = (USHORT)dt.day; month = (USHORT)dt.month; @@ -958,69 +1086,42 @@ int sqlite3Os2CurrentTime( double *prNow ){ return 0; } -/* -** Remember the number of thread-specific-data blocks allocated. -** Use this to verify that we are not leaking thread-specific-data. -** Ticket #1601 -*/ -#ifdef SQLITE_TEST -int sqlite3_tsd_count = 0; -# define TSD_COUNTER_INCR InterlockedIncrement( &sqlite3_tsd_count ) -# define TSD_COUNTER_DECR InterlockedDecrement( &sqlite3_tsd_count ) -#else -# define TSD_COUNTER_INCR /* no-op */ -# define TSD_COUNTER_DECR /* no-op */ -#endif - -/* -** If called with allocateFlag>1, then return a pointer to thread -** specific data for the current thread. Allocate and zero the -** thread-specific data if it does not already exist necessary. -** -** If called with allocateFlag==0, then check the current thread -** specific data. Return it if it exists. If it does not exist, -** then return NULL. -** -** If called with allocateFlag<0, check to see if the thread specific -** data is allocated and is all zero. If it is then deallocate it. -** Return a pointer to the thread specific data or NULL if it is -** unallocated or gets deallocated. -*/ -ThreadData *sqlite3Os2ThreadSpecificData( int allocateFlag ){ - static ThreadData **s_ppTsd = NULL; - static const ThreadData zeroData = {0, 0, 0}; - ThreadData *pTsd; - - if( !s_ppTsd ){ - sqlite3OsEnterMutex(); - if( !s_ppTsd ){ - PULONG pul; - APIRET rc = DosAllocThreadLocalMemory(1, &pul); - if( rc != NO_ERROR ){ - sqlite3OsLeaveMutex(); - return 0; - } - s_ppTsd = (ThreadData **)pul; - } - sqlite3OsLeaveMutex(); - } - pTsd = *s_ppTsd; - if( allocateFlag>0 ){ - if( !pTsd ){ - pTsd = sqlite3OsMalloc( sizeof(zeroData) ); - if( pTsd ){ - *pTsd = zeroData; - *s_ppTsd = pTsd; - TSD_COUNTER_INCR; - } - } - }else if( pTsd!=0 && allocateFlag<0 - && memcmp( pTsd, &zeroData, sizeof(ThreadData) )==0 ){ - sqlite3OsFree(pTsd); - *s_ppTsd = NULL; - TSD_COUNTER_DECR; - pTsd = 0; - } - return pTsd; +static int os2GetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + return 0; } -#endif /* OS_OS2 */ + +/* +** Initialize and deinitialize the operating system interface. +*/ +int sqlite3_os_init(void){ + static sqlite3_vfs os2Vfs = { + 1, /* iVersion */ + sizeof(os2File), /* szOsFile */ + CCHMAXPATH, /* mxPathname */ + 0, /* pNext */ + "os2", /* zName */ + 0, /* pAppData */ + + os2Open, /* xOpen */ + os2Delete, /* xDelete */ + os2Access, /* xAccess */ + os2FullPathname, /* xFullPathname */ + os2DlOpen, /* xDlOpen */ + os2DlError, /* xDlError */ + os2DlSym, /* xDlSym */ + os2DlClose, /* xDlClose */ + os2Randomness, /* xRandomness */ + os2Sleep, /* xSleep */ + os2CurrentTime, /* xCurrentTime */ + os2GetLastError /* xGetLastError */ + }; + sqlite3_vfs_register(&os2Vfs, 1); + initUconvObjects(); + return SQLITE_OK; +} +int sqlite3_os_end(void){ + freeUconvObjects(); + return SQLITE_OK; +} + +#endif /* SQLITE_OS_OS2 */ diff --git a/os_unix.c b/os_unix.c index 9141891..b7e6f35 100644 --- a/os_unix.c +++ b/os_unix.c @@ -10,13 +10,80 @@ ** ****************************************************************************** ** -** This file contains code that is specific to Unix systems. +** This file contains the VFS implementation for unix-like operating systems +** include Linux, MacOSX, *BSD, QNX, VxWorks, AIX, HPUX, and others. +** +** There are actually several different VFS implementations in this file. +** The differences are in the way that file locking is done. The default +** implementation uses Posix Advisory Locks. Alternative implementations +** use flock(), dot-files, various proprietary locking schemas, or simply +** skip locking all together. +** +** This source file is organized into divisions where the logic for various +** subfunctions is contained within the appropriate division. PLEASE +** KEEP THE STRUCTURE OF THIS FILE INTACT. New code should be placed +** in the correct division and should be clearly labeled. +** +** The layout of divisions is as follows: +** +** * General-purpose declarations and utility functions. +** * Unique file ID logic used by VxWorks. +** * Various locking primitive implementations (all except proxy locking): +** + for Posix Advisory Locks +** + for no-op locks +** + for dot-file locks +** + for flock() locking +** + for named semaphore locks (VxWorks only) +** + for AFP filesystem locks (MacOSX only) +** * sqlite3_file methods not associated with locking. +** * Definitions of sqlite3_io_methods objects for all locking +** methods plus "finder" functions for each locking method. +** * sqlite3_vfs method implementations. +** * Locking primitives for the proxy uber-locking-method. (MacOSX only) +** * Definitions of sqlite3_vfs objects for all locking methods +** plus implementations of sqlite3_os_init() and sqlite3_os_end(). +** +** $Id: os_unix.c,v 1.237 2009/01/15 04:30:03 drh Exp $ */ #include "sqliteInt.h" -#include "os.h" -#if OS_UNIX /* This file is used on unix only */ +#if SQLITE_OS_UNIX /* This file is used on unix only */ -/* #define SQLITE_ENABLE_LOCKING_STYLE 0 */ +/* +** There are various methods for file locking used for concurrency +** control: +** +** 1. POSIX locking (the default), +** 2. No locking, +** 3. Dot-file locking, +** 4. flock() locking, +** 5. AFP locking (OSX only), +** 6. Named POSIX semaphores (VXWorks only), +** 7. proxy locking. (OSX only) +** +** Styles 4, 5, and 7 are only available of SQLITE_ENABLE_LOCKING_STYLE +** is defined to 1. The SQLITE_ENABLE_LOCKING_STYLE also enables automatic +** selection of the appropriate locking style based on the filesystem +** where the database is located. +*/ +#if !defined(SQLITE_ENABLE_LOCKING_STYLE) +# if defined(__APPLE__) +# define SQLITE_ENABLE_LOCKING_STYLE 1 +# else +# define SQLITE_ENABLE_LOCKING_STYLE 0 +# endif +#endif + +/* +** Define the OS_VXWORKS pre-processor macro to 1 if building on +** vxworks, or 0 otherwise. +*/ +#ifndef OS_VXWORKS +# if defined(__RTP__) || defined(_WRS_KERNEL) +# define OS_VXWORKS 1 +# else +# define OS_VXWORKS 0 +# endif +#endif /* ** These #defines should enable >2GB file support on Posix if the @@ -30,6 +97,11 @@ ** without this option, LFS is enable. But LFS does not exist in the kernel ** in RedHat 6.0, so the code won't work. Hence, for maximum binary ** portability you should omit LFS. +** +** The previous paragraph was written in 2005. (This paragraph is written +** on 2008-11-28.) These days, all Linux kernels support large files, so +** you should probably leave LFS enabled. But some embedded platforms might +** lack LFS in which case the SQLITE_DISABLE_LFS macro might still be useful. */ #ifndef SQLITE_DISABLE_LFS # define _LARGE_FILE 1 @@ -49,20 +121,24 @@ #include #include #include -#ifdef SQLITE_ENABLE_LOCKING_STYLE -#include -#include -#include + +#if SQLITE_ENABLE_LOCKING_STYLE +# include +# if OS_VXWORKS +# include +# include +# else +# include +# include +# include +# endif #endif /* SQLITE_ENABLE_LOCKING_STYLE */ /* ** If we are to be thread-safe, include the pthreads header and define ** the SQLITE_UNIX_THREADS macro. */ -#ifndef THREADSAFE -# define THREADSAFE 1 -#endif -#if THREADSAFE +#if SQLITE_THREADSAFE # include # define SQLITE_UNIX_THREADS 1 #endif @@ -74,61 +150,72 @@ # define SQLITE_DEFAULT_FILE_PERMISSIONS 0644 #endif +/* + ** Default permissions when creating auto proxy dir + */ +#ifndef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS +# define SQLITE_DEFAULT_PROXYDIR_PERMISSIONS 0755 +#endif + +/* +** Maximum supported path-length. +*/ +#define MAX_PATHNAME 512 + +/* +** Only set the lastErrno if the error code is a real error and not +** a normal expected return code of SQLITE_BUSY or SQLITE_OK +*/ +#define IS_LOCK_ERROR(x) ((x != SQLITE_OK) && (x != SQLITE_BUSY)) /* -** The unixFile structure is subclass of OsFile specific for the unix -** protability layer. +** The unixFile structure is subclass of sqlite3_file specific to the unix +** VFS implementations. */ typedef struct unixFile unixFile; struct unixFile { - IoMethod const *pMethod; /* Always the first entry */ - struct openCnt *pOpen; /* Info about all open fd's on this inode */ - struct lockInfo *pLock; /* Info about locks on this inode */ -#ifdef SQLITE_ENABLE_LOCKING_STYLE - void *lockingContext; /* Locking style specific state */ -#endif /* SQLITE_ENABLE_LOCKING_STYLE */ - int h; /* The file descriptor */ - unsigned char locktype; /* The type of lock held on this fd */ - unsigned char isOpen; /* True if needs to be closed */ - unsigned char fullSync; /* Use F_FULLSYNC if available */ - int dirfd; /* File descriptor for the directory */ - i64 offset; /* Seek offset */ -#ifdef SQLITE_UNIX_THREADS - pthread_t tid; /* The thread that "owns" this OsFile */ + sqlite3_io_methods const *pMethod; /* Always the first entry */ + struct unixOpenCnt *pOpen; /* Info about all open fd's on this inode */ + struct unixLockInfo *pLock; /* Info about locks on this inode */ + int h; /* The file descriptor */ + int dirfd; /* File descriptor for the directory */ + unsigned char locktype; /* The type of lock held on this fd */ + int lastErrno; /* The unix errno from the last I/O error */ + void *lockingContext; /* Locking style specific state */ + int openFlags; /* The flags specified at open */ +#if SQLITE_THREADSAFE && defined(__linux__) + pthread_t tid; /* The thread that "owns" this unixFile */ +#endif +#if OS_VXWORKS + int isDelete; /* Delete on close if true */ + struct vxworksFileId *pId; /* Unique file ID */ +#endif +#ifndef NDEBUG + /* The next group of variables are used to track whether or not the + ** transaction counter in bytes 24-27 of database files are updated + ** whenever any part of the database changes. An assertion fault will + ** occur if a file is updated without also updating the transaction + ** counter. This test is made to avoid new problems similar to the + ** one described by ticket #3584. + */ + unsigned char transCntrChng; /* True if the transaction counter changed */ + unsigned char dbUpdate; /* True if any part of database file changed */ + unsigned char inNormalWrite; /* True if in a normal write operation */ +#endif +#ifdef SQLITE_TEST + /* In test mode, increase the size of this structure a bit so that + ** it is larger than the struct CrashFile defined in test6.c. + */ + char aPadding[32]; #endif }; -/* -** Provide the ability to override some OS-layer functions during -** testing. This is used to simulate OS crashes to verify that -** commits are atomic even in the event of an OS crash. -*/ -#ifdef SQLITE_CRASH_TEST - extern int sqlite3CrashTestEnable; - extern int sqlite3CrashOpenReadWrite(const char*, OsFile**, int*); - extern int sqlite3CrashOpenExclusive(const char*, OsFile**, int); - extern int sqlite3CrashOpenReadOnly(const char*, OsFile**, int); -# define CRASH_TEST_OVERRIDE(X,A,B,C) \ - if(sqlite3CrashTestEnable){ return X(A,B,C); } -#else -# define CRASH_TEST_OVERRIDE(X,A,B,C) /* no-op */ -#endif - - /* ** Include code that is common to all os_*.c files */ #include "os_common.h" -/* -** Do not include any of the File I/O interface procedures if the -** SQLITE_OMIT_DISKIO macro is defined (indicating that the database -** will be in-memory only) -*/ -#ifndef SQLITE_OMIT_DISKIO - - /* ** Define various macros that are missing from some systems. */ @@ -150,7 +237,7 @@ struct unixFile { ** The DJGPP compiler environment looks mostly like Unix, but it ** lacks the fcntl() system call. So redefine fcntl() to be something ** that always succeeds. This means that locking does not occur under -** DJGPP. But it's DOS - what did you expect? +** DJGPP. But it is DOS - what did you expect? */ #ifdef __DJGPP__ # define fcntl(A,B,C) 0 @@ -160,269 +247,42 @@ struct unixFile { ** The threadid macro resolves to the thread-id or to 0. Used for ** testing and debugging only. */ -#ifdef SQLITE_UNIX_THREADS +#if SQLITE_THREADSAFE #define threadid pthread_self() #else #define threadid 0 #endif + /* -** Set or check the OsFile.tid field. This field is set when an OsFile -** is first opened. All subsequent uses of the OsFile verify that the -** same thread is operating on the OsFile. Some operating systems do -** not allow locks to be overridden by other threads and that restriction -** means that sqlite3* database handles cannot be moved from one thread -** to another. This logic makes sure a user does not try to do that -** by mistake. -** -** Version 3.3.1 (2006-01-15): OsFiles can be moved from one thread to -** another as long as we are running on a system that supports threads -** overriding each others locks (which now the most common behavior) -** or if no locks are held. But the OsFile.pLock field needs to be -** recomputed because its key includes the thread-id. See the -** transferOwnership() function below for additional information +** Helper functions to obtain and relinquish the global mutex. */ -#if defined(SQLITE_UNIX_THREADS) -# define SET_THREADID(X) (X)->tid = pthread_self() -# define CHECK_THREADID(X) (threadsOverrideEachOthersLocks==0 && \ - !pthread_equal((X)->tid, pthread_self())) -#else -# define SET_THREADID(X) -# define CHECK_THREADID(X) 0 +static void unixEnterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} +static void unixLeaveMutex(void){ + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +} + + +#ifdef SQLITE_DEBUG +/* +** Helper function for printing out trace information from debugging +** binaries. This returns the string represetation of the supplied +** integer lock-type. +*/ +static const char *locktypeName(int locktype){ + switch( locktype ){ + case NO_LOCK: return "NONE"; + case SHARED_LOCK: return "SHARED"; + case RESERVED_LOCK: return "RESERVED"; + case PENDING_LOCK: return "PENDING"; + case EXCLUSIVE_LOCK: return "EXCLUSIVE"; + } + return "ERROR"; +} #endif -/* -** Here is the dirt on POSIX advisory locks: ANSI STD 1003.1 (1996) -** section 6.5.2.2 lines 483 through 490 specify that when a process -** sets or clears a lock, that operation overrides any prior locks set -** by the same process. It does not explicitly say so, but this implies -** that it overrides locks set by the same process using a different -** file descriptor. Consider this test case: -** -** int fd1 = open("./file1", O_RDWR|O_CREAT, 0644); -** int fd2 = open("./file2", O_RDWR|O_CREAT, 0644); -** -** Suppose ./file1 and ./file2 are really the same file (because -** one is a hard or symbolic link to the other) then if you set -** an exclusive lock on fd1, then try to get an exclusive lock -** on fd2, it works. I would have expected the second lock to -** fail since there was already a lock on the file due to fd1. -** But not so. Since both locks came from the same process, the -** second overrides the first, even though they were on different -** file descriptors opened on different file names. -** -** Bummer. If you ask me, this is broken. Badly broken. It means -** that we cannot use POSIX locks to synchronize file access among -** competing threads of the same process. POSIX locks will work fine -** to synchronize access for threads in separate processes, but not -** threads within the same process. -** -** To work around the problem, SQLite has to manage file locks internally -** on its own. Whenever a new database is opened, we have to find the -** specific inode of the database file (the inode is determined by the -** st_dev and st_ino fields of the stat structure that fstat() fills in) -** and check for locks already existing on that inode. When locks are -** created or removed, we have to look at our own internal record of the -** locks to see if another thread has previously set a lock on that same -** inode. -** -** The OsFile structure for POSIX is no longer just an integer file -** descriptor. It is now a structure that holds the integer file -** descriptor and a pointer to a structure that describes the internal -** locks on the corresponding inode. There is one locking structure -** per inode, so if the same inode is opened twice, both OsFile structures -** point to the same locking structure. The locking structure keeps -** a reference count (so we will know when to delete it) and a "cnt" -** field that tells us its internal lock status. cnt==0 means the -** file is unlocked. cnt==-1 means the file has an exclusive lock. -** cnt>0 means there are cnt shared locks on the file. -** -** Any attempt to lock or unlock a file first checks the locking -** structure. The fcntl() system call is only invoked to set a -** POSIX lock if the internal lock structure transitions between -** a locked and an unlocked state. -** -** 2004-Jan-11: -** More recent discoveries about POSIX advisory locks. (The more -** I discover, the more I realize the a POSIX advisory locks are -** an abomination.) -** -** If you close a file descriptor that points to a file that has locks, -** all locks on that file that are owned by the current process are -** released. To work around this problem, each OsFile structure contains -** a pointer to an openCnt structure. There is one openCnt structure -** per open inode, which means that multiple OsFiles can point to a single -** openCnt. When an attempt is made to close an OsFile, if there are -** other OsFiles open on the same inode that are holding locks, the call -** to close() the file descriptor is deferred until all of the locks clear. -** The openCnt structure keeps a list of file descriptors that need to -** be closed and that list is walked (and cleared) when the last lock -** clears. -** -** First, under Linux threads, because each thread has a separate -** process ID, lock operations in one thread do not override locks -** to the same file in other threads. Linux threads behave like -** separate processes in this respect. But, if you close a file -** descriptor in linux threads, all locks are cleared, even locks -** on other threads and even though the other threads have different -** process IDs. Linux threads is inconsistent in this respect. -** (I'm beginning to think that linux threads is an abomination too.) -** The consequence of this all is that the hash table for the lockInfo -** structure has to include the process id as part of its key because -** locks in different threads are treated as distinct. But the -** openCnt structure should not include the process id in its -** key because close() clears lock on all threads, not just the current -** thread. Were it not for this goofiness in linux threads, we could -** combine the lockInfo and openCnt structures into a single structure. -** -** 2004-Jun-28: -** On some versions of linux, threads can override each others locks. -** On others not. Sometimes you can change the behavior on the same -** system by setting the LD_ASSUME_KERNEL environment variable. The -** POSIX standard is silent as to which behavior is correct, as far -** as I can tell, so other versions of unix might show the same -** inconsistency. There is no little doubt in my mind that posix -** advisory locks and linux threads are profoundly broken. -** -** To work around the inconsistencies, we have to test at runtime -** whether or not threads can override each others locks. This test -** is run once, the first time any lock is attempted. A static -** variable is set to record the results of this test for future -** use. -*/ - -/* -** An instance of the following structure serves as the key used -** to locate a particular lockInfo structure given its inode. -** -** If threads cannot override each others locks, then we set the -** lockKey.tid field to the thread ID. If threads can override -** each others locks then tid is always set to zero. tid is omitted -** if we compile without threading support. -*/ -struct lockKey { - dev_t dev; /* Device number */ - ino_t ino; /* Inode number */ -#ifdef SQLITE_UNIX_THREADS - pthread_t tid; /* Thread ID or zero if threads can override each other */ -#endif -}; - -/* -** An instance of the following structure is allocated for each open -** inode on each thread with a different process ID. (Threads have -** different process IDs on linux, but not on most other unixes.) -** -** A single inode can have multiple file descriptors, so each OsFile -** structure contains a pointer to an instance of this object and this -** object keeps a count of the number of OsFiles pointing to it. -*/ -struct lockInfo { - struct lockKey key; /* The lookup key */ - int cnt; /* Number of SHARED locks held */ - int locktype; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ - int nRef; /* Number of pointers to this structure */ -}; - -/* -** An instance of the following structure serves as the key used -** to locate a particular openCnt structure given its inode. This -** is the same as the lockKey except that the thread ID is omitted. -*/ -struct openKey { - dev_t dev; /* Device number */ - ino_t ino; /* Inode number */ -}; - -/* -** An instance of the following structure is allocated for each open -** inode. This structure keeps track of the number of locks on that -** inode. If a close is attempted against an inode that is holding -** locks, the close is deferred until all locks clear by adding the -** file descriptor to be closed to the pending list. -*/ -struct openCnt { - struct openKey key; /* The lookup key */ - int nRef; /* Number of pointers to this structure */ - int nLock; /* Number of outstanding locks */ - int nPending; /* Number of pending close() operations */ - int *aPending; /* Malloced space holding fd's awaiting a close() */ -}; - -/* -** These hash tables map inodes and file descriptors (really, lockKey and -** openKey structures) into lockInfo and openCnt structures. Access to -** these hash tables must be protected by a mutex. -*/ -static Hash lockHash = {SQLITE_HASH_BINARY, 0, 0, 0, - sqlite3ThreadSafeMalloc, sqlite3ThreadSafeFree, 0, 0}; -static Hash openHash = {SQLITE_HASH_BINARY, 0, 0, 0, - sqlite3ThreadSafeMalloc, sqlite3ThreadSafeFree, 0, 0}; - -#ifdef SQLITE_ENABLE_LOCKING_STYLE -/* -** The locking styles are associated with the different file locking -** capabilities supported by different file systems. -** -** POSIX locking style fully supports shared and exclusive byte-range locks -** ADP locking only supports exclusive byte-range locks -** FLOCK only supports a single file-global exclusive lock -** DOTLOCK isn't a true locking style, it refers to the use of a special -** file named the same as the database file with a '.lock' extension, this -** can be used on file systems that do not offer any reliable file locking -** NO locking means that no locking will be attempted, this is only used for -** read-only file systems currently -** UNSUPPORTED means that no locking will be attempted, this is only used for -** file systems that are known to be unsupported -*/ -typedef enum { - posixLockingStyle = 0, /* standard posix-advisory locks */ - afpLockingStyle, /* use afp locks */ - flockLockingStyle, /* use flock() */ - dotlockLockingStyle, /* use .lock files */ - noLockingStyle, /* useful for read-only file system */ - unsupportedLockingStyle /* indicates unsupported file system */ -} sqlite3LockingStyle; -#endif /* SQLITE_ENABLE_LOCKING_STYLE */ - -#ifdef SQLITE_UNIX_THREADS -/* -** This variable records whether or not threads can override each others -** locks. -** -** 0: No. Threads cannot override each others locks. -** 1: Yes. Threads can override each others locks. -** -1: We don't know yet. -** -** On some systems, we know at compile-time if threads can override each -** others locks. On those systems, the SQLITE_THREAD_OVERRIDE_LOCK macro -** will be set appropriately. On other systems, we have to check at -** runtime. On these latter systems, SQLTIE_THREAD_OVERRIDE_LOCK is -** undefined. -** -** This variable normally has file scope only. But during testing, we make -** it a global so that the test code can change its value in order to verify -** that the right stuff happens in either case. -*/ -#ifndef SQLITE_THREAD_OVERRIDE_LOCK -# define SQLITE_THREAD_OVERRIDE_LOCK -1 -#endif -#ifdef SQLITE_TEST -int threadsOverrideEachOthersLocks = SQLITE_THREAD_OVERRIDE_LOCK; -#else -static int threadsOverrideEachOthersLocks = SQLITE_THREAD_OVERRIDE_LOCK; -#endif - -/* -** This structure holds information passed into individual test -** threads by the testThreadLockingBehavior() routine. -*/ -struct threadTestData { - int fd; /* File to be locked */ - struct flock lock; /* The locking operation */ - int result; /* Result of the locking operation */ -}; - #ifdef SQLITE_LOCK_TRACE /* ** Print out information about all locking operations. @@ -482,19 +342,471 @@ static int lockTrace(int fd, int op, struct flock *p){ #define fcntl lockTrace #endif /* SQLITE_LOCK_TRACE */ + + /* -** The testThreadLockingBehavior() routine launches two separate -** threads on this routine. This routine attempts to lock a file -** descriptor then returns. The success or failure of that attempt -** allows the testThreadLockingBehavior() procedure to determine -** whether or not threads can override each others locks. +** This routine translates a standard POSIX errno code into something +** useful to the clients of the sqlite3 functions. Specifically, it is +** intended to translate a variety of "try again" errors into SQLITE_BUSY +** and a variety of "please close the file descriptor NOW" errors into +** SQLITE_IOERR +** +** Errors during initialization of locks, or file system support for locks, +** should handle ENOLCK, ENOTSUP, EOPNOTSUPP separately. */ -static void *threadLockingTest(void *pArg){ - struct threadTestData *pData = (struct threadTestData*)pArg; - pData->result = fcntl(pData->fd, F_SETLK, &pData->lock); - return pArg; +static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { + switch (posixError) { + case 0: + return SQLITE_OK; + + case EAGAIN: + case ETIMEDOUT: + case EBUSY: + case EINTR: + case ENOLCK: + /* random NFS retry error, unless during file system support + * introspection, in which it actually means what it says */ + return SQLITE_BUSY; + + case EACCES: + /* EACCES is like EAGAIN during locking operations, but not any other time*/ + if( (sqliteIOErr == SQLITE_IOERR_LOCK) || + (sqliteIOErr == SQLITE_IOERR_UNLOCK) || + (sqliteIOErr == SQLITE_IOERR_RDLOCK) || + (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ){ + return SQLITE_BUSY; + } + /* else fall through */ + case EPERM: + return SQLITE_PERM; + + case EDEADLK: + return SQLITE_IOERR_BLOCKED; + +#if EOPNOTSUPP!=ENOTSUP + case EOPNOTSUPP: + /* something went terribly awry, unless during file system support + * introspection, in which it actually means what it says */ +#endif +#ifdef ENOTSUP + case ENOTSUP: + /* invalid fd, unless during file system support introspection, in which + * it actually means what it says */ +#endif + case EIO: + case EBADF: + case EINVAL: + case ENOTCONN: + case ENODEV: + case ENXIO: + case ENOENT: + case ESTALE: + case ENOSYS: + /* these should force the client to close the file and reconnect */ + + default: + return sqliteIOErr; + } } + + +/****************************************************************************** +****************** Begin Unique File ID Utility Used By VxWorks *************** +** +** On most versions of unix, we can get a unique ID for a file by concatenating +** the device number and the inode number. But this does not work on VxWorks. +** On VxWorks, a unique file id must be based on the canonical filename. +** +** A pointer to an instance of the following structure can be used as a +** unique file ID in VxWorks. Each instance of this structure contains +** a copy of the canonical filename. There is also a reference count. +** The structure is reclaimed when the number of pointers to it drops to +** zero. +** +** There are never very many files open at one time and lookups are not +** a performance-critical path, so it is sufficient to put these +** structures on a linked list. +*/ +struct vxworksFileId { + struct vxworksFileId *pNext; /* Next in a list of them all */ + int nRef; /* Number of references to this one */ + int nName; /* Length of the zCanonicalName[] string */ + char *zCanonicalName; /* Canonical filename */ +}; + +#if OS_VXWORKS +/* +** All unique filenames are held on a linked list headed by this +** variable: +*/ +static struct vxworksFileId *vxworksFileList = 0; + +/* +** Simplify a filename into its canonical form +** by making the following changes: +** +** * removing any trailing and duplicate / +** * convert /./ into just / +** * convert /A/../ where A is any simple name into just / +** +** Changes are made in-place. Return the new name length. +** +** The original filename is in z[0..n-1]. Return the number of +** characters in the simplified name. +*/ +static int vxworksSimplifyName(char *z, int n){ + int i, j; + while( n>1 && z[n-1]=='/' ){ n--; } + for(i=j=0; i0 && z[j-1]!='/' ){ j--; } + if( j>0 ){ j--; } + i += 2; + continue; + } + } + z[j++] = z[i]; + } + z[j] = 0; + return j; +} + +/* +** Find a unique file ID for the given absolute pathname. Return +** a pointer to the vxworksFileId object. This pointer is the unique +** file ID. +** +** The nRef field of the vxworksFileId object is incremented before +** the object is returned. A new vxworksFileId object is created +** and added to the global list if necessary. +** +** If a memory allocation error occurs, return NULL. +*/ +static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ + struct vxworksFileId *pNew; /* search key and new file ID */ + struct vxworksFileId *pCandidate; /* For looping over existing file IDs */ + int n; /* Length of zAbsoluteName string */ + + assert( zAbsoluteName[0]=='/' ); + n = (int)strlen(zAbsoluteName); + pNew = sqlite3_malloc( sizeof(*pNew) + (n+1) ); + if( pNew==0 ) return 0; + pNew->zCanonicalName = (char*)&pNew[1]; + memcpy(pNew->zCanonicalName, zAbsoluteName, n+1); + n = vxworksSimplifyName(pNew->zCanonicalName, n); + + /* Search for an existing entry that matching the canonical name. + ** If found, increment the reference count and return a pointer to + ** the existing file ID. + */ + unixEnterMutex(); + for(pCandidate=vxworksFileList; pCandidate; pCandidate=pCandidate->pNext){ + if( pCandidate->nName==n + && memcmp(pCandidate->zCanonicalName, pNew->zCanonicalName, n)==0 + ){ + sqlite3_free(pNew); + pCandidate->nRef++; + unixLeaveMutex(); + return pCandidate; + } + } + + /* No match was found. We will make a new file ID */ + pNew->nRef = 1; + pNew->nName = n; + pNew->pNext = vxworksFileList; + vxworksFileList = pNew; + unixLeaveMutex(); + return pNew; +} + +/* +** Decrement the reference count on a vxworksFileId object. Free +** the object when the reference count reaches zero. +*/ +static void vxworksReleaseFileId(struct vxworksFileId *pId){ + unixEnterMutex(); + assert( pId->nRef>0 ); + pId->nRef--; + if( pId->nRef==0 ){ + struct vxworksFileId **pp; + for(pp=&vxworksFileList; *pp && *pp!=pId; pp = &((*pp)->pNext)){} + assert( *pp==pId ); + *pp = pId->pNext; + sqlite3_free(pId); + } + unixLeaveMutex(); +} +#endif /* OS_VXWORKS */ +/*************** End of Unique File ID Utility Used By VxWorks **************** +******************************************************************************/ + + +/****************************************************************************** +*************************** Posix Advisory Locking **************************** +** +** POSIX advisory locks are broken by design. ANSI STD 1003.1 (1996) +** section 6.5.2.2 lines 483 through 490 specify that when a process +** sets or clears a lock, that operation overrides any prior locks set +** by the same process. It does not explicitly say so, but this implies +** that it overrides locks set by the same process using a different +** file descriptor. Consider this test case: +** +** int fd1 = open("./file1", O_RDWR|O_CREAT, 0644); +** int fd2 = open("./file2", O_RDWR|O_CREAT, 0644); +** +** Suppose ./file1 and ./file2 are really the same file (because +** one is a hard or symbolic link to the other) then if you set +** an exclusive lock on fd1, then try to get an exclusive lock +** on fd2, it works. I would have expected the second lock to +** fail since there was already a lock on the file due to fd1. +** But not so. Since both locks came from the same process, the +** second overrides the first, even though they were on different +** file descriptors opened on different file names. +** +** This means that we cannot use POSIX locks to synchronize file access +** among competing threads of the same process. POSIX locks will work fine +** to synchronize access for threads in separate processes, but not +** threads within the same process. +** +** To work around the problem, SQLite has to manage file locks internally +** on its own. Whenever a new database is opened, we have to find the +** specific inode of the database file (the inode is determined by the +** st_dev and st_ino fields of the stat structure that fstat() fills in) +** and check for locks already existing on that inode. When locks are +** created or removed, we have to look at our own internal record of the +** locks to see if another thread has previously set a lock on that same +** inode. +** +** (Aside: The use of inode numbers as unique IDs does not work on VxWorks. +** For VxWorks, we have to use the alternative unique ID system based on +** canonical filename and implemented in the previous division.) +** +** The sqlite3_file structure for POSIX is no longer just an integer file +** descriptor. It is now a structure that holds the integer file +** descriptor and a pointer to a structure that describes the internal +** locks on the corresponding inode. There is one locking structure +** per inode, so if the same inode is opened twice, both unixFile structures +** point to the same locking structure. The locking structure keeps +** a reference count (so we will know when to delete it) and a "cnt" +** field that tells us its internal lock status. cnt==0 means the +** file is unlocked. cnt==-1 means the file has an exclusive lock. +** cnt>0 means there are cnt shared locks on the file. +** +** Any attempt to lock or unlock a file first checks the locking +** structure. The fcntl() system call is only invoked to set a +** POSIX lock if the internal lock structure transitions between +** a locked and an unlocked state. +** +** But wait: there are yet more problems with POSIX advisory locks. +** +** If you close a file descriptor that points to a file that has locks, +** all locks on that file that are owned by the current process are +** released. To work around this problem, each unixFile structure contains +** a pointer to an unixOpenCnt structure. There is one unixOpenCnt structure +** per open inode, which means that multiple unixFile can point to a single +** unixOpenCnt. When an attempt is made to close an unixFile, if there are +** other unixFile open on the same inode that are holding locks, the call +** to close() the file descriptor is deferred until all of the locks clear. +** The unixOpenCnt structure keeps a list of file descriptors that need to +** be closed and that list is walked (and cleared) when the last lock +** clears. +** +** Yet another problem: LinuxThreads do not play well with posix locks. +** +** Many older versions of linux use the LinuxThreads library which is +** not posix compliant. Under LinuxThreads, a lock created by thread +** A cannot be modified or overridden by a different thread B. +** Only thread A can modify the lock. Locking behavior is correct +** if the appliation uses the newer Native Posix Thread Library (NPTL) +** on linux - with NPTL a lock created by thread A can override locks +** in thread B. But there is no way to know at compile-time which +** threading library is being used. So there is no way to know at +** compile-time whether or not thread A can override locks on thread B. +** We have to do a run-time check to discover the behavior of the +** current process. +** +** On systems where thread A is unable to modify locks created by +** thread B, we have to keep track of which thread created each +** lock. Hence there is an extra field in the key to the unixLockInfo +** structure to record this information. And on those systems it +** is illegal to begin a transaction in one thread and finish it +** in another. For this latter restriction, there is no work-around. +** It is a limitation of LinuxThreads. +*/ + +/* +** Set or check the unixFile.tid field. This field is set when an unixFile +** is first opened. All subsequent uses of the unixFile verify that the +** same thread is operating on the unixFile. Some operating systems do +** not allow locks to be overridden by other threads and that restriction +** means that sqlite3* database handles cannot be moved from one thread +** to another while locks are held. +** +** Version 3.3.1 (2006-01-15): unixFile can be moved from one thread to +** another as long as we are running on a system that supports threads +** overriding each others locks (which is now the most common behavior) +** or if no locks are held. But the unixFile.pLock field needs to be +** recomputed because its key includes the thread-id. See the +** transferOwnership() function below for additional information +*/ +#if SQLITE_THREADSAFE && defined(__linux__) +# define SET_THREADID(X) (X)->tid = pthread_self() +# define CHECK_THREADID(X) (threadsOverrideEachOthersLocks==0 && \ + !pthread_equal((X)->tid, pthread_self())) +#else +# define SET_THREADID(X) +# define CHECK_THREADID(X) 0 +#endif + +/* +** An instance of the following structure serves as the key used +** to locate a particular unixOpenCnt structure given its inode. This +** is the same as the unixLockKey except that the thread ID is omitted. +*/ +struct unixFileId { + dev_t dev; /* Device number */ +#if OS_VXWORKS + struct vxworksFileId *pId; /* Unique file ID for vxworks. */ +#else + ino_t ino; /* Inode number */ +#endif +}; + +/* +** An instance of the following structure serves as the key used +** to locate a particular unixLockInfo structure given its inode. +** +** If threads cannot override each others locks (LinuxThreads), then we +** set the unixLockKey.tid field to the thread ID. If threads can override +** each others locks (Posix and NPTL) then tid is always set to zero. +** tid is omitted if we compile without threading support or on an OS +** other than linux. +*/ +struct unixLockKey { + struct unixFileId fid; /* Unique identifier for the file */ +#if SQLITE_THREADSAFE && defined(__linux__) + pthread_t tid; /* Thread ID of lock owner. Zero if not using LinuxThreads */ +#endif +}; + +/* +** An instance of the following structure is allocated for each open +** inode. Or, on LinuxThreads, there is one of these structures for +** each inode opened by each thread. +** +** A single inode can have multiple file descriptors, so each unixFile +** structure contains a pointer to an instance of this object and this +** object keeps a count of the number of unixFile pointing to it. +*/ +struct unixLockInfo { + struct unixLockKey lockKey; /* The lookup key */ + int cnt; /* Number of SHARED locks held */ + int locktype; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ + int nRef; /* Number of pointers to this structure */ + struct unixLockInfo *pNext; /* List of all unixLockInfo objects */ + struct unixLockInfo *pPrev; /* .... doubly linked */ +}; + +/* +** An instance of the following structure is allocated for each open +** inode. This structure keeps track of the number of locks on that +** inode. If a close is attempted against an inode that is holding +** locks, the close is deferred until all locks clear by adding the +** file descriptor to be closed to the pending list. +** +** TODO: Consider changing this so that there is only a single file +** descriptor for each open file, even when it is opened multiple times. +** The close() system call would only occur when the last database +** using the file closes. +*/ +struct unixOpenCnt { + struct unixFileId fileId; /* The lookup key */ + int nRef; /* Number of pointers to this structure */ + int nLock; /* Number of outstanding locks */ + int nPending; /* Number of pending close() operations */ + int *aPending; /* Malloced space holding fd's awaiting a close() */ +#if OS_VXWORKS + sem_t *pSem; /* Named POSIX semaphore */ + char aSemName[MAX_PATHNAME+1]; /* Name of that semaphore */ +#endif + struct unixOpenCnt *pNext, *pPrev; /* List of all unixOpenCnt objects */ +}; + +/* +** Lists of all unixLockInfo and unixOpenCnt objects. These used to be hash +** tables. But the number of objects is rarely more than a dozen and +** never exceeds a few thousand. And lookup is not on a critical +** path so a simple linked list will suffice. +*/ +static struct unixLockInfo *lockList = 0; +static struct unixOpenCnt *openList = 0; + +/* +** This variable remembers whether or not threads can override each others +** locks. +** +** 0: No. Threads cannot override each others locks. (LinuxThreads) +** 1: Yes. Threads can override each others locks. (Posix & NLPT) +** -1: We don't know yet. +** +** On some systems, we know at compile-time if threads can override each +** others locks. On those systems, the SQLITE_THREAD_OVERRIDE_LOCK macro +** will be set appropriately. On other systems, we have to check at +** runtime. On these latter systems, SQLTIE_THREAD_OVERRIDE_LOCK is +** undefined. +** +** This variable normally has file scope only. But during testing, we make +** it a global so that the test code can change its value in order to verify +** that the right stuff happens in either case. +*/ +#if SQLITE_THREADSAFE && defined(__linux__) +# ifndef SQLITE_THREAD_OVERRIDE_LOCK +# define SQLITE_THREAD_OVERRIDE_LOCK -1 +# endif +# ifdef SQLITE_TEST +int threadsOverrideEachOthersLocks = SQLITE_THREAD_OVERRIDE_LOCK; +# else +static int threadsOverrideEachOthersLocks = SQLITE_THREAD_OVERRIDE_LOCK; +# endif +#endif + +/* +** This structure holds information passed into individual test +** threads by the testThreadLockingBehavior() routine. +*/ +struct threadTestData { + int fd; /* File to be locked */ + struct flock lock; /* The locking operation */ + int result; /* Result of the locking operation */ +}; + +#if SQLITE_THREADSAFE && defined(__linux__) +/* +** This function is used as the main routine for a thread launched by +** testThreadLockingBehavior(). It tests whether the shared-lock obtained +** by the main thread in testThreadLockingBehavior() conflicts with a +** hypothetical write-lock obtained by this thread on the same file. +** +** The write-lock is not actually acquired, as this is not possible if +** the file is open in read-only mode (see ticket #3472). +*/ +static void *threadLockingTest(void *pArg){ + struct threadTestData *pData = (struct threadTestData*)pArg; + pData->result = fcntl(pData->fd, F_GETLK, &pData->lock); + return pArg; +} +#endif /* SQLITE_THREADSAFE && defined(__linux__) */ + + +#if SQLITE_THREADSAFE && defined(__linux__) /* ** This procedure attempts to determine whether or not threads ** can override each others locks then sets the @@ -502,208 +814,194 @@ static void *threadLockingTest(void *pArg){ */ static void testThreadLockingBehavior(int fd_orig){ int fd; - struct threadTestData d[2]; - pthread_t t[2]; + int rc; + struct threadTestData d; + struct flock l; + pthread_t t; fd = dup(fd_orig); if( fd<0 ) return; - memset(d, 0, sizeof(d)); - d[0].fd = fd; - d[0].lock.l_type = F_RDLCK; - d[0].lock.l_len = 1; - d[0].lock.l_start = 0; - d[0].lock.l_whence = SEEK_SET; - d[1] = d[0]; - d[1].lock.l_type = F_WRLCK; - pthread_create(&t[0], 0, threadLockingTest, &d[0]); - pthread_create(&t[1], 0, threadLockingTest, &d[1]); - pthread_join(t[0], 0); - pthread_join(t[1], 0); + memset(&l, 0, sizeof(l)); + l.l_type = F_RDLCK; + l.l_len = 1; + l.l_start = 0; + l.l_whence = SEEK_SET; + rc = fcntl(fd_orig, F_SETLK, &l); + if( rc!=0 ) return; + memset(&d, 0, sizeof(d)); + d.fd = fd; + d.lock = l; + d.lock.l_type = F_WRLCK; + pthread_create(&t, 0, threadLockingTest, &d); + pthread_join(t, 0); close(fd); - threadsOverrideEachOthersLocks = d[0].result==0 && d[1].result==0; + if( d.result!=0 ) return; + threadsOverrideEachOthersLocks = (d.lock.l_type==F_UNLCK); } -#endif /* SQLITE_UNIX_THREADS */ +#endif /* SQLITE_THERADSAFE && defined(__linux__) */ /* -** Release a lockInfo structure previously allocated by findLockInfo(). +** Release a unixLockInfo structure previously allocated by findLockInfo(). */ -static void releaseLockInfo(struct lockInfo *pLock){ - assert( sqlite3OsInMutex(1) ); - if (pLock == NULL) - return; - pLock->nRef--; - if( pLock->nRef==0 ){ - sqlite3HashInsert(&lockHash, &pLock->key, sizeof(pLock->key), 0); - sqlite3ThreadSafeFree(pLock); +static void releaseLockInfo(struct unixLockInfo *pLock){ + if( pLock ){ + pLock->nRef--; + if( pLock->nRef==0 ){ + if( pLock->pPrev ){ + assert( pLock->pPrev->pNext==pLock ); + pLock->pPrev->pNext = pLock->pNext; + }else{ + assert( lockList==pLock ); + lockList = pLock->pNext; + } + if( pLock->pNext ){ + assert( pLock->pNext->pPrev==pLock ); + pLock->pNext->pPrev = pLock->pPrev; + } + sqlite3_free(pLock); + } } } /* -** Release a openCnt structure previously allocated by findLockInfo(). +** Release a unixOpenCnt structure previously allocated by findLockInfo(). */ -static void releaseOpenCnt(struct openCnt *pOpen){ - assert( sqlite3OsInMutex(1) ); - if (pOpen == NULL) - return; - pOpen->nRef--; - if( pOpen->nRef==0 ){ - sqlite3HashInsert(&openHash, &pOpen->key, sizeof(pOpen->key), 0); - free(pOpen->aPending); - sqlite3ThreadSafeFree(pOpen); +static void releaseOpenCnt(struct unixOpenCnt *pOpen){ + if( pOpen ){ + pOpen->nRef--; + if( pOpen->nRef==0 ){ + if( pOpen->pPrev ){ + assert( pOpen->pPrev->pNext==pOpen ); + pOpen->pPrev->pNext = pOpen->pNext; + }else{ + assert( openList==pOpen ); + openList = pOpen->pNext; + } + if( pOpen->pNext ){ + assert( pOpen->pNext->pPrev==pOpen ); + pOpen->pNext->pPrev = pOpen->pPrev; + } + sqlite3_free(pOpen->aPending); + sqlite3_free(pOpen); + } } } -#ifdef SQLITE_ENABLE_LOCKING_STYLE /* -** Tests a byte-range locking query to see if byte range locks are -** supported, if not we fall back to dotlockLockingStyle. -*/ -static sqlite3LockingStyle sqlite3TestLockingStyle(const char *filePath, - int fd) { - /* test byte-range lock using fcntl */ - struct flock lockInfo; - - lockInfo.l_len = 1; - lockInfo.l_start = 0; - lockInfo.l_whence = SEEK_SET; - lockInfo.l_type = F_RDLCK; - - if (fcntl(fd, F_GETLK, &lockInfo) != -1) { - return posixLockingStyle; - } - - /* testing for flock can give false positives. So if if the above test - ** fails, then we fall back to using dot-lock style locking. - */ - return dotlockLockingStyle; -} - -/* -** Examines the f_fstypename entry in the statfs structure as returned by -** stat() for the file system hosting the database file, assigns the -** appropriate locking style based on it's value. These values and -** assignments are based on Darwin/OSX behavior and have not been tested on -** other systems. -*/ -static sqlite3LockingStyle sqlite3DetectLockingStyle(const char *filePath, - int fd) { - -#ifdef SQLITE_FIXED_LOCKING_STYLE - return (sqlite3LockingStyle)SQLITE_FIXED_LOCKING_STYLE; -#else - struct statfs fsInfo; - - if (statfs(filePath, &fsInfo) == -1) - return sqlite3TestLockingStyle(filePath, fd); - - if (fsInfo.f_flags & MNT_RDONLY) - return noLockingStyle; - - if( (!strcmp(fsInfo.f_fstypename, "hfs")) || - (!strcmp(fsInfo.f_fstypename, "ufs")) ) - return posixLockingStyle; - - if(!strcmp(fsInfo.f_fstypename, "afpfs")) - return afpLockingStyle; - - if(!strcmp(fsInfo.f_fstypename, "nfs")) - return sqlite3TestLockingStyle(filePath, fd); - - if(!strcmp(fsInfo.f_fstypename, "smbfs")) - return flockLockingStyle; - - if(!strcmp(fsInfo.f_fstypename, "msdos")) - return dotlockLockingStyle; - - if(!strcmp(fsInfo.f_fstypename, "webdav")) - return unsupportedLockingStyle; - - return sqlite3TestLockingStyle(filePath, fd); -#endif /* SQLITE_FIXED_LOCKING_STYLE */ -} - -#endif /* SQLITE_ENABLE_LOCKING_STYLE */ - -/* -** Given a file descriptor, locate lockInfo and openCnt structures that +** Given a file descriptor, locate unixLockInfo and unixOpenCnt structures that ** describes that file descriptor. Create new ones if necessary. The ** return values might be uninitialized if an error occurs. ** -** Return the number of errors. +** Return an appropriate error code. */ static int findLockInfo( - int fd, /* The file descriptor used in the key */ - struct lockInfo **ppLock, /* Return the lockInfo structure here */ - struct openCnt **ppOpen /* Return the openCnt structure here */ + unixFile *pFile, /* Unix file with file desc used in the key */ + struct unixLockInfo **ppLock, /* Return the unixLockInfo structure here */ + struct unixOpenCnt **ppOpen /* Return the unixOpenCnt structure here */ ){ - int rc; - struct lockKey key1; - struct openKey key2; - struct stat statbuf; - struct lockInfo *pLock; - struct openCnt *pOpen; - rc = fstat(fd, &statbuf); - if( rc!=0 ) return 1; + int rc; /* System call return code */ + int fd; /* The file descriptor for pFile */ + struct unixLockKey lockKey; /* Lookup key for the unixLockInfo structure */ + struct unixFileId fileId; /* Lookup key for the unixOpenCnt struct */ + struct stat statbuf; /* Low-level file information */ + struct unixLockInfo *pLock; /* Candidate unixLockInfo object */ + struct unixOpenCnt *pOpen; /* Candidate unixOpenCnt object */ - assert( sqlite3OsInMutex(1) ); - memset(&key1, 0, sizeof(key1)); - key1.dev = statbuf.st_dev; - key1.ino = statbuf.st_ino; -#ifdef SQLITE_UNIX_THREADS + /* Get low-level information about the file that we can used to + ** create a unique name for the file. + */ + fd = pFile->h; + rc = fstat(fd, &statbuf); + if( rc!=0 ){ + pFile->lastErrno = errno; +#ifdef EOVERFLOW + if( pFile->lastErrno==EOVERFLOW ) return SQLITE_NOLFS; +#endif + return SQLITE_IOERR; + } + + /* On OS X on an msdos filesystem, the inode number is reported + ** incorrectly for zero-size files. See ticket #3260. To work + ** around this problem (we consider it a bug in OS X, not SQLite) + ** we always increase the file size to 1 by writing a single byte + ** prior to accessing the inode number. The one byte written is + ** an ASCII 'S' character which also happens to be the first byte + ** in the header of every SQLite database. In this way, if there + ** is a race condition such that another thread has already populated + ** the first page of the database, no damage is done. + */ + if( statbuf.st_size==0 ){ + write(fd, "S", 1); + rc = fstat(fd, &statbuf); + if( rc!=0 ){ + pFile->lastErrno = errno; + return SQLITE_IOERR; + } + } + + memset(&lockKey, 0, sizeof(lockKey)); + lockKey.fid.dev = statbuf.st_dev; +#if OS_VXWORKS + lockKey.fid.pId = pFile->pId; +#else + lockKey.fid.ino = statbuf.st_ino; +#endif +#if SQLITE_THREADSAFE && defined(__linux__) if( threadsOverrideEachOthersLocks<0 ){ testThreadLockingBehavior(fd); } - key1.tid = threadsOverrideEachOthersLocks ? 0 : pthread_self(); + lockKey.tid = threadsOverrideEachOthersLocks ? 0 : pthread_self(); #endif - memset(&key2, 0, sizeof(key2)); - key2.dev = statbuf.st_dev; - key2.ino = statbuf.st_ino; - pLock = (struct lockInfo*)sqlite3HashFind(&lockHash, &key1, sizeof(key1)); - if( pLock==0 ){ - struct lockInfo *pOld; - pLock = sqlite3ThreadSafeMalloc( sizeof(*pLock) ); + fileId = lockKey.fid; + if( ppLock!=0 ){ + pLock = lockList; + while( pLock && memcmp(&lockKey, &pLock->lockKey, sizeof(lockKey)) ){ + pLock = pLock->pNext; + } if( pLock==0 ){ - rc = 1; - goto exit_findlockinfo; - } - pLock->key = key1; - pLock->nRef = 1; - pLock->cnt = 0; - pLock->locktype = 0; - pOld = sqlite3HashInsert(&lockHash, &pLock->key, sizeof(key1), pLock); - if( pOld!=0 ){ - assert( pOld==pLock ); - sqlite3ThreadSafeFree(pLock); - rc = 1; - goto exit_findlockinfo; - } - }else{ - pLock->nRef++; - } - *ppLock = pLock; - if( ppOpen!=0 ){ - pOpen = (struct openCnt*)sqlite3HashFind(&openHash, &key2, sizeof(key2)); - if( pOpen==0 ){ - struct openCnt *pOld; - pOpen = sqlite3ThreadSafeMalloc( sizeof(*pOpen) ); - if( pOpen==0 ){ - releaseLockInfo(pLock); - rc = 1; + pLock = sqlite3_malloc( sizeof(*pLock) ); + if( pLock==0 ){ + rc = SQLITE_NOMEM; goto exit_findlockinfo; } - pOpen->key = key2; + pLock->lockKey = lockKey; + pLock->nRef = 1; + pLock->cnt = 0; + pLock->locktype = 0; + pLock->pNext = lockList; + pLock->pPrev = 0; + if( lockList ) lockList->pPrev = pLock; + lockList = pLock; + }else{ + pLock->nRef++; + } + *ppLock = pLock; + } + if( ppOpen!=0 ){ + pOpen = openList; + while( pOpen && memcmp(&fileId, &pOpen->fileId, sizeof(fileId)) ){ + pOpen = pOpen->pNext; + } + if( pOpen==0 ){ + pOpen = sqlite3_malloc( sizeof(*pOpen) ); + if( pOpen==0 ){ + releaseLockInfo(pLock); + rc = SQLITE_NOMEM; + goto exit_findlockinfo; + } + pOpen->fileId = fileId; pOpen->nRef = 1; pOpen->nLock = 0; pOpen->nPending = 0; pOpen->aPending = 0; - pOld = sqlite3HashInsert(&openHash, &pOpen->key, sizeof(key2), pOpen); - if( pOld!=0 ){ - assert( pOld==pOpen ); - sqlite3ThreadSafeFree(pOpen); - releaseLockInfo(pLock); - rc = 1; - goto exit_findlockinfo; - } + pOpen->pNext = openList; + pOpen->pPrev = 0; + if( openList ) openList->pPrev = pOpen; + openList = pOpen; +#if OS_VXWORKS + pOpen->pSem = NULL; + pOpen->aSemName[0] = '\0'; +#endif }else{ pOpen->nRef++; } @@ -714,38 +1012,18 @@ exit_findlockinfo: return rc; } -#ifdef SQLITE_DEBUG -/* -** Helper function for printing out trace information from debugging -** binaries. This returns the string represetation of the supplied -** integer lock-type. -*/ -static const char *locktypeName(int locktype){ - switch( locktype ){ - case NO_LOCK: return "NONE"; - case SHARED_LOCK: return "SHARED"; - case RESERVED_LOCK: return "RESERVED"; - case PENDING_LOCK: return "PENDING"; - case EXCLUSIVE_LOCK: return "EXCLUSIVE"; - } - return "ERROR"; -} -#endif - /* ** If we are currently in a different thread than the thread that the ** unixFile argument belongs to, then transfer ownership of the unixFile ** over to the current thread. ** -** A unixFile is only owned by a thread on systems where one thread is -** unable to override locks created by a different thread. RedHat9 is -** an example of such a system. +** A unixFile is only owned by a thread on systems that use LinuxThreads. ** ** Ownership transfer is only allowed if the unixFile is currently unlocked. ** If the unixFile is locked and an ownership is wrong, then return ** SQLITE_MISUSE. SQLITE_OK is returned if everything works. */ -#ifdef SQLITE_UNIX_THREADS +#if SQLITE_THREADSAFE && defined(__linux__) static int transferOwnership(unixFile *pFile){ int rc; pthread_t hSelf; @@ -768,7 +1046,7 @@ static int transferOwnership(unixFile *pFile){ pFile->tid = hSelf; if (pFile->pLock != NULL) { releaseLockInfo(pFile->pLock); - rc = findLockInfo(pFile->h, &pFile->pLock, 0); + rc = findLockInfo(pFile, &pFile->pLock, 0); OSTRACE5("LOCK %d is now %s(%s,%d)\n", pFile->h, locktypeName(pFile->locktype), locktypeName(pFile->pLock->locktype), pFile->pLock->cnt); @@ -777,565 +1055,55 @@ static int transferOwnership(unixFile *pFile){ return SQLITE_OK; } } -#else +#else /* if not SQLITE_THREADSAFE */ /* On single-threaded builds, ownership transfer is a no-op */ # define transferOwnership(X) SQLITE_OK -#endif +#endif /* SQLITE_THREADSAFE */ -/* -** Delete the named file -*/ -int sqlite3UnixDelete(const char *zFilename){ - SimulateIOError(return SQLITE_IOERR_DELETE); - unlink(zFilename); - return SQLITE_OK; -} - -/* -** Return TRUE if the named file exists. -*/ -int sqlite3UnixFileExists(const char *zFilename){ - return access(zFilename, 0)==0; -} - -/* Forward declaration */ -static int allocateUnixFile( - int h, /* File descriptor of the open file */ - OsFile **pId, /* Write the real file descriptor here */ - const char *zFilename, /* Name of the file being opened */ - int delFlag /* If true, make sure the file deletes on close */ -); - -/* -** Attempt to open a file for both reading and writing. If that -** fails, try opening it read-only. If the file does not exist, -** try to create it. -** -** On success, a handle for the open file is written to *id -** and *pReadonly is set to 0 if the file was opened for reading and -** writing or 1 if the file was opened read-only. The function returns -** SQLITE_OK. -** -** On failure, the function returns SQLITE_CANTOPEN and leaves -** *id and *pReadonly unchanged. -*/ -int sqlite3UnixOpenReadWrite( - const char *zFilename, - OsFile **pId, - int *pReadonly -){ - int h; - - CRASH_TEST_OVERRIDE(sqlite3CrashOpenReadWrite, zFilename, pId, pReadonly); - assert( 0==*pId ); - h = open(zFilename, O_RDWR|O_CREAT|O_LARGEFILE|O_BINARY, - SQLITE_DEFAULT_FILE_PERMISSIONS); - if( h<0 ){ -#ifdef EISDIR - if( errno==EISDIR ){ - return SQLITE_CANTOPEN; - } -#endif - h = open(zFilename, O_RDONLY|O_LARGEFILE|O_BINARY); - if( h<0 ){ - return SQLITE_CANTOPEN; - } - *pReadonly = 1; - }else{ - *pReadonly = 0; - } - return allocateUnixFile(h, pId, zFilename, 0); -} - - -/* -** Attempt to open a new file for exclusive access by this process. -** The file will be opened for both reading and writing. To avoid -** a potential security problem, we do not allow the file to have -** previously existed. Nor do we allow the file to be a symbolic -** link. -** -** If delFlag is true, then make arrangements to automatically delete -** the file when it is closed. -** -** On success, write the file handle into *id and return SQLITE_OK. -** -** On failure, return SQLITE_CANTOPEN. -*/ -int sqlite3UnixOpenExclusive(const char *zFilename, OsFile **pId, int delFlag){ - int h; - - CRASH_TEST_OVERRIDE(sqlite3CrashOpenExclusive, zFilename, pId, delFlag); - assert( 0==*pId ); - h = open(zFilename, - O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW|O_LARGEFILE|O_BINARY, - delFlag ? 0600 : SQLITE_DEFAULT_FILE_PERMISSIONS); - if( h<0 ){ - return SQLITE_CANTOPEN; - } - return allocateUnixFile(h, pId, zFilename, delFlag); -} - -/* -** Attempt to open a new file for read-only access. -** -** On success, write the file handle into *id and return SQLITE_OK. -** -** On failure, return SQLITE_CANTOPEN. -*/ -int sqlite3UnixOpenReadOnly(const char *zFilename, OsFile **pId){ - int h; - - CRASH_TEST_OVERRIDE(sqlite3CrashOpenReadOnly, zFilename, pId, 0); - assert( 0==*pId ); - h = open(zFilename, O_RDONLY|O_LARGEFILE|O_BINARY); - if( h<0 ){ - return SQLITE_CANTOPEN; - } - return allocateUnixFile(h, pId, zFilename, 0); -} - -/* -** Attempt to open a file descriptor for the directory that contains a -** file. This file descriptor can be used to fsync() the directory -** in order to make sure the creation of a new file is actually written -** to disk. -** -** This routine is only meaningful for Unix. It is a no-op under -** windows since windows does not support hard links. -** -** If FULL_FSYNC is enabled, this function is not longer useful, -** a FULL_FSYNC sync applies to all pending disk operations. -** -** On success, a handle for a previously open file at *id is -** updated with the new directory file descriptor and SQLITE_OK is -** returned. -** -** On failure, the function returns SQLITE_CANTOPEN and leaves -** *id unchanged. -*/ -static int unixOpenDirectory( - OsFile *id, - const char *zDirname -){ - int h; - unixFile *pFile = (unixFile*)id; - assert( pFile!=0 ); - SET_THREADID(pFile); - assert( pFile->dirfd<0 ); - pFile->dirfd = h = open(zDirname, O_RDONLY|O_BINARY, 0); - if( h<0 ){ - return SQLITE_CANTOPEN; - } -#ifdef FD_CLOEXEC - fcntl(h, F_SETFD, fcntl(h, F_GETFD, 0) | FD_CLOEXEC); -#endif - OSTRACE3("OPENDIR %-3d %s\n", h, zDirname); - return SQLITE_OK; -} - -/* -** Create a temporary file name in zBuf. zBuf must be big enough to -** hold at least SQLITE_TEMPNAME_SIZE characters. -*/ -int sqlite3UnixTempFileName(char *zBuf){ - static const char *azDirs[] = { - 0, - "/var/tmp", - "/usr/tmp", - "/tmp", - ".", - }; - static const unsigned char zChars[] = - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "0123456789"; - int i, j; - struct stat buf; - const char *zDir = "."; - azDirs[0] = sqlite3_temp_directory; - for(i=0; ioffset then read cnt bytes into pBuf. -** Return the number of bytes actually read. Update the offset. -*/ -static int seekAndRead(unixFile *id, void *pBuf, int cnt){ - int got; - i64 newOffset; - TIMER_START; -#if defined(USE_PREAD) - got = pread(id->h, pBuf, cnt, id->offset); - SimulateIOError( got = -1 ); -#elif defined(USE_PREAD64) - got = pread64(id->h, pBuf, cnt, id->offset); - SimulateIOError( got = -1 ); -#else - newOffset = lseek(id->h, id->offset, SEEK_SET); - SimulateIOError( newOffset-- ); - if( newOffset!=id->offset ){ - return -1; - } - got = read(id->h, pBuf, cnt); -#endif - TIMER_END; - OSTRACE5("READ %-3d %5d %7lld %d\n", id->h, got, id->offset, TIMER_ELAPSED); - if( got>0 ){ - id->offset += got; - } - return got; -} - -/* -** Read data from a file into a buffer. Return SQLITE_OK if all -** bytes were read successfully and SQLITE_IOERR if anything goes -** wrong. -*/ -static int unixRead(OsFile *id, void *pBuf, int amt){ - int got; - assert( id ); - got = seekAndRead((unixFile*)id, pBuf, amt); - if( got==amt ){ - return SQLITE_OK; - }else if( got<0 ){ - return SQLITE_IOERR_READ; - }else{ - memset(&((char*)pBuf)[got], 0, amt-got); - return SQLITE_IOERR_SHORT_READ; - } -} - -/* -** Seek to the offset in id->offset then read cnt bytes into pBuf. -** Return the number of bytes actually read. Update the offset. -*/ -static int seekAndWrite(unixFile *id, const void *pBuf, int cnt){ - int got; - i64 newOffset; - TIMER_START; -#if defined(USE_PREAD) - got = pwrite(id->h, pBuf, cnt, id->offset); -#elif defined(USE_PREAD64) - got = pwrite64(id->h, pBuf, cnt, id->offset); -#else - newOffset = lseek(id->h, id->offset, SEEK_SET); - if( newOffset!=id->offset ){ - return -1; - } - got = write(id->h, pBuf, cnt); -#endif - TIMER_END; - OSTRACE5("WRITE %-3d %5d %7lld %d\n", id->h, got, id->offset, TIMER_ELAPSED); - if( got>0 ){ - id->offset += got; - } - return got; -} - - -/* -** Write data from a buffer into a file. Return SQLITE_OK on success -** or some other error code on failure. -*/ -static int unixWrite(OsFile *id, const void *pBuf, int amt){ - int wrote = 0; - assert( id ); - assert( amt>0 ); - while( amt>0 && (wrote = seekAndWrite((unixFile*)id, pBuf, amt))>0 ){ - amt -= wrote; - pBuf = &((char*)pBuf)[wrote]; - } - SimulateIOError(( wrote=(-1), amt=1 )); - SimulateDiskfullError(( wrote=0, amt=1 )); - if( amt>0 ){ - if( wrote<0 ){ - return SQLITE_IOERR_WRITE; - }else{ - return SQLITE_FULL; - } - } - return SQLITE_OK; -} - -/* -** Move the read/write pointer in a file. -*/ -static int unixSeek(OsFile *id, i64 offset){ - assert( id ); -#ifdef SQLITE_TEST - if( offset ) SimulateDiskfullError(return SQLITE_FULL); -#endif - ((unixFile*)id)->offset = offset; - return SQLITE_OK; -} - -#ifdef SQLITE_TEST -/* -** Count the number of fullsyncs and normal syncs. This is used to test -** that syncs and fullsyncs are occuring at the right times. -*/ -int sqlite3_sync_count = 0; -int sqlite3_fullsync_count = 0; -#endif - -/* -** Use the fdatasync() API only if the HAVE_FDATASYNC macro is defined. -** Otherwise use fsync() in its place. -*/ -#ifndef HAVE_FDATASYNC -# define fdatasync fsync -#endif - -/* -** Define HAVE_FULLFSYNC to 0 or 1 depending on whether or not -** the F_FULLFSYNC macro is defined. F_FULLFSYNC is currently -** only available on Mac OS X. But that could change. -*/ -#ifdef F_FULLFSYNC -# define HAVE_FULLFSYNC 1 -#else -# define HAVE_FULLFSYNC 0 -#endif - - -/* -** The fsync() system call does not work as advertised on many -** unix systems. The following procedure is an attempt to make -** it work better. -** -** The SQLITE_NO_SYNC macro disables all fsync()s. This is useful -** for testing when we want to run through the test suite quickly. -** You are strongly advised *not* to deploy with SQLITE_NO_SYNC -** enabled, however, since with SQLITE_NO_SYNC enabled, an OS crash -** or power failure will likely corrupt the database file. -*/ -static int full_fsync(int fd, int fullSync, int dataOnly){ - int rc; - - /* Record the number of times that we do a normal fsync() and - ** FULLSYNC. This is used during testing to verify that this procedure - ** gets called with the correct arguments. - */ -#ifdef SQLITE_TEST - if( fullSync ) sqlite3_fullsync_count++; - sqlite3_sync_count++; -#endif - - /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a - ** no-op - */ -#ifdef SQLITE_NO_SYNC - rc = SQLITE_OK; -#else - -#if HAVE_FULLFSYNC - if( fullSync ){ - rc = fcntl(fd, F_FULLFSYNC, 0); - }else{ - rc = 1; - } - /* If the FULLFSYNC failed, fall back to attempting an fsync(). - * It shouldn't be possible for fullfsync to fail on the local - * file system (on OSX), so failure indicates that FULLFSYNC - * isn't supported for this file system. So, attempt an fsync - * and (for now) ignore the overhead of a superfluous fcntl call. - * It'd be better to detect fullfsync support once and avoid - * the fcntl call every time sync is called. - */ - if( rc ) rc = fsync(fd); - -#else - if( dataOnly ){ - rc = fdatasync(fd); - }else{ - rc = fsync(fd); - } -#endif /* HAVE_FULLFSYNC */ -#endif /* defined(SQLITE_NO_SYNC) */ - - return rc; -} - -/* -** Make sure all writes to a particular file are committed to disk. -** -** If dataOnly==0 then both the file itself and its metadata (file -** size, access time, etc) are synced. If dataOnly!=0 then only the -** file data is synced. -** -** Under Unix, also make sure that the directory entry for the file -** has been created by fsync-ing the directory that contains the file. -** If we do not do this and we encounter a power failure, the directory -** entry for the journal might not exist after we reboot. The next -** SQLite to access the file will not know that the journal exists (because -** the directory entry for the journal was never created) and the transaction -** will not roll back - possibly leading to database corruption. -*/ -static int unixSync(OsFile *id, int dataOnly){ - int rc; - unixFile *pFile = (unixFile*)id; - assert( pFile ); - OSTRACE2("SYNC %-3d\n", pFile->h); - rc = full_fsync(pFile->h, pFile->fullSync, dataOnly); - SimulateIOError( rc=1 ); - if( rc ){ - return SQLITE_IOERR_FSYNC; - } - if( pFile->dirfd>=0 ){ - OSTRACE4("DIRSYNC %-3d (have_fullfsync=%d fullsync=%d)\n", pFile->dirfd, - HAVE_FULLFSYNC, pFile->fullSync); -#ifndef SQLITE_DISABLE_DIRSYNC - /* The directory sync is only attempted if full_fsync is - ** turned off or unavailable. If a full_fsync occurred above, - ** then the directory sync is superfluous. - */ - if( (!HAVE_FULLFSYNC || !pFile->fullSync) && full_fsync(pFile->dirfd,0,0) ){ - /* - ** We have received multiple reports of fsync() returning - ** errors when applied to directories on certain file systems. - ** A failed directory sync is not a big deal. So it seems - ** better to ignore the error. Ticket #1657 - */ - /* return SQLITE_IOERR; */ - } -#endif - close(pFile->dirfd); /* Only need to sync once, so close the directory */ - pFile->dirfd = -1; /* when we are done. */ - } - return SQLITE_OK; -} - -/* -** Sync the directory zDirname. This is a no-op on operating systems other -** than UNIX. -** -** This is used to make sure the master journal file has truely been deleted -** before making changes to individual journals on a multi-database commit. -** The F_FULLFSYNC option is not needed here. -*/ -int sqlite3UnixSyncDirectory(const char *zDirname){ -#ifdef SQLITE_DISABLE_DIRSYNC - return SQLITE_OK; -#else - int fd; - int r; - fd = open(zDirname, O_RDONLY|O_BINARY, 0); - OSTRACE3("DIRSYNC %-3d (%s)\n", fd, zDirname); - if( fd<0 ){ - return SQLITE_CANTOPEN; - } - r = fsync(fd); - close(fd); - SimulateIOError( r=1 ); - if( r ){ - return SQLITE_IOERR_DIR_FSYNC; - }else{ - return SQLITE_OK; - } -#endif -} - -/* -** Truncate an open file to a specified size -*/ -static int unixTruncate(OsFile *id, i64 nByte){ - int rc; - assert( id ); - rc = ftruncate(((unixFile*)id)->h, (off_t)nByte); - SimulateIOError( rc=1 ); - if( rc ){ - return SQLITE_IOERR_TRUNCATE; - }else{ - return SQLITE_OK; - } -} - -/* -** Determine the current size of a file in bytes -*/ -static int unixFileSize(OsFile *id, i64 *pSize){ - int rc; - struct stat buf; - assert( id ); - rc = fstat(((unixFile*)id)->h, &buf); - SimulateIOError( rc=1 ); - if( rc!=0 ){ - return SQLITE_IOERR_FSTAT; - } - *pSize = buf.st_size; - return SQLITE_OK; -} /* ** This routine checks if there is a RESERVED lock held on the specified -** file by this or any other process. If such a lock is held, return -** non-zero. If the file is unlocked or holds only SHARED locks, then -** return zero. +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. */ -static int unixCheckReservedLock(OsFile *id){ - int r = 0; +static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; unixFile *pFile = (unixFile*)id; + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + assert( pFile ); - sqlite3OsEnterMutex(); /* Because pFile->pLock is shared across threads */ + unixEnterMutex(); /* Because pFile->pLock is shared across threads */ /* Check if a thread in this process holds such a lock */ if( pFile->pLock->locktype>SHARED_LOCK ){ - r = 1; + reserved = 1; } /* Otherwise see if some other process holds it. */ - if( !r ){ + if( !reserved ){ struct flock lock; lock.l_whence = SEEK_SET; lock.l_start = RESERVED_BYTE; lock.l_len = 1; lock.l_type = F_WRLCK; - fcntl(pFile->h, F_GETLK, &lock); - if( lock.l_type!=F_UNLCK ){ - r = 1; + if (-1 == fcntl(pFile->h, F_GETLK, &lock)) { + int tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_CHECKRESERVEDLOCK); + pFile->lastErrno = tErrno; + } else if( lock.l_type!=F_UNLCK ){ + reserved = 1; } } - sqlite3OsLeaveMutex(); - OSTRACE3("TEST WR-LOCK %d %d\n", pFile->h, r); + unixLeaveMutex(); + OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved); - return r; + *pResOut = reserved; + return rc; } /* @@ -1362,7 +1130,7 @@ static int unixCheckReservedLock(OsFile *id){ ** This routine will only increase a lock. Use the sqlite3OsUnlock() ** routine to lower a locking level. */ -static int unixLock(OsFile *id, int locktype){ +static int unixLock(sqlite3_file *id, int locktype){ /* The following describes the implementation of the various locks and ** lock transitions in terms of the POSIX advisory shared and exclusive ** lock primitives (called read-locks and write-locks below, to avoid @@ -1403,7 +1171,7 @@ static int unixLock(OsFile *id, int locktype){ */ int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; - struct lockInfo *pLock = pFile->pLock; + struct unixLockInfo *pLock = pFile->pLock; struct flock lock; int s; @@ -1413,8 +1181,8 @@ static int unixLock(OsFile *id, int locktype){ locktypeName(pLock->locktype), pLock->cnt , getpid()); /* If there is already a lock of this type or more restrictive on the - ** OsFile, do nothing. Don't use the end_lock: exit path, as - ** sqlite3OsEnterMutex() hasn't been called yet. + ** unixFile, do nothing. Don't use the end_lock: exit path, as + ** unixEnterMutex() hasn't been called yet. */ if( pFile->locktype>=locktype ){ OSTRACE3("LOCK %d %s ok (already held)\n", pFile->h, @@ -1430,18 +1198,18 @@ static int unixLock(OsFile *id, int locktype){ /* This mutex is needed because pFile->pLock is shared across threads */ - sqlite3OsEnterMutex(); + unixEnterMutex(); /* Make sure the current thread owns the pFile. */ rc = transferOwnership(pFile); if( rc!=SQLITE_OK ){ - sqlite3OsLeaveMutex(); + unixLeaveMutex(); return rc; } pLock = pFile->pLock; - /* If some thread using this PID has a lock via a different OsFile* + /* If some thread using this PID has a lock via a different unixFile* ** handle that precludes the requested lock, return BUSY. */ if( (pFile->locktype!=pLock->locktype && @@ -1481,7 +1249,11 @@ static int unixLock(OsFile *id, int locktype){ lock.l_start = PENDING_BYTE; s = fcntl(pFile->h, F_SETLK, &lock); if( s==(-1) ){ - rc = (errno==EINVAL) ? SQLITE_NOLFS : SQLITE_BUSY; + int tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } goto end_lock; } } @@ -1491,24 +1263,36 @@ static int unixLock(OsFile *id, int locktype){ ** operating system calls for the specified lock. */ if( locktype==SHARED_LOCK ){ + int tErrno = 0; assert( pLock->cnt==0 ); assert( pLock->locktype==0 ); /* Now get the read-lock */ lock.l_start = SHARED_FIRST; lock.l_len = SHARED_SIZE; - s = fcntl(pFile->h, F_SETLK, &lock); - + if( (s = fcntl(pFile->h, F_SETLK, &lock))==(-1) ){ + tErrno = errno; + } /* Drop the temporary PENDING lock */ lock.l_start = PENDING_BYTE; lock.l_len = 1L; lock.l_type = F_UNLCK; if( fcntl(pFile->h, F_SETLK, &lock)!=0 ){ - rc = SQLITE_IOERR_UNLOCK; /* This should never happen */ - goto end_lock; + if( s != -1 ){ + /* This could happen with a network mount */ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_lock; + } } if( s==(-1) ){ - rc = (errno==EINVAL) ? SQLITE_NOLFS : SQLITE_BUSY; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } }else{ pFile->locktype = SHARED_LOCK; pFile->pOpen->nLock++; @@ -1538,10 +1322,32 @@ static int unixLock(OsFile *id, int locktype){ } s = fcntl(pFile->h, F_SETLK, &lock); if( s==(-1) ){ - rc = (errno==EINVAL) ? SQLITE_NOLFS : SQLITE_BUSY; + int tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } } } + +#ifndef NDEBUG + /* Set up the transaction-counter change checking flags when + ** transitioning from a SHARED to a RESERVED lock. The change + ** from SHARED to RESERVED marks the beginning of a normal + ** write operation (not a hot journal rollback). + */ + if( rc==SQLITE_OK + && pFile->locktype<=SHARED_LOCK + && locktype==RESERVED_LOCK + ){ + pFile->transCntrChng = 0; + pFile->dbUpdate = 0; + pFile->inNormalWrite = 1; + } +#endif + + if( rc==SQLITE_OK ){ pFile->locktype = locktype; pLock->locktype = locktype; @@ -1551,7 +1357,7 @@ static int unixLock(OsFile *id, int locktype){ } end_lock: - sqlite3OsLeaveMutex(); + unixLeaveMutex(); OSTRACE4("LOCK %d %s %s\n", pFile->h, locktypeName(locktype), rc==SQLITE_OK ? "ok" : "failed"); return rc; @@ -1564,11 +1370,12 @@ end_lock: ** If the locking level of the file descriptor is already at or below ** the requested locking level, this routine is a no-op. */ -static int unixUnlock(OsFile *id, int locktype){ - struct lockInfo *pLock; +static int unixUnlock(sqlite3_file *id, int locktype){ + struct unixLockInfo *pLock; struct flock lock; int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; + int h; assert( pFile ); OSTRACE7("UNLOCK %d %d was %d(%d,%d) pid=%d\n", pFile->h, locktype, @@ -1581,33 +1388,63 @@ static int unixUnlock(OsFile *id, int locktype){ if( CHECK_THREADID(pFile) ){ return SQLITE_MISUSE; } - sqlite3OsEnterMutex(); + unixEnterMutex(); + h = pFile->h; pLock = pFile->pLock; assert( pLock->cnt!=0 ); if( pFile->locktype>SHARED_LOCK ){ assert( pLock->locktype==pFile->locktype ); + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + +#ifndef NDEBUG + /* When reducing a lock such that other processes can start + ** reading the database file again, make sure that the + ** transaction counter was updated if any part of the database + ** file changed. If the transaction counter is not updated, + ** other connections to the same file might not realize that + ** the file has changed and hence might not know to flush their + ** cache. The use of a stale cache can lead to database corruption. + */ + assert( pFile->inNormalWrite==0 + || pFile->dbUpdate==0 + || pFile->transCntrChng==1 ); + pFile->inNormalWrite = 0; +#endif + + if( locktype==SHARED_LOCK ){ lock.l_type = F_RDLCK; lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = SHARED_SIZE; - if( fcntl(pFile->h, F_SETLK, &lock)==(-1) ){ - /* This should never happen */ - rc = SQLITE_IOERR_RDLOCK; + if( fcntl(h, F_SETLK, &lock)==(-1) ){ + int tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_unlock; } } lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = PENDING_BYTE; lock.l_len = 2L; assert( PENDING_BYTE+1==RESERVED_BYTE ); - if( fcntl(pFile->h, F_SETLK, &lock)!=(-1) ){ + if( fcntl(h, F_SETLK, &lock)!=(-1) ){ pLock->locktype = SHARED_LOCK; }else{ - rc = SQLITE_IOERR_UNLOCK; /* This should never happen */ + int tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_unlock; } } if( locktype==NO_LOCK ){ - struct openCnt *pOpen; + struct unixOpenCnt *pOpen; /* Decrement the shared lock counter. Release the lock using an ** OS call only when all threads in this same process have released @@ -1618,10 +1455,19 @@ static int unixUnlock(OsFile *id, int locktype){ lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = lock.l_len = 0L; - if( fcntl(pFile->h, F_SETLK, &lock)!=(-1) ){ + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + if( fcntl(h, F_SETLK, &lock)!=(-1) ){ pLock->locktype = NO_LOCK; }else{ - rc = SQLITE_IOERR_UNLOCK; /* This should never happen */ + int tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + pLock->cnt = 1; + goto end_unlock; } } @@ -1629,412 +1475,470 @@ static int unixUnlock(OsFile *id, int locktype){ ** count reaches zero, close any other file descriptors whose close ** was deferred because of outstanding locks. */ - pOpen = pFile->pOpen; - pOpen->nLock--; - assert( pOpen->nLock>=0 ); - if( pOpen->nLock==0 && pOpen->nPending>0 ){ - int i; - for(i=0; inPending; i++){ - close(pOpen->aPending[i]); + if( rc==SQLITE_OK ){ + pOpen = pFile->pOpen; + pOpen->nLock--; + assert( pOpen->nLock>=0 ); + if( pOpen->nLock==0 && pOpen->nPending>0 ){ + int i; + for(i=0; inPending; i++){ + /* close pending fds, but if closing fails don't free the array + ** assign -1 to the successfully closed descriptors and record the + ** error. The next attempt to unlock will try again. */ + if( pOpen->aPending[i] < 0 ) continue; + if( close(pOpen->aPending[i]) ){ + pFile->lastErrno = errno; + rc = SQLITE_IOERR_CLOSE; + }else{ + pOpen->aPending[i] = -1; + } + } + if( rc==SQLITE_OK ){ + sqlite3_free(pOpen->aPending); + pOpen->nPending = 0; + pOpen->aPending = 0; + } } - free(pOpen->aPending); - pOpen->nPending = 0; - pOpen->aPending = 0; } } - sqlite3OsLeaveMutex(); - pFile->locktype = locktype; + +end_unlock: + unixLeaveMutex(); + if( rc==SQLITE_OK ) pFile->locktype = locktype; return rc; } +/* +** This function performs the parts of the "close file" operation +** common to all locking schemes. It closes the directory and file +** handles, if they are valid, and sets all fields of the unixFile +** structure to 0. +** +** It is *not* necessary to hold the mutex when this routine is called, +** even on VxWorks. A mutex will be acquired on VxWorks by the +** vxworksReleaseFileId() routine. +*/ +static int closeUnixFile(sqlite3_file *id){ + unixFile *pFile = (unixFile*)id; + if( pFile ){ + if( pFile->dirfd>=0 ){ + int err = close(pFile->dirfd); + if( err ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_DIR_CLOSE; + }else{ + pFile->dirfd=-1; + } + } + if( pFile->h>=0 ){ + int err = close(pFile->h); + if( err ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_CLOSE; + } + } +#if OS_VXWORKS + if( pFile->pId ){ + if( pFile->isDelete ){ + unlink(pFile->pId->zCanonicalName); + } + vxworksReleaseFileId(pFile->pId); + pFile->pId = 0; + } +#endif + OSTRACE2("CLOSE %-3d\n", pFile->h); + OpenCounter(-1); + memset(pFile, 0, sizeof(unixFile)); + } + return SQLITE_OK; +} + /* ** Close a file. */ -static int unixClose(OsFile **pId){ - unixFile *id = (unixFile*)*pId; - - if( !id ) return SQLITE_OK; - unixUnlock(*pId, NO_LOCK); - if( id->dirfd>=0 ) close(id->dirfd); - id->dirfd = -1; - sqlite3OsEnterMutex(); - - if( id->pOpen->nLock ){ - /* If there are outstanding locks, do not actually close the file just - ** yet because that would clear those locks. Instead, add the file - ** descriptor to pOpen->aPending. It will be automatically closed when - ** the last lock is cleared. - */ - int *aNew; - struct openCnt *pOpen = id->pOpen; - aNew = realloc( pOpen->aPending, (pOpen->nPending+1)*sizeof(int) ); - if( aNew==0 ){ - /* If a malloc fails, just leak the file descriptor */ - }else{ - pOpen->aPending = aNew; - pOpen->aPending[pOpen->nPending] = id->h; - pOpen->nPending++; +static int unixClose(sqlite3_file *id){ + int rc = SQLITE_OK; + if( id ){ + unixFile *pFile = (unixFile *)id; + unixUnlock(id, NO_LOCK); + unixEnterMutex(); + if( pFile->pOpen && pFile->pOpen->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pOpen->aPending. It will be automatically closed when + ** the last lock is cleared. + */ + int *aNew; + struct unixOpenCnt *pOpen = pFile->pOpen; + aNew = sqlite3_realloc(pOpen->aPending, (pOpen->nPending+1)*sizeof(int) ); + if( aNew==0 ){ + /* If a malloc fails, just leak the file descriptor */ + }else{ + pOpen->aPending = aNew; + pOpen->aPending[pOpen->nPending] = pFile->h; + pOpen->nPending++; + pFile->h = -1; + } } - }else{ - /* There are no outstanding locks so we can close the file immediately */ - close(id->h); + releaseLockInfo(pFile->pLock); + releaseOpenCnt(pFile->pOpen); + rc = closeUnixFile(id); + unixLeaveMutex(); } - releaseLockInfo(id->pLock); - releaseOpenCnt(id->pOpen); + return rc; +} - sqlite3OsLeaveMutex(); - id->isOpen = 0; - OSTRACE2("CLOSE %-3d\n", id->h); - OpenCounter(-1); - sqlite3ThreadSafeFree(id); - *pId = 0; +/************** End of the posix advisory lock implementation ***************** +******************************************************************************/ + +/****************************************************************************** +****************************** No-op Locking ********************************** +** +** Of the various locking implementations available, this is by far the +** simplest: locking is ignored. No attempt is made to lock the database +** file for reading or writing. +** +** This locking mode is appropriate for use on read-only databases +** (ex: databases that are burned into CD-ROM, for example.) It can +** also be used if the application employs some external mechanism to +** prevent simultaneous access of the same database by two or more +** database connections. But there is a serious risk of database +** corruption if this locking mode is used in situations where multiple +** database connections are accessing the same database file at the same +** time and one or more of those connections are writing. +*/ + +static int nolockCheckReservedLock(sqlite3_file *NotUsed, int *pResOut){ + UNUSED_PARAMETER(NotUsed); + *pResOut = 0; + return SQLITE_OK; +} +static int nolockLock(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return SQLITE_OK; +} +static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); return SQLITE_OK; } - -#ifdef SQLITE_ENABLE_LOCKING_STYLE -#pragma mark AFP Support - /* - ** The afpLockingContext structure contains all afp lock specific state - */ -typedef struct afpLockingContext afpLockingContext; -struct afpLockingContext { - unsigned long long sharedLockByte; - char *filePath; -}; - -struct ByteRangeLockPB2 -{ - unsigned long long offset; /* offset to first byte to lock */ - unsigned long long length; /* nbr of bytes to lock */ - unsigned long long retRangeStart; /* nbr of 1st byte locked if successful */ - unsigned char unLockFlag; /* 1 = unlock, 0 = lock */ - unsigned char startEndFlag; /* 1=rel to end of fork, 0=rel to start */ - int fd; /* file desc to assoc this lock with */ -}; - -#define afpfsByteRangeLock2FSCTL _IOWR('z', 23, struct ByteRangeLockPB2) - -/* return 0 on success, 1 on failure. To match the behavior of the - normal posix file locking (used in unixLock for example), we should - provide 'richer' return codes - specifically to differentiate between - 'file busy' and 'file system error' results */ -static int _AFPFSSetLock(const char *path, int fd, unsigned long long offset, - unsigned long long length, int setLockFlag) -{ - struct ByteRangeLockPB2 pb; - int err; - - pb.unLockFlag = setLockFlag ? 0 : 1; - pb.startEndFlag = 0; - pb.offset = offset; - pb.length = length; - pb.fd = fd; - OSTRACE5("AFPLOCK setting lock %s for %d in range %llx:%llx\n", - (setLockFlag?"ON":"OFF"), fd, offset, length); - err = fsctl(path, afpfsByteRangeLock2FSCTL, &pb, 0); - if ( err==-1 ) { - OSTRACE4("AFPLOCK failed to fsctl() '%s' %d %s\n", path, errno, - strerror(errno)); - return 1; /* error */ - } else { - return 0; - } +** Close the file. +*/ +static int nolockClose(sqlite3_file *id) { + return closeUnixFile(id); } -/* - ** This routine checks if there is a RESERVED lock held on the specified - ** file by this or any other process. If such a lock is held, return - ** non-zero. If the file is unlocked or holds only SHARED locks, then - ** return zero. - */ -static int afpUnixCheckReservedLock(OsFile *id){ - int r = 0; - unixFile *pFile = (unixFile*)id; - - assert( pFile ); - afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; - - /* Check if a thread in this process holds such a lock */ - if( pFile->locktype>SHARED_LOCK ){ - r = 1; - } - - /* Otherwise see if some other process holds it. - */ - if ( !r ) { - /* lock the byte */ - int failed = _AFPFSSetLock(context->filePath, pFile->h, RESERVED_BYTE, 1,1); - if (failed) { - /* if we failed to get the lock then someone else must have it */ - r = 1; - } else { - /* if we succeeded in taking the reserved lock, unlock it to restore - ** the original state */ - _AFPFSSetLock(context->filePath, pFile->h, RESERVED_BYTE, 1, 0); - } - } - OSTRACE3("TEST WR-LOCK %d %d\n", pFile->h, r); - - return r; -} +/******************* End of the no-op lock implementation ********************* +******************************************************************************/ -/* AFP-style locking following the behavior of unixLock, see the unixLock -** function comments for details of lock management. */ -static int afpUnixLock(OsFile *id, int locktype) -{ +/****************************************************************************** +************************* Begin dot-file Locking ****************************** +** +** The dotfile locking implementation uses the existing of separate lock +** files in order to control access to the database. This works on just +** about every filesystem imaginable. But there are serious downsides: +** +** (1) There is zero concurrency. A single reader blocks all other +** connections from reading or writing the database. +** +** (2) An application crash or power loss can leave stale lock files +** sitting around that need to be cleared manually. +** +** Nevertheless, a dotlock is an appropriate locking mode for use if no +** other locking strategy is available. +** +** Dotfile locking works by creating a file in the same directory as the +** database and with the same name but with a ".lock" extension added. +** The existance of a lock file implies an EXCLUSIVE lock. All other lock +** types (SHARED, RESERVED, PENDING) are mapped into EXCLUSIVE. +*/ + +/* +** The file suffix added to the data base filename in order to create the +** lock file. +*/ +#define DOTLOCK_SUFFIX ".lock" + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +** +** In dotfile locking, either a lock exists or it does not. So in this +** variation of CheckReservedLock(), *pResOut is set to true if any lock +** is held on the file and false if the file is unlocked. +*/ +static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { int rc = SQLITE_OK; + int reserved = 0; unixFile *pFile = (unixFile*)id; - afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; - int gotPendingLock = 0; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); assert( pFile ); - OSTRACE5("LOCK %d %s was %s pid=%d\n", pFile->h, - locktypeName(locktype), locktypeName(pFile->locktype), getpid()); - /* If there is already a lock of this type or more restrictive on the - ** OsFile, do nothing. Don't use the afp_end_lock: exit path, as - ** sqlite3OsEnterMutex() hasn't been called yet. - */ - if( pFile->locktype>=locktype ){ - OSTRACE3("LOCK %d %s ok (already held)\n", pFile->h, - locktypeName(locktype)); - return SQLITE_OK; - } - /* Make sure the locking sequence is correct - */ - assert( pFile->locktype!=NO_LOCK || locktype==SHARED_LOCK ); - assert( locktype!=PENDING_LOCK ); - assert( locktype!=RESERVED_LOCK || pFile->locktype==SHARED_LOCK ); - - /* This mutex is needed because pFile->pLock is shared across threads - */ - sqlite3OsEnterMutex(); - - /* Make sure the current thread owns the pFile. - */ - rc = transferOwnership(pFile); - if( rc!=SQLITE_OK ){ - sqlite3OsLeaveMutex(); - return rc; - } - - /* A PENDING lock is needed before acquiring a SHARED lock and before - ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will - ** be released. - */ - if( locktype==SHARED_LOCK - || (locktype==EXCLUSIVE_LOCK && pFile->locktypefilePath, pFile->h, - PENDING_BYTE, 1, 1); - if (failed) { - rc = SQLITE_BUSY; - goto afp_end_lock; - } - } - - /* If control gets to this point, then actually go ahead and make - ** operating system calls for the specified lock. - */ - if( locktype==SHARED_LOCK ){ - int lk, failed; - int tries = 0; - - /* Now get the read-lock */ - /* note that the quality of the randomness doesn't matter that much */ - lk = random(); - context->sharedLockByte = (lk & 0x7fffffff)%(SHARED_SIZE - 1); - failed = _AFPFSSetLock(context->filePath, pFile->h, - SHARED_FIRST+context->sharedLockByte, 1, 1); - - /* Drop the temporary PENDING lock */ - if (_AFPFSSetLock(context->filePath, pFile->h, PENDING_BYTE, 1, 0)) { - rc = SQLITE_IOERR_UNLOCK; /* This should never happen */ - goto afp_end_lock; - } - - if( failed ){ - rc = SQLITE_BUSY; - } else { - pFile->locktype = SHARED_LOCK; - } + /* Check if a thread in this process holds such a lock */ + if( pFile->locktype>SHARED_LOCK ){ + /* Either this connection or some other connection in the same process + ** holds a lock on the file. No need to check further. */ + reserved = 1; }else{ - /* The request was for a RESERVED or EXCLUSIVE lock. It is - ** assumed that there is a SHARED or greater lock on the file - ** already. - */ - int failed = 0; - assert( 0!=pFile->locktype ); - if (locktype >= RESERVED_LOCK && pFile->locktype < RESERVED_LOCK) { - /* Acquire a RESERVED lock */ - failed = _AFPFSSetLock(context->filePath, pFile->h, RESERVED_BYTE, 1,1); - } - if (!failed && locktype == EXCLUSIVE_LOCK) { - /* Acquire an EXCLUSIVE lock */ - - /* Remove the shared lock before trying the range. we'll need to - ** reestablish the shared lock if we can't get the afpUnixUnlock - */ - if (!_AFPFSSetLock(context->filePath, pFile->h, SHARED_FIRST + - context->sharedLockByte, 1, 0)) { - /* now attemmpt to get the exclusive lock range */ - failed = _AFPFSSetLock(context->filePath, pFile->h, SHARED_FIRST, - SHARED_SIZE, 1); - if (failed && _AFPFSSetLock(context->filePath, pFile->h, SHARED_FIRST + - context->sharedLockByte, 1, 1)) { - rc = SQLITE_IOERR_RDLOCK; /* this should never happen */ - } - } else { - /* */ - rc = SQLITE_IOERR_UNLOCK; /* this should never happen */ - } - } - if( failed && rc == SQLITE_OK){ - rc = SQLITE_BUSY; - } + /* The lock is held if and only if the lockfile exists */ + const char *zLockFile = (const char*)pFile->lockingContext; + reserved = access(zLockFile, 0)==0; } - - if( rc==SQLITE_OK ){ - pFile->locktype = locktype; - }else if( locktype==EXCLUSIVE_LOCK ){ - pFile->locktype = PENDING_LOCK; - } - -afp_end_lock: - sqlite3OsLeaveMutex(); - OSTRACE4("LOCK %d %s %s\n", pFile->h, locktypeName(locktype), - rc==SQLITE_OK ? "ok" : "failed"); + OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved); + *pResOut = reserved; return rc; } /* - ** Lower the locking level on file descriptor pFile to locktype. locktype - ** must be either NO_LOCK or SHARED_LOCK. - ** - ** If the locking level of the file descriptor is already at or below - ** the requested locking level, this routine is a no-op. - */ -static int afpUnixUnlock(OsFile *id, int locktype) { - struct flock lock; - int rc = SQLITE_OK; +** Lock the file with the lock specified by parameter locktype - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +** +** With dotfile locking, we really only support state (4): EXCLUSIVE. +** But we track the other locking levels internally. +*/ +static int dotlockLock(sqlite3_file *id, int locktype) { unixFile *pFile = (unixFile*)id; - afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + int fd; + char *zLockFile = (char *)pFile->lockingContext; + int rc = SQLITE_OK; + + + /* If we have any lock, then the lock file already exists. All we have + ** to do is adjust our internal record of the lock level. + */ + if( pFile->locktype > NO_LOCK ){ + pFile->locktype = locktype; +#if !OS_VXWORKS + /* Always update the timestamp on the old file */ + utimes(zLockFile, NULL); +#endif + return SQLITE_OK; + } + + /* grab an exclusive lock */ + fd = open(zLockFile,O_RDONLY|O_CREAT|O_EXCL,0600); + if( fd<0 ){ + /* failed to open/create the file, someone else may have stolen the lock */ + int tErrno = errno; + if( EEXIST == tErrno ){ + rc = SQLITE_BUSY; + } else { + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + } + return rc; + } + if( close(fd) ){ + pFile->lastErrno = errno; + rc = SQLITE_IOERR_CLOSE; + } + + /* got it, set the type and return ok */ + pFile->locktype = locktype; + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to locktype. locktype +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +** +** When the locking level reaches NO_LOCK, delete the lock file. +*/ +static int dotlockUnlock(sqlite3_file *id, int locktype) { + unixFile *pFile = (unixFile*)id; + char *zLockFile = (char *)pFile->lockingContext; assert( pFile ); OSTRACE5("UNLOCK %d %d was %d pid=%d\n", pFile->h, locktype, - pFile->locktype, getpid()); - + pFile->locktype, getpid()); assert( locktype<=SHARED_LOCK ); - if( pFile->locktype<=locktype ){ + + /* no-op if possible */ + if( pFile->locktype==locktype ){ return SQLITE_OK; } - if( CHECK_THREADID(pFile) ){ - return SQLITE_MISUSE; - } - sqlite3OsEnterMutex(); - if( pFile->locktype>SHARED_LOCK ){ - if( locktype==SHARED_LOCK ){ - int failed = 0; - /* unlock the exclusive range - then re-establish the shared lock */ - if (pFile->locktype==EXCLUSIVE_LOCK) { - failed = _AFPFSSetLock(context->filePath, pFile->h, SHARED_FIRST, - SHARED_SIZE, 0); - if (!failed) { - /* successfully removed the exclusive lock */ - if (_AFPFSSetLock(context->filePath, pFile->h, SHARED_FIRST+ - context->sharedLockByte, 1, 1)) { - /* failed to re-establish our shared lock */ - rc = SQLITE_IOERR_RDLOCK; /* This should never happen */ - } - } else { - /* This should never happen - failed to unlock the exclusive range */ - rc = SQLITE_IOERR_UNLOCK; - } + /* To downgrade to shared, simply update our internal notion of the + ** lock state. No need to mess with the file on disk. + */ + if( locktype==SHARED_LOCK ){ + pFile->locktype = SHARED_LOCK; + return SQLITE_OK; + } + + /* To fully unlock the database, delete the lock file */ + assert( locktype==NO_LOCK ); + if( unlink(zLockFile) ){ + int rc, tErrno = errno; + if( ENOENT != tErrno ){ + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + } + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + return rc; + } + pFile->locktype = NO_LOCK; + return SQLITE_OK; +} + +/* +** Close a file. Make sure the lock has been released before closing. +*/ +static int dotlockClose(sqlite3_file *id) { + int rc; + if( id ){ + unixFile *pFile = (unixFile*)id; + dotlockUnlock(id, NO_LOCK); + sqlite3_free(pFile->lockingContext); + } + rc = closeUnixFile(id); + return rc; +} +/****************** End of the dot-file lock implementation ******************* +******************************************************************************/ + +/****************************************************************************** +************************** Begin flock Locking ******************************** +** +** Use the flock() system call to do file locking. +** +** flock() locking is like dot-file locking in that the various +** fine-grain locking levels supported by SQLite are collapsed into +** a single exclusive lock. In other words, SHARED, RESERVED, and +** PENDING locks are the same thing as an EXCLUSIVE lock. SQLite +** still works when you do this, but concurrency is reduced since +** only a single process can be reading the database at a time. +** +** Omit this section if SQLITE_ENABLE_LOCKING_STYLE is turned off or if +** compiling for VXWORKS. +*/ +#if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + + /* Check if a thread in this process holds such a lock */ + if( pFile->locktype>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. */ + if( !reserved ){ + /* attempt to get the lock */ + int lrc = flock(pFile->h, LOCK_EX | LOCK_NB); + if( !lrc ){ + /* got the lock, unlock it */ + lrc = flock(pFile->h, LOCK_UN); + if ( lrc ) { + int tErrno = errno; + /* unlock failed with an error */ + lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(lrc) ){ + pFile->lastErrno = tErrno; + rc = lrc; + } + } + } else { + int tErrno = errno; + reserved = 1; + /* someone else might have it reserved */ + lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(lrc) ){ + pFile->lastErrno = tErrno; + rc = lrc; } } - if (rc == SQLITE_OK && pFile->locktype>=PENDING_LOCK) { - if (_AFPFSSetLock(context->filePath, pFile->h, PENDING_BYTE, 1, 0)){ - /* failed to release the pending lock */ - rc = SQLITE_IOERR_UNLOCK; /* This should never happen */ - } - } - if (rc == SQLITE_OK && pFile->locktype>=RESERVED_LOCK) { - if (_AFPFSSetLock(context->filePath, pFile->h, RESERVED_BYTE, 1, 0)) { - /* failed to release the reserved lock */ - rc = SQLITE_IOERR_UNLOCK; /* This should never happen */ - } - } } - if( locktype==NO_LOCK ){ - int failed = _AFPFSSetLock(context->filePath, pFile->h, - SHARED_FIRST + context->sharedLockByte, 1, 0); - if (failed) { - rc = SQLITE_IOERR_UNLOCK; /* This should never happen */ - } + OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved); + +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ + rc = SQLITE_OK; + reserved=1; } - if (rc == SQLITE_OK) - pFile->locktype = locktype; - sqlite3OsLeaveMutex(); +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + *pResOut = reserved; return rc; } /* - ** Close a file & cleanup AFP specific locking context - */ -static int afpUnixClose(OsFile **pId) { - unixFile *id = (unixFile*)*pId; - - if( !id ) return SQLITE_OK; - afpUnixUnlock(*pId, NO_LOCK); - /* free the AFP locking structure */ - if (id->lockingContext != NULL) { - if (((afpLockingContext *)id->lockingContext)->filePath != NULL) - sqlite3ThreadSafeFree(((afpLockingContext*)id->lockingContext)->filePath); - sqlite3ThreadSafeFree(id->lockingContext); - } - - if( id->dirfd>=0 ) close(id->dirfd); - id->dirfd = -1; - close(id->h); - id->isOpen = 0; - OSTRACE2("CLOSE %-3d\n", id->h); - OpenCounter(-1); - sqlite3ThreadSafeFree(id); - *pId = 0; - return SQLITE_OK; -} - - -#pragma mark flock() style locking - -/* - ** The flockLockingContext is not used - */ -typedef void flockLockingContext; - -static int flockUnixCheckReservedLock(OsFile *id) { +** Lock the file with the lock specified by parameter locktype - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** flock() only really support EXCLUSIVE locks. We track intermediate +** lock states in the sqlite3_file structure, but all locks SHARED or +** above are really EXCLUSIVE locks and exclude all other processes from +** access the file. +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int flockLock(sqlite3_file *id, int locktype) { + int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; - - if (pFile->locktype == RESERVED_LOCK) { - return 1; /* already have a reserved lock */ - } else { - /* attempt to get the lock */ - int rc = flock(pFile->h, LOCK_EX | LOCK_NB); - if (!rc) { - /* got the lock, unlock it */ - flock(pFile->h, LOCK_UN); - return 0; /* no one has it reserved */ - } - return 1; /* someone else might have it reserved */ - } -} -static int flockUnixLock(OsFile *id, int locktype) { - unixFile *pFile = (unixFile*)id; - + assert( pFile ); + /* if we already have a lock, it is exclusive. ** Just adjust level and punt on outta here. */ if (pFile->locktype > NO_LOCK) { @@ -2043,20 +1947,42 @@ static int flockUnixLock(OsFile *id, int locktype) { } /* grab an exclusive lock */ - int rc = flock(pFile->h, LOCK_EX | LOCK_NB); - if (rc) { + + if (flock(pFile->h, LOCK_EX | LOCK_NB)) { + int tErrno = errno; /* didn't get, must be busy */ - return SQLITE_BUSY; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } } else { /* got it, set the type and return ok */ pFile->locktype = locktype; - return SQLITE_OK; } + OSTRACE4("LOCK %d %s %s\n", pFile->h, locktypeName(locktype), + rc==SQLITE_OK ? "ok" : "failed"); +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ + rc = SQLITE_BUSY; + } +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + return rc; } -static int flockUnixUnlock(OsFile *id, int locktype) { + +/* +** Lower the locking level on file descriptor pFile to locktype. locktype +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int flockUnlock(sqlite3_file *id, int locktype) { unixFile *pFile = (unixFile*)id; + assert( pFile ); + OSTRACE5("UNLOCK %d %d was %d pid=%d\n", pFile->h, locktype, + pFile->locktype, getpid()); assert( locktype<=SHARED_LOCK ); /* no-op if possible */ @@ -2072,106 +1998,168 @@ static int flockUnixUnlock(OsFile *id, int locktype) { /* no, really, unlock. */ int rc = flock(pFile->h, LOCK_UN); - if (rc) - return SQLITE_IOERR_UNLOCK; - else { + if (rc) { + int r, tErrno = errno; + r = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(r) ){ + pFile->lastErrno = tErrno; + } +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + if( (r & SQLITE_IOERR) == SQLITE_IOERR ){ + r = SQLITE_BUSY; + } +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + + return r; + } else { pFile->locktype = NO_LOCK; return SQLITE_OK; } } /* - ** Close a file. - */ -static int flockUnixClose(OsFile **pId) { - unixFile *id = (unixFile*)*pId; - - if( !id ) return SQLITE_OK; - flockUnixUnlock(*pId, NO_LOCK); - - if( id->dirfd>=0 ) close(id->dirfd); - id->dirfd = -1; - sqlite3OsEnterMutex(); - - close(id->h); - sqlite3OsLeaveMutex(); - id->isOpen = 0; - OSTRACE2("CLOSE %-3d\n", id->h); - OpenCounter(-1); - sqlite3ThreadSafeFree(id); - *pId = 0; - return SQLITE_OK; +** Close a file. +*/ +static int flockClose(sqlite3_file *id) { + if( id ){ + flockUnlock(id, NO_LOCK); + } + return closeUnixFile(id); } -#pragma mark Old-School .lock file based locking +#endif /* SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORK */ + +/******************* End of the flock lock implementation ********************* +******************************************************************************/ + +/****************************************************************************** +************************ Begin Named Semaphore Locking ************************ +** +** Named semaphore locking is only supported on VxWorks. +** +** Semaphore locking is like dot-lock and flock in that it really only +** supports EXCLUSIVE locking. Only a single process can read or write +** the database file at a time. This reduces potential concurrency, but +** makes the lock implementation much easier. +*/ +#if OS_VXWORKS /* - ** The dotlockLockingContext structure contains all dotlock (.lock) lock - ** specific state - */ -typedef struct dotlockLockingContext dotlockLockingContext; -struct dotlockLockingContext { - char *lockPath; -}; - - -static int dotlockUnixCheckReservedLock(OsFile *id) { +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { + int rc = SQLITE_OK; + int reserved = 0; unixFile *pFile = (unixFile*)id; - dotlockLockingContext *context = - (dotlockLockingContext *) pFile->lockingContext; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - if (pFile->locktype == RESERVED_LOCK) { - return 1; /* already have a reserved lock */ - } else { - struct stat statBuf; - if (lstat(context->lockPath,&statBuf) == 0) - /* file exists, someone else has the lock */ - return 1; - else - /* file does not exist, we could have it if we want it */ - return 0; + assert( pFile ); + + /* Check if a thread in this process holds such a lock */ + if( pFile->locktype>SHARED_LOCK ){ + reserved = 1; } + + /* Otherwise see if some other process holds it. */ + if( !reserved ){ + sem_t *pSem = pFile->pOpen->pSem; + struct stat statBuf; + + if( sem_trywait(pSem)==-1 ){ + int tErrno = errno; + if( EAGAIN != tErrno ){ + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_CHECKRESERVEDLOCK); + pFile->lastErrno = tErrno; + } else { + /* someone else has the lock when we are in NO_LOCK */ + reserved = (pFile->locktype < SHARED_LOCK); + } + }else{ + /* we could have it if we want it */ + sem_post(pSem); + } + } + OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved); + + *pResOut = reserved; + return rc; } -static int dotlockUnixLock(OsFile *id, int locktype) { +/* +** Lock the file with the lock specified by parameter locktype - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** Semaphore locks only really support EXCLUSIVE locks. We track intermediate +** lock states in the sqlite3_file structure, but all locks SHARED or +** above are really EXCLUSIVE locks and exclude all other processes from +** access the file. +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int semLock(sqlite3_file *id, int locktype) { unixFile *pFile = (unixFile*)id; - dotlockLockingContext *context = - (dotlockLockingContext *) pFile->lockingContext; - + int fd; + sem_t *pSem = pFile->pOpen->pSem; + int rc = SQLITE_OK; + /* if we already have a lock, it is exclusive. ** Just adjust level and punt on outta here. */ if (pFile->locktype > NO_LOCK) { pFile->locktype = locktype; - - /* Always update the timestamp on the old file */ - utimes(context->lockPath,NULL); - return SQLITE_OK; + rc = SQLITE_OK; + goto sem_end_lock; } - /* check to see if lock file already exists */ - struct stat statBuf; - if (lstat(context->lockPath,&statBuf) == 0){ - return SQLITE_BUSY; /* it does, busy */ + /* lock semaphore now but bail out when already locked. */ + if( sem_trywait(pSem)==-1 ){ + rc = SQLITE_BUSY; + goto sem_end_lock; } - - /* grab an exclusive lock */ - int fd = open(context->lockPath,O_RDONLY|O_CREAT|O_EXCL,0600); - if (fd < 0) { - /* failed to open/create the file, someone else may have stolen the lock */ - return SQLITE_BUSY; - } - close(fd); - + /* got it, set the type and return ok */ pFile->locktype = locktype; - return SQLITE_OK; + + sem_end_lock: + return rc; } -static int dotlockUnixUnlock(OsFile *id, int locktype) { +/* +** Lower the locking level on file descriptor pFile to locktype. locktype +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int semUnlock(sqlite3_file *id, int locktype) { unixFile *pFile = (unixFile*)id; - dotlockLockingContext *context = - (dotlockLockingContext *) pFile->lockingContext; - + sem_t *pSem = pFile->pOpen->pSem; + + assert( pFile ); + assert( pSem ); + OSTRACE5("UNLOCK %d %d was %d pid=%d\n", pFile->h, locktype, + pFile->locktype, getpid()); assert( locktype<=SHARED_LOCK ); /* no-op if possible */ @@ -2185,8 +2173,15 @@ static int dotlockUnixUnlock(OsFile *id, int locktype) { return SQLITE_OK; } - /* no, really, unlock. */ - unlink(context->lockPath); + /* no, really unlock. */ + if ( sem_post(pSem)==-1 ) { + int rc, tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + return rc; + } pFile->locktype = NO_LOCK; return SQLITE_OK; } @@ -2194,148 +2189,866 @@ static int dotlockUnixUnlock(OsFile *id, int locktype) { /* ** Close a file. */ -static int dotlockUnixClose(OsFile **pId) { - unixFile *id = (unixFile*)*pId; - - if( !id ) return SQLITE_OK; - dotlockUnixUnlock(*pId, NO_LOCK); - /* free the dotlock locking structure */ - if (id->lockingContext != NULL) { - if (((dotlockLockingContext *)id->lockingContext)->lockPath != NULL) - sqlite3ThreadSafeFree( ( (dotlockLockingContext *) - id->lockingContext)->lockPath); - sqlite3ThreadSafeFree(id->lockingContext); +static int semClose(sqlite3_file *id) { + if( id ){ + unixFile *pFile = (unixFile*)id; + semUnlock(id, NO_LOCK); + assert( pFile ); + unixEnterMutex(); + releaseLockInfo(pFile->pLock); + releaseOpenCnt(pFile->pOpen); + closeUnixFile(id); + unixLeaveMutex(); } - - if( id->dirfd>=0 ) close(id->dirfd); - id->dirfd = -1; - sqlite3OsEnterMutex(); - - close(id->h); - - sqlite3OsLeaveMutex(); - id->isOpen = 0; - OSTRACE2("CLOSE %-3d\n", id->h); - OpenCounter(-1); - sqlite3ThreadSafeFree(id); - *pId = 0; return SQLITE_OK; } - -#pragma mark No locking - +#endif /* OS_VXWORKS */ /* - ** The nolockLockingContext is void - */ -typedef void nolockLockingContext; +** Named semaphore locking is only available on VxWorks. +** +*************** End of the named semaphore lock implementation **************** +******************************************************************************/ -static int nolockUnixCheckReservedLock(OsFile *id) { - return 0; -} -static int nolockUnixLock(OsFile *id, int locktype) { - return SQLITE_OK; -} - -static int nolockUnixUnlock(OsFile *id, int locktype) { - return SQLITE_OK; -} - -/* - ** Close a file. - */ -static int nolockUnixClose(OsFile **pId) { - unixFile *id = (unixFile*)*pId; - - if( !id ) return SQLITE_OK; - if( id->dirfd>=0 ) close(id->dirfd); - id->dirfd = -1; - sqlite3OsEnterMutex(); - - close(id->h); - - sqlite3OsLeaveMutex(); - id->isOpen = 0; - OSTRACE2("CLOSE %-3d\n", id->h); - OpenCounter(-1); - sqlite3ThreadSafeFree(id); - *pId = 0; - return SQLITE_OK; -} - -#endif /* SQLITE_ENABLE_LOCKING_STYLE */ - -/* -** Turn a relative pathname into a full pathname. Return a pointer -** to the full pathname stored in space obtained from sqliteMalloc(). -** The calling function is responsible for freeing this space once it -** is no longer needed. +/****************************************************************************** +*************************** Begin AFP Locking ********************************* +** +** AFP is the Apple Filing Protocol. AFP is a network filesystem found +** on Apple Macintosh computers - both OS9 and OSX. +** +** Third-party implementations of AFP are available. But this code here +** only works on OSX. */ -char *sqlite3UnixFullPathname(const char *zRelative){ - char *zFull = 0; - if( zRelative[0]=='/' ){ - sqlite3SetString(&zFull, zRelative, (char*)0); - }else{ - char *zBuf = sqliteMalloc(5000); - if( zBuf==0 ){ - return 0; + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* +** The afpLockingContext structure contains all afp lock specific state +*/ +typedef struct afpLockingContext afpLockingContext; +struct afpLockingContext { + unsigned long long sharedByte; + const char *dbPath; /* Name of the open file */ +}; + +struct ByteRangeLockPB2 +{ + unsigned long long offset; /* offset to first byte to lock */ + unsigned long long length; /* nbr of bytes to lock */ + unsigned long long retRangeStart; /* nbr of 1st byte locked if successful */ + unsigned char unLockFlag; /* 1 = unlock, 0 = lock */ + unsigned char startEndFlag; /* 1=rel to end of fork, 0=rel to start */ + int fd; /* file desc to assoc this lock with */ +}; + +#define afpfsByteRangeLock2FSCTL _IOWR('z', 23, struct ByteRangeLockPB2) + +/* +** This is a utility for setting or clearing a bit-range lock on an +** AFP filesystem. +** +** Return SQLITE_OK on success, SQLITE_BUSY on failure. +*/ +static int afpSetLock( + const char *path, /* Name of the file to be locked or unlocked */ + unixFile *pFile, /* Open file descriptor on path */ + unsigned long long offset, /* First byte to be locked */ + unsigned long long length, /* Number of bytes to lock */ + int setLockFlag /* True to set lock. False to clear lock */ +){ + struct ByteRangeLockPB2 pb; + int err; + + pb.unLockFlag = setLockFlag ? 0 : 1; + pb.startEndFlag = 0; + pb.offset = offset; + pb.length = length; + pb.fd = pFile->h; + + OSTRACE6("AFPSETLOCK [%s] for %d%s in range %llx:%llx\n", + (setLockFlag?"ON":"OFF"), pFile->h, (pb.fd==-1?"[testval-1]":""), + offset, length); + err = fsctl(path, afpfsByteRangeLock2FSCTL, &pb, 0); + if ( err==-1 ) { + int rc; + int tErrno = errno; + OSTRACE4("AFPSETLOCK failed to fsctl() '%s' %d %s\n", + path, tErrno, strerror(tErrno)); +#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS + rc = SQLITE_BUSY; +#else + rc = sqliteErrorFromPosixError(tErrno, + setLockFlag ? SQLITE_IOERR_LOCK : SQLITE_IOERR_UNLOCK); +#endif /* SQLITE_IGNORE_AFP_LOCK_ERRORS */ + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; } - zBuf[0] = 0; - sqlite3SetString(&zFull, getcwd(zBuf, 5000), "/", zRelative, - (char*)0); - sqliteFree(zBuf); + return rc; + } else { + return SQLITE_OK; + } +} + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + + /* Check if a thread in this process holds such a lock */ + if( pFile->locktype>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. + */ + if( !reserved ){ + /* lock the RESERVED byte */ + int lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + if( SQLITE_OK==lrc ){ + /* if we succeeded in taking the reserved lock, unlock it to restore + ** the original state */ + lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); + } else { + /* if we failed to get the lock then someone else must have it */ + reserved = 1; + } + if( IS_LOCK_ERROR(lrc) ){ + rc=lrc; + } + } + + OSTRACE4("TEST WR-LOCK %d %d %d\n", pFile->h, rc, reserved); + + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter locktype - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int afpLock(sqlite3_file *id, int locktype){ + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + + assert( pFile ); + OSTRACE5("LOCK %d %s was %s pid=%d\n", pFile->h, + locktypeName(locktype), locktypeName(pFile->locktype), getpid()); + + /* If there is already a lock of this type or more restrictive on the + ** unixFile, do nothing. Don't use the afp_end_lock: exit path, as + ** unixEnterMutex() hasn't been called yet. + */ + if( pFile->locktype>=locktype ){ + OSTRACE3("LOCK %d %s ok (already held)\n", pFile->h, + locktypeName(locktype)); + return SQLITE_OK; } -#if 0 - /* - ** Remove "/./" path elements and convert "/A/./" path elements - ** to just "/". + /* Make sure the locking sequence is correct */ - if( zFull ){ - int i, j; - for(i=j=0; zFull[i]; i++){ - if( zFull[i]=='/' ){ - if( zFull[i+1]=='/' ) continue; - if( zFull[i+1]=='.' && zFull[i+2]=='/' ){ - i += 1; - continue; + assert( pFile->locktype!=NO_LOCK || locktype==SHARED_LOCK ); + assert( locktype!=PENDING_LOCK ); + assert( locktype!=RESERVED_LOCK || pFile->locktype==SHARED_LOCK ); + + /* This mutex is needed because pFile->pLock is shared across threads + */ + unixEnterMutex(); + + /* Make sure the current thread owns the pFile. + */ + rc = transferOwnership(pFile); + if( rc!=SQLITE_OK ){ + unixLeaveMutex(); + return rc; + } + + /* A PENDING lock is needed before acquiring a SHARED lock and before + ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will + ** be released. + */ + if( locktype==SHARED_LOCK + || (locktype==EXCLUSIVE_LOCK && pFile->locktypedbPath, pFile, PENDING_BYTE, 1, 1); + if (failed) { + rc = failed; + goto afp_end_lock; + } + } + + /* If control gets to this point, then actually go ahead and make + ** operating system calls for the specified lock. + */ + if( locktype==SHARED_LOCK ){ + int lk, lrc1, lrc2, lrc1Errno; + + /* Now get the read-lock SHARED_LOCK */ + /* note that the quality of the randomness doesn't matter that much */ + lk = random(); + context->sharedByte = (lk & 0x7fffffff)%(SHARED_SIZE - 1); + lrc1 = afpSetLock(context->dbPath, pFile, + SHARED_FIRST+context->sharedByte, 1, 1); + if( IS_LOCK_ERROR(lrc1) ){ + lrc1Errno = pFile->lastErrno; + } + /* Drop the temporary PENDING lock */ + lrc2 = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); + + if( IS_LOCK_ERROR(lrc1) ) { + pFile->lastErrno = lrc1Errno; + rc = lrc1; + goto afp_end_lock; + } else if( IS_LOCK_ERROR(lrc2) ){ + rc = lrc2; + goto afp_end_lock; + } else if( lrc1 != SQLITE_OK ) { + rc = lrc1; + } else { + pFile->locktype = SHARED_LOCK; + pFile->pOpen->nLock++; + } + }else{ + /* The request was for a RESERVED or EXCLUSIVE lock. It is + ** assumed that there is a SHARED or greater lock on the file + ** already. + */ + int failed = 0; + assert( 0!=pFile->locktype ); + if (locktype >= RESERVED_LOCK && pFile->locktype < RESERVED_LOCK) { + /* Acquire a RESERVED lock */ + failed = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + } + if (!failed && locktype == EXCLUSIVE_LOCK) { + /* Acquire an EXCLUSIVE lock */ + + /* Remove the shared lock before trying the range. we'll need to + ** reestablish the shared lock if we can't get the afpUnlock + */ + if( !(failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST + + context->sharedByte, 1, 0)) ){ + int failed2 = SQLITE_OK; + /* now attemmpt to get the exclusive lock range */ + failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, + SHARED_SIZE, 1); + if( failed && (failed2 = afpSetLock(context->dbPath, pFile, + SHARED_FIRST + context->sharedByte, 1, 1)) ){ + /* Can't reestablish the shared lock. Sqlite can't deal, this is + ** a critical I/O error + */ + rc = ((failed & SQLITE_IOERR) == SQLITE_IOERR) ? failed2 : + SQLITE_IOERR_LOCK; + goto afp_end_lock; + } + }else{ + rc = failed; + } + } + if( failed ){ + rc = failed; + } + } + + if( rc==SQLITE_OK ){ + pFile->locktype = locktype; + }else if( locktype==EXCLUSIVE_LOCK ){ + pFile->locktype = PENDING_LOCK; + } + +afp_end_lock: + unixLeaveMutex(); + OSTRACE4("LOCK %d %s %s\n", pFile->h, locktypeName(locktype), + rc==SQLITE_OK ? "ok" : "failed"); + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to locktype. locktype +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int afpUnlock(sqlite3_file *id, int locktype) { + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + afpLockingContext *pCtx = (afpLockingContext *) pFile->lockingContext; + + assert( pFile ); + OSTRACE5("UNLOCK %d %d was %d pid=%d\n", pFile->h, locktype, + pFile->locktype, getpid()); + + assert( locktype<=SHARED_LOCK ); + if( pFile->locktype<=locktype ){ + return SQLITE_OK; + } + if( CHECK_THREADID(pFile) ){ + return SQLITE_MISUSE; + } + unixEnterMutex(); + if( pFile->locktype>SHARED_LOCK ){ + + if( pFile->locktype==EXCLUSIVE_LOCK ){ + rc = afpSetLock(pCtx->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 0); + if( rc==SQLITE_OK && locktype==SHARED_LOCK ){ + /* only re-establish the shared lock if necessary */ + int sharedLockByte = SHARED_FIRST+pCtx->sharedByte; + rc = afpSetLock(pCtx->dbPath, pFile, sharedLockByte, 1, 1); + } + } + if( rc==SQLITE_OK && pFile->locktype>=PENDING_LOCK ){ + rc = afpSetLock(pCtx->dbPath, pFile, PENDING_BYTE, 1, 0); + } + if( rc==SQLITE_OK && pFile->locktype>=RESERVED_LOCK ){ + rc = afpSetLock(pCtx->dbPath, pFile, RESERVED_BYTE, 1, 0); + } + }else if( locktype==NO_LOCK ){ + /* clear the shared lock */ + int sharedLockByte = SHARED_FIRST+pCtx->sharedByte; + rc = afpSetLock(pCtx->dbPath, pFile, sharedLockByte, 1, 0); + } + + if( rc==SQLITE_OK ){ + if( locktype==NO_LOCK ){ + struct unixOpenCnt *pOpen = pFile->pOpen; + pOpen->nLock--; + assert( pOpen->nLock>=0 ); + if( pOpen->nLock==0 && pOpen->nPending>0 ){ + int i; + for(i=0; inPending; i++){ + if( pOpen->aPending[i] < 0 ) continue; + if( close(pOpen->aPending[i]) ){ + pFile->lastErrno = errno; + rc = SQLITE_IOERR_CLOSE; + }else{ + pOpen->aPending[i] = -1; + } } - if( zFull[i+1]=='.' && zFull[i+2]=='.' && zFull[i+3]=='/' ){ - while( j>0 && zFull[j-1]!='/' ){ j--; } - i += 3; - continue; + if( rc==SQLITE_OK ){ + sqlite3_free(pOpen->aPending); + pOpen->nPending = 0; + pOpen->aPending = 0; } } - zFull[j++] = zFull[i]; } - zFull[j] = 0; + } + unixLeaveMutex(); + if( rc==SQLITE_OK ) pFile->locktype = locktype; + return rc; +} + +/* +** Close a file & cleanup AFP specific locking context +*/ +static int afpClose(sqlite3_file *id) { + if( id ){ + unixFile *pFile = (unixFile*)id; + afpUnlock(id, NO_LOCK); + unixEnterMutex(); + if( pFile->pOpen && pFile->pOpen->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pOpen->aPending. It will be automatically closed when + ** the last lock is cleared. + */ + int *aNew; + struct unixOpenCnt *pOpen = pFile->pOpen; + aNew = sqlite3_realloc(pOpen->aPending, (pOpen->nPending+1)*sizeof(int) ); + if( aNew==0 ){ + /* If a malloc fails, just leak the file descriptor */ + }else{ + pOpen->aPending = aNew; + pOpen->aPending[pOpen->nPending] = pFile->h; + pOpen->nPending++; + pFile->h = -1; + } + } + releaseOpenCnt(pFile->pOpen); + sqlite3_free(pFile->lockingContext); + closeUnixFile(id); + unixLeaveMutex(); + } + return SQLITE_OK; +} + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The code above is the AFP lock implementation. The code is specific +** to MacOSX and does not work on other unix platforms. No alternative +** is available. If you don't compile for a mac, then the "unix-afp" +** VFS is not available. +** +********************* End of the AFP lock implementation ********************** +******************************************************************************/ + + +/****************************************************************************** +**************** Non-locking sqlite3_file methods ***************************** +** +** The next division contains implementations for all methods of the +** sqlite3_file object other than the locking methods. The locking +** methods were defined in divisions above (one locking method per +** division). Those methods that are common to all locking modes +** are gather together into this division. +*/ + +/* +** Seek to the offset passed as the second argument, then read cnt +** bytes into pBuf. Return the number of bytes actually read. +** +** NB: If you define USE_PREAD or USE_PREAD64, then it might also +** be necessary to define _XOPEN_SOURCE to be 500. This varies from +** one system to another. Since SQLite does not define USE_PREAD +** any any form by default, we will not attempt to define _XOPEN_SOURCE. +** See tickets #2741 and #2681. +** +** To avoid stomping the errno value on a failed read the lastErrno value +** is set before returning. +*/ +static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ + int got; + i64 newOffset; + TIMER_START; +#if defined(USE_PREAD) + got = pread(id->h, pBuf, cnt, offset); + SimulateIOError( got = -1 ); +#elif defined(USE_PREAD64) + got = pread64(id->h, pBuf, cnt, offset); + SimulateIOError( got = -1 ); +#else + newOffset = lseek(id->h, offset, SEEK_SET); + SimulateIOError( newOffset-- ); + if( newOffset!=offset ){ + if( newOffset == -1 ){ + ((unixFile*)id)->lastErrno = errno; + }else{ + ((unixFile*)id)->lastErrno = 0; + } + return -1; + } + got = read(id->h, pBuf, cnt); +#endif + TIMER_END; + if( got<0 ){ + ((unixFile*)id)->lastErrno = errno; + } + OSTRACE5("READ %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED); + return got; +} + +/* +** Read data from a file into a buffer. Return SQLITE_OK if all +** bytes were read successfully and SQLITE_IOERR if anything goes +** wrong. +*/ +static int unixRead( + sqlite3_file *id, + void *pBuf, + int amt, + sqlite3_int64 offset +){ + int got; + assert( id ); + got = seekAndRead((unixFile*)id, offset, pBuf, amt); + if( got==amt ){ + return SQLITE_OK; + }else if( got<0 ){ + /* lastErrno set by seekAndRead */ + return SQLITE_IOERR_READ; + }else{ + ((unixFile*)id)->lastErrno = 0; /* not a system error */ + /* Unread parts of the buffer must be zero-filled */ + memset(&((char*)pBuf)[got], 0, amt-got); + return SQLITE_IOERR_SHORT_READ; + } +} + +/* +** Seek to the offset in id->offset then read cnt bytes into pBuf. +** Return the number of bytes actually read. Update the offset. +** +** To avoid stomping the errno value on a failed write the lastErrno value +** is set before returning. +*/ +static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ + int got; + i64 newOffset; + TIMER_START; +#if defined(USE_PREAD) + got = pwrite(id->h, pBuf, cnt, offset); +#elif defined(USE_PREAD64) + got = pwrite64(id->h, pBuf, cnt, offset); +#else + newOffset = lseek(id->h, offset, SEEK_SET); + if( newOffset!=offset ){ + if( newOffset == -1 ){ + ((unixFile*)id)->lastErrno = errno; + }else{ + ((unixFile*)id)->lastErrno = 0; + } + return -1; + } + got = write(id->h, pBuf, cnt); +#endif + TIMER_END; + if( got<0 ){ + ((unixFile*)id)->lastErrno = errno; + } + + OSTRACE5("WRITE %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED); + return got; +} + + +/* +** Write data from a buffer into a file. Return SQLITE_OK on success +** or some other error code on failure. +*/ +static int unixWrite( + sqlite3_file *id, + const void *pBuf, + int amt, + sqlite3_int64 offset +){ + int wrote = 0; + assert( id ); + assert( amt>0 ); + +#ifndef NDEBUG + /* If we are doing a normal write to a database file (as opposed to + ** doing a hot-journal rollback or a write to some file other than a + ** normal database file) then record the fact that the database + ** has changed. If the transaction counter is modified, record that + ** fact too. + */ + if( ((unixFile*)id)->inNormalWrite ){ + unixFile *pFile = (unixFile*)id; + pFile->dbUpdate = 1; /* The database has been modified */ + if( offset<=24 && offset+amt>=27 ){ + char oldCntr[4]; + SimulateIOErrorBenign(1); + seekAndRead(pFile, 24, oldCntr, 4); + SimulateIOErrorBenign(0); + if( memcmp(oldCntr, &((char*)pBuf)[24-offset], 4)!=0 ){ + pFile->transCntrChng = 1; /* The transaction counter has changed */ + } + } } #endif - return zFull; + while( amt>0 && (wrote = seekAndWrite((unixFile*)id, offset, pBuf, amt))>0 ){ + amt -= wrote; + offset += wrote; + pBuf = &((char*)pBuf)[wrote]; + } + SimulateIOError(( wrote=(-1), amt=1 )); + SimulateDiskfullError(( wrote=0, amt=1 )); + if( amt>0 ){ + if( wrote<0 ){ + /* lastErrno set by seekAndWrite */ + return SQLITE_IOERR_WRITE; + }else{ + ((unixFile*)id)->lastErrno = 0; /* not a system error */ + return SQLITE_FULL; + } + } + return SQLITE_OK; +} + +#ifdef SQLITE_TEST +/* +** Count the number of fullsyncs and normal syncs. This is used to test +** that syncs and fullsyncs are occurring at the right times. +*/ +int sqlite3_sync_count = 0; +int sqlite3_fullsync_count = 0; +#endif + +/* +** Use the fdatasync() API only if the HAVE_FDATASYNC macro is defined. +** Otherwise use fsync() in its place. +*/ +#ifndef HAVE_FDATASYNC +# define fdatasync fsync +#endif + +/* +** Define HAVE_FULLFSYNC to 0 or 1 depending on whether or not +** the F_FULLFSYNC macro is defined. F_FULLFSYNC is currently +** only available on Mac OS X. But that could change. +*/ +#ifdef F_FULLFSYNC +# define HAVE_FULLFSYNC 1 +#else +# define HAVE_FULLFSYNC 0 +#endif + + +/* +** The fsync() system call does not work as advertised on many +** unix systems. The following procedure is an attempt to make +** it work better. +** +** The SQLITE_NO_SYNC macro disables all fsync()s. This is useful +** for testing when we want to run through the test suite quickly. +** You are strongly advised *not* to deploy with SQLITE_NO_SYNC +** enabled, however, since with SQLITE_NO_SYNC enabled, an OS crash +** or power failure will likely corrupt the database file. +*/ +static int full_fsync(int fd, int fullSync, int dataOnly){ + int rc; + + /* The following "ifdef/elif/else/" block has the same structure as + ** the one below. It is replicated here solely to avoid cluttering + ** up the real code with the UNUSED_PARAMETER() macros. + */ +#ifdef SQLITE_NO_SYNC + UNUSED_PARAMETER(fd); + UNUSED_PARAMETER(fullSync); + UNUSED_PARAMETER(dataOnly); +#elif HAVE_FULLFSYNC + UNUSED_PARAMETER(dataOnly); +#else + UNUSED_PARAMETER(fullSync); +#endif + + /* Record the number of times that we do a normal fsync() and + ** FULLSYNC. This is used during testing to verify that this procedure + ** gets called with the correct arguments. + */ +#ifdef SQLITE_TEST + if( fullSync ) sqlite3_fullsync_count++; + sqlite3_sync_count++; +#endif + + /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a + ** no-op + */ +#ifdef SQLITE_NO_SYNC + rc = SQLITE_OK; +#elif HAVE_FULLFSYNC + if( fullSync ){ + rc = fcntl(fd, F_FULLFSYNC, 0); + }else{ + rc = 1; + } + /* If the FULLFSYNC failed, fall back to attempting an fsync(). + ** It shouldn't be possible for fullfsync to fail on the local + ** file system (on OSX), so failure indicates that FULLFSYNC + ** isn't supported for this file system. So, attempt an fsync + ** and (for now) ignore the overhead of a superfluous fcntl call. + ** It'd be better to detect fullfsync support once and avoid + ** the fcntl call every time sync is called. + */ + if( rc ) rc = fsync(fd); + +#else + if( dataOnly ){ + rc = fdatasync(fd); +#if OS_VXWORKS + if( rc==-1 && errno==ENOTSUP ){ + rc = fsync(fd); + } +#endif + }else{ + rc = fsync(fd); + } +#endif /* ifdef SQLITE_NO_SYNC elif HAVE_FULLFSYNC */ + + if( OS_VXWORKS && rc!= -1 ){ + rc = 0; + } + return rc; } /* -** Change the value of the fullsync flag in the given file descriptor. +** Make sure all writes to a particular file are committed to disk. +** +** If dataOnly==0 then both the file itself and its metadata (file +** size, access time, etc) are synced. If dataOnly!=0 then only the +** file data is synced. +** +** Under Unix, also make sure that the directory entry for the file +** has been created by fsync-ing the directory that contains the file. +** If we do not do this and we encounter a power failure, the directory +** entry for the journal might not exist after we reboot. The next +** SQLite to access the file will not know that the journal exists (because +** the directory entry for the journal was never created) and the transaction +** will not roll back - possibly leading to database corruption. */ -static void unixSetFullSync(OsFile *id, int v){ - ((unixFile*)id)->fullSync = v; +static int unixSync(sqlite3_file *id, int flags){ + int rc; + unixFile *pFile = (unixFile*)id; + + int isDataOnly = (flags&SQLITE_SYNC_DATAONLY); + int isFullsync = (flags&0x0F)==SQLITE_SYNC_FULL; + + /* Check that one of SQLITE_SYNC_NORMAL or FULL was passed */ + assert((flags&0x0F)==SQLITE_SYNC_NORMAL + || (flags&0x0F)==SQLITE_SYNC_FULL + ); + + /* Unix cannot, but some systems may return SQLITE_FULL from here. This + ** line is to test that doing so does not cause any problems. + */ + SimulateDiskfullError( return SQLITE_FULL ); + + assert( pFile ); + OSTRACE2("SYNC %-3d\n", pFile->h); + rc = full_fsync(pFile->h, isFullsync, isDataOnly); + SimulateIOError( rc=1 ); + if( rc ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_FSYNC; + } + if( pFile->dirfd>=0 ){ + int err; + OSTRACE4("DIRSYNC %-3d (have_fullfsync=%d fullsync=%d)\n", pFile->dirfd, + HAVE_FULLFSYNC, isFullsync); +#ifndef SQLITE_DISABLE_DIRSYNC + /* The directory sync is only attempted if full_fsync is + ** turned off or unavailable. If a full_fsync occurred above, + ** then the directory sync is superfluous. + */ + if( (!HAVE_FULLFSYNC || !isFullsync) && full_fsync(pFile->dirfd,0,0) ){ + /* + ** We have received multiple reports of fsync() returning + ** errors when applied to directories on certain file systems. + ** A failed directory sync is not a big deal. So it seems + ** better to ignore the error. Ticket #1657 + */ + /* pFile->lastErrno = errno; */ + /* return SQLITE_IOERR; */ + } +#endif + err = close(pFile->dirfd); /* Only need to sync once, so close the */ + if( err==0 ){ /* directory when we are done */ + pFile->dirfd = -1; + }else{ + pFile->lastErrno = errno; + rc = SQLITE_IOERR_DIR_CLOSE; + } + } + return rc; } /* -** Return the underlying file handle for an OsFile +** Truncate an open file to a specified size */ -static int unixFileHandle(OsFile *id){ - return ((unixFile*)id)->h; +static int unixTruncate(sqlite3_file *id, i64 nByte){ + int rc; + assert( id ); + SimulateIOError( return SQLITE_IOERR_TRUNCATE ); + rc = ftruncate(((unixFile*)id)->h, (off_t)nByte); + if( rc ){ + ((unixFile*)id)->lastErrno = errno; + return SQLITE_IOERR_TRUNCATE; + }else{ + return SQLITE_OK; + } } /* -** Return an integer that indices the type of lock currently held -** by this handle. (Used for testing and analysis only.) +** Determine the current size of a file in bytes */ -static int unixLockState(OsFile *id){ - return ((unixFile*)id)->locktype; +static int unixFileSize(sqlite3_file *id, i64 *pSize){ + int rc; + struct stat buf; + assert( id ); + rc = fstat(((unixFile*)id)->h, &buf); + SimulateIOError( rc=1 ); + if( rc!=0 ){ + ((unixFile*)id)->lastErrno = errno; + return SQLITE_IOERR_FSTAT; + } + *pSize = buf.st_size; + + /* When opening a zero-size database, the findLockInfo() procedure + ** writes a single byte into that file in order to work around a bug + ** in the OS-X msdos filesystem. In order to avoid problems with upper + ** layers, we need to report this file size as zero even though it is + ** really 1. Ticket #3260. + */ + if( *pSize==1 ) *pSize = 0; + + + return SQLITE_OK; +} + +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) +/* +** Handler for proxy-locking file-control verbs. Defined below in the +** proxying locking division. +*/ +static int proxyFileControl(sqlite3_file*,int,void*); +#endif + + +/* +** Information and control of an open file handle. +*/ +static int unixFileControl(sqlite3_file *id, int op, void *pArg){ + switch( op ){ + case SQLITE_FCNTL_LOCKSTATE: { + *(int*)pArg = ((unixFile*)id)->locktype; + return SQLITE_OK; + } + case SQLITE_LAST_ERRNO: { + *(int*)pArg = ((unixFile*)id)->lastErrno; + return SQLITE_OK; + } +#ifndef NDEBUG + /* The pager calls this method to signal that it has done + ** a rollback and that the database is therefore unchanged and + ** it hence it is OK for the transaction change counter to be + ** unchanged. + */ + case SQLITE_FCNTL_DB_UNCHANGED: { + ((unixFile*)id)->dbUpdate = 0; + return SQLITE_OK; + } +#endif +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + case SQLITE_SET_LOCKPROXYFILE: + case SQLITE_GET_LOCKPROXYFILE: { + return proxyFileControl(id,op,pArg); + } +#endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ + } + return SQLITE_ERROR; } /* @@ -2345,284 +3058,779 @@ static int unixLockState(OsFile *id){ ** ** SQLite code assumes this function cannot fail. It also assumes that ** if two files are created in the same file-system directory (i.e. -** a database and it's journal file) that the sector size will be the +** a database and its journal file) that the sector size will be the ** same for both. */ -static int unixSectorSize(OsFile *id){ +static int unixSectorSize(sqlite3_file *NotUsed){ + UNUSED_PARAMETER(NotUsed); return SQLITE_DEFAULT_SECTOR_SIZE; } /* -** This vector defines all the methods that can operate on an OsFile -** for unix. +** Return the device characteristics for the file. This is always 0 for unix. */ -static const IoMethod sqlite3UnixIoMethod = { - unixClose, - unixOpenDirectory, - unixRead, - unixWrite, - unixSeek, - unixTruncate, - unixSync, - unixSetFullSync, - unixFileHandle, - unixFileSize, - unixLock, - unixUnlock, - unixLockState, - unixCheckReservedLock, - unixSectorSize, -}; - -#ifdef SQLITE_ENABLE_LOCKING_STYLE -/* - ** This vector defines all the methods that can operate on an OsFile - ** for unix with AFP style file locking. - */ -static const IoMethod sqlite3AFPLockingUnixIoMethod = { - afpUnixClose, - unixOpenDirectory, - unixRead, - unixWrite, - unixSeek, - unixTruncate, - unixSync, - unixSetFullSync, - unixFileHandle, - unixFileSize, - afpUnixLock, - afpUnixUnlock, - unixLockState, - afpUnixCheckReservedLock, - unixSectorSize, -}; +static int unixDeviceCharacteristics(sqlite3_file *NotUsed){ + UNUSED_PARAMETER(NotUsed); + return 0; +} /* - ** This vector defines all the methods that can operate on an OsFile - ** for unix with flock() style file locking. - */ -static const IoMethod sqlite3FlockLockingUnixIoMethod = { - flockUnixClose, - unixOpenDirectory, - unixRead, - unixWrite, - unixSeek, - unixTruncate, - unixSync, - unixSetFullSync, - unixFileHandle, - unixFileSize, - flockUnixLock, - flockUnixUnlock, - unixLockState, - flockUnixCheckReservedLock, - unixSectorSize, -}; +** Here ends the implementation of all sqlite3_file methods. +** +********************** End sqlite3_file Methods ******************************* +******************************************************************************/ /* - ** This vector defines all the methods that can operate on an OsFile - ** for unix with dotlock style file locking. - */ -static const IoMethod sqlite3DotlockLockingUnixIoMethod = { - dotlockUnixClose, - unixOpenDirectory, - unixRead, - unixWrite, - unixSeek, - unixTruncate, - unixSync, - unixSetFullSync, - unixFileHandle, - unixFileSize, - dotlockUnixLock, - dotlockUnixUnlock, - unixLockState, - dotlockUnixCheckReservedLock, - unixSectorSize, -}; - -/* - ** This vector defines all the methods that can operate on an OsFile - ** for unix with dotlock style file locking. - */ -static const IoMethod sqlite3NolockLockingUnixIoMethod = { - nolockUnixClose, - unixOpenDirectory, - unixRead, - unixWrite, - unixSeek, - unixTruncate, - unixSync, - unixSetFullSync, - unixFileHandle, - unixFileSize, - nolockUnixLock, - nolockUnixUnlock, - unixLockState, - nolockUnixCheckReservedLock, - unixSectorSize, -}; - -#endif /* SQLITE_ENABLE_LOCKING_STYLE */ - -/* -** Allocate memory for a new unixFile and initialize that unixFile. -** Write a pointer to the new unixFile into *pId. -** If we run out of memory, close the file and return an error. +** This division contains definitions of sqlite3_io_methods objects that +** implement various file locking strategies. It also contains definitions +** of "finder" functions. A finder-function is used to locate the appropriate +** sqlite3_io_methods object for a particular database file. The pAppData +** field of the sqlite3_vfs VFS objects are initialized to be pointers to +** the correct finder-function for that VFS. +** +** Most finder functions return a pointer to a fixed sqlite3_io_methods +** object. The only interesting finder-function is autolockIoFinder, which +** looks at the filesystem type and tries to guess the best locking +** strategy from that. +** +** For finder-funtion F, two objects are created: +** +** (1) The real finder-function named "FImpt()". +** +** (2) A constant pointer to this functio named just "F". +** +** +** A pointer to the F pointer is used as the pAppData value for VFS +** objects. We have to do this instead of letting pAppData point +** directly at the finder-function since C90 rules prevent a void* +** from be cast into a function pointer. +** +** +** Each instance of this macro generates two objects: +** +** * A constant sqlite3_io_methods object call METHOD that has locking +** methods CLOSE, LOCK, UNLOCK, CKRESLOCK. +** +** * An I/O method finder function called FINDER that returns a pointer +** to the METHOD object in the previous bullet. */ -#ifdef SQLITE_ENABLE_LOCKING_STYLE +#define IOMETHODS(FINDER, METHOD, CLOSE, LOCK, UNLOCK, CKLOCK) \ +static const sqlite3_io_methods METHOD = { \ + 1, /* iVersion */ \ + CLOSE, /* xClose */ \ + unixRead, /* xRead */ \ + unixWrite, /* xWrite */ \ + unixTruncate, /* xTruncate */ \ + unixSync, /* xSync */ \ + unixFileSize, /* xFileSize */ \ + LOCK, /* xLock */ \ + UNLOCK, /* xUnlock */ \ + CKLOCK, /* xCheckReservedLock */ \ + unixFileControl, /* xFileControl */ \ + unixSectorSize, /* xSectorSize */ \ + unixDeviceCharacteristics /* xDeviceCapabilities */ \ +}; \ +static const sqlite3_io_methods *FINDER##Impl(const char *z, int h){ \ + UNUSED_PARAMETER(z); UNUSED_PARAMETER(h); \ + return &METHOD; \ +} \ +static const sqlite3_io_methods *(*const FINDER)(const char*,int) \ + = FINDER##Impl; + +/* +** Here are all of the sqlite3_io_methods objects for each of the +** locking strategies. Functions that return pointers to these methods +** are also created. +*/ +IOMETHODS( + posixIoFinder, /* Finder function name */ + posixIoMethods, /* sqlite3_io_methods object name */ + unixClose, /* xClose method */ + unixLock, /* xLock method */ + unixUnlock, /* xUnlock method */ + unixCheckReservedLock /* xCheckReservedLock method */ +) +IOMETHODS( + nolockIoFinder, /* Finder function name */ + nolockIoMethods, /* sqlite3_io_methods object name */ + nolockClose, /* xClose method */ + nolockLock, /* xLock method */ + nolockUnlock, /* xUnlock method */ + nolockCheckReservedLock /* xCheckReservedLock method */ +) +IOMETHODS( + dotlockIoFinder, /* Finder function name */ + dotlockIoMethods, /* sqlite3_io_methods object name */ + dotlockClose, /* xClose method */ + dotlockLock, /* xLock method */ + dotlockUnlock, /* xUnlock method */ + dotlockCheckReservedLock /* xCheckReservedLock method */ +) + +#if SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + flockIoFinder, /* Finder function name */ + flockIoMethods, /* sqlite3_io_methods object name */ + flockClose, /* xClose method */ + flockLock, /* xLock method */ + flockUnlock, /* xUnlock method */ + flockCheckReservedLock /* xCheckReservedLock method */ +) +#endif + +#if OS_VXWORKS +IOMETHODS( + semIoFinder, /* Finder function name */ + semIoMethods, /* sqlite3_io_methods object name */ + semClose, /* xClose method */ + semLock, /* xLock method */ + semUnlock, /* xUnlock method */ + semCheckReservedLock /* xCheckReservedLock method */ +) +#endif + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + afpIoFinder, /* Finder function name */ + afpIoMethods, /* sqlite3_io_methods object name */ + afpClose, /* xClose method */ + afpLock, /* xLock method */ + afpUnlock, /* xUnlock method */ + afpCheckReservedLock /* xCheckReservedLock method */ +) +#endif + +/* +** The proxy locking method is a "super-method" in the sense that it +** opens secondary file descriptors for the conch and lock files and +** it uses proxy, dot-file, AFP, and flock() locking methods on those +** secondary files. For this reason, the division that implements +** proxy locking is located much further down in the file. But we need +** to go ahead and define the sqlite3_io_methods and finder function +** for proxy locking here. So we forward declare the I/O methods. +*/ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +static int proxyClose(sqlite3_file*); +static int proxyLock(sqlite3_file*, int); +static int proxyUnlock(sqlite3_file*, int); +static int proxyCheckReservedLock(sqlite3_file*, int*); +IOMETHODS( + proxyIoFinder, /* Finder function name */ + proxyIoMethods, /* sqlite3_io_methods object name */ + proxyClose, /* xClose method */ + proxyLock, /* xLock method */ + proxyUnlock, /* xUnlock method */ + proxyCheckReservedLock /* xCheckReservedLock method */ +) +#endif + + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE /* - ** When locking extensions are enabled, the filepath and locking style - ** are needed to determine the unixFile pMethod to use for locking operations. - ** The locking-style specific lockingContext data structure is created - ** and assigned here also. - */ -static int allocateUnixFile( - int h, /* Open file descriptor of file being opened */ - OsFile **pId, /* Write completed initialization here */ - const char *zFilename, /* Name of the file being opened */ - int delFlag /* Delete-on-or-before-close flag */ +** This "finder" function attempts to determine the best locking strategy +** for the database file "filePath". It then returns the sqlite3_io_methods +** object that implements that strategy. +** +** This is for MacOSX only. +*/ +static const sqlite3_io_methods *autolockIoFinderImpl( + const char *filePath, /* name of the database file */ + int fd /* file descriptor open on the database file */ ){ - sqlite3LockingStyle lockingStyle; - unixFile *pNew; - unixFile f; - int rc; + static const struct Mapping { + const char *zFilesystem; /* Filesystem type name */ + const sqlite3_io_methods *pMethods; /* Appropriate locking method */ + } aMap[] = { + { "hfs", &posixIoMethods }, + { "ufs", &posixIoMethods }, + { "afpfs", &afpIoMethods }, +#ifdef SQLITE_ENABLE_AFP_LOCKING_SMB + { "smbfs", &afpIoMethods }, +#else + { "smbfs", &flockIoMethods }, +#endif + { "webdav", &nolockIoMethods }, + { 0, 0 } + }; + int i; + struct statfs fsInfo; + struct flock lockInfo; - memset(&f, 0, sizeof(f)); - lockingStyle = sqlite3DetectLockingStyle(zFilename, h); - if ( lockingStyle == posixLockingStyle ) { - sqlite3OsEnterMutex(); - rc = findLockInfo(h, &f.pLock, &f.pOpen); - sqlite3OsLeaveMutex(); - if( rc ){ - close(h); - unlink(zFilename); - return SQLITE_NOMEM; + if( !filePath ){ + /* If filePath==NULL that means we are dealing with a transient file + ** that does not need to be locked. */ + return &nolockIoMethods; + } + if( statfs(filePath, &fsInfo) != -1 ){ + if( fsInfo.f_flags & MNT_RDONLY ){ + return &nolockIoMethods; + } + for(i=0; aMap[i].zFilesystem; i++){ + if( strcmp(fsInfo.f_fstypename, aMap[i].zFilesystem)==0 ){ + return aMap[i].pMethods; + } } - } else { - /* pLock and pOpen are only used for posix advisory locking */ - f.pLock = NULL; - f.pOpen = NULL; } - if( delFlag ){ - unlink(zFilename); - } - f.dirfd = -1; - f.h = h; - SET_THREADID(&f); - pNew = sqlite3ThreadSafeMalloc( sizeof(unixFile) ); - if( pNew==0 ){ - close(h); - sqlite3OsEnterMutex(); - releaseLockInfo(f.pLock); - releaseOpenCnt(f.pOpen); - sqlite3OsLeaveMutex(); - *pId = 0; - return SQLITE_NOMEM; + + /* Default case. Handles, amongst others, "nfs". + ** Test byte-range lock using fcntl(). If the call succeeds, + ** assume that the file-system supports POSIX style locks. + */ + lockInfo.l_len = 1; + lockInfo.l_start = 0; + lockInfo.l_whence = SEEK_SET; + lockInfo.l_type = F_RDLCK; + if( fcntl(fd, F_GETLK, &lockInfo)!=-1 ) { + return &posixIoMethods; }else{ - *pNew = f; - switch(lockingStyle) { - case afpLockingStyle: { - /* afp locking uses the file path so it needs to be included in - ** the afpLockingContext */ - int nFilename; - pNew->pMethod = &sqlite3AFPLockingUnixIoMethod; - pNew->lockingContext = - sqlite3ThreadSafeMalloc(sizeof(afpLockingContext)); - nFilename = strlen(zFilename)+1; - ((afpLockingContext *)pNew->lockingContext)->filePath = - sqlite3ThreadSafeMalloc(nFilename); - memcpy(((afpLockingContext *)pNew->lockingContext)->filePath, - zFilename, nFilename); - srandomdev(); - break; - } - case flockLockingStyle: - /* flock locking doesn't need additional lockingContext information */ - pNew->pMethod = &sqlite3FlockLockingUnixIoMethod; - break; - case dotlockLockingStyle: { - /* dotlock locking uses the file path so it needs to be included in - ** the dotlockLockingContext */ - int nFilename; - pNew->pMethod = &sqlite3DotlockLockingUnixIoMethod; - pNew->lockingContext = sqlite3ThreadSafeMalloc( - sizeof(dotlockLockingContext)); - nFilename = strlen(zFilename) + 6; - ((dotlockLockingContext *)pNew->lockingContext)->lockPath = - sqlite3ThreadSafeMalloc( nFilename ); - sqlite3_snprintf(nFilename, - ((dotlockLockingContext *)pNew->lockingContext)->lockPath, - "%s.lock", zFilename); - break; - } - case posixLockingStyle: - /* posix locking doesn't need additional lockingContext information */ - pNew->pMethod = &sqlite3UnixIoMethod; - break; - case noLockingStyle: - case unsupportedLockingStyle: - default: - pNew->pMethod = &sqlite3NolockLockingUnixIoMethod; - } - *pId = (OsFile*)pNew; - OpenCounter(+1); - return SQLITE_OK; + return &dotlockIoMethods; } } -#else /* SQLITE_ENABLE_LOCKING_STYLE */ -static int allocateUnixFile( - int h, /* Open file descriptor on file being opened */ - OsFile **pId, /* Write the resul unixFile structure here */ - const char *zFilename, /* Name of the file being opened */ - int delFlag /* If true, delete the file on or before closing */ +static const sqlite3_io_methods *(*const autolockIoFinder)(const char*,int) + = autolockIoFinderImpl; + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ + +/* +** An abstract type for a pointer to a IO method finder function: +*/ +typedef const sqlite3_io_methods *(*finder_type)(const char*,int); + + +/**************************************************************************** +**************************** sqlite3_vfs methods **************************** +** +** This division contains the implementation of methods on the +** sqlite3_vfs object. +*/ + +/* +** Initialize the contents of the unixFile structure pointed to by pId. +*/ +static int fillInUnixFile( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + int h, /* Open file descriptor of file being opened */ + int dirfd, /* Directory file descriptor */ + sqlite3_file *pId, /* Write to the unixFile structure here */ + const char *zFilename, /* Name of the file being opened */ + int noLock, /* Omit locking if true */ + int isDelete /* Delete on close if true */ ){ - unixFile *pNew; - unixFile f; - int rc; + const sqlite3_io_methods *pLockingStyle; + unixFile *pNew = (unixFile *)pId; + int rc = SQLITE_OK; + + assert( pNew->pLock==NULL ); + assert( pNew->pOpen==NULL ); + + /* Parameter isDelete is only used on vxworks. + ** Express this explicitly here to prevent compiler warnings + ** about unused parameters. + */ +#if !OS_VXWORKS + UNUSED_PARAMETER(isDelete); +#endif + + OSTRACE3("OPEN %-3d %s\n", h, zFilename); + pNew->h = h; + pNew->dirfd = dirfd; + SET_THREADID(pNew); + +#if OS_VXWORKS + pNew->pId = vxworksFindFileId(zFilename); + if( pNew->pId==0 ){ + noLock = 1; + rc = SQLITE_NOMEM; + } +#endif + + if( noLock ){ + pLockingStyle = &nolockIoMethods; + }else{ + pLockingStyle = (**(finder_type*)pVfs->pAppData)(zFilename, h); +#if SQLITE_ENABLE_LOCKING_STYLE + /* Cache zFilename in the locking context (AFP and dotlock override) for + ** proxyLock activation is possible (remote proxy is based on db name) + ** zFilename remains valid until file is closed, to support */ + pNew->lockingContext = (void*)zFilename; +#endif + } + + if( pLockingStyle == &posixIoMethods ){ + unixEnterMutex(); + rc = findLockInfo(pNew, &pNew->pLock, &pNew->pOpen); + unixLeaveMutex(); + } + +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + else if( pLockingStyle == &afpIoMethods ){ + /* AFP locking uses the file path so it needs to be included in + ** the afpLockingContext. + */ + afpLockingContext *pCtx; + pNew->lockingContext = pCtx = sqlite3_malloc( sizeof(*pCtx) ); + if( pCtx==0 ){ + rc = SQLITE_NOMEM; + }else{ + /* NB: zFilename exists and remains valid until the file is closed + ** according to requirement F11141. So we do not need to make a + ** copy of the filename. */ + pCtx->dbPath = zFilename; + srandomdev(); + unixEnterMutex(); + rc = findLockInfo(pNew, NULL, &pNew->pOpen); + unixLeaveMutex(); + } + } +#endif + + else if( pLockingStyle == &dotlockIoMethods ){ + /* Dotfile locking uses the file path so it needs to be included in + ** the dotlockLockingContext + */ + char *zLockFile; + int nFilename; + nFilename = (int)strlen(zFilename) + 6; + zLockFile = (char *)sqlite3_malloc(nFilename); + if( zLockFile==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_snprintf(nFilename, zLockFile, "%s" DOTLOCK_SUFFIX, zFilename); + } + pNew->lockingContext = zLockFile; + } + +#if OS_VXWORKS + else if( pLockingStyle == &semIoMethods ){ + /* Named semaphore locking uses the file path so it needs to be + ** included in the semLockingContext + */ + unixEnterMutex(); + rc = findLockInfo(pNew, &pNew->pLock, &pNew->pOpen); + if( (rc==SQLITE_OK) && (pNew->pOpen->pSem==NULL) ){ + char *zSemName = pNew->pOpen->aSemName; + int n; + sqlite3_snprintf(MAX_PATHNAME, zSemName, "%s.sem", + pNew->pId->zCanonicalName); + for( n=0; zSemName[n]; n++ ) + if( zSemName[n]=='/' ) zSemName[n] = '_'; + pNew->pOpen->pSem = sem_open(zSemName, O_CREAT, 0666, 1); + if( pNew->pOpen->pSem == SEM_FAILED ){ + rc = SQLITE_NOMEM; + pNew->pOpen->aSemName[0] = '\0'; + } + } + unixLeaveMutex(); + } +#endif + + pNew->lastErrno = 0; +#if OS_VXWORKS + if( rc!=SQLITE_OK ){ + unlink(zFilename); + isDelete = 0; + } + pNew->isDelete = isDelete; +#endif + if( rc!=SQLITE_OK ){ + if( dirfd>=0 ) close(dirfd); /* silent leak if fail, already in error */ + close(h); + }else{ + pNew->pMethod = pLockingStyle; + OpenCounter(+1); + } + return rc; +} + +/* +** Open a file descriptor to the directory containing file zFilename. +** If successful, *pFd is set to the opened file descriptor and +** SQLITE_OK is returned. If an error occurs, either SQLITE_NOMEM +** or SQLITE_CANTOPEN is returned and *pFd is set to an undefined +** value. +** +** If SQLITE_OK is returned, the caller is responsible for closing +** the file descriptor *pFd using close(). +*/ +static int openDirectory(const char *zFilename, int *pFd){ + int ii; + int fd = -1; + char zDirname[MAX_PATHNAME+1]; + + sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); + for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--); + if( ii>0 ){ + zDirname[ii] = '\0'; + fd = open(zDirname, O_RDONLY|O_BINARY, 0); + if( fd>=0 ){ +#ifdef FD_CLOEXEC + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD, 0) | FD_CLOEXEC); +#endif + OSTRACE3("OPENDIR %-3d %s\n", fd, zDirname); + } + } + *pFd = fd; + return (fd>=0?SQLITE_OK:SQLITE_CANTOPEN); +} + +/* +** Create a temporary file name in zBuf. zBuf must be allocated +** by the calling process and must be big enough to hold at least +** pVfs->mxPathname bytes. +*/ +static int getTempname(int nBuf, char *zBuf){ + static const char *azDirs[] = { + 0, + 0, + "/var/tmp", + "/usr/tmp", + "/tmp", + ".", + }; + static const unsigned char zChars[] = + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789"; + unsigned int i, j; + struct stat buf; + const char *zDir = "."; + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. + */ + SimulateIOError( return SQLITE_IOERR ); + + azDirs[0] = sqlite3_temp_directory; + if (NULL == azDirs[1]) { + azDirs[1] = getenv("TMPDIR"); + } + + for(i=0; i= (size_t)nBuf ){ + return SQLITE_ERROR; + } + + do{ + sqlite3_snprintf(nBuf-17, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX, zDir); + j = (int)strlen(zBuf); + sqlite3_randomness(15, &zBuf[j]); + for(i=0; i<15; i++, j++){ + zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; + } + zBuf[j] = 0; + }while( access(zBuf,0)==0 ); + return SQLITE_OK; +} + +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) +/* +** Routine to transform a unixFile into a proxy-locking unixFile. +** Implementation in the proxy-lock division, but used by unixOpen() +** if SQLITE_PREFER_PROXY_LOCKING is defined. +*/ +static int proxyTransformUnixFile(unixFile*, const char*); +#endif + + +/* +** Open the file zPath. +** +** Previously, the SQLite OS layer used three functions in place of this +** one: +** +** sqlite3OsOpenReadWrite(); +** sqlite3OsOpenReadOnly(); +** sqlite3OsOpenExclusive(); +** +** These calls correspond to the following combinations of flags: +** +** ReadWrite() -> (READWRITE | CREATE) +** ReadOnly() -> (READONLY) +** OpenExclusive() -> (READWRITE | CREATE | EXCLUSIVE) +** +** The old OpenExclusive() accepted a boolean argument - "delFlag". If +** true, the file was configured to be automatically deleted when the +** file handle closed. To achieve the same effect using this new +** interface, add the DELETEONCLOSE flag to those specified above for +** OpenExclusive(). +*/ +static int unixOpen( + sqlite3_vfs *pVfs, /* The VFS for which this is the xOpen method */ + const char *zPath, /* Pathname of file to be opened */ + sqlite3_file *pFile, /* The file descriptor to be filled in */ + int flags, /* Input flags to control the opening */ + int *pOutFlags /* Output flags returned to SQLite core */ +){ + int fd = 0; /* File descriptor returned by open() */ + int dirfd = -1; /* Directory file descriptor */ + int openFlags = 0; /* Flags to pass to open() */ + int eType = flags&0xFFFFFF00; /* Type of file to open */ + int noLock; /* True to omit locking primitives */ + int rc = SQLITE_OK; + + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isCreate = (flags & SQLITE_OPEN_CREATE); + int isReadonly = (flags & SQLITE_OPEN_READONLY); + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); + + /* If creating a master or main-file journal, this function will open + ** a file-descriptor on the directory too. The first time unixSync() + ** is called the directory file descriptor will be fsync()ed and close()d. + */ + int isOpenDirectory = (isCreate && + (eType==SQLITE_OPEN_MASTER_JOURNAL || eType==SQLITE_OPEN_MAIN_JOURNAL) + ); + + /* If argument zPath is a NULL pointer, this function is required to open + ** a temporary file. Use this buffer to store the file name in. + */ + char zTmpname[MAX_PATHNAME+1]; + const char *zName = zPath; + + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); + + /* The main DB, main journal, and master journal are never automatically + ** deleted + */ + assert( eType!=SQLITE_OPEN_MAIN_DB || !isDelete ); + assert( eType!=SQLITE_OPEN_MAIN_JOURNAL || !isDelete ); + assert( eType!=SQLITE_OPEN_MASTER_JOURNAL || !isDelete ); + + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB + ); + + memset(pFile, 0, sizeof(unixFile)); + + if( !zName ){ + assert(isDelete && !isOpenDirectory); + rc = getTempname(MAX_PATHNAME+1, zTmpname); + if( rc!=SQLITE_OK ){ + return rc; + } + zName = zTmpname; + } + + if( isReadonly ) openFlags |= O_RDONLY; + if( isReadWrite ) openFlags |= O_RDWR; + if( isCreate ) openFlags |= O_CREAT; + if( isExclusive ) openFlags |= (O_EXCL|O_NOFOLLOW); + openFlags |= (O_LARGEFILE|O_BINARY); + + fd = open(zName, openFlags, isDelete?0600:SQLITE_DEFAULT_FILE_PERMISSIONS); + OSTRACE4("OPENX %-3d %s 0%o\n", fd, zName, openFlags); + if( fd<0 && errno!=EISDIR && isReadWrite && !isExclusive ){ + /* Failed to open the file for read/write access. Try read-only. */ + flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); + flags |= SQLITE_OPEN_READONLY; + return unixOpen(pVfs, zPath, pFile, flags, pOutFlags); + } + if( fd<0 ){ + return SQLITE_CANTOPEN; + } + if( isDelete ){ +#if OS_VXWORKS + zPath = zName; +#else + unlink(zName); +#endif + } +#if SQLITE_ENABLE_LOCKING_STYLE + else{ + ((unixFile*)pFile)->openFlags = openFlags; + } +#endif + if( pOutFlags ){ + *pOutFlags = flags; + } + + assert(fd!=0); + if( isOpenDirectory ){ + rc = openDirectory(zPath, &dirfd); + if( rc!=SQLITE_OK ){ + close(fd); /* silently leak if fail, already in error */ + return rc; + } + } #ifdef FD_CLOEXEC - fcntl(h, F_SETFD, fcntl(h, F_GETFD, 0) | FD_CLOEXEC); + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD, 0) | FD_CLOEXEC); #endif - memset(&f, 0, sizeof(f)); - sqlite3OsEnterMutex(); - rc = findLockInfo(h, &f.pLock, &f.pOpen); - sqlite3OsLeaveMutex(); - if( delFlag ){ - unlink(zFilename); - } - if( rc ){ - close(h); - return SQLITE_NOMEM; - } - OSTRACE3("OPEN %-3d %s\n", h, zFilename); - f.dirfd = -1; - f.h = h; - SET_THREADID(&f); - pNew = sqlite3ThreadSafeMalloc( sizeof(unixFile) ); - if( pNew==0 ){ - close(h); - sqlite3OsEnterMutex(); - releaseLockInfo(f.pLock); - releaseOpenCnt(f.pOpen); - sqlite3OsLeaveMutex(); - *pId = 0; - return SQLITE_NOMEM; - }else{ - *pNew = f; - pNew->pMethod = &sqlite3UnixIoMethod; - *pId = (OsFile*)pNew; - OpenCounter(+1); - return SQLITE_OK; - } -} -#endif /* SQLITE_ENABLE_LOCKING_STYLE */ -#endif /* SQLITE_OMIT_DISKIO */ -/*************************************************************************** -** Everything above deals with file I/O. Everything that follows deals -** with other miscellanous aspects of the operating system interface -****************************************************************************/ + noLock = eType!=SQLITE_OPEN_MAIN_DB; + +#if SQLITE_PREFER_PROXY_LOCKING + if( zPath!=NULL && !noLock ){ + char *envforce = getenv("SQLITE_FORCE_PROXY_LOCKING"); + int useProxy = 0; + + /* SQLITE_FORCE_PROXY_LOCKING==1 means force always use proxy, + ** 0 means never use proxy, NULL means use proxy for non-local files only + */ + if( envforce!=NULL ){ + useProxy = atoi(envforce)>0; + }else{ + struct statfs fsInfo; + + if( statfs(zPath, &fsInfo) == -1 ){ + ((unixFile*)pFile)->lastErrno = errno; + if( dirfd>=0 ) close(dirfd); /* silently leak if fail, in error */ + close(fd); /* silently leak if fail, in error */ + return SQLITE_IOERR_ACCESS; + } + useProxy = !(fsInfo.f_flags&MNT_LOCAL); + } + if( useProxy ){ + rc = fillInUnixFile(pVfs, fd, dirfd, pFile, zPath, noLock, isDelete); + if( rc==SQLITE_OK ){ + rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); + } + return rc; + } + } +#endif + + return fillInUnixFile(pVfs, fd, dirfd, pFile, zPath, noLock, isDelete); +} + +/* +** Delete the file at zPath. If the dirSync argument is true, fsync() +** the directory after deleting the file. +*/ +static int unixDelete( + sqlite3_vfs *NotUsed, /* VFS containing this as the xDelete method */ + const char *zPath, /* Name of file to be deleted */ + int dirSync /* If true, fsync() directory after deleting file */ +){ + int rc = SQLITE_OK; + UNUSED_PARAMETER(NotUsed); + SimulateIOError(return SQLITE_IOERR_DELETE); + unlink(zPath); +#ifndef SQLITE_DISABLE_DIRSYNC + if( dirSync ){ + int fd; + rc = openDirectory(zPath, &fd); + if( rc==SQLITE_OK ){ +#if OS_VXWORKS + if( fsync(fd)==-1 ) +#else + if( fsync(fd) ) +#endif + { + rc = SQLITE_IOERR_DIR_FSYNC; + } + if( close(fd)&&!rc ){ + rc = SQLITE_IOERR_DIR_CLOSE; + } + } + } +#endif + return rc; +} + +/* +** Test the existance of or access permissions of file zPath. The +** test performed depends on the value of flags: +** +** SQLITE_ACCESS_EXISTS: Return 1 if the file exists +** SQLITE_ACCESS_READWRITE: Return 1 if the file is read and writable. +** SQLITE_ACCESS_READONLY: Return 1 if the file is readable. +** +** Otherwise return 0. +*/ +static int unixAccess( + sqlite3_vfs *NotUsed, /* The VFS containing this xAccess method */ + const char *zPath, /* Path of the file to examine */ + int flags, /* What do we want to learn about the zPath file? */ + int *pResOut /* Write result boolean here */ +){ + int amode = 0; + UNUSED_PARAMETER(NotUsed); + SimulateIOError( return SQLITE_IOERR_ACCESS; ); + switch( flags ){ + case SQLITE_ACCESS_EXISTS: + amode = F_OK; + break; + case SQLITE_ACCESS_READWRITE: + amode = W_OK|R_OK; + break; + case SQLITE_ACCESS_READ: + amode = R_OK; + break; + + default: + assert(!"Invalid flags argument"); + } + *pResOut = (access(zPath, amode)==0); + return SQLITE_OK; +} + + +/* +** Turn a relative pathname into a full pathname. The relative path +** is stored as a nul-terminated string in the buffer pointed to by +** zPath. +** +** zOut points to a buffer of at least sqlite3_vfs.mxPathname bytes +** (in this case, MAX_PATHNAME bytes). The full-path is written to +** this buffer before returning. +*/ +static int unixFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zPath, /* Possibly relative input path */ + int nOut, /* Size of output buffer in bytes */ + char *zOut /* Output buffer */ +){ + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. This function could fail if, for example, the + ** current working directory has been unlinked. + */ + SimulateIOError( return SQLITE_ERROR ); + + assert( pVfs->mxPathname==MAX_PATHNAME ); + UNUSED_PARAMETER(pVfs); + + zOut[nOut-1] = '\0'; + if( zPath[0]=='/' ){ + sqlite3_snprintf(nOut, zOut, "%s", zPath); + }else{ + int nCwd; + if( getcwd(zOut, nOut-1)==0 ){ + return SQLITE_CANTOPEN; + } + nCwd = (int)strlen(zOut); + sqlite3_snprintf(nOut-nCwd, &zOut[nCwd], "/%s", zPath); + } + return SQLITE_OK; +} #ifndef SQLITE_OMIT_LOAD_EXTENSION @@ -2631,23 +3839,69 @@ static int allocateUnixFile( ** within the shared library, and closing the shared library. */ #include -void *sqlite3UnixDlopen(const char *zFilename){ +static void *unixDlOpen(sqlite3_vfs *NotUsed, const char *zFilename){ + UNUSED_PARAMETER(NotUsed); return dlopen(zFilename, RTLD_NOW | RTLD_GLOBAL); } -void *sqlite3UnixDlsym(void *pHandle, const char *zSymbol){ - return dlsym(pHandle, zSymbol); -} -int sqlite3UnixDlclose(void *pHandle){ - return dlclose(pHandle); -} -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ /* -** Get information to seed the random number generator. The seed -** is written into the buffer zBuf[256]. The calling function must -** supply a sufficiently large buffer. +** SQLite calls this function immediately after a call to unixDlSym() or +** unixDlOpen() fails (returns a null pointer). If a more detailed error +** message is available, it is written to zBufOut. If no error message +** is available, zBufOut is left unmodified and SQLite uses a default +** error message. */ -int sqlite3UnixRandomSeed(char *zBuf){ +static void unixDlError(sqlite3_vfs *NotUsed, int nBuf, char *zBufOut){ + char *zErr; + UNUSED_PARAMETER(NotUsed); + unixEnterMutex(); + zErr = dlerror(); + if( zErr ){ + sqlite3_snprintf(nBuf, zBufOut, "%s", zErr); + } + unixLeaveMutex(); +} +static void (*unixDlSym(sqlite3_vfs *NotUsed, void *p, const char*zSym))(void){ + /* + ** GCC with -pedantic-errors says that C90 does not allow a void* to be + ** cast into a pointer to a function. And yet the library dlsym() routine + ** returns a void* which is really a pointer to a function. So how do we + ** use dlsym() with -pedantic-errors? + ** + ** Variable x below is defined to be a pointer to a function taking + ** parameters void* and const char* and returning a pointer to a function. + ** We initialize x by assigning it a pointer to the dlsym() function. + ** (That assignment requires a cast.) Then we call the function that + ** x points to. + ** + ** This work-around is unlikely to work correctly on any system where + ** you really cannot cast a function pointer into void*. But then, on the + ** other hand, dlsym() will not work on such a system either, so we have + ** not really lost anything. + */ + void (*(*x)(void*,const char*))(void); + UNUSED_PARAMETER(NotUsed); + x = (void(*(*)(void*,const char*))(void))dlsym; + return (*x)(p, zSym); +} +static void unixDlClose(sqlite3_vfs *NotUsed, void *pHandle){ + UNUSED_PARAMETER(NotUsed); + dlclose(pHandle); +} +#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ + #define unixDlOpen 0 + #define unixDlError 0 + #define unixDlSym 0 + #define unixDlClose 0 +#endif + +/* +** Write nBuf bytes of random data to the supplied buffer zBuf. +*/ +static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ + UNUSED_PARAMETER(NotUsed); + assert((size_t)nBuf>=(sizeof(time_t)+sizeof(int))); + /* We have to initialize zBuf to prevent valgrind from reporting ** errors. The reports issued by valgrind are incorrect - we would ** prefer that the randomness be increased by making use of the @@ -2660,7 +3914,7 @@ int sqlite3UnixRandomSeed(char *zBuf){ ** that we always use the same random number sequence. This makes the ** tests repeatable. */ - memset(zBuf, 0, 256); + memset(zBuf, 0, nBuf); #if !defined(SQLITE_TEST) { int pid, fd; @@ -2670,243 +3924,53 @@ int sqlite3UnixRandomSeed(char *zBuf){ time(&t); memcpy(zBuf, &t, sizeof(t)); pid = getpid(); - memcpy(&zBuf[sizeof(time_t)], &pid, sizeof(pid)); + memcpy(&zBuf[sizeof(t)], &pid, sizeof(pid)); + assert( sizeof(t)+sizeof(pid)<=(size_t)nBuf ); + nBuf = sizeof(t) + sizeof(pid); }else{ - read(fd, zBuf, 256); + nBuf = read(fd, zBuf, nBuf); close(fd); } } #endif - return SQLITE_OK; + return nBuf; } + /* ** Sleep for a little while. Return the amount of time slept. -** The argument is the number of milliseconds we want to sleep. +** The argument is the number of microseconds we want to sleep. +** The return value is the number of microseconds of sleep actually +** requested from the underlying operating system, a number which +** might be greater than or equal to the argument, but not less +** than the argument. */ -int sqlite3UnixSleep(int ms){ -#if defined(HAVE_USLEEP) && HAVE_USLEEP - usleep(ms*1000); - return ms; +static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ +#if OS_VXWORKS + struct timespec sp; + + sp.tv_sec = microseconds / 1000000; + sp.tv_nsec = (microseconds % 1000000) * 1000; + nanosleep(&sp, NULL); + return microseconds; +#elif defined(HAVE_USLEEP) && HAVE_USLEEP + usleep(microseconds); + return microseconds; #else - sleep((ms+999)/1000); - return 1000*((ms+999)/1000); + int seconds = (microseconds+999999)/1000000; + sleep(seconds); + return seconds*1000000; #endif + UNUSED_PARAMETER(NotUsed); } /* -** Static variables used for thread synchronization. -** -** inMutex the nesting depth of the recursive mutex. The thread -** holding mutexMain can read this variable at any time. -** But is must hold mutexAux to change this variable. Other -** threads must hold mutexAux to read the variable and can -** never write. -** -** mutexOwner The thread id of the thread holding mutexMain. Same -** access rules as for inMutex. -** -** mutexOwnerValid True if the value in mutexOwner is valid. The same -** access rules apply as for inMutex. -** -** mutexMain The main mutex. Hold this mutex in order to get exclusive -** access to SQLite data structures. -** -** mutexAux An auxiliary mutex needed to access variables defined above. -** -** Mutexes are always acquired in this order: mutexMain mutexAux. It -** is not necessary to acquire mutexMain in order to get mutexAux - just -** do not attempt to acquire them in the reverse order: mutexAux mutexMain. -** Either get the mutexes with mutexMain first or get mutexAux only. -** -** When running on a platform where the three variables inMutex, mutexOwner, -** and mutexOwnerValid can be set atomically, the mutexAux is not required. -** On many systems, all three are 32-bit integers and writing to a 32-bit -** integer is atomic. I think. But there are no guarantees. So it seems -** safer to protect them using mutexAux. -*/ -static int inMutex = 0; -#ifdef SQLITE_UNIX_THREADS -static pthread_t mutexOwner; /* Thread holding mutexMain */ -static int mutexOwnerValid = 0; /* True if mutexOwner is valid */ -static pthread_mutex_t mutexMain = PTHREAD_MUTEX_INITIALIZER; /* The mutex */ -static pthread_mutex_t mutexAux = PTHREAD_MUTEX_INITIALIZER; /* Aux mutex */ -#endif - -/* -** The following pair of routine implement mutual exclusion for -** multi-threaded processes. Only a single thread is allowed to -** executed code that is surrounded by EnterMutex() and LeaveMutex(). -** -** SQLite uses only a single Mutex. There is not much critical -** code and what little there is executes quickly and without blocking. -** -** As of version 3.3.2, this mutex must be recursive. -*/ -void sqlite3UnixEnterMutex(){ -#ifdef SQLITE_UNIX_THREADS - pthread_mutex_lock(&mutexAux); - if( !mutexOwnerValid || !pthread_equal(mutexOwner, pthread_self()) ){ - pthread_mutex_unlock(&mutexAux); - pthread_mutex_lock(&mutexMain); - assert( inMutex==0 ); - assert( !mutexOwnerValid ); - pthread_mutex_lock(&mutexAux); - mutexOwner = pthread_self(); - mutexOwnerValid = 1; - } - inMutex++; - pthread_mutex_unlock(&mutexAux); -#else - inMutex++; -#endif -} -void sqlite3UnixLeaveMutex(){ - assert( inMutex>0 ); -#ifdef SQLITE_UNIX_THREADS - pthread_mutex_lock(&mutexAux); - inMutex--; - assert( pthread_equal(mutexOwner, pthread_self()) ); - if( inMutex==0 ){ - assert( mutexOwnerValid ); - mutexOwnerValid = 0; - pthread_mutex_unlock(&mutexMain); - } - pthread_mutex_unlock(&mutexAux); -#else - inMutex--; -#endif -} - -/* -** Return TRUE if the mutex is currently held. -** -** If the thisThrd parameter is true, return true only if the -** calling thread holds the mutex. If the parameter is false, return -** true if any thread holds the mutex. -*/ -int sqlite3UnixInMutex(int thisThrd){ -#ifdef SQLITE_UNIX_THREADS - int rc; - pthread_mutex_lock(&mutexAux); - rc = inMutex>0 && (thisThrd==0 || pthread_equal(mutexOwner,pthread_self())); - pthread_mutex_unlock(&mutexAux); - return rc; -#else - return inMutex>0; -#endif -} - -/* -** Remember the number of thread-specific-data blocks allocated. -** Use this to verify that we are not leaking thread-specific-data. -** Ticket #1601 +** The following variable, if set to a non-zero value, is interpreted as +** the number of seconds since 1970 and is used to set the result of +** sqlite3OsCurrentTime() during testing. */ #ifdef SQLITE_TEST -int sqlite3_tsd_count = 0; -# ifdef SQLITE_UNIX_THREADS - static pthread_mutex_t tsd_counter_mutex = PTHREAD_MUTEX_INITIALIZER; -# define TSD_COUNTER(N) \ - pthread_mutex_lock(&tsd_counter_mutex); \ - sqlite3_tsd_count += N; \ - pthread_mutex_unlock(&tsd_counter_mutex); -# else -# define TSD_COUNTER(N) sqlite3_tsd_count += N -# endif -#else -# define TSD_COUNTER(N) /* no-op */ -#endif - -/* -** If called with allocateFlag>0, then return a pointer to thread -** specific data for the current thread. Allocate and zero the -** thread-specific data if it does not already exist. -** -** If called with allocateFlag==0, then check the current thread -** specific data. Return it if it exists. If it does not exist, -** then return NULL. -** -** If called with allocateFlag<0, check to see if the thread specific -** data is allocated and is all zero. If it is then deallocate it. -** Return a pointer to the thread specific data or NULL if it is -** unallocated or gets deallocated. -*/ -ThreadData *sqlite3UnixThreadSpecificData(int allocateFlag){ - static const ThreadData zeroData = {0}; /* Initializer to silence warnings - ** from broken compilers */ -#ifdef SQLITE_UNIX_THREADS - static pthread_key_t key; - static int keyInit = 0; - ThreadData *pTsd; - - if( !keyInit ){ - sqlite3OsEnterMutex(); - if( !keyInit ){ - int rc; - rc = pthread_key_create(&key, 0); - if( rc ){ - sqlite3OsLeaveMutex(); - return 0; - } - keyInit = 1; - } - sqlite3OsLeaveMutex(); - } - - pTsd = pthread_getspecific(key); - if( allocateFlag>0 ){ - if( pTsd==0 ){ - if( !sqlite3TestMallocFail() ){ - pTsd = sqlite3OsMalloc(sizeof(zeroData)); - } -#ifdef SQLITE_MEMDEBUG - sqlite3_isFail = 0; -#endif - if( pTsd ){ - *pTsd = zeroData; - pthread_setspecific(key, pTsd); - TSD_COUNTER(+1); - } - } - }else if( pTsd!=0 && allocateFlag<0 - && memcmp(pTsd, &zeroData, sizeof(ThreadData))==0 ){ - sqlite3OsFree(pTsd); - pthread_setspecific(key, 0); - TSD_COUNTER(-1); - pTsd = 0; - } - return pTsd; -#else - static ThreadData *pTsd = 0; - if( allocateFlag>0 ){ - if( pTsd==0 ){ - if( !sqlite3TestMallocFail() ){ - pTsd = sqlite3OsMalloc( sizeof(zeroData) ); - } -#ifdef SQLITE_MEMDEBUG - sqlite3_isFail = 0; -#endif - if( pTsd ){ - *pTsd = zeroData; - TSD_COUNTER(+1); - } - } - }else if( pTsd!=0 && allocateFlag<0 - && memcmp(pTsd, &zeroData, sizeof(ThreadData))==0 ){ - sqlite3OsFree(pTsd); - TSD_COUNTER(-1); - pTsd = 0; - } - return pTsd; -#endif -} - -/* -** The following variable, if set to a non-zero value, becomes the result -** returned from sqlite3OsCurrentTime(). This is used for testing. -*/ -#ifdef SQLITE_TEST -int sqlite3_current_time = 0; +int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ #endif /* @@ -2914,22 +3978,1078 @@ int sqlite3_current_time = 0; ** current time and date as a Julian Day number into *prNow and ** return 0. Return 1 if the time and date cannot be found. */ -int sqlite3UnixCurrentTime(double *prNow){ -#ifdef NO_GETTOD +static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ +#if defined(NO_GETTOD) time_t t; time(&t); *prNow = t/86400.0 + 2440587.5; +#elif OS_VXWORKS + struct timespec sNow; + clock_gettime(CLOCK_REALTIME, &sNow); + *prNow = 2440587.5 + sNow.tv_sec/86400.0 + sNow.tv_nsec/86400000000000.0; #else struct timeval sNow; gettimeofday(&sNow, 0); *prNow = 2440587.5 + sNow.tv_sec/86400.0 + sNow.tv_usec/86400000000.0; #endif + #ifdef SQLITE_TEST if( sqlite3_current_time ){ *prNow = sqlite3_current_time/86400.0 + 2440587.5; } #endif + UNUSED_PARAMETER(NotUsed); return 0; } -#endif /* OS_UNIX */ +/* +** We added the xGetLastError() method with the intention of providing +** better low-level error messages when operating-system problems come up +** during SQLite operation. But so far, none of that has been implemented +** in the core. So this routine is never called. For now, it is merely +** a place-holder. +*/ +static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ + UNUSED_PARAMETER(NotUsed); + UNUSED_PARAMETER(NotUsed2); + UNUSED_PARAMETER(NotUsed3); + return 0; +} + +/* +************************ End of sqlite3_vfs methods *************************** +******************************************************************************/ + +/****************************************************************************** +************************** Begin Proxy Locking ******************************** +** +** Proxy locking is a "uber-locking-method" in this sense: It uses the +** other locking methods on secondary lock files. Proxy locking is a +** meta-layer over top of the primitive locking implemented above. For +** this reason, the division that implements of proxy locking is deferred +** until late in the file (here) after all of the other I/O methods have +** been defined - so that the primitive locking methods are available +** as services to help with the implementation of proxy locking. +** +**** +** +** The default locking schemes in SQLite use byte-range locks on the +** database file to coordinate safe, concurrent access by multiple readers +** and writers [http://sqlite.org/lockingv3.html]. The five file locking +** states (UNLOCKED, PENDING, SHARED, RESERVED, EXCLUSIVE) are implemented +** as POSIX read & write locks over fixed set of locations (via fsctl), +** on AFP and SMB only exclusive byte-range locks are available via fsctl +** with _IOWR('z', 23, struct ByteRangeLockPB2) to track the same 5 states. +** To simulate a F_RDLCK on the shared range, on AFP a randomly selected +** address in the shared range is taken for a SHARED lock, the entire +** shared range is taken for an EXCLUSIVE lock): +** +** PENDING_BYTE 0x40000000 +** RESERVED_BYTE 0x40000001 +** SHARED_RANGE 0x40000002 -> 0x40000200 +** +** This works well on the local file system, but shows a nearly 100x +** slowdown in read performance on AFP because the AFP client disables +** the read cache when byte-range locks are present. Enabling the read +** cache exposes a cache coherency problem that is present on all OS X +** supported network file systems. NFS and AFP both observe the +** close-to-open semantics for ensuring cache coherency +** [http://nfs.sourceforge.net/#faq_a8], which does not effectively +** address the requirements for concurrent database access by multiple +** readers and writers +** [http://www.nabble.com/SQLite-on-NFS-cache-coherency-td15655701.html]. +** +** To address the performance and cache coherency issues, proxy file locking +** changes the way database access is controlled by limiting access to a +** single host at a time and moving file locks off of the database file +** and onto a proxy file on the local file system. +** +** +** Using proxy locks +** ----------------- +** +** C APIs +** +** sqlite3_file_control(db, dbname, SQLITE_SET_LOCKPROXYFILE, +** | ":auto:"); +** sqlite3_file_control(db, dbname, SQLITE_GET_LOCKPROXYFILE, &); +** +** +** SQL pragmas +** +** PRAGMA [database.]lock_proxy_file= | :auto: +** PRAGMA [database.]lock_proxy_file +** +** Specifying ":auto:" means that if there is a conch file with a matching +** host ID in it, the proxy path in the conch file will be used, otherwise +** a proxy path based on the user's temp dir +** (via confstr(_CS_DARWIN_USER_TEMP_DIR,...)) will be used and the +** actual proxy file name is generated from the name and path of the +** database file. For example: +** +** For database path "/Users/me/foo.db" +** The lock path will be "/sqliteplocks/_Users_me_foo.db:auto:") +** +** Once a lock proxy is configured for a database connection, it can not +** be removed, however it may be switched to a different proxy path via +** the above APIs (assuming the conch file is not being held by another +** connection or process). +** +** +** How proxy locking works +** ----------------------- +** +** Proxy file locking relies primarily on two new supporting files: +** +** * conch file to limit access to the database file to a single host +** at a time +** +** * proxy file to act as a proxy for the advisory locks normally +** taken on the database +** +** The conch file - to use a proxy file, sqlite must first "hold the conch" +** by taking an sqlite-style shared lock on the conch file, reading the +** contents and comparing the host's unique host ID (see below) and lock +** proxy path against the values stored in the conch. The conch file is +** stored in the same directory as the database file and the file name +** is patterned after the database file name as ".-conch". +** If the conch file does not exist, or it's contents do not match the +** host ID and/or proxy path, then the lock is escalated to an exclusive +** lock and the conch file contents is updated with the host ID and proxy +** path and the lock is downgraded to a shared lock again. If the conch +** is held by another process (with a shared lock), the exclusive lock +** will fail and SQLITE_BUSY is returned. +** +** The proxy file - a single-byte file used for all advisory file locks +** normally taken on the database file. This allows for safe sharing +** of the database file for multiple readers and writers on the same +** host (the conch ensures that they all use the same local lock file). +** +** There is a third file - the host ID file - used as a persistent record +** of a unique identifier for the host, a 128-byte unique host id file +** in the path defined by the HOSTIDPATH macro (default value is +** /Library/Caches/.com.apple.sqliteConchHostId). +** +** Requesting the lock proxy does not immediately take the conch, it is +** only taken when the first request to lock database file is made. +** This matches the semantics of the traditional locking behavior, where +** opening a connection to a database file does not take a lock on it. +** The shared lock and an open file descriptor are maintained until +** the connection to the database is closed. +** +** The proxy file and the lock file are never deleted so they only need +** to be created the first time they are used. +** +** Configuration options +** --------------------- +** +** SQLITE_PREFER_PROXY_LOCKING +** +** Database files accessed on non-local file systems are +** automatically configured for proxy locking, lock files are +** named automatically using the same logic as +** PRAGMA lock_proxy_file=":auto:" +** +** SQLITE_PROXY_DEBUG +** +** Enables the logging of error messages during host id file +** retrieval and creation +** +** HOSTIDPATH +** +** Overrides the default host ID file path location +** +** LOCKPROXYDIR +** +** Overrides the default directory used for lock proxy files that +** are named automatically via the ":auto:" setting +** +** SQLITE_DEFAULT_PROXYDIR_PERMISSIONS +** +** Permissions to use when creating a directory for storing the +** lock proxy files, only used when LOCKPROXYDIR is not set. +** +** +** As mentioned above, when compiled with SQLITE_PREFER_PROXY_LOCKING, +** setting the environment variable SQLITE_FORCE_PROXY_LOCKING to 1 will +** force proxy locking to be used for every database file opened, and 0 +** will force automatic proxy locking to be disabled for all database +** files (explicity calling the SQLITE_SET_LOCKPROXYFILE pragma or +** sqlite_file_control API is not affected by SQLITE_FORCE_PROXY_LOCKING). +*/ + +/* +** Proxy locking is only available on MacOSX +*/ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + +#ifdef SQLITE_TEST +/* simulate multiple hosts by creating unique hostid file paths */ +int sqlite3_hostid_num = 0; +#endif + +/* +** The proxyLockingContext has the path and file structures for the remote +** and local proxy files in it +*/ +typedef struct proxyLockingContext proxyLockingContext; +struct proxyLockingContext { + unixFile *conchFile; /* Open conch file */ + char *conchFilePath; /* Name of the conch file */ + unixFile *lockProxy; /* Open proxy lock file */ + char *lockProxyPath; /* Name of the proxy lock file */ + char *dbPath; /* Name of the open file */ + int conchHeld; /* True if the conch is currently held */ + void *oldLockingContext; /* Original lockingcontext to restore on close */ + sqlite3_io_methods const *pOldMethod; /* Original I/O methods for close */ +}; + +/* HOSTIDLEN and CONCHLEN both include space for the string +** terminating nul +*/ +#define HOSTIDLEN 128 +#define CONCHLEN (MAXPATHLEN+HOSTIDLEN+1) +#ifndef HOSTIDPATH +# define HOSTIDPATH "/Library/Caches/.com.apple.sqliteConchHostId" +#endif + +/* basically a copy of unixRandomness with different +** test behavior built in */ +static int proxyGenerateHostID(char *pHostID){ + int pid, fd, len; + unsigned char *key = (unsigned char *)pHostID; + + memset(key, 0, HOSTIDLEN); + len = 0; + fd = open("/dev/urandom", O_RDONLY); + if( fd>=0 ){ + len = read(fd, key, HOSTIDLEN); + close(fd); /* silently leak the fd if it fails */ + } + if( len < HOSTIDLEN ){ + time_t t; + time(&t); + memcpy(key, &t, sizeof(t)); + pid = getpid(); + memcpy(&key[sizeof(t)], &pid, sizeof(pid)); + } + +#ifdef MAKE_PRETTY_HOSTID + { + int i; + /* filter the bytes into printable ascii characters and NUL terminate */ + key[(HOSTIDLEN-1)] = 0x00; + for( i=0; i<(HOSTIDLEN-1); i++ ){ + unsigned char pa = key[i]&0x7F; + if( pa<0x20 ){ + key[i] = (key[i]&0x80 == 0x80) ? pa+0x40 : pa+0x20; + }else if( pa==0x7F ){ + key[i] = (key[i]&0x80 == 0x80) ? pa=0x20 : pa+0x7E; + } + } + } +#endif + return SQLITE_OK; +} + +/* writes the host id path to path, path should be an pre-allocated buffer +** with enough space for a path +*/ +static void proxyGetHostIDPath(char *path, size_t len){ + strlcpy(path, HOSTIDPATH, len); +#ifdef SQLITE_TEST + if( sqlite3_hostid_num>0 ){ + char suffix[2] = "1"; + suffix[0] = suffix[0] + sqlite3_hostid_num; + strlcat(path, suffix, len); + } +#endif + OSTRACE3("GETHOSTIDPATH %s pid=%d\n", path, getpid()); +} + +/* get the host ID from a sqlite hostid file stored in the +** user-specific tmp directory, create the ID if it's not there already +*/ +static int proxyGetHostID(char *pHostID, int *pError){ + int fd; + char path[MAXPATHLEN]; + size_t len; + int rc=SQLITE_OK; + + proxyGetHostIDPath(path, MAXPATHLEN); + /* try to create the host ID file, if it already exists read the contents */ + fd = open(path, O_CREAT|O_WRONLY|O_EXCL, 0644); + if( fd<0 ){ + int err=errno; + + if( err!=EEXIST ){ +#ifdef SQLITE_PROXY_DEBUG /* set the sqlite error message instead */ + fprintf(stderr, "sqlite error creating host ID file %s: %s\n", + path, strerror(err)); +#endif + return SQLITE_PERM; + } + /* couldn't create the file, read it instead */ + fd = open(path, O_RDONLY|O_EXCL); + if( fd<0 ){ +#ifdef SQLITE_PROXY_DEBUG /* set the sqlite error message instead */ + int err = errno; + fprintf(stderr, "sqlite error opening host ID file %s: %s\n", + path, strerror(err)); +#endif + return SQLITE_PERM; + } + len = pread(fd, pHostID, HOSTIDLEN, 0); + if( len<0 ){ + *pError = errno; + rc = SQLITE_IOERR_READ; + }else if( lenlockingContext; + + if( pCtx->conchHeld>0 ){ + return SQLITE_OK; + }else{ + unixFile *conchFile = pCtx->conchFile; + char testValue[CONCHLEN]; + char conchValue[CONCHLEN]; + char lockPath[MAXPATHLEN]; + char *tLockPath = NULL; + int rc = SQLITE_OK; + int readRc = SQLITE_OK; + int syncPerms = 0; + + OSTRACE4("TAKECONCH %d for %s pid=%d\n", conchFile->h, + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), getpid()); + + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); + if( rc==SQLITE_OK ){ + int pError = 0; + memset(testValue, 0, CONCHLEN); /* conch is fixed size */ + rc = proxyGetHostID(testValue, &pError); + if( (rc&0xff)==SQLITE_IOERR ){ + pFile->lastErrno = pError; + } + if( pCtx->lockProxyPath ){ + strlcpy(&testValue[HOSTIDLEN], pCtx->lockProxyPath, MAXPATHLEN); + } + } + if( rc!=SQLITE_OK ){ + goto end_takeconch; + } + + readRc = unixRead((sqlite3_file *)conchFile, conchValue, CONCHLEN, 0); + if( readRc!=SQLITE_IOERR_SHORT_READ ){ + if( readRc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_IOERR ){ + pFile->lastErrno = conchFile->lastErrno; + } + rc = readRc; + goto end_takeconch; + } + /* if the conch has data compare the contents */ + if( !pCtx->lockProxyPath ){ + /* for auto-named local lock file, just check the host ID and we'll + ** use the local lock file path that's already in there */ + if( !memcmp(testValue, conchValue, HOSTIDLEN) ){ + tLockPath = (char *)&conchValue[HOSTIDLEN]; + goto end_takeconch; + } + }else{ + /* we've got the conch if conchValue matches our path and host ID */ + if( !memcmp(testValue, conchValue, CONCHLEN) ){ + goto end_takeconch; + } + } + }else{ + /* a short read means we're "creating" the conch (even though it could + ** have been user-intervention), if we acquire the exclusive lock, + ** we'll try to match the current on-disk permissions of the database + */ + syncPerms = 1; + } + + /* either conch was emtpy or didn't match */ + if( !pCtx->lockProxyPath ){ + proxyGetLockPath(pCtx->dbPath, lockPath, MAXPATHLEN); + tLockPath = lockPath; + strlcpy(&testValue[HOSTIDLEN], lockPath, MAXPATHLEN); + } + + /* update conch with host and path (this will fail if other process + ** has a shared lock already) */ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, EXCLUSIVE_LOCK); + if( rc==SQLITE_OK ){ + rc = unixWrite((sqlite3_file *)conchFile, testValue, CONCHLEN, 0); + if( rc==SQLITE_OK && syncPerms ){ + struct stat buf; + int err = fstat(pFile->h, &buf); + if( err==0 ){ + /* try to match the database file permissions, ignore failure */ +#ifndef SQLITE_PROXY_DEBUG + fchmod(conchFile->h, buf.st_mode); +#else + if( fchmod(conchFile->h, buf.st_mode)!=0 ){ + int code = errno; + fprintf(stderr, "fchmod %o FAILED with %d %s\n", + buf.st_mode, code, strerror(code)); + } else { + fprintf(stderr, "fchmod %o SUCCEDED\n",buf.st_mode); + } + }else{ + int code = errno; + fprintf(stderr, "STAT FAILED[%d] with %d %s\n", + err, code, strerror(code)); +#endif + } + } + } + conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, SHARED_LOCK); + +end_takeconch: + OSTRACE2("TRANSPROXY: CLOSE %d\n", pFile->h); + if( rc==SQLITE_OK && pFile->openFlags ){ + if( pFile->h>=0 ){ +#ifdef STRICT_CLOSE_ERROR + if( close(pFile->h) ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_CLOSE; + } +#else + close(pFile->h); /* silently leak fd if fail */ +#endif + } + pFile->h = -1; + int fd = open(pCtx->dbPath, pFile->openFlags, + SQLITE_DEFAULT_FILE_PERMISSIONS); + OSTRACE2("TRANSPROXY: OPEN %d\n", fd); + if( fd>=0 ){ + pFile->h = fd; + }else{ + rc=SQLITE_CANTOPEN; /* SQLITE_BUSY? proxyTakeConch called + during locking */ + } + } + if( rc==SQLITE_OK && !pCtx->lockProxy ){ + char *path = tLockPath ? tLockPath : pCtx->lockProxyPath; + /* ACS: Need to make a copy of path sometimes */ + rc = proxyCreateUnixFile(path, &pCtx->lockProxy); + } + if( rc==SQLITE_OK ){ + pCtx->conchHeld = 1; + + if( tLockPath ){ + pCtx->lockProxyPath = sqlite3DbStrDup(0, tLockPath); + if( pCtx->lockProxy->pMethod == &afpIoMethods ){ + ((afpLockingContext *)pCtx->lockProxy->lockingContext)->dbPath = + pCtx->lockProxyPath; + } + } + } else { + conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + } + OSTRACE3("TAKECONCH %d %s\n", conchFile->h, rc==SQLITE_OK?"ok":"failed"); + return rc; + } +} + +/* +** If pFile holds a lock on a conch file, then release that lock. +*/ +static int proxyReleaseConch(unixFile *pFile){ + int rc; /* Subroutine return code */ + proxyLockingContext *pCtx; /* The locking context for the proxy lock */ + unixFile *conchFile; /* Name of the conch file */ + + pCtx = (proxyLockingContext *)pFile->lockingContext; + conchFile = pCtx->conchFile; + OSTRACE4("RELEASECONCH %d for %s pid=%d\n", conchFile->h, + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), + getpid()); + pCtx->conchHeld = 0; + rc = conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + OSTRACE3("RELEASECONCH %d %s\n", conchFile->h, + (rc==SQLITE_OK ? "ok" : "failed")); + return rc; +} + +/* +** Given the name of a database file, compute the name of its conch file. +** Store the conch filename in memory obtained from sqlite3_malloc(). +** Make *pConchPath point to the new name. Return SQLITE_OK on success +** or SQLITE_NOMEM if unable to obtain memory. +** +** The caller is responsible for ensuring that the allocated memory +** space is eventually freed. +** +** *pConchPath is set to NULL if a memory allocation error occurs. +*/ +static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ + int i; /* Loop counter */ + int len = (int)strlen(dbPath); /* Length of database filename - dbPath */ + char *conchPath; /* buffer in which to construct conch name */ + + /* Allocate space for the conch filename and initialize the name to + ** the name of the original database file. */ + *pConchPath = conchPath = (char *)sqlite3_malloc(len + 8); + if( conchPath==0 ){ + return SQLITE_NOMEM; + } + memcpy(conchPath, dbPath, len+1); + + /* now insert a "." before the last / character */ + for( i=(len-1); i>=0; i-- ){ + if( conchPath[i]=='/' ){ + i++; + break; + } + } + conchPath[i]='.'; + while ( ilockingContext; + char *oldPath = pCtx->lockProxyPath; + int rc = SQLITE_OK; + + if( pFile->locktype!=NO_LOCK ){ + return SQLITE_BUSY; + } + + /* nothing to do if the path is NULL, :auto: or matches the existing path */ + if( !path || path[0]=='\0' || !strcmp(path, ":auto:") || + (oldPath && !strncmp(oldPath, path, MAXPATHLEN)) ){ + return SQLITE_OK; + }else{ + unixFile *lockProxy = pCtx->lockProxy; + pCtx->lockProxy=NULL; + pCtx->conchHeld = 0; + if( lockProxy!=NULL ){ + rc=lockProxy->pMethod->xClose((sqlite3_file *)lockProxy); + if( rc ) return rc; + sqlite3_free(lockProxy); + } + sqlite3_free(oldPath); + pCtx->lockProxyPath = sqlite3DbStrDup(0, path); + } + + return rc; +} + +/* +** pFile is a file that has been opened by a prior xOpen call. dbPath +** is a string buffer at least MAXPATHLEN+1 characters in size. +** +** This routine find the filename associated with pFile and writes it +** int dbPath. +*/ +static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ +#if defined(__APPLE__) + if( pFile->pMethod == &afpIoMethods ){ + /* afp style keeps a reference to the db path in the filePath field + ** of the struct */ + assert( (int)strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); + strcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath); + }else +#endif + if( pFile->pMethod == &dotlockIoMethods ){ + /* dot lock style uses the locking context to store the dot lock + ** file path */ + int len = strlen((char *)pFile->lockingContext) - strlen(DOTLOCK_SUFFIX); + memcpy(dbPath, (char *)pFile->lockingContext, len + 1); + }else{ + /* all other styles use the locking context to store the db file path */ + assert( strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); + strcpy(dbPath, (char *)pFile->lockingContext); + } + return SQLITE_OK; +} + +/* +** Takes an already filled in unix file and alters it so all file locking +** will be performed on the local proxy lock file. The following fields +** are preserved in the locking context so that they can be restored and +** the unix structure properly cleaned up at close time: +** ->lockingContext +** ->pMethod +*/ +static int proxyTransformUnixFile(unixFile *pFile, const char *path) { + proxyLockingContext *pCtx; + char dbPath[MAXPATHLEN+1]; /* Name of the database file */ + char *lockPath=NULL; + int rc = SQLITE_OK; + + if( pFile->locktype!=NO_LOCK ){ + return SQLITE_BUSY; + } + proxyGetDbPathForUnixFile(pFile, dbPath); + if( !path || path[0]=='\0' || !strcmp(path, ":auto:") ){ + lockPath=NULL; + }else{ + lockPath=(char *)path; + } + + OSTRACE4("TRANSPROXY %d for %s pid=%d\n", pFile->h, + (lockPath ? lockPath : ":auto:"), getpid()); + + pCtx = sqlite3_malloc( sizeof(*pCtx) ); + if( pCtx==0 ){ + return SQLITE_NOMEM; + } + memset(pCtx, 0, sizeof(*pCtx)); + + rc = proxyCreateConchPathname(dbPath, &pCtx->conchFilePath); + if( rc==SQLITE_OK ){ + rc = proxyCreateUnixFile(pCtx->conchFilePath, &pCtx->conchFile); + } + if( rc==SQLITE_OK && lockPath ){ + pCtx->lockProxyPath = sqlite3DbStrDup(0, lockPath); + } + + if( rc==SQLITE_OK ){ + /* all memory is allocated, proxys are created and assigned, + ** switch the locking context and pMethod then return. + */ + pCtx->dbPath = sqlite3DbStrDup(0, dbPath); + pCtx->oldLockingContext = pFile->lockingContext; + pFile->lockingContext = pCtx; + pCtx->pOldMethod = pFile->pMethod; + pFile->pMethod = &proxyIoMethods; + }else{ + if( pCtx->conchFile ){ + rc = pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); + if( rc ) return rc; + sqlite3_free(pCtx->conchFile); + } + sqlite3_free(pCtx->conchFilePath); + sqlite3_free(pCtx); + } + OSTRACE3("TRANSPROXY %d %s\n", pFile->h, + (rc==SQLITE_OK ? "ok" : "failed")); + return rc; +} + + +/* +** This routine handles sqlite3_file_control() calls that are specific +** to proxy locking. +*/ +static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ + switch( op ){ + case SQLITE_GET_LOCKPROXYFILE: { + unixFile *pFile = (unixFile*)id; + if( pFile->pMethod == &proxyIoMethods ){ + proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; + proxyTakeConch(pFile); + if( pCtx->lockProxyPath ){ + *(const char **)pArg = pCtx->lockProxyPath; + }else{ + *(const char **)pArg = ":auto: (not held)"; + } + } else { + *(const char **)pArg = NULL; + } + return SQLITE_OK; + } + case SQLITE_SET_LOCKPROXYFILE: { + unixFile *pFile = (unixFile*)id; + int rc = SQLITE_OK; + int isProxyStyle = (pFile->pMethod == &proxyIoMethods); + if( pArg==NULL || (const char *)pArg==0 ){ + if( isProxyStyle ){ + /* turn off proxy locking - not supported */ + rc = SQLITE_ERROR /*SQLITE_PROTOCOL? SQLITE_MISUSE?*/; + }else{ + /* turn off proxy locking - already off - NOOP */ + rc = SQLITE_OK; + } + }else{ + const char *proxyPath = (const char *)pArg; + if( isProxyStyle ){ + proxyLockingContext *pCtx = + (proxyLockingContext*)pFile->lockingContext; + if( !strcmp(pArg, ":auto:") + || (pCtx->lockProxyPath && + !strncmp(pCtx->lockProxyPath, proxyPath, MAXPATHLEN)) + ){ + rc = SQLITE_OK; + }else{ + rc = switchLockProxyPath(pFile, proxyPath); + } + }else{ + /* turn on proxy file locking */ + rc = proxyTransformUnixFile(pFile, proxyPath); + } + } + return rc; + } + default: { + assert( 0 ); /* The call assures that only valid opcodes are sent */ + } + } + /*NOTREACHED*/ + return SQLITE_ERROR; +} + +/* +** Within this division (the proxying locking implementation) the procedures +** above this point are all utilities. The lock-related methods of the +** proxy-locking sqlite3_io_method object follow. +*/ + + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int proxyCheckReservedLock(sqlite3_file *id, int *pResOut) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *proxy = pCtx->lockProxy; + return proxy->pMethod->xCheckReservedLock((sqlite3_file*)proxy, pResOut); + } + return rc; +} + +/* +** Lock the file with the lock specified by parameter locktype - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int proxyLock(sqlite3_file *id, int locktype) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *proxy = pCtx->lockProxy; + rc = proxy->pMethod->xLock((sqlite3_file*)proxy, locktype); + pFile->locktype = proxy->locktype; + } + return rc; +} + + +/* +** Lower the locking level on file descriptor pFile to locktype. locktype +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int proxyUnlock(sqlite3_file *id, int locktype) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *proxy = pCtx->lockProxy; + rc = proxy->pMethod->xUnlock((sqlite3_file*)proxy, locktype); + pFile->locktype = proxy->locktype; + } + return rc; +} + +/* +** Close a file that uses proxy locks. +*/ +static int proxyClose(sqlite3_file *id) { + if( id ){ + unixFile *pFile = (unixFile*)id; + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *lockProxy = pCtx->lockProxy; + unixFile *conchFile = pCtx->conchFile; + int rc = SQLITE_OK; + + if( lockProxy ){ + rc = lockProxy->pMethod->xUnlock((sqlite3_file*)lockProxy, NO_LOCK); + if( rc ) return rc; + rc = lockProxy->pMethod->xClose((sqlite3_file*)lockProxy); + if( rc ) return rc; + sqlite3_free(lockProxy); + pCtx->lockProxy = 0; + } + if( conchFile ){ + if( pCtx->conchHeld ){ + rc = proxyReleaseConch(pFile); + if( rc ) return rc; + } + rc = conchFile->pMethod->xClose((sqlite3_file*)conchFile); + if( rc ) return rc; + sqlite3_free(conchFile); + } + sqlite3_free(pCtx->lockProxyPath); + sqlite3_free(pCtx->conchFilePath); + sqlite3_free(pCtx->dbPath); + /* restore the original locking context and pMethod then close it */ + pFile->lockingContext = pCtx->oldLockingContext; + pFile->pMethod = pCtx->pOldMethod; + sqlite3_free(pCtx); + return pFile->pMethod->xClose(id); + } + return SQLITE_OK; +} + + + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The proxy locking style is intended for use with AFP filesystems. +** And since AFP is only supported on MacOSX, the proxy locking is also +** restricted to MacOSX. +** +** +******************* End of the proxy lock implementation ********************** +******************************************************************************/ + +/* +** Initialize the operating system interface. +** +** This routine registers all VFS implementations for unix-like operating +** systems. This routine, and the sqlite3_os_end() routine that follows, +** should be the only routines in this file that are visible from other +** files. +** +** This routine is called once during SQLite initialization and by a +** single thread. The memory allocation and mutex subsystems have not +** necessarily been initialized when this routine is called, and so they +** should not be used. +*/ +int sqlite3_os_init(void){ + /* + ** The following macro defines an initializer for an sqlite3_vfs object. + ** The name of the VFS is NAME. The pAppData is a pointer to a pointer + ** to the "finder" function. (pAppData is a pointer to a pointer because + ** silly C90 rules prohibit a void* from being cast to a function pointer + ** and so we have to go through the intermediate pointer to avoid problems + ** when compiling with -pedantic-errors on GCC.) + ** + ** The FINDER parameter to this macro is the name of the pointer to the + ** finder-function. The finder-function returns a pointer to the + ** sqlite_io_methods object that implements the desired locking + ** behaviors. See the division above that contains the IOMETHODS + ** macro for addition information on finder-functions. + ** + ** Most finders simply return a pointer to a fixed sqlite3_io_methods + ** object. But the "autolockIoFinder" available on MacOSX does a little + ** more than that; it looks at the filesystem type that hosts the + ** database file and tries to choose an locking method appropriate for + ** that filesystem time. + */ + #define UNIXVFS(VFSNAME, FINDER) { \ + 1, /* iVersion */ \ + sizeof(unixFile), /* szOsFile */ \ + MAX_PATHNAME, /* mxPathname */ \ + 0, /* pNext */ \ + VFSNAME, /* zName */ \ + (void*)&FINDER, /* pAppData */ \ + unixOpen, /* xOpen */ \ + unixDelete, /* xDelete */ \ + unixAccess, /* xAccess */ \ + unixFullPathname, /* xFullPathname */ \ + unixDlOpen, /* xDlOpen */ \ + unixDlError, /* xDlError */ \ + unixDlSym, /* xDlSym */ \ + unixDlClose, /* xDlClose */ \ + unixRandomness, /* xRandomness */ \ + unixSleep, /* xSleep */ \ + unixCurrentTime, /* xCurrentTime */ \ + unixGetLastError /* xGetLastError */ \ + } + + /* + ** All default VFSes for unix are contained in the following array. + ** + ** Note that the sqlite3_vfs.pNext field of the VFS object is modified + ** by the SQLite core when the VFS is registered. So the following + ** array cannot be const. + */ + static sqlite3_vfs aVfs[] = { +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + UNIXVFS("unix", autolockIoFinder ), +#else + UNIXVFS("unix", posixIoFinder ), +#endif + UNIXVFS("unix-none", nolockIoFinder ), + UNIXVFS("unix-dotfile", dotlockIoFinder ), +#if OS_VXWORKS + UNIXVFS("unix-namedsem", semIoFinder ), +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + UNIXVFS("unix-posix", posixIoFinder ), + UNIXVFS("unix-flock", flockIoFinder ), +#endif +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + UNIXVFS("unix-afp", afpIoFinder ), + UNIXVFS("unix-proxy", proxyIoFinder ), +#endif + }; + unsigned int i; /* Loop counter */ + + /* Register all VFSes defined in the aVfs[] array */ + for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ + sqlite3_vfs_register(&aVfs[i], i==0); + } + return SQLITE_OK; +} + +/* +** Shutdown the operating system interface. +** +** Some operating systems might need to do some cleanup in this routine, +** to release dynamically allocated objects. But not on unix. +** This routine is a no-op for unix. +*/ +int sqlite3_os_end(void){ + return SQLITE_OK; +} + +#endif /* SQLITE_OS_UNIX */ diff --git a/os_win.c b/os_win.c index 9ebb9dc..8d4f1e8 100644 --- a/os_win.c +++ b/os_win.c @@ -11,10 +11,38 @@ ****************************************************************************** ** ** This file contains code that is specific to windows. +** +** $Id: os_win.c,v 1.145 2008/12/11 02:58:27 shane Exp $ */ #include "sqliteInt.h" -#include "os.h" -#if OS_WIN /* This file is used for windows only */ +#if SQLITE_OS_WIN /* This file is used for windows only */ + + +/* +** A Note About Memory Allocation: +** +** This driver uses malloc()/free() directly rather than going through +** the SQLite-wrappers sqlite3_malloc()/sqlite3_free(). Those wrappers +** are designed for use on embedded systems where memory is scarce and +** malloc failures happen frequently. Win32 does not typically run on +** embedded systems, and when it does the developers normally have bigger +** problems to worry about than running out of memory. So there is not +** a compelling need to use the wrappers. +** +** But there is a good reason to not use the wrappers. If we use the +** wrappers then we will get simulated malloc() failures within this +** driver. And that causes all kinds of problems for our tests. We +** could enhance SQLite to deal with simulated malloc failures within +** the OS driver, but the code to deal with those failure would not +** be exercised on Linux (which does not need to malloc() in the driver) +** and so we would have difficulty writing coverage tests for that +** code. Better to leave the code out, we think. +** +** The point of this discussion is as follows: When creating a new +** OS layer for an embedded system, if you use this file as an example, +** avoid the use of malloc()/free(). Those routines work ok on windows +** desktops but not so well in embedded systems. +*/ #include @@ -34,22 +62,26 @@ */ #include "os_common.h" +/* +** Some microsoft compilers lack this definition. +*/ +#ifndef INVALID_FILE_ATTRIBUTES +# define INVALID_FILE_ATTRIBUTES ((DWORD)-1) +#endif + /* ** Determine if we are dealing with WindowsCE - which has a much ** reduced API. */ -#if defined(_WIN32_WCE) -# define OS_WINCE 1 +#if SQLITE_OS_WINCE # define AreFileApisANSI() 1 -#else -# define OS_WINCE 0 #endif /* ** WinCE lacks native support for file locking so we have to fake it ** with some code of our own. */ -#if OS_WINCE +#if SQLITE_OS_WINCE typedef struct winceLock { int nReaders; /* Number of reader locks obtained */ BOOL bPending; /* Indicates a pending lock has been obtained */ @@ -59,16 +91,16 @@ typedef struct winceLock { #endif /* -** The winFile structure is a subclass of OsFile specific to the win32 +** The winFile structure is a subclass of sqlite3_file* specific to the win32 ** portability layer. */ typedef struct winFile winFile; struct winFile { - IoMethod const *pMethod;/* Must be first */ + const sqlite3_io_methods *pMethod;/* Must be first */ HANDLE h; /* Handle for accessing the file */ unsigned char locktype; /* Type of lock currently held on this file */ short sharedLockByte; /* Randomly chosen byte used as a shared lock */ -#if OS_WINCE +#if SQLITE_OS_WINCE WCHAR *zDeleteOnClose; /* Name of file to delete when closing */ HANDLE hMutex; /* Mutex used to control access to shared lock */ HANDLE hShared; /* Shared memory segment used for locking */ @@ -78,13 +110,6 @@ struct winFile { }; -/* -** Do not include any of the File I/O interface procedures if the -** SQLITE_OMIT_DISKIO macro is defined (indicating that there database -** will be in-memory only) -*/ -#ifndef SQLITE_OMIT_DISKIO - /* ** The following variable is (normally) set once and never changes ** thereafter. It records whether the operating system is Win95 @@ -97,7 +122,11 @@ struct winFile { ** In order to facilitate testing on a WinNT system, the test fixture ** can manually set this value to 1 to emulate Win98 behavior. */ +#ifdef SQLITE_TEST int sqlite3_os_type = 0; +#else +static int sqlite3_os_type = 0; +#endif /* ** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, @@ -110,7 +139,7 @@ int sqlite3_os_type = 0; ** WinNT/2K/XP so that we will know whether or not we can safely call ** the LockFileEx() API. */ -#if OS_WINCE +#if SQLITE_OS_WINCE # define isNT() (1) #else static int isNT(void){ @@ -122,25 +151,25 @@ int sqlite3_os_type = 0; } return sqlite3_os_type==2; } -#endif /* OS_WINCE */ +#endif /* SQLITE_OS_WINCE */ /* ** Convert a UTF-8 string to microsoft unicode (UTF-16?). ** -** Space to hold the returned string is obtained from sqliteMalloc. +** Space to hold the returned string is obtained from malloc. */ static WCHAR *utf8ToUnicode(const char *zFilename){ int nChar; WCHAR *zWideFilename; nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); - zWideFilename = sqliteMalloc( nChar*sizeof(zWideFilename[0]) ); + zWideFilename = malloc( nChar*sizeof(zWideFilename[0]) ); if( zWideFilename==0 ){ return 0; } nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, nChar); if( nChar==0 ){ - sqliteFree(zWideFilename); + free(zWideFilename); zWideFilename = 0; } return zWideFilename; @@ -148,21 +177,21 @@ static WCHAR *utf8ToUnicode(const char *zFilename){ /* ** Convert microsoft unicode to UTF-8. Space to hold the returned string is -** obtained from sqliteMalloc(). +** obtained from malloc(). */ static char *unicodeToUtf8(const WCHAR *zWideFilename){ int nByte; char *zFilename; nByte = WideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, 0, 0, 0, 0); - zFilename = sqliteMalloc( nByte ); + zFilename = malloc( nByte ); if( zFilename==0 ){ return 0; } nByte = WideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, zFilename, nByte, 0, 0); if( nByte == 0 ){ - sqliteFree(zFilename); + free(zFilename); zFilename = 0; } return zFilename; @@ -173,7 +202,7 @@ static char *unicodeToUtf8(const WCHAR *zWideFilename){ ** current codepage settings for file apis. ** ** Space to hold the returned string is obtained -** from sqliteMalloc. +** from malloc. */ static WCHAR *mbcsToUnicode(const char *zFilename){ int nByte; @@ -181,13 +210,13 @@ static WCHAR *mbcsToUnicode(const char *zFilename){ int codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; nByte = MultiByteToWideChar(codepage, 0, zFilename, -1, NULL,0)*sizeof(WCHAR); - zMbcsFilename = sqliteMalloc( nByte*sizeof(zMbcsFilename[0]) ); + zMbcsFilename = malloc( nByte*sizeof(zMbcsFilename[0]) ); if( zMbcsFilename==0 ){ return 0; } nByte = MultiByteToWideChar(codepage, 0, zFilename, -1, zMbcsFilename, nByte); if( nByte==0 ){ - sqliteFree(zMbcsFilename); + free(zMbcsFilename); zMbcsFilename = 0; } return zMbcsFilename; @@ -198,7 +227,7 @@ static WCHAR *mbcsToUnicode(const char *zFilename){ ** user's Ansi codepage. ** ** Space to hold the returned string is obtained from -** sqliteMalloc(). +** malloc(). */ static char *unicodeToMbcs(const WCHAR *zWideFilename){ int nByte; @@ -206,14 +235,14 @@ static char *unicodeToMbcs(const WCHAR *zWideFilename){ int codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; nByte = WideCharToMultiByte(codepage, 0, zWideFilename, -1, 0, 0, 0, 0); - zFilename = sqliteMalloc( nByte ); + zFilename = malloc( nByte ); if( zFilename==0 ){ return 0; } nByte = WideCharToMultiByte(codepage, 0, zWideFilename, -1, zFilename, nByte, 0, 0); if( nByte == 0 ){ - sqliteFree(zFilename); + free(zFilename); zFilename = 0; } return zFilename; @@ -221,9 +250,9 @@ static char *unicodeToMbcs(const WCHAR *zWideFilename){ /* ** Convert multibyte character string to UTF-8. Space to hold the -** returned string is obtained from sqliteMalloc(). +** returned string is obtained from malloc(). */ -static char *mbcsToUtf8(const char *zFilename){ +char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ char *zFilenameUtf8; WCHAR *zTmpWide; @@ -232,13 +261,13 @@ static char *mbcsToUtf8(const char *zFilename){ return 0; } zFilenameUtf8 = unicodeToUtf8(zTmpWide); - sqliteFree(zTmpWide); + free(zTmpWide); return zFilenameUtf8; } /* ** Convert UTF-8 to multibyte character string. Space to hold the -** returned string is obtained from sqliteMalloc(). +** returned string is obtained from malloc(). */ static char *utf8ToMbcs(const char *zFilename){ char *zFilenameMbcs; @@ -249,11 +278,11 @@ static char *utf8ToMbcs(const char *zFilename){ return 0; } zFilenameMbcs = unicodeToMbcs(zTmpWide); - sqliteFree(zTmpWide); + free(zTmpWide); return zFilenameMbcs; } -#if OS_WINCE +#if SQLITE_OS_WINCE /************************************************************************* ** This section contains code for WinCE only. */ @@ -267,7 +296,7 @@ struct tm *__cdecl localtime(const time_t *t) static struct tm y; FILETIME uTm, lTm; SYSTEMTIME pTm; - i64 t64; + sqlite3_int64 t64; t64 = *t; t64 = (t64 + 11644473600)*10000000; uTm.dwLowDateTime = t64 & 0xFFFFFFFF; @@ -329,7 +358,7 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ /* Create/open the named mutex */ pFile->hMutex = CreateMutexW(NULL, FALSE, zName); if (!pFile->hMutex){ - sqliteFree(zName); + free(zName); return FALSE; } @@ -351,7 +380,7 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ bInit = FALSE; } - sqliteFree(zName); + free(zName); /* If we succeeded in making the shared memory handle, map it. */ if (pFile->hShared){ @@ -408,12 +437,6 @@ static void winceDestroyLock(winFile *pFile){ UnmapViewOfFile(pFile->shared); CloseHandle(pFile->hShared); - if( pFile->zDeleteOnClose ){ - DeleteFileW(pFile->zDeleteOnClose); - sqliteFree(pFile->zDeleteOnClose); - pFile->zDeleteOnClose = 0; - } - /* Done with the mutex */ winceMutexRelease(pFile->hMutex); CloseHandle(pFile->hMutex); @@ -563,402 +586,12 @@ static BOOL winceLockFileEx( /* ** End of the special code for wince *****************************************************************************/ -#endif /* OS_WINCE */ +#endif /* SQLITE_OS_WINCE */ -/* -** Convert a UTF-8 filename into whatever form the underlying -** operating system wants filenames in. Space to hold the result -** is obtained from sqliteMalloc and must be freed by the calling -** function. -*/ -static void *convertUtf8Filename(const char *zFilename){ - void *zConverted = 0; - if( isNT() ){ - zConverted = utf8ToUnicode(zFilename); - }else{ - zConverted = utf8ToMbcs(zFilename); - } - /* caller will handle out of memory */ - return zConverted; -} - -/* -** Delete the named file. -** -** Note that windows does not allow a file to be deleted if some other -** process has it open. Sometimes a virus scanner or indexing program -** will open a journal file shortly after it is created in order to do -** whatever it is it does. While this other process is holding the -** file open, we will be unable to delete it. To work around this -** problem, we delay 100 milliseconds and try to delete again. Up -** to MX_DELETION_ATTEMPTs deletion attempts are run before giving -** up and returning an error. -*/ -#define MX_DELETION_ATTEMPTS 3 -int sqlite3WinDelete(const char *zFilename){ - int cnt = 0; - int rc; - void *zConverted = convertUtf8Filename(zFilename); - if( zConverted==0 ){ - return SQLITE_NOMEM; - } - SimulateIOError(return SQLITE_IOERR_DELETE); - if( isNT() ){ - do{ - rc = DeleteFileW(zConverted); - }while( rc==0 && GetFileAttributesW(zConverted)!=0xffffffff - && cnt++ < MX_DELETION_ATTEMPTS && (Sleep(100), 1) ); - }else{ -#if OS_WINCE - return SQLITE_NOMEM; -#else - do{ - rc = DeleteFileA(zConverted); - }while( rc==0 && GetFileAttributesA(zConverted)!=0xffffffff - && cnt++ < MX_DELETION_ATTEMPTS && (Sleep(100), 1) ); -#endif - } - sqliteFree(zConverted); - OSTRACE2("DELETE \"%s\"\n", zFilename); - return rc!=0 ? SQLITE_OK : SQLITE_IOERR; -} - -/* -** Return TRUE if the named file exists. -*/ -int sqlite3WinFileExists(const char *zFilename){ - int exists = 0; - void *zConverted = convertUtf8Filename(zFilename); - if( zConverted==0 ){ - return SQLITE_NOMEM; - } - if( isNT() ){ - exists = GetFileAttributesW((WCHAR*)zConverted) != 0xffffffff; - }else{ -#if OS_WINCE - return SQLITE_NOMEM; -#else - exists = GetFileAttributesA((char*)zConverted) != 0xffffffff; -#endif - } - sqliteFree(zConverted); - return exists; -} - -/* Forward declaration */ -static int allocateWinFile(winFile *pInit, OsFile **pId); - -/* -** Attempt to open a file for both reading and writing. If that -** fails, try opening it read-only. If the file does not exist, -** try to create it. -** -** On success, a handle for the open file is written to *id -** and *pReadonly is set to 0 if the file was opened for reading and -** writing or 1 if the file was opened read-only. The function returns -** SQLITE_OK. -** -** On failure, the function returns SQLITE_CANTOPEN and leaves -** *id and *pReadonly unchanged. -*/ -int sqlite3WinOpenReadWrite( - const char *zFilename, - OsFile **pId, - int *pReadonly -){ - winFile f; - HANDLE h; - void *zConverted = convertUtf8Filename(zFilename); - if( zConverted==0 ){ - return SQLITE_NOMEM; - } - assert( *pId==0 ); - - if( isNT() ){ - h = CreateFileW((WCHAR*)zConverted, - GENERIC_READ | GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE, - NULL, - OPEN_ALWAYS, - FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, - NULL - ); - if( h==INVALID_HANDLE_VALUE ){ - h = CreateFileW((WCHAR*)zConverted, - GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE, - NULL, - OPEN_ALWAYS, - FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, - NULL - ); - if( h==INVALID_HANDLE_VALUE ){ - sqliteFree(zConverted); - return SQLITE_CANTOPEN; - } - *pReadonly = 1; - }else{ - *pReadonly = 0; - } -#if OS_WINCE - if (!winceCreateLock(zFilename, &f)){ - CloseHandle(h); - sqliteFree(zConverted); - return SQLITE_CANTOPEN; - } -#endif - }else{ -#if OS_WINCE - return SQLITE_NOMEM; -#else - h = CreateFileA((char*)zConverted, - GENERIC_READ | GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE, - NULL, - OPEN_ALWAYS, - FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, - NULL - ); - if( h==INVALID_HANDLE_VALUE ){ - h = CreateFileA((char*)zConverted, - GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE, - NULL, - OPEN_ALWAYS, - FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, - NULL - ); - if( h==INVALID_HANDLE_VALUE ){ - sqliteFree(zConverted); - return SQLITE_CANTOPEN; - } - *pReadonly = 1; - }else{ - *pReadonly = 0; - } -#endif /* OS_WINCE */ - } - - sqliteFree(zConverted); - - f.h = h; -#if OS_WINCE - f.zDeleteOnClose = 0; -#endif - OSTRACE3("OPEN R/W %d \"%s\"\n", h, zFilename); - return allocateWinFile(&f, pId); -} - - -/* -** Attempt to open a new file for exclusive access by this process. -** The file will be opened for both reading and writing. To avoid -** a potential security problem, we do not allow the file to have -** previously existed. Nor do we allow the file to be a symbolic -** link. -** -** If delFlag is true, then make arrangements to automatically delete -** the file when it is closed. -** -** On success, write the file handle into *id and return SQLITE_OK. -** -** On failure, return SQLITE_CANTOPEN. -** -** Sometimes if we have just deleted a prior journal file, windows -** will fail to open a new one because there is a "pending delete". -** To work around this bug, we pause for 100 milliseconds and attempt -** a second open after the first one fails. The whole operation only -** fails if both open attempts are unsuccessful. -*/ -int sqlite3WinOpenExclusive(const char *zFilename, OsFile **pId, int delFlag){ - winFile f; - HANDLE h; - DWORD fileflags; - void *zConverted = convertUtf8Filename(zFilename); - if( zConverted==0 ){ - return SQLITE_NOMEM; - } - assert( *pId == 0 ); - fileflags = FILE_FLAG_RANDOM_ACCESS; -#if !OS_WINCE - if( delFlag ){ - fileflags |= FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE; - } -#endif - if( isNT() ){ - int cnt = 0; - do{ - h = CreateFileW((WCHAR*)zConverted, - GENERIC_READ | GENERIC_WRITE, - 0, - NULL, - CREATE_ALWAYS, - fileflags, - NULL - ); - }while( h==INVALID_HANDLE_VALUE && cnt++ < 2 && (Sleep(100), 1) ); - }else{ -#if OS_WINCE - return SQLITE_NOMEM; -#else - int cnt = 0; - do{ - h = CreateFileA((char*)zConverted, - GENERIC_READ | GENERIC_WRITE, - 0, - NULL, - CREATE_ALWAYS, - fileflags, - NULL - ); - }while( h==INVALID_HANDLE_VALUE && cnt++ < 2 && (Sleep(100), 1) ); -#endif /* OS_WINCE */ - } -#if OS_WINCE - if( delFlag && h!=INVALID_HANDLE_VALUE ){ - f.zDeleteOnClose = zConverted; - zConverted = 0; - } - f.hMutex = NULL; -#endif - sqliteFree(zConverted); - if( h==INVALID_HANDLE_VALUE ){ - return SQLITE_CANTOPEN; - } - f.h = h; - OSTRACE3("OPEN EX %d \"%s\"\n", h, zFilename); - return allocateWinFile(&f, pId); -} - -/* -** Attempt to open a new file for read-only access. -** -** On success, write the file handle into *id and return SQLITE_OK. -** -** On failure, return SQLITE_CANTOPEN. -*/ -int sqlite3WinOpenReadOnly(const char *zFilename, OsFile **pId){ - winFile f; - HANDLE h; - void *zConverted = convertUtf8Filename(zFilename); - if( zConverted==0 ){ - return SQLITE_NOMEM; - } - assert( *pId==0 ); - if( isNT() ){ - h = CreateFileW((WCHAR*)zConverted, - GENERIC_READ, - 0, - NULL, - OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, - NULL - ); - }else{ -#if OS_WINCE - return SQLITE_NOMEM; -#else - h = CreateFileA((char*)zConverted, - GENERIC_READ, - 0, - NULL, - OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, - NULL - ); -#endif - } - sqliteFree(zConverted); - if( h==INVALID_HANDLE_VALUE ){ - return SQLITE_CANTOPEN; - } - f.h = h; -#if OS_WINCE - f.zDeleteOnClose = 0; - f.hMutex = NULL; -#endif - OSTRACE3("OPEN RO %d \"%s\"\n", h, zFilename); - return allocateWinFile(&f, pId); -} - -/* -** Attempt to open a file descriptor for the directory that contains a -** file. This file descriptor can be used to fsync() the directory -** in order to make sure the creation of a new file is actually written -** to disk. -** -** This routine is only meaningful for Unix. It is a no-op under -** windows since windows does not support hard links. -** -** On success, a handle for a previously open file is at *id is -** updated with the new directory file descriptor and SQLITE_OK is -** returned. -** -** On failure, the function returns SQLITE_CANTOPEN and leaves -** *id unchanged. -*/ -static int winOpenDirectory( - OsFile *id, - const char *zDirname -){ - return SQLITE_OK; -} - -/* -** Create a temporary file name in zBuf. zBuf must be big enough to -** hold at least SQLITE_TEMPNAME_SIZE characters. -*/ -int sqlite3WinTempFileName(char *zBuf){ - static char zChars[] = - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "0123456789"; - int i, j; - char zTempPath[SQLITE_TEMPNAME_SIZE]; - if( sqlite3_temp_directory ){ - strncpy(zTempPath, sqlite3_temp_directory, SQLITE_TEMPNAME_SIZE-30); - zTempPath[SQLITE_TEMPNAME_SIZE-30] = 0; - }else if( isNT() ){ - char *zMulti; - WCHAR zWidePath[SQLITE_TEMPNAME_SIZE]; - GetTempPathW(SQLITE_TEMPNAME_SIZE-30, zWidePath); - zMulti = unicodeToUtf8(zWidePath); - if( zMulti ){ - strncpy(zTempPath, zMulti, SQLITE_TEMPNAME_SIZE-30); - zTempPath[SQLITE_TEMPNAME_SIZE-30] = 0; - sqliteFree(zMulti); - }else{ - return SQLITE_NOMEM; - } - }else{ - char *zUtf8; - char zMbcsPath[SQLITE_TEMPNAME_SIZE]; - GetTempPathA(SQLITE_TEMPNAME_SIZE-30, zMbcsPath); - zUtf8 = mbcsToUtf8(zMbcsPath); - if( zUtf8 ){ - strncpy(zTempPath, zUtf8, SQLITE_TEMPNAME_SIZE-30); - zTempPath[SQLITE_TEMPNAME_SIZE-30] = 0; - sqliteFree(zUtf8); - }else{ - return SQLITE_NOMEM; - } - } - for(i=strlen(zTempPath); i>0 && zTempPath[i-1]=='\\'; i--){} - zTempPath[i] = 0; - for(;;){ - sqlite3_snprintf(SQLITE_TEMPNAME_SIZE, zBuf, - "%s\\"TEMP_FILE_PREFIX, zTempPath); - j = strlen(zBuf); - sqlite3Randomness(15, &zBuf[j]); - for(i=0; i<15; i++, j++){ - zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; - } - zBuf[j] = 0; - if( !sqlite3OsFileExists(zBuf) ) break; - } - OSTRACE2("TEMP FILENAME: %s\n", zBuf); - return SQLITE_OK; -} +/***************************************************************************** +** The next group of routines implement the I/O methods specified +** by the sqlite3_io_methods object. +******************************************************************************/ /* ** Close a file. @@ -971,69 +604,32 @@ int sqlite3WinTempFileName(char *zBuf){ ** giving up and returning an error. */ #define MX_CLOSE_ATTEMPT 3 -static int winClose(OsFile **pId){ - winFile *pFile; - int rc = 1; - if( pId && (pFile = (winFile*)*pId)!=0 ){ - int rc, cnt = 0; - OSTRACE2("CLOSE %d\n", pFile->h); - do{ - rc = CloseHandle(pFile->h); - }while( rc==0 && cnt++ < MX_CLOSE_ATTEMPT && (Sleep(100), 1) ); -#if OS_WINCE - winceDestroyLock(pFile); +static int winClose(sqlite3_file *id){ + int rc, cnt = 0; + winFile *pFile = (winFile*)id; + OSTRACE2("CLOSE %d\n", pFile->h); + do{ + rc = CloseHandle(pFile->h); + }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (Sleep(100), 1) ); +#if SQLITE_OS_WINCE +#define WINCE_DELETION_ATTEMPTS 3 + winceDestroyLock(pFile); + if( pFile->zDeleteOnClose ){ + int cnt = 0; + while( + DeleteFileW(pFile->zDeleteOnClose)==0 + && GetFileAttributesW(pFile->zDeleteOnClose)!=0xffffffff + && cnt++ < WINCE_DELETION_ATTEMPTS + ){ + Sleep(100); /* Wait a little before trying again */ + } + free(pFile->zDeleteOnClose); + } #endif - OpenCounter(-1); - sqliteFree(pFile); - *pId = 0; - } + OpenCounter(-1); return rc ? SQLITE_OK : SQLITE_IOERR; } -/* -** Read data from a file into a buffer. Return SQLITE_OK if all -** bytes were read successfully and SQLITE_IOERR if anything goes -** wrong. -*/ -static int winRead(OsFile *id, void *pBuf, int amt){ - DWORD got; - assert( id!=0 ); - SimulateIOError(return SQLITE_IOERR_READ); - OSTRACE3("READ %d lock=%d\n", ((winFile*)id)->h, ((winFile*)id)->locktype); - if( !ReadFile(((winFile*)id)->h, pBuf, amt, &got, 0) ){ - return SQLITE_IOERR_READ; - } - if( got==(DWORD)amt ){ - return SQLITE_OK; - }else{ - memset(&((char*)pBuf)[got], 0, amt-got); - return SQLITE_IOERR_SHORT_READ; - } -} - -/* -** Write data from a buffer into a file. Return SQLITE_OK on success -** or some other error code on failure. -*/ -static int winWrite(OsFile *id, const void *pBuf, int amt){ - int rc = 0; - DWORD wrote; - assert( id!=0 ); - SimulateIOError(return SQLITE_IOERR_READ); - SimulateDiskfullError(return SQLITE_FULL); - OSTRACE3("WRITE %d lock=%d\n", ((winFile*)id)->h, ((winFile*)id)->locktype); - assert( amt>0 ); - while( amt>0 && (rc = WriteFile(((winFile*)id)->h, pBuf, amt, &wrote, 0))!=0 - && wrote>0 ){ - amt -= wrote; - pBuf = &((char*)pBuf)[wrote]; - } - if( !rc || amt>(int)wrote ){ - return SQLITE_FULL; - } - return SQLITE_OK; -} - /* ** Some microsoft compilers lack this definition. */ @@ -1042,68 +638,148 @@ static int winWrite(OsFile *id, const void *pBuf, int amt){ #endif /* -** Move the read/write pointer in a file. +** Read data from a file into a buffer. Return SQLITE_OK if all +** bytes were read successfully and SQLITE_IOERR if anything goes +** wrong. */ -static int winSeek(OsFile *id, i64 offset){ - LONG upperBits = offset>>32; - LONG lowerBits = offset & 0xffffffff; +static int winRead( + sqlite3_file *id, /* File to read from */ + void *pBuf, /* Write content into this buffer */ + int amt, /* Number of bytes to read */ + sqlite3_int64 offset /* Begin reading at this offset */ +){ + LONG upperBits = (LONG)((offset>>32) & 0x7fffffff); + LONG lowerBits = (LONG)(offset & 0xffffffff); DWORD rc; + DWORD got; + winFile *pFile = (winFile*)id; assert( id!=0 ); -#ifdef SQLITE_TEST - if( offset ) SimulateDiskfullError(return SQLITE_FULL); -#endif - rc = SetFilePointer(((winFile*)id)->h, lowerBits, &upperBits, FILE_BEGIN); - OSTRACE3("SEEK %d %lld\n", ((winFile*)id)->h, offset); + SimulateIOError(return SQLITE_IOERR_READ); + OSTRACE3("READ %d lock=%d\n", pFile->h, pFile->locktype); + rc = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); if( rc==INVALID_SET_FILE_POINTER && GetLastError()!=NO_ERROR ){ return SQLITE_FULL; } + if( !ReadFile(pFile->h, pBuf, amt, &got, 0) ){ + return SQLITE_IOERR_READ; + } + if( got==(DWORD)amt ){ + return SQLITE_OK; + }else{ + /* Unread parts of the buffer must be zero-filled */ + memset(&((char*)pBuf)[got], 0, amt-got); + return SQLITE_IOERR_SHORT_READ; + } +} + +/* +** Write data from a buffer into a file. Return SQLITE_OK on success +** or some other error code on failure. +*/ +static int winWrite( + sqlite3_file *id, /* File to write into */ + const void *pBuf, /* The bytes to be written */ + int amt, /* Number of bytes to write */ + sqlite3_int64 offset /* Offset into the file to begin writing at */ +){ + LONG upperBits = (LONG)((offset>>32) & 0x7fffffff); + LONG lowerBits = (LONG)(offset & 0xffffffff); + DWORD rc; + DWORD wrote = 0; + winFile *pFile = (winFile*)id; + assert( id!=0 ); + SimulateIOError(return SQLITE_IOERR_WRITE); + SimulateDiskfullError(return SQLITE_FULL); + OSTRACE3("WRITE %d lock=%d\n", pFile->h, pFile->locktype); + rc = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); + if( rc==INVALID_SET_FILE_POINTER && GetLastError()!=NO_ERROR ){ + return SQLITE_FULL; + } + assert( amt>0 ); + while( + amt>0 + && (rc = WriteFile(pFile->h, pBuf, amt, &wrote, 0))!=0 + && wrote>0 + ){ + amt -= wrote; + pBuf = &((char*)pBuf)[wrote]; + } + if( !rc || amt>(int)wrote ){ + return SQLITE_FULL; + } return SQLITE_OK; } +/* +** Truncate an open file to a specified size +*/ +static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ + DWORD rc; + LONG upperBits = (LONG)((nByte>>32) & 0x7fffffff); + LONG lowerBits = (LONG)(nByte & 0xffffffff); + winFile *pFile = (winFile*)id; + OSTRACE3("TRUNCATE %d %lld\n", pFile->h, nByte); + SimulateIOError(return SQLITE_IOERR_TRUNCATE); + rc = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); + if( INVALID_SET_FILE_POINTER != rc ){ + /* SetEndOfFile will fail if nByte is negative */ + if( SetEndOfFile(pFile->h) ){ + return SQLITE_OK; + } + } + return SQLITE_IOERR_TRUNCATE; +} + +#ifdef SQLITE_TEST +/* +** Count the number of fullsyncs and normal syncs. This is used to test +** that syncs and fullsyncs are occuring at the right times. +*/ +int sqlite3_sync_count = 0; +int sqlite3_fullsync_count = 0; +#endif + /* ** Make sure all writes to a particular file are committed to disk. */ -static int winSync(OsFile *id, int dataOnly){ - assert( id!=0 ); - OSTRACE3("SYNC %d lock=%d\n", ((winFile*)id)->h, ((winFile*)id)->locktype); - if( FlushFileBuffers(((winFile*)id)->h) ){ +static int winSync(sqlite3_file *id, int flags){ +#ifndef SQLITE_NO_SYNC + winFile *pFile = (winFile*)id; +#else + UNUSED_PARAMETER(id); +#endif + OSTRACE3("SYNC %d lock=%d\n", pFile->h, pFile->locktype); +#ifndef SQLITE_TEST + UNUSED_PARAMETER(flags); +#else + if( flags & SQLITE_SYNC_FULL ){ + sqlite3_fullsync_count++; + } + sqlite3_sync_count++; +#endif + /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a + ** no-op + */ +#ifdef SQLITE_NO_SYNC + return SQLITE_OK; +#else + if( FlushFileBuffers(pFile->h) ){ return SQLITE_OK; }else{ return SQLITE_IOERR; } -} - -/* -** Sync the directory zDirname. This is a no-op on operating systems other -** than UNIX. -*/ -int sqlite3WinSyncDirectory(const char *zDirname){ - SimulateIOError(return SQLITE_IOERR_READ); - return SQLITE_OK; -} - -/* -** Truncate an open file to a specified size -*/ -static int winTruncate(OsFile *id, i64 nByte){ - LONG upperBits = nByte>>32; - assert( id!=0 ); - OSTRACE3("TRUNCATE %d %lld\n", ((winFile*)id)->h, nByte); - SimulateIOError(return SQLITE_IOERR_TRUNCATE); - SetFilePointer(((winFile*)id)->h, nByte, &upperBits, FILE_BEGIN); - SetEndOfFile(((winFile*)id)->h); - return SQLITE_OK; +#endif } /* ** Determine the current size of a file in bytes */ -static int winFileSize(OsFile *id, i64 *pSize){ +static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ + winFile *pFile = (winFile*)id; DWORD upperBits, lowerBits; - assert( id!=0 ); SimulateIOError(return SQLITE_IOERR_FSTAT); - lowerBits = GetFileSize(((winFile*)id)->h, &upperBits); - *pSize = (((i64)upperBits)<<32) + lowerBits; + lowerBits = GetFileSize(pFile->h, &upperBits); + *pSize = (((sqlite3_int64)upperBits)<<32) + lowerBits; return SQLITE_OK; } @@ -1119,19 +795,24 @@ static int winFileSize(OsFile *id, i64 *pSize){ ** Different API routines are called depending on whether or not this ** is Win95 or WinNT. */ -static int getReadLock(winFile *id){ +static int getReadLock(winFile *pFile){ int res; if( isNT() ){ OVERLAPPED ovlp; ovlp.Offset = SHARED_FIRST; ovlp.OffsetHigh = 0; ovlp.hEvent = 0; - res = LockFileEx(id->h, LOCKFILE_FAIL_IMMEDIATELY, 0, SHARED_SIZE,0,&ovlp); + res = LockFileEx(pFile->h, LOCKFILE_FAIL_IMMEDIATELY, + 0, SHARED_SIZE, 0, &ovlp); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +*/ +#if SQLITE_OS_WINCE==0 }else{ int lk; - sqlite3Randomness(sizeof(lk), &lk); - id->sharedLockByte = (lk & 0x7fffffff)%(SHARED_SIZE - 1); - res = LockFile(id->h, SHARED_FIRST+id->sharedLockByte, 0, 1, 0); + sqlite3_randomness(sizeof(lk), &lk); + pFile->sharedLockByte = (short)((lk & 0x7fffffff)%(SHARED_SIZE - 1)); + res = LockFile(pFile->h, SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); +#endif } return res; } @@ -1143,45 +824,16 @@ static int unlockReadLock(winFile *pFile){ int res; if( isNT() ){ res = UnlockFile(pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +*/ +#if SQLITE_OS_WINCE==0 }else{ res = UnlockFile(pFile->h, SHARED_FIRST + pFile->sharedLockByte, 0, 1, 0); +#endif } return res; } -#ifndef SQLITE_OMIT_PAGER_PRAGMAS -/* -** Check that a given pathname is a directory and is writable -** -*/ -int sqlite3WinIsDirWritable(char *zDirname){ - int fileAttr; - void *zConverted; - if( zDirname==0 ) return 0; - if( !isNT() && strlen(zDirname)>MAX_PATH ) return 0; - - zConverted = convertUtf8Filename(zDirname); - if( zConverted==0 ){ - return SQLITE_NOMEM; - } - if( isNT() ){ - fileAttr = GetFileAttributesW((WCHAR*)zConverted); - }else{ -#if OS_WINCE - return 0; -#else - fileAttr = GetFileAttributesA((char*)zConverted); -#endif - } - sqliteFree(zConverted); - if( fileAttr == 0xffffffff ) return 0; - if( (fileAttr & FILE_ATTRIBUTE_DIRECTORY) != FILE_ATTRIBUTE_DIRECTORY ){ - return 0; - } - return 1; -} -#endif /* SQLITE_OMIT_PAGER_PRAGMAS */ - /* ** Lock the file with the lock specified by parameter locktype - one ** of the following: @@ -1208,10 +860,10 @@ int sqlite3WinIsDirWritable(char *zDirname){ ** It is not possible to lower the locking level one step at a time. You ** must go straight to locking level 0. */ -static int winLock(OsFile *id, int locktype){ +static int winLock(sqlite3_file *id, int locktype){ int rc = SQLITE_OK; /* Return code from subroutines */ int res = 1; /* Result of a windows lock call */ - int newLocktype; /* Set id->locktype to this value before exiting */ + int newLocktype; /* Set pFile->locktype to this value before exiting */ int gotPendingLock = 0;/* True if we acquired a PENDING lock this time */ winFile *pFile = (winFile*)id; @@ -1311,7 +963,7 @@ static int winLock(OsFile *id, int locktype){ locktype, newLocktype); rc = SQLITE_BUSY; } - pFile->locktype = newLocktype; + pFile->locktype = (u8)newLocktype; return rc; } @@ -1320,7 +972,7 @@ static int winLock(OsFile *id, int locktype){ ** file by this or any other process. If such a lock is held, return ** non-zero, otherwise zero. */ -static int winCheckReservedLock(OsFile *id){ +static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ int rc; winFile *pFile = (winFile*)id; assert( pFile!=0 ); @@ -1335,7 +987,8 @@ static int winCheckReservedLock(OsFile *id){ rc = !rc; OSTRACE3("TEST WR-LOCK %d %d (remote)\n", pFile->h, rc); } - return rc; + *pResOut = rc; + return SQLITE_OK; } /* @@ -1349,10 +1002,10 @@ static int winCheckReservedLock(OsFile *id){ ** is NO_LOCK. If the second argument is SHARED_LOCK then this routine ** might return SQLITE_IOERR; */ -static int winUnlock(OsFile *id, int locktype){ +static int winUnlock(sqlite3_file *id, int locktype){ int type; - int rc = SQLITE_OK; winFile *pFile = (winFile*)id; + int rc = SQLITE_OK; assert( pFile!=0 ); assert( locktype<=SHARED_LOCK ); OSTRACE5("UNLOCK %d to %d was %d(%d)\n", pFile->h, locktype, @@ -1375,80 +1028,21 @@ static int winUnlock(OsFile *id, int locktype){ if( type>=PENDING_LOCK ){ UnlockFile(pFile->h, PENDING_BYTE, 0, 1, 0); } - pFile->locktype = locktype; + pFile->locktype = (u8)locktype; return rc; } /* -** Turn a relative pathname into a full pathname. Return a pointer -** to the full pathname stored in space obtained from sqliteMalloc(). -** The calling function is responsible for freeing this space once it -** is no longer needed. +** Control and query of the open file handle. */ -char *sqlite3WinFullPathname(const char *zRelative){ - char *zFull; -#if defined(__CYGWIN__) - int nByte; - nByte = strlen(zRelative) + MAX_PATH + 1001; - zFull = sqliteMalloc( nByte ); - if( zFull==0 ) return 0; - if( cygwin_conv_to_full_win32_path(zRelative, zFull) ) return 0; -#elif OS_WINCE - /* WinCE has no concept of a relative pathname, or so I am told. */ - zFull = sqliteStrDup(zRelative); -#else - int nByte; - void *zConverted; - zConverted = convertUtf8Filename(zRelative); - if( isNT() ){ - WCHAR *zTemp; - nByte = GetFullPathNameW((WCHAR*)zConverted, 0, 0, 0) + 3; - zTemp = sqliteMalloc( nByte*sizeof(zTemp[0]) ); - if( zTemp==0 ){ - sqliteFree(zConverted); - return 0; +static int winFileControl(sqlite3_file *id, int op, void *pArg){ + switch( op ){ + case SQLITE_FCNTL_LOCKSTATE: { + *(int*)pArg = ((winFile*)id)->locktype; + return SQLITE_OK; } - GetFullPathNameW((WCHAR*)zConverted, nByte, zTemp, 0); - sqliteFree(zConverted); - zFull = unicodeToUtf8(zTemp); - sqliteFree(zTemp); - }else{ - char *zTemp; - nByte = GetFullPathNameA((char*)zConverted, 0, 0, 0) + 3; - zTemp = sqliteMalloc( nByte*sizeof(zTemp[0]) ); - if( zTemp==0 ){ - sqliteFree(zConverted); - return 0; - } - GetFullPathNameA((char*)zConverted, nByte, zTemp, 0); - sqliteFree(zConverted); - zFull = mbcsToUtf8(zTemp); - sqliteFree(zTemp); } -#endif - return zFull; -} - -/* -** The fullSync option is meaningless on windows. This is a no-op. -*/ -static void winSetFullSync(OsFile *id, int v){ - return; -} - -/* -** Return the underlying file handle for an OsFile -*/ -static int winFileHandle(OsFile *id){ - return (int)((winFile*)id)->h; -} - -/* -** Return an integer that indices the type of lock currently held -** by this handle. (Used for testing and analysis only.) -*/ -static int winLockState(OsFile *id){ - return ((winFile*)id)->locktype; + return SQLITE_ERROR; } /* @@ -1458,200 +1052,568 @@ static int winLockState(OsFile *id){ ** ** SQLite code assumes this function cannot fail. It also assumes that ** if two files are created in the same file-system directory (i.e. -** a database and it's journal file) that the sector size will be the +** a database and its journal file) that the sector size will be the ** same for both. */ -static int winSectorSize(OsFile *id){ +static int winSectorSize(sqlite3_file *id){ + UNUSED_PARAMETER(id); return SQLITE_DEFAULT_SECTOR_SIZE; } /* -** This vector defines all the methods that can operate on an OsFile -** for win32. +** Return a vector of device characteristics. */ -static const IoMethod sqlite3WinIoMethod = { +static int winDeviceCharacteristics(sqlite3_file *id){ + UNUSED_PARAMETER(id); + return 0; +} + +/* +** This vector defines all the methods that can operate on an +** sqlite3_file for win32. +*/ +static const sqlite3_io_methods winIoMethod = { + 1, /* iVersion */ winClose, - winOpenDirectory, winRead, winWrite, - winSeek, winTruncate, winSync, - winSetFullSync, - winFileHandle, winFileSize, winLock, winUnlock, - winLockState, winCheckReservedLock, + winFileControl, winSectorSize, + winDeviceCharacteristics }; +/*************************************************************************** +** Here ends the I/O methods that form the sqlite3_io_methods object. +** +** The next block of code implements the VFS methods. +****************************************************************************/ + /* -** Allocate memory for an OsFile. Initialize the new OsFile -** to the value given in pInit and return a pointer to the new -** OsFile. If we run out of memory, close the file and return NULL. +** Convert a UTF-8 filename into whatever form the underlying +** operating system wants filenames in. Space to hold the result +** is obtained from malloc and must be freed by the calling +** function. */ -static int allocateWinFile(winFile *pInit, OsFile **pId){ - winFile *pNew; - pNew = sqliteMalloc( sizeof(*pNew) ); - if( pNew==0 ){ - CloseHandle(pInit->h); -#if OS_WINCE - sqliteFree(pInit->zDeleteOnClose); -#endif - *pId = 0; - return SQLITE_NOMEM; +static void *convertUtf8Filename(const char *zFilename){ + void *zConverted = 0; + if( isNT() ){ + zConverted = utf8ToUnicode(zFilename); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +*/ +#if SQLITE_OS_WINCE==0 }else{ - *pNew = *pInit; - pNew->pMethod = &sqlite3WinIoMethod; - pNew->locktype = NO_LOCK; - pNew->sharedLockByte = 0; - *pId = (OsFile*)pNew; - OpenCounter(+1); - return SQLITE_OK; + zConverted = utf8ToMbcs(zFilename); +#endif } + /* caller will handle out of memory */ + return zConverted; +} + +/* +** Create a temporary file name in zBuf. zBuf must be big enough to +** hold at pVfs->mxPathname characters. +*/ +static int getTempname(int nBuf, char *zBuf){ + static char zChars[] = + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789"; + size_t i, j; + char zTempPath[MAX_PATH+1]; + if( sqlite3_temp_directory ){ + sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", sqlite3_temp_directory); + }else if( isNT() ){ + char *zMulti; + WCHAR zWidePath[MAX_PATH]; + GetTempPathW(MAX_PATH-30, zWidePath); + zMulti = unicodeToUtf8(zWidePath); + if( zMulti ){ + sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zMulti); + free(zMulti); + }else{ + return SQLITE_NOMEM; + } +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. +*/ +#if SQLITE_OS_WINCE==0 + }else{ + char *zUtf8; + char zMbcsPath[MAX_PATH]; + GetTempPathA(MAX_PATH-30, zMbcsPath); + zUtf8 = sqlite3_win32_mbcs_to_utf8(zMbcsPath); + if( zUtf8 ){ + sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zUtf8); + free(zUtf8); + }else{ + return SQLITE_NOMEM; + } +#endif + } + for(i=sqlite3Strlen30(zTempPath); i>0 && zTempPath[i-1]=='\\'; i--){} + zTempPath[i] = 0; + sqlite3_snprintf(nBuf-30, zBuf, + "%s\\"SQLITE_TEMP_FILE_PREFIX, zTempPath); + j = sqlite3Strlen30(zBuf); + sqlite3_randomness(20, &zBuf[j]); + for(i=0; i<20; i++, j++){ + zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; + } + zBuf[j] = 0; + OSTRACE2("TEMP FILENAME: %s\n", zBuf); + return SQLITE_OK; +} + +/* +** The return value of getLastErrorMsg +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). +*/ +static int getLastErrorMsg(int nBuf, char *zBuf){ + DWORD error = GetLastError(); + +#if SQLITE_OS_WINCE + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", error, error); +#else + /* FormatMessage returns 0 on failure. Otherwise it + ** returns the number of TCHARs written to the output + ** buffer, excluding the terminating null char. + */ + if (!FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, + NULL, + error, + 0, + zBuf, + nBuf-1, + 0)) + { + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", error, error); + } +#endif + + return 0; } -#endif /* SQLITE_OMIT_DISKIO */ -/*************************************************************************** -** Everything above deals with file I/O. Everything that follows deals -** with other miscellanous aspects of the operating system interface -****************************************************************************/ +/* +** Open a file. +*/ +static int winOpen( + sqlite3_vfs *pVfs, /* Not used */ + const char *zName, /* Name of the file (UTF-8) */ + sqlite3_file *id, /* Write the SQLite file handle here */ + int flags, /* Open mode flags */ + int *pOutFlags /* Status return flags */ +){ + HANDLE h; + DWORD dwDesiredAccess; + DWORD dwShareMode; + DWORD dwCreationDisposition; + DWORD dwFlagsAndAttributes = 0; +#if SQLITE_OS_WINCE + int isTemp = 0; +#endif + winFile *pFile = (winFile*)id; + void *zConverted; /* Filename in OS encoding */ + const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ + char zTmpname[MAX_PATH+1]; /* Buffer used to create temp filename */ -#if !defined(SQLITE_OMIT_LOAD_EXTENSION) + UNUSED_PARAMETER(pVfs); + + /* If the second argument to this function is NULL, generate a + ** temporary file name to use + */ + if( !zUtf8Name ){ + int rc = getTempname(MAX_PATH+1, zTmpname); + if( rc!=SQLITE_OK ){ + return rc; + } + zUtf8Name = zTmpname; + } + + /* Convert the filename to the system encoding. */ + zConverted = convertUtf8Filename(zUtf8Name); + if( zConverted==0 ){ + return SQLITE_NOMEM; + } + + if( flags & SQLITE_OPEN_READWRITE ){ + dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; + }else{ + dwDesiredAccess = GENERIC_READ; + } + if( flags & SQLITE_OPEN_CREATE ){ + dwCreationDisposition = OPEN_ALWAYS; + }else{ + dwCreationDisposition = OPEN_EXISTING; + } + if( flags & SQLITE_OPEN_MAIN_DB ){ + dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; + }else{ + dwShareMode = 0; + } + if( flags & SQLITE_OPEN_DELETEONCLOSE ){ +#if SQLITE_OS_WINCE + dwFlagsAndAttributes = FILE_ATTRIBUTE_HIDDEN; + isTemp = 1; +#else + dwFlagsAndAttributes = FILE_ATTRIBUTE_TEMPORARY + | FILE_ATTRIBUTE_HIDDEN + | FILE_FLAG_DELETE_ON_CLOSE; +#endif + }else{ + dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; + } + /* Reports from the internet are that performance is always + ** better if FILE_FLAG_RANDOM_ACCESS is used. Ticket #2699. */ +#if SQLITE_OS_WINCE + dwFlagsAndAttributes |= FILE_FLAG_RANDOM_ACCESS; +#endif + if( isNT() ){ + h = CreateFileW((WCHAR*)zConverted, + dwDesiredAccess, + dwShareMode, + NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL + ); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. +*/ +#if SQLITE_OS_WINCE==0 + }else{ + h = CreateFileA((char*)zConverted, + dwDesiredAccess, + dwShareMode, + NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL + ); +#endif + } + if( h==INVALID_HANDLE_VALUE ){ + free(zConverted); + if( flags & SQLITE_OPEN_READWRITE ){ + return winOpen(0, zName, id, + ((flags|SQLITE_OPEN_READONLY)&~SQLITE_OPEN_READWRITE), pOutFlags); + }else{ + return SQLITE_CANTOPEN; + } + } + if( pOutFlags ){ + if( flags & SQLITE_OPEN_READWRITE ){ + *pOutFlags = SQLITE_OPEN_READWRITE; + }else{ + *pOutFlags = SQLITE_OPEN_READONLY; + } + } + memset(pFile, 0, sizeof(*pFile)); + pFile->pMethod = &winIoMethod; + pFile->h = h; +#if SQLITE_OS_WINCE + if( (flags & (SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB)) == + (SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB) + && !winceCreateLock(zName, pFile) + ){ + CloseHandle(h); + free(zConverted); + return SQLITE_CANTOPEN; + } + if( isTemp ){ + pFile->zDeleteOnClose = zConverted; + }else +#endif + { + free(zConverted); + } + OpenCounter(+1); + return SQLITE_OK; +} + +/* +** Delete the named file. +** +** Note that windows does not allow a file to be deleted if some other +** process has it open. Sometimes a virus scanner or indexing program +** will open a journal file shortly after it is created in order to do +** whatever it does. While this other process is holding the +** file open, we will be unable to delete it. To work around this +** problem, we delay 100 milliseconds and try to delete again. Up +** to MX_DELETION_ATTEMPTs deletion attempts are run before giving +** up and returning an error. +*/ +#define MX_DELETION_ATTEMPTS 5 +static int winDelete( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to delete */ + int syncDir /* Not used on win32 */ +){ + int cnt = 0; + DWORD rc; + DWORD error = 0; + void *zConverted = convertUtf8Filename(zFilename); + UNUSED_PARAMETER(pVfs); + UNUSED_PARAMETER(syncDir); + if( zConverted==0 ){ + return SQLITE_NOMEM; + } + SimulateIOError(return SQLITE_IOERR_DELETE); + if( isNT() ){ + do{ + DeleteFileW(zConverted); + }while( ( ((rc = GetFileAttributesW(zConverted)) != INVALID_FILE_ATTRIBUTES) + || ((error = GetLastError()) == ERROR_ACCESS_DENIED)) + && (++cnt < MX_DELETION_ATTEMPTS) + && (Sleep(100), 1) ); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. +*/ +#if SQLITE_OS_WINCE==0 + }else{ + do{ + DeleteFileA(zConverted); + }while( ( ((rc = GetFileAttributesA(zConverted)) != INVALID_FILE_ATTRIBUTES) + || ((error = GetLastError()) == ERROR_ACCESS_DENIED)) + && (++cnt < MX_DELETION_ATTEMPTS) + && (Sleep(100), 1) ); +#endif + } + free(zConverted); + OSTRACE2("DELETE \"%s\"\n", zFilename); + return ( (rc == INVALID_FILE_ATTRIBUTES) + && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK : SQLITE_IOERR_DELETE; +} + +/* +** Check the existance and status of a file. +*/ +static int winAccess( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to check */ + int flags, /* Type of test to make on this file */ + int *pResOut /* OUT: Result */ +){ + DWORD attr; + int rc = 0; + void *zConverted = convertUtf8Filename(zFilename); + UNUSED_PARAMETER(pVfs); + if( zConverted==0 ){ + return SQLITE_NOMEM; + } + if( isNT() ){ + attr = GetFileAttributesW((WCHAR*)zConverted); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. +*/ +#if SQLITE_OS_WINCE==0 + }else{ + attr = GetFileAttributesA((char*)zConverted); +#endif + } + free(zConverted); + switch( flags ){ + case SQLITE_ACCESS_READ: + case SQLITE_ACCESS_EXISTS: + rc = attr!=INVALID_FILE_ATTRIBUTES; + break; + case SQLITE_ACCESS_READWRITE: + rc = (attr & FILE_ATTRIBUTE_READONLY)==0; + break; + default: + assert(!"Invalid flags argument"); + } + *pResOut = rc; + return SQLITE_OK; +} + + +/* +** Turn a relative pathname into a full pathname. Write the full +** pathname into zOut[]. zOut[] will be at least pVfs->mxPathname +** bytes in size. +*/ +static int winFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zRelative, /* Possibly relative input path */ + int nFull, /* Size of output buffer in bytes */ + char *zFull /* Output buffer */ +){ + +#if defined(__CYGWIN__) + UNUSED_PARAMETER(nFull); + cygwin_conv_to_full_win32_path(zRelative, zFull); + return SQLITE_OK; +#endif + +#if SQLITE_OS_WINCE + UNUSED_PARAMETER(nFull); + /* WinCE has no concept of a relative pathname, or so I am told. */ + sqlite3_snprintf(pVfs->mxPathname, zFull, "%s", zRelative); + return SQLITE_OK; +#endif + +#if !SQLITE_OS_WINCE && !defined(__CYGWIN__) + int nByte; + void *zConverted; + char *zOut; + UNUSED_PARAMETER(nFull); + zConverted = convertUtf8Filename(zRelative); + if( isNT() ){ + WCHAR *zTemp; + nByte = GetFullPathNameW((WCHAR*)zConverted, 0, 0, 0) + 3; + zTemp = malloc( nByte*sizeof(zTemp[0]) ); + if( zTemp==0 ){ + free(zConverted); + return SQLITE_NOMEM; + } + GetFullPathNameW((WCHAR*)zConverted, nByte, zTemp, 0); + free(zConverted); + zOut = unicodeToUtf8(zTemp); + free(zTemp); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. +*/ +#if SQLITE_OS_WINCE==0 + }else{ + char *zTemp; + nByte = GetFullPathNameA((char*)zConverted, 0, 0, 0) + 3; + zTemp = malloc( nByte*sizeof(zTemp[0]) ); + if( zTemp==0 ){ + free(zConverted); + return SQLITE_NOMEM; + } + GetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + free(zConverted); + zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + free(zTemp); +#endif + } + if( zOut ){ + sqlite3_snprintf(pVfs->mxPathname, zFull, "%s", zOut); + free(zOut); + return SQLITE_OK; + }else{ + return SQLITE_NOMEM; + } +#endif +} + +#ifndef SQLITE_OMIT_LOAD_EXTENSION /* ** Interfaces for opening a shared library, finding entry points ** within the shared library, and closing the shared library. */ -void *sqlite3WinDlopen(const char *zFilename){ +/* +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. +*/ +static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ HANDLE h; void *zConverted = convertUtf8Filename(zFilename); + UNUSED_PARAMETER(pVfs); if( zConverted==0 ){ return 0; } if( isNT() ){ h = LoadLibraryW((WCHAR*)zConverted); +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. +*/ +#if SQLITE_OS_WINCE==0 }else{ -#if OS_WINCE - return 0; -#else h = LoadLibraryA((char*)zConverted); #endif } - sqliteFree(zConverted); + free(zConverted); return (void*)h; - } -void *sqlite3WinDlsym(void *pHandle, const char *zSymbol){ -#if OS_WINCE +static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ + UNUSED_PARAMETER(pVfs); + getLastErrorMsg(nBuf, zBufOut); +} +void (*winDlSym(sqlite3_vfs *pVfs, void *pHandle, const char *zSymbol))(void){ + UNUSED_PARAMETER(pVfs); +#if SQLITE_OS_WINCE /* The GetProcAddressA() routine is only available on wince. */ - return GetProcAddressA((HANDLE)pHandle, zSymbol); + return (void(*)(void))GetProcAddressA((HANDLE)pHandle, zSymbol); #else /* All other windows platforms expect GetProcAddress() to take ** an Ansi string regardless of the _UNICODE setting */ - return GetProcAddress((HANDLE)pHandle, zSymbol); + return (void(*)(void))GetProcAddress((HANDLE)pHandle, zSymbol); #endif } -int sqlite3WinDlclose(void *pHandle){ - return FreeLibrary((HANDLE)pHandle); +void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ + UNUSED_PARAMETER(pVfs); + FreeLibrary((HANDLE)pHandle); } -#endif /* !SQLITE_OMIT_LOAD_EXTENSION */ +#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ + #define winDlOpen 0 + #define winDlError 0 + #define winDlSym 0 + #define winDlClose 0 +#endif + /* -** Get information to seed the random number generator. The seed -** is written into the buffer zBuf[256]. The calling function must -** supply a sufficiently large buffer. +** Write up to nBuf bytes of randomness into zBuf. */ -int sqlite3WinRandomSeed(char *zBuf){ - /* We have to initialize zBuf to prevent valgrind from reporting - ** errors. The reports issued by valgrind are incorrect - we would - ** prefer that the randomness be increased by making use of the - ** uninitialized space in zBuf - but valgrind errors tend to worry - ** some users. Rather than argue, it seems easier just to initialize - ** the whole array and silence valgrind, even if that means less randomness - ** in the random seed. - ** - ** When testing, initializing zBuf[] to zero is all we do. That means - ** that we always use the same random number sequence.* This makes the - ** tests repeatable. - */ - memset(zBuf, 0, 256); - GetSystemTime((LPSYSTEMTIME)zBuf); - return SQLITE_OK; +static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + int n = 0; + UNUSED_PARAMETER(pVfs); +#if defined(SQLITE_TEST) + n = nBuf; + memset(zBuf, 0, nBuf); +#else + if( sizeof(SYSTEMTIME)<=nBuf-n ){ + SYSTEMTIME x; + GetSystemTime(&x); + memcpy(&zBuf[n], &x, sizeof(x)); + n += sizeof(x); + } + if( sizeof(DWORD)<=nBuf-n ){ + DWORD pid = GetCurrentProcessId(); + memcpy(&zBuf[n], &pid, sizeof(pid)); + n += sizeof(pid); + } + if( sizeof(DWORD)<=nBuf-n ){ + DWORD cnt = GetTickCount(); + memcpy(&zBuf[n], &cnt, sizeof(cnt)); + n += sizeof(cnt); + } + if( sizeof(LARGE_INTEGER)<=nBuf-n ){ + LARGE_INTEGER i; + QueryPerformanceCounter(&i); + memcpy(&zBuf[n], &i, sizeof(i)); + n += sizeof(i); + } +#endif + return n; } + /* ** Sleep for a little while. Return the amount of time slept. */ -int sqlite3WinSleep(int ms){ - Sleep(ms); - return ms; +static int winSleep(sqlite3_vfs *pVfs, int microsec){ + Sleep((microsec+999)/1000); + UNUSED_PARAMETER(pVfs); + return ((microsec+999)/1000)*1000; } -/* -** Static variables used for thread synchronization -*/ -static int inMutex = 0; -#ifdef SQLITE_W32_THREADS - static DWORD mutexOwner; - static CRITICAL_SECTION cs; -#endif - -/* -** The following pair of routines implement mutual exclusion for -** multi-threaded processes. Only a single thread is allowed to -** executed code that is surrounded by EnterMutex() and LeaveMutex(). -** -** SQLite uses only a single Mutex. There is not much critical -** code and what little there is executes quickly and without blocking. -** -** Version 3.3.1 and earlier used a simple mutex. Beginning with -** version 3.3.2, a recursive mutex is required. -*/ -void sqlite3WinEnterMutex(){ -#ifdef SQLITE_W32_THREADS - static int isInit = 0; - while( !isInit ){ - static long lock = 0; - if( InterlockedIncrement(&lock)==1 ){ - InitializeCriticalSection(&cs); - isInit = 1; - }else{ - Sleep(1); - } - } - EnterCriticalSection(&cs); - mutexOwner = GetCurrentThreadId(); -#endif - inMutex++; -} -void sqlite3WinLeaveMutex(){ - assert( inMutex ); - inMutex--; -#ifdef SQLITE_W32_THREADS - assert( mutexOwner==GetCurrentThreadId() ); - LeaveCriticalSection(&cs); -#endif -} - -/* -** Return TRUE if the mutex is currently held. -** -** If the thisThreadOnly parameter is true, return true if and only if the -** calling thread holds the mutex. If the parameter is false, return -** true if any thread holds the mutex. -*/ -int sqlite3WinInMutex(int thisThreadOnly){ -#ifdef SQLITE_W32_THREADS - return inMutex>0 && (thisThreadOnly==0 || mutexOwner==GetCurrentThreadId()); -#else - return inMutex>0; -#endif -} - - /* ** The following variable, if set to a non-zero value, becomes the result ** returned from sqlite3OsCurrentTime(). This is used for testing. @@ -1665,19 +1627,23 @@ int sqlite3_current_time = 0; ** current time and date as a Julian Day number into *prNow and ** return 0. Return 1 if the time and date cannot be found. */ -int sqlite3WinCurrentTime(double *prNow){ +int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ FILETIME ft; /* FILETIME structure is a 64-bit value representing the number of 100-nanosecond intervals since January 1, 1601 (= JD 2305813.5). */ double now; -#if OS_WINCE +#if SQLITE_OS_WINCE SYSTEMTIME time; GetSystemTime(&time); - SystemTimeToFileTime(&time,&ft); + /* if SystemTimeToFileTime() fails, it returns zero. */ + if (!SystemTimeToFileTime(&time,&ft)){ + return 1; + } #else GetSystemTimeAsFileTime( &ft ); #endif + UNUSED_PARAMETER(pVfs); now = ((double)ft.dwHighDateTime) * 4294967296.0; *prNow = (now + ft.dwLowDateTime)/864000000000.0 + 2305813.5; #ifdef SQLITE_TEST @@ -1689,70 +1655,70 @@ int sqlite3WinCurrentTime(double *prNow){ } /* -** Remember the number of thread-specific-data blocks allocated. -** Use this to verify that we are not leaking thread-specific-data. -** Ticket #1601 +** The idea is that this function works like a combination of +** GetLastError() and FormatMessage() on windows (or errno and +** strerror_r() on unix). After an error is returned by an OS +** function, SQLite calls this function with zBuf pointing to +** a buffer of nBuf bytes. The OS layer should populate the +** buffer with a nul-terminated UTF-8 encoded error message +** describing the last IO error to have occured within the calling +** thread. +** +** If the error message is too large for the supplied buffer, +** it should be truncated. The return value of xGetLastError +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). If non-zero is returned, +** then it is not necessary to include the nul-terminator character +** in the output buffer. +** +** Not supplying an error message will have no adverse effect +** on SQLite. It is fine to have an implementation that never +** returns an error message: +** +** int xGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ +** assert(zBuf[0]=='\0'); +** return 0; +** } +** +** However if an error message is supplied, it will be incorporated +** by sqlite into the error message available to the user using +** sqlite3_errmsg(), possibly making IO errors easier to debug. */ -#ifdef SQLITE_TEST -int sqlite3_tsd_count = 0; -# define TSD_COUNTER_INCR InterlockedIncrement(&sqlite3_tsd_count) -# define TSD_COUNTER_DECR InterlockedDecrement(&sqlite3_tsd_count) -#else -# define TSD_COUNTER_INCR /* no-op */ -# define TSD_COUNTER_DECR /* no-op */ -#endif - - +static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + UNUSED_PARAMETER(pVfs); + return getLastErrorMsg(nBuf, zBuf); +} /* -** If called with allocateFlag>1, then return a pointer to thread -** specific data for the current thread. Allocate and zero the -** thread-specific data if it does not already exist necessary. -** -** If called with allocateFlag==0, then check the current thread -** specific data. Return it if it exists. If it does not exist, -** then return NULL. -** -** If called with allocateFlag<0, check to see if the thread specific -** data is allocated and is all zero. If it is then deallocate it. -** Return a pointer to the thread specific data or NULL if it is -** unallocated or gets deallocated. +** Initialize and deinitialize the operating system interface. */ -ThreadData *sqlite3WinThreadSpecificData(int allocateFlag){ - static int key; - static int keyInit = 0; - static const ThreadData zeroData = {0}; - ThreadData *pTsd; - - if( !keyInit ){ - sqlite3OsEnterMutex(); - if( !keyInit ){ - key = TlsAlloc(); - if( key==0xffffffff ){ - sqlite3OsLeaveMutex(); - return 0; - } - keyInit = 1; - } - sqlite3OsLeaveMutex(); - } - pTsd = TlsGetValue(key); - if( allocateFlag>0 ){ - if( !pTsd ){ - pTsd = sqlite3OsMalloc( sizeof(zeroData) ); - if( pTsd ){ - *pTsd = zeroData; - TlsSetValue(key, pTsd); - TSD_COUNTER_INCR; - } - } - }else if( pTsd!=0 && allocateFlag<0 - && memcmp(pTsd, &zeroData, sizeof(ThreadData))==0 ){ - sqlite3OsFree(pTsd); - TlsSetValue(key, 0); - TSD_COUNTER_DECR; - pTsd = 0; - } - return pTsd; +int sqlite3_os_init(void){ + static sqlite3_vfs winVfs = { + 1, /* iVersion */ + sizeof(winFile), /* szOsFile */ + MAX_PATH, /* mxPathname */ + 0, /* pNext */ + "win32", /* zName */ + 0, /* pAppData */ + + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError /* xGetLastError */ + }; + sqlite3_vfs_register(&winVfs, 1); + return SQLITE_OK; } -#endif /* OS_WIN */ +int sqlite3_os_end(void){ + return SQLITE_OK; +} + +#endif /* SQLITE_OS_WIN */ diff --git a/pager.c b/pager.c index 1202b77..81e30bb 100644 --- a/pager.c +++ b/pager.c @@ -18,40 +18,29 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.355 2007/08/11 00:26:21 drh Exp $ +** @(#) $Id: pager.c,v 1.551 2009/01/14 23:03:41 drh Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" -#include "os.h" -#include "pager.h" -#include -#include /* ** Macros for troubleshooting. Normally turned off */ #if 0 +int sqlite3PagerTrace=1; /* True to enable tracing */ #define sqlite3DebugPrintf printf -#define PAGERTRACE1(X) sqlite3DebugPrintf(X) -#define PAGERTRACE2(X,Y) sqlite3DebugPrintf(X,Y) -#define PAGERTRACE3(X,Y,Z) sqlite3DebugPrintf(X,Y,Z) -#define PAGERTRACE4(X,Y,Z,W) sqlite3DebugPrintf(X,Y,Z,W) -#define PAGERTRACE5(X,Y,Z,W,V) sqlite3DebugPrintf(X,Y,Z,W,V) +#define PAGERTRACE(X) if( sqlite3PagerTrace ){ sqlite3DebugPrintf X; } #else -#define PAGERTRACE1(X) -#define PAGERTRACE2(X,Y) -#define PAGERTRACE3(X,Y,Z) -#define PAGERTRACE4(X,Y,Z,W) -#define PAGERTRACE5(X,Y,Z,W,V) +#define PAGERTRACE(X) #endif /* -** The following two macros are used within the PAGERTRACEX() macros above +** The following two macros are used within the PAGERTRACE() macros above ** to print out file-descriptors. ** -** PAGERID() takes a pointer to a Pager struct as it's argument. The -** associated file-descriptor is returned. FILEHANDLEID() takes an OsFile -** struct as it's argument. +** PAGERID() takes a pointer to a Pager struct as its argument. The +** associated file-descriptor is returned. FILEHANDLEID() takes an sqlite3_file +** struct as its argument. */ #define PAGERID(p) ((int)(p->fd)) #define FILEHANDLEID(fd) ((int)fd) @@ -110,160 +99,12 @@ #define PAGER_EXCLUSIVE 4 /* same as EXCLUSIVE_LOCK */ #define PAGER_SYNCED 5 -/* -** If the SQLITE_BUSY_RESERVED_LOCK macro is set to true at compile-time, -** then failed attempts to get a reserved lock will invoke the busy callback. -** This is off by default. To see why, consider the following scenario: -** -** Suppose thread A already has a shared lock and wants a reserved lock. -** Thread B already has a reserved lock and wants an exclusive lock. If -** both threads are using their busy callbacks, it might be a long time -** be for one of the threads give up and allows the other to proceed. -** But if the thread trying to get the reserved lock gives up quickly -** (if it never invokes its busy callback) then the contention will be -** resolved quickly. -*/ -#ifndef SQLITE_BUSY_RESERVED_LOCK -# define SQLITE_BUSY_RESERVED_LOCK 0 -#endif - /* ** This macro rounds values up so that if the value is an address it ** is guaranteed to be an address that is aligned to an 8-byte boundary. */ #define FORCE_ALIGNMENT(X) (((X)+7)&~7) -/* -** Each in-memory image of a page begins with the following header. -** This header is only visible to this pager module. The client -** code that calls pager sees only the data that follows the header. -** -** Client code should call sqlite3PagerWrite() on a page prior to making -** any modifications to that page. The first time sqlite3PagerWrite() -** is called, the original page contents are written into the rollback -** journal and PgHdr.inJournal and PgHdr.needSync are set. Later, once -** the journal page has made it onto the disk surface, PgHdr.needSync -** is cleared. The modified page cannot be written back into the original -** database file until the journal pages has been synced to disk and the -** PgHdr.needSync has been cleared. -** -** The PgHdr.dirty flag is set when sqlite3PagerWrite() is called and -** is cleared again when the page content is written back to the original -** database file. -** -** Details of important structure elements: -** -** needSync -** -** If this is true, this means that it is not safe to write the page -** content to the database because the original content needed -** for rollback has not by synced to the main rollback journal. -** The original content may have been written to the rollback journal -** but it has not yet been synced. So we cannot write to the database -** file because power failure might cause the page in the journal file -** to never reach the disk. It is as if the write to the journal file -** does not occur until the journal file is synced. -** -** This flag is false if the page content exactly matches what -** currently exists in the database file. The needSync flag is also -** false if the original content has been written to the main rollback -** journal and synced. If the page represents a new page that has -** been added onto the end of the database during the current -** transaction, the needSync flag is true until the original database -** size in the journal header has been synced to disk. -** -** inJournal -** -** This is true if the original page has been written into the main -** rollback journal. This is always false for new pages added to -** the end of the database file during the current transaction. -** And this flag says nothing about whether or not the journal -** has been synced to disk. For pages that are in the original -** database file, the following expression should always be true: -** -** inJournal = (pPager->aInJournal[(pgno-1)/8] & (1<<((pgno-1)%8))!=0 -** -** The pPager->aInJournal[] array is only valid for the original -** pages of the database, not new pages that are added to the end -** of the database, so obviously the above expression cannot be -** valid for new pages. For new pages inJournal is always 0. -** -** dirty -** -** When true, this means that the content of the page has been -** modified and needs to be written back to the database file. -** If false, it means that either the content of the page is -** unchanged or else the content is unimportant and we do not -** care whether or not it is preserved. -** -** alwaysRollback -** -** This means that the sqlite3PagerDontRollback() API should be -** ignored for this page. The DontRollback() API attempts to say -** that the content of the page on disk is unimportant (it is an -** unused page on the freelist) so that it is unnecessary to -** rollback changes to this page because the content of the page -** can change without changing the meaning of the database. This -** flag overrides any DontRollback() attempt. This flag is set -** when a page that originally contained valid data is added to -** the freelist. Later in the same transaction, this page might -** be pulled from the freelist and reused for something different -** and at that point the DontRollback() API will be called because -** pages taken from the freelist do not need to be protected by -** the rollback journal. But this flag says that the page was -** not originally part of the freelist so that it still needs to -** be rolled back in spite of any subsequent DontRollback() calls. -** -** needRead -** -** This flag means (when true) that the content of the page has -** not yet been loaded from disk. The in-memory content is just -** garbage. (Actually, we zero the content, but you should not -** make any assumptions about the content nevertheless.) If the -** content is needed in the future, it should be read from the -** original database file. -*/ -typedef struct PgHdr PgHdr; -struct PgHdr { - Pager *pPager; /* The pager to which this page belongs */ - Pgno pgno; /* The page number for this page */ - PgHdr *pNextHash, *pPrevHash; /* Hash collision chain for PgHdr.pgno */ - PgHdr *pNextFree, *pPrevFree; /* Freelist of pages where nRef==0 */ - PgHdr *pNextAll; /* A list of all pages */ - u8 inJournal; /* TRUE if has been written to journal */ - u8 dirty; /* TRUE if we need to write back changes */ - u8 needSync; /* Sync journal before writing this page */ - u8 alwaysRollback; /* Disable DontRollback() for this page */ - u8 needRead; /* Read content if PagerWrite() is called */ - short int nRef; /* Number of users of this page */ - PgHdr *pDirty, *pPrevDirty; /* Dirty pages */ - u32 notUsed; /* Buffer space */ -#ifdef SQLITE_CHECK_PAGES - u32 pageHash; -#endif - /* pPager->pageSize bytes of page data follow this header */ - /* Pager.nExtra bytes of local data follow the page data */ -}; - -/* -** For an in-memory only database, some extra information is recorded about -** each page so that changes can be rolled back. (Journal files are not -** used for in-memory databases.) The following information is added to -** the end of every EXTRA block for in-memory databases. -** -** This information could have been added directly to the PgHdr structure. -** But then it would take up an extra 8 bytes of storage on every PgHdr -** even for disk-based databases. Splitting it out saves 8 bytes. This -** is only a savings of 0.8% but those percentages add up. -*/ -typedef struct PgHistory PgHistory; -struct PgHistory { - u8 *pOrig; /* Original page text. Restore to this on a full rollback */ - u8 *pStmt; /* Text as it was at the beginning of the current statement */ - PgHdr *pNextStmt, *pPrevStmt; /* List of pages in the statement journal */ - u8 inStmt; /* TRUE if in the statement subjournal */ -}; - /* ** A macro used for invoking the codec if there is one */ @@ -276,14 +117,34 @@ struct PgHistory { #endif /* -** Convert a pointer to a PgHdr into a pointer to its data -** and back again. +** The maximum allowed sector size. 16MB. If the xSectorsize() method +** returns a value larger than this, then MAX_SECTOR_SIZE is used instead. +** This could conceivably cause corruption following a power failure on +** such a system. This is currently an undocumented limit. */ -#define PGHDR_TO_DATA(P) ((void*)(&(P)[1])) -#define DATA_TO_PGHDR(D) (&((PgHdr*)(D))[-1]) -#define PGHDR_TO_EXTRA(G,P) ((void*)&((char*)(&(G)[1]))[(P)->pageSize]) -#define PGHDR_TO_HIST(P,PGR) \ - ((PgHistory*)&((char*)(&(P)[1]))[(PGR)->pageSize+(PGR)->nExtra]) +#define MAX_SECTOR_SIZE 0x0100000 + +/* +** An instance of the following structure is allocated for each active +** savepoint and statement transaction in the system. All such structures +** are stored in the Pager.aSavepoint[] array, which is allocated and +** resized using sqlite3Realloc(). +** +** When a savepoint is created, the PagerSavepoint.iHdrOffset field is +** set to 0. If a journal-header is written into the main journal while +** the savepoint is active, then iHdrOffset is set to the byte offset +** immediately following the last journal record written into the main +** journal before the journal-header. This is required during savepoint +** rollback (see pagerPlaybackSavepoint()). +*/ +typedef struct PagerSavepoint PagerSavepoint; +struct PagerSavepoint { + i64 iOffset; /* Starting offset in main journal */ + i64 iHdrOffset; /* See above */ + Bitvec *pInSavepoint; /* Set of pages in this savepoint */ + Pgno nOrig; /* Original number of pages in file */ + Pgno iSubRec; /* Index of first record in sub-journal */ +}; /* ** A open page cache is an instance of the following structure. @@ -295,78 +156,81 @@ struct PgHistory { ** next successful rollback is performed on the pager cache. Also, ** SQLITE_FULL does not affect the sqlite3PagerGet() and sqlite3PagerLookup() ** APIs, they may still be used successfully. +** +** Managing the size of the database file in pages is a little complicated. +** The variable Pager.dbSize contains the number of pages that the database +** image currently contains. As the database image grows or shrinks this +** variable is updated. The variable Pager.dbFileSize contains the number +** of pages in the database file. This may be different from Pager.dbSize +** if some pages have been appended to the database image but not yet written +** out from the cache to the actual file on disk. Or if the image has been +** truncated by an incremental-vacuum operation. The Pager.dbOrigSize variable +** contains the number of pages in the database image when the current +** transaction was opened. The contents of all three of these variables is +** only guaranteed to be correct if the boolean Pager.dbSizeValid is true. */ struct Pager { + sqlite3_vfs *pVfs; /* OS functions to use for IO */ u8 journalOpen; /* True if journal file descriptors is valid */ u8 journalStarted; /* True if header of journal is synced */ u8 useJournal; /* Use a rollback journal on this file */ u8 noReadlock; /* Do not bother to obtain readlocks */ - u8 stmtOpen; /* True if the statement subjournal is open */ - u8 stmtInUse; /* True we are in a statement subtransaction */ - u8 stmtAutoopen; /* Open stmt journal when main journal is opened*/ u8 noSync; /* Do not sync the journal if true */ u8 fullSync; /* Do extra syncs of the journal for robustness */ - u8 full_fsync; /* Use F_FULLFSYNC when available */ + u8 sync_flags; /* One of SYNC_NORMAL or SYNC_FULL */ u8 state; /* PAGER_UNLOCK, _SHARED, _RESERVED, etc. */ u8 tempFile; /* zFilename is a temporary file */ u8 readOnly; /* True for a read-only database */ u8 needSync; /* True if an fsync() is needed on the journal */ u8 dirtyCache; /* True if cached pages have changed */ - u8 alwaysRollback; /* Disable DontRollback() for all pages */ u8 memDb; /* True to inhibit all file I/O */ u8 setMaster; /* True if a m-j name has been written to jrnl */ u8 doNotSync; /* Boolean. While true, do not spill the cache */ u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ + u8 journalMode; /* On of the PAGER_JOURNALMODE_* values */ + u8 dbModified; /* True if there are any changes to the Db */ u8 changeCountDone; /* Set after incrementing the change-counter */ + u8 dbSizeValid; /* Set when dbSize is correct */ + Pgno dbSize; /* Number of pages in the database */ + Pgno dbOrigSize; /* dbSize before the current transaction */ + Pgno dbFileSize; /* Number of pages in the database file */ + u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */ int errCode; /* One of several kinds of errors */ - int dbSize; /* Number of pages in the file */ - int origDbSize; /* dbSize before the current change */ - int stmtSize; /* Size of database (in pages) at stmt_begin() */ int nRec; /* Number of pages written to the journal */ u32 cksumInit; /* Quasi-random value added to every checksum */ int stmtNRec; /* Number of records in stmt subjournal */ int nExtra; /* Add this many bytes to each in-memory page */ int pageSize; /* Number of bytes in a page */ int nPage; /* Total number of in-memory pages */ - int nRef; /* Number of in-memory pages with PgHdr.nRef>0 */ int mxPage; /* Maximum number of pages to hold in cache */ Pgno mxPgno; /* Maximum allowed size of the database */ - u8 *aInJournal; /* One bit for each page in the database file */ - u8 *aInStmt; /* One bit for each page in the database */ + Bitvec *pInJournal; /* One bit for each page in the database file */ + Bitvec *pAlwaysRollback; /* One bit for each page marked always-rollback */ char *zFilename; /* Name of the database file */ char *zJournal; /* Name of the journal file */ char *zDirectory; /* Directory hold database and journal files */ - OsFile *fd, *jfd; /* File descriptors for database and journal */ - OsFile *stfd; /* File descriptor for the statement subjournal*/ - BusyHandler *pBusyHandler; /* Pointer to sqlite.busyHandler */ - PgHdr *pFirst, *pLast; /* List of free pages */ - PgHdr *pFirstSynced; /* First free page with PgHdr.needSync==0 */ - PgHdr *pAll; /* List of all pages */ - PgHdr *pStmt; /* List of pages in the statement subjournal */ - PgHdr *pDirty; /* List of all dirty pages */ + sqlite3_file *fd, *jfd; /* File descriptors for database and journal */ + sqlite3_file *sjfd; /* File descriptor for the sub-journal*/ + int (*xBusyHandler)(void*); /* Function to call when busy */ + void *pBusyHandlerArg; /* Context argument for xBusyHandler */ i64 journalOff; /* Current byte offset in the journal file */ i64 journalHdr; /* Byte offset to previous journal header */ - i64 stmtHdrOff; /* First journal header written this statement */ - i64 stmtCksum; /* cksumInit when statement was started */ - i64 stmtJSize; /* Size of journal at stmt_begin() */ - int sectorSize; /* Assumed sector size during rollback */ + u32 sectorSize; /* Assumed sector size during rollback */ #ifdef SQLITE_TEST int nHit, nMiss; /* Cache hits and missing */ int nRead, nWrite; /* Database pages read/written */ #endif - void (*xDestructor)(DbPage*,int); /* Call this routine when freeing pages */ - void (*xReiniter)(DbPage*,int); /* Call this routine when reloading pages */ + void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ #ifdef SQLITE_HAS_CODEC void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */ void *pCodecArg; /* First argument to xCodec() */ -#endif - int nHash; /* Size of the pager hash table */ - PgHdr **aHash; /* Hash table to map page number to PgHdr */ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - Pager *pNext; /* Linked list of pagers in this thread */ #endif char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ char dbFileVers[16]; /* Changes whenever database file changes */ + i64 journalSizeLimit; /* Size limit for persistent journal files */ + PCache *pPCache; /* Pointer to page cache object */ + PagerSavepoint *aSavepoint; /* Array of active savepoints */ + int nSavepoint; /* Number of elements in aSavepoint[] */ }; /* @@ -378,7 +242,6 @@ struct Pager { int sqlite3_pager_readdb_count = 0; /* Number of full pages read from DB */ int sqlite3_pager_writedb_count = 0; /* Number of full pages written to DB */ int sqlite3_pager_writej_count = 0; /* Number of pages written to journal */ -int sqlite3_pager_pgfree_count = 0; /* Number of cache pages freed */ # define PAGER_INCR(v) v++ #else # define PAGER_INCR(v) @@ -391,7 +254,7 @@ int sqlite3_pager_pgfree_count = 0; /* Number of cache pages freed */ ** was obtained from /dev/random. It is used only as a sanity check. ** ** Since version 2.8.0, the journal format contains additional sanity -** checking information. If the power fails while the journal is begin +** checking information. If the power fails while the journal is being ** written, semi-random garbage data might appear in the journal ** file after power is restored. If an attempt is then made ** to roll the journal back, the database could be corrupted. The additional @@ -446,7 +309,7 @@ static const unsigned char aJournalMagic[] = { ** roll back. See comments for function writeMasterJournal() for details. */ /* #define PAGER_MJ_PGNO(x) (PENDING_BYTE/((x)->pageSize)) */ -#define PAGER_MJ_PGNO(x) ((PENDING_BYTE/((x)->pageSize))+1) +#define PAGER_MJ_PGNO(x) ((Pgno)((PENDING_BYTE/((x)->pageSize))+1)) /* ** The maximum legal page number is (2^31 - 1). @@ -454,69 +317,32 @@ static const unsigned char aJournalMagic[] = { #define PAGER_MAX_PGNO 2147483647 /* -** Enable reference count tracking (for debugging) here: +** Return true if it is necessary to write page *pPg into the sub-journal. +** A page needs to be written into the sub-journal if there exists one +** or more open savepoints for which: +** +** * The page-number is less than or equal to PagerSavepoint.nOrig, and +** * The bit corresponding to the page-number is not set in +** PagerSavepoint.pInSavepoint. */ -#ifdef SQLITE_DEBUG - int pager3_refinfo_enable = 0; - static void pager_refinfo(PgHdr *p){ - static int cnt = 0; - if( !pager3_refinfo_enable ) return; - sqlite3DebugPrintf( - "REFCNT: %4d addr=%p nRef=%-3d total=%d\n", - p->pgno, PGHDR_TO_DATA(p), p->nRef, p->pPager->nRef - ); - cnt++; /* Something to set a breakpoint on */ - } -# define REFINFO(X) pager_refinfo(X) -#else -# define REFINFO(X) -#endif - -/* -** Return true if page *pPg has already been written to the statement -** journal (or statement snapshot has been created, if *pPg is part -** of an in-memory database). -*/ -static int pageInStatement(PgHdr *pPg){ +static int subjRequiresPage(PgHdr *pPg){ + Pgno pgno = pPg->pgno; Pager *pPager = pPg->pPager; - if( MEMDB ){ - return PGHDR_TO_HIST(pPg, pPager)->inStmt; - }else{ - Pgno pgno = pPg->pgno; - u8 *a = pPager->aInStmt; - return (a && (int)pgno<=pPager->stmtSize && (a[pgno/8] & (1<<(pgno&7)))); + int i; + for(i=0; inSavepoint; i++){ + PagerSavepoint *p = &pPager->aSavepoint[i]; + if( p->nOrig>=pgno && 0==sqlite3BitvecTest(p->pInSavepoint, pgno) ){ + return 1; + } } + return 0; } /* -** Change the size of the pager hash table to N. N must be a power -** of two. +** Return true if the page is already in the journal file. */ -static void pager_resize_hash_table(Pager *pPager, int N){ - PgHdr **aHash, *pPg; - assert( N>0 && (N&(N-1))==0 ); - aHash = sqliteMalloc( sizeof(aHash[0])*N ); - if( aHash==0 ){ - /* Failure to rehash is not an error. It is only a performance hit. */ - return; - } - sqliteFree(pPager->aHash); - pPager->nHash = N; - pPager->aHash = aHash; - for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ - int h; - if( pPg->pgno==0 ){ - assert( pPg->pNextHash==0 && pPg->pPrevHash==0 ); - continue; - } - h = pPg->pgno & (N-1); - pPg->pNextHash = aHash[h]; - if( aHash[h] ){ - aHash[h]->pPrevHash = pPg; - } - aHash[h] = pPg; - pPg->pPrevHash = 0; - } +static int pageInJournal(PgHdr *pPg){ + return sqlite3BitvecTest(pPg->pPager->pInJournal, pPg->pgno); } /* @@ -526,9 +352,9 @@ static void pager_resize_hash_table(Pager *pPager, int N){ ** ** All values are stored on disk as big-endian. */ -static int read32bits(OsFile *fd, u32 *pRes){ +static int read32bits(sqlite3_file *fd, i64 offset, u32 *pRes){ unsigned char ac[4]; - int rc = sqlite3OsRead(fd, ac, sizeof(ac)); + int rc = sqlite3OsRead(fd, ac, sizeof(ac), offset); if( rc==SQLITE_OK ){ *pRes = sqlite3Get4byte(ac); } @@ -544,12 +370,59 @@ static int read32bits(OsFile *fd, u32 *pRes){ ** Write a 32-bit integer into the given file descriptor. Return SQLITE_OK ** on success or an error code is something goes wrong. */ -static int write32bits(OsFile *fd, u32 val){ +static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ char ac[4]; put32bits(ac, val); - return sqlite3OsWrite(fd, ac, 4); + return sqlite3OsWrite(fd, ac, 4, offset); } +/* +** If file pFd is open, call sqlite3OsUnlock() on it. +*/ +static int osUnlock(sqlite3_file *pFd, int eLock){ + if( !pFd->pMethods ){ + return SQLITE_OK; + } + return sqlite3OsUnlock(pFd, eLock); +} + +/* +** This function determines whether or not the atomic-write optimization +** can be used with this pager. The optimization can be used if: +** +** (a) the value returned by OsDeviceCharacteristics() indicates that +** a database page may be written atomically, and +** (b) the value returned by OsSectorSize() is less than or equal +** to the page size. +** +** If the optimization cannot be used, 0 is returned. If it can be used, +** then the value returned is the size of the journal file when it +** contains rollback data for exactly one page. +*/ +#ifdef SQLITE_ENABLE_ATOMIC_WRITE +static int jrnlBufferSize(Pager *pPager){ + int dc; /* Device characteristics */ + int nSector; /* Sector size */ + int szPage; /* Page size */ + sqlite3_file *fd = pPager->fd; + + if( fd->pMethods ){ + dc = sqlite3OsDeviceCharacteristics(fd); + nSector = pPager->sectorSize; + szPage = pPager->pageSize; + } + + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + + if( !fd->pMethods || + (dc & (SQLITE_IOCAP_ATOMIC|(szPage>>8)) && nSector<=szPage) ){ + return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); + } + return 0; +} +#endif + /* ** This function should be called when an error occurs within the pager ** code. The first argument is a pointer to the pager structure, the @@ -557,9 +430,17 @@ static int write32bits(OsFile *fd, u32 val){ ** The value returned is a copy of the second argument to this function. ** ** If the second argument is SQLITE_IOERR, SQLITE_CORRUPT, or SQLITE_FULL -** the error becomes persistent. All subsequent API calls on this Pager -** will immediately return the same error code. +** the error becomes persistent. Until the persisten error is cleared, +** subsequent API calls on this Pager will immediately return the same +** error code. +** +** A persistent error indicates that the contents of the pager-cache +** cannot be trusted. This state can be cleared by completely discarding +** the contents of the pager-cache. If a transaction was active when +** the persistent error occured, then the rollback journal may need +** to be replayed. */ +static void pager_unlock(Pager *pPager); static int pager_error(Pager *pPager, int rc){ int rc2 = rc & 0xff; assert( @@ -573,6 +454,15 @@ static int pager_error(Pager *pPager, int rc){ rc2==SQLITE_CORRUPT ){ pPager->errCode = rc; + if( pPager->state==PAGER_UNLOCK + && sqlite3PcacheRefCount(pPager->pPCache)==0 + ){ + /* If the pager is already unlocked, call pager_unlock() now to + ** clear the error state and ensure that the pager-cache is + ** completely empty. + */ + pager_unlock(pPager); + } } return rc; } @@ -595,8 +485,10 @@ static u32 pager_datahash(int nByte, unsigned char *pData){ return hash; } static u32 pager_pagehash(PgHdr *pPage){ - return pager_datahash(pPage->pPager->pageSize, - (unsigned char *)PGHDR_TO_DATA(pPage)); + return pager_datahash(pPage->pPager->pageSize, (unsigned char *)pPage->pData); +} +static void pager_set_pagehash(PgHdr *pPage){ + pPage->pageHash = pager_pagehash(pPage); } /* @@ -607,79 +499,74 @@ static u32 pager_pagehash(PgHdr *pPage){ #define CHECK_PAGE(x) checkPage(x) static void checkPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; - assert( !pPg->pageHash || pPager->errCode || MEMDB || pPg->dirty || - pPg->pageHash==pager_pagehash(pPg) ); + assert( !pPg->pageHash || pPager->errCode + || (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); } #else #define pager_datahash(X,Y) 0 #define pager_pagehash(X) 0 #define CHECK_PAGE(x) -#endif +#endif /* SQLITE_CHECK_PAGES */ /* ** When this is called the journal file for pager pPager must be open. ** The master journal file name is read from the end of the file and -** written into memory obtained from sqliteMalloc(). *pzMaster is -** set to point at the memory and SQLITE_OK returned. The caller must -** sqliteFree() *pzMaster. +** written into memory supplied by the caller. ** -** If no master journal file name is present *pzMaster is set to 0 and +** zMaster must point to a buffer of at least nMaster bytes allocated by +** the caller. This should be sqlite3_vfs.mxPathname+1 (to ensure there is +** enough space to write the master journal name). If the master journal +** name in the journal is longer than nMaster bytes (including a +** nul-terminator), then this is handled as if no master journal name +** were present in the journal. +** +** If no master journal file name is present zMaster[0] is set to 0 and ** SQLITE_OK returned. */ -static int readMasterJournal(OsFile *pJrnl, char **pzMaster){ +static int readMasterJournal(sqlite3_file *pJrnl, char *zMaster, u32 nMaster){ int rc; u32 len; i64 szJ; u32 cksum; - int i; + u32 u; /* Unsigned loop counter */ unsigned char aMagic[8]; /* A buffer to hold the magic header */ - *pzMaster = 0; + zMaster[0] = '\0'; rc = sqlite3OsFileSize(pJrnl, &szJ); if( rc!=SQLITE_OK || szJ<16 ) return rc; - rc = sqlite3OsSeek(pJrnl, szJ-16); - if( rc!=SQLITE_OK ) return rc; - - rc = read32bits(pJrnl, &len); + rc = read32bits(pJrnl, szJ-16, &len); if( rc!=SQLITE_OK ) return rc; - rc = read32bits(pJrnl, &cksum); + if( len>=nMaster ){ + return SQLITE_OK; + } + + rc = read32bits(pJrnl, szJ-12, &cksum); if( rc!=SQLITE_OK ) return rc; - rc = sqlite3OsRead(pJrnl, aMagic, 8); + rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8); if( rc!=SQLITE_OK || memcmp(aMagic, aJournalMagic, 8) ) return rc; - rc = sqlite3OsSeek(pJrnl, szJ-16-len); - if( rc!=SQLITE_OK ) return rc; - - *pzMaster = (char *)sqliteMalloc(len+1); - if( !*pzMaster ){ - return SQLITE_NOMEM; - } - rc = sqlite3OsRead(pJrnl, *pzMaster, len); + rc = sqlite3OsRead(pJrnl, zMaster, len, szJ-16-len); if( rc!=SQLITE_OK ){ - sqliteFree(*pzMaster); - *pzMaster = 0; return rc; } + zMaster[len] = '\0'; /* See if the checksum matches the master journal name */ - for(i=0; ijournalOff; if( c ){ @@ -709,8 +596,49 @@ static int seekJournalHdr(Pager *pPager){ assert( offset%JOURNAL_HDR_SZ(pPager)==0 ); assert( offset>=c ); assert( (offset-c)journalOff = offset; - return sqlite3OsSeek(pPager->jfd, pPager->journalOff); + return offset; +} +static void seekJournalHdr(Pager *pPager){ + pPager->journalOff = journalHdrOffset(pPager); +} + +/* +** Write zeros over the header of the journal file. This has the +** effect of invalidating the journal file and committing the +** transaction. +*/ +static int zeroJournalHdr(Pager *pPager, int doTruncate){ + int rc = SQLITE_OK; + static const char zeroHdr[28] = {0}; + + if( pPager->journalOff ){ + i64 iLimit = pPager->journalSizeLimit; + + IOTRACE(("JZEROHDR %p\n", pPager)) + if( doTruncate || iLimit==0 ){ + rc = sqlite3OsTruncate(pPager->jfd, 0); + }else{ + rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0); + } + if( rc==SQLITE_OK && !pPager->noSync ){ + rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->sync_flags); + } + + /* At this point the transaction is committed but the write lock + ** is still held on the file. If there is a size limit configured for + ** the persistent journal and the journal file currently consumes more + ** space than that limit allows for, truncate it now. There is no need + ** to sync the file following this operation. + */ + if( rc==SQLITE_OK && iLimit>0 ){ + i64 sz; + rc = sqlite3OsFileSize(pPager->jfd, &sz); + if( rc==SQLITE_OK && sz>iLimit ){ + rc = sqlite3OsTruncate(pPager->jfd, iLimit); + } + } + } + return rc; } /* @@ -724,62 +652,101 @@ static int seekJournalHdr(Pager *pPager){ ** - 4 bytes: Random number used for page hash. ** - 4 bytes: Initial database page count. ** - 4 bytes: Sector size used by the process that wrote this journal. +** - 4 bytes: Database page size. ** -** Followed by (JOURNAL_HDR_SZ - 24) bytes of unused space. +** Followed by (JOURNAL_HDR_SZ - 28) bytes of unused space. */ static int writeJournalHdr(Pager *pPager){ - char zHeader[sizeof(aJournalMagic)+16]; - int rc; + int rc = SQLITE_OK; + char *zHeader = pPager->pTmpSpace; + u32 nHeader = pPager->pageSize; + u32 nWrite; + int ii; - if( pPager->stmtHdrOff==0 ){ - pPager->stmtHdrOff = pPager->journalOff; + if( nHeader>JOURNAL_HDR_SZ(pPager) ){ + nHeader = JOURNAL_HDR_SZ(pPager); } - rc = seekJournalHdr(pPager); - if( rc ) return rc; - - pPager->journalHdr = pPager->journalOff; - pPager->journalOff += JOURNAL_HDR_SZ(pPager); - - /* FIX ME: - ** - ** Possibly for a pager not in no-sync mode, the journal magic should not - ** be written until nRec is filled in as part of next syncJournal(). - ** - ** Actually maybe the whole journal header should be delayed until that - ** point. Think about this. + /* If there are active savepoints and any of them were created since the + ** most recent journal header was written, update the PagerSavepoint.iHdrOff + ** fields now. */ - memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); - /* The nRec Field. 0xFFFFFFFF for no-sync journals. */ - put32bits(&zHeader[sizeof(aJournalMagic)], pPager->noSync ? 0xffffffff : 0); - /* The random check-hash initialiser */ - sqlite3Randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); - put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); - /* The initial database size */ - put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbSize); - /* The assumed sector size for this process */ - put32bits(&zHeader[sizeof(aJournalMagic)+12], pPager->sectorSize); - IOTRACE(("JHDR %p %lld %d\n", pPager, pPager->journalHdr, sizeof(zHeader))) - rc = sqlite3OsWrite(pPager->jfd, zHeader, sizeof(zHeader)); - - /* The journal header has been written successfully. Seek the journal - ** file descriptor to the end of the journal header sector. - */ - if( rc==SQLITE_OK ){ - IOTRACE(("JTAIL %p %lld\n", pPager, pPager->journalOff-1)) - rc = sqlite3OsSeek(pPager->jfd, pPager->journalOff-1); - if( rc==SQLITE_OK ){ - rc = sqlite3OsWrite(pPager->jfd, "\000", 1); + for(ii=0; iinSavepoint; ii++){ + if( pPager->aSavepoint[ii].iHdrOffset==0 ){ + pPager->aSavepoint[ii].iHdrOffset = pPager->journalOff; } } + + seekJournalHdr(pPager); + pPager->journalHdr = pPager->journalOff; + + memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); + + /* + ** Write the nRec Field - the number of page records that follow this + ** journal header. Normally, zero is written to this value at this time. + ** After the records are added to the journal (and the journal synced, + ** if in full-sync mode), the zero is overwritten with the true number + ** of records (see syncJournal()). + ** + ** A faster alternative is to write 0xFFFFFFFF to the nRec field. When + ** reading the journal this value tells SQLite to assume that the + ** rest of the journal file contains valid page records. This assumption + ** is dangerous, as if a failure occured whilst writing to the journal + ** file it may contain some garbage data. There are two scenarios + ** where this risk can be ignored: + ** + ** * When the pager is in no-sync mode. Corruption can follow a + ** power failure in this case anyway. + ** + ** * When the SQLITE_IOCAP_SAFE_APPEND flag is set. This guarantees + ** that garbage data is never appended to the journal file. + */ + assert(pPager->fd->pMethods||pPager->noSync); + if( (pPager->noSync) || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) + || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) + ){ + put32bits(&zHeader[sizeof(aJournalMagic)], 0xffffffff); + }else{ + put32bits(&zHeader[sizeof(aJournalMagic)], 0); + } + + /* The random check-hash initialiser */ + sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); + put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); + /* The initial database size */ + put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize); + /* The assumed sector size for this process */ + put32bits(&zHeader[sizeof(aJournalMagic)+12], pPager->sectorSize); + + /* Initializing the tail of the buffer is not necessary. Everything + ** works find if the following memset() is omitted. But initializing + ** the memory prevents valgrind from complaining, so we are willing to + ** take the performance hit. + */ + memset(&zHeader[sizeof(aJournalMagic)+16], 0, + nHeader-(sizeof(aJournalMagic)+16)); + + if( pPager->journalHdr==0 ){ + /* The page size */ + put32bits(&zHeader[sizeof(aJournalMagic)+16], pPager->pageSize); + } + + for(nWrite=0; rc==SQLITE_OK&&nWritejournalHdr, nHeader)) + rc = sqlite3OsWrite(pPager->jfd, zHeader, nHeader, pPager->journalOff); + pPager->journalOff += nHeader; + } + return rc; } /* ** The journal file must be open when this is called. A journal header file ** (JOURNAL_HDR_SZ bytes) is read from the current location in the journal -** file. See comments above function writeJournalHdr() for a description of -** the journal header format. +** file. The current location in the journal file is given by +** pPager->journalOff. See comments above function writeJournalHdr() for +** a description of the journal header format. ** ** If the header is read successfully, *nRec is set to the number of ** page records following this header and *dbSize is set to the size of the @@ -788,7 +755,7 @@ static int writeJournalHdr(Pager *pPager){ ** in this case. ** ** If the journal header file appears to be corrupted, SQLITE_DONE is -** returned and *nRec and *dbSize are not set. If JOURNAL_HDR_SZ bytes +** returned and *nRec and *dbSize are undefined. If JOURNAL_HDR_SZ bytes ** cannot be read from the journal file an error code is returned. */ static int readJournalHdr( @@ -799,42 +766,72 @@ static int readJournalHdr( ){ int rc; unsigned char aMagic[8]; /* A buffer to hold the magic header */ + i64 jrnlOff; + u32 iPageSize; + u32 iSectorSize; - rc = seekJournalHdr(pPager); - if( rc ) return rc; - + seekJournalHdr(pPager); if( pPager->journalOff+JOURNAL_HDR_SZ(pPager) > journalSize ){ return SQLITE_DONE; } + jrnlOff = pPager->journalOff; - rc = sqlite3OsRead(pPager->jfd, aMagic, sizeof(aMagic)); + rc = sqlite3OsRead(pPager->jfd, aMagic, sizeof(aMagic), jrnlOff); if( rc ) return rc; + jrnlOff += sizeof(aMagic); if( memcmp(aMagic, aJournalMagic, sizeof(aMagic))!=0 ){ return SQLITE_DONE; } - rc = read32bits(pPager->jfd, pNRec); + rc = read32bits(pPager->jfd, jrnlOff, pNRec); if( rc ) return rc; - rc = read32bits(pPager->jfd, &pPager->cksumInit); + rc = read32bits(pPager->jfd, jrnlOff+4, &pPager->cksumInit); if( rc ) return rc; - rc = read32bits(pPager->jfd, pDbSize); + rc = read32bits(pPager->jfd, jrnlOff+8, pDbSize); if( rc ) return rc; - /* Update the assumed sector-size to match the value used by - ** the process that created this journal. If this journal was - ** created by a process other than this one, then this routine - ** is being called from within pager_playback(). The local value - ** of Pager.sectorSize is restored at the end of that routine. - */ - rc = read32bits(pPager->jfd, (u32 *)&pPager->sectorSize); - if( rc ) return rc; + if( pPager->journalOff==0 ){ + rc = read32bits(pPager->jfd, jrnlOff+16, &iPageSize); + if( rc ) return rc; + + if( iPageSize<512 + || iPageSize>SQLITE_MAX_PAGE_SIZE + || ((iPageSize-1)&iPageSize)!=0 + ){ + /* If the page-size in the journal-header is invalid, then the process + ** that wrote the journal-header must have crashed before the header + ** was synced. In this case stop reading the journal file here. + */ + rc = SQLITE_DONE; + }else{ + u16 pagesize = (u16)iPageSize; + rc = sqlite3PagerSetPagesize(pPager, &pagesize); + assert( rc!=SQLITE_OK || pagesize==(u16)iPageSize ); + } + if( rc ) return rc; + + /* Update the assumed sector-size to match the value used by + ** the process that created this journal. If this journal was + ** created by a process other than this one, then this routine + ** is being called from within pager_playback(). The local value + ** of Pager.sectorSize is restored at the end of that routine. + */ + rc = read32bits(pPager->jfd, jrnlOff+12, &iSectorSize); + if( rc ) return rc; + if( (iSectorSize&(iSectorSize-1)) + || iSectorSize<512 + || iSectorSize>MAX_SECTOR_SIZE + ){ + return SQLITE_DONE; + } + pPager->sectorSize = iSectorSize; + } pPager->journalOff += JOURNAL_HDR_SZ(pPager); - rc = sqlite3OsSeek(pPager->jfd, pPager->journalOff); - return rc; + return SQLITE_OK; } @@ -861,13 +858,16 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ int rc; int len; int i; + i64 jrnlOff; + i64 jrnlSize; u32 cksum = 0; char zBuf[sizeof(aJournalMagic)+2*4]; - if( !zMaster || pPager->setMaster) return SQLITE_OK; + if( !zMaster || pPager->setMaster ) return SQLITE_OK; + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ) return SQLITE_OK; pPager->setMaster = 1; - len = strlen(zMaster); + len = sqlite3Strlen30(zMaster); for(i=0; ifullSync ){ - rc = seekJournalHdr(pPager); - if( rc!=SQLITE_OK ) return rc; + seekJournalHdr(pPager); } + jrnlOff = pPager->journalOff; pPager->journalOff += (len+20); - rc = write32bits(pPager->jfd, PAGER_MJ_PGNO(pPager)); + rc = write32bits(pPager->jfd, jrnlOff, PAGER_MJ_PGNO(pPager)); if( rc!=SQLITE_OK ) return rc; + jrnlOff += 4; - rc = sqlite3OsWrite(pPager->jfd, zMaster, len); + rc = sqlite3OsWrite(pPager->jfd, zMaster, len, jrnlOff); if( rc!=SQLITE_OK ) return rc; + jrnlOff += len; put32bits(zBuf, len); put32bits(&zBuf[4], cksum); memcpy(&zBuf[8], aJournalMagic, sizeof(aJournalMagic)); - rc = sqlite3OsWrite(pPager->jfd, zBuf, 8+sizeof(aJournalMagic)); + rc = sqlite3OsWrite(pPager->jfd, zBuf, 8+sizeof(aJournalMagic), jrnlOff); + jrnlOff += 8+sizeof(aJournalMagic); pPager->needSync = !pPager->noSync; - return rc; -} -/* -** Add or remove a page from the list of all pages that are in the -** statement journal. -** -** The Pager keeps a separate list of pages that are currently in -** the statement journal. This helps the sqlite3PagerStmtCommit() -** routine run MUCH faster for the common case where there are many -** pages in memory but only a few are in the statement journal. -*/ -static void page_add_to_stmt_list(PgHdr *pPg){ - Pager *pPager = pPg->pPager; - PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager); - assert( MEMDB ); - if( !pHist->inStmt ){ - assert( pHist->pPrevStmt==0 && pHist->pNextStmt==0 ); - if( pPager->pStmt ){ - PGHDR_TO_HIST(pPager->pStmt, pPager)->pPrevStmt = pPg; - } - pHist->pNextStmt = pPager->pStmt; - pPager->pStmt = pPg; - pHist->inStmt = 1; + /* If the pager is in peristent-journal mode, then the physical + ** journal-file may extend past the end of the master-journal name + ** and 8 bytes of magic data just written to the file. This is + ** dangerous because the code to rollback a hot-journal file + ** will not be able to find the master-journal name to determine + ** whether or not the journal is hot. + ** + ** Easiest thing to do in this scenario is to truncate the journal + ** file to the required size. + */ + if( (rc==SQLITE_OK) + && (rc = sqlite3OsFileSize(pPager->jfd, &jrnlSize))==SQLITE_OK + && jrnlSize>jrnlOff + ){ + rc = sqlite3OsTruncate(pPager->jfd, jrnlOff); } + return rc; } /* @@ -926,46 +922,10 @@ static void page_add_to_stmt_list(PgHdr *pPg){ */ static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ PgHdr *p; - if( pPager->aHash==0 ) return 0; - p = pPager->aHash[pgno & (pPager->nHash-1)]; - while( p && p->pgno!=pgno ){ - p = p->pNextHash; - } + sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &p); return p; } -/* -** Unlock the database file. -*/ -static void pager_unlock(Pager *pPager){ - if( !pPager->exclusiveMode ){ - if( !MEMDB ){ - sqlite3OsUnlock(pPager->fd, NO_LOCK); - pPager->dbSize = -1; - IOTRACE(("UNLOCK %p\n", pPager)) - } - pPager->state = PAGER_UNLOCK; - pPager->changeCountDone = 0; - } -} - -/* -** Execute a rollback if a transaction is active and unlock the -** database file. This is a no-op if the pager has already entered -** the error-state. -*/ -static void pagerUnlockAndRollback(Pager *p){ - if( p->errCode ) return; - assert( p->state>=PAGER_RESERVED || p->journalOpen==0 ); - if( p->state>=PAGER_RESERVED ){ - sqlite3PagerRollback(p); - } - pager_unlock(p); - assert( p->errCode || !p->journalOpen || (p->exclusiveMode&&!p->journalOff) ); - assert( p->errCode || !p->stmtOpen || p->exclusiveMode ); -} - - /* ** Clear the in-memory cache. This routine ** sets the state of the pager back to what it was when it was first @@ -973,24 +933,108 @@ static void pagerUnlockAndRollback(Pager *p){ ** to access those pages will likely result in a coredump. */ static void pager_reset(Pager *pPager){ - PgHdr *pPg, *pNext; if( pPager->errCode ) return; - for(pPg=pPager->pAll; pPg; pPg=pNext){ - IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno)); - PAGER_INCR(sqlite3_pager_pgfree_count); - pNext = pPg->pNextAll; - sqliteFree(pPg); + sqlite3PcacheClear(pPager->pPCache); +} + +/* +** Free all structures in the Pager.aSavepoint[] array and set both +** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal +** if it is open and the pager is not in exclusive mode. +*/ +static void releaseAllSavepoint(Pager *pPager){ + int ii; + for(ii=0; iinSavepoint; ii++){ + sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); } - pPager->pStmt = 0; - pPager->pFirst = 0; - pPager->pFirstSynced = 0; - pPager->pLast = 0; - pPager->pAll = 0; - pPager->nHash = 0; - sqliteFree(pPager->aHash); - pPager->nPage = 0; - pPager->aHash = 0; - pPager->nRef = 0; + if( !pPager->exclusiveMode ){ + sqlite3OsClose(pPager->sjfd); + } + sqlite3_free(pPager->aSavepoint); + pPager->aSavepoint = 0; + pPager->nSavepoint = 0; + pPager->stmtNRec = 0; +} + +/* +** Set the bit number pgno in the PagerSavepoint.pInSavepoint bitvecs of +** all open savepoints. +*/ +static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ + int ii; /* Loop counter */ + int rc = SQLITE_OK; /* Result code */ + + for(ii=0; iinSavepoint; ii++){ + PagerSavepoint *p = &pPager->aSavepoint[ii]; + if( pgno<=p->nOrig ){ + rc |= sqlite3BitvecSet(p->pInSavepoint, pgno); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + } + } + return rc; +} + +/* +** Unlock the database file. +** +** If the pager is currently in error state, discard the contents of +** the cache and reset the Pager structure internal state. If there is +** an open journal-file, then the next time a shared-lock is obtained +** on the pager file (by this or any other process), it will be +** treated as a hot-journal and rolled back. +*/ +static void pager_unlock(Pager *pPager){ + if( !pPager->exclusiveMode ){ + int rc; + + /* Always close the journal file when dropping the database lock. + ** Otherwise, another connection with journal_mode=delete might + ** delete the file out from under us. + */ + if( pPager->journalOpen ){ + sqlite3OsClose(pPager->jfd); + pPager->journalOpen = 0; + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + sqlite3BitvecDestroy(pPager->pAlwaysRollback); + pPager->pAlwaysRollback = 0; + } + + rc = osUnlock(pPager->fd, NO_LOCK); + if( rc ) pPager->errCode = rc; + pPager->dbSizeValid = 0; + IOTRACE(("UNLOCK %p\n", pPager)) + + /* If Pager.errCode is set, the contents of the pager cache cannot be + ** trusted. Now that the pager file is unlocked, the contents of the + ** cache can be discarded and the error code safely cleared. + */ + if( pPager->errCode ){ + if( rc==SQLITE_OK ) pPager->errCode = SQLITE_OK; + pager_reset(pPager); + releaseAllSavepoint(pPager); + pPager->journalOff = 0; + pPager->journalStarted = 0; + pPager->dbOrigSize = 0; + } + + pPager->state = PAGER_UNLOCK; + pPager->changeCountDone = 0; + } +} + +/* +** Execute a rollback if a transaction is active and unlock the +** database file. If the pager has already entered the error state, +** do not attempt the rollback. +*/ +static void pagerUnlockAndRollback(Pager *p){ + if( p->errCode==SQLITE_OK && p->state>=PAGER_RESERVED ){ + sqlite3BeginBenignMalloc(); + sqlite3PagerRollback(p); + sqlite3EndBenignMalloc(); + } + pager_unlock(p); } /* @@ -1010,62 +1054,70 @@ static void pager_reset(Pager *pPager){ ** This might give a performance improvement on windows where opening ** a file is an expensive operation. */ -static int pager_end_transaction(Pager *pPager){ - PgHdr *pPg; +static int pager_end_transaction(Pager *pPager, int hasMaster){ int rc = SQLITE_OK; int rc2 = SQLITE_OK; - assert( !MEMDB ); if( pPager->statestmtOpen && !pPager->exclusiveMode ){ - sqlite3OsClose(&pPager->stfd); - pPager->stmtOpen = 0; - } + releaseAllSavepoint(pPager); if( pPager->journalOpen ){ - if( pPager->exclusiveMode - && (rc = sqlite3OsTruncate(pPager->jfd, 0))==SQLITE_OK ){; - sqlite3OsSeek(pPager->jfd, 0); + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ + int isMemoryJournal = sqlite3IsMemJournal(pPager->jfd); + sqlite3OsClose(pPager->jfd); + pPager->journalOpen = 0; + if( !isMemoryJournal ){ + rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + } + }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE + && (rc = sqlite3OsTruncate(pPager->jfd, 0))==SQLITE_OK ){ + pPager->journalOff = 0; + pPager->journalStarted = 0; + }else if( pPager->exclusiveMode + || pPager->journalMode==PAGER_JOURNALMODE_PERSIST + ){ + rc = zeroJournalHdr(pPager, hasMaster); + pager_error(pPager, rc); pPager->journalOff = 0; pPager->journalStarted = 0; }else{ - sqlite3OsClose(&pPager->jfd); + assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE || rc ); + sqlite3OsClose(pPager->jfd); pPager->journalOpen = 0; - if( rc==SQLITE_OK ){ - rc = sqlite3OsDelete(pPager->zJournal); + if( rc==SQLITE_OK && !pPager->tempFile ){ + rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } } - sqliteFree( pPager->aInJournal ); - pPager->aInJournal = 0; - for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ - pPg->inJournal = 0; - pPg->dirty = 0; - pPg->needSync = 0; - pPg->alwaysRollback = 0; + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + sqlite3BitvecDestroy(pPager->pAlwaysRollback); + pPager->pAlwaysRollback = 0; #ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); + sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); #endif - } - pPager->pDirty = 0; + sqlite3PcacheCleanAll(pPager->pPCache); pPager->dirtyCache = 0; pPager->nRec = 0; }else{ - assert( pPager->aInJournal==0 ); - assert( pPager->dirtyCache==0 || pPager->useJournal==0 ); + assert( pPager->pInJournal==0 ); } if( !pPager->exclusiveMode ){ - rc2 = sqlite3OsUnlock(pPager->fd, SHARED_LOCK); + rc2 = osUnlock(pPager->fd, SHARED_LOCK); pPager->state = PAGER_SHARED; + pPager->changeCountDone = 0; }else if( pPager->state==PAGER_SYNCED ){ pPager->state = PAGER_EXCLUSIVE; } - pPager->origDbSize = 0; + pPager->dbOrigSize = 0; pPager->setMaster = 0; pPager->needSync = 0; - pPager->pFirstSynced = pPager->pFirst; - pPager->dbSize = -1; + /* lruListSetFirstSynced(pPager); */ + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); + if( !MEMDB ){ + pPager->dbSizeValid = 0; + } + pPager->dbModified = 0; return (rc==SQLITE_OK?rc2:rc); } @@ -1100,35 +1152,51 @@ static u32 pager_cksum(Pager *pPager, const u8 *aData){ return cksum; } -/* Forward declaration */ -static void makeClean(PgHdr*); - /* -** Read a single page from the journal file opened on file descriptor -** jfd. Playback this one page. +** Read a single page from either the journal file (if isMainJrnl==1) or +** from the sub-journal (if isMainJrnl==0) and playback that page. +** The page begins at offset *pOffset into the file. The *pOffset +** value is increased to the start of the next page in the journal. ** -** If useCksum==0 it means this journal does not use checksums. Checksums -** are not used in statement journals because statement journals do not -** need to survive power failures. +** The isMainJrnl flag is true if this is the main rollback journal and +** false for the statement journal. The main rollback journal uses +** checksums - the statement journal does not. +** +** If pDone is not NULL, then it is a record of pages that have already +** been played back. If the page at *pOffset has already been played back +** (if the corresponding pDone bit is set) then skip the playback. +** Make sure the pDone bit corresponding to the *pOffset page is set +** prior to returning. */ -static int pager_playback_one_page(Pager *pPager, OsFile *jfd, int useCksum){ +static int pager_playback_one_page( + Pager *pPager, /* The pager being played back */ + int isMainJrnl, /* 1 -> main journal. 0 -> sub-journal. */ + i64 *pOffset, /* Offset of record to playback */ + int isSavepnt, /* True for a savepoint rollback */ + Bitvec *pDone /* Bitvec of pages already played back */ +){ int rc; PgHdr *pPg; /* An existing page in the cache */ Pgno pgno; /* The page number of a page in journal */ u32 cksum; /* Checksum used for sanity checking */ - u8 *aData = (u8 *)pPager->pTmpSpace; /* Temp storage for a page */ + u8 *aData; /* Temporary storage for the page */ + sqlite3_file *jfd; /* The file descriptor for the journal file */ - /* useCksum should be true for the main journal and false for - ** statement journals. Verify that this is always the case - */ - assert( jfd == (useCksum ? pPager->jfd : pPager->stfd) ); - assert( aData ); + assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */ + assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */ + assert( isMainJrnl || pDone ); /* pDone always used on sub-journals */ + assert( isSavepnt || pDone==0 ); /* pDone never used on non-savepoint */ - rc = read32bits(jfd, &pgno); + aData = (u8*)pPager->pTmpSpace; + assert( aData ); /* Temp storage must have already been allocated */ + + jfd = isMainJrnl ? pPager->jfd : pPager->sjfd; + + rc = read32bits(jfd, *pOffset, &pgno); if( rc!=SQLITE_OK ) return rc; - rc = sqlite3OsRead(jfd, aData, pPager->pageSize); + rc = sqlite3OsRead(jfd, aData, pPager->pageSize, (*pOffset)+4); if( rc!=SQLITE_OK ) return rc; - pPager->journalOff += pPager->pageSize + 4; + *pOffset += pPager->pageSize + 4 + isMainJrnl*4; /* Sanity checking on the page. This is more important that I originally ** thought. If a power failure occurs while the journal is being written, @@ -1138,17 +1206,19 @@ static int pager_playback_one_page(Pager *pPager, OsFile *jfd, int useCksum){ if( pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){ return SQLITE_DONE; } - if( pgno>(unsigned)pPager->dbSize ){ + if( pgno>(Pgno)pPager->dbSize || sqlite3BitvecTest(pDone, pgno) ){ return SQLITE_OK; } - if( useCksum ){ - rc = read32bits(jfd, &cksum); + if( isMainJrnl ){ + rc = read32bits(jfd, (*pOffset)-4, &cksum); if( rc ) return rc; - pPager->journalOff += 4; - if( pager_cksum(pPager, aData)!=cksum ){ + if( !isSavepnt && pager_cksum(pPager, aData)!=cksum ){ return SQLITE_DONE; } } + if( pDone && (rc = sqlite3BitvecSet(pDone, pgno)) ){ + return rc; + } assert( pPager->state==PAGER_RESERVED || pPager->state>=PAGER_EXCLUSIVE ); @@ -1179,18 +1249,48 @@ static int pager_playback_one_page(Pager *pPager, OsFile *jfd, int useCksum){ ** locked. (2) we know that the original page content is fully synced ** in the main journal either because the page is not in cache or else ** the page is marked as needSync==0. + ** + ** 2008-04-14: When attempting to vacuum a corrupt database file, it + ** is possible to fail a statement on a database that does not yet exist. + ** Do not attempt to write if database file has never been opened. */ pPg = pager_lookup(pPager, pgno); - PAGERTRACE4("PLAYBACK %d page %d hash(%08x)\n", - PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, aData)); - if( pPager->state>=PAGER_EXCLUSIVE && (pPg==0 || pPg->needSync==0) ){ - rc = sqlite3OsSeek(pPager->fd, (pgno-1)*(i64)pPager->pageSize); - if( rc==SQLITE_OK ){ - rc = sqlite3OsWrite(pPager->fd, aData, pPager->pageSize); + PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", + PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, aData), + (isMainJrnl?"main-journal":"sub-journal") + )); + if( (pPager->state>=PAGER_EXCLUSIVE) + && (pPg==0 || 0==(pPg->flags&PGHDR_NEED_SYNC)) + && (pPager->fd->pMethods) + ){ + i64 ofst = (pgno-1)*(i64)pPager->pageSize; + rc = sqlite3OsWrite(pPager->fd, aData, pPager->pageSize, ofst); + if( pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pgno; } - if( pPg ){ - makeClean(pPg); + }else if( !isMainJrnl && pPg==0 ){ + /* If this is a rollback of a savepoint and data was not written to + ** the database and the page is not in-memory, there is a potential + ** problem. When the page is next fetched by the b-tree layer, it + ** will be read from the database file, which may or may not be + ** current. + ** + ** There are a couple of different ways this can happen. All are quite + ** obscure. When running in synchronous mode, this can only happen + ** if the page is on the free-list at the start of the transaction, then + ** populated, then moved using sqlite3PagerMovepage(). + ** + ** The solution is to add an in-memory page to the cache containing + ** the data just read from the sub-journal. Mark the page as dirty + ** and if the pager requires a journal-sync, then mark the page as + ** requiring a journal-sync before it is written. + */ + assert( isSavepnt ); + if( (rc = sqlite3PagerAcquire(pPager, pgno, &pPg, 1)) ){ + return rc; } + pPg->flags &= ~PGHDR_NEED_READ; + sqlite3PcacheMakeDirty(pPg); } if( pPg ){ /* No page should ever be explicitly rolled back that is in use, except @@ -1200,11 +1300,31 @@ static int pager_playback_one_page(Pager *pPager, OsFile *jfd, int useCksum){ ** sqlite3PagerRollback(). */ void *pData; - /* assert( pPg->nRef==0 || pPg->pgno==1 ); */ - pData = PGHDR_TO_DATA(pPg); + pData = pPg->pData; memcpy(pData, aData, pPager->pageSize); if( pPager->xReiniter ){ - pPager->xReiniter(pPg, pPager->pageSize); + pPager->xReiniter(pPg); + } + if( isMainJrnl && (!isSavepnt || pPager->journalOff<=pPager->journalHdr) ){ + /* If the contents of this page were just restored from the main + ** journal file, then its content must be as they were when the + ** transaction was first opened. In this case we can mark the page + ** as clean, since there will be no need to write it out to the. + ** + ** There is one exception to this rule. If the page is being rolled + ** back as part of a savepoint (or statement) rollback from an + ** unsynced portion of the main journal file, then it is not safe + ** to mark the page as clean. This is because marking the page as + ** clean will clear the PGHDR_NEED_SYNC flag. Since the page is + ** already in the journal file (recorded in Pager.pInJournal) and + ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to + ** again within this transaction, it will be marked as dirty but + ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially + ** be written out into the database file before its journal file + ** segment is synced. If a crash occurs during or following this, + ** database corruption may ensue. + */ + sqlite3PcacheMakeClean(pPg); } #ifdef SQLITE_CHECK_PAGES pPg->pageHash = pager_pagehash(pPg); @@ -1217,113 +1337,188 @@ static int pager_playback_one_page(Pager *pPager, OsFile *jfd, int useCksum){ /* Decode the page just read from disk */ CODEC1(pPager, pData, pPg->pgno, 3); + sqlite3PcacheRelease(pPg); } return rc; } +#if !defined(NDEBUG) || defined(SQLITE_COVERAGE_TEST) +/* +** This routine looks ahead into the main journal file and determines +** whether or not the next record (the record that begins at file +** offset pPager->journalOff) is a well-formed page record consisting +** of a valid page number, pPage->pageSize bytes of content, followed +** by a valid checksum. +** +** The pager never needs to know this in order to do its job. This +** routine is only used from with assert() and testcase() macros. +*/ +static int pagerNextJournalPageIsValid(Pager *pPager){ + Pgno pgno; /* The page number of the page */ + u32 cksum; /* The page checksum */ + int rc; /* Return code from read operations */ + sqlite3_file *fd; /* The file descriptor from which we are reading */ + u8 *aData; /* Content of the page */ + + /* Read the page number header */ + fd = pPager->jfd; + rc = read32bits(fd, pPager->journalOff, &pgno); + if( rc!=SQLITE_OK ){ return 0; } /*NO_TEST*/ + if( pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){ return 0; } /*NO_TEST*/ + if( pgno>(Pgno)pPager->dbSize ){ return 0; } /*NO_TEST*/ + + /* Read the checksum */ + rc = read32bits(fd, pPager->journalOff+pPager->pageSize+4, &cksum); + if( rc!=SQLITE_OK ){ return 0; } /*NO_TEST*/ + + /* Read the data and verify the checksum */ + aData = (u8*)pPager->pTmpSpace; + rc = sqlite3OsRead(fd, aData, pPager->pageSize, pPager->journalOff+4); + if( rc!=SQLITE_OK ){ return 0; } /*NO_TEST*/ + if( pager_cksum(pPager, aData)!=cksum ){ return 0; } /*NO_TEST*/ + + /* Reach this point only if the page is valid */ + return 1; +} +#endif /* !defined(NDEBUG) || defined(SQLITE_COVERAGE_TEST) */ + /* ** Parameter zMaster is the name of a master journal file. A single journal ** file that referred to the master journal file has just been rolled back. ** This routine checks if it is possible to delete the master journal file, ** and does so if it is. ** +** Argument zMaster may point to Pager.pTmpSpace. So that buffer is not +** available for use within this function. +** +** ** The master journal file contains the names of all child journals. ** To tell if a master journal can be deleted, check to each of the ** children. If all children are either missing or do not refer to ** a different master journal, then this master journal can be deleted. */ -static int pager_delmaster(const char *zMaster){ +static int pager_delmaster(Pager *pPager, const char *zMaster){ + sqlite3_vfs *pVfs = pPager->pVfs; int rc; int master_open = 0; - OsFile *master = 0; + sqlite3_file *pMaster; + sqlite3_file *pJournal; char *zMasterJournal = 0; /* Contents of master journal file */ i64 nMasterJournal; /* Size of master journal file */ /* Open the master journal file exclusively in case some other process ** is running this routine also. Not that it makes too much difference. */ - rc = sqlite3OsOpenReadOnly(zMaster, &master); - assert( rc!=SQLITE_OK || master ); + pMaster = (sqlite3_file *)sqlite3Malloc(pVfs->szOsFile * 2); + pJournal = (sqlite3_file *)(((u8 *)pMaster) + pVfs->szOsFile); + if( !pMaster ){ + rc = SQLITE_NOMEM; + }else{ + int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); + rc = sqlite3OsOpen(pVfs, zMaster, pMaster, flags, 0); + } if( rc!=SQLITE_OK ) goto delmaster_out; master_open = 1; - rc = sqlite3OsFileSize(master, &nMasterJournal); + + rc = sqlite3OsFileSize(pMaster, &nMasterJournal); if( rc!=SQLITE_OK ) goto delmaster_out; if( nMasterJournal>0 ){ char *zJournal; char *zMasterPtr = 0; + int nMasterPtr = pPager->pVfs->mxPathname+1; /* Load the entire master journal file into space obtained from - ** sqliteMalloc() and pointed to by zMasterJournal. + ** sqlite3_malloc() and pointed to by zMasterJournal. */ - zMasterJournal = (char *)sqliteMalloc(nMasterJournal); + zMasterJournal = (char *)sqlite3Malloc((int)nMasterJournal + nMasterPtr); if( !zMasterJournal ){ rc = SQLITE_NOMEM; goto delmaster_out; } - rc = sqlite3OsRead(master, zMasterJournal, nMasterJournal); + zMasterPtr = &zMasterJournal[nMasterJournal]; + rc = sqlite3OsRead(pMaster, zMasterJournal, (int)nMasterJournal, 0); if( rc!=SQLITE_OK ) goto delmaster_out; zJournal = zMasterJournal; while( (zJournal-zMasterJournal)state>=PAGER_EXCLUSIVE ){ - rc = sqlite3OsTruncate(pPager->fd, pPager->pageSize*(i64)nPage); - } - if( rc==SQLITE_OK ){ - pPager->dbSize = nPage; - pager_truncate_cache(pPager); + if( pPager->state>=PAGER_EXCLUSIVE && pPager->fd->pMethods ){ + i64 currentSize, newSize; + rc = sqlite3OsFileSize(pPager->fd, ¤tSize); + newSize = pPager->pageSize*(i64)nPage; + if( rc==SQLITE_OK && currentSize!=newSize ){ + if( currentSize>newSize ){ + rc = sqlite3OsTruncate(pPager->fd, newSize); + }else{ + rc = sqlite3OsWrite(pPager->fd, "", 1, newSize-1); + } + if( rc==SQLITE_OK ){ + pPager->dbFileSize = nPage; + } + } } return rc; } @@ -1331,13 +1526,23 @@ static int pager_truncate(Pager *pPager, int nPage){ /* ** Set the sectorSize for the given pager. ** -** The sector size is the larger of the sector size reported -** by sqlite3OsSectorSize() and the pageSize. +** The sector size is at least as big as the sector size reported +** by sqlite3OsSectorSize(). The minimum sector size is 512. */ static void setSectorSize(Pager *pPager){ - pPager->sectorSize = sqlite3OsSectorSize(pPager->fd); - if( pPager->sectorSizepageSize ){ - pPager->sectorSize = pPager->pageSize; + assert(pPager->fd->pMethods||pPager->tempFile); + if( !pPager->tempFile ){ + /* Sector size doesn't matter for temporary files. Also, the file + ** may not have been opened yet, in whcih case the OsSectorSize() + ** call will segfault. + */ + pPager->sectorSize = sqlite3OsSectorSize(pPager->fd); + } + if( pPager->sectorSize<512 ){ + pPager->sectorSize = 512; + } + if( pPager->sectorSize>MAX_SECTOR_SIZE ){ + pPager->sectorSize = MAX_SECTOR_SIZE; } } @@ -1355,20 +1560,23 @@ static void setSectorSize(Pager *pPager){ ** sanity checksum. ** (4) 4 byte integer which is the number of pages to truncate the ** database to during a rollback. -** (5) 4 byte integer which is the number of bytes in the master journal +** (5) 4 byte big-endian integer which is the sector size. The header +** is this many bytes in size. +** (6) 4 byte big-endian integer which is the page case. +** (7) 4 byte integer which is the number of bytes in the master journal ** name. The value may be zero (indicate that there is no master ** journal.) -** (6) N bytes of the master journal name. The name will be nul-terminated +** (8) N bytes of the master journal name. The name will be nul-terminated ** and might be shorter than the value read from (5). If the first byte ** of the name is \000 then there is no master journal. The master ** journal name is stored in UTF-8. -** (7) Zero or more pages instances, each as follows: +** (9) Zero or more pages instances, each as follows: ** + 4 byte page number. ** + pPager->pageSize bytes of data. ** + 4 byte checksum ** -** When we speak of the journal header, we mean the first 6 items above. -** Each entry in the journal is an instance of the 7th item. +** When we speak of the journal header, we mean the first 8 items above. +** Each entry in the journal is an instance of the 9th item. ** ** Call the value from the second bullet "nRec". nRec is the number of ** valid page entries in the journal. In most cases, you can compute the @@ -1395,11 +1603,13 @@ static void setSectorSize(Pager *pPager){ ** and an error code is returned. */ static int pager_playback(Pager *pPager, int isHot){ + sqlite3_vfs *pVfs = pPager->pVfs; i64 szJ; /* Size of the journal file in bytes */ u32 nRec; /* Number of Records in the journal */ - int i; /* Loop counter */ + u32 u; /* Unsigned loop counter */ Pgno mxPg = 0; /* Size of the original file in pages */ int rc; /* Result code of a subroutine */ + int res = 1; /* Value returned by sqlite3OsAccess() */ char *zMaster = 0; /* Name of master journal file if any */ /* Figure out how many records are in the journal. Abort early if @@ -1416,15 +1626,15 @@ static int pager_playback(Pager *pPager, int isHot){ ** present on disk, then the journal is not hot and does not need to be ** played back. */ - rc = readMasterJournal(pPager->jfd, &zMaster); - assert( rc!=SQLITE_DONE ); - if( rc!=SQLITE_OK || (zMaster && !sqlite3OsFileExists(zMaster)) ){ - sqliteFree(zMaster); - zMaster = 0; - if( rc==SQLITE_DONE ) rc = SQLITE_OK; + zMaster = pPager->pTmpSpace; + rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); + if( rc==SQLITE_OK && zMaster[0] ){ + rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); + } + zMaster = 0; + if( rc!=SQLITE_OK || !res ){ goto end_playback; } - sqlite3OsSeek(pPager->jfd, 0); pPager->journalOff = 0; /* This loop terminates either when the readJournalHdr() call returns @@ -1451,7 +1661,7 @@ static int pager_playback(Pager *pPager, int isHot){ */ if( nRec==0xffffffff ){ assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ); - nRec = (szJ - JOURNAL_HDR_SZ(pPager))/JOURNAL_PG_SZ(pPager); + nRec = (int)((szJ - JOURNAL_HDR_SZ(pPager))/JOURNAL_PG_SZ(pPager)); } /* If nRec is 0 and this rollback is of a transaction created by this @@ -1461,32 +1671,49 @@ static int pager_playback(Pager *pPager, int isHot){ ** size of the file. ** ** The third term of the test was added to fix ticket #2565. + ** When rolling back a hot journal, nRec==0 always means that the next + ** chunk of the journal contains zero pages to be rolled back. But + ** when doing a ROLLBACK and the nRec==0 chunk is the last chunk in + ** the journal, it means that the journal might contain additional + ** pages that need to be rolled back and that the number of pages + ** should be computed based on the journal file size. */ + testcase( nRec==0 && !isHot + && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)!=pPager->journalOff + && ((szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager))>0 + && pagerNextJournalPageIsValid(pPager) + ); if( nRec==0 && !isHot && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff ){ - nRec = (szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager); + nRec = (int)((szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager)); } /* If this is the first header read from the journal, truncate the - ** database file back to it's original size. + ** database file back to its original size. */ if( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ){ rc = pager_truncate(pPager, mxPg); if( rc!=SQLITE_OK ){ goto end_playback; } + pPager->dbSize = mxPg; } /* Copy original pages out of the journal and back into the database file. */ - for(i=0; ijfd, 1); + for(u=0; ujournalOff, 0, 0); if( rc!=SQLITE_OK ){ if( rc==SQLITE_DONE ){ rc = SQLITE_OK; pPager->journalOff = szJ; break; }else{ + /* If we are unable to rollback, then the database is probably + ** going to end up being corrupt. It is corrupt to us, anyhow. + ** Perhaps the next process to come along can fix it.... + */ + rc = SQLITE_CORRUPT_BKPT; goto end_playback; } } @@ -1496,17 +1723,28 @@ static int pager_playback(Pager *pPager, int isHot){ assert( 0 ); end_playback: + /* Following a rollback, the database file should be back in its original + ** state prior to the start of the transaction, so invoke the + ** SQLITE_FCNTL_DB_UNCHANGED file-control method to disable the + ** assertion that the transaction counter was modified. + */ + assert( + pPager->fd->pMethods==0 || + sqlite3OsFileControl(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0)>=SQLITE_OK + ); + if( rc==SQLITE_OK ){ - rc = pager_end_transaction(pPager); + zMaster = pPager->pTmpSpace; + rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); } - if( zMaster ){ + if( rc==SQLITE_OK ){ + rc = pager_end_transaction(pPager, zMaster[0]!='\0'); + } + if( rc==SQLITE_OK && zMaster[0] && res ){ /* If there was a master journal and this routine will return success, ** see if it is possible to delete the master journal. */ - if( rc==SQLITE_OK ){ - rc = pager_delmaster(zMaster); - } - sqliteFree(zMaster); + rc = pager_delmaster(pPager, zMaster); } /* The Pager.sectorSize variable may have been updated while rolling @@ -1518,113 +1756,107 @@ end_playback: } /* -** Playback the statement journal. +** Playback savepoint pSavepoint. Or, if pSavepoint==NULL, then playback +** the entire master journal file. ** -** This is similar to playing back the transaction journal but with -** a few extra twists. -** -** (1) The number of pages in the database file at the start of -** the statement is stored in pPager->stmtSize, not in the -** journal file itself. -** -** (2) In addition to playing back the statement journal, also -** playback all pages of the transaction journal beginning -** at offset pPager->stmtJSize. +** The case pSavepoint==NULL occurs when a ROLLBACK TO command is invoked +** on a SAVEPOINT that is a transaction savepoint. */ -static int pager_stmt_playback(Pager *pPager){ - i64 szJ; /* Size of the full journal */ - i64 hdrOff; - int nRec; /* Number of Records */ - int i; /* Loop counter */ - int rc; +static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ + i64 szJ; /* Effective size of the main journal */ + i64 iHdrOff; /* End of first segment of main-journal records */ + Pgno ii; /* Loop counter */ + int rc = SQLITE_OK; /* Return code */ + Bitvec *pDone = 0; /* Bitvec to ensure pages played back only once */ - szJ = pPager->journalOff; -#ifndef NDEBUG - { - i64 os_szJ; - rc = sqlite3OsFileSize(pPager->jfd, &os_szJ); - if( rc!=SQLITE_OK ) return rc; - assert( szJ==os_szJ ); + /* Allocate a bitvec to use to store the set of pages rolled back */ + if( pSavepoint ){ + pDone = sqlite3BitvecCreate(pSavepoint->nOrig); + if( !pDone ){ + return SQLITE_NOMEM; + } } -#endif - /* Set hdrOff to be the offset just after the end of the last journal - ** page written before the first journal-header for this statement - ** transaction was written, or the end of the file if no journal - ** header was written. + /* Truncate the database back to the size it was before the + ** savepoint being reverted was opened. */ - hdrOff = pPager->stmtHdrOff; - assert( pPager->fullSync || !hdrOff ); - if( !hdrOff ){ - hdrOff = szJ; - } - - /* Truncate the database back to its original size. - */ - rc = pager_truncate(pPager, pPager->stmtSize); + pPager->dbSize = pSavepoint ? pSavepoint->nOrig : pPager->dbOrigSize; assert( pPager->state>=PAGER_SHARED ); - /* Figure out how many records are in the statement journal. + /* Use pPager->journalOff as the effective size of the main rollback + ** journal. The actual file might be larger than this in + ** PAGER_JOURNALMODE_TRUNCATE or PAGER_JOURNALMODE_PERSIST. But anything + ** past pPager->journalOff is off-limits to us. */ - assert( pPager->stmtInUse && pPager->journalOpen ); - sqlite3OsSeek(pPager->stfd, 0); - nRec = pPager->stmtNRec; - - /* Copy original pages out of the statement journal and back into the - ** database file. Note that the statement journal omits checksums from - ** each record since power-failure recovery is not important to statement - ** journals. + szJ = pPager->journalOff; + + /* Begin by rolling back records from the main journal starting at + ** PagerSavepoint.iOffset and continuing to the next journal header. + ** There might be records in the main journal that have a page number + ** greater than the current database size (pPager->dbSize) but those + ** will be skipped automatically. Pages are added to pDone as they + ** are played back. */ - for(i=nRec-1; i>=0; i--){ - rc = pager_playback_one_page(pPager, pPager->stfd, 0); - assert( rc!=SQLITE_DONE ); - if( rc!=SQLITE_OK ) goto end_stmt_playback; + if( pSavepoint ){ + iHdrOff = pSavepoint->iHdrOffset ? pSavepoint->iHdrOffset : szJ; + pPager->journalOff = pSavepoint->iOffset; + while( rc==SQLITE_OK && pPager->journalOffjournalOff, 1, pDone); + assert( rc!=SQLITE_DONE ); + } + }else{ + pPager->journalOff = 0; } - /* Now roll some pages back from the transaction journal. Pager.stmtJSize - ** was the size of the journal file when this statement was started, so - ** everything after that needs to be rolled back, either into the - ** database, the memory cache, or both. - ** - ** If it is not zero, then Pager.stmtHdrOff is the offset to the start - ** of the first journal header written during this statement transaction. + /* Continue rolling back records out of the main journal starting at + ** the first journal header seen and continuing until the effective end + ** of the main journal file. Continue to skip out-of-range pages and + ** continue adding pages rolled back to pDone. */ - rc = sqlite3OsSeek(pPager->jfd, pPager->stmtJSize); - if( rc!=SQLITE_OK ){ - goto end_stmt_playback; - } - pPager->journalOff = pPager->stmtJSize; - pPager->cksumInit = pPager->stmtCksum; - while( pPager->journalOff < hdrOff ){ - rc = pager_playback_one_page(pPager, pPager->jfd, 1); - assert( rc!=SQLITE_DONE ); - if( rc!=SQLITE_OK ) goto end_stmt_playback; - } - - while( pPager->journalOff < szJ ){ - u32 nJRec; /* Number of Journal Records */ + while( rc==SQLITE_OK && pPager->journalOffjournalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff" + ** test is related to ticket #2565. See the discussion in the + ** pager_playback() function for additional information. + */ + assert( !(nJRec==0 + && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)!=pPager->journalOff + && ((szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager))>0 + && pagerNextJournalPageIsValid(pPager)) + ); + if( nJRec==0 + && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff + ){ + nJRec = (szJ - pPager->journalOff)/JOURNAL_PG_SZ(pPager); } - if( nJRec==0 ){ - nJRec = (szJ - pPager->journalOff) / (pPager->pageSize+8); - } - for(i=nJRec-1; i>=0 && pPager->journalOff < szJ; i--){ - rc = pager_playback_one_page(pPager, pPager->jfd, 1); + for(ii=0; rc==SQLITE_OK && iijournalOffjournalOff, 1, pDone); + assert( rc!=SQLITE_DONE ); + } + } + assert( rc!=SQLITE_OK || pPager->journalOff==szJ ); + + /* Finally, rollback pages from the sub-journal. Page that were + ** previously rolled back out of the main journal (and are hence in pDone) + ** will be skipped. Out-of-range pages are also skipped. + */ + if( pSavepoint ){ + i64 offset = pSavepoint->iSubRec*(4+pPager->pageSize); + for(ii=pSavepoint->iSubRec; rc==SQLITE_OK&&ii<(u32)pPager->stmtNRec; ii++){ + assert( offset == ii*(4+pPager->pageSize) ); + rc = pager_playback_one_page(pPager, 0, &offset, 1, pDone); assert( rc!=SQLITE_DONE ); - if( rc!=SQLITE_OK ) goto end_stmt_playback; } } - pPager->journalOff = szJ; - -end_stmt_playback: - if( rc==SQLITE_OK) { + sqlite3BitvecDestroy(pDone); + if( rc==SQLITE_OK ){ pPager->journalOff = szJ; - /* pager_reload_cache(pPager); */ } return rc; } @@ -1633,11 +1865,7 @@ end_stmt_playback: ** Change the maximum number of in-memory pages that are allowed. */ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ - if( mxPage>10 ){ - pPager->mxPage = mxPage; - }else{ - pPager->mxPage = 10; - } + sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } /* @@ -1667,10 +1895,10 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ ** and FULL=3. */ #ifndef SQLITE_OMIT_PAGER_PRAGMAS -void sqlite3PagerSetSafetyLevel(Pager *pPager, int level, int full_fsync){ - pPager->noSync = level==1 || pPager->tempFile; - pPager->fullSync = level==3 && !pPager->tempFile; - pPager->full_fsync = full_fsync; +void sqlite3PagerSetSafetyLevel(Pager *pPager, int level, int bFullFsync){ + pPager->noSync = (level==1 || pPager->tempFile) ?1:0; + pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; + pPager->sync_flags = (bFullFsync?SQLITE_SYNC_FULL:SQLITE_SYNC_NORMAL); if( pPager->noSync ) pPager->needSync = 0; } #endif @@ -1688,28 +1916,29 @@ int sqlite3_opentemp_count = 0; ** Open a temporary file. ** ** Write the file descriptor into *fd. Return SQLITE_OK on success or some -** other error code if we fail. -** -** The OS will automatically delete the temporary file when it is -** closed. +** other error code if we fail. The OS will automatically delete the temporary +** file when it is closed. */ -static int sqlite3PagerOpentemp(OsFile **pFd){ - int cnt = 8; +static int sqlite3PagerOpentemp( + Pager *pPager, /* The pager object */ + sqlite3_file *pFile, /* Write the file descriptor here */ + int vfsFlags /* Flags passed through to the VFS */ +){ int rc; - char zFile[SQLITE_TEMPNAME_SIZE]; #ifdef SQLITE_TEST sqlite3_opentemp_count++; /* Used for testing and analysis only */ #endif - do{ - cnt--; - sqlite3OsTempFileName(zFile); - rc = sqlite3OsOpenExclusive(zFile, pFd, 1); - assert( rc!=SQLITE_OK || *pFd ); - }while( cnt>0 && rc!=SQLITE_OK && rc!=SQLITE_NOMEM ); + + vfsFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; + rc = sqlite3OsOpen(pPager->pVfs, 0, pFile, vfsFlags, 0); + assert( rc!=SQLITE_OK || pFile->pMethods ); return rc; } +static int pagerStress(void *,PgHdr *); + /* ** Create a new page cache and put a pointer to the page cache in *ppPager. ** The file to be cached need not exist. The file is not locked until @@ -1725,15 +1954,15 @@ static int sqlite3PagerOpentemp(OsFile **pFd){ ** in-memory database. */ int sqlite3PagerOpen( + sqlite3_vfs *pVfs, /* The virtual file system to use */ Pager **ppPager, /* Return the Pager structure here */ const char *zFilename, /* Name of the database file to open */ int nExtra, /* Extra bytes append to each in-memory page */ - int flags /* flags controlling this file */ + int flags, /* flags controlling this file */ + int vfsFlags /* flags passed through to sqlite3_vfs.xOpen() */ ){ + u8 *pPtr; Pager *pPager = 0; - char *zFullPathname = 0; - int nameLen; /* Compiler is wrong. This is always initialized before use */ - OsFile *fd = 0; int rc = SQLITE_OK; int i; int tempFile = 0; @@ -1741,154 +1970,218 @@ int sqlite3PagerOpen( int readOnly = 0; int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; int noReadlock = (flags & PAGER_NO_READLOCK)!=0; - char zTemp[SQLITE_TEMPNAME_SIZE]; -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - /* A malloc() cannot fail in sqlite3ThreadData() as one or more calls to - ** malloc() must have already been made by this thread before it gets - ** to this point. This means the ThreadData must have been allocated already - ** so that ThreadData.nAlloc can be set. It would be nice to assert - ** that ThreadData.nAlloc is non-zero, but alas this breaks test cases - ** written to invoke the pager directly. - */ - ThreadData *pTsd = sqlite3ThreadData(); - assert( pTsd ); -#endif + int journalFileSize; + int pcacheSize = sqlite3PcacheSize(); + int szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; + char *zPathname = 0; + int nPathname = 0; - /* We used to test if malloc() had already failed before proceeding. - ** But the way this function is used in SQLite means that can never - ** happen. Furthermore, if the malloc-failed flag is already set, - ** either the call to sqliteStrDup() or sqliteMalloc() below will - ** fail shortly and SQLITE_NOMEM returned anyway. - */ + if( sqlite3JournalSize(pVfs)>sqlite3MemJournalSize() ){ + journalFileSize = sqlite3JournalSize(pVfs); + }else{ + journalFileSize = sqlite3MemJournalSize(); + } + + /* The default return is a NULL pointer */ *ppPager = 0; - /* Open the pager file and set zFullPathname to point at malloc()ed - ** memory containing the complete filename (i.e. including the directory). + /* Compute and store the full pathname in an allocated buffer pointed + ** to by zPathname, length nPathname. Or, if this is a temporary file, + ** leave both nPathname and zPathname set to 0. */ if( zFilename && zFilename[0] ){ + nPathname = pVfs->mxPathname+1; + zPathname = sqlite3Malloc(nPathname*2); + if( zPathname==0 ){ + return SQLITE_NOMEM; + } #ifndef SQLITE_OMIT_MEMORYDB if( strcmp(zFilename,":memory:")==0 ){ memDb = 1; - zFullPathname = sqliteStrDup(""); + zPathname[0] = 0; }else #endif { - zFullPathname = sqlite3OsFullPathname(zFilename); - if( zFullPathname ){ - rc = sqlite3OsOpenReadWrite(zFullPathname, &fd, &readOnly); - assert( rc!=SQLITE_OK || fd ); + rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); + } + if( rc!=SQLITE_OK ){ + sqlite3_free(zPathname); + return rc; + } + nPathname = sqlite3Strlen30(zPathname); + } + + /* Allocate memory for the pager structure */ + pPager = sqlite3MallocZero( + sizeof(*pPager) + /* Pager structure */ + pcacheSize + /* PCache object */ + journalFileSize + /* The journal file structure */ + pVfs->szOsFile + /* The main db file */ + journalFileSize * 2 + /* The two journal files */ + 3*nPathname + 40 /* zFilename, zDirectory, zJournal */ + ); + if( !pPager ){ + sqlite3_free(zPathname); + return SQLITE_NOMEM; + } + pPager->pPCache = (PCache *)&pPager[1]; + pPtr = ((u8 *)&pPager[1]) + pcacheSize; + pPager->vfsFlags = vfsFlags; + pPager->fd = (sqlite3_file*)&pPtr[pVfs->szOsFile*0]; + pPager->sjfd = (sqlite3_file*)&pPtr[pVfs->szOsFile]; + pPager->jfd = (sqlite3_file*)&pPtr[pVfs->szOsFile+journalFileSize]; + pPager->zFilename = (char*)&pPtr[pVfs->szOsFile+2*journalFileSize]; + pPager->zDirectory = &pPager->zFilename[nPathname+1]; + pPager->zJournal = &pPager->zDirectory[nPathname+1]; + pPager->pVfs = pVfs; + if( zPathname ){ + memcpy(pPager->zFilename, zPathname, nPathname+1); + sqlite3_free(zPathname); + } + + /* Open the pager file. + */ + if( zFilename && zFilename[0] && !memDb ){ + if( nPathname>(pVfs->mxPathname - (int)sizeof("-journal")) ){ + rc = SQLITE_CANTOPEN; + }else{ + int fout = 0; + rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, + pPager->vfsFlags, &fout); + readOnly = (fout&SQLITE_OPEN_READONLY); + + /* If the file was successfully opened for read/write access, + ** choose a default page size in case we have to create the + ** database file. The default page size is the maximum of: + ** + ** + SQLITE_DEFAULT_PAGE_SIZE, + ** + The value returned by sqlite3OsSectorSize() + ** + The largest page size that can be written atomically. + */ + if( rc==SQLITE_OK && !readOnly ){ + setSectorSize(pPager); + if( szPageDfltsectorSize ){ + szPageDflt = pPager->sectorSize; + } +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + { + int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); + int ii; + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536); + for(ii=szPageDflt; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){ + if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ) szPageDflt = ii; + } + } +#endif + if( szPageDflt>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ + szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; + } } } }else{ - rc = sqlite3PagerOpentemp(&fd); - sqlite3OsTempFileName(zTemp); - zFilename = zTemp; - zFullPathname = sqlite3OsFullPathname(zFilename); - if( rc==SQLITE_OK ){ - tempFile = 1; - } + /* If a temporary file is requested, it is not opened immediately. + ** In this case we accept the default page size and delay actually + ** opening the file until the first call to OsWrite(). + ** + ** This branch is also run for an in-memory database. An in-memory + ** database is the same as a temp-file that is never written out to + ** disk and uses an in-memory rollback journal. + */ + tempFile = 1; + pPager->state = PAGER_EXCLUSIVE; } - /* Allocate the Pager structure. As part of the same allocation, allocate - ** space for the full paths of the file, directory and journal - ** (Pager.zFilename, Pager.zDirectory and Pager.zJournal). - */ - if( zFullPathname ){ - nameLen = strlen(zFullPathname); - pPager = sqliteMalloc( sizeof(*pPager) + nameLen*3 + 30 ); - if( pPager && rc==SQLITE_OK ){ - pPager->pTmpSpace = (char *)sqliteMallocRaw(SQLITE_DEFAULT_PAGE_SIZE); - } + if( pPager && rc==SQLITE_OK ){ + pPager->pTmpSpace = sqlite3PageMalloc(szPageDflt); } - - /* If an error occured in either of the blocks above, free the memory - ** pointed to by zFullPathname, free the Pager structure and close the - ** file. Since the pager is not allocated there is no need to set + /* If an error occured in either of the blocks above. + ** Free the Pager structure and close the file. + ** Since the pager is not allocated there is no need to set ** any Pager.errMask variables. */ - if( !pPager || !zFullPathname || !pPager->pTmpSpace || rc!=SQLITE_OK ){ - sqlite3OsClose(&fd); - sqliteFree(zFullPathname); - sqliteFree(pPager); + if( !pPager || !pPager->pTmpSpace ){ + sqlite3OsClose(pPager->fd); + sqlite3_free(pPager); return ((rc==SQLITE_OK)?SQLITE_NOMEM:rc); } + nExtra = FORCE_ALIGNMENT(nExtra); + sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, + !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); - PAGERTRACE3("OPEN %d %s\n", FILEHANDLEID(fd), zFullPathname); - IOTRACE(("OPEN %p %s\n", pPager, zFullPathname)) - pPager->zFilename = (char*)&pPager[1]; - pPager->zDirectory = &pPager->zFilename[nameLen+1]; - pPager->zJournal = &pPager->zDirectory[nameLen+1]; - memcpy(pPager->zFilename, zFullPathname, nameLen+1); - memcpy(pPager->zDirectory, zFullPathname, nameLen+1); + PAGERTRACE(("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename)); + IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename)) - for(i=nameLen; i>0 && pPager->zDirectory[i-1]!='/'; i--){} + /* Fill in Pager.zDirectory[] */ + memcpy(pPager->zDirectory, pPager->zFilename, nPathname+1); + for(i=sqlite3Strlen30(pPager->zDirectory); + i>0 && pPager->zDirectory[i-1]!='/'; i--){} if( i>0 ) pPager->zDirectory[i-1] = 0; - memcpy(pPager->zJournal, zFullPathname,nameLen); - sqliteFree(zFullPathname); - memcpy(&pPager->zJournal[nameLen], "-journal",sizeof("-journal")); - pPager->fd = fd; + + /* Fill in Pager.zJournal[] */ + if( zPathname ){ + memcpy(pPager->zJournal, pPager->zFilename, nPathname); + memcpy(&pPager->zJournal[nPathname], "-journal", 9); + }else{ + pPager->zJournal = 0; + } + /* pPager->journalOpen = 0; */ - pPager->useJournal = useJournal && !memDb; - pPager->noReadlock = noReadlock && readOnly; + pPager->useJournal = (u8)useJournal; + pPager->noReadlock = (noReadlock && readOnly) ?1:0; /* pPager->stmtOpen = 0; */ /* pPager->stmtInUse = 0; */ /* pPager->nRef = 0; */ - pPager->dbSize = memDb-1; - pPager->pageSize = SQLITE_DEFAULT_PAGE_SIZE; + pPager->dbSizeValid = (u8)memDb; + pPager->pageSize = szPageDflt; /* pPager->stmtSize = 0; */ /* pPager->stmtJSize = 0; */ /* pPager->nPage = 0; */ pPager->mxPage = 100; pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; - assert( PAGER_UNLOCK==0 ); /* pPager->state = PAGER_UNLOCK; */ + assert( pPager->state == (tempFile ? PAGER_EXCLUSIVE : PAGER_UNLOCK) ); /* pPager->errMask = 0; */ - pPager->tempFile = tempFile; + pPager->tempFile = (u8)tempFile; assert( tempFile==PAGER_LOCKINGMODE_NORMAL || tempFile==PAGER_LOCKINGMODE_EXCLUSIVE ); assert( PAGER_LOCKINGMODE_EXCLUSIVE==1 ); - pPager->exclusiveMode = tempFile; - pPager->memDb = memDb; - pPager->readOnly = readOnly; + pPager->exclusiveMode = (u8)tempFile; + pPager->memDb = (u8)memDb; + pPager->readOnly = (u8)readOnly; /* pPager->needSync = 0; */ - pPager->noSync = pPager->tempFile || !useJournal; - pPager->fullSync = (pPager->noSync?0:1); + pPager->noSync = (pPager->tempFile || !useJournal) ?1:0; + pPager->fullSync = pPager->noSync ?0:1; + pPager->sync_flags = SQLITE_SYNC_NORMAL; /* pPager->pFirst = 0; */ /* pPager->pFirstSynced = 0; */ /* pPager->pLast = 0; */ - pPager->nExtra = FORCE_ALIGNMENT(nExtra); - assert(fd||memDb); - if( !memDb ){ - setSectorSize(pPager); + pPager->nExtra = nExtra; + pPager->journalSizeLimit = SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT; + assert(pPager->fd->pMethods||tempFile); + setSectorSize(pPager); + if( memDb ){ + pPager->journalMode = PAGER_JOURNALMODE_MEMORY; } - /* pPager->pBusyHandler = 0; */ + /* pPager->xBusyHandler = 0; */ + /* pPager->pBusyHandlerArg = 0; */ /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ *ppPager = pPager; -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - pPager->pNext = pTsd->pPager; - pTsd->pPager = pPager; -#endif return SQLITE_OK; } /* ** Set the busy handler function. */ -void sqlite3PagerSetBusyhandler(Pager *pPager, BusyHandler *pBusyHandler){ - pPager->pBusyHandler = pBusyHandler; -} - -/* -** Set the destructor for this pager. If not NULL, the destructor is called -** when the reference count on each page reaches zero. The destructor can -** be used to clean up information in the extra segment appended to each page. -** -** The destructor is not called as a result sqlite3PagerClose(). -** Destructors are only called by sqlite3PagerUnref(). -*/ -void sqlite3PagerSetDestructor(Pager *pPager, void (*xDesc)(DbPage*,int)){ - pPager->xDestructor = xDesc; +void sqlite3PagerSetBusyhandler( + Pager *pPager, + int (*xBusyHandler)(void *), + void *pBusyHandlerArg +){ + pPager->xBusyHandler = xBusyHandler; + pPager->pBusyHandlerArg = pBusyHandlerArg; } /* @@ -1898,23 +2191,51 @@ void sqlite3PagerSetDestructor(Pager *pPager, void (*xDesc)(DbPage*,int)){ ** an opportunity to restore the EXTRA section to agree with the restored ** page data. */ -void sqlite3PagerSetReiniter(Pager *pPager, void (*xReinit)(DbPage*,int)){ +void sqlite3PagerSetReiniter(Pager *pPager, void (*xReinit)(DbPage*)){ pPager->xReiniter = xReinit; } /* -** Set the page size. Return the new size. If the suggest new page -** size is inappropriate, then an alternative page size is selected -** and returned. +** Set the page size to *pPageSize. If the suggest new page size is +** inappropriate, then an alternative page size is set to that +** value before returning. */ -int sqlite3PagerSetPagesize(Pager *pPager, int pageSize){ - assert( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE ); - if( !pPager->memDb && pPager->nRef==0 ){ - pager_reset(pPager); - pPager->pageSize = pageSize; - pPager->pTmpSpace = sqlite3ReallocOrFree(pPager->pTmpSpace, pageSize); +int sqlite3PagerSetPagesize(Pager *pPager, u16 *pPageSize){ + int rc = pPager->errCode; + if( rc==SQLITE_OK ){ + u16 pageSize = *pPageSize; + assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); + if( pageSize && pageSize!=pPager->pageSize + && (pPager->memDb==0 || pPager->dbSize==0) + && sqlite3PcacheRefCount(pPager->pPCache)==0 + ){ + char *pNew = (char *)sqlite3PageMalloc(pageSize); + if( !pNew ){ + rc = SQLITE_NOMEM; + }else{ + pager_reset(pPager); + pPager->pageSize = pageSize; + if( !pPager->memDb ) setSectorSize(pPager); + sqlite3PageFree(pPager->pTmpSpace); + pPager->pTmpSpace = pNew; + sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); + } + } + *pPageSize = (u16)pPager->pageSize; } - return pPager->pageSize; + return rc; +} + +/* +** Return a pointer to the "temporary page" buffer held internally +** by the pager. This is a buffer that is big enough to hold the +** entire content of a database page. This buffer is used internally +** during rollback and will be overwritten whenever a rollback +** occurs. But other modules are free to use it too, as long as +** no rollbacks are happening. +*/ +void *sqlite3PagerTempSpace(Pager *pPager){ + return pPager->pTmpSpace; } /* @@ -1928,7 +2249,7 @@ int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ if( mxPage>0 ){ pPager->mxPgno = mxPage; } - sqlite3PagerPagecount(pPager); + sqlite3PagerPagecount(pPager, 0); return pPager->mxPgno; } @@ -1969,12 +2290,10 @@ void enable_simulated_io_errors(void){ int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ int rc = SQLITE_OK; memset(pDest, 0, N); - if( MEMDB==0 ){ - disable_simulated_io_errors(); - sqlite3OsSeek(pPager->fd, 0); - enable_simulated_io_errors(); + assert(pPager->fd->pMethods||pPager->tempFile); + if( pPager->fd->pMethods ){ IOTRACE(("DBHDR %p 0 %d\n", pPager, N)) - rc = sqlite3OsRead(pPager->fd, pDest, N); + rc = sqlite3OsRead(pPager->fd, pDest, N, 0); if( rc==SQLITE_IOERR_SHORT_READ ){ rc = SQLITE_OK; } @@ -1991,19 +2310,22 @@ int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ ** PENDING_BYTE is byte 4096 (the first byte of page 5) and the size of the ** file is 4096 bytes, 5 is returned instead of 4. */ -int sqlite3PagerPagecount(Pager *pPager){ - i64 n; +int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ + i64 n = 0; int rc; assert( pPager!=0 ); if( pPager->errCode ){ - return 0; + rc = pPager->errCode; + return rc; } - if( pPager->dbSize>=0 ){ + if( pPager->dbSizeValid ){ n = pPager->dbSize; } else { - if( (rc = sqlite3OsFileSize(pPager->fd, &n))!=SQLITE_OK ){ + assert(pPager->fd->pMethods||pPager->tempFile); + if( (pPager->fd->pMethods) + && (rc = sqlite3OsFileSize(pPager->fd, &n))!=SQLITE_OK ){ pager_error(pPager, rc); - return 0; + return rc; } if( n>0 && npageSize ){ n = 1; @@ -2011,135 +2333,28 @@ int sqlite3PagerPagecount(Pager *pPager){ n /= pPager->pageSize; } if( pPager->state!=PAGER_UNLOCK ){ - pPager->dbSize = n; + pPager->dbSize = (Pgno)n; + pPager->dbFileSize = (Pgno)n; + pPager->dbSizeValid = 1; } } if( n==(PENDING_BYTE/pPager->pageSize) ){ n++; } if( n>pPager->mxPgno ){ - pPager->mxPgno = n; + pPager->mxPgno = (Pgno)n; } - return n; + if( pnPage ){ + *pnPage = (int)n; + } + return SQLITE_OK; } - -#ifndef SQLITE_OMIT_MEMORYDB -/* -** Clear a PgHistory block -*/ -static void clearHistory(PgHistory *pHist){ - sqliteFree(pHist->pOrig); - sqliteFree(pHist->pStmt); - pHist->pOrig = 0; - pHist->pStmt = 0; -} -#else -#define clearHistory(x) -#endif - /* ** Forward declaration */ static int syncJournal(Pager*); -/* -** Unlink pPg from it's hash chain. Also set the page number to 0 to indicate -** that the page is not part of any hash chain. This is required because the -** sqlite3PagerMovepage() routine can leave a page in the -** pNextFree/pPrevFree list that is not a part of any hash-chain. -*/ -static void unlinkHashChain(Pager *pPager, PgHdr *pPg){ - if( pPg->pgno==0 ){ - assert( pPg->pNextHash==0 && pPg->pPrevHash==0 ); - return; - } - if( pPg->pNextHash ){ - pPg->pNextHash->pPrevHash = pPg->pPrevHash; - } - if( pPg->pPrevHash ){ - assert( pPager->aHash[pPg->pgno & (pPager->nHash-1)]!=pPg ); - pPg->pPrevHash->pNextHash = pPg->pNextHash; - }else{ - int h = pPg->pgno & (pPager->nHash-1); - pPager->aHash[h] = pPg->pNextHash; - } - if( MEMDB ){ - clearHistory(PGHDR_TO_HIST(pPg, pPager)); - } - pPg->pgno = 0; - pPg->pNextHash = pPg->pPrevHash = 0; -} - -/* -** Unlink a page from the free list (the list of all pages where nRef==0) -** and from its hash collision chain. -*/ -static void unlinkPage(PgHdr *pPg){ - Pager *pPager = pPg->pPager; - - /* Keep the pFirstSynced pointer pointing at the first synchronized page */ - if( pPg==pPager->pFirstSynced ){ - PgHdr *p = pPg->pNextFree; - while( p && p->needSync ){ p = p->pNextFree; } - pPager->pFirstSynced = p; - } - - /* Unlink from the freelist */ - if( pPg->pPrevFree ){ - pPg->pPrevFree->pNextFree = pPg->pNextFree; - }else{ - assert( pPager->pFirst==pPg ); - pPager->pFirst = pPg->pNextFree; - } - if( pPg->pNextFree ){ - pPg->pNextFree->pPrevFree = pPg->pPrevFree; - }else{ - assert( pPager->pLast==pPg ); - pPager->pLast = pPg->pPrevFree; - } - pPg->pNextFree = pPg->pPrevFree = 0; - - /* Unlink from the pgno hash table */ - unlinkHashChain(pPager, pPg); -} - -/* -** This routine is used to truncate the cache when a database -** is truncated. Drop from the cache all pages whose pgno is -** larger than pPager->dbSize and is unreferenced. -** -** Referenced pages larger than pPager->dbSize are zeroed. -** -** Actually, at the point this routine is called, it would be -** an error to have a referenced page. But rather than delete -** that page and guarantee a subsequent segfault, it seems better -** to zero it and hope that we error out sanely. -*/ -static void pager_truncate_cache(Pager *pPager){ - PgHdr *pPg; - PgHdr **ppPg; - int dbSize = pPager->dbSize; - - ppPg = &pPager->pAll; - while( (pPg = *ppPg)!=0 ){ - if( pPg->pgno<=dbSize ){ - ppPg = &pPg->pNextAll; - }else if( pPg->nRef>0 ){ - memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize); - ppPg = &pPg->pNextAll; - }else{ - *ppPg = pPg->pNextAll; - IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno)); - PAGER_INCR(sqlite3_pager_pgfree_count); - unlinkPage(pPg); - makeClean(pPg); - sqliteFree(pPg); - pPager->nPage--; - } - } -} - /* ** Try to obtain a lock on a file. Invoke the busy callback if the lock ** is currently not available. Repeat until the busy callback returns @@ -2157,56 +2372,51 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ assert( PAGER_EXCLUSIVE==EXCLUSIVE_LOCK ); /* If the file is currently unlocked then the size must be unknown */ - assert( pPager->state>=PAGER_SHARED || pPager->dbSize<0 || MEMDB ); + assert( pPager->state>=PAGER_SHARED || pPager->dbSizeValid==0 ); if( pPager->state>=locktype ){ rc = SQLITE_OK; }else{ do { rc = sqlite3OsLock(pPager->fd, locktype); - }while( rc==SQLITE_BUSY && sqlite3InvokeBusyHandler(pPager->pBusyHandler) ); + }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); if( rc==SQLITE_OK ){ - pPager->state = locktype; + pPager->state = (u8)locktype; IOTRACE(("LOCK %p %d\n", pPager, locktype)) } } return rc; } +#ifndef SQLITE_OMIT_AUTOVACUUM /* -** Truncate the file to the number of pages specified. +** Truncate the in-memory database file image to nPage pages. This +** function does not actually modify the database file on disk. It +** just sets the internal state of the pager object so that the +** truncation will be done when the current transaction is committed. */ -int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){ - int rc; - assert( pPager->state>=PAGER_SHARED || MEMDB ); - sqlite3PagerPagecount(pPager); - if( pPager->errCode ){ - rc = pPager->errCode; - return rc; - } - if( nPage>=(unsigned)pPager->dbSize ){ - return SQLITE_OK; - } - if( MEMDB ){ - pPager->dbSize = nPage; - pager_truncate_cache(pPager); - return SQLITE_OK; - } - rc = syncJournal(pPager); - if( rc!=SQLITE_OK ){ - return rc; - } - - /* Get an exclusive lock on the database before truncating. */ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - if( rc!=SQLITE_OK ){ - return rc; - } - - rc = pager_truncate(pPager, nPage); - return rc; +void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ + assert( pPager->dbSizeValid ); + assert( pPager->dbSize>=nPage ); + pPager->dbSize = nPage; } +/* +** Return the current size of the database file image in pages. This +** function differs from sqlite3PagerPagecount() in two ways: +** +** a) It may only be called when at least one reference to a database +** page is held. This guarantees that the database size is already +** known and a call to sqlite3OsFileSize() is not required. +** +** b) The return value is not adjusted for the locking page. +*/ +Pgno sqlite3PagerImageSize(Pager *pPager){ + assert( pPager->dbSizeValid ); + return pPager->dbSize; +} +#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */ + /* ** Shutdown the page cache. Free all memory and close all files. ** @@ -2222,55 +2432,42 @@ int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){ ** to the caller. */ int sqlite3PagerClose(Pager *pPager){ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - /* A malloc() cannot fail in sqlite3ThreadData() as one or more calls to - ** malloc() must have already been made by this thread before it gets - ** to this point. This means the ThreadData must have been allocated already - ** so that ThreadData.nAlloc can be set. - */ - ThreadData *pTsd = sqlite3ThreadData(); - assert( pPager ); - assert( pTsd && pTsd->nAlloc ); -#endif disable_simulated_io_errors(); + sqlite3BeginBenignMalloc(); pPager->errCode = 0; pPager->exclusiveMode = 0; pager_reset(pPager); - pagerUnlockAndRollback(pPager); + if( !MEMDB ){ + /* Set Pager.journalHdr to -1 for the benefit of the pager_playback() + ** call which may be made from within pagerUnlockAndRollback(). If it + ** is not -1, then the unsynced portion of an open journal file may + ** be played back into the database. If a power failure occurs while + ** this is happening, the database may become corrupt. + */ + pPager->journalHdr = -1; + pagerUnlockAndRollback(pPager); + } enable_simulated_io_errors(); - PAGERTRACE2("CLOSE %d\n", PAGERID(pPager)); + sqlite3EndBenignMalloc(); + PAGERTRACE(("CLOSE %d\n", PAGERID(pPager))); IOTRACE(("CLOSE %p\n", pPager)) - assert( pPager->errCode || (pPager->journalOpen==0 && pPager->stmtOpen==0) ); if( pPager->journalOpen ){ - sqlite3OsClose(&pPager->jfd); + sqlite3OsClose(pPager->jfd); } - sqliteFree(pPager->aInJournal); - if( pPager->stmtOpen ){ - sqlite3OsClose(&pPager->stfd); - } - sqlite3OsClose(&pPager->fd); + sqlite3BitvecDestroy(pPager->pInJournal); + sqlite3BitvecDestroy(pPager->pAlwaysRollback); + releaseAllSavepoint(pPager); + sqlite3OsClose(pPager->fd); /* Temp files are automatically deleted by the OS ** if( pPager->tempFile ){ ** sqlite3OsDelete(pPager->zFilename); ** } */ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - /* Remove the pager from the linked list of pagers starting at - ** ThreadData.pPager if memory-management is enabled. - */ - if( pPager==pTsd->pPager ){ - pTsd->pPager = pPager->pNext; - }else{ - Pager *pTmp; - for(pTmp = pTsd->pPager; pTmp->pNext!=pPager; pTmp=pTmp->pNext){} - pTmp->pNext = pPager->pNext; - } -#endif - sqliteFree(pPager->aHash); - sqliteFree(pPager->pTmpSpace); - sqliteFree(pPager); + sqlite3PageFree(pPager->pTmpSpace); + sqlite3PcacheClose(pPager->pPCache); + sqlite3_free(pPager); return SQLITE_OK; } @@ -2283,57 +2480,12 @@ Pgno sqlite3PagerPagenumber(DbPage *p){ } #endif -/* -** The page_ref() function increments the reference count for a page. -** If the page is currently on the freelist (the reference count is zero) then -** remove it from the freelist. -** -** For non-test systems, page_ref() is a macro that calls _page_ref() -** online of the reference count is zero. For test systems, page_ref() -** is a real function so that we can set breakpoints and trace it. -*/ -static void _page_ref(PgHdr *pPg){ - if( pPg->nRef==0 ){ - /* The page is currently on the freelist. Remove it. */ - if( pPg==pPg->pPager->pFirstSynced ){ - PgHdr *p = pPg->pNextFree; - while( p && p->needSync ){ p = p->pNextFree; } - pPg->pPager->pFirstSynced = p; - } - if( pPg->pPrevFree ){ - pPg->pPrevFree->pNextFree = pPg->pNextFree; - }else{ - pPg->pPager->pFirst = pPg->pNextFree; - } - if( pPg->pNextFree ){ - pPg->pNextFree->pPrevFree = pPg->pPrevFree; - }else{ - pPg->pPager->pLast = pPg->pPrevFree; - } - pPg->pPager->nRef++; - } - pPg->nRef++; - REFINFO(pPg); -} -#ifdef SQLITE_DEBUG - static void page_ref(PgHdr *pPg){ - if( pPg->nRef==0 ){ - _page_ref(pPg); - }else{ - pPg->nRef++; - REFINFO(pPg); - } - } -#else -# define page_ref(P) ((P)->nRef==0?_page_ref(P):(void)(P)->nRef++) -#endif - /* ** Increment the reference count for a page. The input pointer is ** a reference to the page data. */ int sqlite3PagerRef(DbPage *pPg){ - page_ref(pPg); + sqlite3PcacheRef(pPg); return SQLITE_OK; } @@ -2352,167 +2504,107 @@ int sqlite3PagerRef(DbPage *pPg){ ** is synced, then the nRec field is updated, then a second sync occurs. ** ** For temporary databases, we do not care if we are able to rollback -** after a power failure, so sync occurs. +** after a power failure, so no sync occurs. +** +** If the IOCAP_SEQUENTIAL flag is set for the persistent media on which +** the database is stored, then OsSync() is never called on the journal +** file. In this case all that is required is to update the nRec field in +** the journal header. ** ** This routine clears the needSync field of every page current held in ** memory. */ static int syncJournal(Pager *pPager){ - PgHdr *pPg; int rc = SQLITE_OK; /* Sync the journal before modifying the main database ** (assuming there is a journal and it needs to be synced.) */ if( pPager->needSync ){ - if( !pPager->tempFile ){ + assert( !pPager->tempFile ); + if( pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ + int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); assert( pPager->journalOpen ); - /* assert( !pPager->noSync ); // noSync might be set if synchronous - ** was turned off after the transaction was started. Ticket #615 */ -#ifndef NDEBUG - { - /* Make sure the pPager->nRec counter we are keeping agrees - ** with the nRec computed from the size of the journal file. + + if( 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + i64 jrnlOff = journalHdrOffset(pPager); + u8 zMagic[8]; + + /* This block deals with an obscure problem. If the last connection + ** that wrote to this database was operating in persistent-journal + ** mode, then the journal file may at this point actually be larger + ** than Pager.journalOff bytes. If the next thing in the journal + ** file happens to be a journal-header (written as part of the + ** previous connections transaction), and a crash or power-failure + ** occurs after nRec is updated but before this connection writes + ** anything else to the journal file (or commits/rolls back its + ** transaction), then SQLite may become confused when doing the + ** hot-journal rollback following recovery. It may roll back all + ** of this connections data, then proceed to rolling back the old, + ** out-of-date data that follows it. Database corruption. + ** + ** To work around this, if the journal file does appear to contain + ** a valid header following Pager.journalOff, then write a 0x00 + ** byte to the start of it to prevent it from being recognized. */ - i64 jSz; - rc = sqlite3OsFileSize(pPager->jfd, &jSz); - if( rc!=0 ) return rc; - assert( pPager->journalOff==jSz ); - } -#endif - { + rc = sqlite3OsRead(pPager->jfd, zMagic, 8, jrnlOff); + if( rc==SQLITE_OK && 0==memcmp(zMagic, aJournalMagic, 8) ){ + static const u8 zerobyte = 0; + rc = sqlite3OsWrite(pPager->jfd, &zerobyte, 1, jrnlOff); + } + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ + return rc; + } + /* Write the nRec value into the journal file header. If in ** full-synchronous mode, sync the journal first. This ensures that ** all data has really hit the disk before nRec is updated to mark - ** it as a candidate for rollback. + ** it as a candidate for rollback. + ** + ** This is not required if the persistent media supports the + ** SAFE_APPEND property. Because in this case it is not possible + ** for garbage data to be appended to the file, the nRec field + ** is populated with 0xFFFFFFFF when the journal header is written + ** and never needs to be updated. */ - if( pPager->fullSync ){ - PAGERTRACE2("SYNC journal of %d\n", PAGERID(pPager)); + if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ + PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); IOTRACE(("JSYNC %p\n", pPager)) - rc = sqlite3OsSync(pPager->jfd, 0); + rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags); if( rc!=0 ) return rc; } - rc = sqlite3OsSeek(pPager->jfd, - pPager->journalHdr + sizeof(aJournalMagic)); - if( rc ) return rc; - IOTRACE(("JHDR %p %lld %d\n", pPager, - pPager->journalHdr + sizeof(aJournalMagic), 4)) - rc = write32bits(pPager->jfd, pPager->nRec); - if( rc ) return rc; - rc = sqlite3OsSeek(pPager->jfd, pPager->journalOff); + jrnlOff = pPager->journalHdr + sizeof(aJournalMagic); + IOTRACE(("JHDR %p %lld %d\n", pPager, jrnlOff, 4)); + rc = write32bits(pPager->jfd, jrnlOff, pPager->nRec); if( rc ) return rc; } - PAGERTRACE2("SYNC journal of %d\n", PAGERID(pPager)); - IOTRACE(("JSYNC %p\n", pPager)) - rc = sqlite3OsSync(pPager->jfd, pPager->full_fsync); - if( rc!=0 ) return rc; + if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ + PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); + IOTRACE(("JSYNC %p\n", pPager)) + rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags| + (pPager->sync_flags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0) + ); + if( rc!=0 ) return rc; + } pPager->journalStarted = 1; } pPager->needSync = 0; /* Erase the needSync flag from every page. */ - for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ - pPg->needSync = 0; - } - pPager->pFirstSynced = pPager->pFirst; + sqlite3PcacheClearSyncFlags(pPager->pPCache); } -#ifndef NDEBUG - /* If the Pager.needSync flag is clear then the PgHdr.needSync - ** flag must also be clear for all pages. Verify that this - ** invariant is true. - */ - else{ - for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ - assert( pPg->needSync==0 ); - } - assert( pPager->pFirstSynced==pPager->pFirst ); - } -#endif - return rc; } -/* -** Merge two lists of pages connected by pDirty and in pgno order. -** Do not both fixing the pPrevDirty pointers. -*/ -static PgHdr *merge_pagelist(PgHdr *pA, PgHdr *pB){ - PgHdr result, *pTail; - pTail = &result; - while( pA && pB ){ - if( pA->pgnopgno ){ - pTail->pDirty = pA; - pTail = pA; - pA = pA->pDirty; - }else{ - pTail->pDirty = pB; - pTail = pB; - pB = pB->pDirty; - } - } - if( pA ){ - pTail->pDirty = pA; - }else if( pB ){ - pTail->pDirty = pB; - }else{ - pTail->pDirty = 0; - } - return result.pDirty; -} - -/* -** Sort the list of pages in accending order by pgno. Pages are -** connected by pDirty pointers. The pPrevDirty pointers are -** corrupted by this sort. -*/ -#define N_SORT_BUCKET_ALLOC 25 -#define N_SORT_BUCKET 25 -#ifdef SQLITE_TEST - int sqlite3_pager_n_sort_bucket = 0; - #undef N_SORT_BUCKET - #define N_SORT_BUCKET \ - (sqlite3_pager_n_sort_bucket?sqlite3_pager_n_sort_bucket:N_SORT_BUCKET_ALLOC) -#endif -static PgHdr *sort_pagelist(PgHdr *pIn){ - PgHdr *a[N_SORT_BUCKET_ALLOC], *p; - int i; - memset(a, 0, sizeof(a)); - while( pIn ){ - p = pIn; - pIn = p->pDirty; - p->pDirty = 0; - for(i=0; idirty ); - rc = sqlite3OsSeek(pPager->fd, (pList->pgno-1)*(i64)pPager->pageSize); - if( rc ) return rc; + + /* If the file has not yet been opened, open it now. */ + if( !pPager->fd->pMethods ){ + assert(pPager->tempFile); + rc = sqlite3PagerOpentemp(pPager, pPager->fd, pPager->vfsFlags); + if( rc ) return rc; + } + /* If there are dirty pages in the page cache with page numbers greater - ** than Pager.dbSize, this means sqlite3PagerTruncate() was called to + ** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to ** make the file smaller (presumably by auto-vacuum code). Do not write ** any such pages to the file. */ - if( pList->pgno<=pPager->dbSize ){ - char *pData = CODEC2(pPager, PGHDR_TO_DATA(pList), pList->pgno, 6); - PAGERTRACE4("STORE %d page %d hash(%08x)\n", - PAGERID(pPager), pList->pgno, pager_pagehash(pList)); + if( pList->pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){ + i64 offset = (pList->pgno-1)*(i64)pPager->pageSize; + char *pData = CODEC2(pPager, pList->pData, pList->pgno, 6); + + PAGERTRACE(("STORE %d page %d hash(%08x)\n", + PAGERID(pPager), pList->pgno, pager_pagehash(pList))); IOTRACE(("PGOUT %p %d\n", pPager, pList->pgno)); - rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize); + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); PAGER_INCR(sqlite3_pager_writedb_count); PAGER_INCR(pPager->nWrite); if( pList->pgno==1 ){ memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers)); } + if( pList->pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pList->pgno; + } } #ifndef NDEBUG else{ - PAGERTRACE3("NOSTORE %d page %d\n", PAGERID(pPager), pList->pgno); + PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pList->pgno)); } #endif if( rc ) return rc; - pList->dirty = 0; #ifdef SQLITE_CHECK_PAGES pList->pageHash = pager_pagehash(pList); #endif pList = pList->pDirty; } + return SQLITE_OK; } /* -** Collect every dirty page into a dirty list and -** return a pointer to the head of that list. All pages are -** collected even if they are still in use. +** Add the page to the sub-journal. It is the callers responsibility to +** use subjRequiresPage() to check that it is really required before +** calling this function. */ -static PgHdr *pager_get_all_dirty_pages(Pager *pPager){ - return pPager->pDirty; +static int subjournalPage(PgHdr *pPg){ + int rc; + void *pData = pPg->pData; + Pager *pPager = pPg->pPager; + i64 offset = pPager->stmtNRec*(4+pPager->pageSize); + char *pData2 = CODEC2(pPager, pData, pPg->pgno, 7); + + PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); + + assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); + rc = write32bits(pPager->sjfd, offset, pPg->pgno); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4); + } + if( rc==SQLITE_OK ){ + pPager->stmtNRec++; + assert( pPager->nSavepoint>0 ); + rc = addToSavepointBitvecs(pPager, pPg->pgno); + } + return rc; } + /* -** Return TRUE if there is a hot journal on the given pager. +** This function is called by the pcache layer when it has reached some +** soft memory limit. The argument is a pointer to a purgeable Pager +** object. This function attempts to make a single dirty page that has no +** outstanding references (if one exists) clean so that it can be recycled +** by the pcache layer. +*/ +static int pagerStress(void *p, PgHdr *pPg){ + Pager *pPager = (Pager *)p; + int rc = SQLITE_OK; + + if( pPager->doNotSync ){ + return SQLITE_OK; + } + + assert( pPg->flags&PGHDR_DIRTY ); + if( pPager->errCode==SQLITE_OK ){ + if( pPg->flags&PGHDR_NEED_SYNC ){ + rc = syncJournal(pPager); + if( rc==SQLITE_OK && pPager->fullSync && + !(pPager->journalMode==PAGER_JOURNALMODE_MEMORY) && + !(sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) + ){ + pPager->nRec = 0; + rc = writeJournalHdr(pPager); + } + } + if( rc==SQLITE_OK ){ + pPg->pDirty = 0; + if( pPg->pgno>pPager->dbSize && subjRequiresPage(pPg) ){ + rc = subjournalPage(pPg); + } + if( rc==SQLITE_OK ){ + rc = pager_write_pagelist(pPg); + } + } + if( rc!=SQLITE_OK ){ + pager_error(pPager, rc); + } + } + + if( rc==SQLITE_OK ){ + PAGERTRACE(("STRESS %d page %d\n", PAGERID(pPager), pPg->pgno)); + sqlite3PcacheMakeClean(pPg); + } + return rc; +} + + +/* +** Return 1 if there is a hot journal on the given pager. ** A hot journal is one that needs to be played back. ** ** If the current size of the database file is 0 but a journal file ** exists, that is probably an old journal left over from a prior ** database with the same name. Just delete the journal. -*/ -static int hasHotJournal(Pager *pPager){ - if( !pPager->useJournal ) return 0; - if( !sqlite3OsFileExists(pPager->zJournal) ){ - return 0; - } - if( sqlite3OsCheckReservedLock(pPager->fd) ){ - return 0; - } - if( sqlite3PagerPagecount(pPager)==0 ){ - sqlite3OsDelete(pPager->zJournal); - return 0; - }else{ - return 1; - } -} - -/* -** Try to find a page in the cache that can be recycled. ** -** This routine may return SQLITE_IOERR, SQLITE_FULL or SQLITE_OK. It -** does not set the pPager->errCode variable. -*/ -static int pager_recycle(Pager *pPager, int syncOk, PgHdr **ppPg){ - PgHdr *pPg; - *ppPg = 0; - - assert(!MEMDB); - - /* Find a page to recycle. Try to locate a page that does not - ** require us to do an fsync() on the journal. - */ - pPg = pPager->pFirstSynced; - - /* If we could not find a page that does not require an fsync() - ** on the journal file then fsync the journal file. This is a - ** very slow operation, so we work hard to avoid it. But sometimes - ** it can't be helped. - */ - if( pPg==0 && pPager->pFirst && syncOk && !MEMDB){ - int rc = syncJournal(pPager); - if( rc!=0 ){ - return rc; - } - if( pPager->fullSync ){ - /* If in full-sync mode, write a new journal header into the - ** journal file. This is done to avoid ever modifying a journal - ** header that is involved in the rollback of pages that have - ** already been written to the database (in case the header is - ** trashed when the nRec field is updated). - */ - pPager->nRec = 0; - assert( pPager->journalOff > 0 ); - assert( pPager->doNotSync==0 ); - rc = writeJournalHdr(pPager); - if( rc!=0 ){ - return rc; - } - } - pPg = pPager->pFirst; - } - if( pPg==0 ){ - return SQLITE_OK; - } - - assert( pPg->nRef==0 ); - - /* Write the page to the database file if it is dirty. - */ - if( pPg->dirty ){ - int rc; - assert( pPg->needSync==0 ); - makeClean(pPg); - pPg->dirty = 1; - pPg->pDirty = 0; - rc = pager_write_pagelist( pPg ); - if( rc!=SQLITE_OK ){ - return rc; - } - } - assert( pPg->dirty==0 ); - - /* If the page we are recycling is marked as alwaysRollback, then - ** set the global alwaysRollback flag, thus disabling the - ** sqlite3PagerDontRollback() optimization for the rest of this transaction. - ** It is necessary to do this because the page marked alwaysRollback - ** might be reloaded at a later time but at that point we won't remember - ** that is was marked alwaysRollback. This means that all pages must - ** be marked as alwaysRollback from here on out. - */ - if( pPg->alwaysRollback ){ - IOTRACE(("ALWAYS_ROLLBACK %p\n", pPager)) - pPager->alwaysRollback = 1; - } - - /* Unlink the old page from the free list and the hash table - */ - unlinkPage(pPg); - assert( pPg->pgno==0 ); - - *ppPg = pPg; - return SQLITE_OK; -} - -/* -** This function is called to free superfluous dynamically allocated memory -** held by the pager system. Memory in use by any SQLite pager allocated -** by the current thread may be sqliteFree()ed. +** Return negative if unable to determine the status of the journal. ** -** nReq is the number of bytes of memory required. Once this much has -** been released, the function returns. A negative value for nReq means -** free as much memory as possible. The return value is the total number -** of bytes of memory released. +** This routine does not open the journal file to examine its +** content. Hence, the journal might contain the name of a master +** journal file that has been deleted, and hence not be hot. Or +** the header of the journal might be zeroed out. This routine +** does not discover these cases of a non-hot journal - if the +** journal file exists and is not empty this routine assumes it +** is hot. The pager_playback() routine will discover that the +** journal file is not really hot and will no-op. */ -#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) && !defined(SQLITE_OMIT_DISKIO) -int sqlite3PagerReleaseMemory(int nReq){ - const ThreadData *pTsdro = sqlite3ThreadDataReadOnly(); - int nReleased = 0; - int i; - - /* If the the global mutex is held, this subroutine becomes a - ** o-op; zero bytes of memory are freed. This is because - ** some of the code invoked by this function may also - ** try to obtain the mutex, resulting in a deadlock. - */ - if( sqlite3OsInMutex(0) ){ - return 0; +static int hasHotJournal(Pager *pPager, int *pExists){ + sqlite3_vfs *pVfs = pPager->pVfs; + int rc = SQLITE_OK; + int exists = 0; + int locked = 0; + assert( pPager!=0 ); + assert( pPager->useJournal ); + assert( pPager->fd->pMethods ); + *pExists = 0; + rc = sqlite3OsAccess(pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &exists); + if( rc==SQLITE_OK && exists ){ + rc = sqlite3OsCheckReservedLock(pPager->fd, &locked); } - - /* Outermost loop runs for at most two iterations. First iteration we - ** try to find memory that can be released without calling fsync(). Second - ** iteration (which only runs if the first failed to free nReq bytes of - ** memory) is permitted to call fsync(). This is of course much more - ** expensive. - */ - for(i=0; i<=1; i++){ - - /* Loop through all the SQLite pagers opened by the current thread. */ - Pager *pPager = pTsdro->pPager; - for( ; pPager && (nReq<0 || nReleasedpNext){ - PgHdr *pPg; - int rc; - - if( MEMDB ){ - continue; - } - - /* For each pager, try to free as many pages as possible (without - ** calling fsync() if this is the first iteration of the outermost - ** loop). - */ - while( SQLITE_OK==(rc = pager_recycle(pPager, i, &pPg)) && pPg) { - /* We've found a page to free. At this point the page has been - ** removed from the page hash-table, free-list and synced-list - ** (pFirstSynced). It is still in the all pages (pAll) list. - ** Remove it from this list before freeing. - ** - ** Todo: Check the Pager.pStmt list to make sure this is Ok. It - ** probably is though. - */ - PgHdr *pTmp; - assert( pPg ); - if( pPg==pPager->pAll ){ - pPager->pAll = pPg->pNextAll; - }else{ - for( pTmp=pPager->pAll; pTmp->pNextAll!=pPg; pTmp=pTmp->pNextAll ){} - pTmp->pNextAll = pPg->pNextAll; - } - nReleased += sqliteAllocSize(pPg); - IOTRACE(("PGFREE %p %d *\n", pPager, pPg->pgno)); - PAGER_INCR(sqlite3_pager_pgfree_count); - sqliteFree(pPg); - } - - if( rc!=SQLITE_OK ){ - /* An error occured whilst writing to the database file or - ** journal in pager_recycle(). The error is not returned to the - ** caller of this function. Instead, set the Pager.errCode variable. - ** The error will be returned to the user (or users, in the case - ** of a shared pager cache) of the pager for which the error occured. - */ - assert( - (rc&0xff)==SQLITE_IOERR || - rc==SQLITE_FULL || - rc==SQLITE_BUSY - ); - assert( pPager->state>=PAGER_RESERVED ); - pager_error(pPager, rc); + if( rc==SQLITE_OK && exists && !locked ){ + int nPage; + rc = sqlite3PagerPagecount(pPager, &nPage); + if( rc==SQLITE_OK ){ + if( nPage==0 ){ + sqlite3OsDelete(pVfs, pPager->zJournal, 0); + }else{ + *pExists = 1; } } } - - return nReleased; + return rc; } -#endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT && !SQLITE_OMIT_DISKIO */ /* ** Read the content of page pPg out of the database file. */ static int readDbPage(Pager *pPager, PgHdr *pPg, Pgno pgno){ int rc; + i64 offset; assert( MEMDB==0 ); - rc = sqlite3OsSeek(pPager->fd, (pgno-1)*(i64)pPager->pageSize); - if( rc==SQLITE_OK ){ - rc = sqlite3OsRead(pPager->fd, PGHDR_TO_DATA(pPg), - pPager->pageSize); + assert(pPager->fd->pMethods||pPager->tempFile); + if( !pPager->fd->pMethods ){ + return SQLITE_IOERR_SHORT_READ; } + offset = (pgno-1)*(i64)pPager->pageSize; + rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, offset); PAGER_INCR(sqlite3_pager_readdb_count); PAGER_INCR(pPager->nRead); IOTRACE(("PGIN %p %d\n", pPager, pgno)); if( pgno==1 ){ - memcpy(&pPager->dbFileVers, &((u8*)PGHDR_TO_DATA(pPg))[24], + memcpy(&pPager->dbFileVers, &((u8*)pPg->pData)[24], sizeof(pPager->dbFileVers)); } - CODEC1(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3); - PAGERTRACE4("FETCH %d page %d hash(%08x)\n", - PAGERID(pPager), pPg->pgno, pager_pagehash(pPg)); + CODEC1(pPager, pPg->pData, pPg->pgno, 3); + PAGERTRACE(("FETCH %d page %d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, pager_pagehash(pPg))); return rc; } @@ -2827,220 +2841,177 @@ static int readDbPage(Pager *pPager, PgHdr *pPg, Pgno pgno){ */ static int pagerSharedLock(Pager *pPager){ int rc = SQLITE_OK; + int isErrorReset = 0; - if( pPager->state==PAGER_UNLOCK ){ - if( !MEMDB ){ - assert( pPager->nRef==0 ); - if( !pPager->noReadlock ){ - rc = pager_wait_on_lock(pPager, SHARED_LOCK); - if( rc!=SQLITE_OK ){ - return pager_error(pPager, rc); - } - assert( pPager->state>=SHARED_LOCK ); - } - - /* If a journal file exists, and there is no RESERVED lock on the - ** database file, then it either needs to be played back or deleted. - */ - if( hasHotJournal(pPager) ){ - /* Get an EXCLUSIVE lock on the database file. At this point it is - ** important that a RESERVED lock is not obtained on the way to the - ** EXCLUSIVE lock. If it were, another process might open the - ** database file, detect the RESERVED lock, and conclude that the - ** database is safe to read while this process is still rolling it - ** back. - ** - ** Because the intermediate RESERVED lock is not requested, the - ** second process will get to this point in the code and fail to - ** obtain it's own EXCLUSIVE lock on the database file. - */ - rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK); - if( rc!=SQLITE_OK ){ - pager_unlock(pPager); - return pager_error(pPager, rc); - } - pPager->state = PAGER_EXCLUSIVE; - - /* Open the journal for reading only. Return SQLITE_BUSY if - ** we are unable to open the journal file. - ** - ** The journal file does not need to be locked itself. The - ** journal file is never open unless the main database file holds - ** a write lock, so there is never any chance of two or more - ** processes opening the journal at the same time. - ** - ** Open the journal for read/write access. This is because in - ** exclusive-access mode the file descriptor will be kept open and - ** possibly used for a transaction later on. On some systems, the - ** OsTruncate() call used in exclusive-access mode also requires - ** a read/write file handle. - */ - rc = SQLITE_BUSY; - if( sqlite3OsFileExists(pPager->zJournal) ){ - int ro; - assert( !pPager->tempFile ); - rc = sqlite3OsOpenReadWrite(pPager->zJournal, &pPager->jfd, &ro); - assert( rc!=SQLITE_OK || pPager->jfd ); - if( ro ){ - rc = SQLITE_BUSY; - sqlite3OsClose(&pPager->jfd); - } - } - if( rc!=SQLITE_OK ){ - pager_unlock(pPager); - return SQLITE_BUSY; - } - pPager->journalOpen = 1; - pPager->journalStarted = 0; - pPager->journalOff = 0; - pPager->setMaster = 0; - pPager->journalHdr = 0; - - /* Playback and delete the journal. Drop the database write - ** lock and reacquire the read lock. - */ - rc = pager_playback(pPager, 1); - if( rc!=SQLITE_OK ){ - return pager_error(pPager, rc); - } - assert(pPager->state==PAGER_SHARED || - (pPager->exclusiveMode && pPager->state>PAGER_SHARED) - ); - } - - if( pPager->pAll ){ - /* The shared-lock has just been acquired on the database file - ** and there are already pages in the cache (from a previous - ** read or write transaction). Check to see if the database - ** has been modified. If the database has changed, flush the - ** cache. - ** - ** Database changes is detected by looking at 15 bytes beginning - ** at offset 24 into the file. The first 4 of these 16 bytes are - ** a 32-bit counter that is incremented with each change. The - ** other bytes change randomly with each file change when - ** a codec is in use. - ** - ** There is a vanishingly small chance that a change will not be - ** detected. The chance of an undetected change is so small that - ** it can be neglected. - */ - char dbFileVers[sizeof(pPager->dbFileVers)]; - sqlite3PagerPagecount(pPager); - - if( pPager->errCode ){ - return pPager->errCode; - } - - if( pPager->dbSize>0 ){ - IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); - rc = sqlite3OsSeek(pPager->fd, 24); - if( rc!=SQLITE_OK ){ - return rc; - } - rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers)); - if( rc!=SQLITE_OK ){ - return rc; - } - }else{ - memset(dbFileVers, 0, sizeof(dbFileVers)); - } - - if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ - pager_reset(pPager); - } - } + /* If this database is opened for exclusive access, has no outstanding + ** page references and is in an error-state, now is the chance to clear + ** the error. Discard the contents of the pager-cache and treat any + ** open journal file as a hot-journal. + */ + if( !MEMDB && pPager->exclusiveMode + && sqlite3PcacheRefCount(pPager->pPCache)==0 && pPager->errCode + ){ + if( pPager->journalOpen ){ + isErrorReset = 1; } - assert( pPager->exclusiveMode || pPager->state<=PAGER_SHARED ); - if( pPager->state==PAGER_UNLOCK ){ + pPager->errCode = SQLITE_OK; + pager_reset(pPager); + } + + /* If the pager is still in an error state, do not proceed. The error + ** state will be cleared at some point in the future when all page + ** references are dropped and the cache can be discarded. + */ + if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ + return pPager->errCode; + } + + if( pPager->state==PAGER_UNLOCK || isErrorReset ){ + sqlite3_vfs *pVfs = pPager->pVfs; + int isHotJournal = 0; + assert( !MEMDB ); + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); + if( !pPager->noReadlock ){ + rc = pager_wait_on_lock(pPager, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + assert( pPager->state==PAGER_UNLOCK ); + return pager_error(pPager, rc); + } + }else if( pPager->state==PAGER_UNLOCK ){ pPager->state = PAGER_SHARED; } - } + assert( pPager->state>=SHARED_LOCK ); - return rc; -} - -/* -** Allocate a PgHdr object. Either create a new one or reuse -** an existing one that is not otherwise in use. -** -** A new PgHdr structure is created if any of the following are -** true: -** -** (1) We have not exceeded our maximum allocated cache size -** as set by the "PRAGMA cache_size" command. -** -** (2) There are no unused PgHdr objects available at this time. -** -** (3) This is an in-memory database. -** -** (4) There are no PgHdr objects that do not require a journal -** file sync and a sync of the journal file is currently -** prohibited. -** -** Otherwise, reuse an existing PgHdr. In other words, reuse an -** existing PgHdr if all of the following are true: -** -** (1) We have reached or exceeded the maximum cache size -** allowed by "PRAGMA cache_size". -** -** (2) There is a PgHdr available with PgHdr->nRef==0 -** -** (3) We are not in an in-memory database -** -** (4) Either there is an available PgHdr that does not need -** to be synced to disk or else disk syncing is currently -** allowed. -*/ -static int pagerAllocatePage(Pager *pPager, PgHdr **ppPg){ - int rc = SQLITE_OK; - PgHdr *pPg; - - /* Create a new PgHdr if any of the four conditions defined - ** above is met: */ - if( pPager->nPagemxPage - || pPager->pFirst==0 - || MEMDB - || (pPager->pFirstSynced==0 && pPager->doNotSync) - ){ - if( pPager->nPage>=pPager->nHash ){ - pager_resize_hash_table(pPager, - pPager->nHash<256 ? 256 : pPager->nHash*2); - if( pPager->nHash==0 ){ - rc = SQLITE_NOMEM; - goto pager_allocate_out; + /* If a journal file exists, and there is no RESERVED lock on the + ** database file, then it either needs to be played back or deleted. + */ + if( !isErrorReset ){ + rc = hasHotJournal(pPager, &isHotJournal); + if( rc!=SQLITE_OK ){ + goto failed; } } - pPg = sqliteMallocRaw( sizeof(*pPg) + pPager->pageSize - + sizeof(u32) + pPager->nExtra - + MEMDB*sizeof(PgHistory) ); - if( pPg==0 ){ - rc = SQLITE_NOMEM; - goto pager_allocate_out; + if( isErrorReset || isHotJournal ){ + /* Get an EXCLUSIVE lock on the database file. At this point it is + ** important that a RESERVED lock is not obtained on the way to the + ** EXCLUSIVE lock. If it were, another process might open the + ** database file, detect the RESERVED lock, and conclude that the + ** database is safe to read while this process is still rolling it + ** back. + ** + ** Because the intermediate RESERVED lock is not requested, the + ** second process will get to this point in the code and fail to + ** obtain its own EXCLUSIVE lock on the database file. + */ + if( pPager->statefd, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + rc = pager_error(pPager, rc); + goto failed; + } + pPager->state = PAGER_EXCLUSIVE; + } + + /* Open the journal for read/write access. This is because in + ** exclusive-access mode the file descriptor will be kept open and + ** possibly used for a transaction later on. On some systems, the + ** OsTruncate() call used in exclusive-access mode also requires + ** a read/write file handle. + */ + if( !isErrorReset && pPager->journalOpen==0 ){ + int res; + rc = sqlite3OsAccess(pVfs,pPager->zJournal,SQLITE_ACCESS_EXISTS,&res); + if( rc==SQLITE_OK ){ + if( res ){ + int fout = 0; + int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; + assert( !pPager->tempFile ); + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); + assert( rc!=SQLITE_OK || pPager->jfd->pMethods ); + if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ + rc = SQLITE_CANTOPEN; + sqlite3OsClose(pPager->jfd); + } + }else{ + /* If the journal does not exist, that means some other process + ** has already rolled it back */ + rc = SQLITE_BUSY; + } + } + } + if( rc!=SQLITE_OK ){ + goto failed; + } + pPager->journalOpen = 1; + pPager->journalStarted = 0; + pPager->journalOff = 0; + pPager->setMaster = 0; + pPager->journalHdr = 0; + + /* Playback and delete the journal. Drop the database write + ** lock and reacquire the read lock. Purge the cache before + ** playing back the hot-journal so that we don't end up with + ** an inconsistent cache. + */ + sqlite3PcacheClear(pPager->pPCache); + rc = pager_playback(pPager, 1); + if( rc!=SQLITE_OK ){ + rc = pager_error(pPager, rc); + goto failed; + } + assert(pPager->state==PAGER_SHARED || + (pPager->exclusiveMode && pPager->state>PAGER_SHARED) + ); } - memset(pPg, 0, sizeof(*pPg)); - if( MEMDB ){ - memset(PGHDR_TO_HIST(pPg, pPager), 0, sizeof(PgHistory)); - } - pPg->pPager = pPager; - pPg->pNextAll = pPager->pAll; - pPager->pAll = pPg; - pPager->nPage++; - }else{ - /* Recycle an existing page with a zero ref-count. */ - rc = pager_recycle(pPager, 1, &pPg); - if( rc==SQLITE_BUSY ){ - rc = SQLITE_IOERR_BLOCKED; - } - if( rc!=SQLITE_OK ){ - goto pager_allocate_out; - } - assert( pPager->state>=SHARED_LOCK ); - assert(pPg); - } - *ppPg = pPg; -pager_allocate_out: + if( sqlite3PcachePagecount(pPager->pPCache)>0 ){ + /* The shared-lock has just been acquired on the database file + ** and there are already pages in the cache (from a previous + ** read or write transaction). Check to see if the database + ** has been modified. If the database has changed, flush the + ** cache. + ** + ** Database changes is detected by looking at 15 bytes beginning + ** at offset 24 into the file. The first 4 of these 16 bytes are + ** a 32-bit counter that is incremented with each change. The + ** other bytes change randomly with each file change when + ** a codec is in use. + ** + ** There is a vanishingly small chance that a change will not be + ** detected. The chance of an undetected change is so small that + ** it can be neglected. + */ + char dbFileVers[sizeof(pPager->dbFileVers)]; + sqlite3PagerPagecount(pPager, 0); + + if( pPager->errCode ){ + rc = pPager->errCode; + goto failed; + } + + assert( pPager->dbSizeValid ); + if( pPager->dbSize>0 ){ + IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); + if( rc!=SQLITE_OK ){ + goto failed; + } + }else{ + memset(dbFileVers, 0, sizeof(dbFileVers)); + } + + if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ + pager_reset(pPager); + } + } + assert( pPager->exclusiveMode || pPager->state==PAGER_SHARED ); + } + + failed: + if( rc!=SQLITE_OK ){ + /* pager_unlock() is a no-op for exclusive mode and in-memory databases. */ + pager_unlock(pPager); + } return rc; } @@ -3052,10 +3023,10 @@ pager_allocate_out: ** have it. Read it in if we do not have it already. */ static int pager_get_content(PgHdr *pPg){ - if( pPg->needRead ){ + if( pPg->flags&PGHDR_NEED_READ ){ int rc = readDbPage(pPg->pPager, pPg, pPg->pgno); if( rc==SQLITE_OK ){ - pPg->needRead = 0; + pPg->flags &= ~PGHDR_NEED_READ; }else{ return rc; } @@ -3063,6 +3034,30 @@ static int pager_get_content(PgHdr *pPg){ return SQLITE_OK; } +/* +** If the reference count has reached zero, and the pager is not in the +** middle of a write transaction or opened in exclusive mode, unlock it. +*/ +static void pagerUnlockIfUnused(Pager *pPager){ + if( (sqlite3PcacheRefCount(pPager->pPCache)==0) + && (!pPager->exclusiveMode || pPager->journalOff>0) + ){ + pagerUnlockAndRollback(pPager); + } +} + +/* +** Drop a page from the cache using sqlite3PcacheDrop(). +** +** If this means there are now no pages with references to them, a rollback +** occurs and the lock on the database is removed. +*/ +static void pagerDropPage(DbPage *pPg){ + Pager *pPager = pPg->pPager; + sqlite3PcacheDrop(pPg); + pagerUnlockIfUnused(pPager); +} + /* ** Acquire a page. ** @@ -3101,10 +3096,13 @@ int sqlite3PagerAcquire( DbPage **ppPage, /* Write a pointer to the page here */ int noContent /* Do not bother reading content from disk if true */ ){ - PgHdr *pPg; + PgHdr *pPg = 0; int rc; - assert( pPager->state==PAGER_UNLOCK || pPager->nRef>0 || pgno==1 ); + assert( pPager->state==PAGER_UNLOCK + || sqlite3PcacheRefCount(pPager->pPCache)>0 + || pgno==1 + ); /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page ** number greater than this, or zero, is requested. @@ -3117,9 +3115,6 @@ int sqlite3PagerAcquire( */ assert( pPager!=0 ); *ppPage = 0; - if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ - return pPager->errCode; - } /* If this is the first page accessed, then get a SHARED lock ** on the database file. pagerSharedLock() is a no-op if @@ -3131,90 +3126,59 @@ int sqlite3PagerAcquire( } assert( pPager->state!=PAGER_UNLOCK ); - pPg = pager_lookup(pPager, pgno); - if( pPg==0 ){ - /* The requested page is not in the page cache. */ - int nMax; - int h; - PAGER_INCR(pPager->nMiss); - rc = pagerAllocatePage(pPager, &pPg); - if( rc!=SQLITE_OK ){ - return rc; - } - - pPg->pgno = pgno; - assert( !MEMDB || pgno>pPager->stmtSize ); - if( pPager->aInJournal && (int)pgno<=pPager->origDbSize ){ - sqlite3CheckMemory(pPager->aInJournal, pgno/8); - assert( pPager->journalOpen ); - pPg->inJournal = (pPager->aInJournal[pgno/8] & (1<<(pgno&7)))!=0; - pPg->needSync = 0; - }else{ - pPg->inJournal = 0; - pPg->needSync = 0; - } - - makeClean(pPg); - pPg->nRef = 1; - REFINFO(pPg); - - pPager->nRef++; - if( pPager->nExtra>0 ){ - memset(PGHDR_TO_EXTRA(pPg, pPager), 0, pPager->nExtra); - } - nMax = sqlite3PagerPagecount(pPager); - if( pPager->errCode ){ - sqlite3PagerUnref(pPg); - rc = pPager->errCode; - return rc; - } - - /* Populate the page with data, either by reading from the database - ** file, or by setting the entire page to zero. + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, &pPg); + if( rc!=SQLITE_OK ){ + return rc; + } + if( pPg->pPager==0 ){ + /* The pager cache has created a new page. Its content needs to + ** be initialized. */ - if( nMax<(int)pgno || MEMDB || (noContent && !pPager->alwaysRollback) ){ + int nMax; + PAGER_INCR(pPager->nMiss); + pPg->pPager = pPager; + memset(pPg->pExtra, 0, pPager->nExtra); + + rc = sqlite3PagerPagecount(pPager, &nMax); + if( rc!=SQLITE_OK ){ + sqlite3PagerUnref(pPg); + return rc; + } + + if( nMax<(int)pgno || MEMDB || noContent ){ if( pgno>pPager->mxPgno ){ sqlite3PagerUnref(pPg); return SQLITE_FULL; } - memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize); - pPg->needRead = noContent && !pPager->alwaysRollback; + memset(pPg->pData, 0, pPager->pageSize); + if( noContent ){ + pPg->flags |= PGHDR_NEED_READ; + } IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ rc = readDbPage(pPager, pPg, pgno); if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ - pPg->pgno = 0; - sqlite3PagerUnref(pPg); + /* sqlite3PagerUnref(pPg); */ + pagerDropPage(pPg); return rc; } - pPg->needRead = 0; } - - /* Link the page into the page hash table */ - h = pgno & (pPager->nHash-1); - assert( pgno!=0 ); - pPg->pNextHash = pPager->aHash[h]; - pPager->aHash[h] = pPg; - if( pPg->pNextHash ){ - assert( pPg->pNextHash->pPrevHash==0 ); - pPg->pNextHash->pPrevHash = pPg; - } - #ifdef SQLITE_CHECK_PAGES pPg->pageHash = pager_pagehash(pPg); #endif }else{ /* The requested page is in the page cache. */ - assert(pPager->nRef>0 || pgno==1); + assert(sqlite3PcacheRefCount(pPager->pPCache)>0 || pgno==1); PAGER_INCR(pPager->nHit); if( !noContent ){ rc = pager_get_content(pPg); if( rc ){ + sqlite3PagerUnref(pPg); return rc; } } - page_ref(pPg); } + *ppPage = pPg; return SQLITE_OK; } @@ -3231,21 +3195,16 @@ int sqlite3PagerAcquire( ** has ever happened. */ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ - PgHdr *pPg; - + PgHdr *pPg = 0; assert( pPager!=0 ); assert( pgno!=0 ); - if( pPager->state==PAGER_UNLOCK ){ - assert( !pPager->pAll || pPager->exclusiveMode ); - return 0; + if( (pPager->state!=PAGER_UNLOCK) + && (pPager->errCode==SQLITE_OK || pPager->errCode==SQLITE_FULL) + ){ + sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); } - if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ - return 0; - } - pPg = pager_lookup(pPager, pgno); - if( pPg==0 ) return 0; - page_ref(pPg); + return pPg; } @@ -3258,48 +3217,35 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ ** removed. */ int sqlite3PagerUnref(DbPage *pPg){ - - /* Decrement the reference count for this page - */ - assert( pPg->nRef>0 ); - pPg->nRef--; - REFINFO(pPg); - - CHECK_PAGE(pPg); - - /* When the number of references to a page reach 0, call the - ** destructor and add the page to the freelist. - */ - if( pPg->nRef==0 ){ - Pager *pPager; - pPager = pPg->pPager; - pPg->pNextFree = 0; - pPg->pPrevFree = pPager->pLast; - pPager->pLast = pPg; - if( pPg->pPrevFree ){ - pPg->pPrevFree->pNextFree = pPg; - }else{ - pPager->pFirst = pPg; - } - if( pPg->needSync==0 && pPager->pFirstSynced==0 ){ - pPager->pFirstSynced = pPg; - } - if( pPager->xDestructor ){ - pPager->xDestructor(pPg, pPager->pageSize); - } - - /* When all pages reach the freelist, drop the read lock from - ** the database file. - */ - pPager->nRef--; - assert( pPager->nRef>=0 ); - if( pPager->nRef==0 && (!pPager->exclusiveMode || pPager->journalOff>0) ){ - pagerUnlockAndRollback(pPager); - } + if( pPg ){ + Pager *pPager = pPg->pPager; + sqlite3PcacheRelease(pPg); + pagerUnlockIfUnused(pPager); } return SQLITE_OK; } +/* +** If the main journal file has already been opened, ensure that the +** sub-journal file is open too. If the main journal is not open, +** this function is a no-op. +** +** SQLITE_OK is returned if everything goes according to plan. An +** SQLITE_IOERR_XXX error code is returned if the call to +** sqlite3OsOpen() fails. +*/ +static int openSubJournal(Pager *pPager){ + int rc = SQLITE_OK; + if( pPager->journalOpen && !pPager->sjfd->pMethods ){ + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ + sqlite3MemJournalOpen(pPager->sjfd); + }else{ + rc = sqlite3PagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL); + } + } + return rc; +} + /* ** Create a journal file for pPager. There should already be a RESERVED ** or EXCLUSIVE lock on the database file when this routine is called. @@ -3308,51 +3254,66 @@ int sqlite3PagerUnref(DbPage *pPg){ ** write lock if anything goes wrong. */ static int pager_open_journal(Pager *pPager){ + sqlite3_vfs *pVfs = pPager->pVfs; + int flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_EXCLUSIVE|SQLITE_OPEN_CREATE); + int rc; - assert( !MEMDB ); assert( pPager->state>=PAGER_RESERVED ); - assert( pPager->journalOpen==0 ); assert( pPager->useJournal ); - assert( pPager->aInJournal==0 ); - sqlite3PagerPagecount(pPager); - pPager->aInJournal = sqliteMalloc( pPager->dbSize/8 + 1 ); - if( pPager->aInJournal==0 ){ + assert( pPager->pInJournal==0 ); + sqlite3PagerPagecount(pPager, 0); + pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); + if( pPager->pInJournal==0 ){ rc = SQLITE_NOMEM; goto failed_to_open_journal; } - rc = sqlite3OsOpenExclusive(pPager->zJournal, &pPager->jfd, - pPager->tempFile); - assert( rc!=SQLITE_OK || pPager->jfd ); - pPager->journalOff = 0; - pPager->setMaster = 0; - pPager->journalHdr = 0; - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ){ - sqlite3OsDelete(pPager->zJournal); + + if( pPager->journalOpen==0 ){ + if( pPager->tempFile ){ + flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + }else{ + flags |= (SQLITE_OPEN_MAIN_JOURNAL); + } + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ + sqlite3MemJournalOpen(pPager->jfd); + rc = SQLITE_OK; + }else{ +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + rc = sqlite3JournalOpen( + pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) + ); +#else + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); +#endif + } + assert( rc!=SQLITE_OK || pPager->jfd->pMethods ); + pPager->journalOff = 0; + pPager->setMaster = 0; + pPager->journalHdr = 0; + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ){ + sqlite3OsDelete(pVfs, pPager->zJournal, 0); + } + goto failed_to_open_journal; } - goto failed_to_open_journal; } - sqlite3OsSetFullSync(pPager->jfd, pPager->full_fsync); - sqlite3OsSetFullSync(pPager->fd, pPager->full_fsync); - sqlite3OsOpenDirectory(pPager->jfd, pPager->zDirectory); pPager->journalOpen = 1; pPager->journalStarted = 0; pPager->needSync = 0; - pPager->alwaysRollback = 0; pPager->nRec = 0; if( pPager->errCode ){ rc = pPager->errCode; goto failed_to_open_journal; } - pPager->origDbSize = pPager->dbSize; + pPager->dbOrigSize = pPager->dbSize; rc = writeJournalHdr(pPager); - if( pPager->stmtAutoopen && rc==SQLITE_OK ){ - rc = sqlite3PagerStmtBegin(pPager); + if( pPager->nSavepoint && rc==SQLITE_OK ){ + rc = openSubJournal(pPager); } - if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ - rc = pager_end_transaction(pPager); + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && rc!=SQLITE_IOERR_NOMEM ){ + rc = pager_end_transaction(pPager, 0); if( rc==SQLITE_OK ){ rc = SQLITE_FULL; } @@ -3360,8 +3321,8 @@ static int pager_open_journal(Pager *pPager){ return rc; failed_to_open_journal: - sqliteFree(pPager->aInJournal); - pPager->aInJournal = 0; + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; return rc; } @@ -3398,42 +3359,40 @@ int sqlite3PagerBegin(DbPage *pPg, int exFlag){ assert( pPg->nRef>0 ); assert( pPager->state!=PAGER_UNLOCK ); if( pPager->state==PAGER_SHARED ){ - assert( pPager->aInJournal==0 ); - if( MEMDB ){ - pPager->state = PAGER_EXCLUSIVE; - pPager->origDbSize = pPager->dbSize; - }else{ - rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); - if( rc==SQLITE_OK ){ - pPager->state = PAGER_RESERVED; - if( exFlag ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - } - } - if( rc!=SQLITE_OK ){ - return rc; - } - pPager->dirtyCache = 0; - PAGERTRACE2("TRANSACTION %d\n", PAGERID(pPager)); - if( pPager->useJournal && !pPager->tempFile ){ - rc = pager_open_journal(pPager); + assert( pPager->pInJournal==0 ); + assert( !MEMDB ); + rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); + if( rc==SQLITE_OK ){ + pPager->state = PAGER_RESERVED; + if( exFlag ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } } + if( rc!=SQLITE_OK ){ + return rc; + } + pPager->dirtyCache = 0; + PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager))); + if( pPager->useJournal && !pPager->tempFile + && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + rc = pager_open_journal(pPager); + } }else if( pPager->journalOpen && pPager->journalOff==0 ){ - /* This happens when the pager was in exclusive-access mode last + /* This happens when the pager was in exclusive-access mode the last ** time a (read or write) transaction was successfully concluded ** by this connection. Instead of deleting the journal file it was - ** kept open and truncated to 0 bytes. + ** kept open and either was truncated to 0 bytes or its header was + ** overwritten with zeros. */ assert( pPager->nRec==0 ); - assert( pPager->origDbSize==0 ); - assert( pPager->aInJournal==0 ); - sqlite3PagerPagecount(pPager); - pPager->aInJournal = sqliteMalloc( pPager->dbSize/8 + 1 ); - if( !pPager->aInJournal ){ + assert( pPager->dbOrigSize==0 ); + assert( pPager->pInJournal==0 ); + sqlite3PagerPagecount(pPager, 0); + pPager->pInJournal = sqlite3BitvecCreate( pPager->dbSize ); + if( !pPager->pInJournal ){ rc = SQLITE_NOMEM; }else{ - pPager->origDbSize = pPager->dbSize; + pPager->dbOrigSize = pPager->dbSize; rc = writeJournalHdr(pPager); } } @@ -3441,42 +3400,6 @@ int sqlite3PagerBegin(DbPage *pPg, int exFlag){ return rc; } -/* -** Make a page dirty. Set its dirty flag and add it to the dirty -** page list. -*/ -static void makeDirty(PgHdr *pPg){ - if( pPg->dirty==0 ){ - Pager *pPager = pPg->pPager; - pPg->dirty = 1; - pPg->pDirty = pPager->pDirty; - if( pPager->pDirty ){ - pPager->pDirty->pPrevDirty = pPg; - } - pPg->pPrevDirty = 0; - pPager->pDirty = pPg; - } -} - -/* -** Make a page clean. Clear its dirty bit and remove it from the -** dirty page list. -*/ -static void makeClean(PgHdr *pPg){ - if( pPg->dirty ){ - pPg->dirty = 0; - if( pPg->pDirty ){ - pPg->pDirty->pPrevDirty = pPg->pPrevDirty; - } - if( pPg->pPrevDirty ){ - pPg->pPrevDirty->pDirty = pPg->pDirty; - }else{ - pPg->pPager->pDirty = pPg->pDirty; - } - } -} - - /* ** Mark a data page as writeable. The page is written into the journal ** if it is not there already. This routine must be called before making @@ -3495,7 +3418,7 @@ static void makeClean(PgHdr *pPg){ ** reset. */ static int pager_write(PgHdr *pPg){ - void *pData = PGHDR_TO_DATA(pPg); + void *pData = pPg->pData; Pager *pPager = pPg->pPager; int rc = SQLITE_OK; @@ -3528,9 +3451,10 @@ static int pager_write(PgHdr *pPg){ /* Mark the page as dirty. If the page has already been written ** to the journal then we can return right away. */ - makeDirty(pPg); - if( pPg->inJournal && (pageInStatement(pPg) || pPager->stmtInUse==0) ){ + sqlite3PcacheMakeDirty(pPg); + if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){ pPager->dirtyCache = 1; + pPager->dbModified = 1; }else{ /* If we get this far, it means that the page needs to be @@ -3546,76 +3470,84 @@ static int pager_write(PgHdr *pPg){ return rc; } assert( pPager->state>=PAGER_RESERVED ); - if( !pPager->journalOpen && pPager->useJournal ){ + if( !pPager->journalOpen && pPager->useJournal + && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ rc = pager_open_journal(pPager); if( rc!=SQLITE_OK ) return rc; } - assert( pPager->journalOpen || !pPager->useJournal ); pPager->dirtyCache = 1; + pPager->dbModified = 1; /* The transaction journal now exists and we have a RESERVED or an ** EXCLUSIVE lock on the main database file. Write the current page to ** the transaction journal if it is not there already. */ - if( !pPg->inJournal && (pPager->useJournal || MEMDB) ){ - if( (int)pPg->pgno <= pPager->origDbSize ){ - int szPg; - if( MEMDB ){ - PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager); - PAGERTRACE3("JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno); - assert( pHist->pOrig==0 ); - pHist->pOrig = sqliteMallocRaw( pPager->pageSize ); - if( pHist->pOrig ){ - memcpy(pHist->pOrig, PGHDR_TO_DATA(pPg), pPager->pageSize); - } - }else{ - u32 cksum, saved; - char *pData2, *pEnd; - /* We should never write to the journal file the page that - ** contains the database locks. The following assert verifies - ** that we do not. */ - assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); - pData2 = CODEC2(pPager, pData, pPg->pgno, 7); - cksum = pager_cksum(pPager, (u8*)pData2); - pEnd = pData2 + pPager->pageSize; - pData2 -= 4; - saved = *(u32*)pEnd; - put32bits(pEnd, cksum); - szPg = pPager->pageSize+8; - put32bits(pData2, pPg->pgno); - rc = sqlite3OsWrite(pPager->jfd, pData2, szPg); - IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, - pPager->journalOff, szPg)); - PAGER_INCR(sqlite3_pager_writej_count); - pPager->journalOff += szPg; - PAGERTRACE5("JOURNAL %d page %d needSync=%d hash(%08x)\n", - PAGERID(pPager), pPg->pgno, pPg->needSync, pager_pagehash(pPg)); - *(u32*)pEnd = saved; + if( !pageInJournal(pPg) && pPager->journalOpen ){ + if( pPg->pgno<=pPager->dbOrigSize ){ + u32 cksum; + char *pData2; - /* An error has occured writing to the journal file. The - ** transaction will be rolled back by the layer above. - */ - if( rc!=SQLITE_OK ){ - return rc; - } + /* We should never write to the journal file the page that + ** contains the database locks. The following assert verifies + ** that we do not. */ + assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); + pData2 = CODEC2(pPager, pData, pPg->pgno, 7); + cksum = pager_cksum(pPager, (u8*)pData2); + rc = write32bits(pPager->jfd, pPager->journalOff, pPg->pgno); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, + pPager->journalOff + 4); + pPager->journalOff += pPager->pageSize+4; + } + if( rc==SQLITE_OK ){ + rc = write32bits(pPager->jfd, pPager->journalOff, cksum); + pPager->journalOff += 4; + } + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, + pPager->journalOff, pPager->pageSize)); + PAGER_INCR(sqlite3_pager_writej_count); + PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); - pPager->nRec++; - assert( pPager->aInJournal!=0 ); - pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7); - pPg->needSync = !pPager->noSync; - if( pPager->stmtInUse ){ - pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7); - } + /* Even if an IO or diskfull error occurred while journalling the + ** page in the block above, set the need-sync flag for the page. + ** Otherwise, when the transaction is rolled back, the logic in + ** playback_one_page() will think that the page needs to be restored + ** in the database file. And if an IO error occurs while doing so, + ** then corruption may follow. + */ + if( !pPager->noSync ){ + pPg->flags |= PGHDR_NEED_SYNC; + pPager->needSync = 1; + } + + /* An error has occured writing to the journal file. The + ** transaction will be rolled back by the layer above. + */ + if( rc!=SQLITE_OK ){ + return rc; + } + + pPager->nRec++; + assert( pPager->pInJournal!=0 ); + rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + rc |= addToSavepointBitvecs(pPager, pPg->pgno); + if( rc!=SQLITE_OK ){ + assert( rc==SQLITE_NOMEM ); + return rc; } }else{ - pPg->needSync = !pPager->journalStarted && !pPager->noSync; - PAGERTRACE4("APPEND %d page %d needSync=%d\n", - PAGERID(pPager), pPg->pgno, pPg->needSync); + if( !pPager->journalStarted && !pPager->noSync ){ + pPg->flags |= PGHDR_NEED_SYNC; + pPager->needSync = 1; + } + PAGERTRACE(("APPEND %d page %d needSync=%d\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); } - if( pPg->needSync ){ - pPager->needSync = 1; - } - pPg->inJournal = 1; } /* If the statement journal is open and the page is not in it, @@ -3623,41 +3555,17 @@ static int pager_write(PgHdr *pPg){ ** the statement journal format differs from the standard journal format ** in that it omits the checksums and the header. */ - if( pPager->stmtInUse - && !pageInStatement(pPg) - && (int)pPg->pgno<=pPager->stmtSize - ){ - assert( pPg->inJournal || (int)pPg->pgno>pPager->origDbSize ); - if( MEMDB ){ - PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager); - assert( pHist->pStmt==0 ); - pHist->pStmt = sqliteMallocRaw( pPager->pageSize ); - if( pHist->pStmt ){ - memcpy(pHist->pStmt, PGHDR_TO_DATA(pPg), pPager->pageSize); - } - PAGERTRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno); - page_add_to_stmt_list(pPg); - }else{ - char *pData2 = CODEC2(pPager, pData, pPg->pgno, 7)-4; - put32bits(pData2, pPg->pgno); - rc = sqlite3OsWrite(pPager->stfd, pData2, pPager->pageSize+4); - PAGERTRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno); - if( rc!=SQLITE_OK ){ - return rc; - } - pPager->stmtNRec++; - assert( pPager->aInStmt!=0 ); - pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7); - } + if( subjRequiresPage(pPg) ){ + rc = subjournalPage(pPg); } } /* Update the database size and return. */ assert( pPager->state>=PAGER_SHARED ); - if( pPager->dbSize<(int)pPg->pgno ){ + if( pPager->dbSizepgno ){ pPager->dbSize = pPg->pgno; - if( !MEMDB && pPager->dbSize==PENDING_BYTE/pPager->pageSize ){ + if( pPager->dbSize==(PAGER_MJ_PGNO(pPager)-1) ){ pPager->dbSize++; } } @@ -3681,15 +3589,17 @@ int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); - if( !MEMDB && nPagePerSector>1 ){ + if( nPagePerSector>1 ){ Pgno nPageCount; /* Total number of pages in database file */ Pgno pg1; /* First page of the sector pPg is located on. */ int nPage; /* Number of pages starting at pg1 to journal */ int ii; + int needSync = 0; /* Set the doNotSync flag to 1. This is because we cannot allow a journal ** header to be written between the pages journaled by this function. */ + assert( !MEMDB ); assert( pPager->doNotSync==0 ); pPager->doNotSync = 1; @@ -3699,7 +3609,7 @@ int sqlite3PagerWrite(DbPage *pDbPage){ */ pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; - nPageCount = sqlite3PagerPagecount(pPager); + sqlite3PagerPagecount(pPager, (int *)&nPageCount); if( pPg->pgno>nPageCount ){ nPage = (pPg->pgno - pg1)+1; }else if( (pg1+nPagePerSector-1)>nPageCount ){ @@ -3713,20 +3623,45 @@ int sqlite3PagerWrite(DbPage *pDbPage){ for(ii=0; iiaInJournal || pg==pPg->pgno || - pg>pPager->origDbSize || !(pPager->aInJournal[pg/8]&(1<<(pg&7))) - ) { + PgHdr *pPage; + if( pg==pPg->pgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ if( pg!=PAGER_MJ_PGNO(pPager) ){ - PgHdr *pPage; rc = sqlite3PagerGet(pPager, pg, &pPage); if( rc==SQLITE_OK ){ rc = pager_write(pPage); + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + assert(pPager->needSync); + } sqlite3PagerUnref(pPage); } } + }else if( (pPage = pager_lookup(pPager, pg))!=0 ){ + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnref(pPage); } } + /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages + ** starting at pg1, then it needs to be set for all of them. Because + ** writing to any of these nPage pages may damage the others, the + ** journal file must contain sync()ed copies of all of them + ** before any of them can be written out to the database file. + */ + if( needSync ){ + assert( !MEMDB && pPager->noSync==0 ); + for(ii=0; iiflags |= PGHDR_NEED_SYNC; + sqlite3PagerUnref(pPage); + } + } + assert(pPager->needSync); + } + assert( pPager->doNotSync==1 ); pPager->doNotSync = 0; }else{ @@ -3742,35 +3677,16 @@ int sqlite3PagerWrite(DbPage *pDbPage){ */ #ifndef NDEBUG int sqlite3PagerIswriteable(DbPage *pPg){ - return pPg->dirty; -} -#endif - -#ifndef SQLITE_OMIT_VACUUM -/* -** Replace the content of a single page with the information in the third -** argument. -*/ -int sqlite3PagerOverwrite(Pager *pPager, Pgno pgno, void *pData){ - PgHdr *pPg; - int rc; - - rc = sqlite3PagerGet(pPager, pgno, &pPg); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerWrite(pPg); - if( rc==SQLITE_OK ){ - memcpy(sqlite3PagerGetData(pPg), pData, pPager->pageSize); - } - sqlite3PagerUnref(pPg); - } - return rc; + return pPg->flags&PGHDR_DIRTY; } #endif /* ** A call to this routine tells the pager that it is not necessary to ** write the information on page pPg back to the disk, even though -** that page might be marked as dirty. +** that page might be marked as dirty. This happens, for example, when +** the page has been added as a leaf of the freelist and so its +** content no longer matters. ** ** The overlying software layer calls this routine when all of the data ** on the given page is unused. The pager marks the page as clean so @@ -3780,26 +3696,37 @@ int sqlite3PagerOverwrite(Pager *pPager, Pgno pgno, void *pData){ ** sqlite3PagerDontRollback() below, more than double the speed ** of large INSERT operations and quadruple the speed of large DELETEs. ** -** When this routine is called, set the alwaysRollback flag to true. -** Subsequent calls to sqlite3PagerDontRollback() for the same page -** will thereafter be ignored. This is necessary to avoid a problem -** where a page with data is added to the freelist during one part of -** a transaction then removed from the freelist during a later part -** of the same transaction and reused for some other purpose. When it -** is first added to the freelist, this routine is called. When reused, -** the sqlite3PagerDontRollback() routine is called. But because the -** page contains critical data, we still need to be sure it gets -** rolled back in spite of the sqlite3PagerDontRollback() call. +** When this routine is called, set the bit corresponding to pDbPage in +** the Pager.pAlwaysRollback bitvec. Subsequent calls to +** sqlite3PagerDontRollback() for the same page will thereafter be ignored. +** This is necessary to avoid a problem where a page with data is added to +** the freelist during one part of a transaction then removed from the +** freelist during a later part of the same transaction and reused for some +** other purpose. When it is first added to the freelist, this routine is +** called. When reused, the sqlite3PagerDontRollback() routine is called. +** But because the page contains critical data, we still need to be sure it +** gets rolled back in spite of the sqlite3PagerDontRollback() call. */ -void sqlite3PagerDontWrite(DbPage *pDbPage){ +int sqlite3PagerDontWrite(DbPage *pDbPage){ PgHdr *pPg = pDbPage; Pager *pPager = pPg->pPager; + int rc; - if( MEMDB ) return; - pPg->alwaysRollback = 1; - if( pPg->dirty && !pPager->stmtInUse ){ + if( pPg->pgno>pPager->dbOrigSize ){ + return SQLITE_OK; + } + if( pPager->pAlwaysRollback==0 ){ + assert( pPager->pInJournal ); + pPager->pAlwaysRollback = sqlite3BitvecCreate(pPager->dbOrigSize); + if( !pPager->pAlwaysRollback ){ + return SQLITE_NOMEM; + } + } + rc = sqlite3BitvecSet(pPager->pAlwaysRollback, pPg->pgno); + + if( rc==SQLITE_OK && (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ assert( pPager->state>=PAGER_SHARED ); - if( pPager->dbSize==(int)pPg->pgno && pPager->origDbSizedbSize ){ + if( pPager->dbSize==pPg->pgno && pPager->dbOrigSizedbSize ){ /* If this pages is the last page in the file and the file has grown ** during the current transaction, then do NOT mark the page as clean. ** When the database file grows, we must make sure that the last page @@ -3809,14 +3736,15 @@ void sqlite3PagerDontWrite(DbPage *pDbPage){ ** corruption during the next transaction. */ }else{ - PAGERTRACE3("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager)); + PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) - makeClean(pPg); + pPg->flags |= PGHDR_DONT_WRITE; #ifdef SQLITE_CHECK_PAGES pPg->pageHash = pager_pagehash(pPg); #endif } } + return rc; } /* @@ -3832,29 +3760,59 @@ void sqlite3PagerDontWrite(DbPage *pDbPage){ */ void sqlite3PagerDontRollback(DbPage *pPg){ Pager *pPager = pPg->pPager; + TESTONLY( int rc; ) /* Return value from sqlite3BitvecSet() */ assert( pPager->state>=PAGER_RESERVED ); - if( pPager->journalOpen==0 ) return; - if( pPg->alwaysRollback || pPager->alwaysRollback || MEMDB ) return; - if( !pPg->inJournal && (int)pPg->pgno <= pPager->origDbSize ){ - assert( pPager->aInJournal!=0 ); - pPager->aInJournal[pPg->pgno/8] |= 1<<(pPg->pgno&7); - pPg->inJournal = 1; - pPg->needRead = 0; - if( pPager->stmtInUse ){ - pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7); - } - PAGERTRACE3("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager)); - IOTRACE(("GARBAGE %p %d\n", pPager, pPg->pgno)) - } - if( pPager->stmtInUse - && !pageInStatement(pPg) - && (int)pPg->pgno<=pPager->stmtSize + + /* If the journal file is not open, or DontWrite() has been called on + ** this page (DontWrite() sets the Pager.pAlwaysRollback bit), then this + ** function is a no-op. + */ + if( pPager->journalOpen==0 + || sqlite3BitvecTest(pPager->pAlwaysRollback, pPg->pgno) + || pPg->pgno>pPager->dbOrigSize ){ - assert( pPg->inJournal || (int)pPg->pgno>pPager->origDbSize ); - assert( pPager->aInStmt!=0 ); - pPager->aInStmt[pPg->pgno/8] |= 1<<(pPg->pgno&7); + return; } + +#ifdef SQLITE_SECURE_DELETE + if( sqlite3BitvecTest(pPager->pInJournal, pPg->pgno)!=0 + || pPg->pgno>pPager->dbOrigSize ){ + return; + } +#endif + + /* If SECURE_DELETE is disabled, then there is no way that this + ** routine can be called on a page for which sqlite3PagerDontWrite() + ** has not been previously called during the same transaction. + ** And if DontWrite() has previously been called, the following + ** conditions must be met. + ** + ** (Later:) Not true. If the database is corrupted by having duplicate + ** pages on the freelist (ex: corrupt9.test) then the following is not + ** necessarily true: + */ + /* assert( !pPg->inJournal && (int)pPg->pgno <= pPager->dbOrigSize ); */ + + assert( pPager->pInJournal!=0 ); + pPg->flags &= ~PGHDR_NEED_READ; + + /* Failure to set the bits in the InJournal bit-vectors is benign. + ** It merely means that we might do some extra work to journal a page + ** that does not need to be journaled. Nevertheless, be sure to test the + ** case where a malloc error occurs while trying to set a bit in a + ** bit vector. + */ + sqlite3BeginBenignMalloc(); + TESTONLY( rc = ) sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); + testcase( rc==SQLITE_NOMEM ); + TESTONLY( rc = ) addToSavepointBitvecs(pPager, pPg->pgno); + testcase( rc==SQLITE_NOMEM ); + sqlite3EndBenignMalloc(); + + + PAGERTRACE(("DONT_ROLLBACK page %d of %d\n", pPg->pgno, PAGERID(pPager))); + IOTRACE(("GARBAGE %p %d\n", pPager, pPg->pgno)) } @@ -3862,27 +3820,58 @@ void sqlite3PagerDontRollback(DbPage *pPg){ ** This routine is called to increment the database file change-counter, ** stored at byte 24 of the pager file. */ -static int pager_incr_changecounter(Pager *pPager){ +static int pager_incr_changecounter(Pager *pPager, int isDirect){ PgHdr *pPgHdr; u32 change_counter; - int rc; + int rc = SQLITE_OK; - if( !pPager->changeCountDone ){ +#ifndef SQLITE_ENABLE_ATOMIC_WRITE + assert( isDirect==0 ); /* isDirect is only true for atomic writes */ +#endif + if( !pPager->changeCountDone && pPager->dbSize>0 ){ /* Open page 1 of the file for writing. */ rc = sqlite3PagerGet(pPager, 1, &pPgHdr); if( rc!=SQLITE_OK ) return rc; - rc = sqlite3PagerWrite(pPgHdr); - if( rc!=SQLITE_OK ) return rc; - + + if( !isDirect ){ + rc = sqlite3PagerWrite(pPgHdr); + if( rc!=SQLITE_OK ){ + sqlite3PagerUnref(pPgHdr); + return rc; + } + } + /* Increment the value just read and write it back to byte 24. */ change_counter = sqlite3Get4byte((u8*)pPager->dbFileVers); change_counter++; - put32bits(((char*)PGHDR_TO_DATA(pPgHdr))+24, change_counter); + put32bits(((char*)pPgHdr->pData)+24, change_counter); + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + if( isDirect && pPager->fd->pMethods ){ + const void *zBuf = pPgHdr->pData; + assert( pPager->dbFileSize>0 ); + rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); + } +#endif + /* Release the page reference. */ sqlite3PagerUnref(pPgHdr); pPager->changeCountDone = 1; } - return SQLITE_OK; + return rc; +} + +/* +** Sync the pager file to disk. +*/ +int sqlite3PagerSync(Pager *pPager){ + int rc; + if( MEMDB ){ + rc = SQLITE_OK; + }else{ + rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); + } + return rc; } /* @@ -3899,21 +3888,83 @@ static int pager_incr_changecounter(Pager *pPager){ ** Note that if zMaster==NULL, this does not overwrite a previous value ** passed to an sqlite3PagerCommitPhaseOne() call. ** -** If parameter nTrunc is non-zero, then the pager file is truncated to -** nTrunc pages (this is used by auto-vacuum databases). +** If the final parameter - noSync - is true, then the database file itself +** is not synced. The caller must call sqlite3PagerSync() directly to +** sync the database file before calling CommitPhaseTwo() to delete the +** journal file in this case. */ -int sqlite3PagerCommitPhaseOne(Pager *pPager, const char *zMaster, Pgno nTrunc){ +int sqlite3PagerCommitPhaseOne( + Pager *pPager, + const char *zMaster, + int noSync +){ int rc = SQLITE_OK; - PAGERTRACE4("DATABASE SYNC: File=%s zMaster=%s nTrunc=%d\n", - pPager->zFilename, zMaster, nTrunc); + if( pPager->errCode ){ + return pPager->errCode; + } + + /* If no changes have been made, we can leave the transaction early. + */ + if( pPager->dbModified==0 && + (pPager->journalMode!=PAGER_JOURNALMODE_DELETE || + pPager->exclusiveMode!=0) ){ + assert( pPager->dirtyCache==0 || pPager->journalOpen==0 ); + return SQLITE_OK; + } + + PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", + pPager->zFilename, zMaster, pPager->dbSize)); /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is a no-op. */ if( pPager->state!=PAGER_SYNCED && !MEMDB && pPager->dirtyCache ){ PgHdr *pPg; - assert( pPager->journalOpen ); + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + /* The atomic-write optimization can be used if all of the + ** following are true: + ** + ** + The file-system supports the atomic-write property for + ** blocks of size page-size, and + ** + This commit is not part of a multi-file transaction, and + ** + Exactly one page has been modified and store in the journal file. + ** + ** If the optimization can be used, then the journal file will never + ** be created for this transaction. + */ + int useAtomicWrite; + pPg = sqlite3PcacheDirtyList(pPager->pPCache); + useAtomicWrite = ( + !zMaster && + pPager->journalOpen && + pPager->journalOff==jrnlBufferSize(pPager) && + pPager->dbSize>=pPager->dbFileSize && + (pPg==0 || pPg->pDirty==0) + ); + assert( pPager->journalOpen || pPager->journalMode==PAGER_JOURNALMODE_OFF ); + if( useAtomicWrite ){ + /* Update the nRec field in the journal file. */ + int offset = pPager->journalHdr + sizeof(aJournalMagic); + assert(pPager->nRec==1); + rc = write32bits(pPager->jfd, offset, pPager->nRec); + + /* Update the db file change counter. The following call will modify + ** the in-memory representation of page 1 to include the updated + ** change counter and then write page 1 directly to the database + ** file. Because of the atomic-write property of the host file-system, + ** this is safe. + */ + if( rc==SQLITE_OK ){ + rc = pager_incr_changecounter(pPager, 1); + } + }else{ + rc = sqlite3JournalCreate(pPager->jfd); + } + + if( !useAtomicWrite && rc==SQLITE_OK ) +#endif /* If a master journal file name has already been written to the ** journal file, then no sync is required. This happens when it is @@ -3922,62 +3973,74 @@ int sqlite3PagerCommitPhaseOne(Pager *pPager, const char *zMaster, Pgno nTrunc){ ** transaction the m-j name will have already been written. */ if( !pPager->setMaster ){ - rc = pager_incr_changecounter(pPager); + rc = pager_incr_changecounter(pPager, 0); if( rc!=SQLITE_OK ) goto sync_exit; + if( pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ #ifndef SQLITE_OMIT_AUTOVACUUM - if( nTrunc!=0 ){ - /* If this transaction has made the database smaller, then all pages - ** being discarded by the truncation must be written to the journal - ** file. - */ - Pgno i; - int iSkip = PAGER_MJ_PGNO(pPager); - for( i=nTrunc+1; i<=pPager->origDbSize; i++ ){ - if( !(pPager->aInJournal[i/8] & (1<<(i&7))) && i!=iSkip ){ - rc = sqlite3PagerGet(pPager, i, &pPg); - if( rc!=SQLITE_OK ) goto sync_exit; - rc = sqlite3PagerWrite(pPg); - sqlite3PagerUnref(pPg); - if( rc!=SQLITE_OK ) goto sync_exit; - } - } + if( pPager->dbSizedbOrigSize ){ + /* If this transaction has made the database smaller, then all pages + ** being discarded by the truncation must be written to the journal + ** file. + */ + Pgno i; + Pgno iSkip = PAGER_MJ_PGNO(pPager); + Pgno dbSize = pPager->dbSize; + pPager->dbSize = pPager->dbOrigSize; + for( i=dbSize+1; i<=pPager->dbOrigSize; i++ ){ + if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){ + rc = sqlite3PagerGet(pPager, i, &pPg); + if( rc!=SQLITE_OK ) goto sync_exit; + rc = sqlite3PagerWrite(pPg); + sqlite3PagerUnref(pPg); + if( rc!=SQLITE_OK ) goto sync_exit; + } + } + pPager->dbSize = dbSize; + } +#endif + rc = writeMasterJournal(pPager, zMaster); + if( rc!=SQLITE_OK ) goto sync_exit; + rc = syncJournal(pPager); } -#endif - rc = writeMasterJournal(pPager, zMaster); - if( rc!=SQLITE_OK ) goto sync_exit; - rc = syncJournal(pPager); - if( rc!=SQLITE_OK ) goto sync_exit; } - -#ifndef SQLITE_OMIT_AUTOVACUUM - if( nTrunc!=0 ){ - rc = sqlite3PagerTruncate(pPager, nTrunc); - if( rc!=SQLITE_OK ) goto sync_exit; - } -#endif + if( rc!=SQLITE_OK ) goto sync_exit; /* Write all dirty pages to the database file */ - pPg = pager_get_all_dirty_pages(pPager); + pPg = sqlite3PcacheDirtyList(pPager->pPCache); rc = pager_write_pagelist(pPg); - if( rc!=SQLITE_OK ) goto sync_exit; - pPager->pDirty = 0; + if( rc!=SQLITE_OK ){ + assert( rc!=SQLITE_IOERR_BLOCKED ); + /* The error might have left the dirty list all fouled up here, + ** but that does not matter because if the if the dirty list did + ** get corrupted, then the transaction will roll back and + ** discard the dirty list. There is an assert in + ** pager_get_all_dirty_pages() that verifies that no attempt + ** is made to use an invalid dirty list. + */ + goto sync_exit; + } + sqlite3PcacheCleanAll(pPager->pPCache); + + if( pPager->dbSizedbFileSize ){ + assert( pPager->state>=PAGER_EXCLUSIVE ); + rc = pager_truncate(pPager, pPager->dbSize); + if( rc!=SQLITE_OK ) goto sync_exit; + } /* Sync the database file. */ - if( !pPager->noSync ){ - rc = sqlite3OsSync(pPager->fd, 0); + if( !pPager->noSync && !noSync ){ + rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); } IOTRACE(("DBSYNC %p\n", pPager)) pPager->state = PAGER_SYNCED; - }else if( MEMDB && nTrunc!=0 ){ - rc = sqlite3PagerTruncate(pPager, nTrunc); } sync_exit: if( rc==SQLITE_IOERR_BLOCKED ){ /* pager_incr_changecounter() may attempt to obtain an exclusive * lock to spill the cache and return IOERR_BLOCKED. But since - * there is no chance the cache is inconsistent, it's + * there is no chance the cache is inconsistent, it is * better to return SQLITE_BUSY. */ rc = SQLITE_BUSY; @@ -3994,8 +4057,7 @@ sync_exit: ** is returned. */ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ - int rc; - PgHdr *pPg; + int rc = SQLITE_OK; if( pPager->errCode ){ return pPager->errCode; @@ -4003,36 +4065,17 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ if( pPager->statedirty = 0; - pPg->inJournal = 0; - pHist->inStmt = 0; - pPg->needSync = 0; - pHist->pPrevStmt = pHist->pNextStmt = 0; - pPg = pPg->pDirty; - } - pPager->pDirty = 0; -#ifndef NDEBUG - for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ - PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager); - assert( !pPg->alwaysRollback ); - assert( !pHist->pOrig ); - assert( !pHist->pStmt ); - } -#endif - pPager->pStmt = 0; - pPager->state = PAGER_SHARED; + if( pPager->dbModified==0 && + (pPager->journalMode!=PAGER_JOURNALMODE_DELETE || + pPager->exclusiveMode!=0) ){ + assert( pPager->dirtyCache==0 || pPager->journalOpen==0 ); return SQLITE_OK; } - assert( pPager->journalOpen || !pPager->dirtyCache ); - assert( pPager->state==PAGER_SYNCED || !pPager->dirtyCache ); - rc = pager_end_transaction(pPager); - return pager_error(pPager, rc); + PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); + assert( pPager->state==PAGER_SYNCED || MEMDB || !pPager->dirtyCache ); + rc = pager_end_transaction(pPager, pPager->setMaster); + rc = pager_error(pPager, rc); + return rc; } /* @@ -4048,80 +4091,45 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** SQLITE_OK is returned. */ int sqlite3PagerRollback(Pager *pPager){ - int rc; - PAGERTRACE2("ROLLBACK %d\n", PAGERID(pPager)); - if( MEMDB ){ - PgHdr *p; - for(p=pPager->pAll; p; p=p->pNextAll){ - PgHistory *pHist; - assert( !p->alwaysRollback ); - if( !p->dirty ){ - assert( !((PgHistory *)PGHDR_TO_HIST(p, pPager))->pOrig ); - assert( !((PgHistory *)PGHDR_TO_HIST(p, pPager))->pStmt ); - continue; - } - - pHist = PGHDR_TO_HIST(p, pPager); - if( pHist->pOrig ){ - memcpy(PGHDR_TO_DATA(p), pHist->pOrig, pPager->pageSize); - PAGERTRACE3("ROLLBACK-PAGE %d of %d\n", p->pgno, PAGERID(pPager)); - }else{ - PAGERTRACE3("PAGE %d is clean on %d\n", p->pgno, PAGERID(pPager)); - } - clearHistory(pHist); - p->dirty = 0; - p->inJournal = 0; - pHist->inStmt = 0; - pHist->pPrevStmt = pHist->pNextStmt = 0; - if( pPager->xReiniter ){ - pPager->xReiniter(p, pPager->pageSize); - } - } - pPager->pDirty = 0; - pPager->pStmt = 0; - pPager->dbSize = pPager->origDbSize; - pager_truncate_cache(pPager); - pPager->stmtInUse = 0; - pPager->state = PAGER_SHARED; - return SQLITE_OK; - } - + int rc = SQLITE_OK; + PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); if( !pPager->dirtyCache || !pPager->journalOpen ){ - rc = pager_end_transaction(pPager); - return rc; - } - - if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ + rc = pager_end_transaction(pPager, pPager->setMaster); + }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ if( pPager->state>=PAGER_EXCLUSIVE ){ pager_playback(pPager, 0); } - return pPager->errCode; - } - if( pPager->state==PAGER_RESERVED ){ - int rc2; - rc = pager_playback(pPager, 0); - rc2 = pager_end_transaction(pPager); - if( rc==SQLITE_OK ){ - rc = rc2; - } + rc = pPager->errCode; }else{ - rc = pager_playback(pPager, 0); - } - /* pager_reset(pPager); */ - pPager->dbSize = -1; + if( pPager->state==PAGER_RESERVED ){ + int rc2; + rc = pager_playback(pPager, 0); + rc2 = pager_end_transaction(pPager, pPager->setMaster); + if( rc==SQLITE_OK ){ + rc = rc2; + } + }else{ + rc = pager_playback(pPager, 0); + } - /* If an error occurs during a ROLLBACK, we can no longer trust the pager - ** cache. So call pager_error() on the way out to make any error - ** persistent. - */ - return pager_error(pPager, rc); + if( !MEMDB ){ + pPager->dbSizeValid = 0; + } + + /* If an error occurs during a ROLLBACK, we can no longer trust the pager + ** cache. So call pager_error() on the way out to make any error + ** persistent. + */ + rc = pager_error(pPager, rc); + } + return rc; } /* ** Return TRUE if the database file is opened read-only. Return FALSE ** if the database is (in theory) writable. */ -int sqlite3PagerIsreadonly(Pager *pPager){ +u8 sqlite3PagerIsreadonly(Pager *pPager){ return pPager->readOnly; } @@ -4129,7 +4137,14 @@ int sqlite3PagerIsreadonly(Pager *pPager){ ** Return the number of references to the pager. */ int sqlite3PagerRefcount(Pager *pPager){ - return pPager->nRef; + return sqlite3PcacheRefCount(pPager->pPCache); +} + +/* +** Return the number of references to the specified page. +*/ +int sqlite3PagerPageRefcount(DbPage *pPage){ + return sqlite3PcachePageRefcount(pPage); } #ifdef SQLITE_TEST @@ -4138,10 +4153,10 @@ int sqlite3PagerRefcount(Pager *pPager){ */ int *sqlite3PagerStats(Pager *pPager){ static int a[11]; - a[0] = pPager->nRef; - a[1] = pPager->nPage; - a[2] = pPager->mxPage; - a[3] = pPager->dbSize; + a[0] = sqlite3PcacheRefCount(pPager->pPCache); + a[1] = sqlite3PcachePagecount(pPager->pPCache); + a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); + a[3] = pPager->dbSizeValid ? (int) pPager->dbSize : -1; a[4] = pPager->state; a[5] = pPager->errCode; a[6] = pPager->nHit; @@ -4151,122 +4166,105 @@ int *sqlite3PagerStats(Pager *pPager){ a[10] = pPager->nWrite; return a; } +int sqlite3PagerIsMemdb(Pager *pPager){ + return MEMDB; +} #endif /* -** Set the statement rollback point. -** -** This routine should be called with the transaction journal already -** open. A new statement journal is created that can be used to rollback -** changes of a single SQL command within a larger transaction. +** Ensure that there are at least nSavepoint savepoints open. */ -int sqlite3PagerStmtBegin(Pager *pPager){ - int rc; - assert( !pPager->stmtInUse ); - assert( pPager->state>=PAGER_SHARED ); - assert( pPager->dbSize>=0 ); - PAGERTRACE2("STMT-BEGIN %d\n", PAGERID(pPager)); - if( MEMDB ){ - pPager->stmtInUse = 1; - pPager->stmtSize = pPager->dbSize; - return SQLITE_OK; - } - if( !pPager->journalOpen ){ - pPager->stmtAutoopen = 1; - return SQLITE_OK; - } - assert( pPager->journalOpen ); - pPager->aInStmt = sqliteMalloc( pPager->dbSize/8 + 1 ); - if( pPager->aInStmt==0 ){ - /* sqlite3OsLock(pPager->fd, SHARED_LOCK); */ - return SQLITE_NOMEM; - } -#ifndef NDEBUG - rc = sqlite3OsFileSize(pPager->jfd, &pPager->stmtJSize); - if( rc ) goto stmt_begin_failed; - assert( pPager->stmtJSize == pPager->journalOff ); -#endif - pPager->stmtJSize = pPager->journalOff; - pPager->stmtSize = pPager->dbSize; - pPager->stmtHdrOff = 0; - pPager->stmtCksum = pPager->cksumInit; - if( !pPager->stmtOpen ){ - rc = sqlite3PagerOpentemp(&pPager->stfd); - if( rc ) goto stmt_begin_failed; - pPager->stmtOpen = 1; - pPager->stmtNRec = 0; - } - pPager->stmtInUse = 1; - return SQLITE_OK; - -stmt_begin_failed: - if( pPager->aInStmt ){ - sqliteFree(pPager->aInStmt); - pPager->aInStmt = 0; +int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ + int rc = SQLITE_OK; + + if( nSavepoint>pPager->nSavepoint && pPager->useJournal ){ + int ii; + PagerSavepoint *aNew; + + /* Either there is no active journal or the sub-journal is open or + ** the journal is always stored in memory */ + assert( pPager->nSavepoint==0 || pPager->sjfd->pMethods || + pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); + + /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM + ** if the allocation fails. Otherwise, zero the new portion in case a + ** malloc failure occurs while populating it in the for(...) loop below. + */ + aNew = (PagerSavepoint *)sqlite3Realloc( + pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint + ); + if( !aNew ){ + return SQLITE_NOMEM; + } + memset(&aNew[pPager->nSavepoint], 0, + (nSavepoint - pPager->nSavepoint) * sizeof(PagerSavepoint) + ); + pPager->aSavepoint = aNew; + ii = pPager->nSavepoint; + pPager->nSavepoint = nSavepoint; + + /* Populate the PagerSavepoint structures just allocated. */ + for(/* no-op */; iidbSizeValid ); + aNew[ii].nOrig = pPager->dbSize; + if( pPager->journalOpen && pPager->journalOff>0 ){ + aNew[ii].iOffset = pPager->journalOff; + }else{ + aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); + } + aNew[ii].iSubRec = pPager->stmtNRec; + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); + if( !aNew[ii].pInSavepoint ){ + return SQLITE_NOMEM; + } + } + + /* Open the sub-journal, if it is not already opened. */ + rc = openSubJournal(pPager); } + return rc; } /* -** Commit a statement. -*/ -int sqlite3PagerStmtCommit(Pager *pPager){ - if( pPager->stmtInUse ){ - PgHdr *pPg, *pNext; - PAGERTRACE2("STMT-COMMIT %d\n", PAGERID(pPager)); - if( !MEMDB ){ - sqlite3OsSeek(pPager->stfd, 0); - /* sqlite3OsTruncate(pPager->stfd, 0); */ - sqliteFree( pPager->aInStmt ); - pPager->aInStmt = 0; - }else{ - for(pPg=pPager->pStmt; pPg; pPg=pNext){ - PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager); - pNext = pHist->pNextStmt; - assert( pHist->inStmt ); - pHist->inStmt = 0; - pHist->pPrevStmt = pHist->pNextStmt = 0; - sqliteFree(pHist->pStmt); - pHist->pStmt = 0; - } - } - pPager->stmtNRec = 0; - pPager->stmtInUse = 0; - pPager->pStmt = 0; - } - pPager->stmtAutoopen = 0; - return SQLITE_OK; -} +** Parameter op is always either SAVEPOINT_ROLLBACK or SAVEPOINT_RELEASE. +** If it is SAVEPOINT_RELEASE, then release and destroy the savepoint with +** index iSavepoint. If it is SAVEPOINT_ROLLBACK, then rollback all changes +** that have occured since savepoint iSavepoint was created. +** +** In either case, all savepoints with an index greater than iSavepoint +** are destroyed. +** +** If there are less than (iSavepoint+1) active savepoints when this +** function is called it is a no-op. +*/ +int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ + int rc = SQLITE_OK; -/* -** Rollback a statement. -*/ -int sqlite3PagerStmtRollback(Pager *pPager){ - int rc; - if( pPager->stmtInUse ){ - PAGERTRACE2("STMT-ROLLBACK %d\n", PAGERID(pPager)); - if( MEMDB ){ - PgHdr *pPg; - PgHistory *pHist; - for(pPg=pPager->pStmt; pPg; pPg=pHist->pNextStmt){ - pHist = PGHDR_TO_HIST(pPg, pPager); - if( pHist->pStmt ){ - memcpy(PGHDR_TO_DATA(pPg), pHist->pStmt, pPager->pageSize); - sqliteFree(pHist->pStmt); - pHist->pStmt = 0; - } - } - pPager->dbSize = pPager->stmtSize; - pager_truncate_cache(pPager); - rc = SQLITE_OK; - }else{ - rc = pager_stmt_playback(pPager); + assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); + + if( iSavepointnSavepoint ){ + int ii; + int nNew = iSavepoint + (op==SAVEPOINT_ROLLBACK); + for(ii=nNew; iinSavepoint; ii++){ + sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); + } + pPager->nSavepoint = nNew; + + if( op==SAVEPOINT_ROLLBACK && pPager->jfd->pMethods ){ + PagerSavepoint *pSavepoint = (nNew==0)?0:&pPager->aSavepoint[nNew-1]; + rc = pagerPlaybackSavepoint(pPager, pSavepoint); + assert(rc!=SQLITE_DONE); + } + + /* If this is a release of the outermost savepoint, truncate + ** the sub-journal. */ + if( nNew==0 && op==SAVEPOINT_RELEASE && pPager->sjfd->pMethods ){ + assert( rc==SQLITE_OK ); + rc = sqlite3OsTruncate(pPager->sjfd, 0); + pPager->stmtNRec = 0; } - sqlite3PagerStmtCommit(pPager); - }else{ - rc = SQLITE_OK; } - pPager->stmtAutoopen = 0; return rc; } @@ -4277,6 +4275,22 @@ const char *sqlite3PagerFilename(Pager *pPager){ return pPager->zFilename; } +/* +** Return the VFS structure for the pager. +*/ +const sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){ + return pPager->pVfs; +} + +/* +** Return the file handle for the database file associated +** with the pager. This might return NULL if the file has +** not yet been opened. +*/ +sqlite3_file *sqlite3PagerFile(Pager *pPager){ + return pPager->fd; +} + /* ** Return the directory of the database file. */ @@ -4315,11 +4329,11 @@ void sqlite3PagerSetCodec( #ifndef SQLITE_OMIT_AUTOVACUUM /* -** Move the page pPg to location pgno in the file. +** Move the page pPg to location pgno in the file. ** ** There must be no references to the page previously located at ** pgno (which we call pPgOld) though that page is allowed to be -** in cache. If the page previous located at pgno is not already +** in cache. If the page previously located at pgno is not already ** in the rollback journal, it is not put there by by this routine. ** ** References to the page pPg remain valid. Updating any @@ -4330,85 +4344,112 @@ void sqlite3PagerSetCodec( ** required that a statement transaction was not active, but this restriction ** has been removed (CREATE INDEX needs to move a page when a statement ** transaction is active). +** +** If the fourth argument, isCommit, is non-zero, then this page is being +** moved as part of a database reorganization just before the transaction +** is being committed. In this case, it is guaranteed that the database page +** pPg refers to will not be written to again within this transaction. */ -int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno){ +int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ PgHdr *pPgOld; /* The page being overwritten. */ - int h; Pgno needSyncPgno = 0; + int rc; assert( pPg->nRef>0 ); - PAGERTRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", - PAGERID(pPager), pPg->pgno, pPg->needSync, pgno); + /* If the page being moved is dirty and has not been saved by the latest + ** savepoint, then save the current contents of the page into the + ** sub-journal now. This is required to handle the following scenario: + ** + ** BEGIN; + ** + ** SAVEPOINT one; + ** + ** ROLLBACK TO one; + ** + ** If page X were not written to the sub-journal here, it would not + ** be possible to restore its contents when the "ROLLBACK TO one" + ** statement were processed. + */ + if( pPg->flags&PGHDR_DIRTY + && subjRequiresPage(pPg) + && SQLITE_OK!=(rc = subjournalPage(pPg)) + ){ + return rc; + } + + PAGERTRACE(("MOVE %d page %d (needSync=%d) moves to %d\n", + PAGERID(pPager), pPg->pgno, (pPg->flags&PGHDR_NEED_SYNC)?1:0, pgno)); IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno)) pager_get_content(pPg); - if( pPg->needSync ){ + + /* If the journal needs to be sync()ed before page pPg->pgno can + ** be written to, store pPg->pgno in local variable needSyncPgno. + ** + ** If the isCommit flag is set, there is no need to remember that + ** the journal needs to be sync()ed before database page pPg->pgno + ** can be written to. The caller has already promised not to write to it. + */ + if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ needSyncPgno = pPg->pgno; - assert( pPg->inJournal || (int)pgno>pPager->origDbSize ); - assert( pPg->dirty ); + assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); + assert( pPg->flags&PGHDR_DIRTY ); assert( pPager->needSync ); } - /* Unlink pPg from it's hash-chain */ - unlinkHashChain(pPager, pPg); - /* If the cache contains a page with page-number pgno, remove it - ** from it's hash chain. Also, if the PgHdr.needSync was set for + ** from its hash chain. Also, if the PgHdr.needSync was set for ** page pgno before the 'move' operation, it needs to be retained ** for the page moved there. */ - pPg->needSync = 0; + pPg->flags &= ~PGHDR_NEED_SYNC; pPgOld = pager_lookup(pPager, pgno); + assert( !pPgOld || pPgOld->nRef==1 ); if( pPgOld ){ - assert( pPgOld->nRef==0 ); - unlinkHashChain(pPager, pPgOld); - makeClean(pPgOld); - pPg->needSync = pPgOld->needSync; - }else{ - pPg->needSync = 0; - } - if( pPager->aInJournal && (int)pgno<=pPager->origDbSize ){ - pPg->inJournal = (pPager->aInJournal[pgno/8] & (1<<(pgno&7)))!=0; - }else{ - pPg->inJournal = 0; - assert( pPg->needSync==0 || (int)pgno>pPager->origDbSize ); + pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); } - /* Change the page number for pPg and insert it into the new hash-chain. */ - assert( pgno!=0 ); - pPg->pgno = pgno; - h = pgno & (pPager->nHash-1); - if( pPager->aHash[h] ){ - assert( pPager->aHash[h]->pPrevHash==0 ); - pPager->aHash[h]->pPrevHash = pPg; + sqlite3PcacheMove(pPg, pgno); + if( pPgOld ){ + sqlite3PcacheDrop(pPgOld); } - pPg->pNextHash = pPager->aHash[h]; - pPager->aHash[h] = pPg; - pPg->pPrevHash = 0; - makeDirty(pPg); + sqlite3PcacheMakeDirty(pPg); pPager->dirtyCache = 1; + pPager->dbModified = 1; if( needSyncPgno ){ /* If needSyncPgno is non-zero, then the journal file needs to be ** sync()ed before any data is written to database file page needSyncPgno. ** Currently, no such page exists in the page-cache and the - ** Pager.aInJournal bit has been set. This needs to be remedied by loading - ** the page into the pager-cache and setting the PgHdr.needSync flag. + ** "is journaled" bitvec flag has been set. This needs to be remedied by + ** loading the page into the pager-cache and setting the PgHdr.needSync + ** flag. + ** + ** If the attempt to load the page into the page-cache fails, (due + ** to a malloc() or IO failure), clear the bit in the pInJournal[] + ** array. Otherwise, if the page is loaded and written again in + ** this transaction, it may be written to the database file before + ** it is synced into the journal file. This way, it may end up in + ** the journal file twice, but that is not a problem. ** ** The sqlite3PagerGet() call may cause the journal to sync. So make ** sure the Pager.needSync flag is set too. */ - int rc; PgHdr *pPgHdr; assert( pPager->needSync ); rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + if( pPager->pInJournal && needSyncPgno<=pPager->dbOrigSize ){ + sqlite3BitvecClear(pPager->pInJournal, needSyncPgno); + } + return rc; + } pPager->needSync = 1; - pPgHdr->needSync = 1; - pPgHdr->inJournal = 1; - makeDirty(pPgHdr); + assert( pPager->noSync==0 && !MEMDB ); + pPgHdr->flags |= PGHDR_NEED_SYNC; + sqlite3PcacheMakeDirty(pPgHdr); sqlite3PagerUnref(pPgHdr); } @@ -4420,7 +4461,8 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno){ ** Return a pointer to the data for the specified page. */ void *sqlite3PagerGetData(DbPage *pPg){ - return PGHDR_TO_DATA(pPg); + assert( pPg->nRef>0 || pPg->pPager->memDb ); + return pPg->pData; } /* @@ -4429,7 +4471,7 @@ void *sqlite3PagerGetData(DbPage *pPg){ */ void *sqlite3PagerGetExtra(DbPage *pPg){ Pager *pPager = pPg->pPager; - return (pPager?PGHDR_TO_EXTRA(pPg, pPager):0); + return (pPager?pPg->pExtra:0); } /* @@ -4449,34 +4491,52 @@ int sqlite3PagerLockingMode(Pager *pPager, int eMode){ assert( PAGER_LOCKINGMODE_QUERY<0 ); assert( PAGER_LOCKINGMODE_NORMAL>=0 && PAGER_LOCKINGMODE_EXCLUSIVE>=0 ); if( eMode>=0 && !pPager->tempFile ){ - pPager->exclusiveMode = eMode; + pPager->exclusiveMode = (u8)eMode; } return (int)pPager->exclusiveMode; } -#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* -** Return the current state of the file lock for the given pager. -** The return value is one of NO_LOCK, SHARED_LOCK, RESERVED_LOCK, -** PENDING_LOCK, or EXCLUSIVE_LOCK. +** Get/set the journal-mode for this pager. Parameter eMode must be one of: +** +** PAGER_JOURNALMODE_QUERY +** PAGER_JOURNALMODE_DELETE +** PAGER_JOURNALMODE_TRUNCATE +** PAGER_JOURNALMODE_PERSIST +** PAGER_JOURNALMODE_OFF +** +** If the parameter is not _QUERY, then the journal-mode is set to the +** value specified. +** +** The returned indicate the current (possibly updated) +** journal-mode. */ -int sqlite3PagerLockstate(Pager *pPager){ - return sqlite3OsLockState(pPager->fd); -} -#endif - -#ifdef SQLITE_DEBUG -/* -** Print a listing of all referenced pages and their ref count. -*/ -void sqlite3PagerRefdump(Pager *pPager){ - PgHdr *pPg; - for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ - if( pPg->nRef<=0 ) continue; - sqlite3DebugPrintf("PAGE %3d addr=%p nRef=%d\n", - pPg->pgno, PGHDR_TO_DATA(pPg), pPg->nRef); +int sqlite3PagerJournalMode(Pager *pPager, int eMode){ + if( !MEMDB ){ + assert( eMode==PAGER_JOURNALMODE_QUERY + || eMode==PAGER_JOURNALMODE_DELETE + || eMode==PAGER_JOURNALMODE_TRUNCATE + || eMode==PAGER_JOURNALMODE_PERSIST + || eMode==PAGER_JOURNALMODE_OFF + || eMode==PAGER_JOURNALMODE_MEMORY ); + assert( PAGER_JOURNALMODE_QUERY<0 ); + if( eMode>=0 ){ + pPager->journalMode = (u8)eMode; + }else{ + assert( eMode==PAGER_JOURNALMODE_QUERY ); + } } + return (int)pPager->journalMode; +} + +/* +** Get/set the size-limit used for persistent journal files. +*/ +i64 sqlite3PagerJournalSizeLimit(Pager *pPager, i64 iLimit){ + if( iLimit>=-1 ){ + pPager->journalSizeLimit = iLimit; + } + return pPager->journalSizeLimit; } -#endif #endif /* SQLITE_OMIT_DISKIO */ diff --git a/pager.h b/pager.h index ca193da..9d466d8 100644 --- a/pager.h +++ b/pager.h @@ -13,17 +13,25 @@ ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** -** @(#) $Id: pager.h,v 1.61 2007/05/08 21:45:28 drh Exp $ +** @(#) $Id: pager.h,v 1.93 2009/01/07 15:18:21 danielk1977 Exp $ */ #ifndef _PAGER_H_ #define _PAGER_H_ +/* +** If defined as non-zero, auto-vacuum is enabled by default. Otherwise +** it must be turned on for each database using "PRAGMA auto_vacuum = 1". +*/ +#ifndef SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT + #define SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT -1 +#endif + /* ** The type used to represent a page number. The first page in a file ** is called page 1. 0 is used to represent "not a page". */ -typedef unsigned int Pgno; +typedef u32 Pgno; /* ** Each open file is managed by a separate instance of the "Pager" structure. @@ -50,16 +58,24 @@ typedef struct PgHdr DbPage; #define PAGER_LOCKINGMODE_NORMAL 0 #define PAGER_LOCKINGMODE_EXCLUSIVE 1 +/* +** Valid values for the second argument to sqlite3PagerJournalMode(). +*/ +#define PAGER_JOURNALMODE_QUERY -1 +#define PAGER_JOURNALMODE_DELETE 0 /* Commit by deleting journal file */ +#define PAGER_JOURNALMODE_PERSIST 1 /* Commit by zeroing journal header */ +#define PAGER_JOURNALMODE_OFF 2 /* Journal omitted. */ +#define PAGER_JOURNALMODE_TRUNCATE 3 /* Commit by truncating journal */ +#define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ + /* ** See source code comments for a detailed description of the following ** routines: */ -int sqlite3PagerOpen(Pager **ppPager, const char *zFilename, - int nExtra, int flags); -void sqlite3PagerSetBusyhandler(Pager*, BusyHandler *pBusyHandler); -void sqlite3PagerSetDestructor(Pager*, void(*)(DbPage*,int)); -void sqlite3PagerSetReiniter(Pager*, void(*)(DbPage*,int)); -int sqlite3PagerSetPagesize(Pager*, int); +int sqlite3PagerOpen(sqlite3_vfs *, Pager **ppPager, const char*, int,int,int); +void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); +void sqlite3PagerSetReiniter(Pager*, void(*)(DbPage*)); +int sqlite3PagerSetPagesize(Pager*, u16*); int sqlite3PagerMaxPageCount(Pager*, int); int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); void sqlite3PagerSetCachesize(Pager*, int); @@ -67,35 +83,41 @@ int sqlite3PagerClose(Pager *pPager); int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); #define sqlite3PagerGet(A,B,C) sqlite3PagerAcquire(A,B,C,0) DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); +int sqlite3PagerPageRefcount(DbPage*); int sqlite3PagerRef(DbPage*); int sqlite3PagerUnref(DbPage*); int sqlite3PagerWrite(DbPage*); -int sqlite3PagerOverwrite(Pager *pPager, Pgno pgno, void*); -int sqlite3PagerPagecount(Pager*); -int sqlite3PagerTruncate(Pager*,Pgno); +int sqlite3PagerPagecount(Pager*, int*); int sqlite3PagerBegin(DbPage*, int exFlag); -int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, Pgno); +int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); int sqlite3PagerCommitPhaseTwo(Pager*); int sqlite3PagerRollback(Pager*); -int sqlite3PagerIsreadonly(Pager*); -int sqlite3PagerStmtBegin(Pager*); -int sqlite3PagerStmtCommit(Pager*); -int sqlite3PagerStmtRollback(Pager*); +u8 sqlite3PagerIsreadonly(Pager*); void sqlite3PagerDontRollback(DbPage*); -void sqlite3PagerDontWrite(DbPage*); +int sqlite3PagerDontWrite(DbPage*); int sqlite3PagerRefcount(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int); const char *sqlite3PagerFilename(Pager*); +const sqlite3_vfs *sqlite3PagerVfs(Pager*); +sqlite3_file *sqlite3PagerFile(Pager*); const char *sqlite3PagerDirname(Pager*); const char *sqlite3PagerJournalname(Pager*); int sqlite3PagerNosync(Pager*); -int sqlite3PagerMovepage(Pager*,DbPage*,Pgno); +int sqlite3PagerMovepage(Pager*,DbPage*,Pgno,int); void *sqlite3PagerGetData(DbPage *); void *sqlite3PagerGetExtra(DbPage *); int sqlite3PagerLockingMode(Pager *, int); +int sqlite3PagerJournalMode(Pager *, int); +i64 sqlite3PagerJournalSizeLimit(Pager *, i64); +void *sqlite3PagerTempSpace(Pager*); +int sqlite3PagerSync(Pager *pPager); -#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) && !defined(SQLITE_OMIT_DISKIO) - int sqlite3PagerReleaseMemory(int); +int sqlite3PagerOpenSavepoint(Pager *pPager, int n); +int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); + +#ifndef SQLITE_OMIT_AUTOVACUUM + void sqlite3PagerTruncateImage(Pager*,Pgno); + Pgno sqlite3PagerImageSize(Pager *); #endif #ifdef SQLITE_HAS_CODEC @@ -107,14 +129,10 @@ int sqlite3PagerLockingMode(Pager *, int); int sqlite3PagerIswriteable(DbPage*); #endif -#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - int sqlite3PagerLockstate(Pager*); -#endif - #ifdef SQLITE_TEST int *sqlite3PagerStats(Pager*); void sqlite3PagerRefdump(Pager*); - int pager3_refinfo_enable; + int sqlite3PagerIsMemdb(Pager*); #endif #ifdef SQLITE_TEST diff --git a/parse.c b/parse.c index 33366e2..c31f74f 100644 --- a/parse.c +++ b/parse.c @@ -1,13 +1,12 @@ /* Driver template for the LEMON parser generator. ** The author disclaims copyright to this source code. */ -/* First off, code is include which follows the "include" declaration -** in the input file. */ +/* First off, code is included that follows the "include" declaration +** in the input grammar file. */ #include -#line 56 "parse.y" +#line 53 "parse.y" #include "sqliteInt.h" -#include "parse.h" /* ** An instance of this structure holds information about the @@ -43,7 +42,7 @@ struct TrigEvent { int a; IdList * b; }; */ struct AttachKey { int type; Token key; }; -#line 48 "parse.c" +#line 46 "parse.c" /* Next is all token values, in a form suitable for use by makeheaders. ** This section will be null unless lemon is run with the -m switch. */ @@ -94,26 +93,24 @@ struct AttachKey { int type; Token key; }; ** defined, then do no error processing. */ #define YYCODETYPE unsigned char -#define YYNOCODE 248 +#define YYNOCODE 251 #define YYACTIONTYPE unsigned short int -#define YYWILDCARD 59 +#define YYWILDCARD 62 #define sqlite3ParserTOKENTYPE Token typedef union { + int yyinit; sqlite3ParserTOKENTYPE yy0; - int yy46; - struct LikeOp yy72; - Expr* yy172; - ExprList* yy174; - Select* yy219; - struct LimitVal yy234; - TriggerStep* yy243; - struct TrigEvent yy370; - SrcList* yy373; - Expr * yy386; - struct {int value; int mask;} yy405; - Token yy410; - IdList* yy432; - int yy495; + struct LimitVal yy64; + Expr* yy122; + Select* yy159; + IdList* yy180; + struct {int value; int mask;} yy207; + struct LikeOp yy318; + TriggerStep* yy327; + SrcList* yy347; + int yy392; + struct TrigEvent yy410; + ExprList* yy442; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -122,16 +119,19 @@ typedef union { #define sqlite3ParserARG_PDECL ,Parse *pParse #define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse #define sqlite3ParserARG_STORE yypParser->pParse = pParse -#define YYNSTATE 586 -#define YYNRULE 311 -#define YYERRORSYMBOL 138 -#define YYERRSYMDT yy495 +#define YYNSTATE 610 +#define YYNRULE 319 #define YYFALLBACK 1 #define YY_NO_ACTION (YYNSTATE+YYNRULE+2) #define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) #define YY_ERROR_ACTION (YYNSTATE+YYNRULE) -/* Next are that tables used to determine what action to take based on the +/* The yyzerominor constant is used to initialize instances of +** YYMINORTYPE objects to zero. */ +static const YYMINORTYPE yyzerominor = { 0 }; + + +/* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement ** functions that take a state number and lookahead value and return an ** action integer. @@ -179,413 +179,429 @@ typedef union { ** yy_default[] Default action for each state. */ static const YYACTIONTYPE yy_action[] = { - /* 0 */ 290, 427, 888, 400, 888, 160, 181, 494, 67, 67, - /* 10 */ 67, 67, 551, 69, 69, 69, 69, 70, 70, 71, - /* 20 */ 71, 71, 72, 220, 510, 284, 517, 508, 80, 69, - /* 30 */ 69, 69, 69, 70, 70, 71, 71, 71, 72, 220, - /* 40 */ 536, 514, 478, 473, 66, 90, 301, 501, 500, 506, - /* 50 */ 506, 68, 68, 67, 67, 67, 67, 245, 69, 69, - /* 60 */ 69, 69, 70, 70, 71, 71, 71, 72, 220, 290, - /* 70 */ 567, 542, 494, 402, 9, 64, 168, 527, 478, 546, - /* 80 */ 69, 69, 69, 69, 70, 70, 71, 71, 71, 72, - /* 90 */ 220, 79, 457, 84, 153, 517, 508, 479, 244, 60, - /* 100 */ 70, 70, 71, 71, 71, 72, 220, 456, 452, 521, - /* 110 */ 521, 521, 290, 66, 90, 301, 501, 500, 506, 506, - /* 120 */ 68, 68, 67, 67, 67, 67, 311, 69, 69, 69, - /* 130 */ 69, 70, 70, 71, 71, 71, 72, 220, 517, 508, - /* 140 */ 89, 312, 519, 456, 452, 447, 211, 216, 539, 45, - /* 150 */ 426, 397, 396, 495, 174, 290, 66, 90, 301, 501, - /* 160 */ 500, 506, 506, 68, 68, 67, 67, 67, 67, 620, - /* 170 */ 69, 69, 69, 69, 70, 70, 71, 71, 71, 72, - /* 180 */ 220, 517, 508, 446, 292, 478, 129, 372, 536, 289, - /* 190 */ 329, 552, 475, 543, 176, 338, 304, 488, 486, 66, - /* 200 */ 90, 301, 501, 500, 506, 506, 68, 68, 67, 67, - /* 210 */ 67, 67, 377, 69, 69, 69, 69, 70, 70, 71, - /* 220 */ 71, 71, 72, 220, 523, 386, 437, 290, 544, 150, - /* 230 */ 308, 404, 410, 371, 238, 241, 544, 571, 22, 318, - /* 240 */ 385, 497, 497, 1, 211, 460, 22, 478, 579, 535, - /* 250 */ 456, 452, 166, 517, 508, 343, 346, 347, 361, 478, - /* 260 */ 210, 147, 580, 220, 364, 821, 348, 534, 533, 254, - /* 270 */ 82, 66, 90, 301, 501, 500, 506, 506, 68, 68, - /* 280 */ 67, 67, 67, 67, 379, 69, 69, 69, 69, 70, - /* 290 */ 70, 71, 71, 71, 72, 220, 290, 166, 282, 308, - /* 300 */ 343, 346, 347, 211, 167, 116, 253, 341, 276, 342, - /* 310 */ 204, 348, 456, 452, 479, 205, 378, 281, 318, 195, - /* 320 */ 497, 497, 517, 508, 456, 452, 418, 358, 418, 531, - /* 330 */ 210, 163, 164, 165, 532, 71, 71, 71, 72, 220, - /* 340 */ 66, 90, 301, 501, 500, 506, 506, 68, 68, 67, - /* 350 */ 67, 67, 67, 536, 69, 69, 69, 69, 70, 70, - /* 360 */ 71, 71, 71, 72, 220, 196, 154, 282, 290, 211, - /* 370 */ 336, 466, 401, 112, 116, 253, 341, 276, 342, 204, - /* 380 */ 351, 311, 370, 465, 245, 478, 281, 166, 516, 515, - /* 390 */ 343, 346, 347, 334, 517, 508, 464, 562, 395, 431, - /* 400 */ 434, 348, 2, 539, 46, 546, 161, 161, 390, 511, - /* 410 */ 509, 74, 66, 90, 301, 501, 500, 506, 506, 68, - /* 420 */ 68, 67, 67, 67, 67, 382, 69, 69, 69, 69, - /* 430 */ 70, 70, 71, 71, 71, 72, 220, 290, 512, 414, - /* 440 */ 434, 408, 411, 331, 213, 466, 311, 478, 578, 573, - /* 450 */ 456, 452, 311, 388, 584, 577, 79, 465, 84, 153, - /* 460 */ 156, 293, 293, 517, 508, 72, 220, 281, 539, 46, - /* 470 */ 464, 468, 325, 193, 539, 45, 384, 427, 889, 277, - /* 480 */ 889, 66, 90, 301, 501, 500, 506, 506, 68, 68, - /* 490 */ 67, 67, 67, 67, 536, 69, 69, 69, 69, 70, - /* 500 */ 70, 71, 71, 71, 72, 220, 290, 311, 535, 207, - /* 510 */ 574, 472, 456, 452, 544, 151, 329, 514, 478, 311, - /* 520 */ 246, 328, 311, 581, 22, 79, 534, 84, 153, 539, - /* 530 */ 26, 132, 517, 508, 898, 133, 471, 403, 440, 2, - /* 540 */ 270, 539, 33, 564, 539, 45, 557, 565, 569, 536, - /* 550 */ 66, 90, 301, 501, 500, 506, 506, 68, 68, 67, - /* 560 */ 67, 67, 67, 563, 69, 69, 69, 69, 70, 70, - /* 570 */ 71, 71, 71, 72, 220, 290, 483, 483, 233, 271, - /* 580 */ 230, 478, 311, 456, 452, 311, 333, 423, 380, 210, - /* 590 */ 442, 422, 547, 263, 320, 488, 486, 318, 190, 497, - /* 600 */ 497, 517, 508, 360, 539, 50, 582, 539, 50, 441, - /* 610 */ 79, 454, 84, 153, 318, 300, 497, 497, 453, 66, - /* 620 */ 90, 301, 501, 500, 506, 506, 68, 68, 67, 67, - /* 630 */ 67, 67, 536, 69, 69, 69, 69, 70, 70, 71, - /* 640 */ 71, 71, 72, 220, 290, 337, 456, 452, 20, 86, - /* 650 */ 297, 311, 494, 306, 311, 218, 369, 158, 210, 232, - /* 660 */ 494, 311, 55, 311, 475, 321, 176, 539, 3, 279, - /* 670 */ 517, 508, 267, 539, 39, 221, 539, 36, 268, 365, - /* 680 */ 552, 245, 442, 539, 40, 539, 30, 290, 66, 90, - /* 690 */ 301, 501, 500, 506, 506, 68, 68, 67, 67, 67, - /* 700 */ 67, 311, 69, 69, 69, 69, 70, 70, 71, 71, - /* 710 */ 71, 72, 220, 517, 508, 206, 252, 494, 256, 586, - /* 720 */ 523, 386, 491, 539, 41, 494, 552, 552, 552, 552, - /* 730 */ 290, 66, 90, 301, 501, 500, 506, 506, 68, 68, - /* 740 */ 67, 67, 67, 67, 311, 69, 69, 69, 69, 70, - /* 750 */ 70, 71, 71, 71, 72, 220, 517, 508, 247, 479, - /* 760 */ 205, 294, 302, 303, 295, 316, 539, 38, 540, 433, - /* 770 */ 424, 416, 550, 290, 66, 88, 301, 501, 500, 506, - /* 780 */ 506, 68, 68, 67, 67, 67, 67, 311, 69, 69, - /* 790 */ 69, 69, 70, 70, 71, 71, 71, 72, 220, 517, - /* 800 */ 508, 18, 307, 120, 247, 247, 247, 247, 187, 539, - /* 810 */ 27, 436, 350, 132, 132, 450, 290, 339, 90, 301, - /* 820 */ 501, 500, 506, 506, 68, 68, 67, 67, 67, 67, - /* 830 */ 311, 69, 69, 69, 69, 70, 70, 71, 71, 71, - /* 840 */ 72, 220, 517, 508, 540, 522, 326, 85, 550, 430, - /* 850 */ 8, 193, 539, 115, 152, 317, 160, 558, 494, 132, - /* 860 */ 159, 5, 301, 501, 500, 506, 506, 68, 68, 67, - /* 870 */ 67, 67, 67, 435, 69, 69, 69, 69, 70, 70, - /* 880 */ 71, 71, 71, 72, 220, 77, 322, 311, 4, 311, - /* 890 */ 434, 311, 299, 239, 536, 132, 311, 75, 311, 57, - /* 900 */ 319, 77, 322, 311, 4, 428, 146, 81, 299, 539, - /* 910 */ 47, 539, 25, 539, 52, 311, 319, 324, 539, 97, - /* 920 */ 539, 96, 311, 494, 428, 539, 100, 473, 311, 224, - /* 930 */ 311, 505, 311, 324, 490, 572, 21, 539, 101, 476, - /* 940 */ 19, 85, 142, 473, 539, 111, 311, 63, 73, 222, - /* 950 */ 539, 113, 539, 114, 539, 34, 77, 314, 315, 231, - /* 960 */ 428, 527, 570, 63, 73, 413, 359, 203, 539, 24, - /* 970 */ 200, 549, 77, 314, 315, 77, 322, 527, 4, 211, - /* 980 */ 311, 537, 299, 311, 59, 298, 444, 396, 554, 258, - /* 990 */ 319, 189, 154, 521, 521, 521, 520, 518, 11, 560, - /* 1000 */ 260, 536, 539, 28, 536, 539, 54, 324, 559, 521, - /* 1010 */ 521, 521, 520, 518, 11, 117, 485, 473, 287, 175, - /* 1020 */ 421, 310, 477, 473, 265, 229, 235, 103, 368, 242, - /* 1030 */ 173, 368, 480, 311, 525, 526, 311, 63, 73, 487, - /* 1040 */ 186, 245, 280, 149, 148, 269, 77, 314, 315, 248, - /* 1050 */ 311, 527, 311, 402, 245, 539, 44, 527, 539, 16, - /* 1060 */ 266, 169, 482, 446, 536, 245, 443, 172, 345, 245, - /* 1070 */ 191, 245, 539, 98, 539, 42, 311, 245, 311, 438, - /* 1080 */ 311, 327, 240, 521, 521, 521, 520, 518, 11, 521, - /* 1090 */ 521, 521, 261, 219, 311, 376, 208, 461, 539, 35, - /* 1100 */ 539, 49, 539, 43, 311, 264, 496, 367, 197, 311, - /* 1110 */ 499, 311, 236, 375, 237, 241, 539, 51, 502, 291, - /* 1120 */ 311, 366, 311, 234, 211, 363, 539, 10, 311, 429, - /* 1130 */ 311, 539, 23, 539, 95, 228, 311, 355, 311, 250, - /* 1140 */ 255, 257, 539, 37, 539, 53, 259, 513, 381, 455, - /* 1150 */ 539, 31, 539, 32, 484, 392, 548, 144, 539, 29, - /* 1160 */ 539, 48, 323, 573, 275, 554, 459, 451, 528, 243, - /* 1170 */ 555, 185, 135, 107, 285, 87, 58, 529, 335, 582, - /* 1180 */ 583, 330, 362, 180, 225, 398, 374, 223, 125, 492, - /* 1190 */ 432, 124, 122, 406, 393, 81, 507, 119, 340, 126, - /* 1200 */ 188, 134, 561, 445, 305, 226, 356, 7, 409, 249, - /* 1210 */ 553, 313, 568, 198, 352, 556, 399, 131, 199, 585, - /* 1220 */ 354, 283, 201, 286, 391, 389, 296, 140, 463, 91, - /* 1230 */ 415, 419, 481, 458, 202, 209, 194, 217, 373, 212, - /* 1240 */ 130, 566, 78, 227, 332, 541, 273, 56, 575, 545, - /* 1250 */ 278, 530, 104, 99, 274, 62, 412, 61, 17, 170, - /* 1260 */ 504, 576, 102, 420, 621, 214, 121, 622, 309, 171, - /* 1270 */ 503, 498, 76, 470, 489, 474, 141, 155, 183, 6, - /* 1280 */ 469, 467, 12, 462, 439, 136, 83, 14, 137, 407, - /* 1290 */ 405, 215, 157, 65, 344, 105, 349, 251, 109, 118, - /* 1300 */ 93, 106, 162, 260, 353, 143, 394, 138, 357, 193, - /* 1310 */ 192, 110, 288, 449, 272, 184, 94, 92, 139, 13, - /* 1320 */ 425, 179, 417, 178, 123, 177, 127, 493, 524, 128, - /* 1330 */ 15, 108, 182, 538, 262, 383, 387, 145, 448, + /* 0 */ 304, 74, 581, 83, 164, 205, 587, 543, 86, 86, + /* 10 */ 86, 86, 350, 96, 96, 96, 96, 93, 93, 94, + /* 20 */ 94, 94, 91, 227, 188, 158, 532, 524, 88, 96, + /* 30 */ 96, 96, 96, 93, 93, 94, 94, 94, 91, 227, + /* 40 */ 489, 315, 505, 504, 92, 82, 309, 518, 516, 522, + /* 50 */ 522, 95, 95, 86, 86, 86, 86, 434, 96, 96, + /* 60 */ 96, 96, 93, 93, 94, 94, 94, 91, 227, 304, + /* 70 */ 450, 308, 543, 525, 540, 63, 96, 96, 96, 96, + /* 80 */ 93, 93, 94, 94, 94, 91, 227, 93, 93, 94, + /* 90 */ 94, 94, 91, 227, 586, 532, 524, 332, 596, 77, + /* 100 */ 239, 59, 20, 233, 226, 563, 543, 538, 538, 538, + /* 110 */ 182, 345, 269, 92, 82, 309, 518, 516, 522, 522, + /* 120 */ 95, 95, 86, 86, 86, 86, 592, 96, 96, 96, + /* 130 */ 96, 93, 93, 94, 94, 94, 91, 227, 304, 230, + /* 140 */ 489, 481, 308, 229, 550, 176, 114, 289, 340, 290, + /* 150 */ 325, 170, 451, 447, 480, 191, 607, 394, 262, 264, + /* 160 */ 152, 150, 406, 22, 532, 524, 416, 295, 379, 589, + /* 170 */ 450, 543, 604, 920, 540, 920, 301, 589, 293, 492, + /* 180 */ 453, 206, 92, 82, 309, 518, 516, 522, 522, 95, + /* 190 */ 95, 86, 86, 86, 86, 373, 96, 96, 96, 96, + /* 200 */ 93, 93, 94, 94, 94, 91, 227, 538, 538, 538, + /* 210 */ 230, 304, 599, 74, 153, 83, 164, 114, 289, 340, + /* 220 */ 290, 325, 170, 94, 94, 94, 91, 227, 278, 262, + /* 230 */ 474, 446, 494, 219, 588, 585, 171, 532, 524, 371, + /* 240 */ 378, 383, 588, 585, 281, 339, 505, 504, 546, 606, + /* 250 */ 388, 445, 555, 556, 90, 92, 82, 309, 518, 516, + /* 260 */ 522, 522, 95, 95, 86, 86, 86, 86, 466, 96, + /* 270 */ 96, 96, 96, 93, 93, 94, 94, 94, 91, 227, + /* 280 */ 322, 357, 171, 488, 304, 371, 378, 383, 171, 61, + /* 290 */ 154, 371, 378, 383, 231, 559, 388, 74, 357, 83, + /* 300 */ 164, 56, 388, 321, 551, 28, 240, 163, 226, 464, + /* 310 */ 532, 524, 324, 119, 558, 557, 346, 568, 511, 511, + /* 320 */ 490, 551, 49, 346, 550, 511, 511, 76, 92, 82, + /* 330 */ 309, 518, 516, 522, 522, 95, 95, 86, 86, 86, + /* 340 */ 86, 550, 96, 96, 96, 96, 93, 93, 94, 94, + /* 350 */ 94, 91, 227, 304, 349, 423, 451, 307, 109, 402, + /* 360 */ 234, 238, 357, 331, 560, 357, 566, 22, 341, 346, + /* 370 */ 229, 511, 511, 531, 530, 554, 226, 559, 284, 532, + /* 380 */ 524, 374, 387, 464, 490, 551, 49, 472, 551, 40, + /* 390 */ 151, 410, 159, 573, 527, 526, 558, 92, 82, 309, + /* 400 */ 518, 516, 522, 522, 95, 95, 86, 86, 86, 86, + /* 410 */ 165, 96, 96, 96, 96, 93, 93, 94, 94, 94, + /* 420 */ 91, 227, 304, 528, 91, 227, 550, 331, 353, 222, + /* 430 */ 521, 385, 348, 402, 234, 238, 400, 357, 157, 158, + /* 440 */ 357, 589, 565, 473, 229, 204, 318, 225, 532, 524, + /* 450 */ 60, 436, 205, 525, 543, 413, 424, 403, 226, 296, + /* 460 */ 551, 49, 420, 551, 40, 306, 92, 82, 309, 518, + /* 470 */ 516, 522, 522, 95, 95, 86, 86, 86, 86, 550, + /* 480 */ 96, 96, 96, 96, 93, 93, 94, 94, 94, 91, + /* 490 */ 227, 304, 479, 564, 461, 286, 494, 288, 357, 297, + /* 500 */ 357, 436, 334, 376, 217, 141, 588, 585, 374, 74, + /* 510 */ 357, 83, 164, 930, 130, 609, 2, 532, 524, 543, + /* 520 */ 17, 551, 43, 551, 50, 391, 525, 23, 272, 604, + /* 530 */ 921, 589, 921, 551, 43, 92, 82, 309, 518, 516, + /* 540 */ 522, 522, 95, 95, 86, 86, 86, 86, 592, 96, + /* 550 */ 96, 96, 96, 93, 93, 94, 94, 94, 91, 227, + /* 560 */ 304, 510, 281, 481, 429, 274, 249, 319, 242, 599, + /* 570 */ 142, 589, 459, 459, 218, 263, 480, 589, 338, 311, + /* 580 */ 589, 439, 440, 204, 1, 149, 532, 524, 589, 451, + /* 590 */ 392, 346, 169, 511, 511, 589, 588, 585, 221, 590, + /* 600 */ 22, 492, 359, 206, 92, 82, 309, 518, 516, 522, + /* 610 */ 522, 95, 95, 86, 86, 86, 86, 550, 96, 96, + /* 620 */ 96, 96, 93, 93, 94, 94, 94, 91, 227, 304, + /* 630 */ 397, 470, 208, 454, 475, 357, 588, 585, 469, 161, + /* 640 */ 161, 376, 588, 585, 256, 588, 585, 346, 277, 511, + /* 650 */ 511, 236, 196, 588, 585, 532, 524, 165, 551, 31, + /* 660 */ 588, 585, 458, 396, 459, 459, 550, 437, 401, 437, + /* 670 */ 595, 542, 303, 92, 82, 309, 518, 516, 522, 522, + /* 680 */ 95, 95, 86, 86, 86, 86, 550, 96, 96, 96, + /* 690 */ 96, 93, 93, 94, 94, 94, 91, 227, 304, 357, + /* 700 */ 442, 227, 421, 355, 441, 357, 543, 357, 179, 357, + /* 710 */ 506, 357, 306, 471, 357, 169, 386, 262, 610, 600, + /* 720 */ 360, 490, 551, 16, 532, 524, 551, 3, 551, 111, + /* 730 */ 551, 33, 551, 24, 551, 34, 190, 551, 51, 582, + /* 740 */ 579, 578, 92, 82, 309, 518, 516, 522, 522, 95, + /* 750 */ 95, 86, 86, 86, 86, 357, 96, 96, 96, 96, + /* 760 */ 93, 93, 94, 94, 94, 91, 227, 304, 357, 552, + /* 770 */ 550, 543, 357, 553, 357, 326, 357, 169, 551, 42, + /* 780 */ 357, 605, 606, 320, 357, 282, 357, 201, 211, 166, + /* 790 */ 287, 551, 112, 532, 524, 551, 37, 551, 98, 551, + /* 800 */ 54, 649, 552, 551, 52, 167, 553, 551, 113, 551, + /* 810 */ 35, 92, 82, 309, 518, 516, 522, 522, 95, 95, + /* 820 */ 86, 86, 86, 86, 357, 96, 96, 96, 96, 93, + /* 830 */ 93, 94, 94, 94, 91, 227, 304, 276, 399, 550, + /* 840 */ 155, 357, 509, 357, 21, 357, 343, 551, 55, 357, + /* 850 */ 598, 302, 412, 357, 601, 2, 426, 229, 494, 219, + /* 860 */ 600, 360, 532, 524, 551, 25, 551, 29, 551, 41, + /* 870 */ 525, 381, 551, 39, 550, 281, 551, 10, 276, 223, + /* 880 */ 92, 81, 309, 518, 516, 522, 522, 95, 95, 86, + /* 890 */ 86, 86, 86, 357, 96, 96, 96, 96, 93, 93, + /* 900 */ 94, 94, 94, 91, 227, 304, 357, 281, 281, 228, + /* 910 */ 357, 323, 357, 285, 357, 169, 551, 26, 357, 487, + /* 920 */ 512, 191, 495, 357, 70, 357, 409, 407, 460, 551, + /* 930 */ 46, 532, 524, 551, 30, 551, 32, 551, 48, 18, + /* 940 */ 276, 551, 47, 314, 310, 602, 551, 103, 551, 108, + /* 950 */ 82, 309, 518, 516, 522, 522, 95, 95, 86, 86, + /* 960 */ 86, 86, 357, 96, 96, 96, 96, 93, 93, 94, + /* 970 */ 94, 94, 91, 227, 304, 513, 276, 276, 19, 357, + /* 980 */ 146, 357, 517, 357, 603, 551, 97, 357, 8, 455, + /* 990 */ 356, 169, 357, 498, 498, 199, 545, 62, 70, 80, + /* 1000 */ 532, 524, 551, 102, 551, 100, 551, 99, 266, 452, + /* 1010 */ 551, 38, 368, 414, 336, 551, 45, 276, 362, 214, + /* 1020 */ 309, 518, 516, 522, 522, 95, 95, 86, 86, 86, + /* 1030 */ 86, 357, 96, 96, 96, 96, 93, 93, 94, 94, + /* 1040 */ 94, 91, 227, 71, 352, 237, 4, 357, 276, 369, + /* 1050 */ 316, 456, 276, 357, 551, 110, 357, 276, 344, 523, + /* 1060 */ 449, 572, 539, 357, 570, 294, 71, 352, 404, 4, + /* 1070 */ 551, 44, 351, 316, 260, 330, 551, 36, 370, 551, + /* 1080 */ 27, 344, 162, 435, 247, 489, 551, 53, 496, 478, + /* 1090 */ 520, 853, 462, 279, 246, 78, 583, 243, 330, 271, + /* 1100 */ 375, 292, 395, 438, 499, 84, 73, 476, 489, 465, + /* 1110 */ 533, 501, 258, 541, 72, 354, 358, 337, 493, 540, + /* 1120 */ 424, 298, 483, 497, 536, 147, 259, 549, 84, 73, + /* 1130 */ 116, 569, 203, 571, 57, 195, 580, 72, 354, 358, + /* 1140 */ 78, 367, 540, 500, 267, 363, 463, 115, 229, 425, + /* 1150 */ 432, 508, 538, 538, 538, 535, 534, 12, 172, 5, + /* 1160 */ 229, 187, 241, 245, 248, 251, 183, 273, 576, 104, + /* 1170 */ 128, 575, 364, 123, 390, 538, 538, 538, 535, 534, + /* 1180 */ 12, 125, 168, 327, 235, 448, 329, 71, 352, 574, + /* 1190 */ 4, 261, 193, 591, 316, 126, 393, 200, 342, 143, + /* 1200 */ 313, 268, 344, 185, 529, 464, 257, 312, 107, 175, + /* 1210 */ 561, 87, 202, 213, 58, 7, 333, 67, 594, 330, + /* 1220 */ 69, 431, 250, 79, 482, 244, 270, 366, 265, 489, + /* 1230 */ 135, 174, 275, 548, 428, 577, 427, 122, 133, 299, + /* 1240 */ 224, 215, 347, 467, 181, 415, 433, 216, 194, 84, + /* 1250 */ 73, 300, 412, 105, 398, 384, 254, 238, 72, 354, + /* 1260 */ 358, 305, 417, 540, 411, 408, 229, 418, 207, 291, + /* 1270 */ 405, 173, 101, 422, 584, 75, 280, 544, 232, 317, + /* 1280 */ 210, 132, 118, 129, 335, 593, 485, 184, 253, 380, + /* 1290 */ 562, 377, 136, 209, 160, 145, 538, 538, 538, 535, + /* 1300 */ 534, 12, 547, 650, 468, 127, 13, 382, 15, 514, + /* 1310 */ 502, 389, 144, 444, 443, 486, 191, 255, 66, 430, + /* 1320 */ 131, 121, 137, 106, 519, 65, 608, 89, 192, 85, + /* 1330 */ 283, 419, 134, 252, 457, 156, 140, 64, 148, 372, + /* 1340 */ 597, 120, 567, 6, 537, 212, 186, 503, 189, 177, + /* 1350 */ 328, 491, 651, 507, 365, 477, 197, 9, 14, 293, + /* 1360 */ 484, 178, 68, 138, 220, 180, 139, 515, 117, 198, + /* 1370 */ 11, 124, 931, 931, 931, 931, 931, 931, 931, 931, + /* 1380 */ 361, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 16, 19, 20, 160, 22, 21, 155, 23, 69, 70, - /* 10 */ 71, 72, 169, 74, 75, 76, 77, 78, 79, 80, - /* 20 */ 81, 82, 83, 84, 244, 245, 42, 43, 73, 74, - /* 30 */ 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, - /* 40 */ 189, 59, 23, 58, 60, 61, 62, 63, 64, 65, - /* 50 */ 66, 67, 68, 69, 70, 71, 72, 147, 74, 75, - /* 60 */ 76, 77, 78, 79, 80, 81, 82, 83, 84, 16, - /* 70 */ 167, 168, 88, 88, 19, 22, 19, 92, 23, 176, - /* 80 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, - /* 90 */ 84, 217, 218, 219, 220, 42, 43, 78, 188, 46, - /* 100 */ 78, 79, 80, 81, 82, 83, 84, 88, 89, 124, - /* 110 */ 125, 126, 16, 60, 61, 62, 63, 64, 65, 66, - /* 120 */ 67, 68, 69, 70, 71, 72, 147, 74, 75, 76, - /* 130 */ 77, 78, 79, 80, 81, 82, 83, 84, 42, 43, - /* 140 */ 44, 142, 143, 88, 89, 11, 110, 148, 169, 170, - /* 150 */ 18, 185, 186, 20, 155, 16, 60, 61, 62, 63, - /* 160 */ 64, 65, 66, 67, 68, 69, 70, 71, 72, 112, - /* 170 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, - /* 180 */ 84, 42, 43, 49, 150, 23, 102, 55, 189, 158, - /* 190 */ 211, 147, 161, 162, 163, 216, 164, 165, 166, 60, - /* 200 */ 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, - /* 210 */ 71, 72, 213, 74, 75, 76, 77, 78, 79, 80, - /* 220 */ 81, 82, 83, 84, 1, 2, 94, 16, 147, 22, - /* 230 */ 16, 20, 20, 99, 100, 101, 147, 156, 157, 106, - /* 240 */ 241, 108, 109, 19, 110, 156, 157, 23, 181, 147, - /* 250 */ 88, 89, 90, 42, 43, 93, 94, 95, 224, 23, - /* 260 */ 228, 180, 181, 84, 230, 133, 104, 165, 166, 225, - /* 270 */ 131, 60, 61, 62, 63, 64, 65, 66, 67, 68, - /* 280 */ 69, 70, 71, 72, 91, 74, 75, 76, 77, 78, - /* 290 */ 79, 80, 81, 82, 83, 84, 16, 90, 84, 16, - /* 300 */ 93, 94, 95, 110, 90, 91, 92, 93, 94, 95, - /* 310 */ 96, 104, 88, 89, 78, 79, 123, 103, 106, 155, - /* 320 */ 108, 109, 42, 43, 88, 89, 99, 100, 101, 227, - /* 330 */ 228, 99, 100, 101, 98, 80, 81, 82, 83, 84, - /* 340 */ 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - /* 350 */ 70, 71, 72, 189, 74, 75, 76, 77, 78, 79, - /* 360 */ 80, 81, 82, 83, 84, 201, 202, 84, 16, 110, - /* 370 */ 206, 12, 169, 21, 91, 92, 93, 94, 95, 96, - /* 380 */ 16, 147, 123, 24, 147, 23, 103, 90, 42, 43, - /* 390 */ 93, 94, 95, 147, 42, 43, 37, 168, 39, 141, - /* 400 */ 161, 104, 144, 169, 170, 176, 43, 43, 49, 63, - /* 410 */ 64, 131, 60, 61, 62, 63, 64, 65, 66, 67, - /* 420 */ 68, 69, 70, 71, 72, 188, 74, 75, 76, 77, - /* 430 */ 78, 79, 80, 81, 82, 83, 84, 16, 92, 200, - /* 440 */ 161, 20, 80, 209, 210, 12, 147, 23, 176, 177, - /* 450 */ 88, 89, 147, 7, 8, 183, 217, 24, 219, 220, - /* 460 */ 155, 98, 98, 42, 43, 83, 84, 103, 169, 170, - /* 470 */ 37, 20, 39, 22, 169, 170, 239, 19, 20, 200, - /* 480 */ 22, 60, 61, 62, 63, 64, 65, 66, 67, 68, - /* 490 */ 69, 70, 71, 72, 189, 74, 75, 76, 77, 78, - /* 500 */ 79, 80, 81, 82, 83, 84, 16, 147, 147, 210, - /* 510 */ 20, 20, 88, 89, 147, 155, 211, 59, 23, 147, - /* 520 */ 153, 216, 147, 156, 157, 217, 165, 219, 220, 169, - /* 530 */ 170, 22, 42, 43, 139, 140, 141, 169, 114, 144, - /* 540 */ 14, 169, 170, 30, 169, 170, 238, 91, 92, 189, - /* 550 */ 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - /* 560 */ 70, 71, 72, 50, 74, 75, 76, 77, 78, 79, - /* 570 */ 80, 81, 82, 83, 84, 16, 124, 125, 52, 20, - /* 580 */ 54, 23, 147, 88, 89, 147, 211, 25, 227, 228, - /* 590 */ 22, 29, 181, 221, 164, 165, 166, 106, 155, 108, - /* 600 */ 109, 42, 43, 41, 169, 170, 97, 169, 170, 114, - /* 610 */ 217, 27, 219, 220, 106, 102, 108, 109, 34, 60, - /* 620 */ 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, - /* 630 */ 71, 72, 189, 74, 75, 76, 77, 78, 79, 80, - /* 640 */ 81, 82, 83, 84, 16, 147, 88, 89, 19, 21, - /* 650 */ 215, 147, 23, 215, 147, 192, 213, 89, 228, 133, - /* 660 */ 23, 147, 199, 147, 161, 162, 163, 169, 170, 190, - /* 670 */ 42, 43, 147, 169, 170, 147, 169, 170, 14, 236, - /* 680 */ 147, 147, 114, 169, 170, 169, 170, 16, 60, 61, - /* 690 */ 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - /* 700 */ 72, 147, 74, 75, 76, 77, 78, 79, 80, 81, - /* 710 */ 82, 83, 84, 42, 43, 182, 52, 88, 54, 0, - /* 720 */ 1, 2, 188, 169, 170, 88, 147, 147, 147, 147, - /* 730 */ 16, 60, 61, 62, 63, 64, 65, 66, 67, 68, - /* 740 */ 69, 70, 71, 72, 147, 74, 75, 76, 77, 78, - /* 750 */ 79, 80, 81, 82, 83, 84, 42, 43, 225, 78, - /* 760 */ 79, 182, 182, 182, 182, 147, 169, 170, 107, 7, - /* 770 */ 8, 9, 111, 16, 60, 61, 62, 63, 64, 65, - /* 780 */ 66, 67, 68, 69, 70, 71, 72, 147, 74, 75, - /* 790 */ 76, 77, 78, 79, 80, 81, 82, 83, 84, 42, - /* 800 */ 43, 19, 242, 243, 225, 225, 225, 225, 22, 169, - /* 810 */ 170, 20, 20, 22, 22, 22, 16, 80, 61, 62, - /* 820 */ 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - /* 830 */ 147, 74, 75, 76, 77, 78, 79, 80, 81, 82, - /* 840 */ 83, 84, 42, 43, 107, 20, 186, 22, 111, 20, - /* 850 */ 68, 22, 169, 170, 147, 16, 21, 20, 23, 22, - /* 860 */ 155, 191, 62, 63, 64, 65, 66, 67, 68, 69, - /* 870 */ 70, 71, 72, 147, 74, 75, 76, 77, 78, 79, - /* 880 */ 80, 81, 82, 83, 84, 16, 17, 147, 19, 147, - /* 890 */ 161, 147, 23, 20, 189, 22, 147, 130, 147, 132, - /* 900 */ 31, 16, 17, 147, 19, 147, 113, 121, 23, 169, - /* 910 */ 170, 169, 170, 169, 170, 147, 31, 48, 169, 170, - /* 920 */ 169, 170, 147, 88, 147, 169, 170, 58, 147, 200, - /* 930 */ 147, 92, 147, 48, 20, 147, 22, 169, 170, 20, - /* 940 */ 19, 22, 21, 58, 169, 170, 147, 78, 79, 147, - /* 950 */ 169, 170, 169, 170, 169, 170, 87, 88, 89, 145, - /* 960 */ 147, 92, 178, 78, 79, 80, 208, 155, 169, 170, - /* 970 */ 155, 147, 87, 88, 89, 16, 17, 92, 19, 110, - /* 980 */ 147, 20, 23, 147, 19, 208, 185, 186, 23, 92, - /* 990 */ 31, 201, 202, 124, 125, 126, 127, 128, 129, 178, - /* 1000 */ 103, 189, 169, 170, 189, 169, 170, 48, 178, 124, - /* 1010 */ 125, 126, 127, 128, 129, 147, 20, 58, 22, 5, - /* 1020 */ 59, 208, 161, 58, 10, 11, 12, 13, 147, 147, - /* 1030 */ 155, 147, 229, 147, 147, 147, 147, 78, 79, 147, - /* 1040 */ 26, 147, 28, 78, 79, 147, 87, 88, 89, 35, - /* 1050 */ 147, 92, 147, 88, 147, 169, 170, 92, 169, 170, - /* 1060 */ 147, 47, 147, 49, 189, 147, 203, 53, 173, 147, - /* 1070 */ 56, 147, 169, 170, 169, 170, 147, 147, 147, 147, - /* 1080 */ 147, 147, 188, 124, 125, 126, 127, 128, 129, 124, - /* 1090 */ 125, 126, 147, 212, 147, 188, 212, 147, 169, 170, - /* 1100 */ 169, 170, 169, 170, 147, 147, 188, 147, 232, 147, - /* 1110 */ 188, 147, 188, 99, 100, 101, 169, 170, 188, 105, - /* 1120 */ 147, 147, 147, 147, 110, 147, 169, 170, 147, 147, - /* 1130 */ 147, 169, 170, 169, 170, 147, 147, 233, 147, 147, - /* 1140 */ 147, 147, 169, 170, 169, 170, 147, 194, 134, 172, - /* 1150 */ 169, 170, 169, 170, 229, 149, 172, 191, 169, 170, - /* 1160 */ 169, 170, 223, 177, 193, 23, 194, 172, 160, 172, - /* 1170 */ 172, 112, 152, 19, 174, 120, 98, 198, 115, 97, - /* 1180 */ 173, 116, 152, 6, 204, 146, 15, 197, 187, 146, - /* 1190 */ 205, 187, 187, 152, 146, 121, 194, 60, 98, 184, - /* 1200 */ 184, 19, 171, 184, 152, 171, 38, 22, 152, 84, - /* 1210 */ 160, 154, 33, 152, 15, 171, 171, 214, 151, 179, - /* 1220 */ 152, 234, 151, 174, 152, 171, 40, 214, 137, 130, - /* 1230 */ 146, 152, 194, 152, 151, 226, 151, 226, 152, 222, - /* 1240 */ 187, 171, 119, 196, 118, 179, 195, 237, 171, 160, - /* 1250 */ 204, 189, 240, 159, 194, 237, 235, 130, 231, 112, - /* 1260 */ 1, 20, 175, 205, 112, 175, 243, 112, 246, 112, - /* 1270 */ 92, 107, 19, 11, 20, 20, 19, 19, 22, 117, - /* 1280 */ 20, 20, 117, 20, 114, 19, 22, 22, 20, 20, - /* 1290 */ 20, 44, 112, 19, 44, 19, 44, 20, 19, 32, - /* 1300 */ 19, 19, 96, 103, 16, 21, 17, 98, 36, 22, - /* 1310 */ 98, 19, 5, 1, 133, 122, 68, 68, 45, 19, - /* 1320 */ 45, 113, 51, 14, 113, 115, 102, 17, 123, 122, - /* 1330 */ 19, 14, 135, 20, 136, 57, 3, 19, 4, + /* 0 */ 19, 221, 222, 223, 224, 24, 150, 26, 72, 73, + /* 10 */ 74, 75, 19, 77, 78, 79, 80, 81, 82, 83, + /* 20 */ 84, 85, 86, 87, 205, 206, 45, 46, 76, 77, + /* 30 */ 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, + /* 40 */ 61, 168, 169, 170, 63, 64, 65, 66, 67, 68, + /* 50 */ 69, 70, 71, 72, 73, 74, 75, 150, 77, 78, + /* 60 */ 79, 80, 81, 82, 83, 84, 85, 86, 87, 19, + /* 70 */ 91, 19, 91, 165, 95, 25, 77, 78, 79, 80, + /* 80 */ 81, 82, 83, 84, 85, 86, 87, 81, 82, 83, + /* 90 */ 84, 85, 86, 87, 150, 45, 46, 146, 147, 49, + /* 100 */ 150, 22, 22, 152, 231, 26, 26, 128, 129, 130, + /* 110 */ 159, 118, 204, 63, 64, 65, 66, 67, 68, 69, + /* 120 */ 70, 71, 72, 73, 74, 75, 12, 77, 78, 79, + /* 130 */ 80, 81, 82, 83, 84, 85, 86, 87, 19, 87, + /* 140 */ 61, 27, 19, 113, 193, 93, 94, 95, 96, 97, + /* 150 */ 98, 99, 150, 23, 40, 25, 42, 127, 106, 157, + /* 160 */ 81, 82, 160, 161, 45, 46, 52, 95, 217, 26, + /* 170 */ 91, 91, 22, 23, 95, 25, 162, 26, 106, 165, + /* 180 */ 166, 167, 63, 64, 65, 66, 67, 68, 69, 70, + /* 190 */ 71, 72, 73, 74, 75, 244, 77, 78, 79, 80, + /* 200 */ 81, 82, 83, 84, 85, 86, 87, 128, 129, 130, + /* 210 */ 87, 19, 62, 221, 25, 223, 224, 94, 95, 96, + /* 220 */ 97, 98, 99, 83, 84, 85, 86, 87, 194, 106, + /* 230 */ 23, 33, 81, 82, 91, 92, 93, 45, 46, 96, + /* 240 */ 97, 98, 91, 92, 150, 168, 169, 170, 189, 190, + /* 250 */ 107, 53, 101, 102, 135, 63, 64, 65, 66, 67, + /* 260 */ 68, 69, 70, 71, 72, 73, 74, 75, 11, 77, + /* 270 */ 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, + /* 280 */ 186, 150, 93, 23, 19, 96, 97, 98, 93, 24, + /* 290 */ 159, 96, 97, 98, 196, 150, 107, 221, 150, 223, + /* 300 */ 224, 203, 107, 105, 173, 174, 150, 159, 231, 52, + /* 310 */ 45, 46, 245, 246, 169, 170, 109, 241, 111, 112, + /* 320 */ 150, 173, 174, 109, 193, 111, 112, 135, 63, 64, + /* 330 */ 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + /* 340 */ 75, 193, 77, 78, 79, 80, 81, 82, 83, 84, + /* 350 */ 85, 86, 87, 19, 19, 164, 150, 154, 24, 102, + /* 360 */ 103, 104, 150, 215, 173, 150, 160, 161, 220, 109, + /* 370 */ 113, 111, 112, 45, 46, 230, 231, 150, 150, 45, + /* 380 */ 46, 150, 212, 52, 150, 173, 174, 182, 173, 174, + /* 390 */ 184, 185, 159, 150, 66, 67, 169, 63, 64, 65, + /* 400 */ 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, + /* 410 */ 46, 77, 78, 79, 80, 81, 82, 83, 84, 85, + /* 420 */ 86, 87, 19, 95, 86, 87, 193, 215, 213, 214, + /* 430 */ 95, 228, 220, 102, 103, 104, 233, 150, 205, 206, + /* 440 */ 150, 26, 171, 172, 113, 159, 212, 216, 45, 46, + /* 450 */ 47, 180, 24, 165, 26, 180, 181, 230, 231, 17, + /* 460 */ 173, 174, 187, 173, 174, 101, 63, 64, 65, 66, + /* 470 */ 67, 68, 69, 70, 71, 72, 73, 74, 75, 193, + /* 480 */ 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, + /* 490 */ 87, 19, 204, 172, 25, 23, 81, 55, 150, 57, + /* 500 */ 150, 180, 215, 217, 214, 23, 91, 92, 150, 221, + /* 510 */ 150, 223, 224, 142, 143, 144, 145, 45, 46, 91, + /* 520 */ 234, 173, 174, 173, 174, 239, 165, 22, 17, 22, + /* 530 */ 23, 26, 25, 173, 174, 63, 64, 65, 66, 67, + /* 540 */ 68, 69, 70, 71, 72, 73, 74, 75, 12, 77, + /* 550 */ 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, + /* 560 */ 19, 23, 150, 27, 23, 204, 55, 219, 57, 62, + /* 570 */ 23, 26, 25, 25, 216, 225, 40, 26, 42, 219, + /* 580 */ 26, 94, 95, 159, 22, 116, 45, 46, 26, 150, + /* 590 */ 23, 109, 25, 111, 112, 26, 91, 92, 186, 160, + /* 600 */ 161, 165, 166, 167, 63, 64, 65, 66, 67, 68, + /* 610 */ 69, 70, 71, 72, 73, 74, 75, 193, 77, 78, + /* 620 */ 79, 80, 81, 82, 83, 84, 85, 86, 87, 19, + /* 630 */ 19, 30, 159, 23, 83, 150, 91, 92, 37, 92, + /* 640 */ 92, 217, 91, 92, 150, 91, 92, 109, 137, 111, + /* 650 */ 112, 150, 159, 91, 92, 45, 46, 46, 173, 174, + /* 660 */ 91, 92, 117, 239, 117, 117, 193, 102, 103, 104, + /* 670 */ 23, 117, 25, 63, 64, 65, 66, 67, 68, 69, + /* 680 */ 70, 71, 72, 73, 74, 75, 193, 77, 78, 79, + /* 690 */ 80, 81, 82, 83, 84, 85, 86, 87, 19, 150, + /* 700 */ 28, 87, 23, 150, 32, 150, 26, 150, 22, 150, + /* 710 */ 150, 150, 101, 23, 150, 25, 44, 106, 0, 1, + /* 720 */ 2, 150, 173, 174, 45, 46, 173, 174, 173, 174, + /* 730 */ 173, 174, 173, 174, 173, 174, 159, 173, 174, 7, + /* 740 */ 8, 9, 63, 64, 65, 66, 67, 68, 69, 70, + /* 750 */ 71, 72, 73, 74, 75, 150, 77, 78, 79, 80, + /* 760 */ 81, 82, 83, 84, 85, 86, 87, 19, 150, 110, + /* 770 */ 193, 91, 150, 114, 150, 83, 150, 25, 173, 174, + /* 780 */ 150, 189, 190, 212, 150, 150, 150, 102, 103, 104, + /* 790 */ 150, 173, 174, 45, 46, 173, 174, 173, 174, 173, + /* 800 */ 174, 115, 110, 173, 174, 159, 114, 173, 174, 173, + /* 810 */ 174, 63, 64, 65, 66, 67, 68, 69, 70, 71, + /* 820 */ 72, 73, 74, 75, 150, 77, 78, 79, 80, 81, + /* 830 */ 82, 83, 84, 85, 86, 87, 19, 150, 94, 193, + /* 840 */ 159, 150, 23, 150, 25, 150, 150, 173, 174, 150, + /* 850 */ 247, 248, 100, 150, 144, 145, 150, 113, 81, 82, + /* 860 */ 1, 2, 45, 46, 173, 174, 173, 174, 173, 174, + /* 870 */ 165, 127, 173, 174, 193, 150, 173, 174, 150, 192, + /* 880 */ 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, + /* 890 */ 73, 74, 75, 150, 77, 78, 79, 80, 81, 82, + /* 900 */ 83, 84, 85, 86, 87, 19, 150, 150, 150, 204, + /* 910 */ 150, 186, 150, 23, 150, 25, 173, 174, 150, 23, + /* 920 */ 192, 25, 23, 150, 25, 150, 7, 8, 207, 173, + /* 930 */ 174, 45, 46, 173, 174, 173, 174, 173, 174, 22, + /* 940 */ 150, 173, 174, 186, 186, 23, 173, 174, 173, 174, + /* 950 */ 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, + /* 960 */ 74, 75, 150, 77, 78, 79, 80, 81, 82, 83, + /* 970 */ 84, 85, 86, 87, 19, 21, 150, 150, 22, 150, + /* 980 */ 24, 150, 192, 150, 62, 173, 174, 150, 71, 23, + /* 990 */ 150, 25, 150, 128, 129, 25, 23, 134, 25, 136, + /* 1000 */ 45, 46, 173, 174, 173, 174, 173, 174, 148, 173, + /* 1010 */ 173, 174, 58, 185, 150, 173, 174, 150, 192, 192, + /* 1020 */ 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + /* 1030 */ 75, 150, 77, 78, 79, 80, 81, 82, 83, 84, + /* 1040 */ 85, 86, 87, 19, 20, 150, 22, 150, 150, 236, + /* 1050 */ 26, 97, 150, 150, 173, 174, 150, 150, 34, 192, + /* 1060 */ 173, 150, 150, 150, 150, 150, 19, 20, 242, 22, + /* 1070 */ 173, 174, 227, 26, 150, 51, 173, 174, 150, 173, + /* 1080 */ 174, 34, 150, 165, 150, 61, 173, 174, 232, 198, + /* 1090 */ 192, 137, 176, 150, 192, 125, 176, 150, 51, 192, + /* 1100 */ 177, 176, 150, 182, 232, 81, 82, 83, 61, 176, + /* 1110 */ 150, 176, 208, 165, 90, 91, 92, 190, 165, 95, + /* 1120 */ 181, 150, 165, 185, 198, 195, 197, 150, 81, 82, + /* 1130 */ 63, 153, 188, 36, 240, 115, 156, 90, 91, 92, + /* 1140 */ 125, 156, 95, 198, 208, 150, 188, 150, 113, 182, + /* 1150 */ 150, 150, 128, 129, 130, 131, 132, 133, 5, 195, + /* 1160 */ 113, 6, 201, 10, 11, 12, 13, 14, 149, 16, + /* 1170 */ 22, 149, 156, 191, 18, 128, 129, 130, 131, 132, + /* 1180 */ 133, 218, 29, 120, 31, 210, 149, 19, 20, 13, + /* 1190 */ 22, 38, 151, 140, 26, 188, 18, 155, 122, 218, + /* 1200 */ 156, 199, 34, 50, 198, 52, 198, 43, 243, 56, + /* 1210 */ 183, 134, 59, 226, 240, 25, 158, 134, 149, 51, + /* 1220 */ 101, 238, 209, 123, 210, 200, 209, 41, 237, 61, + /* 1230 */ 191, 151, 209, 193, 156, 156, 210, 191, 22, 178, + /* 1240 */ 229, 87, 101, 175, 156, 175, 175, 229, 155, 81, + /* 1250 */ 82, 178, 100, 179, 156, 102, 103, 104, 90, 91, + /* 1260 */ 92, 108, 175, 95, 175, 183, 113, 177, 155, 175, + /* 1270 */ 175, 155, 163, 156, 156, 124, 150, 202, 179, 249, + /* 1280 */ 235, 156, 246, 191, 121, 150, 23, 115, 139, 208, + /* 1290 */ 23, 138, 48, 116, 115, 24, 128, 129, 130, 131, + /* 1300 */ 132, 133, 127, 115, 1, 105, 22, 19, 25, 20, + /* 1310 */ 1, 47, 22, 48, 20, 11, 25, 137, 22, 118, + /* 1320 */ 105, 126, 22, 17, 95, 71, 4, 22, 126, 71, + /* 1330 */ 23, 23, 22, 15, 23, 22, 22, 25, 22, 47, + /* 1340 */ 1, 101, 23, 119, 117, 120, 17, 54, 25, 101, + /* 1350 */ 3, 23, 115, 23, 39, 23, 115, 5, 119, 106, + /* 1360 */ 23, 99, 22, 22, 47, 119, 116, 110, 35, 15, + /* 1370 */ 22, 22, 250, 250, 250, 250, 250, 250, 250, 250, + /* 1380 */ 60, }; -#define YY_SHIFT_USE_DFLT (-62) -#define YY_SHIFT_MAX 387 +#define YY_SHIFT_USE_DFLT (-65) +#define YY_SHIFT_MAX 404 static const short yy_shift_ofst[] = { - /* 0 */ 223, 869, 1014, -16, 869, 959, 959, 162, 236, 36, - /* 10 */ 96, 959, 959, 959, 959, 959, -45, 134, 19, 558, - /* 20 */ 681, 681, 637, 53, 139, 490, 421, 628, 352, 559, - /* 30 */ 211, 280, 671, 671, 671, 671, 671, 671, 671, 671, - /* 40 */ 671, 671, 671, 714, 671, 671, 671, 671, 671, 671, - /* 50 */ 671, 671, 757, 800, 800, 885, 959, 959, 959, 959, - /* 60 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 70 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 80 */ 959, 959, 959, 959, 959, 959, 959, 959, 959, 959, - /* 90 */ 959, 959, 959, 959, 959, -61, -61, 6, 6, 214, - /* 100 */ 22, 255, 364, 132, 558, 558, 558, 558, 558, 558, - /* 110 */ 558, 382, 637, 179, -62, -62, 965, 283, 359, 359, - /* 120 */ -18, 458, 558, 558, 558, 558, 558, 558, 558, 558, - /* 130 */ 558, 193, 558, 719, 558, 835, 558, 835, 558, 558, - /* 140 */ 259, 36, 36, 36, -62, -62, -62, 207, -15, -15, - /* 150 */ 297, 212, 424, 224, 55, 433, 133, 362, 495, 491, - /* 160 */ 558, 558, 558, 513, 513, 513, 558, 558, 737, 558, - /* 170 */ 558, 558, 558, 508, 508, 762, 629, 558, 558, 558, - /* 180 */ 558, 508, 558, 558, 558, 558, 558, 558, 558, 568, - /* 190 */ 508, 558, 558, 558, 558, 508, 568, 227, 562, 558, - /* 200 */ 508, 558, 558, 508, 558, 452, 509, 584, 786, 584, - /* 210 */ 452, 661, 767, 584, 363, 584, 921, 584, 793, 786, - /* 220 */ 1142, 1059, 1154, 1055, 1078, 1063, 1082, 1065, 1059, 1177, - /* 230 */ 1171, 1177, 1171, 1171, 1059, 1177, 1074, 1137, 1137, 1100, - /* 240 */ 1182, 1137, 1059, 1100, 1168, 1059, 1185, 1125, 1179, 1142, - /* 250 */ 1059, 1100, 1199, 1100, 1125, 1059, 1199, 1059, 1100, 1154, - /* 260 */ 1186, 1182, 1091, 1099, 1059, 1177, 1074, 1059, 1199, 1059, - /* 270 */ 1171, 1100, 1199, 1123, 1126, 1074, 1100, 1078, 1063, 36, - /* 280 */ 1179, 1186, 1142, 1127, -62, -62, -62, -62, -62, -62, - /* 290 */ 346, 526, 664, 232, 792, 791, 446, 825, 829, 57, - /* 300 */ 456, 839, 837, 873, 914, 782, 919, 996, 897, 961, - /* 310 */ 451, 1147, 1259, 1241, 1152, 1155, 1157, 1178, 1164, 1253, - /* 320 */ 1254, 1255, 1257, 1262, 1258, 1260, 1256, 1261, 1263, 1264, - /* 330 */ 1162, 1265, 1165, 1264, 1170, 1266, 1268, 1180, 1269, 1270, - /* 340 */ 1267, 1247, 1274, 1250, 1276, 1277, 1279, 1281, 1252, 1282, - /* 350 */ 1206, 1200, 1288, 1289, 1284, 1209, 1272, 1271, 1273, 1287, - /* 360 */ 1275, 1181, 1212, 1292, 1307, 1312, 84, 1248, 1249, 1193, - /* 370 */ 1300, 1208, 1309, 1210, 1310, 1211, 1224, 1207, 1311, 1205, - /* 380 */ 1313, 1317, 1278, 1197, 1198, 1318, 1333, 1334, + /* 0 */ 859, 1047, 1153, -19, 1047, 1168, 1168, 143, 151, 331, + /* 10 */ 403, 1168, 1168, 1168, 1168, 1168, -48, 257, 415, 569, + /* 20 */ 777, 777, 680, 1035, 50, 610, 541, 472, 679, 192, + /* 30 */ 119, 265, 334, 748, 748, 748, 748, 748, 748, 748, + /* 40 */ 748, 748, 748, 748, 748, 748, 748, 748, 748, 748, + /* 50 */ 748, 748, 817, 886, 955, 955, 1024, 1168, 1168, 1168, + /* 60 */ 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1168, + /* 70 */ 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1168, + /* 80 */ 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1168, + /* 90 */ 1168, 1168, 1168, 1168, 1168, 1168, 1168, -64, -64, -1, + /* 100 */ -1, 52, 6, 140, 954, 611, 569, 569, 338, 680, + /* 110 */ 614, -65, -65, -65, 79, 123, 114, 114, 507, 150, + /* 120 */ 569, 569, 569, 569, 569, 744, 569, 569, 569, 569, + /* 130 */ 718, 569, 428, 569, 569, 569, 569, 569, 569, 569, + /* 140 */ 569, 428, 428, 30, 1035, 1035, 1035, -65, -65, -65, + /* 150 */ -21, 189, -21, 195, 207, 260, 536, 547, 505, 482, + /* 160 */ 551, 545, 554, 538, 562, 569, 198, 214, 569, 569, + /* 170 */ 569, 569, 732, 569, 569, 569, 569, 569, 569, 692, + /* 180 */ 569, 672, 214, 569, 569, 569, 569, 569, 548, 569, + /* 190 */ 214, 569, 569, 569, 569, 569, 214, 569, 569, 569, + /* 200 */ 569, 198, 569, 569, 214, 569, 80, 569, 214, 569, + /* 210 */ 565, 198, 569, 863, -7, 680, 601, 601, 970, 865, + /* 220 */ 601, 752, 601, -7, 601, 970, 865, 680, -7, 659, + /* 230 */ 680, 469, 364, 956, 1067, 1097, 1020, 1015, 1067, 1020, + /* 240 */ 1020, 1151, 1156, 1020, 1163, 1155, 1148, 1020, 1155, 1156, + /* 250 */ 1063, 1155, 1176, 1053, 1067, 1178, 1148, 1076, 1015, 1015, + /* 260 */ 1020, 1097, 1164, 1077, 1190, 1083, 1155, 1119, 1100, 1119, + /* 270 */ 1063, 1186, 1156, 1176, 1119, 1063, 1020, 1156, 1035, 1020, + /* 280 */ 1216, 1154, 1154, 1141, 1020, 1141, 1141, 1216, 1178, 1141, + /* 290 */ 1141, 1152, 1141, 1164, 1020, 1141, 1178, 1178, 1020, -65, + /* 300 */ -65, -65, -65, -65, 328, 511, 685, 442, 72, 335, + /* 310 */ 966, 973, 919, 917, 890, 819, 686, 922, 896, 899, + /* 320 */ 130, 487, 690, 567, 647, 1305, 1308, 1310, 1322, 1318, + /* 330 */ 1313, 1312, 1339, 1319, 1312, 1224, 1227, 1323, 1263, 1330, + /* 340 */ 1317, 1332, 1239, 1337, 1340, 1246, 1257, 1333, 1311, 1229, + /* 350 */ 1201, 1304, 1290, 1283, 1188, 1179, 1172, 1241, 1237, 1328, + /* 360 */ 1347, 1354, 1320, 1349, 1248, 1293, 1315, 1225, 1329, 1240, + /* 370 */ 1258, 1292, 1314, 1316, 1254, 1307, 1202, 1306, 1300, 1195, + /* 380 */ 1215, 1348, 1294, 1296, 1250, 1180, 1265, 1291, 1264, 1341, + /* 390 */ 1289, 1309, 1262, 1288, 1284, 1200, 1303, 1253, 1271, 1175, + /* 400 */ 1352, 1244, 1177, 1267, 1149, }; #define YY_REDUCE_USE_DFLT (-221) -#define YY_REDUCE_MAX 289 +#define YY_REDUCE_MAX 303 static const short yy_reduce_ofst[] = { - /* 0 */ 395, 305, -1, 239, 360, -21, 234, 81, 102, 164, - /* 10 */ 308, 372, 375, 438, 299, 435, -126, 443, 361, 367, - /* 20 */ 32, 430, 31, 393, 393, 393, 393, 393, 393, 393, - /* 30 */ 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, - /* 40 */ 393, 393, 393, 393, 393, 393, 393, 393, 393, 393, - /* 50 */ 393, 393, 393, 393, 393, 498, 504, 507, 514, 516, - /* 60 */ 554, 597, 640, 683, 740, 742, 744, 749, 751, 756, - /* 70 */ 768, 775, 781, 783, 785, 799, 833, 836, 886, 889, - /* 80 */ 903, 905, 929, 931, 933, 947, 957, 962, 964, 973, - /* 90 */ 975, 981, 983, 989, 991, 393, 393, 393, 393, -97, - /* 100 */ 393, 393, 272, 34, 89, 533, 579, 580, 237, 581, - /* 110 */ 582, 393, 503, 393, 393, 393, -157, 229, -34, 801, - /* 120 */ -220, -220, 930, 924, 922, 918, 907, 884, 894, 881, - /* 130 */ 534, 875, 44, 258, 813, 729, 777, 279, -90, 758, - /* 140 */ -149, 705, 812, 815, 463, 560, 790, 67, 203, 368, - /* 150 */ 411, 479, 246, 525, 528, 660, 479, 618, 707, 479, - /* 160 */ 726, 788, 802, 784, 821, 830, 824, 868, 670, 882, - /* 170 */ 618, 887, 888, 479, 479, 814, 861, 892, 898, 913, - /* 180 */ 915, 479, 932, 934, 945, 950, 958, 960, 974, 863, - /* 190 */ 479, 976, 978, 982, 988, 479, 863, 904, 876, 992, - /* 200 */ 479, 993, 994, 479, 999, 803, 895, 977, 953, 984, - /* 210 */ 925, 966, 939, 995, 986, 997, 1006, 998, 971, 972, - /* 220 */ 1008, 1020, 1000, 979, 980, 985, 1007, 990, 1030, 1039, - /* 230 */ 1001, 1043, 1004, 1005, 1041, 1048, 1002, 1015, 1016, 1031, - /* 240 */ 1003, 1019, 1052, 1034, 987, 1056, 1057, 1009, 1010, 1050, - /* 250 */ 1061, 1044, 1067, 1045, 1011, 1068, 1071, 1072, 1054, 1049, - /* 260 */ 1040, 1013, 1012, 1017, 1079, 1084, 1038, 1081, 1083, 1086, - /* 270 */ 1053, 1070, 1085, 1047, 1051, 1060, 1077, 1046, 1058, 1062, - /* 280 */ 1018, 1066, 1089, 1021, 1022, 1087, 1090, 1023, 1027, 1094, + /* 0 */ 371, 148, -49, 288, 131, 212, 215, 206, 145, 286, + /* 10 */ 76, 360, 350, 348, 287, 290, -220, 424, 227, 2, + /* 20 */ -127, 77, 14, 233, -8, -8, -8, -8, -8, -8, + /* 30 */ -8, -8, -8, -8, -8, -8, -8, -8, -8, -8, + /* 40 */ -8, -8, -8, -8, -8, -8, -8, -8, -8, -8, + /* 50 */ -8, -8, -8, -8, -8, -8, 553, 485, 557, 691, + /* 60 */ 622, 703, 760, 768, 903, 837, 906, 897, 762, 756, + /* 70 */ 699, 674, 618, 555, 549, 559, 561, 564, 605, 695, + /* 80 */ 764, 624, 626, 630, 634, 636, 831, 693, 833, 743, + /* 90 */ 842, 881, 913, 773, 775, 812, 829, -8, -8, -8, + /* 100 */ -8, 271, -8, -8, 203, 275, 826, 439, -8, 436, + /* 110 */ -8, -8, -8, -8, 191, 321, 59, 592, 603, 603, + /* 120 */ 907, 902, 898, 867, 758, 473, 827, 358, 234, 790, + /* 130 */ 710, 231, 705, 94, 571, 728, 170, 757, 725, 687, + /* 140 */ 412, -92, 361, 646, 681, 493, 577, 98, 67, -181, + /* 150 */ 887, 828, 836, 938, 34, 34, 927, 721, 947, 34, + /* 160 */ 840, 932, 864, 34, 156, -93, 205, 34, 501, 635, + /* 170 */ 640, 706, 860, 915, 914, 977, 997, 995, 1126, 964, + /* 180 */ 1000, 1045, 34, 911, 912, 924, -50, 1135, 721, 696, + /* 190 */ 34, 560, 494, 243, 228, -144, 34, 840, -56, 928, + /* 200 */ 934, 921, 943, 952, 34, 960, 953, 971, 34, 895, + /* 210 */ 813, 967, 1001, 845, 1081, 918, 916, 920, 891, 856, + /* 220 */ 925, 923, 933, 904, 935, 926, 872, 948, 936, 930, + /* 230 */ 957, 929, 939, 978, 944, 894, 980, 945, 958, 985, + /* 240 */ 1118, 1075, 1092, 1125, 961, 1019, 963, 1016, 1022, 982, + /* 250 */ 975, 1037, 1041, 965, 1007, 1042, 981, 1002, 1006, 1008, + /* 260 */ 1044, 974, 1027, 987, 1058, 983, 1069, 1013, 1025, 1017, + /* 270 */ 1014, 991, 1039, 1080, 1023, 1026, 1078, 1046, 1040, 1079, + /* 280 */ 1061, 1011, 1018, 1068, 1088, 1070, 1071, 1073, 1093, 1087, + /* 290 */ 1089, 1090, 1094, 1082, 1098, 1095, 1113, 1116, 1117, 1074, + /* 300 */ 1099, 1109, 1030, 1036, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 592, 816, 897, 707, 897, 816, 897, 897, 843, 711, - /* 10 */ 872, 814, 897, 897, 897, 897, 789, 897, 843, 897, - /* 20 */ 843, 843, 623, 740, 897, 897, 897, 897, 897, 897, - /* 30 */ 897, 897, 748, 813, 810, 809, 811, 873, 856, 871, - /* 40 */ 723, 741, 745, 897, 738, 818, 731, 742, 747, 817, - /* 50 */ 755, 754, 777, 776, 795, 897, 897, 897, 897, 897, - /* 60 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 70 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 80 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 90 */ 897, 897, 897, 897, 897, 800, 779, 778, 788, 616, - /* 100 */ 780, 781, 676, 611, 897, 897, 897, 897, 897, 897, - /* 110 */ 897, 782, 897, 783, 796, 797, 897, 897, 897, 897, - /* 120 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 130 */ 897, 897, 897, 592, 897, 707, 897, 707, 897, 897, - /* 140 */ 897, 897, 897, 897, 701, 890, 711, 667, 897, 897, - /* 150 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 160 */ 897, 897, 897, 897, 897, 897, 897, 897, 699, 897, - /* 170 */ 897, 897, 830, 750, 690, 599, 625, 897, 897, 897, - /* 180 */ 597, 861, 897, 897, 897, 897, 878, 897, 897, 709, - /* 190 */ 863, 876, 897, 897, 897, 714, 715, 897, 849, 897, - /* 200 */ 688, 897, 897, 613, 897, 897, 649, 734, 744, 734, - /* 210 */ 897, 699, 812, 734, 646, 734, 897, 734, 708, 744, - /* 220 */ 897, 716, 822, 739, 724, 726, 649, 727, 716, 596, - /* 230 */ 687, 596, 687, 687, 716, 596, 744, 680, 680, 678, - /* 240 */ 756, 680, 716, 678, 853, 716, 666, 827, 875, 897, - /* 250 */ 716, 678, 608, 678, 827, 716, 608, 716, 678, 822, - /* 260 */ 663, 756, 883, 897, 716, 596, 744, 716, 608, 716, - /* 270 */ 687, 678, 608, 737, 735, 744, 678, 724, 726, 897, - /* 280 */ 875, 663, 897, 855, 895, 651, 651, 890, 858, 633, - /* 290 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 763, - /* 300 */ 897, 897, 897, 897, 897, 836, 897, 897, 897, 897, - /* 310 */ 897, 897, 897, 897, 768, 764, 765, 897, 693, 897, - /* 320 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 815, - /* 330 */ 897, 728, 897, 736, 897, 897, 897, 897, 897, 897, - /* 340 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 350 */ 897, 897, 897, 897, 897, 897, 897, 897, 851, 852, - /* 360 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 370 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 380 */ 897, 897, 882, 897, 897, 885, 593, 897, 664, 642, - /* 390 */ 684, 607, 606, 605, 609, 683, 682, 679, 604, 641, - /* 400 */ 640, 639, 768, 638, 637, 774, 877, 773, 761, 718, - /* 410 */ 804, 704, 845, 703, 702, 603, 602, 854, 850, 879, - /* 420 */ 713, 896, 846, 847, 601, 848, 610, 894, 759, 758, - /* 430 */ 725, 587, 712, 600, 706, 705, 819, 820, 880, 722, - /* 440 */ 721, 720, 719, 710, 681, 752, 753, 844, 594, 857, - /* 450 */ 700, 730, 622, 733, 732, 729, 621, 790, 806, 859, - /* 460 */ 881, 717, 803, 884, 868, 867, 866, 865, 757, 864, - /* 470 */ 808, 588, 807, 775, 772, 628, 860, 629, 620, 842, - /* 480 */ 840, 862, 598, 841, 839, 886, 631, 887, 630, 627, - /* 490 */ 626, 685, 595, 686, 619, 805, 689, 695, 694, 829, - /* 500 */ 801, 798, 869, 794, 591, 792, 791, 743, 786, 802, - /* 510 */ 891, 799, 793, 746, 892, 787, 785, 784, 770, 589, - /* 520 */ 769, 767, 749, 590, 751, 766, 831, 762, 771, 696, - /* 530 */ 692, 832, 833, 834, 837, 838, 691, 893, 835, 760, - /* 540 */ 698, 662, 635, 624, 617, 648, 647, 668, 825, 671, - /* 550 */ 697, 636, 826, 828, 618, 824, 672, 870, 823, 656, - /* 560 */ 655, 673, 634, 660, 659, 658, 674, 632, 874, 657, - /* 570 */ 654, 614, 653, 652, 645, 644, 612, 675, 677, 669, - /* 580 */ 670, 615, 650, 643, 665, 661, + /* 0 */ 615, 848, 929, 736, 929, 848, 929, 929, 875, 929, + /* 10 */ 904, 929, 846, 929, 929, 929, 820, 929, 875, 929, + /* 20 */ 875, 875, 652, 740, 771, 929, 929, 929, 929, 929, + /* 30 */ 929, 929, 929, 903, 842, 778, 849, 905, 779, 785, + /* 40 */ 762, 769, 776, 786, 888, 841, 751, 773, 843, 850, + /* 50 */ 845, 772, 929, 808, 807, 826, 929, 929, 929, 929, + /* 60 */ 929, 929, 929, 929, 929, 929, 929, 929, 929, 929, + /* 70 */ 929, 929, 929, 929, 929, 929, 929, 929, 929, 929, + /* 80 */ 929, 929, 929, 929, 929, 929, 929, 929, 929, 929, + /* 90 */ 929, 929, 929, 929, 929, 929, 929, 810, 832, 819, + /* 100 */ 809, 645, 811, 812, 640, 705, 929, 929, 813, 929, + /* 110 */ 814, 828, 827, 829, 929, 929, 929, 929, 929, 929, + /* 120 */ 929, 929, 929, 929, 929, 929, 929, 929, 929, 929, + /* 130 */ 615, 929, 736, 929, 929, 929, 929, 929, 929, 929, + /* 140 */ 929, 736, 736, 929, 929, 929, 929, 730, 922, 740, + /* 150 */ 929, 696, 929, 929, 929, 929, 929, 929, 929, 929, + /* 160 */ 929, 929, 929, 929, 929, 929, 929, 893, 910, 929, + /* 170 */ 929, 929, 623, 929, 929, 861, 929, 929, 929, 728, + /* 180 */ 929, 881, 719, 929, 929, 929, 929, 621, 738, 929, + /* 190 */ 642, 929, 929, 929, 929, 929, 717, 929, 929, 929, + /* 200 */ 929, 929, 908, 929, 895, 929, 654, 929, 781, 929, + /* 210 */ 929, 929, 929, 844, 753, 929, 765, 765, 775, 929, + /* 220 */ 765, 678, 765, 753, 765, 775, 929, 929, 753, 728, + /* 230 */ 929, 737, 675, 929, 709, 907, 744, 775, 709, 744, + /* 240 */ 744, 770, 716, 744, 758, 620, 787, 744, 620, 716, + /* 250 */ 757, 620, 631, 915, 709, 637, 787, 766, 775, 775, + /* 260 */ 744, 907, 692, 929, 695, 887, 620, 752, 768, 752, + /* 270 */ 757, 885, 716, 631, 752, 757, 744, 716, 929, 744, + /* 280 */ 854, 858, 858, 707, 744, 707, 707, 854, 637, 707, + /* 290 */ 707, 678, 707, 692, 744, 707, 637, 637, 744, 680, + /* 300 */ 680, 662, 927, 922, 929, 929, 929, 929, 929, 929, + /* 310 */ 929, 929, 929, 868, 929, 929, 794, 929, 929, 929, + /* 320 */ 929, 929, 929, 929, 929, 929, 929, 929, 929, 629, + /* 330 */ 929, 847, 929, 929, 767, 929, 929, 929, 929, 929, + /* 340 */ 929, 929, 929, 929, 929, 929, 722, 929, 929, 929, + /* 350 */ 929, 929, 929, 759, 799, 929, 796, 929, 795, 929, + /* 360 */ 616, 929, 914, 929, 929, 929, 929, 929, 929, 929, + /* 370 */ 929, 929, 929, 917, 929, 929, 929, 929, 929, 929, + /* 380 */ 929, 929, 929, 929, 929, 929, 929, 884, 929, 929, + /* 390 */ 929, 929, 929, 929, 929, 929, 929, 929, 929, 929, + /* 400 */ 929, 883, 929, 929, 929, 671, 644, 694, 690, 693, + /* 410 */ 699, 673, 679, 706, 698, 702, 713, 670, 672, 805, + /* 420 */ 704, 792, 636, 669, 681, 684, 700, 743, 746, 674, + /* 430 */ 755, 877, 754, 703, 682, 859, 676, 882, 683, 687, + /* 440 */ 686, 878, 879, 638, 880, 689, 688, 756, 741, 668, + /* 450 */ 799, 646, 667, 653, 666, 851, 852, 804, 748, 747, + /* 460 */ 739, 729, 856, 783, 784, 761, 876, 701, 889, 764, + /* 470 */ 763, 855, 685, 664, 836, 733, 732, 835, 891, 731, + /* 480 */ 900, 899, 742, 677, 897, 896, 840, 788, 839, 806, + /* 490 */ 790, 803, 657, 658, 874, 892, 872, 697, 873, 871, + /* 500 */ 894, 857, 890, 886, 660, 659, 789, 656, 919, 655, + /* 510 */ 837, 724, 714, 639, 715, 723, 833, 718, 830, 825, + /* 520 */ 860, 823, 822, 901, 817, 735, 834, 831, 824, 774, + /* 530 */ 818, 816, 815, 734, 801, 800, 777, 750, 798, 797, + /* 540 */ 793, 802, 749, 648, 725, 780, 710, 782, 721, 862, + /* 550 */ 720, 791, 727, 726, 863, 864, 865, 866, 869, 870, + /* 560 */ 665, 691, 867, 647, 663, 661, 643, 641, 902, 635, + /* 570 */ 633, 906, 632, 634, 630, 628, 627, 909, 626, 625, + /* 580 */ 911, 821, 624, 760, 838, 651, 912, 745, 650, 649, + /* 590 */ 913, 916, 898, 622, 619, 918, 618, 614, 923, 924, + /* 600 */ 613, 611, 925, 928, 926, 708, 711, 712, 617, 612, }; #define YY_SZ_ACTTAB (int)(sizeof(yy_action)/sizeof(yy_action[0])) @@ -594,7 +610,7 @@ static const YYACTIONTYPE yy_default[] = { ** ** %fallback ID X Y Z. ** -** appears in the grammer, then ID becomes a fallback token for X, Y, +** appears in the grammar, then ID becomes a fallback token for X, Y, ** and Z. Whenever one of the tokens X, Y, or Z is input to the parser ** but it does not parse, the type of the token is changed to ID and ** the parse is retried before an error is thrown. @@ -603,63 +619,66 @@ static const YYACTIONTYPE yy_default[] = { static const YYCODETYPE yyFallback[] = { 0, /* $ => nothing */ 0, /* SEMI => nothing */ - 23, /* EXPLAIN => ID */ - 23, /* QUERY => ID */ - 23, /* PLAN => ID */ - 23, /* BEGIN => ID */ + 26, /* EXPLAIN => ID */ + 26, /* QUERY => ID */ + 26, /* PLAN => ID */ + 26, /* BEGIN => ID */ 0, /* TRANSACTION => nothing */ - 23, /* DEFERRED => ID */ - 23, /* IMMEDIATE => ID */ - 23, /* EXCLUSIVE => ID */ + 26, /* DEFERRED => ID */ + 26, /* IMMEDIATE => ID */ + 26, /* EXCLUSIVE => ID */ 0, /* COMMIT => nothing */ - 23, /* END => ID */ + 26, /* END => ID */ 0, /* ROLLBACK => nothing */ + 0, /* SAVEPOINT => nothing */ + 0, /* RELEASE => nothing */ + 0, /* TO => nothing */ 0, /* CREATE => nothing */ 0, /* TABLE => nothing */ - 23, /* IF => ID */ + 26, /* IF => ID */ 0, /* NOT => nothing */ 0, /* EXISTS => nothing */ - 23, /* TEMP => ID */ + 26, /* TEMP => ID */ 0, /* LP => nothing */ 0, /* RP => nothing */ 0, /* AS => nothing */ 0, /* COMMA => nothing */ 0, /* ID => nothing */ - 23, /* ABORT => ID */ - 23, /* AFTER => ID */ - 23, /* ANALYZE => ID */ - 23, /* ASC => ID */ - 23, /* ATTACH => ID */ - 23, /* BEFORE => ID */ - 23, /* CASCADE => ID */ - 23, /* CAST => ID */ - 23, /* CONFLICT => ID */ - 23, /* DATABASE => ID */ - 23, /* DESC => ID */ - 23, /* DETACH => ID */ - 23, /* EACH => ID */ - 23, /* FAIL => ID */ - 23, /* FOR => ID */ - 23, /* IGNORE => ID */ - 23, /* INITIALLY => ID */ - 23, /* INSTEAD => ID */ - 23, /* LIKE_KW => ID */ - 23, /* MATCH => ID */ - 23, /* KEY => ID */ - 23, /* OF => ID */ - 23, /* OFFSET => ID */ - 23, /* PRAGMA => ID */ - 23, /* RAISE => ID */ - 23, /* REPLACE => ID */ - 23, /* RESTRICT => ID */ - 23, /* ROW => ID */ - 23, /* TRIGGER => ID */ - 23, /* VACUUM => ID */ - 23, /* VIEW => ID */ - 23, /* VIRTUAL => ID */ - 23, /* REINDEX => ID */ - 23, /* RENAME => ID */ - 23, /* CTIME_KW => ID */ + 26, /* ABORT => ID */ + 26, /* AFTER => ID */ + 26, /* ANALYZE => ID */ + 26, /* ASC => ID */ + 26, /* ATTACH => ID */ + 26, /* BEFORE => ID */ + 26, /* CASCADE => ID */ + 26, /* CAST => ID */ + 26, /* CONFLICT => ID */ + 26, /* DATABASE => ID */ + 26, /* DESC => ID */ + 26, /* DETACH => ID */ + 26, /* EACH => ID */ + 26, /* FAIL => ID */ + 26, /* FOR => ID */ + 26, /* IGNORE => ID */ + 26, /* INITIALLY => ID */ + 26, /* INSTEAD => ID */ + 26, /* LIKE_KW => ID */ + 26, /* MATCH => ID */ + 26, /* KEY => ID */ + 26, /* OF => ID */ + 26, /* OFFSET => ID */ + 26, /* PRAGMA => ID */ + 26, /* RAISE => ID */ + 26, /* REPLACE => ID */ + 26, /* RESTRICT => ID */ + 26, /* ROW => ID */ + 26, /* TRIGGER => ID */ + 26, /* VACUUM => ID */ + 26, /* VIEW => ID */ + 26, /* VIRTUAL => ID */ + 26, /* REINDEX => ID */ + 26, /* RENAME => ID */ + 26, /* CTIME_KW => ID */ 0, /* ANY => nothing */ 0, /* OR => nothing */ 0, /* AND => nothing */ @@ -716,9 +735,10 @@ static const YYCODETYPE yyFallback[] = { 0, /* DOT => nothing */ 0, /* FROM => nothing */ 0, /* JOIN => nothing */ + 0, /* INDEXED => nothing */ + 0, /* BY => nothing */ 0, /* USING => nothing */ 0, /* ORDER => nothing */ - 0, /* BY => nothing */ 0, /* GROUP => nothing */ 0, /* HAVING => nothing */ 0, /* LIMIT => nothing */ @@ -736,7 +756,6 @@ static const YYCODETYPE yyFallback[] = { 0, /* ELSE => nothing */ 0, /* INDEX => nothing */ 0, /* ALTER => nothing */ - 0, /* TO => nothing */ 0, /* ADD => nothing */ 0, /* COLUMNKW => nothing */ }; @@ -755,11 +774,11 @@ static const YYCODETYPE yyFallback[] = { ** It is sometimes called the "minor" token. */ struct yyStackEntry { - int stateno; /* The state-number */ - int major; /* The major token value. This is the code - ** number for the token at this stack level */ - YYMINORTYPE minor; /* The user-supplied minor token value. This - ** is the value of the token */ + YYACTIONTYPE stateno; /* The state-number */ + YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ }; typedef struct yyStackEntry yyStackEntry; @@ -767,6 +786,9 @@ typedef struct yyStackEntry yyStackEntry; ** the following structure */ struct yyParser { int yyidx; /* Index of top element in stack */ +#ifdef YYTRACKMAXSTACKDEPTH + int yyidxMax; /* Maximum value of yyidx */ +#endif int yyerrcnt; /* Shifts left before out of the error */ sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ #if YYSTACKDEPTH<=0 @@ -817,40 +839,41 @@ static const char *const yyTokenName[] = { "$", "SEMI", "EXPLAIN", "QUERY", "PLAN", "BEGIN", "TRANSACTION", "DEFERRED", "IMMEDIATE", "EXCLUSIVE", "COMMIT", "END", - "ROLLBACK", "CREATE", "TABLE", "IF", - "NOT", "EXISTS", "TEMP", "LP", - "RP", "AS", "COMMA", "ID", - "ABORT", "AFTER", "ANALYZE", "ASC", - "ATTACH", "BEFORE", "CASCADE", "CAST", - "CONFLICT", "DATABASE", "DESC", "DETACH", - "EACH", "FAIL", "FOR", "IGNORE", - "INITIALLY", "INSTEAD", "LIKE_KW", "MATCH", - "KEY", "OF", "OFFSET", "PRAGMA", - "RAISE", "REPLACE", "RESTRICT", "ROW", - "TRIGGER", "VACUUM", "VIEW", "VIRTUAL", - "REINDEX", "RENAME", "CTIME_KW", "ANY", - "OR", "AND", "IS", "BETWEEN", - "IN", "ISNULL", "NOTNULL", "NE", - "EQ", "GT", "LE", "LT", - "GE", "ESCAPE", "BITAND", "BITOR", - "LSHIFT", "RSHIFT", "PLUS", "MINUS", - "STAR", "SLASH", "REM", "CONCAT", - "COLLATE", "UMINUS", "UPLUS", "BITNOT", - "STRING", "JOIN_KW", "CONSTRAINT", "DEFAULT", - "NULL", "PRIMARY", "UNIQUE", "CHECK", - "REFERENCES", "AUTOINCR", "ON", "DELETE", - "UPDATE", "INSERT", "SET", "DEFERRABLE", - "FOREIGN", "DROP", "UNION", "ALL", - "EXCEPT", "INTERSECT", "SELECT", "DISTINCT", - "DOT", "FROM", "JOIN", "USING", - "ORDER", "BY", "GROUP", "HAVING", + "ROLLBACK", "SAVEPOINT", "RELEASE", "TO", + "CREATE", "TABLE", "IF", "NOT", + "EXISTS", "TEMP", "LP", "RP", + "AS", "COMMA", "ID", "ABORT", + "AFTER", "ANALYZE", "ASC", "ATTACH", + "BEFORE", "CASCADE", "CAST", "CONFLICT", + "DATABASE", "DESC", "DETACH", "EACH", + "FAIL", "FOR", "IGNORE", "INITIALLY", + "INSTEAD", "LIKE_KW", "MATCH", "KEY", + "OF", "OFFSET", "PRAGMA", "RAISE", + "REPLACE", "RESTRICT", "ROW", "TRIGGER", + "VACUUM", "VIEW", "VIRTUAL", "REINDEX", + "RENAME", "CTIME_KW", "ANY", "OR", + "AND", "IS", "BETWEEN", "IN", + "ISNULL", "NOTNULL", "NE", "EQ", + "GT", "LE", "LT", "GE", + "ESCAPE", "BITAND", "BITOR", "LSHIFT", + "RSHIFT", "PLUS", "MINUS", "STAR", + "SLASH", "REM", "CONCAT", "COLLATE", + "UMINUS", "UPLUS", "BITNOT", "STRING", + "JOIN_KW", "CONSTRAINT", "DEFAULT", "NULL", + "PRIMARY", "UNIQUE", "CHECK", "REFERENCES", + "AUTOINCR", "ON", "DELETE", "UPDATE", + "INSERT", "SET", "DEFERRABLE", "FOREIGN", + "DROP", "UNION", "ALL", "EXCEPT", + "INTERSECT", "SELECT", "DISTINCT", "DOT", + "FROM", "JOIN", "INDEXED", "BY", + "USING", "ORDER", "GROUP", "HAVING", "LIMIT", "WHERE", "INTO", "VALUES", "INTEGER", "FLOAT", "BLOB", "REGISTER", "VARIABLE", "CASE", "WHEN", "THEN", - "ELSE", "INDEX", "ALTER", "TO", - "ADD", "COLUMNKW", "error", "input", - "cmdlist", "ecmd", "cmdx", "cmd", - "explain", "transtype", "trans_opt", "nm", + "ELSE", "INDEX", "ALTER", "ADD", + "COLUMNKW", "error", "input", "cmdlist", + "ecmd", "explain", "cmdx", "cmd", + "transtype", "trans_opt", "nm", "savepoint_opt", "create_table", "create_table_args", "temp", "ifnotexists", "dbnm", "columnlist", "conslist_opt", "select", "column", "columnid", "type", "carglist", @@ -865,17 +888,17 @@ static const char *const yyTokenName[] = { "selcollist", "from", "where_opt", "groupby_opt", "having_opt", "orderby_opt", "limit_opt", "sclp", "as", "seltablist", "stl_prefix", "joinop", - "on_opt", "using_opt", "seltablist_paren", "joinop2", + "indexed_opt", "on_opt", "using_opt", "joinop2", "inscollist", "sortlist", "sortitem", "nexprlist", "setlist", "insert_cmd", "inscollist_opt", "itemlist", "exprlist", "likeop", "escape", "between_op", "in_op", "case_operand", "case_exprlist", "case_else", - "uniqueflag", "idxitem", "collate", "nmnum", - "plus_opt", "number", "trigger_decl", "trigger_cmd_list", - "trigger_time", "trigger_event", "foreach_clause", "when_clause", - "trigger_cmd", "database_kw_opt", "key_opt", "add_column_fullname", - "kwcolumn_opt", "create_vtab", "vtabarglist", "vtabarg", - "vtabargtoken", "lp", "anylist", + "uniqueflag", "collate", "nmnum", "plus_opt", + "number", "trigger_decl", "trigger_cmd_list", "trigger_time", + "trigger_event", "foreach_clause", "when_clause", "trigger_cmd", + "database_kw_opt", "key_opt", "add_column_fullname", "kwcolumn_opt", + "create_vtab", "vtabarglist", "vtabarg", "vtabargtoken", + "lp", "anylist", }; #endif /* NDEBUG */ @@ -886,12 +909,12 @@ static const char *const yyRuleName[] = { /* 0 */ "input ::= cmdlist", /* 1 */ "cmdlist ::= cmdlist ecmd", /* 2 */ "cmdlist ::= ecmd", - /* 3 */ "cmdx ::= cmd", - /* 4 */ "ecmd ::= SEMI", - /* 5 */ "ecmd ::= explain cmdx SEMI", - /* 6 */ "explain ::=", - /* 7 */ "explain ::= EXPLAIN", - /* 8 */ "explain ::= EXPLAIN QUERY PLAN", + /* 3 */ "ecmd ::= SEMI", + /* 4 */ "ecmd ::= explain cmdx SEMI", + /* 5 */ "explain ::=", + /* 6 */ "explain ::= EXPLAIN", + /* 7 */ "explain ::= EXPLAIN QUERY PLAN", + /* 8 */ "cmdx ::= cmd", /* 9 */ "cmd ::= BEGIN transtype trans_opt", /* 10 */ "trans_opt ::=", /* 11 */ "trans_opt ::= TRANSACTION", @@ -903,297 +926,305 @@ static const char *const yyRuleName[] = { /* 17 */ "cmd ::= COMMIT trans_opt", /* 18 */ "cmd ::= END trans_opt", /* 19 */ "cmd ::= ROLLBACK trans_opt", - /* 20 */ "cmd ::= create_table create_table_args", - /* 21 */ "create_table ::= CREATE temp TABLE ifnotexists nm dbnm", - /* 22 */ "ifnotexists ::=", - /* 23 */ "ifnotexists ::= IF NOT EXISTS", - /* 24 */ "temp ::= TEMP", - /* 25 */ "temp ::=", - /* 26 */ "create_table_args ::= LP columnlist conslist_opt RP", - /* 27 */ "create_table_args ::= AS select", - /* 28 */ "columnlist ::= columnlist COMMA column", - /* 29 */ "columnlist ::= column", - /* 30 */ "column ::= columnid type carglist", - /* 31 */ "columnid ::= nm", - /* 32 */ "id ::= ID", - /* 33 */ "ids ::= ID|STRING", - /* 34 */ "nm ::= ID", - /* 35 */ "nm ::= STRING", - /* 36 */ "nm ::= JOIN_KW", - /* 37 */ "type ::=", - /* 38 */ "type ::= typetoken", - /* 39 */ "typetoken ::= typename", - /* 40 */ "typetoken ::= typename LP signed RP", - /* 41 */ "typetoken ::= typename LP signed COMMA signed RP", - /* 42 */ "typename ::= ids", - /* 43 */ "typename ::= typename ids", - /* 44 */ "signed ::= plus_num", - /* 45 */ "signed ::= minus_num", - /* 46 */ "carglist ::= carglist carg", - /* 47 */ "carglist ::=", - /* 48 */ "carg ::= CONSTRAINT nm ccons", - /* 49 */ "carg ::= ccons", - /* 50 */ "ccons ::= DEFAULT term", - /* 51 */ "ccons ::= DEFAULT LP expr RP", - /* 52 */ "ccons ::= DEFAULT PLUS term", - /* 53 */ "ccons ::= DEFAULT MINUS term", - /* 54 */ "ccons ::= DEFAULT id", - /* 55 */ "ccons ::= NULL onconf", - /* 56 */ "ccons ::= NOT NULL onconf", - /* 57 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", - /* 58 */ "ccons ::= UNIQUE onconf", - /* 59 */ "ccons ::= CHECK LP expr RP", - /* 60 */ "ccons ::= REFERENCES nm idxlist_opt refargs", - /* 61 */ "ccons ::= defer_subclause", - /* 62 */ "ccons ::= COLLATE id", - /* 63 */ "autoinc ::=", - /* 64 */ "autoinc ::= AUTOINCR", - /* 65 */ "refargs ::=", - /* 66 */ "refargs ::= refargs refarg", - /* 67 */ "refarg ::= MATCH nm", - /* 68 */ "refarg ::= ON DELETE refact", - /* 69 */ "refarg ::= ON UPDATE refact", - /* 70 */ "refarg ::= ON INSERT refact", - /* 71 */ "refact ::= SET NULL", - /* 72 */ "refact ::= SET DEFAULT", - /* 73 */ "refact ::= CASCADE", - /* 74 */ "refact ::= RESTRICT", - /* 75 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", - /* 76 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", - /* 77 */ "init_deferred_pred_opt ::=", - /* 78 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", - /* 79 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", - /* 80 */ "conslist_opt ::=", - /* 81 */ "conslist_opt ::= COMMA conslist", - /* 82 */ "conslist ::= conslist COMMA tcons", - /* 83 */ "conslist ::= conslist tcons", - /* 84 */ "conslist ::= tcons", - /* 85 */ "tcons ::= CONSTRAINT nm", - /* 86 */ "tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf", - /* 87 */ "tcons ::= UNIQUE LP idxlist RP onconf", - /* 88 */ "tcons ::= CHECK LP expr RP onconf", - /* 89 */ "tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt", - /* 90 */ "defer_subclause_opt ::=", - /* 91 */ "defer_subclause_opt ::= defer_subclause", - /* 92 */ "onconf ::=", - /* 93 */ "onconf ::= ON CONFLICT resolvetype", - /* 94 */ "orconf ::=", - /* 95 */ "orconf ::= OR resolvetype", - /* 96 */ "resolvetype ::= raisetype", - /* 97 */ "resolvetype ::= IGNORE", - /* 98 */ "resolvetype ::= REPLACE", - /* 99 */ "cmd ::= DROP TABLE ifexists fullname", - /* 100 */ "ifexists ::= IF EXISTS", - /* 101 */ "ifexists ::=", - /* 102 */ "cmd ::= CREATE temp VIEW ifnotexists nm dbnm AS select", - /* 103 */ "cmd ::= DROP VIEW ifexists fullname", - /* 104 */ "cmd ::= select", - /* 105 */ "select ::= oneselect", - /* 106 */ "select ::= select multiselect_op oneselect", - /* 107 */ "multiselect_op ::= UNION", - /* 108 */ "multiselect_op ::= UNION ALL", - /* 109 */ "multiselect_op ::= EXCEPT|INTERSECT", - /* 110 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", - /* 111 */ "distinct ::= DISTINCT", - /* 112 */ "distinct ::= ALL", - /* 113 */ "distinct ::=", - /* 114 */ "sclp ::= selcollist COMMA", - /* 115 */ "sclp ::=", - /* 116 */ "selcollist ::= sclp expr as", - /* 117 */ "selcollist ::= sclp STAR", - /* 118 */ "selcollist ::= sclp nm DOT STAR", - /* 119 */ "as ::= AS nm", - /* 120 */ "as ::= ids", - /* 121 */ "as ::=", - /* 122 */ "from ::=", - /* 123 */ "from ::= FROM seltablist", - /* 124 */ "stl_prefix ::= seltablist joinop", - /* 125 */ "stl_prefix ::=", - /* 126 */ "seltablist ::= stl_prefix nm dbnm as on_opt using_opt", - /* 127 */ "seltablist ::= stl_prefix LP seltablist_paren RP as on_opt using_opt", - /* 128 */ "seltablist_paren ::= select", - /* 129 */ "seltablist_paren ::= seltablist", - /* 130 */ "dbnm ::=", - /* 131 */ "dbnm ::= DOT nm", - /* 132 */ "fullname ::= nm dbnm", - /* 133 */ "joinop ::= COMMA|JOIN", - /* 134 */ "joinop ::= JOIN_KW JOIN", - /* 135 */ "joinop ::= JOIN_KW nm JOIN", - /* 136 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 137 */ "on_opt ::= ON expr", - /* 138 */ "on_opt ::=", - /* 139 */ "using_opt ::= USING LP inscollist RP", - /* 140 */ "using_opt ::=", - /* 141 */ "orderby_opt ::=", - /* 142 */ "orderby_opt ::= ORDER BY sortlist", - /* 143 */ "sortlist ::= sortlist COMMA sortitem sortorder", - /* 144 */ "sortlist ::= sortitem sortorder", - /* 145 */ "sortitem ::= expr", - /* 146 */ "sortorder ::= ASC", - /* 147 */ "sortorder ::= DESC", - /* 148 */ "sortorder ::=", - /* 149 */ "groupby_opt ::=", - /* 150 */ "groupby_opt ::= GROUP BY nexprlist", - /* 151 */ "having_opt ::=", - /* 152 */ "having_opt ::= HAVING expr", - /* 153 */ "limit_opt ::=", - /* 154 */ "limit_opt ::= LIMIT expr", - /* 155 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 156 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 157 */ "cmd ::= DELETE FROM fullname where_opt", - /* 158 */ "where_opt ::=", - /* 159 */ "where_opt ::= WHERE expr", - /* 160 */ "cmd ::= UPDATE orconf fullname SET setlist where_opt", - /* 161 */ "setlist ::= setlist COMMA nm EQ expr", - /* 162 */ "setlist ::= nm EQ expr", - /* 163 */ "cmd ::= insert_cmd INTO fullname inscollist_opt VALUES LP itemlist RP", - /* 164 */ "cmd ::= insert_cmd INTO fullname inscollist_opt select", - /* 165 */ "cmd ::= insert_cmd INTO fullname inscollist_opt DEFAULT VALUES", - /* 166 */ "insert_cmd ::= INSERT orconf", - /* 167 */ "insert_cmd ::= REPLACE", - /* 168 */ "itemlist ::= itemlist COMMA expr", - /* 169 */ "itemlist ::= expr", - /* 170 */ "inscollist_opt ::=", - /* 171 */ "inscollist_opt ::= LP inscollist RP", - /* 172 */ "inscollist ::= inscollist COMMA nm", - /* 173 */ "inscollist ::= nm", - /* 174 */ "expr ::= term", - /* 175 */ "expr ::= LP expr RP", - /* 176 */ "term ::= NULL", - /* 177 */ "expr ::= ID", - /* 178 */ "expr ::= JOIN_KW", - /* 179 */ "expr ::= nm DOT nm", - /* 180 */ "expr ::= nm DOT nm DOT nm", - /* 181 */ "term ::= INTEGER|FLOAT|BLOB", - /* 182 */ "term ::= STRING", - /* 183 */ "expr ::= REGISTER", - /* 184 */ "expr ::= VARIABLE", - /* 185 */ "expr ::= expr COLLATE id", - /* 186 */ "expr ::= CAST LP expr AS typetoken RP", - /* 187 */ "expr ::= ID LP distinct exprlist RP", - /* 188 */ "expr ::= ID LP STAR RP", - /* 189 */ "term ::= CTIME_KW", - /* 190 */ "expr ::= expr AND expr", - /* 191 */ "expr ::= expr OR expr", - /* 192 */ "expr ::= expr LT|GT|GE|LE expr", - /* 193 */ "expr ::= expr EQ|NE expr", - /* 194 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 195 */ "expr ::= expr PLUS|MINUS expr", - /* 196 */ "expr ::= expr STAR|SLASH|REM expr", - /* 197 */ "expr ::= expr CONCAT expr", - /* 198 */ "likeop ::= LIKE_KW", - /* 199 */ "likeop ::= NOT LIKE_KW", - /* 200 */ "likeop ::= MATCH", - /* 201 */ "likeop ::= NOT MATCH", - /* 202 */ "escape ::= ESCAPE expr", - /* 203 */ "escape ::=", - /* 204 */ "expr ::= expr likeop expr escape", - /* 205 */ "expr ::= expr ISNULL|NOTNULL", - /* 206 */ "expr ::= expr IS NULL", - /* 207 */ "expr ::= expr NOT NULL", - /* 208 */ "expr ::= expr IS NOT NULL", - /* 209 */ "expr ::= NOT|BITNOT expr", - /* 210 */ "expr ::= MINUS expr", - /* 211 */ "expr ::= PLUS expr", - /* 212 */ "between_op ::= BETWEEN", - /* 213 */ "between_op ::= NOT BETWEEN", - /* 214 */ "expr ::= expr between_op expr AND expr", - /* 215 */ "in_op ::= IN", - /* 216 */ "in_op ::= NOT IN", - /* 217 */ "expr ::= expr in_op LP exprlist RP", - /* 218 */ "expr ::= LP select RP", - /* 219 */ "expr ::= expr in_op LP select RP", - /* 220 */ "expr ::= expr in_op nm dbnm", - /* 221 */ "expr ::= EXISTS LP select RP", - /* 222 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 223 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 224 */ "case_exprlist ::= WHEN expr THEN expr", - /* 225 */ "case_else ::= ELSE expr", - /* 226 */ "case_else ::=", - /* 227 */ "case_operand ::= expr", - /* 228 */ "case_operand ::=", - /* 229 */ "exprlist ::= nexprlist", - /* 230 */ "exprlist ::=", - /* 231 */ "nexprlist ::= nexprlist COMMA expr", - /* 232 */ "nexprlist ::= expr", - /* 233 */ "cmd ::= CREATE uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP", - /* 234 */ "uniqueflag ::= UNIQUE", - /* 235 */ "uniqueflag ::=", - /* 236 */ "idxlist_opt ::=", - /* 237 */ "idxlist_opt ::= LP idxlist RP", - /* 238 */ "idxlist ::= idxlist COMMA idxitem collate sortorder", - /* 239 */ "idxlist ::= idxitem collate sortorder", - /* 240 */ "idxitem ::= nm", - /* 241 */ "collate ::=", - /* 242 */ "collate ::= COLLATE id", - /* 243 */ "cmd ::= DROP INDEX ifexists fullname", - /* 244 */ "cmd ::= VACUUM", - /* 245 */ "cmd ::= VACUUM nm", - /* 246 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 247 */ "cmd ::= PRAGMA nm dbnm EQ ON", - /* 248 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 249 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 250 */ "cmd ::= PRAGMA nm dbnm", - /* 251 */ "nmnum ::= plus_num", - /* 252 */ "nmnum ::= nm", - /* 253 */ "plus_num ::= plus_opt number", - /* 254 */ "minus_num ::= MINUS number", - /* 255 */ "number ::= INTEGER|FLOAT", - /* 256 */ "plus_opt ::= PLUS", - /* 257 */ "plus_opt ::=", - /* 258 */ "cmd ::= CREATE trigger_decl BEGIN trigger_cmd_list END", - /* 259 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 260 */ "trigger_time ::= BEFORE", - /* 261 */ "trigger_time ::= AFTER", - /* 262 */ "trigger_time ::= INSTEAD OF", - /* 263 */ "trigger_time ::=", - /* 264 */ "trigger_event ::= DELETE|INSERT", - /* 265 */ "trigger_event ::= UPDATE", - /* 266 */ "trigger_event ::= UPDATE OF inscollist", - /* 267 */ "foreach_clause ::=", - /* 268 */ "foreach_clause ::= FOR EACH ROW", - /* 269 */ "when_clause ::=", - /* 270 */ "when_clause ::= WHEN expr", - /* 271 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 272 */ "trigger_cmd_list ::=", - /* 273 */ "trigger_cmd ::= UPDATE orconf nm SET setlist where_opt", - /* 274 */ "trigger_cmd ::= insert_cmd INTO nm inscollist_opt VALUES LP itemlist RP", - /* 275 */ "trigger_cmd ::= insert_cmd INTO nm inscollist_opt select", - /* 276 */ "trigger_cmd ::= DELETE FROM nm where_opt", - /* 277 */ "trigger_cmd ::= select", - /* 278 */ "expr ::= RAISE LP IGNORE RP", - /* 279 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 280 */ "raisetype ::= ROLLBACK", - /* 281 */ "raisetype ::= ABORT", - /* 282 */ "raisetype ::= FAIL", - /* 283 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 284 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 285 */ "cmd ::= DETACH database_kw_opt expr", - /* 286 */ "key_opt ::=", - /* 287 */ "key_opt ::= KEY expr", - /* 288 */ "database_kw_opt ::= DATABASE", - /* 289 */ "database_kw_opt ::=", - /* 290 */ "cmd ::= REINDEX", - /* 291 */ "cmd ::= REINDEX nm dbnm", - /* 292 */ "cmd ::= ANALYZE", - /* 293 */ "cmd ::= ANALYZE nm dbnm", - /* 294 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 295 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column", - /* 296 */ "add_column_fullname ::= fullname", - /* 297 */ "kwcolumn_opt ::=", - /* 298 */ "kwcolumn_opt ::= COLUMNKW", - /* 299 */ "cmd ::= create_vtab", - /* 300 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 301 */ "create_vtab ::= CREATE VIRTUAL TABLE nm dbnm USING nm", - /* 302 */ "vtabarglist ::= vtabarg", - /* 303 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 304 */ "vtabarg ::=", - /* 305 */ "vtabarg ::= vtabarg vtabargtoken", - /* 306 */ "vtabargtoken ::= ANY", - /* 307 */ "vtabargtoken ::= lp anylist RP", - /* 308 */ "lp ::= LP", - /* 309 */ "anylist ::=", - /* 310 */ "anylist ::= anylist ANY", + /* 20 */ "savepoint_opt ::= SAVEPOINT", + /* 21 */ "savepoint_opt ::=", + /* 22 */ "cmd ::= SAVEPOINT nm", + /* 23 */ "cmd ::= RELEASE savepoint_opt nm", + /* 24 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", + /* 25 */ "cmd ::= create_table create_table_args", + /* 26 */ "create_table ::= CREATE temp TABLE ifnotexists nm dbnm", + /* 27 */ "ifnotexists ::=", + /* 28 */ "ifnotexists ::= IF NOT EXISTS", + /* 29 */ "temp ::= TEMP", + /* 30 */ "temp ::=", + /* 31 */ "create_table_args ::= LP columnlist conslist_opt RP", + /* 32 */ "create_table_args ::= AS select", + /* 33 */ "columnlist ::= columnlist COMMA column", + /* 34 */ "columnlist ::= column", + /* 35 */ "column ::= columnid type carglist", + /* 36 */ "columnid ::= nm", + /* 37 */ "id ::= ID", + /* 38 */ "ids ::= ID|STRING", + /* 39 */ "nm ::= ID", + /* 40 */ "nm ::= STRING", + /* 41 */ "nm ::= JOIN_KW", + /* 42 */ "type ::=", + /* 43 */ "type ::= typetoken", + /* 44 */ "typetoken ::= typename", + /* 45 */ "typetoken ::= typename LP signed RP", + /* 46 */ "typetoken ::= typename LP signed COMMA signed RP", + /* 47 */ "typename ::= ids", + /* 48 */ "typename ::= typename ids", + /* 49 */ "signed ::= plus_num", + /* 50 */ "signed ::= minus_num", + /* 51 */ "carglist ::= carglist carg", + /* 52 */ "carglist ::=", + /* 53 */ "carg ::= CONSTRAINT nm ccons", + /* 54 */ "carg ::= ccons", + /* 55 */ "ccons ::= DEFAULT term", + /* 56 */ "ccons ::= DEFAULT LP expr RP", + /* 57 */ "ccons ::= DEFAULT PLUS term", + /* 58 */ "ccons ::= DEFAULT MINUS term", + /* 59 */ "ccons ::= DEFAULT id", + /* 60 */ "ccons ::= NULL onconf", + /* 61 */ "ccons ::= NOT NULL onconf", + /* 62 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", + /* 63 */ "ccons ::= UNIQUE onconf", + /* 64 */ "ccons ::= CHECK LP expr RP", + /* 65 */ "ccons ::= REFERENCES nm idxlist_opt refargs", + /* 66 */ "ccons ::= defer_subclause", + /* 67 */ "ccons ::= COLLATE ids", + /* 68 */ "autoinc ::=", + /* 69 */ "autoinc ::= AUTOINCR", + /* 70 */ "refargs ::=", + /* 71 */ "refargs ::= refargs refarg", + /* 72 */ "refarg ::= MATCH nm", + /* 73 */ "refarg ::= ON DELETE refact", + /* 74 */ "refarg ::= ON UPDATE refact", + /* 75 */ "refarg ::= ON INSERT refact", + /* 76 */ "refact ::= SET NULL", + /* 77 */ "refact ::= SET DEFAULT", + /* 78 */ "refact ::= CASCADE", + /* 79 */ "refact ::= RESTRICT", + /* 80 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", + /* 81 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", + /* 82 */ "init_deferred_pred_opt ::=", + /* 83 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", + /* 84 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", + /* 85 */ "conslist_opt ::=", + /* 86 */ "conslist_opt ::= COMMA conslist", + /* 87 */ "conslist ::= conslist COMMA tcons", + /* 88 */ "conslist ::= conslist tcons", + /* 89 */ "conslist ::= tcons", + /* 90 */ "tcons ::= CONSTRAINT nm", + /* 91 */ "tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf", + /* 92 */ "tcons ::= UNIQUE LP idxlist RP onconf", + /* 93 */ "tcons ::= CHECK LP expr RP onconf", + /* 94 */ "tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt", + /* 95 */ "defer_subclause_opt ::=", + /* 96 */ "defer_subclause_opt ::= defer_subclause", + /* 97 */ "onconf ::=", + /* 98 */ "onconf ::= ON CONFLICT resolvetype", + /* 99 */ "orconf ::=", + /* 100 */ "orconf ::= OR resolvetype", + /* 101 */ "resolvetype ::= raisetype", + /* 102 */ "resolvetype ::= IGNORE", + /* 103 */ "resolvetype ::= REPLACE", + /* 104 */ "cmd ::= DROP TABLE ifexists fullname", + /* 105 */ "ifexists ::= IF EXISTS", + /* 106 */ "ifexists ::=", + /* 107 */ "cmd ::= CREATE temp VIEW ifnotexists nm dbnm AS select", + /* 108 */ "cmd ::= DROP VIEW ifexists fullname", + /* 109 */ "cmd ::= select", + /* 110 */ "select ::= oneselect", + /* 111 */ "select ::= select multiselect_op oneselect", + /* 112 */ "multiselect_op ::= UNION", + /* 113 */ "multiselect_op ::= UNION ALL", + /* 114 */ "multiselect_op ::= EXCEPT|INTERSECT", + /* 115 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", + /* 116 */ "distinct ::= DISTINCT", + /* 117 */ "distinct ::= ALL", + /* 118 */ "distinct ::=", + /* 119 */ "sclp ::= selcollist COMMA", + /* 120 */ "sclp ::=", + /* 121 */ "selcollist ::= sclp expr as", + /* 122 */ "selcollist ::= sclp STAR", + /* 123 */ "selcollist ::= sclp nm DOT STAR", + /* 124 */ "as ::= AS nm", + /* 125 */ "as ::= ids", + /* 126 */ "as ::=", + /* 127 */ "from ::=", + /* 128 */ "from ::= FROM seltablist", + /* 129 */ "stl_prefix ::= seltablist joinop", + /* 130 */ "stl_prefix ::=", + /* 131 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", + /* 132 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 133 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 134 */ "dbnm ::=", + /* 135 */ "dbnm ::= DOT nm", + /* 136 */ "fullname ::= nm dbnm", + /* 137 */ "joinop ::= COMMA|JOIN", + /* 138 */ "joinop ::= JOIN_KW JOIN", + /* 139 */ "joinop ::= JOIN_KW nm JOIN", + /* 140 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 141 */ "on_opt ::= ON expr", + /* 142 */ "on_opt ::=", + /* 143 */ "indexed_opt ::=", + /* 144 */ "indexed_opt ::= INDEXED BY nm", + /* 145 */ "indexed_opt ::= NOT INDEXED", + /* 146 */ "using_opt ::= USING LP inscollist RP", + /* 147 */ "using_opt ::=", + /* 148 */ "orderby_opt ::=", + /* 149 */ "orderby_opt ::= ORDER BY sortlist", + /* 150 */ "sortlist ::= sortlist COMMA sortitem sortorder", + /* 151 */ "sortlist ::= sortitem sortorder", + /* 152 */ "sortitem ::= expr", + /* 153 */ "sortorder ::= ASC", + /* 154 */ "sortorder ::= DESC", + /* 155 */ "sortorder ::=", + /* 156 */ "groupby_opt ::=", + /* 157 */ "groupby_opt ::= GROUP BY nexprlist", + /* 158 */ "having_opt ::=", + /* 159 */ "having_opt ::= HAVING expr", + /* 160 */ "limit_opt ::=", + /* 161 */ "limit_opt ::= LIMIT expr", + /* 162 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 163 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 164 */ "cmd ::= DELETE FROM fullname indexed_opt where_opt", + /* 165 */ "where_opt ::=", + /* 166 */ "where_opt ::= WHERE expr", + /* 167 */ "cmd ::= UPDATE orconf fullname indexed_opt SET setlist where_opt", + /* 168 */ "setlist ::= setlist COMMA nm EQ expr", + /* 169 */ "setlist ::= nm EQ expr", + /* 170 */ "cmd ::= insert_cmd INTO fullname inscollist_opt VALUES LP itemlist RP", + /* 171 */ "cmd ::= insert_cmd INTO fullname inscollist_opt select", + /* 172 */ "cmd ::= insert_cmd INTO fullname inscollist_opt DEFAULT VALUES", + /* 173 */ "insert_cmd ::= INSERT orconf", + /* 174 */ "insert_cmd ::= REPLACE", + /* 175 */ "itemlist ::= itemlist COMMA expr", + /* 176 */ "itemlist ::= expr", + /* 177 */ "inscollist_opt ::=", + /* 178 */ "inscollist_opt ::= LP inscollist RP", + /* 179 */ "inscollist ::= inscollist COMMA nm", + /* 180 */ "inscollist ::= nm", + /* 181 */ "expr ::= term", + /* 182 */ "expr ::= LP expr RP", + /* 183 */ "term ::= NULL", + /* 184 */ "expr ::= ID", + /* 185 */ "expr ::= JOIN_KW", + /* 186 */ "expr ::= nm DOT nm", + /* 187 */ "expr ::= nm DOT nm DOT nm", + /* 188 */ "term ::= INTEGER|FLOAT|BLOB", + /* 189 */ "term ::= STRING", + /* 190 */ "expr ::= REGISTER", + /* 191 */ "expr ::= VARIABLE", + /* 192 */ "expr ::= expr COLLATE ids", + /* 193 */ "expr ::= CAST LP expr AS typetoken RP", + /* 194 */ "expr ::= ID LP distinct exprlist RP", + /* 195 */ "expr ::= ID LP STAR RP", + /* 196 */ "term ::= CTIME_KW", + /* 197 */ "expr ::= expr AND expr", + /* 198 */ "expr ::= expr OR expr", + /* 199 */ "expr ::= expr LT|GT|GE|LE expr", + /* 200 */ "expr ::= expr EQ|NE expr", + /* 201 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 202 */ "expr ::= expr PLUS|MINUS expr", + /* 203 */ "expr ::= expr STAR|SLASH|REM expr", + /* 204 */ "expr ::= expr CONCAT expr", + /* 205 */ "likeop ::= LIKE_KW", + /* 206 */ "likeop ::= NOT LIKE_KW", + /* 207 */ "likeop ::= MATCH", + /* 208 */ "likeop ::= NOT MATCH", + /* 209 */ "escape ::= ESCAPE expr", + /* 210 */ "escape ::=", + /* 211 */ "expr ::= expr likeop expr escape", + /* 212 */ "expr ::= expr ISNULL|NOTNULL", + /* 213 */ "expr ::= expr IS NULL", + /* 214 */ "expr ::= expr NOT NULL", + /* 215 */ "expr ::= expr IS NOT NULL", + /* 216 */ "expr ::= NOT expr", + /* 217 */ "expr ::= BITNOT expr", + /* 218 */ "expr ::= MINUS expr", + /* 219 */ "expr ::= PLUS expr", + /* 220 */ "between_op ::= BETWEEN", + /* 221 */ "between_op ::= NOT BETWEEN", + /* 222 */ "expr ::= expr between_op expr AND expr", + /* 223 */ "in_op ::= IN", + /* 224 */ "in_op ::= NOT IN", + /* 225 */ "expr ::= expr in_op LP exprlist RP", + /* 226 */ "expr ::= LP select RP", + /* 227 */ "expr ::= expr in_op LP select RP", + /* 228 */ "expr ::= expr in_op nm dbnm", + /* 229 */ "expr ::= EXISTS LP select RP", + /* 230 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 231 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 232 */ "case_exprlist ::= WHEN expr THEN expr", + /* 233 */ "case_else ::= ELSE expr", + /* 234 */ "case_else ::=", + /* 235 */ "case_operand ::= expr", + /* 236 */ "case_operand ::=", + /* 237 */ "exprlist ::= nexprlist", + /* 238 */ "exprlist ::=", + /* 239 */ "nexprlist ::= nexprlist COMMA expr", + /* 240 */ "nexprlist ::= expr", + /* 241 */ "cmd ::= CREATE uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP", + /* 242 */ "uniqueflag ::= UNIQUE", + /* 243 */ "uniqueflag ::=", + /* 244 */ "idxlist_opt ::=", + /* 245 */ "idxlist_opt ::= LP idxlist RP", + /* 246 */ "idxlist ::= idxlist COMMA nm collate sortorder", + /* 247 */ "idxlist ::= nm collate sortorder", + /* 248 */ "collate ::=", + /* 249 */ "collate ::= COLLATE ids", + /* 250 */ "cmd ::= DROP INDEX ifexists fullname", + /* 251 */ "cmd ::= VACUUM", + /* 252 */ "cmd ::= VACUUM nm", + /* 253 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 254 */ "cmd ::= PRAGMA nm dbnm EQ ON", + /* 255 */ "cmd ::= PRAGMA nm dbnm EQ DELETE", + /* 256 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 257 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 258 */ "cmd ::= PRAGMA nm dbnm", + /* 259 */ "nmnum ::= plus_num", + /* 260 */ "nmnum ::= nm", + /* 261 */ "plus_num ::= plus_opt number", + /* 262 */ "minus_num ::= MINUS number", + /* 263 */ "number ::= INTEGER|FLOAT", + /* 264 */ "plus_opt ::= PLUS", + /* 265 */ "plus_opt ::=", + /* 266 */ "cmd ::= CREATE trigger_decl BEGIN trigger_cmd_list END", + /* 267 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 268 */ "trigger_time ::= BEFORE", + /* 269 */ "trigger_time ::= AFTER", + /* 270 */ "trigger_time ::= INSTEAD OF", + /* 271 */ "trigger_time ::=", + /* 272 */ "trigger_event ::= DELETE|INSERT", + /* 273 */ "trigger_event ::= UPDATE", + /* 274 */ "trigger_event ::= UPDATE OF inscollist", + /* 275 */ "foreach_clause ::=", + /* 276 */ "foreach_clause ::= FOR EACH ROW", + /* 277 */ "when_clause ::=", + /* 278 */ "when_clause ::= WHEN expr", + /* 279 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 280 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 281 */ "trigger_cmd ::= UPDATE orconf nm SET setlist where_opt", + /* 282 */ "trigger_cmd ::= insert_cmd INTO nm inscollist_opt VALUES LP itemlist RP", + /* 283 */ "trigger_cmd ::= insert_cmd INTO nm inscollist_opt select", + /* 284 */ "trigger_cmd ::= DELETE FROM nm where_opt", + /* 285 */ "trigger_cmd ::= select", + /* 286 */ "expr ::= RAISE LP IGNORE RP", + /* 287 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 288 */ "raisetype ::= ROLLBACK", + /* 289 */ "raisetype ::= ABORT", + /* 290 */ "raisetype ::= FAIL", + /* 291 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 292 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 293 */ "cmd ::= DETACH database_kw_opt expr", + /* 294 */ "key_opt ::=", + /* 295 */ "key_opt ::= KEY expr", + /* 296 */ "database_kw_opt ::= DATABASE", + /* 297 */ "database_kw_opt ::=", + /* 298 */ "cmd ::= REINDEX", + /* 299 */ "cmd ::= REINDEX nm dbnm", + /* 300 */ "cmd ::= ANALYZE", + /* 301 */ "cmd ::= ANALYZE nm dbnm", + /* 302 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 303 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column", + /* 304 */ "add_column_fullname ::= fullname", + /* 305 */ "kwcolumn_opt ::=", + /* 306 */ "kwcolumn_opt ::= COLUMNKW", + /* 307 */ "cmd ::= create_vtab", + /* 308 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 309 */ "create_vtab ::= CREATE VIRTUAL TABLE nm dbnm USING nm", + /* 310 */ "vtabarglist ::= vtabarg", + /* 311 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 312 */ "vtabarg ::=", + /* 313 */ "vtabarg ::= vtabarg vtabargtoken", + /* 314 */ "vtabargtoken ::= ANY", + /* 315 */ "vtabargtoken ::= lp anylist RP", + /* 316 */ "lp ::= LP", + /* 317 */ "anylist ::=", + /* 318 */ "anylist ::= anylist ANY", }; #endif /* NDEBUG */ @@ -1238,7 +1269,12 @@ void *sqlite3ParserAlloc(void *(*mallocProc)(size_t)){ pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) ); if( pParser ){ pParser->yyidx = -1; +#ifdef YYTRACKMAXSTACKDEPTH + pParser->yyidxMax = 0; +#endif #if YYSTACKDEPTH<=0 + pParser->yystack = NULL; + pParser->yystksz = 0; yyGrowStack(pParser); #endif } @@ -1250,7 +1286,12 @@ void *sqlite3ParserAlloc(void *(*mallocProc)(size_t)){ ** "yymajor" is the symbol code, and "yypminor" is a pointer to ** the value. */ -static void yy_destructor(YYCODETYPE yymajor, YYMINORTYPE *yypminor){ +static void yy_destructor( + yyParser *yypParser, /* The parser */ + YYCODETYPE yymajor, /* Type code for object to destroy */ + YYMINORTYPE *yypminor /* The object to be destroyed */ +){ + sqlite3ParserARG_FETCH; switch( yymajor ){ /* Here is inserted the actions which take place when a ** terminal or non-terminal is destroyed. This can happen @@ -1262,73 +1303,82 @@ static void yy_destructor(YYCODETYPE yymajor, YYMINORTYPE *yypminor){ ** which appear on the RHS of the rule, but which are not used ** inside the C code. */ - case 155: - case 189: - case 206: -#line 374 "parse.y" -{sqlite3SelectDelete((yypminor->yy219));} -#line 1272 "parse.c" + case 159: /* select */ + case 193: /* oneselect */ +{ +#line 384 "parse.y" +sqlite3SelectDelete(pParse->db, (yypminor->yy159)); +#line 1312 "parse.c" +} break; - case 169: - case 170: - case 194: - case 196: - case 204: - case 210: - case 218: - case 221: - case 223: - case 235: -#line 627 "parse.y" -{sqlite3ExprDelete((yypminor->yy172));} -#line 1286 "parse.c" + case 173: /* term */ + case 174: /* expr */ + case 198: /* where_opt */ + case 200: /* having_opt */ + case 209: /* on_opt */ + case 214: /* sortitem */ + case 222: /* escape */ + case 225: /* case_operand */ + case 227: /* case_else */ + case 238: /* when_clause */ + case 241: /* key_opt */ +{ +#line 695 "parse.y" +sqlite3ExprDelete(pParse->db, (yypminor->yy122)); +#line 1329 "parse.c" +} break; - case 174: - case 182: - case 192: - case 195: - case 197: - case 199: - case 209: - case 211: - case 212: - case 215: - case 216: - case 222: -#line 876 "parse.y" -{sqlite3ExprListDelete((yypminor->yy174));} -#line 1302 "parse.c" + case 178: /* idxlist_opt */ + case 186: /* idxlist */ + case 196: /* selcollist */ + case 199: /* groupby_opt */ + case 201: /* orderby_opt */ + case 203: /* sclp */ + case 213: /* sortlist */ + case 215: /* nexprlist */ + case 216: /* setlist */ + case 219: /* itemlist */ + case 220: /* exprlist */ + case 226: /* case_exprlist */ +{ +#line 953 "parse.y" +sqlite3ExprListDelete(pParse->db, (yypminor->yy442)); +#line 1347 "parse.c" +} break; - case 188: - case 193: - case 201: - case 202: -#line 489 "parse.y" -{sqlite3SrcListDelete((yypminor->yy373));} -#line 1310 "parse.c" + case 192: /* fullname */ + case 197: /* from */ + case 205: /* seltablist */ + case 206: /* stl_prefix */ +{ +#line 513 "parse.y" +sqlite3SrcListDelete(pParse->db, (yypminor->yy347)); +#line 1357 "parse.c" +} break; - case 205: - case 208: - case 214: -#line 506 "parse.y" -{sqlite3IdListDelete((yypminor->yy432));} -#line 1317 "parse.c" + case 210: /* using_opt */ + case 212: /* inscollist */ + case 218: /* inscollist_opt */ +{ +#line 545 "parse.y" +sqlite3IdListDelete(pParse->db, (yypminor->yy180)); +#line 1366 "parse.c" +} break; - case 231: - case 236: -#line 979 "parse.y" -{sqlite3DeleteTriggerStep((yypminor->yy243));} -#line 1323 "parse.c" + case 234: /* trigger_cmd_list */ + case 239: /* trigger_cmd */ +{ +#line 1057 "parse.y" +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy327)); +#line 1374 "parse.c" +} break; - case 233: -#line 965 "parse.y" -{sqlite3IdListDelete((yypminor->yy370).b);} -#line 1328 "parse.c" - break; - case 238: -#line 1052 "parse.y" -{sqlite3ExprDelete((yypminor->yy386));} -#line 1333 "parse.c" + case 236: /* trigger_event */ +{ +#line 1043 "parse.y" +sqlite3IdListDelete(pParse->db, (yypminor->yy410).b); +#line 1381 "parse.c" +} break; default: break; /* If no destructor action specified: do nothing */ } @@ -1355,7 +1405,7 @@ static int yy_pop_parser_stack(yyParser *pParser){ } #endif yymajor = yytos->major; - yy_destructor( yymajor, &yytos->minor); + yy_destructor(pParser, yymajor, &yytos->minor); pParser->yyidx--; return yymajor; } @@ -1385,6 +1435,16 @@ void sqlite3ParserFree( (*freeProc)((void*)pParser); } +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef YYTRACKMAXSTACKDEPTH +int sqlite3ParserStackPeak(void *p){ + yyParser *pParser = (yyParser*)p; + return pParser->yyidxMax; +} +#endif + /* ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. @@ -1403,14 +1463,12 @@ static int yy_find_shift_action( if( stateno>YY_SHIFT_MAX || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ return yy_default[stateno]; } - if( iLookAhead==YYNOCODE ){ - return YY_NO_ACTION; - } + assert( iLookAhead!=YYNOCODE ); i += iLookAhead; if( i<0 || i>=YY_SZ_ACTTAB || yy_lookahead[i]!=iLookAhead ){ if( iLookAhead>0 ){ #ifdef YYFALLBACK - int iFallback; /* Fallback token */ + YYCODETYPE iFallback; /* Fallback token */ if( iLookAheadyystack[pParser->yyidx].stateno; */ - - if( stateno>YY_REDUCE_MAX || - (i = yy_reduce_ofst[stateno])==YY_REDUCE_USE_DFLT ){ +#ifdef YYERRORSYMBOL + if( stateno>YY_REDUCE_MAX ){ return yy_default[stateno]; } - if( iLookAhead==YYNOCODE ){ - return YY_NO_ACTION; - } +#else + assert( stateno<=YY_REDUCE_MAX ); +#endif + i = yy_reduce_ofst[stateno]; + assert( i!=YY_REDUCE_USE_DFLT ); + assert( iLookAhead!=YYNOCODE ); i += iLookAhead; +#ifdef YYERRORSYMBOL if( i<0 || i>=YY_SZ_ACTTAB || yy_lookahead[i]!=iLookAhead ){ return yy_default[stateno]; - }else{ - return yy_action[i]; } +#else + assert( i>=0 && iyyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will execute if the parser ** stack every overflows */ -#line 44 "parse.y" +#line 40 "parse.y" + UNUSED_PARAMETER(yypMinor); /* Silence some compiler warnings */ sqlite3ErrorMsg(pParse, "parser stack overflow"); pParse->parseError = 1; -#line 1497 "parse.c" +#line 1559 "parse.c" sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument var */ } @@ -1502,10 +1566,15 @@ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ - YYMINORTYPE *yypMinor /* Pointer ot the minor token to shift in */ + YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */ ){ yyStackEntry *yytos; yypParser->yyidx++; +#ifdef YYTRACKMAXSTACKDEPTH + if( yypParser->yyidx>yypParser->yyidxMax ){ + yypParser->yyidxMax = yypParser->yyidx; + } +#endif #if YYSTACKDEPTH>0 if( yypParser->yyidx>=YYSTACKDEPTH ){ yyStackOverflow(yypParser, yypMinor); @@ -1521,8 +1590,8 @@ static void yy_shift( } #endif yytos = &yypParser->yystack[yypParser->yyidx]; - yytos->stateno = yyNewState; - yytos->major = yyMajor; + yytos->stateno = (YYACTIONTYPE)yyNewState; + yytos->major = (YYCODETYPE)yyMajor; yytos->minor = *yypMinor; #ifndef NDEBUG if( yyTraceFILE && yypParser->yyidx>0 ){ @@ -1543,317 +1612,325 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ unsigned char nrhs; /* Number of right-hand side symbols in the rule */ } yyRuleInfo[] = { - { 139, 1 }, - { 140, 2 }, - { 140, 1 }, { 142, 1 }, - { 141, 1 }, - { 141, 3 }, - { 144, 0 }, + { 143, 2 }, + { 143, 1 }, { 144, 1 }, { 144, 3 }, - { 143, 3 }, - { 146, 0 }, - { 146, 1 }, - { 146, 2 }, { 145, 0 }, { 145, 1 }, - { 145, 1 }, - { 145, 1 }, - { 143, 2 }, - { 143, 2 }, - { 143, 2 }, - { 143, 2 }, - { 148, 6 }, - { 151, 0 }, - { 151, 3 }, - { 150, 1 }, - { 150, 0 }, - { 149, 4 }, + { 145, 3 }, + { 146, 1 }, + { 147, 3 }, + { 149, 0 }, + { 149, 1 }, { 149, 2 }, - { 153, 3 }, - { 153, 1 }, - { 156, 3 }, + { 148, 0 }, + { 148, 1 }, + { 148, 1 }, + { 148, 1 }, + { 147, 2 }, + { 147, 2 }, + { 147, 2 }, + { 151, 1 }, + { 151, 0 }, + { 147, 2 }, + { 147, 3 }, + { 147, 5 }, + { 147, 2 }, + { 152, 6 }, + { 155, 0 }, + { 155, 3 }, + { 154, 1 }, + { 154, 0 }, + { 153, 4 }, + { 153, 2 }, + { 157, 3 }, { 157, 1 }, - { 160, 1 }, + { 160, 3 }, { 161, 1 }, - { 147, 1 }, - { 147, 1 }, - { 147, 1 }, - { 158, 0 }, - { 158, 1 }, + { 164, 1 }, + { 165, 1 }, + { 150, 1 }, + { 150, 1 }, + { 150, 1 }, + { 162, 0 }, { 162, 1 }, - { 162, 4 }, - { 162, 6 }, - { 163, 1 }, - { 163, 2 }, - { 164, 1 }, - { 164, 1 }, - { 159, 2 }, - { 159, 0 }, - { 167, 3 }, + { 166, 1 }, + { 166, 4 }, + { 166, 6 }, { 167, 1 }, - { 168, 2 }, - { 168, 4 }, - { 168, 3 }, - { 168, 3 }, - { 168, 2 }, - { 168, 2 }, - { 168, 3 }, - { 168, 5 }, - { 168, 2 }, - { 168, 4 }, - { 168, 4 }, + { 167, 2 }, { 168, 1 }, - { 168, 2 }, - { 173, 0 }, - { 173, 1 }, - { 175, 0 }, - { 175, 2 }, - { 177, 2 }, - { 177, 3 }, - { 177, 3 }, - { 177, 3 }, - { 178, 2 }, - { 178, 2 }, - { 178, 1 }, - { 178, 1 }, - { 176, 3 }, - { 176, 2 }, + { 168, 1 }, + { 163, 2 }, + { 163, 0 }, + { 171, 3 }, + { 171, 1 }, + { 172, 2 }, + { 172, 4 }, + { 172, 3 }, + { 172, 3 }, + { 172, 2 }, + { 172, 2 }, + { 172, 3 }, + { 172, 5 }, + { 172, 2 }, + { 172, 4 }, + { 172, 4 }, + { 172, 1 }, + { 172, 2 }, + { 177, 0 }, + { 177, 1 }, { 179, 0 }, { 179, 2 }, - { 179, 2 }, - { 154, 0 }, - { 154, 2 }, + { 181, 2 }, + { 181, 3 }, + { 181, 3 }, + { 181, 3 }, + { 182, 2 }, + { 182, 2 }, + { 182, 1 }, + { 182, 1 }, { 180, 3 }, { 180, 2 }, - { 180, 1 }, - { 181, 2 }, - { 181, 7 }, - { 181, 5 }, - { 181, 5 }, - { 181, 10 }, { 183, 0 }, - { 183, 1 }, - { 171, 0 }, - { 171, 3 }, - { 184, 0 }, + { 183, 2 }, + { 183, 2 }, + { 158, 0 }, + { 158, 2 }, + { 184, 3 }, { 184, 2 }, - { 185, 1 }, - { 185, 1 }, - { 185, 1 }, - { 143, 4 }, - { 187, 2 }, + { 184, 1 }, + { 185, 2 }, + { 185, 7 }, + { 185, 5 }, + { 185, 5 }, + { 185, 10 }, { 187, 0 }, - { 143, 8 }, - { 143, 4 }, - { 143, 1 }, - { 155, 1 }, - { 155, 3 }, - { 190, 1 }, - { 190, 2 }, - { 190, 1 }, - { 189, 9 }, - { 191, 1 }, - { 191, 1 }, - { 191, 0 }, - { 199, 2 }, - { 199, 0 }, - { 192, 3 }, - { 192, 2 }, - { 192, 4 }, - { 200, 2 }, - { 200, 1 }, - { 200, 0 }, - { 193, 0 }, - { 193, 2 }, - { 202, 2 }, - { 202, 0 }, - { 201, 6 }, - { 201, 7 }, - { 206, 1 }, - { 206, 1 }, - { 152, 0 }, - { 152, 2 }, + { 187, 1 }, + { 175, 0 }, + { 175, 3 }, + { 188, 0 }, { 188, 2 }, - { 203, 1 }, - { 203, 2 }, - { 203, 3 }, - { 203, 4 }, - { 204, 2 }, - { 204, 0 }, - { 205, 4 }, - { 205, 0 }, - { 197, 0 }, - { 197, 3 }, - { 209, 4 }, - { 209, 2 }, - { 210, 1 }, - { 172, 1 }, - { 172, 1 }, - { 172, 0 }, + { 189, 1 }, + { 189, 1 }, + { 189, 1 }, + { 147, 4 }, + { 191, 2 }, + { 191, 0 }, + { 147, 8 }, + { 147, 4 }, + { 147, 1 }, + { 159, 1 }, + { 159, 3 }, + { 194, 1 }, + { 194, 2 }, + { 194, 1 }, + { 193, 9 }, + { 195, 1 }, + { 195, 1 }, { 195, 0 }, - { 195, 3 }, - { 196, 0 }, + { 203, 2 }, + { 203, 0 }, + { 196, 3 }, { 196, 2 }, + { 196, 4 }, + { 204, 2 }, + { 204, 1 }, + { 204, 0 }, + { 197, 0 }, + { 197, 2 }, + { 206, 2 }, + { 206, 0 }, + { 205, 7 }, + { 205, 7 }, + { 205, 7 }, + { 156, 0 }, + { 156, 2 }, + { 192, 2 }, + { 207, 1 }, + { 207, 2 }, + { 207, 3 }, + { 207, 4 }, + { 209, 2 }, + { 209, 0 }, + { 208, 0 }, + { 208, 3 }, + { 208, 2 }, + { 210, 4 }, + { 210, 0 }, + { 201, 0 }, + { 201, 3 }, + { 213, 4 }, + { 213, 2 }, + { 214, 1 }, + { 176, 1 }, + { 176, 1 }, + { 176, 0 }, + { 199, 0 }, + { 199, 3 }, + { 200, 0 }, + { 200, 2 }, + { 202, 0 }, + { 202, 2 }, + { 202, 4 }, + { 202, 4 }, + { 147, 5 }, { 198, 0 }, { 198, 2 }, - { 198, 4 }, - { 198, 4 }, - { 143, 4 }, - { 194, 0 }, - { 194, 2 }, - { 143, 6 }, - { 212, 5 }, + { 147, 7 }, + { 216, 5 }, + { 216, 3 }, + { 147, 8 }, + { 147, 5 }, + { 147, 6 }, + { 217, 2 }, + { 217, 1 }, + { 219, 3 }, + { 219, 1 }, + { 218, 0 }, + { 218, 3 }, { 212, 3 }, - { 143, 8 }, - { 143, 5 }, - { 143, 6 }, - { 213, 2 }, - { 213, 1 }, + { 212, 1 }, + { 174, 1 }, + { 174, 3 }, + { 173, 1 }, + { 174, 1 }, + { 174, 1 }, + { 174, 3 }, + { 174, 5 }, + { 173, 1 }, + { 173, 1 }, + { 174, 1 }, + { 174, 1 }, + { 174, 3 }, + { 174, 6 }, + { 174, 5 }, + { 174, 4 }, + { 173, 1 }, + { 174, 3 }, + { 174, 3 }, + { 174, 3 }, + { 174, 3 }, + { 174, 3 }, + { 174, 3 }, + { 174, 3 }, + { 174, 3 }, + { 221, 1 }, + { 221, 2 }, + { 221, 1 }, + { 221, 2 }, + { 222, 2 }, + { 222, 0 }, + { 174, 4 }, + { 174, 2 }, + { 174, 3 }, + { 174, 3 }, + { 174, 4 }, + { 174, 2 }, + { 174, 2 }, + { 174, 2 }, + { 174, 2 }, + { 223, 1 }, + { 223, 2 }, + { 174, 5 }, + { 224, 1 }, + { 224, 2 }, + { 174, 5 }, + { 174, 3 }, + { 174, 5 }, + { 174, 4 }, + { 174, 4 }, + { 174, 5 }, + { 226, 5 }, + { 226, 4 }, + { 227, 2 }, + { 227, 0 }, + { 225, 1 }, + { 225, 0 }, + { 220, 1 }, + { 220, 0 }, { 215, 3 }, { 215, 1 }, - { 214, 0 }, - { 214, 3 }, - { 208, 3 }, - { 208, 1 }, - { 170, 1 }, - { 170, 3 }, - { 169, 1 }, - { 170, 1 }, - { 170, 1 }, - { 170, 3 }, - { 170, 5 }, - { 169, 1 }, - { 169, 1 }, - { 170, 1 }, - { 170, 1 }, - { 170, 3 }, - { 170, 6 }, - { 170, 5 }, - { 170, 4 }, - { 169, 1 }, - { 170, 3 }, - { 170, 3 }, - { 170, 3 }, - { 170, 3 }, - { 170, 3 }, - { 170, 3 }, - { 170, 3 }, - { 170, 3 }, - { 217, 1 }, - { 217, 2 }, - { 217, 1 }, - { 217, 2 }, - { 218, 2 }, - { 218, 0 }, - { 170, 4 }, - { 170, 2 }, - { 170, 3 }, - { 170, 3 }, - { 170, 4 }, - { 170, 2 }, - { 170, 2 }, - { 170, 2 }, - { 219, 1 }, - { 219, 2 }, - { 170, 5 }, - { 220, 1 }, - { 220, 2 }, - { 170, 5 }, - { 170, 3 }, - { 170, 5 }, - { 170, 4 }, - { 170, 4 }, - { 170, 5 }, - { 222, 5 }, - { 222, 4 }, - { 223, 2 }, - { 223, 0 }, - { 221, 1 }, - { 221, 0 }, - { 216, 1 }, - { 216, 0 }, - { 211, 3 }, - { 211, 1 }, - { 143, 11 }, - { 224, 1 }, - { 224, 0 }, - { 174, 0 }, - { 174, 3 }, - { 182, 5 }, - { 182, 3 }, - { 225, 1 }, - { 226, 0 }, - { 226, 2 }, - { 143, 4 }, - { 143, 1 }, - { 143, 2 }, - { 143, 5 }, - { 143, 5 }, - { 143, 5 }, - { 143, 6 }, - { 143, 3 }, - { 227, 1 }, - { 227, 1 }, - { 165, 2 }, - { 166, 2 }, - { 229, 1 }, + { 147, 11 }, { 228, 1 }, { 228, 0 }, - { 143, 5 }, - { 230, 11 }, + { 178, 0 }, + { 178, 3 }, + { 186, 5 }, + { 186, 3 }, + { 229, 0 }, + { 229, 2 }, + { 147, 4 }, + { 147, 1 }, + { 147, 2 }, + { 147, 5 }, + { 147, 5 }, + { 147, 5 }, + { 147, 5 }, + { 147, 6 }, + { 147, 3 }, + { 230, 1 }, + { 230, 1 }, + { 169, 2 }, + { 170, 2 }, { 232, 1 }, - { 232, 1 }, - { 232, 2 }, - { 232, 0 }, - { 233, 1 }, - { 233, 1 }, - { 233, 3 }, - { 234, 0 }, - { 234, 3 }, - { 235, 0 }, - { 235, 2 }, - { 231, 3 }, + { 231, 1 }, { 231, 0 }, - { 236, 6 }, - { 236, 8 }, - { 236, 5 }, - { 236, 4 }, + { 147, 5 }, + { 233, 11 }, + { 235, 1 }, + { 235, 1 }, + { 235, 2 }, + { 235, 0 }, { 236, 1 }, - { 170, 4 }, - { 170, 6 }, - { 186, 1 }, - { 186, 1 }, - { 186, 1 }, - { 143, 4 }, - { 143, 6 }, - { 143, 3 }, + { 236, 1 }, + { 236, 3 }, + { 237, 0 }, + { 237, 3 }, { 238, 0 }, { 238, 2 }, - { 237, 1 }, - { 237, 0 }, - { 143, 1 }, - { 143, 3 }, - { 143, 1 }, - { 143, 3 }, - { 143, 6 }, - { 143, 6 }, + { 234, 3 }, + { 234, 2 }, + { 239, 6 }, + { 239, 8 }, + { 239, 5 }, + { 239, 4 }, { 239, 1 }, - { 240, 0 }, + { 174, 4 }, + { 174, 6 }, + { 190, 1 }, + { 190, 1 }, + { 190, 1 }, + { 147, 4 }, + { 147, 6 }, + { 147, 3 }, + { 241, 0 }, + { 241, 2 }, { 240, 1 }, - { 143, 1 }, - { 143, 4 }, - { 241, 7 }, + { 240, 0 }, + { 147, 1 }, + { 147, 3 }, + { 147, 1 }, + { 147, 3 }, + { 147, 6 }, + { 147, 6 }, { 242, 1 }, - { 242, 3 }, { 243, 0 }, - { 243, 2 }, - { 244, 1 }, - { 244, 3 }, + { 243, 1 }, + { 147, 1 }, + { 147, 4 }, + { 244, 7 }, { 245, 1 }, + { 245, 3 }, { 246, 0 }, { 246, 2 }, + { 247, 1 }, + { 247, 3 }, + { 248, 1 }, + { 249, 0 }, + { 249, 2 }, }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1895,7 +1972,8 @@ static void yy_reduce( ** from wireshark this week. Clearly they are stressing Lemon in ways ** that it has not been previously stressed... (SQLite ticket #2172) */ - memset(&yygotominor, 0, sizeof(yygotominor)); + /*memset(&yygotominor, 0, sizeof(yygotominor));*/ + yygotominor = yyzerominor; switch( yyruleno ){ @@ -1907,1359 +1985,1398 @@ static void yy_reduce( ** #line ** break; */ - case 0: - case 1: - case 2: - case 4: - case 5: - case 10: - case 11: - case 12: - case 20: - case 28: - case 29: - case 37: - case 44: - case 45: - case 46: - case 47: - case 48: - case 49: - case 55: - case 82: - case 83: - case 84: - case 85: - case 256: - case 257: - case 267: - case 268: - case 288: - case 289: - case 297: - case 298: - case 302: - case 303: - case 305: - case 309: -#line 97 "parse.y" + case 0: /* input ::= cmdlist */ + case 1: /* cmdlist ::= cmdlist ecmd */ + case 2: /* cmdlist ::= ecmd */ + case 3: /* ecmd ::= SEMI */ + case 4: /* ecmd ::= explain cmdx SEMI */ + case 10: /* trans_opt ::= */ + case 11: /* trans_opt ::= TRANSACTION */ + case 12: /* trans_opt ::= TRANSACTION nm */ + case 20: /* savepoint_opt ::= SAVEPOINT */ + case 21: /* savepoint_opt ::= */ + case 25: /* cmd ::= create_table create_table_args */ + case 33: /* columnlist ::= columnlist COMMA column */ + case 34: /* columnlist ::= column */ + case 42: /* type ::= */ + case 49: /* signed ::= plus_num */ + case 50: /* signed ::= minus_num */ + case 51: /* carglist ::= carglist carg */ + case 52: /* carglist ::= */ + case 53: /* carg ::= CONSTRAINT nm ccons */ + case 54: /* carg ::= ccons */ + case 60: /* ccons ::= NULL onconf */ + case 87: /* conslist ::= conslist COMMA tcons */ + case 88: /* conslist ::= conslist tcons */ + case 89: /* conslist ::= tcons */ + case 90: /* tcons ::= CONSTRAINT nm */ + case 264: /* plus_opt ::= PLUS */ + case 265: /* plus_opt ::= */ + case 275: /* foreach_clause ::= */ + case 276: /* foreach_clause ::= FOR EACH ROW */ + case 296: /* database_kw_opt ::= DATABASE */ + case 297: /* database_kw_opt ::= */ + case 305: /* kwcolumn_opt ::= */ + case 306: /* kwcolumn_opt ::= COLUMNKW */ + case 310: /* vtabarglist ::= vtabarg */ + case 311: /* vtabarglist ::= vtabarglist COMMA vtabarg */ + case 313: /* vtabarg ::= vtabarg vtabargtoken */ + case 317: /* anylist ::= */ +#line 93 "parse.y" { } -#line 1951 "parse.c" +#line 2029 "parse.c" break; - case 3: -#line 100 "parse.y" -{ sqlite3FinishCoding(pParse); } -#line 1956 "parse.c" - break; - case 6: -#line 103 "parse.y" + case 5: /* explain ::= */ +#line 98 "parse.y" { sqlite3BeginParse(pParse, 0); } -#line 1961 "parse.c" +#line 2034 "parse.c" break; - case 7: -#line 105 "parse.y" + case 6: /* explain ::= EXPLAIN */ +#line 100 "parse.y" { sqlite3BeginParse(pParse, 1); } -#line 1966 "parse.c" +#line 2039 "parse.c" break; - case 8: -#line 106 "parse.y" + case 7: /* explain ::= EXPLAIN QUERY PLAN */ +#line 101 "parse.y" { sqlite3BeginParse(pParse, 2); } -#line 1971 "parse.c" +#line 2044 "parse.c" break; - case 9: -#line 112 "parse.y" -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy46);} -#line 1976 "parse.c" + case 8: /* cmdx ::= cmd */ +#line 103 "parse.y" +{ sqlite3FinishCoding(pParse); } +#line 2049 "parse.c" break; - case 13: + case 9: /* cmd ::= BEGIN transtype trans_opt */ +#line 108 "parse.y" +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy392);} +#line 2054 "parse.c" + break; + case 13: /* transtype ::= */ +#line 113 "parse.y" +{yygotominor.yy392 = TK_DEFERRED;} +#line 2059 "parse.c" + break; + case 14: /* transtype ::= DEFERRED */ + case 15: /* transtype ::= IMMEDIATE */ + case 16: /* transtype ::= EXCLUSIVE */ + case 112: /* multiselect_op ::= UNION */ + case 114: /* multiselect_op ::= EXCEPT|INTERSECT */ +#line 114 "parse.y" +{yygotominor.yy392 = yymsp[0].major;} +#line 2068 "parse.c" + break; + case 17: /* cmd ::= COMMIT trans_opt */ + case 18: /* cmd ::= END trans_opt */ #line 117 "parse.y" -{yygotominor.yy46 = TK_DEFERRED;} -#line 1981 "parse.c" - break; - case 14: - case 15: - case 16: - case 107: - case 109: -#line 118 "parse.y" -{yygotominor.yy46 = yymsp[0].major;} -#line 1990 "parse.c" - break; - case 17: - case 18: -#line 121 "parse.y" {sqlite3CommitTransaction(pParse);} -#line 1996 "parse.c" +#line 2074 "parse.c" break; - case 19: -#line 123 "parse.y" + case 19: /* cmd ::= ROLLBACK trans_opt */ +#line 119 "parse.y" {sqlite3RollbackTransaction(pParse);} -#line 2001 "parse.c" +#line 2079 "parse.c" break; - case 21: -#line 128 "parse.y" + case 22: /* cmd ::= SAVEPOINT nm */ +#line 123 "parse.y" { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy410,&yymsp[0].minor.yy410,yymsp[-4].minor.yy46,0,0,yymsp[-2].minor.yy46); + sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); } -#line 2008 "parse.c" +#line 2086 "parse.c" break; - case 22: - case 25: - case 63: - case 77: - case 79: - case 90: - case 101: - case 112: - case 113: - case 212: - case 215: -#line 132 "parse.y" -{yygotominor.yy46 = 0;} -#line 2023 "parse.c" - break; - case 23: - case 24: - case 64: - case 78: - case 100: - case 111: - case 213: - case 216: -#line 133 "parse.y" -{yygotominor.yy46 = 1;} -#line 2035 "parse.c" - break; - case 26: -#line 139 "parse.y" + case 23: /* cmd ::= RELEASE savepoint_opt nm */ +#line 126 "parse.y" { - sqlite3EndTable(pParse,&yymsp[-1].minor.yy410,&yymsp[0].minor.yy0,0); + sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); } -#line 2042 "parse.c" +#line 2093 "parse.c" break; - case 27: -#line 142 "parse.y" + case 24: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ +#line 129 "parse.y" { - sqlite3EndTable(pParse,0,0,yymsp[0].minor.yy219); - sqlite3SelectDelete(yymsp[0].minor.yy219); + sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); } -#line 2050 "parse.c" +#line 2100 "parse.c" break; - case 30: -#line 154 "parse.y" + case 26: /* create_table ::= CREATE temp TABLE ifnotexists nm dbnm */ +#line 136 "parse.y" { - yygotominor.yy410.z = yymsp[-2].minor.yy410.z; - yygotominor.yy410.n = (pParse->sLastToken.z-yymsp[-2].minor.yy410.z) + pParse->sLastToken.n; + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy392,0,0,yymsp[-2].minor.yy392); } -#line 2058 "parse.c" +#line 2107 "parse.c" break; - case 31: -#line 158 "parse.y" + case 27: /* ifnotexists ::= */ + case 30: /* temp ::= */ + case 68: /* autoinc ::= */ + case 82: /* init_deferred_pred_opt ::= */ + case 84: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + case 95: /* defer_subclause_opt ::= */ + case 106: /* ifexists ::= */ + case 117: /* distinct ::= ALL */ + case 118: /* distinct ::= */ + case 220: /* between_op ::= BETWEEN */ + case 223: /* in_op ::= IN */ +#line 140 "parse.y" +{yygotominor.yy392 = 0;} +#line 2122 "parse.c" + break; + case 28: /* ifnotexists ::= IF NOT EXISTS */ + case 29: /* temp ::= TEMP */ + case 69: /* autoinc ::= AUTOINCR */ + case 83: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ + case 105: /* ifexists ::= IF EXISTS */ + case 116: /* distinct ::= DISTINCT */ + case 221: /* between_op ::= NOT BETWEEN */ + case 224: /* in_op ::= NOT IN */ +#line 141 "parse.y" +{yygotominor.yy392 = 1;} +#line 2134 "parse.c" + break; + case 31: /* create_table_args ::= LP columnlist conslist_opt RP */ +#line 147 "parse.y" { - sqlite3AddColumn(pParse,&yymsp[0].minor.yy410); - yygotominor.yy410 = yymsp[0].minor.yy410; + sqlite3EndTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0); } -#line 2066 "parse.c" +#line 2141 "parse.c" break; - case 32: - case 33: - case 34: - case 35: - case 36: - case 255: -#line 168 "parse.y" -{yygotominor.yy410 = yymsp[0].minor.yy0;} -#line 2076 "parse.c" - break; - case 38: -#line 229 "parse.y" -{sqlite3AddColumnType(pParse,&yymsp[0].minor.yy410);} -#line 2081 "parse.c" - break; - case 39: - case 42: - case 119: - case 120: - case 131: - case 240: - case 242: - case 251: - case 252: - case 253: - case 254: -#line 230 "parse.y" -{yygotominor.yy410 = yymsp[0].minor.yy410;} -#line 2096 "parse.c" - break; - case 40: -#line 231 "parse.y" + case 32: /* create_table_args ::= AS select */ +#line 150 "parse.y" { - yygotominor.yy410.z = yymsp[-3].minor.yy410.z; - yygotominor.yy410.n = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy410.z; + sqlite3EndTable(pParse,0,0,yymsp[0].minor.yy159); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy159); } -#line 2104 "parse.c" - break; - case 41: -#line 235 "parse.y" -{ - yygotominor.yy410.z = yymsp[-5].minor.yy410.z; - yygotominor.yy410.n = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy410.z; -} -#line 2112 "parse.c" - break; - case 43: -#line 241 "parse.y" -{yygotominor.yy410.z=yymsp[-1].minor.yy410.z; yygotominor.yy410.n=yymsp[0].minor.yy410.n+(yymsp[0].minor.yy410.z-yymsp[-1].minor.yy410.z);} -#line 2117 "parse.c" - break; - case 50: - case 52: -#line 252 "parse.y" -{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy172);} -#line 2123 "parse.c" - break; - case 51: -#line 253 "parse.y" -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy172);} -#line 2128 "parse.c" - break; - case 53: -#line 255 "parse.y" -{ - Expr *p = sqlite3Expr(TK_UMINUS, yymsp[0].minor.yy172, 0, 0); - sqlite3AddDefaultValue(pParse,p); -} -#line 2136 "parse.c" - break; - case 54: -#line 259 "parse.y" -{ - Expr *p = sqlite3Expr(TK_STRING, 0, 0, &yymsp[0].minor.yy410); - sqlite3AddDefaultValue(pParse,p); -} -#line 2144 "parse.c" - break; - case 56: -#line 268 "parse.y" -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy46);} #line 2149 "parse.c" break; - case 57: -#line 270 "parse.y" -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy46,yymsp[0].minor.yy46,yymsp[-2].minor.yy46);} -#line 2154 "parse.c" + case 35: /* column ::= columnid type carglist */ +#line 162 "parse.y" +{ + yygotominor.yy0.z = yymsp[-2].minor.yy0.z; + yygotominor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-2].minor.yy0.z) + pParse->sLastToken.n; +} +#line 2157 "parse.c" break; - case 58: -#line 271 "parse.y" -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy46,0,0,0,0);} -#line 2159 "parse.c" + case 36: /* columnid ::= nm */ +#line 166 "parse.y" +{ + sqlite3AddColumn(pParse,&yymsp[0].minor.yy0); + yygotominor.yy0 = yymsp[0].minor.yy0; +} +#line 2165 "parse.c" break; - case 59: -#line 272 "parse.y" -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy172);} -#line 2164 "parse.c" + case 37: /* id ::= ID */ + case 38: /* ids ::= ID|STRING */ + case 39: /* nm ::= ID */ + case 40: /* nm ::= STRING */ + case 41: /* nm ::= JOIN_KW */ + case 44: /* typetoken ::= typename */ + case 47: /* typename ::= ids */ + case 124: /* as ::= AS nm */ + case 125: /* as ::= ids */ + case 135: /* dbnm ::= DOT nm */ + case 144: /* indexed_opt ::= INDEXED BY nm */ + case 249: /* collate ::= COLLATE ids */ + case 259: /* nmnum ::= plus_num */ + case 260: /* nmnum ::= nm */ + case 261: /* plus_num ::= plus_opt number */ + case 262: /* minus_num ::= MINUS number */ + case 263: /* number ::= INTEGER|FLOAT */ +#line 176 "parse.y" +{yygotominor.yy0 = yymsp[0].minor.yy0;} +#line 2186 "parse.c" break; - case 60: -#line 274 "parse.y" -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy410,yymsp[-1].minor.yy174,yymsp[0].minor.yy46);} -#line 2169 "parse.c" + case 43: /* type ::= typetoken */ +#line 237 "parse.y" +{sqlite3AddColumnType(pParse,&yymsp[0].minor.yy0);} +#line 2191 "parse.c" break; - case 61: -#line 275 "parse.y" -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy46);} -#line 2174 "parse.c" - break; - case 62: -#line 276 "parse.y" -{sqlite3AddCollateType(pParse, (char*)yymsp[0].minor.yy410.z, yymsp[0].minor.yy410.n);} -#line 2179 "parse.c" - break; - case 65: -#line 289 "parse.y" -{ yygotominor.yy46 = OE_Restrict * 0x010101; } -#line 2184 "parse.c" - break; - case 66: -#line 290 "parse.y" -{ yygotominor.yy46 = (yymsp[-1].minor.yy46 & yymsp[0].minor.yy405.mask) | yymsp[0].minor.yy405.value; } -#line 2189 "parse.c" - break; - case 67: -#line 292 "parse.y" -{ yygotominor.yy405.value = 0; yygotominor.yy405.mask = 0x000000; } -#line 2194 "parse.c" - break; - case 68: -#line 293 "parse.y" -{ yygotominor.yy405.value = yymsp[0].minor.yy46; yygotominor.yy405.mask = 0x0000ff; } + case 45: /* typetoken ::= typename LP signed RP */ +#line 239 "parse.y" +{ + yygotominor.yy0.z = yymsp[-3].minor.yy0.z; + yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); +} #line 2199 "parse.c" break; - case 69: -#line 294 "parse.y" -{ yygotominor.yy405.value = yymsp[0].minor.yy46<<8; yygotominor.yy405.mask = 0x00ff00; } -#line 2204 "parse.c" + case 46: /* typetoken ::= typename LP signed COMMA signed RP */ +#line 243 "parse.y" +{ + yygotominor.yy0.z = yymsp[-5].minor.yy0.z; + yygotominor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); +} +#line 2207 "parse.c" break; - case 70: -#line 295 "parse.y" -{ yygotominor.yy405.value = yymsp[0].minor.yy46<<16; yygotominor.yy405.mask = 0xff0000; } -#line 2209 "parse.c" + case 48: /* typename ::= typename ids */ +#line 249 "parse.y" +{yygotominor.yy0.z=yymsp[-1].minor.yy0.z; yygotominor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} +#line 2212 "parse.c" break; - case 71: -#line 297 "parse.y" -{ yygotominor.yy46 = OE_SetNull; } -#line 2214 "parse.c" + case 55: /* ccons ::= DEFAULT term */ + case 57: /* ccons ::= DEFAULT PLUS term */ +#line 260 "parse.y" +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy122);} +#line 2218 "parse.c" break; - case 72: -#line 298 "parse.y" -{ yygotominor.yy46 = OE_SetDflt; } -#line 2219 "parse.c" + case 56: /* ccons ::= DEFAULT LP expr RP */ +#line 261 "parse.y" +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy122);} +#line 2223 "parse.c" break; - case 73: -#line 299 "parse.y" -{ yygotominor.yy46 = OE_Cascade; } -#line 2224 "parse.c" + case 58: /* ccons ::= DEFAULT MINUS term */ +#line 263 "parse.y" +{ + Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy122, 0, 0); + sqlite3ExprSpan(p,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy122->span); + sqlite3AddDefaultValue(pParse,p); +} +#line 2232 "parse.c" break; - case 74: -#line 300 "parse.y" -{ yygotominor.yy46 = OE_Restrict; } -#line 2229 "parse.c" - break; - case 75: - case 76: - case 91: - case 93: - case 95: - case 96: - case 166: -#line 302 "parse.y" -{yygotominor.yy46 = yymsp[0].minor.yy46;} + case 59: /* ccons ::= DEFAULT id */ +#line 268 "parse.y" +{ + Expr *p = sqlite3PExpr(pParse, TK_STRING, 0, 0, &yymsp[0].minor.yy0); + sqlite3AddDefaultValue(pParse,p); +} #line 2240 "parse.c" break; - case 80: -#line 312 "parse.y" -{yygotominor.yy410.n = 0; yygotominor.yy410.z = 0;} + case 61: /* ccons ::= NOT NULL onconf */ +#line 277 "parse.y" +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy392);} #line 2245 "parse.c" break; - case 81: -#line 313 "parse.y" -{yygotominor.yy410 = yymsp[-1].minor.yy0;} + case 62: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ +#line 279 "parse.y" +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy392,yymsp[0].minor.yy392,yymsp[-2].minor.yy392);} #line 2250 "parse.c" break; - case 86: -#line 319 "parse.y" -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy174,yymsp[0].minor.yy46,yymsp[-2].minor.yy46,0);} + case 63: /* ccons ::= UNIQUE onconf */ +#line 280 "parse.y" +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy392,0,0,0,0);} #line 2255 "parse.c" break; - case 87: -#line 321 "parse.y" -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy174,yymsp[0].minor.yy46,0,0,0,0);} + case 64: /* ccons ::= CHECK LP expr RP */ +#line 281 "parse.y" +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy122);} #line 2260 "parse.c" break; - case 88: -#line 322 "parse.y" -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy172);} + case 65: /* ccons ::= REFERENCES nm idxlist_opt refargs */ +#line 283 "parse.y" +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy442,yymsp[0].minor.yy392);} #line 2265 "parse.c" break; - case 89: -#line 324 "parse.y" -{ - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy174, &yymsp[-3].minor.yy410, yymsp[-2].minor.yy174, yymsp[-1].minor.yy46); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy46); -} -#line 2273 "parse.c" + case 66: /* ccons ::= defer_subclause */ +#line 284 "parse.y" +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy392);} +#line 2270 "parse.c" break; - case 92: - case 94: -#line 338 "parse.y" -{yygotominor.yy46 = OE_Default;} -#line 2279 "parse.c" + case 67: /* ccons ::= COLLATE ids */ +#line 285 "parse.y" +{sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} +#line 2275 "parse.c" break; - case 97: -#line 343 "parse.y" -{yygotominor.yy46 = OE_Ignore;} -#line 2284 "parse.c" + case 70: /* refargs ::= */ +#line 298 "parse.y" +{ yygotominor.yy392 = OE_Restrict * 0x010101; } +#line 2280 "parse.c" break; - case 98: - case 167: -#line 344 "parse.y" -{yygotominor.yy46 = OE_Replace;} + case 71: /* refargs ::= refargs refarg */ +#line 299 "parse.y" +{ yygotominor.yy392 = (yymsp[-1].minor.yy392 & ~yymsp[0].minor.yy207.mask) | yymsp[0].minor.yy207.value; } +#line 2285 "parse.c" + break; + case 72: /* refarg ::= MATCH nm */ +#line 301 "parse.y" +{ yygotominor.yy207.value = 0; yygotominor.yy207.mask = 0x000000; } #line 2290 "parse.c" break; - case 99: -#line 348 "parse.y" -{ - sqlite3DropTable(pParse, yymsp[0].minor.yy373, 0, yymsp[-1].minor.yy46); -} -#line 2297 "parse.c" + case 73: /* refarg ::= ON DELETE refact */ +#line 302 "parse.y" +{ yygotominor.yy207.value = yymsp[0].minor.yy392; yygotominor.yy207.mask = 0x0000ff; } +#line 2295 "parse.c" break; - case 102: -#line 358 "parse.y" -{ - sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy410, &yymsp[-2].minor.yy410, yymsp[0].minor.yy219, yymsp[-6].minor.yy46, yymsp[-4].minor.yy46); -} -#line 2304 "parse.c" + case 74: /* refarg ::= ON UPDATE refact */ +#line 303 "parse.y" +{ yygotominor.yy207.value = yymsp[0].minor.yy392<<8; yygotominor.yy207.mask = 0x00ff00; } +#line 2300 "parse.c" break; - case 103: -#line 361 "parse.y" -{ - sqlite3DropTable(pParse, yymsp[0].minor.yy373, 1, yymsp[-1].minor.yy46); -} -#line 2311 "parse.c" + case 75: /* refarg ::= ON INSERT refact */ +#line 304 "parse.y" +{ yygotominor.yy207.value = yymsp[0].minor.yy392<<16; yygotominor.yy207.mask = 0xff0000; } +#line 2305 "parse.c" break; - case 104: -#line 368 "parse.y" -{ - sqlite3Select(pParse, yymsp[0].minor.yy219, SRT_Callback, 0, 0, 0, 0, 0); - sqlite3SelectDelete(yymsp[0].minor.yy219); -} -#line 2319 "parse.c" + case 76: /* refact ::= SET NULL */ +#line 306 "parse.y" +{ yygotominor.yy392 = OE_SetNull; } +#line 2310 "parse.c" break; - case 105: - case 128: -#line 378 "parse.y" -{yygotominor.yy219 = yymsp[0].minor.yy219;} + case 77: /* refact ::= SET DEFAULT */ +#line 307 "parse.y" +{ yygotominor.yy392 = OE_SetDflt; } +#line 2315 "parse.c" + break; + case 78: /* refact ::= CASCADE */ +#line 308 "parse.y" +{ yygotominor.yy392 = OE_Cascade; } +#line 2320 "parse.c" + break; + case 79: /* refact ::= RESTRICT */ +#line 309 "parse.y" +{ yygotominor.yy392 = OE_Restrict; } #line 2325 "parse.c" break; - case 106: -#line 380 "parse.y" -{ - if( yymsp[0].minor.yy219 ){ - yymsp[0].minor.yy219->op = yymsp[-1].minor.yy46; - yymsp[0].minor.yy219->pPrior = yymsp[-2].minor.yy219; - }else{ - sqlite3SelectDelete(yymsp[-2].minor.yy219); - } - yygotominor.yy219 = yymsp[0].minor.yy219; -} -#line 2338 "parse.c" + case 80: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + case 81: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + case 96: /* defer_subclause_opt ::= defer_subclause */ + case 98: /* onconf ::= ON CONFLICT resolvetype */ + case 100: /* orconf ::= OR resolvetype */ + case 101: /* resolvetype ::= raisetype */ + case 173: /* insert_cmd ::= INSERT orconf */ +#line 311 "parse.y" +{yygotominor.yy392 = yymsp[0].minor.yy392;} +#line 2336 "parse.c" break; - case 108: -#line 391 "parse.y" -{yygotominor.yy46 = TK_ALL;} -#line 2343 "parse.c" + case 85: /* conslist_opt ::= */ +#line 321 "parse.y" +{yygotominor.yy0.n = 0; yygotominor.yy0.z = 0;} +#line 2341 "parse.c" break; - case 110: -#line 395 "parse.y" -{ - yygotominor.yy219 = sqlite3SelectNew(yymsp[-6].minor.yy174,yymsp[-5].minor.yy373,yymsp[-4].minor.yy172,yymsp[-3].minor.yy174,yymsp[-2].minor.yy172,yymsp[-1].minor.yy174,yymsp[-7].minor.yy46,yymsp[0].minor.yy234.pLimit,yymsp[0].minor.yy234.pOffset); -} -#line 2350 "parse.c" + case 86: /* conslist_opt ::= COMMA conslist */ +#line 322 "parse.y" +{yygotominor.yy0 = yymsp[-1].minor.yy0;} +#line 2346 "parse.c" break; - case 114: - case 237: -#line 416 "parse.y" -{yygotominor.yy174 = yymsp[-1].minor.yy174;} + case 91: /* tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf */ +#line 328 "parse.y" +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy442,yymsp[0].minor.yy392,yymsp[-2].minor.yy392,0);} +#line 2351 "parse.c" + break; + case 92: /* tcons ::= UNIQUE LP idxlist RP onconf */ +#line 330 "parse.y" +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy442,yymsp[0].minor.yy392,0,0,0,0);} #line 2356 "parse.c" break; - case 115: - case 141: - case 149: - case 230: - case 236: -#line 417 "parse.y" -{yygotominor.yy174 = 0;} -#line 2365 "parse.c" + case 93: /* tcons ::= CHECK LP expr RP onconf */ +#line 331 "parse.y" +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy122);} +#line 2361 "parse.c" break; - case 116: -#line 418 "parse.y" + case 94: /* tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt */ +#line 333 "parse.y" { - yygotominor.yy174 = sqlite3ExprListAppend(yymsp[-2].minor.yy174,yymsp[-1].minor.yy172,yymsp[0].minor.yy410.n?&yymsp[0].minor.yy410:0); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy442, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy442, yymsp[-1].minor.yy392); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy392); } -#line 2372 "parse.c" +#line 2369 "parse.c" break; - case 117: -#line 421 "parse.y" + case 97: /* onconf ::= */ + case 99: /* orconf ::= */ +#line 347 "parse.y" +{yygotominor.yy392 = OE_Default;} +#line 2375 "parse.c" + break; + case 102: /* resolvetype ::= IGNORE */ +#line 352 "parse.y" +{yygotominor.yy392 = OE_Ignore;} +#line 2380 "parse.c" + break; + case 103: /* resolvetype ::= REPLACE */ + case 174: /* insert_cmd ::= REPLACE */ +#line 353 "parse.y" +{yygotominor.yy392 = OE_Replace;} +#line 2386 "parse.c" + break; + case 104: /* cmd ::= DROP TABLE ifexists fullname */ +#line 357 "parse.y" { - yygotominor.yy174 = sqlite3ExprListAppend(yymsp[-1].minor.yy174, sqlite3Expr(TK_ALL, 0, 0, 0), 0); + sqlite3DropTable(pParse, yymsp[0].minor.yy347, 0, yymsp[-1].minor.yy392); } -#line 2379 "parse.c" - break; - case 118: -#line 424 "parse.y" -{ - Expr *pRight = sqlite3Expr(TK_ALL, 0, 0, 0); - Expr *pLeft = sqlite3Expr(TK_ID, 0, 0, &yymsp[-2].minor.yy410); - yygotominor.yy174 = sqlite3ExprListAppend(yymsp[-3].minor.yy174, sqlite3Expr(TK_DOT, pLeft, pRight, 0), 0); -} -#line 2388 "parse.c" - break; - case 121: -#line 436 "parse.y" -{yygotominor.yy410.n = 0;} #line 2393 "parse.c" break; - case 122: -#line 448 "parse.y" -{yygotominor.yy373 = sqliteMalloc(sizeof(*yygotominor.yy373));} -#line 2398 "parse.c" - break; - case 123: -#line 449 "parse.y" + case 107: /* cmd ::= CREATE temp VIEW ifnotexists nm dbnm AS select */ +#line 367 "parse.y" { - yygotominor.yy373 = yymsp[0].minor.yy373; - sqlite3SrcListShiftJoinType(yygotominor.yy373); + sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy159, yymsp[-6].minor.yy392, yymsp[-4].minor.yy392); } -#line 2406 "parse.c" +#line 2400 "parse.c" break; - case 124: -#line 457 "parse.y" + case 108: /* cmd ::= DROP VIEW ifexists fullname */ +#line 370 "parse.y" { - yygotominor.yy373 = yymsp[-1].minor.yy373; - if( yygotominor.yy373 && yygotominor.yy373->nSrc>0 ) yygotominor.yy373->a[yygotominor.yy373->nSrc-1].jointype = yymsp[0].minor.yy46; + sqlite3DropTable(pParse, yymsp[0].minor.yy347, 1, yymsp[-1].minor.yy392); } -#line 2414 "parse.c" +#line 2407 "parse.c" break; - case 125: -#line 461 "parse.y" -{yygotominor.yy373 = 0;} -#line 2419 "parse.c" - break; - case 126: -#line 462 "parse.y" + case 109: /* cmd ::= select */ +#line 377 "parse.y" { - yygotominor.yy373 = sqlite3SrcListAppendFromTerm(yymsp[-5].minor.yy373,&yymsp[-4].minor.yy410,&yymsp[-3].minor.yy410,&yymsp[-2].minor.yy410,0,yymsp[-1].minor.yy172,yymsp[0].minor.yy432); + SelectDest dest = {SRT_Output, 0, 0, 0, 0}; + sqlite3Select(pParse, yymsp[0].minor.yy159, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy159); } -#line 2426 "parse.c" +#line 2416 "parse.c" break; - case 127: -#line 467 "parse.y" + case 110: /* select ::= oneselect */ +#line 388 "parse.y" +{yygotominor.yy159 = yymsp[0].minor.yy159;} +#line 2421 "parse.c" + break; + case 111: /* select ::= select multiselect_op oneselect */ +#line 390 "parse.y" { - yygotominor.yy373 = sqlite3SrcListAppendFromTerm(yymsp[-6].minor.yy373,0,0,&yymsp[-2].minor.yy410,yymsp[-4].minor.yy219,yymsp[-1].minor.yy172,yymsp[0].minor.yy432); + if( yymsp[0].minor.yy159 ){ + yymsp[0].minor.yy159->op = (u8)yymsp[-1].minor.yy392; + yymsp[0].minor.yy159->pPrior = yymsp[-2].minor.yy159; + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy159); } -#line 2433 "parse.c" + yygotominor.yy159 = yymsp[0].minor.yy159; +} +#line 2434 "parse.c" break; - case 129: -#line 478 "parse.y" + case 113: /* multiselect_op ::= UNION ALL */ +#line 401 "parse.y" +{yygotominor.yy392 = TK_ALL;} +#line 2439 "parse.c" + break; + case 115: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ +#line 405 "parse.y" { - sqlite3SrcListShiftJoinType(yymsp[0].minor.yy373); - yygotominor.yy219 = sqlite3SelectNew(0,yymsp[0].minor.yy373,0,0,0,0,0,0,0); - } -#line 2441 "parse.c" - break; - case 130: -#line 485 "parse.y" -{yygotominor.yy410.z=0; yygotominor.yy410.n=0;} + yygotominor.yy159 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy442,yymsp[-5].minor.yy347,yymsp[-4].minor.yy122,yymsp[-3].minor.yy442,yymsp[-2].minor.yy122,yymsp[-1].minor.yy442,yymsp[-7].minor.yy392,yymsp[0].minor.yy64.pLimit,yymsp[0].minor.yy64.pOffset); +} #line 2446 "parse.c" break; - case 132: -#line 490 "parse.y" -{yygotominor.yy373 = sqlite3SrcListAppend(0,&yymsp[-1].minor.yy410,&yymsp[0].minor.yy410);} -#line 2451 "parse.c" + case 119: /* sclp ::= selcollist COMMA */ + case 245: /* idxlist_opt ::= LP idxlist RP */ +#line 426 "parse.y" +{yygotominor.yy442 = yymsp[-1].minor.yy442;} +#line 2452 "parse.c" break; - case 133: -#line 494 "parse.y" -{ yygotominor.yy46 = JT_INNER; } -#line 2456 "parse.c" - break; - case 134: -#line 495 "parse.y" -{ yygotominor.yy46 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } + case 120: /* sclp ::= */ + case 148: /* orderby_opt ::= */ + case 156: /* groupby_opt ::= */ + case 238: /* exprlist ::= */ + case 244: /* idxlist_opt ::= */ +#line 427 "parse.y" +{yygotominor.yy442 = 0;} #line 2461 "parse.c" break; - case 135: -#line 496 "parse.y" -{ yygotominor.yy46 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy410,0); } -#line 2466 "parse.c" + case 121: /* selcollist ::= sclp expr as */ +#line 428 "parse.y" +{ + yygotominor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy442,yymsp[-1].minor.yy122,yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); +} +#line 2468 "parse.c" break; - case 136: -#line 498 "parse.y" -{ yygotominor.yy46 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy410,&yymsp[-1].minor.yy410); } -#line 2471 "parse.c" + case 122: /* selcollist ::= sclp STAR */ +#line 431 "parse.y" +{ + Expr *p = sqlite3PExpr(pParse, TK_ALL, 0, 0, 0); + yygotominor.yy442 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy442, p, 0); +} +#line 2476 "parse.c" break; - case 137: - case 145: - case 152: - case 159: - case 174: - case 202: - case 225: - case 227: -#line 502 "parse.y" -{yygotominor.yy172 = yymsp[0].minor.yy172;} -#line 2483 "parse.c" + case 123: /* selcollist ::= sclp nm DOT STAR */ +#line 435 "parse.y" +{ + Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &yymsp[0].minor.yy0); + Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); + Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + yygotominor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy442, pDot, 0); +} +#line 2486 "parse.c" break; - case 138: - case 151: - case 158: - case 203: - case 226: - case 228: -#line 503 "parse.y" -{yygotominor.yy172 = 0;} -#line 2493 "parse.c" + case 126: /* as ::= */ +#line 448 "parse.y" +{yygotominor.yy0.n = 0;} +#line 2491 "parse.c" break; - case 139: - case 171: -#line 507 "parse.y" -{yygotominor.yy432 = yymsp[-1].minor.yy432;} -#line 2499 "parse.c" + case 127: /* from ::= */ +#line 460 "parse.y" +{yygotominor.yy347 = sqlite3DbMallocZero(pParse->db, sizeof(*yygotominor.yy347));} +#line 2496 "parse.c" break; - case 140: - case 170: -#line 508 "parse.y" -{yygotominor.yy432 = 0;} -#line 2505 "parse.c" + case 128: /* from ::= FROM seltablist */ +#line 461 "parse.y" +{ + yygotominor.yy347 = yymsp[0].minor.yy347; + sqlite3SrcListShiftJoinType(yygotominor.yy347); +} +#line 2504 "parse.c" break; - case 142: - case 150: - case 229: -#line 519 "parse.y" -{yygotominor.yy174 = yymsp[0].minor.yy174;} + case 129: /* stl_prefix ::= seltablist joinop */ +#line 469 "parse.y" +{ + yygotominor.yy347 = yymsp[-1].minor.yy347; + if( yygotominor.yy347 && yygotominor.yy347->nSrc>0 ) yygotominor.yy347->a[yygotominor.yy347->nSrc-1].jointype = (u8)yymsp[0].minor.yy392; +} #line 2512 "parse.c" break; - case 143: + case 130: /* stl_prefix ::= */ +#line 473 "parse.y" +{yygotominor.yy347 = 0;} +#line 2517 "parse.c" + break; + case 131: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ +#line 474 "parse.y" +{ + yygotominor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); + sqlite3SrcListIndexedBy(pParse, yygotominor.yy347, &yymsp[-2].minor.yy0); +} +#line 2525 "parse.c" + break; + case 132: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ +#line 480 "parse.y" +{ + yygotominor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy159,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); + } +#line 2532 "parse.c" + break; + case 133: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ +#line 484 "parse.y" +{ + if( yymsp[-6].minor.yy347==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy122==0 && yymsp[0].minor.yy180==0 ){ + yygotominor.yy347 = yymsp[-4].minor.yy347; + }else{ + Select *pSubquery; + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy347); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy347,0,0,0,0,0,0,0); + yygotominor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); + } + } +#line 2546 "parse.c" + break; + case 134: /* dbnm ::= */ + case 143: /* indexed_opt ::= */ +#line 509 "parse.y" +{yygotominor.yy0.z=0; yygotominor.yy0.n=0;} +#line 2552 "parse.c" + break; + case 136: /* fullname ::= nm dbnm */ +#line 514 "parse.y" +{yygotominor.yy347 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} +#line 2557 "parse.c" + break; + case 137: /* joinop ::= COMMA|JOIN */ +#line 518 "parse.y" +{ yygotominor.yy392 = JT_INNER; } +#line 2562 "parse.c" + break; + case 138: /* joinop ::= JOIN_KW JOIN */ +#line 519 "parse.y" +{ yygotominor.yy392 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } +#line 2567 "parse.c" + break; + case 139: /* joinop ::= JOIN_KW nm JOIN */ #line 520 "parse.y" -{ - yygotominor.yy174 = sqlite3ExprListAppend(yymsp[-3].minor.yy174,yymsp[-1].minor.yy172,0); - if( yygotominor.yy174 ) yygotominor.yy174->a[yygotominor.yy174->nExpr-1].sortOrder = yymsp[0].minor.yy46; -} -#line 2520 "parse.c" - break; - case 144: -#line 524 "parse.y" -{ - yygotominor.yy174 = sqlite3ExprListAppend(0,yymsp[-1].minor.yy172,0); - if( yygotominor.yy174 && yygotominor.yy174->a ) yygotominor.yy174->a[0].sortOrder = yymsp[0].minor.yy46; -} -#line 2528 "parse.c" - break; - case 146: - case 148: -#line 532 "parse.y" -{yygotominor.yy46 = SQLITE_SO_ASC;} -#line 2534 "parse.c" - break; - case 147: -#line 533 "parse.y" -{yygotominor.yy46 = SQLITE_SO_DESC;} -#line 2539 "parse.c" - break; - case 153: -#line 559 "parse.y" -{yygotominor.yy234.pLimit = 0; yygotominor.yy234.pOffset = 0;} -#line 2544 "parse.c" - break; - case 154: -#line 560 "parse.y" -{yygotominor.yy234.pLimit = yymsp[0].minor.yy172; yygotominor.yy234.pOffset = 0;} -#line 2549 "parse.c" - break; - case 155: -#line 562 "parse.y" -{yygotominor.yy234.pLimit = yymsp[-2].minor.yy172; yygotominor.yy234.pOffset = yymsp[0].minor.yy172;} -#line 2554 "parse.c" - break; - case 156: -#line 564 "parse.y" -{yygotominor.yy234.pOffset = yymsp[-2].minor.yy172; yygotominor.yy234.pLimit = yymsp[0].minor.yy172;} -#line 2559 "parse.c" - break; - case 157: -#line 568 "parse.y" -{sqlite3DeleteFrom(pParse,yymsp[-1].minor.yy373,yymsp[0].minor.yy172);} -#line 2564 "parse.c" - break; - case 160: -#line 578 "parse.y" -{ - sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy174,SQLITE_MAX_COLUMN,"set list"); - sqlite3Update(pParse,yymsp[-3].minor.yy373,yymsp[-1].minor.yy174,yymsp[0].minor.yy172,yymsp[-4].minor.yy46); -} +{ yygotominor.yy392 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } #line 2572 "parse.c" break; - case 161: -#line 587 "parse.y" -{yygotominor.yy174 = sqlite3ExprListAppend(yymsp[-4].minor.yy174,yymsp[0].minor.yy172,&yymsp[-2].minor.yy410);} + case 140: /* joinop ::= JOIN_KW nm nm JOIN */ +#line 522 "parse.y" +{ yygotominor.yy392 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } #line 2577 "parse.c" break; - case 162: -#line 588 "parse.y" -{yygotominor.yy174 = sqlite3ExprListAppend(0,yymsp[0].minor.yy172,&yymsp[-2].minor.yy410);} -#line 2582 "parse.c" + case 141: /* on_opt ::= ON expr */ + case 152: /* sortitem ::= expr */ + case 159: /* having_opt ::= HAVING expr */ + case 166: /* where_opt ::= WHERE expr */ + case 181: /* expr ::= term */ + case 209: /* escape ::= ESCAPE expr */ + case 233: /* case_else ::= ELSE expr */ + case 235: /* case_operand ::= expr */ +#line 526 "parse.y" +{yygotominor.yy122 = yymsp[0].minor.yy122;} +#line 2589 "parse.c" break; - case 163: -#line 594 "parse.y" -{sqlite3Insert(pParse, yymsp[-5].minor.yy373, yymsp[-1].minor.yy174, 0, yymsp[-4].minor.yy432, yymsp[-7].minor.yy46);} -#line 2587 "parse.c" + case 142: /* on_opt ::= */ + case 158: /* having_opt ::= */ + case 165: /* where_opt ::= */ + case 210: /* escape ::= */ + case 234: /* case_else ::= */ + case 236: /* case_operand ::= */ +#line 527 "parse.y" +{yygotominor.yy122 = 0;} +#line 2599 "parse.c" break; - case 164: -#line 596 "parse.y" -{sqlite3Insert(pParse, yymsp[-2].minor.yy373, 0, yymsp[0].minor.yy219, yymsp[-1].minor.yy432, yymsp[-4].minor.yy46);} -#line 2592 "parse.c" + case 145: /* indexed_opt ::= NOT INDEXED */ +#line 542 "parse.y" +{yygotominor.yy0.z=0; yygotominor.yy0.n=1;} +#line 2604 "parse.c" break; - case 165: -#line 598 "parse.y" -{sqlite3Insert(pParse, yymsp[-3].minor.yy373, 0, 0, yymsp[-2].minor.yy432, yymsp[-5].minor.yy46);} -#line 2597 "parse.c" + case 146: /* using_opt ::= USING LP inscollist RP */ + case 178: /* inscollist_opt ::= LP inscollist RP */ +#line 546 "parse.y" +{yygotominor.yy180 = yymsp[-1].minor.yy180;} +#line 2610 "parse.c" break; - case 168: - case 231: -#line 608 "parse.y" -{yygotominor.yy174 = sqlite3ExprListAppend(yymsp[-2].minor.yy174,yymsp[0].minor.yy172,0);} -#line 2603 "parse.c" + case 147: /* using_opt ::= */ + case 177: /* inscollist_opt ::= */ +#line 547 "parse.y" +{yygotominor.yy180 = 0;} +#line 2616 "parse.c" break; - case 169: - case 232: -#line 609 "parse.y" -{yygotominor.yy174 = sqlite3ExprListAppend(0,yymsp[0].minor.yy172,0);} -#line 2609 "parse.c" + case 149: /* orderby_opt ::= ORDER BY sortlist */ + case 157: /* groupby_opt ::= GROUP BY nexprlist */ + case 237: /* exprlist ::= nexprlist */ +#line 558 "parse.y" +{yygotominor.yy442 = yymsp[0].minor.yy442;} +#line 2623 "parse.c" break; - case 172: -#line 618 "parse.y" -{yygotominor.yy432 = sqlite3IdListAppend(yymsp[-2].minor.yy432,&yymsp[0].minor.yy410);} -#line 2614 "parse.c" - break; - case 173: -#line 619 "parse.y" -{yygotominor.yy432 = sqlite3IdListAppend(0,&yymsp[0].minor.yy410);} -#line 2619 "parse.c" - break; - case 175: -#line 630 "parse.y" -{yygotominor.yy172 = yymsp[-1].minor.yy172; sqlite3ExprSpan(yygotominor.yy172,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); } -#line 2624 "parse.c" - break; - case 176: - case 181: - case 182: -#line 631 "parse.y" -{yygotominor.yy172 = sqlite3Expr(yymsp[0].major, 0, 0, &yymsp[0].minor.yy0);} + case 150: /* sortlist ::= sortlist COMMA sortitem sortorder */ +#line 559 "parse.y" +{ + yygotominor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy442,yymsp[-1].minor.yy122,0); + if( yygotominor.yy442 ) yygotominor.yy442->a[yygotominor.yy442->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy392; +} #line 2631 "parse.c" break; - case 177: - case 178: -#line 632 "parse.y" -{yygotominor.yy172 = sqlite3Expr(TK_ID, 0, 0, &yymsp[0].minor.yy0);} -#line 2637 "parse.c" - break; - case 179: -#line 634 "parse.y" + case 151: /* sortlist ::= sortitem sortorder */ +#line 563 "parse.y" { - Expr *temp1 = sqlite3Expr(TK_ID, 0, 0, &yymsp[-2].minor.yy410); - Expr *temp2 = sqlite3Expr(TK_ID, 0, 0, &yymsp[0].minor.yy410); - yygotominor.yy172 = sqlite3Expr(TK_DOT, temp1, temp2, 0); + yygotominor.yy442 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy122,0); + if( yygotominor.yy442 && yygotominor.yy442->a ) yygotominor.yy442->a[0].sortOrder = (u8)yymsp[0].minor.yy392; } -#line 2646 "parse.c" +#line 2639 "parse.c" break; - case 180: -#line 639 "parse.y" + case 153: /* sortorder ::= ASC */ + case 155: /* sortorder ::= */ +#line 571 "parse.y" +{yygotominor.yy392 = SQLITE_SO_ASC;} +#line 2645 "parse.c" + break; + case 154: /* sortorder ::= DESC */ +#line 572 "parse.y" +{yygotominor.yy392 = SQLITE_SO_DESC;} +#line 2650 "parse.c" + break; + case 160: /* limit_opt ::= */ +#line 598 "parse.y" +{yygotominor.yy64.pLimit = 0; yygotominor.yy64.pOffset = 0;} +#line 2655 "parse.c" + break; + case 161: /* limit_opt ::= LIMIT expr */ +#line 599 "parse.y" +{yygotominor.yy64.pLimit = yymsp[0].minor.yy122; yygotominor.yy64.pOffset = 0;} +#line 2660 "parse.c" + break; + case 162: /* limit_opt ::= LIMIT expr OFFSET expr */ +#line 601 "parse.y" +{yygotominor.yy64.pLimit = yymsp[-2].minor.yy122; yygotominor.yy64.pOffset = yymsp[0].minor.yy122;} +#line 2665 "parse.c" + break; + case 163: /* limit_opt ::= LIMIT expr COMMA expr */ +#line 603 "parse.y" +{yygotominor.yy64.pOffset = yymsp[-2].minor.yy122; yygotominor.yy64.pLimit = yymsp[0].minor.yy122;} +#line 2670 "parse.c" + break; + case 164: /* cmd ::= DELETE FROM fullname indexed_opt where_opt */ +#line 616 "parse.y" { - Expr *temp1 = sqlite3Expr(TK_ID, 0, 0, &yymsp[-4].minor.yy410); - Expr *temp2 = sqlite3Expr(TK_ID, 0, 0, &yymsp[-2].minor.yy410); - Expr *temp3 = sqlite3Expr(TK_ID, 0, 0, &yymsp[0].minor.yy410); - Expr *temp4 = sqlite3Expr(TK_DOT, temp2, temp3, 0); - yygotominor.yy172 = sqlite3Expr(TK_DOT, temp1, temp4, 0); -} -#line 2657 "parse.c" - break; - case 183: -#line 648 "parse.y" -{yygotominor.yy172 = sqlite3RegisterExpr(pParse, &yymsp[0].minor.yy0);} -#line 2662 "parse.c" - break; - case 184: -#line 649 "parse.y" -{ - Token *pToken = &yymsp[0].minor.yy0; - Expr *pExpr = yygotominor.yy172 = sqlite3Expr(TK_VARIABLE, 0, 0, pToken); - sqlite3ExprAssignVarNumber(pParse, pExpr); -} -#line 2671 "parse.c" - break; - case 185: -#line 654 "parse.y" -{ - yygotominor.yy172 = sqlite3ExprSetColl(pParse, yymsp[-2].minor.yy172, &yymsp[0].minor.yy410); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy347, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy347,yymsp[0].minor.yy122); } #line 2678 "parse.c" break; - case 186: + case 167: /* cmd ::= UPDATE orconf fullname indexed_opt SET setlist where_opt */ +#line 639 "parse.y" +{ + sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy347, &yymsp[-3].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy442,"set list"); + sqlite3Update(pParse,yymsp[-4].minor.yy347,yymsp[-1].minor.yy442,yymsp[0].minor.yy122,yymsp[-5].minor.yy392); +} +#line 2687 "parse.c" + break; + case 168: /* setlist ::= setlist COMMA nm EQ expr */ +#line 650 "parse.y" +{yygotominor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy442,yymsp[0].minor.yy122,&yymsp[-2].minor.yy0);} +#line 2692 "parse.c" + break; + case 169: /* setlist ::= nm EQ expr */ +#line 652 "parse.y" +{yygotominor.yy442 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy122,&yymsp[-2].minor.yy0);} +#line 2697 "parse.c" + break; + case 170: /* cmd ::= insert_cmd INTO fullname inscollist_opt VALUES LP itemlist RP */ #line 658 "parse.y" -{ - yygotominor.yy172 = sqlite3Expr(TK_CAST, yymsp[-3].minor.yy172, 0, &yymsp[-1].minor.yy410); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); -} -#line 2686 "parse.c" +{sqlite3Insert(pParse, yymsp[-5].minor.yy347, yymsp[-1].minor.yy442, 0, yymsp[-4].minor.yy180, yymsp[-7].minor.yy392);} +#line 2702 "parse.c" break; - case 187: -#line 663 "parse.y" -{ - if( yymsp[-1].minor.yy174 && yymsp[-1].minor.yy174->nExpr>SQLITE_MAX_FUNCTION_ARG ){ - sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); - } - yygotominor.yy172 = sqlite3ExprFunction(yymsp[-1].minor.yy174, &yymsp[-4].minor.yy0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); - if( yymsp[-2].minor.yy46 && yygotominor.yy172 ){ - yygotominor.yy172->flags |= EP_Distinct; - } -} -#line 2700 "parse.c" + case 171: /* cmd ::= insert_cmd INTO fullname inscollist_opt select */ +#line 660 "parse.y" +{sqlite3Insert(pParse, yymsp[-2].minor.yy347, 0, yymsp[0].minor.yy159, yymsp[-1].minor.yy180, yymsp[-4].minor.yy392);} +#line 2707 "parse.c" break; - case 188: + case 172: /* cmd ::= insert_cmd INTO fullname inscollist_opt DEFAULT VALUES */ +#line 662 "parse.y" +{sqlite3Insert(pParse, yymsp[-3].minor.yy347, 0, 0, yymsp[-2].minor.yy180, yymsp[-5].minor.yy392);} +#line 2712 "parse.c" + break; + case 175: /* itemlist ::= itemlist COMMA expr */ + case 239: /* nexprlist ::= nexprlist COMMA expr */ #line 673 "parse.y" -{ - yygotominor.yy172 = sqlite3ExprFunction(0, &yymsp[-3].minor.yy0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); -} -#line 2708 "parse.c" +{yygotominor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy442,yymsp[0].minor.yy122,0);} +#line 2718 "parse.c" break; - case 189: -#line 677 "parse.y" -{ - /* The CURRENT_TIME, CURRENT_DATE, and CURRENT_TIMESTAMP values are - ** treated as functions that return constants */ - yygotominor.yy172 = sqlite3ExprFunction(0,&yymsp[0].minor.yy0); - if( yygotominor.yy172 ){ - yygotominor.yy172->op = TK_CONST_FUNC; - yygotominor.yy172->span = yymsp[0].minor.yy0; - } -} -#line 2721 "parse.c" + case 176: /* itemlist ::= expr */ + case 240: /* nexprlist ::= expr */ +#line 675 "parse.y" +{yygotominor.yy442 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy122,0);} +#line 2724 "parse.c" break; - case 190: - case 191: - case 192: - case 193: - case 194: - case 195: - case 196: - case 197: -#line 686 "parse.y" -{yygotominor.yy172 = sqlite3Expr(yymsp[-1].major, yymsp[-2].minor.yy172, yymsp[0].minor.yy172, 0);} -#line 2733 "parse.c" + case 179: /* inscollist ::= inscollist COMMA nm */ +#line 685 "parse.y" +{yygotominor.yy180 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy180,&yymsp[0].minor.yy0);} +#line 2729 "parse.c" break; - case 198: - case 200: -#line 696 "parse.y" -{yygotominor.yy72.eOperator = yymsp[0].minor.yy0; yygotominor.yy72.not = 0;} + case 180: /* inscollist ::= nm */ +#line 687 "parse.y" +{yygotominor.yy180 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} +#line 2734 "parse.c" + break; + case 182: /* expr ::= LP expr RP */ +#line 698 "parse.y" +{yygotominor.yy122 = yymsp[-1].minor.yy122; sqlite3ExprSpan(yygotominor.yy122,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); } #line 2739 "parse.c" break; - case 199: - case 201: -#line 697 "parse.y" -{yygotominor.yy72.eOperator = yymsp[0].minor.yy0; yygotominor.yy72.not = 1;} -#line 2745 "parse.c" + case 183: /* term ::= NULL */ + case 188: /* term ::= INTEGER|FLOAT|BLOB */ + case 189: /* term ::= STRING */ +#line 699 "parse.y" +{yygotominor.yy122 = sqlite3PExpr(pParse, yymsp[0].major, 0, 0, &yymsp[0].minor.yy0);} +#line 2746 "parse.c" break; - case 204: -#line 704 "parse.y" + case 184: /* expr ::= ID */ + case 185: /* expr ::= JOIN_KW */ +#line 700 "parse.y" +{yygotominor.yy122 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0);} +#line 2752 "parse.c" + break; + case 186: /* expr ::= nm DOT nm */ +#line 702 "parse.y" { - ExprList *pList; - pList = sqlite3ExprListAppend(0, yymsp[-1].minor.yy172, 0); - pList = sqlite3ExprListAppend(pList, yymsp[-3].minor.yy172, 0); - if( yymsp[0].minor.yy172 ){ - pList = sqlite3ExprListAppend(pList, yymsp[0].minor.yy172, 0); - } - yygotominor.yy172 = sqlite3ExprFunction(pList, &yymsp[-2].minor.yy72.eOperator); - if( yymsp[-2].minor.yy72.not ) yygotominor.yy172 = sqlite3Expr(TK_NOT, yygotominor.yy172, 0, 0); - sqlite3ExprSpan(yygotominor.yy172, &yymsp[-3].minor.yy172->span, &yymsp[-1].minor.yy172->span); - if( yygotominor.yy172 ) yygotominor.yy172->flags |= EP_InfixFunc; + Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); + Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); + yygotominor.yy122 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); } #line 2761 "parse.c" break; - case 205: -#line 717 "parse.y" + case 187: /* expr ::= nm DOT nm DOT nm */ +#line 707 "parse.y" { - yygotominor.yy172 = sqlite3Expr(yymsp[0].major, yymsp[-1].minor.yy172, 0, 0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-1].minor.yy172->span,&yymsp[0].minor.yy0); + Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0); + Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); + Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); + Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); + yygotominor.yy122 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); } -#line 2769 "parse.c" +#line 2772 "parse.c" break; - case 206: -#line 721 "parse.y" -{ - yygotominor.yy172 = sqlite3Expr(TK_ISNULL, yymsp[-2].minor.yy172, 0, 0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-2].minor.yy172->span,&yymsp[0].minor.yy0); -} + case 190: /* expr ::= REGISTER */ +#line 716 "parse.y" +{yygotominor.yy122 = sqlite3RegisterExpr(pParse, &yymsp[0].minor.yy0);} #line 2777 "parse.c" break; - case 207: -#line 725 "parse.y" + case 191: /* expr ::= VARIABLE */ +#line 717 "parse.y" { - yygotominor.yy172 = sqlite3Expr(TK_NOTNULL, yymsp[-2].minor.yy172, 0, 0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-2].minor.yy172->span,&yymsp[0].minor.yy0); + Token *pToken = &yymsp[0].minor.yy0; + Expr *pExpr = yygotominor.yy122 = sqlite3PExpr(pParse, TK_VARIABLE, 0, 0, pToken); + sqlite3ExprAssignVarNumber(pParse, pExpr); } -#line 2785 "parse.c" +#line 2786 "parse.c" break; - case 208: -#line 729 "parse.y" + case 192: /* expr ::= expr COLLATE ids */ +#line 722 "parse.y" { - yygotominor.yy172 = sqlite3Expr(TK_NOTNULL, yymsp[-3].minor.yy172, 0, 0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-3].minor.yy172->span,&yymsp[0].minor.yy0); + yygotominor.yy122 = sqlite3ExprSetColl(pParse, yymsp[-2].minor.yy122, &yymsp[0].minor.yy0); } #line 2793 "parse.c" break; - case 209: -#line 733 "parse.y" + case 193: /* expr ::= CAST LP expr AS typetoken RP */ +#line 726 "parse.y" { - yygotominor.yy172 = sqlite3Expr(yymsp[-1].major, yymsp[0].minor.yy172, 0, 0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy172->span); + yygotominor.yy122 = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy122, 0, &yymsp[-1].minor.yy0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); } #line 2801 "parse.c" break; - case 210: -#line 737 "parse.y" + case 194: /* expr ::= ID LP distinct exprlist RP */ +#line 731 "parse.y" { - yygotominor.yy172 = sqlite3Expr(TK_UMINUS, yymsp[0].minor.yy172, 0, 0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy172->span); + if( yymsp[-1].minor.yy442 && yymsp[-1].minor.yy442->nExpr>SQLITE_MAX_FUNCTION_ARG ){ + sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); + } + yygotominor.yy122 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy442, &yymsp[-4].minor.yy0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); + if( yymsp[-2].minor.yy392 && yygotominor.yy122 ){ + yygotominor.yy122->flags |= EP_Distinct; + } } -#line 2809 "parse.c" +#line 2815 "parse.c" break; - case 211: + case 195: /* expr ::= ID LP STAR RP */ #line 741 "parse.y" { - yygotominor.yy172 = sqlite3Expr(TK_UPLUS, yymsp[0].minor.yy172, 0, 0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy172->span); + yygotominor.yy122 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); } -#line 2817 "parse.c" +#line 2823 "parse.c" break; - case 214: -#line 748 "parse.y" + case 196: /* term ::= CTIME_KW */ +#line 745 "parse.y" { - ExprList *pList = sqlite3ExprListAppend(0, yymsp[-2].minor.yy172, 0); - pList = sqlite3ExprListAppend(pList, yymsp[0].minor.yy172, 0); - yygotominor.yy172 = sqlite3Expr(TK_BETWEEN, yymsp[-4].minor.yy172, 0, 0); - if( yygotominor.yy172 ){ - yygotominor.yy172->pList = pList; - }else{ - sqlite3ExprListDelete(pList); - } - if( yymsp[-3].minor.yy46 ) yygotominor.yy172 = sqlite3Expr(TK_NOT, yygotominor.yy172, 0, 0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-4].minor.yy172->span,&yymsp[0].minor.yy172->span); -} -#line 2833 "parse.c" - break; - case 217: -#line 764 "parse.y" -{ - yygotominor.yy172 = sqlite3Expr(TK_IN, yymsp[-4].minor.yy172, 0, 0); - if( yygotominor.yy172 ){ - yygotominor.yy172->pList = yymsp[-1].minor.yy174; - sqlite3ExprSetHeight(yygotominor.yy172); - }else{ - sqlite3ExprListDelete(yymsp[-1].minor.yy174); - } - if( yymsp[-3].minor.yy46 ) yygotominor.yy172 = sqlite3Expr(TK_NOT, yygotominor.yy172, 0, 0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-4].minor.yy172->span,&yymsp[0].minor.yy0); + /* The CURRENT_TIME, CURRENT_DATE, and CURRENT_TIMESTAMP values are + ** treated as functions that return constants */ + yygotominor.yy122 = sqlite3ExprFunction(pParse, 0,&yymsp[0].minor.yy0); + if( yygotominor.yy122 ){ + yygotominor.yy122->op = TK_CONST_FUNC; + yygotominor.yy122->span = yymsp[0].minor.yy0; } +} +#line 2836 "parse.c" + break; + case 197: /* expr ::= expr AND expr */ + case 198: /* expr ::= expr OR expr */ + case 199: /* expr ::= expr LT|GT|GE|LE expr */ + case 200: /* expr ::= expr EQ|NE expr */ + case 201: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + case 202: /* expr ::= expr PLUS|MINUS expr */ + case 203: /* expr ::= expr STAR|SLASH|REM expr */ + case 204: /* expr ::= expr CONCAT expr */ +#line 754 "parse.y" +{yygotominor.yy122 = sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy122,yymsp[0].minor.yy122,0);} #line 2848 "parse.c" break; - case 218: -#line 775 "parse.y" -{ - yygotominor.yy172 = sqlite3Expr(TK_SELECT, 0, 0, 0); - if( yygotominor.yy172 ){ - yygotominor.yy172->pSelect = yymsp[-1].minor.yy219; - sqlite3ExprSetHeight(yygotominor.yy172); - }else{ - sqlite3SelectDelete(yymsp[-1].minor.yy219); - } - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); - } -#line 2862 "parse.c" + case 205: /* likeop ::= LIKE_KW */ + case 207: /* likeop ::= MATCH */ +#line 766 "parse.y" +{yygotominor.yy318.eOperator = yymsp[0].minor.yy0; yygotominor.yy318.not = 0;} +#line 2854 "parse.c" break; - case 219: -#line 785 "parse.y" -{ - yygotominor.yy172 = sqlite3Expr(TK_IN, yymsp[-4].minor.yy172, 0, 0); - if( yygotominor.yy172 ){ - yygotominor.yy172->pSelect = yymsp[-1].minor.yy219; - sqlite3ExprSetHeight(yygotominor.yy172); - }else{ - sqlite3SelectDelete(yymsp[-1].minor.yy219); - } - if( yymsp[-3].minor.yy46 ) yygotominor.yy172 = sqlite3Expr(TK_NOT, yygotominor.yy172, 0, 0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-4].minor.yy172->span,&yymsp[0].minor.yy0); - } -#line 2877 "parse.c" + case 206: /* likeop ::= NOT LIKE_KW */ + case 208: /* likeop ::= NOT MATCH */ +#line 767 "parse.y" +{yygotominor.yy318.eOperator = yymsp[0].minor.yy0; yygotominor.yy318.not = 1;} +#line 2860 "parse.c" break; - case 220: -#line 796 "parse.y" + case 211: /* expr ::= expr likeop expr escape */ +#line 774 "parse.y" { - SrcList *pSrc = sqlite3SrcListAppend(0,&yymsp[-1].minor.yy410,&yymsp[0].minor.yy410); - yygotominor.yy172 = sqlite3Expr(TK_IN, yymsp[-3].minor.yy172, 0, 0); - if( yygotominor.yy172 ){ - yygotominor.yy172->pSelect = sqlite3SelectNew(0,pSrc,0,0,0,0,0,0,0); - sqlite3ExprSetHeight(yygotominor.yy172); - }else{ - sqlite3SrcListDelete(pSrc); - } - if( yymsp[-2].minor.yy46 ) yygotominor.yy172 = sqlite3Expr(TK_NOT, yygotominor.yy172, 0, 0); - sqlite3ExprSpan(yygotominor.yy172,&yymsp[-3].minor.yy172->span,yymsp[0].minor.yy410.z?&yymsp[0].minor.yy410:&yymsp[-1].minor.yy410); + ExprList *pList; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-1].minor.yy122, 0); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-3].minor.yy122, 0); + if( yymsp[0].minor.yy122 ){ + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy122, 0); } -#line 2893 "parse.c" + yygotominor.yy122 = sqlite3ExprFunction(pParse, pList, &yymsp[-2].minor.yy318.eOperator); + if( yymsp[-2].minor.yy318.not ) yygotominor.yy122 = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy122, 0, 0); + sqlite3ExprSpan(yygotominor.yy122, &yymsp[-3].minor.yy122->span, &yymsp[-1].minor.yy122->span); + if( yygotominor.yy122 ) yygotominor.yy122->flags |= EP_InfixFunc; +} +#line 2876 "parse.c" break; - case 221: -#line 808 "parse.y" + case 212: /* expr ::= expr ISNULL|NOTNULL */ +#line 787 "parse.y" { - Expr *p = yygotominor.yy172 = sqlite3Expr(TK_EXISTS, 0, 0, 0); - if( p ){ - p->pSelect = yymsp[-1].minor.yy219; - sqlite3ExprSpan(p,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); - sqlite3ExprSetHeight(yygotominor.yy172); - }else{ - sqlite3SelectDelete(yymsp[-1].minor.yy219); - } - } -#line 2907 "parse.c" + yygotominor.yy122 = sqlite3PExpr(pParse, yymsp[0].major, yymsp[-1].minor.yy122, 0, 0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-1].minor.yy122->span,&yymsp[0].minor.yy0); +} +#line 2884 "parse.c" break; - case 222: -#line 821 "parse.y" + case 213: /* expr ::= expr IS NULL */ +#line 791 "parse.y" { - yygotominor.yy172 = sqlite3Expr(TK_CASE, yymsp[-3].minor.yy172, yymsp[-1].minor.yy172, 0); - if( yygotominor.yy172 ){ - yygotominor.yy172->pList = yymsp[-2].minor.yy174; - sqlite3ExprSetHeight(yygotominor.yy172); + yygotominor.yy122 = sqlite3PExpr(pParse, TK_ISNULL, yymsp[-2].minor.yy122, 0, 0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-2].minor.yy122->span,&yymsp[0].minor.yy0); +} +#line 2892 "parse.c" + break; + case 214: /* expr ::= expr NOT NULL */ +#line 795 "parse.y" +{ + yygotominor.yy122 = sqlite3PExpr(pParse, TK_NOTNULL, yymsp[-2].minor.yy122, 0, 0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-2].minor.yy122->span,&yymsp[0].minor.yy0); +} +#line 2900 "parse.c" + break; + case 215: /* expr ::= expr IS NOT NULL */ +#line 799 "parse.y" +{ + yygotominor.yy122 = sqlite3PExpr(pParse, TK_NOTNULL, yymsp[-3].minor.yy122, 0, 0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-3].minor.yy122->span,&yymsp[0].minor.yy0); +} +#line 2908 "parse.c" + break; + case 216: /* expr ::= NOT expr */ + case 217: /* expr ::= BITNOT expr */ +#line 803 "parse.y" +{ + yygotominor.yy122 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy122, 0, 0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy122->span); +} +#line 2917 "parse.c" + break; + case 218: /* expr ::= MINUS expr */ +#line 811 "parse.y" +{ + yygotominor.yy122 = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy122, 0, 0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy122->span); +} +#line 2925 "parse.c" + break; + case 219: /* expr ::= PLUS expr */ +#line 815 "parse.y" +{ + yygotominor.yy122 = sqlite3PExpr(pParse, TK_UPLUS, yymsp[0].minor.yy122, 0, 0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy122->span); +} +#line 2933 "parse.c" + break; + case 222: /* expr ::= expr between_op expr AND expr */ +#line 822 "parse.y" +{ + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy122, 0); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy122, 0); + yygotominor.yy122 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy122, 0, 0); + if( yygotominor.yy122 ){ + yygotominor.yy122->pList = pList; }else{ - sqlite3ExprListDelete(yymsp[-2].minor.yy174); + sqlite3ExprListDelete(pParse->db, pList); + } + if( yymsp[-3].minor.yy392 ) yygotominor.yy122 = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy122, 0, 0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-4].minor.yy122->span,&yymsp[0].minor.yy122->span); +} +#line 2949 "parse.c" + break; + case 225: /* expr ::= expr in_op LP exprlist RP */ +#line 838 "parse.y" +{ + yygotominor.yy122 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy122, 0, 0); + if( yygotominor.yy122 ){ + yygotominor.yy122->pList = yymsp[-1].minor.yy442; + sqlite3ExprSetHeight(pParse, yygotominor.yy122); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy442); + } + if( yymsp[-3].minor.yy392 ) yygotominor.yy122 = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy122, 0, 0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-4].minor.yy122->span,&yymsp[0].minor.yy0); } - sqlite3ExprSpan(yygotominor.yy172, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0); -} -#line 2921 "parse.c" +#line 2964 "parse.c" break; - case 223: -#line 833 "parse.y" + case 226: /* expr ::= LP select RP */ +#line 849 "parse.y" { - yygotominor.yy174 = sqlite3ExprListAppend(yymsp[-4].minor.yy174, yymsp[-2].minor.yy172, 0); - yygotominor.yy174 = sqlite3ExprListAppend(yygotominor.yy174, yymsp[0].minor.yy172, 0); -} -#line 2929 "parse.c" + yygotominor.yy122 = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); + if( yygotominor.yy122 ){ + yygotominor.yy122->pSelect = yymsp[-1].minor.yy159; + sqlite3ExprSetHeight(pParse, yygotominor.yy122); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy159); + } + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + } +#line 2978 "parse.c" break; - case 224: -#line 837 "parse.y" + case 227: /* expr ::= expr in_op LP select RP */ +#line 859 "parse.y" { - yygotominor.yy174 = sqlite3ExprListAppend(0, yymsp[-2].minor.yy172, 0); - yygotominor.yy174 = sqlite3ExprListAppend(yygotominor.yy174, yymsp[0].minor.yy172, 0); -} -#line 2937 "parse.c" + yygotominor.yy122 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy122, 0, 0); + if( yygotominor.yy122 ){ + yygotominor.yy122->pSelect = yymsp[-1].minor.yy159; + sqlite3ExprSetHeight(pParse, yygotominor.yy122); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy159); + } + if( yymsp[-3].minor.yy392 ) yygotominor.yy122 = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy122, 0, 0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-4].minor.yy122->span,&yymsp[0].minor.yy0); + } +#line 2993 "parse.c" break; - case 233: -#line 864 "parse.y" -{ - sqlite3CreateIndex(pParse, &yymsp[-6].minor.yy410, &yymsp[-5].minor.yy410, sqlite3SrcListAppend(0,&yymsp[-3].minor.yy410,0), yymsp[-1].minor.yy174, yymsp[-9].minor.yy46, - &yymsp[-10].minor.yy0, &yymsp[0].minor.yy0, SQLITE_SO_ASC, yymsp[-7].minor.yy46); -} -#line 2945 "parse.c" - break; - case 234: - case 281: + case 228: /* expr ::= expr in_op nm dbnm */ #line 870 "parse.y" -{yygotominor.yy46 = OE_Abort;} -#line 2951 "parse.c" - break; - case 235: -#line 871 "parse.y" -{yygotominor.yy46 = OE_None;} -#line 2956 "parse.c" - break; - case 238: -#line 881 "parse.y" { - Expr *p = 0; - if( yymsp[-1].minor.yy410.n>0 ){ - p = sqlite3Expr(TK_COLUMN, 0, 0, 0); - if( p ) p->pColl = sqlite3LocateCollSeq(pParse, (char*)yymsp[-1].minor.yy410.z, yymsp[-1].minor.yy410.n); + SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); + yygotominor.yy122 = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy122, 0, 0); + if( yygotominor.yy122 ){ + yygotominor.yy122->pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); + sqlite3ExprSetHeight(pParse, yygotominor.yy122); + }else{ + sqlite3SrcListDelete(pParse->db, pSrc); + } + if( yymsp[-2].minor.yy392 ) yygotominor.yy122 = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy122, 0, 0); + sqlite3ExprSpan(yygotominor.yy122,&yymsp[-3].minor.yy122->span,yymsp[0].minor.yy0.z?&yymsp[0].minor.yy0:&yymsp[-1].minor.yy0); } - yygotominor.yy174 = sqlite3ExprListAppend(yymsp[-4].minor.yy174, p, &yymsp[-2].minor.yy410); - sqlite3ExprListCheckLength(pParse, yygotominor.yy174, SQLITE_MAX_COLUMN, "index"); - if( yygotominor.yy174 ) yygotominor.yy174->a[yygotominor.yy174->nExpr-1].sortOrder = yymsp[0].minor.yy46; -} -#line 2970 "parse.c" +#line 3009 "parse.c" break; - case 239: -#line 891 "parse.y" + case 229: /* expr ::= EXISTS LP select RP */ +#line 882 "parse.y" { - Expr *p = 0; - if( yymsp[-1].minor.yy410.n>0 ){ - p = sqlite3Expr(TK_COLUMN, 0, 0, 0); - if( p ) p->pColl = sqlite3LocateCollSeq(pParse, (char*)yymsp[-1].minor.yy410.z, yymsp[-1].minor.yy410.n); + Expr *p = yygotominor.yy122 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); + if( p ){ + p->pSelect = yymsp[-1].minor.yy159; + sqlite3ExprSpan(p,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); + sqlite3ExprSetHeight(pParse, yygotominor.yy122); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy159); + } } - yygotominor.yy174 = sqlite3ExprListAppend(0, p, &yymsp[-2].minor.yy410); - sqlite3ExprListCheckLength(pParse, yygotominor.yy174, SQLITE_MAX_COLUMN, "index"); - if( yygotominor.yy174 ) yygotominor.yy174->a[yygotominor.yy174->nExpr-1].sortOrder = yymsp[0].minor.yy46; -} -#line 2984 "parse.c" +#line 3023 "parse.c" break; - case 241: -#line 904 "parse.y" -{yygotominor.yy410.z = 0; yygotominor.yy410.n = 0;} -#line 2989 "parse.c" - break; - case 243: -#line 910 "parse.y" -{sqlite3DropIndex(pParse, yymsp[0].minor.yy373, yymsp[-1].minor.yy46);} -#line 2994 "parse.c" - break; - case 244: - case 245: -#line 916 "parse.y" -{sqlite3Vacuum(pParse);} -#line 3000 "parse.c" - break; - case 246: -#line 924 "parse.y" -{sqlite3Pragma(pParse,&yymsp[-3].minor.yy410,&yymsp[-2].minor.yy410,&yymsp[0].minor.yy410,0);} -#line 3005 "parse.c" - break; - case 247: -#line 925 "parse.y" -{sqlite3Pragma(pParse,&yymsp[-3].minor.yy410,&yymsp[-2].minor.yy410,&yymsp[0].minor.yy0,0);} -#line 3010 "parse.c" - break; - case 248: -#line 926 "parse.y" + case 230: /* expr ::= CASE case_operand case_exprlist case_else END */ +#line 895 "parse.y" { - sqlite3Pragma(pParse,&yymsp[-3].minor.yy410,&yymsp[-2].minor.yy410,&yymsp[0].minor.yy410,1); -} -#line 3017 "parse.c" - break; - case 249: -#line 929 "parse.y" -{sqlite3Pragma(pParse,&yymsp[-4].minor.yy410,&yymsp[-3].minor.yy410,&yymsp[-1].minor.yy410,0);} -#line 3022 "parse.c" - break; - case 250: -#line 930 "parse.y" -{sqlite3Pragma(pParse,&yymsp[-1].minor.yy410,&yymsp[0].minor.yy410,0,0);} -#line 3027 "parse.c" - break; - case 258: -#line 944 "parse.y" -{ - Token all; - all.z = yymsp[-3].minor.yy410.z; - all.n = (yymsp[0].minor.yy0.z - yymsp[-3].minor.yy410.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy243, &all); + yygotominor.yy122 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy122, yymsp[-1].minor.yy122, 0); + if( yygotominor.yy122 ){ + yygotominor.yy122->pList = yymsp[-2].minor.yy442; + sqlite3ExprSetHeight(pParse, yygotominor.yy122); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy442); + } + sqlite3ExprSpan(yygotominor.yy122, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0); } #line 3037 "parse.c" break; - case 259: -#line 953 "parse.y" + case 231: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ +#line 907 "parse.y" { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy410, &yymsp[-6].minor.yy410, yymsp[-5].minor.yy46, yymsp[-4].minor.yy370.a, yymsp[-4].minor.yy370.b, yymsp[-2].minor.yy373, yymsp[0].minor.yy172, yymsp[-10].minor.yy46, yymsp[-8].minor.yy46); - yygotominor.yy410 = (yymsp[-6].minor.yy410.n==0?yymsp[-7].minor.yy410:yymsp[-6].minor.yy410); + yygotominor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy442, yymsp[-2].minor.yy122, 0); + yygotominor.yy442 = sqlite3ExprListAppend(pParse,yygotominor.yy442, yymsp[0].minor.yy122, 0); } #line 3045 "parse.c" break; - case 260: - case 263: -#line 959 "parse.y" -{ yygotominor.yy46 = TK_BEFORE; } -#line 3051 "parse.c" - break; - case 261: -#line 960 "parse.y" -{ yygotominor.yy46 = TK_AFTER; } -#line 3056 "parse.c" - break; - case 262: -#line 961 "parse.y" -{ yygotominor.yy46 = TK_INSTEAD;} -#line 3061 "parse.c" - break; - case 264: - case 265: -#line 966 "parse.y" -{yygotominor.yy370.a = yymsp[0].major; yygotominor.yy370.b = 0;} -#line 3067 "parse.c" - break; - case 266: -#line 968 "parse.y" -{yygotominor.yy370.a = TK_UPDATE; yygotominor.yy370.b = yymsp[0].minor.yy432;} -#line 3072 "parse.c" - break; - case 269: -#line 975 "parse.y" -{ yygotominor.yy172 = 0; } -#line 3077 "parse.c" - break; - case 270: -#line 976 "parse.y" -{ yygotominor.yy172 = yymsp[0].minor.yy172; } -#line 3082 "parse.c" - break; - case 271: -#line 980 "parse.y" + case 232: /* case_exprlist ::= WHEN expr THEN expr */ +#line 911 "parse.y" { - if( yymsp[-2].minor.yy243 ){ - yymsp[-2].minor.yy243->pLast->pNext = yymsp[-1].minor.yy243; - }else{ - yymsp[-2].minor.yy243 = yymsp[-1].minor.yy243; - } - yymsp[-2].minor.yy243->pLast = yymsp[-1].minor.yy243; - yygotominor.yy243 = yymsp[-2].minor.yy243; + yygotominor.yy442 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy122, 0); + yygotominor.yy442 = sqlite3ExprListAppend(pParse,yygotominor.yy442, yymsp[0].minor.yy122, 0); } -#line 3095 "parse.c" +#line 3053 "parse.c" break; - case 272: -#line 989 "parse.y" -{ yygotominor.yy243 = 0; } -#line 3100 "parse.c" + case 241: /* cmd ::= CREATE uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP */ +#line 940 "parse.y" +{ + sqlite3CreateIndex(pParse, &yymsp[-6].minor.yy0, &yymsp[-5].minor.yy0, + sqlite3SrcListAppend(pParse->db,0,&yymsp[-3].minor.yy0,0), yymsp[-1].minor.yy442, yymsp[-9].minor.yy392, + &yymsp[-10].minor.yy0, &yymsp[0].minor.yy0, SQLITE_SO_ASC, yymsp[-7].minor.yy392); +} +#line 3062 "parse.c" break; - case 273: -#line 995 "parse.y" -{ yygotominor.yy243 = sqlite3TriggerUpdateStep(&yymsp[-3].minor.yy410, yymsp[-1].minor.yy174, yymsp[0].minor.yy172, yymsp[-4].minor.yy46); } -#line 3105 "parse.c" + case 242: /* uniqueflag ::= UNIQUE */ + case 289: /* raisetype ::= ABORT */ +#line 947 "parse.y" +{yygotominor.yy392 = OE_Abort;} +#line 3068 "parse.c" break; - case 274: + case 243: /* uniqueflag ::= */ +#line 948 "parse.y" +{yygotominor.yy392 = OE_None;} +#line 3073 "parse.c" + break; + case 246: /* idxlist ::= idxlist COMMA nm collate sortorder */ +#line 957 "parse.y" +{ + Expr *p = 0; + if( yymsp[-1].minor.yy0.n>0 ){ + p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); + sqlite3ExprSetColl(pParse, p, &yymsp[-1].minor.yy0); + } + yygotominor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy442, p, &yymsp[-2].minor.yy0); + sqlite3ExprListCheckLength(pParse, yygotominor.yy442, "index"); + if( yygotominor.yy442 ) yygotominor.yy442->a[yygotominor.yy442->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy392; +} +#line 3087 "parse.c" + break; + case 247: /* idxlist ::= nm collate sortorder */ +#line 967 "parse.y" +{ + Expr *p = 0; + if( yymsp[-1].minor.yy0.n>0 ){ + p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); + sqlite3ExprSetColl(pParse, p, &yymsp[-1].minor.yy0); + } + yygotominor.yy442 = sqlite3ExprListAppend(pParse,0, p, &yymsp[-2].minor.yy0); + sqlite3ExprListCheckLength(pParse, yygotominor.yy442, "index"); + if( yygotominor.yy442 ) yygotominor.yy442->a[yygotominor.yy442->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy392; +} +#line 3101 "parse.c" + break; + case 248: /* collate ::= */ +#line 979 "parse.y" +{yygotominor.yy0.z = 0; yygotominor.yy0.n = 0;} +#line 3106 "parse.c" + break; + case 250: /* cmd ::= DROP INDEX ifexists fullname */ +#line 985 "parse.y" +{sqlite3DropIndex(pParse, yymsp[0].minor.yy347, yymsp[-1].minor.yy392);} +#line 3111 "parse.c" + break; + case 251: /* cmd ::= VACUUM */ + case 252: /* cmd ::= VACUUM nm */ +#line 991 "parse.y" +{sqlite3Vacuum(pParse);} +#line 3117 "parse.c" + break; + case 253: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 254: /* cmd ::= PRAGMA nm dbnm EQ ON */ + case 255: /* cmd ::= PRAGMA nm dbnm EQ DELETE */ #line 1000 "parse.y" -{yygotominor.yy243 = sqlite3TriggerInsertStep(&yymsp[-5].minor.yy410, yymsp[-4].minor.yy432, yymsp[-1].minor.yy174, 0, yymsp[-7].minor.yy46);} -#line 3110 "parse.c" +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} +#line 3124 "parse.c" break; - case 275: + case 256: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ #line 1003 "parse.y" -{yygotominor.yy243 = sqlite3TriggerInsertStep(&yymsp[-2].minor.yy410, yymsp[-1].minor.yy432, 0, yymsp[0].minor.yy219, yymsp[-4].minor.yy46);} -#line 3115 "parse.c" - break; - case 276: -#line 1007 "parse.y" -{yygotominor.yy243 = sqlite3TriggerDeleteStep(&yymsp[-1].minor.yy410, yymsp[0].minor.yy172);} -#line 3120 "parse.c" - break; - case 277: -#line 1010 "parse.y" -{yygotominor.yy243 = sqlite3TriggerSelectStep(yymsp[0].minor.yy219); } -#line 3125 "parse.c" - break; - case 278: -#line 1013 "parse.y" { - yygotominor.yy172 = sqlite3Expr(TK_RAISE, 0, 0, 0); - if( yygotominor.yy172 ){ - yygotominor.yy172->iColumn = OE_Ignore; - sqlite3ExprSpan(yygotominor.yy172, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0); - } + sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1); } +#line 3131 "parse.c" + break; + case 257: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ +#line 1006 "parse.y" +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} #line 3136 "parse.c" break; - case 279: -#line 1020 "parse.y" + case 258: /* cmd ::= PRAGMA nm dbnm */ +#line 1007 "parse.y" +{sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} +#line 3141 "parse.c" + break; + case 266: /* cmd ::= CREATE trigger_decl BEGIN trigger_cmd_list END */ +#line 1022 "parse.y" { - yygotominor.yy172 = sqlite3Expr(TK_RAISE, 0, 0, &yymsp[-1].minor.yy410); - if( yygotominor.yy172 ) { - yygotominor.yy172->iColumn = yymsp[-3].minor.yy46; - sqlite3ExprSpan(yygotominor.yy172, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0); - } + Token all; + all.z = yymsp[-3].minor.yy0.z; + all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy327, &all); } -#line 3147 "parse.c" +#line 3151 "parse.c" break; - case 280: -#line 1030 "parse.y" -{yygotominor.yy46 = OE_Rollback;} -#line 3152 "parse.c" + case 267: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ +#line 1031 "parse.y" +{ + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy392, yymsp[-4].minor.yy410.a, yymsp[-4].minor.yy410.b, yymsp[-2].minor.yy347, yymsp[0].minor.yy122, yymsp[-10].minor.yy392, yymsp[-8].minor.yy392); + yygotominor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); +} +#line 3159 "parse.c" break; - case 282: -#line 1032 "parse.y" -{yygotominor.yy46 = OE_Fail;} -#line 3157 "parse.c" - break; - case 283: + case 268: /* trigger_time ::= BEFORE */ + case 271: /* trigger_time ::= */ #line 1037 "parse.y" -{ - sqlite3DropTrigger(pParse,yymsp[0].minor.yy373,yymsp[-1].minor.yy46); -} -#line 3164 "parse.c" +{ yygotominor.yy392 = TK_BEFORE; } +#line 3165 "parse.c" break; - case 284: + case 269: /* trigger_time ::= AFTER */ +#line 1038 "parse.y" +{ yygotominor.yy392 = TK_AFTER; } +#line 3170 "parse.c" + break; + case 270: /* trigger_time ::= INSTEAD OF */ +#line 1039 "parse.y" +{ yygotominor.yy392 = TK_INSTEAD;} +#line 3175 "parse.c" + break; + case 272: /* trigger_event ::= DELETE|INSERT */ + case 273: /* trigger_event ::= UPDATE */ #line 1044 "parse.y" -{ - sqlite3Attach(pParse, yymsp[-3].minor.yy172, yymsp[-1].minor.yy172, yymsp[0].minor.yy386); -} -#line 3171 "parse.c" +{yygotominor.yy410.a = yymsp[0].major; yygotominor.yy410.b = 0;} +#line 3181 "parse.c" break; - case 285: -#line 1047 "parse.y" -{ - sqlite3Detach(pParse, yymsp[0].minor.yy172); -} -#line 3178 "parse.c" + case 274: /* trigger_event ::= UPDATE OF inscollist */ +#line 1046 "parse.y" +{yygotominor.yy410.a = TK_UPDATE; yygotominor.yy410.b = yymsp[0].minor.yy180;} +#line 3186 "parse.c" break; - case 286: + case 277: /* when_clause ::= */ + case 294: /* key_opt ::= */ #line 1053 "parse.y" -{ yygotominor.yy386 = 0; } -#line 3183 "parse.c" +{ yygotominor.yy122 = 0; } +#line 3192 "parse.c" break; - case 287: + case 278: /* when_clause ::= WHEN expr */ + case 295: /* key_opt ::= KEY expr */ #line 1054 "parse.y" -{ yygotominor.yy386 = yymsp[0].minor.yy172; } -#line 3188 "parse.c" - break; - case 290: -#line 1062 "parse.y" -{sqlite3Reindex(pParse, 0, 0);} -#line 3193 "parse.c" - break; - case 291: -#line 1063 "parse.y" -{sqlite3Reindex(pParse, &yymsp[-1].minor.yy410, &yymsp[0].minor.yy410);} +{ yygotominor.yy122 = yymsp[0].minor.yy122; } #line 3198 "parse.c" break; - case 292: -#line 1068 "parse.y" -{sqlite3Analyze(pParse, 0, 0);} -#line 3203 "parse.c" - break; - case 293: -#line 1069 "parse.y" -{sqlite3Analyze(pParse, &yymsp[-1].minor.yy410, &yymsp[0].minor.yy410);} -#line 3208 "parse.c" - break; - case 294: -#line 1074 "parse.y" + case 279: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ +#line 1058 "parse.y" { - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy373,&yymsp[0].minor.yy410); +/* + if( yymsp[-2].minor.yy327 ){ + yymsp[-2].minor.yy327->pLast->pNext = yymsp[-1].minor.yy327; + }else{ + yymsp[-2].minor.yy327 = yymsp[-1].minor.yy327; + } +*/ + assert( yymsp[-2].minor.yy327!=0 ); + yymsp[-2].minor.yy327->pLast->pNext = yymsp[-1].minor.yy327; + yymsp[-2].minor.yy327->pLast = yymsp[-1].minor.yy327; + yygotominor.yy327 = yymsp[-2].minor.yy327; } #line 3215 "parse.c" break; - case 295: -#line 1077 "parse.y" -{ - sqlite3AlterFinishAddColumn(pParse, &yymsp[0].minor.yy410); + case 280: /* trigger_cmd_list ::= trigger_cmd SEMI */ +#line 1071 "parse.y" +{ + /* if( yymsp[-1].minor.yy327 ) */ + assert( yymsp[-1].minor.yy327!=0 ); + yymsp[-1].minor.yy327->pLast = yymsp[-1].minor.yy327; + yygotominor.yy327 = yymsp[-1].minor.yy327; } -#line 3222 "parse.c" +#line 3225 "parse.c" break; - case 296: -#line 1080 "parse.y" -{ - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy373); -} -#line 3229 "parse.c" + case 281: /* trigger_cmd ::= UPDATE orconf nm SET setlist where_opt */ +#line 1082 "parse.y" +{ yygotominor.yy327 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy442, yymsp[0].minor.yy122, yymsp[-4].minor.yy392); } +#line 3230 "parse.c" break; - case 299: -#line 1089 "parse.y" -{sqlite3VtabFinishParse(pParse,0);} -#line 3234 "parse.c" + case 282: /* trigger_cmd ::= insert_cmd INTO nm inscollist_opt VALUES LP itemlist RP */ +#line 1087 "parse.y" +{yygotominor.yy327 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy180, yymsp[-1].minor.yy442, 0, yymsp[-7].minor.yy392);} +#line 3235 "parse.c" break; - case 300: + case 283: /* trigger_cmd ::= insert_cmd INTO nm inscollist_opt select */ #line 1090 "parse.y" -{sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} -#line 3239 "parse.c" +{yygotominor.yy327 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy180, 0, yymsp[0].minor.yy159, yymsp[-4].minor.yy392);} +#line 3240 "parse.c" break; - case 301: -#line 1091 "parse.y" + case 284: /* trigger_cmd ::= DELETE FROM nm where_opt */ +#line 1094 "parse.y" +{yygotominor.yy327 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-1].minor.yy0, yymsp[0].minor.yy122);} +#line 3245 "parse.c" + break; + case 285: /* trigger_cmd ::= select */ +#line 1097 "parse.y" +{yygotominor.yy327 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy159); } +#line 3250 "parse.c" + break; + case 286: /* expr ::= RAISE LP IGNORE RP */ +#line 1100 "parse.y" { - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy410, &yymsp[-2].minor.yy410, &yymsp[0].minor.yy410); + yygotominor.yy122 = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); + if( yygotominor.yy122 ){ + yygotominor.yy122->iColumn = OE_Ignore; + sqlite3ExprSpan(yygotominor.yy122, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0); + } } -#line 3246 "parse.c" +#line 3261 "parse.c" break; - case 304: -#line 1096 "parse.y" + case 287: /* expr ::= RAISE LP raisetype COMMA nm RP */ +#line 1107 "parse.y" +{ + yygotominor.yy122 = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); + if( yygotominor.yy122 ) { + yygotominor.yy122->iColumn = yymsp[-3].minor.yy392; + sqlite3ExprSpan(yygotominor.yy122, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0); + } +} +#line 3272 "parse.c" + break; + case 288: /* raisetype ::= ROLLBACK */ +#line 1117 "parse.y" +{yygotominor.yy392 = OE_Rollback;} +#line 3277 "parse.c" + break; + case 290: /* raisetype ::= FAIL */ +#line 1119 "parse.y" +{yygotominor.yy392 = OE_Fail;} +#line 3282 "parse.c" + break; + case 291: /* cmd ::= DROP TRIGGER ifexists fullname */ +#line 1124 "parse.y" +{ + sqlite3DropTrigger(pParse,yymsp[0].minor.yy347,yymsp[-1].minor.yy392); +} +#line 3289 "parse.c" + break; + case 292: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ +#line 1131 "parse.y" +{ + sqlite3Attach(pParse, yymsp[-3].minor.yy122, yymsp[-1].minor.yy122, yymsp[0].minor.yy122); +} +#line 3296 "parse.c" + break; + case 293: /* cmd ::= DETACH database_kw_opt expr */ +#line 1134 "parse.y" +{ + sqlite3Detach(pParse, yymsp[0].minor.yy122); +} +#line 3303 "parse.c" + break; + case 298: /* cmd ::= REINDEX */ +#line 1149 "parse.y" +{sqlite3Reindex(pParse, 0, 0);} +#line 3308 "parse.c" + break; + case 299: /* cmd ::= REINDEX nm dbnm */ +#line 1150 "parse.y" +{sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} +#line 3313 "parse.c" + break; + case 300: /* cmd ::= ANALYZE */ +#line 1155 "parse.y" +{sqlite3Analyze(pParse, 0, 0);} +#line 3318 "parse.c" + break; + case 301: /* cmd ::= ANALYZE nm dbnm */ +#line 1156 "parse.y" +{sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} +#line 3323 "parse.c" + break; + case 302: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ +#line 1161 "parse.y" +{ + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy347,&yymsp[0].minor.yy0); +} +#line 3330 "parse.c" + break; + case 303: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column */ +#line 1164 "parse.y" +{ + sqlite3AlterFinishAddColumn(pParse, &yymsp[0].minor.yy0); +} +#line 3337 "parse.c" + break; + case 304: /* add_column_fullname ::= fullname */ +#line 1167 "parse.y" +{ + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy347); +} +#line 3344 "parse.c" + break; + case 307: /* cmd ::= create_vtab */ +#line 1176 "parse.y" +{sqlite3VtabFinishParse(pParse,0);} +#line 3349 "parse.c" + break; + case 308: /* cmd ::= create_vtab LP vtabarglist RP */ +#line 1177 "parse.y" +{sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} +#line 3354 "parse.c" + break; + case 309: /* create_vtab ::= CREATE VIRTUAL TABLE nm dbnm USING nm */ +#line 1178 "parse.y" +{ + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); +} +#line 3361 "parse.c" + break; + case 312: /* vtabarg ::= */ +#line 1183 "parse.y" {sqlite3VtabArgInit(pParse);} -#line 3251 "parse.c" +#line 3366 "parse.c" break; - case 306: - case 307: - case 308: - case 310: -#line 1098 "parse.y" + case 314: /* vtabargtoken ::= ANY */ + case 315: /* vtabargtoken ::= lp anylist RP */ + case 316: /* lp ::= LP */ + case 318: /* anylist ::= anylist ANY */ +#line 1185 "parse.y" {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} -#line 3259 "parse.c" +#line 3374 "parse.c" break; }; yygoto = yyRuleInfo[yyruleno].lhs; yysize = yyRuleInfo[yyruleno].nrhs; yypParser->yyidx -= yysize; - yyact = yy_find_reduce_action(yymsp[-yysize].stateno,yygoto); + yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); if( yyact < YYNSTATE ){ #ifdef NDEBUG /* If we are not debugging and the reduce action popped at least @@ -3277,7 +3394,8 @@ static void yy_reduce( { yy_shift(yypParser,yyact,yygoto,&yygotominor); } - }else if( yyact == YYNSTATE + YYNRULE + 1 ){ + }else{ + assert( yyact == YYNSTATE + YYNRULE + 1 ); yy_accept(yypParser); } } @@ -3312,15 +3430,11 @@ static void yy_syntax_error( #define TOKEN (yyminor.yy0) #line 34 "parse.y" - if( !pParse->parseError ){ - if( TOKEN.z[0] ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); - }else{ - sqlite3ErrorMsg(pParse, "incomplete SQL statement"); - } - pParse->parseError = 1; - } -#line 3327 "parse.c" + UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ + assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); + pParse->parseError = 1; +#line 3438 "parse.c" sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ } @@ -3370,7 +3484,9 @@ void sqlite3Parser( YYMINORTYPE yyminorunion; int yyact; /* The parser action. */ int yyendofinput; /* True if we are at the end of input */ +#ifdef YYERRORSYMBOL int yyerrorhit = 0; /* True if yymajor has invoked an error */ +#endif yyParser *yypParser; /* The parser */ /* (re)initialize the parser, if necessary */ @@ -3378,7 +3494,8 @@ void sqlite3Parser( if( yypParser->yyidx<0 ){ #if YYSTACKDEPTH<=0 if( yypParser->yystksz <=0 ){ - memset(&yyminorunion, 0, sizeof(yyminorunion)); + /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/ + yyminorunion = yyzerominor; yyStackOverflow(yypParser, &yyminorunion); return; } @@ -3399,19 +3516,19 @@ void sqlite3Parser( #endif do{ - yyact = yy_find_shift_action(yypParser,yymajor); + yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); if( yyactyyerrcnt--; - if( yyendofinput && yypParser->yyidx>=0 ){ - yymajor = 0; - }else{ - yymajor = YYNOCODE; - } + yymajor = YYNOCODE; }else if( yyact < YYNSTATE + YYNRULE ){ yy_reduce(yypParser,yyact-YYNSTATE); - }else if( yyact == YY_ERROR_ACTION ){ + }else{ + assert( yyact == YY_ERROR_ACTION ); +#ifdef YYERRORSYMBOL int yymx; +#endif #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); @@ -3448,7 +3565,7 @@ void sqlite3Parser( yyTracePrompt,yyTokenName[yymajor]); } #endif - yy_destructor(yymajor,&yyminorunion); + yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion); yymajor = YYNOCODE; }else{ while( @@ -3461,7 +3578,7 @@ void sqlite3Parser( yy_pop_parser_stack(yypParser); } if( yypParser->yyidx < 0 || yymajor==0 ){ - yy_destructor(yymajor,&yyminorunion); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); yy_parse_failed(yypParser); yymajor = YYNOCODE; }else if( yymx!=YYERRORSYMBOL ){ @@ -3486,15 +3603,12 @@ void sqlite3Parser( yy_syntax_error(yypParser,yymajor,yyminorunion); } yypParser->yyerrcnt = 3; - yy_destructor(yymajor,&yyminorunion); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); if( yyendofinput ){ yy_parse_failed(yypParser); } yymajor = YYNOCODE; #endif - }else{ - yy_accept(yypParser); - yymajor = YYNOCODE; } }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); return; diff --git a/parse.h b/parse.h index ed848ec..1194891 100644 --- a/parse.h +++ b/parse.h @@ -10,143 +10,145 @@ #define TK_COMMIT 10 #define TK_END 11 #define TK_ROLLBACK 12 -#define TK_CREATE 13 -#define TK_TABLE 14 -#define TK_IF 15 -#define TK_NOT 16 -#define TK_EXISTS 17 -#define TK_TEMP 18 -#define TK_LP 19 -#define TK_RP 20 -#define TK_AS 21 -#define TK_COMMA 22 -#define TK_ID 23 -#define TK_ABORT 24 -#define TK_AFTER 25 -#define TK_ANALYZE 26 -#define TK_ASC 27 -#define TK_ATTACH 28 -#define TK_BEFORE 29 -#define TK_CASCADE 30 -#define TK_CAST 31 -#define TK_CONFLICT 32 -#define TK_DATABASE 33 -#define TK_DESC 34 -#define TK_DETACH 35 -#define TK_EACH 36 -#define TK_FAIL 37 -#define TK_FOR 38 -#define TK_IGNORE 39 -#define TK_INITIALLY 40 -#define TK_INSTEAD 41 -#define TK_LIKE_KW 42 -#define TK_MATCH 43 -#define TK_KEY 44 -#define TK_OF 45 -#define TK_OFFSET 46 -#define TK_PRAGMA 47 -#define TK_RAISE 48 -#define TK_REPLACE 49 -#define TK_RESTRICT 50 -#define TK_ROW 51 -#define TK_TRIGGER 52 -#define TK_VACUUM 53 -#define TK_VIEW 54 -#define TK_VIRTUAL 55 -#define TK_REINDEX 56 -#define TK_RENAME 57 -#define TK_CTIME_KW 58 -#define TK_ANY 59 -#define TK_OR 60 -#define TK_AND 61 -#define TK_IS 62 -#define TK_BETWEEN 63 -#define TK_IN 64 -#define TK_ISNULL 65 -#define TK_NOTNULL 66 -#define TK_NE 67 -#define TK_EQ 68 -#define TK_GT 69 -#define TK_LE 70 -#define TK_LT 71 -#define TK_GE 72 -#define TK_ESCAPE 73 -#define TK_BITAND 74 -#define TK_BITOR 75 -#define TK_LSHIFT 76 -#define TK_RSHIFT 77 -#define TK_PLUS 78 -#define TK_MINUS 79 -#define TK_STAR 80 -#define TK_SLASH 81 -#define TK_REM 82 -#define TK_CONCAT 83 -#define TK_COLLATE 84 -#define TK_UMINUS 85 -#define TK_UPLUS 86 -#define TK_BITNOT 87 -#define TK_STRING 88 -#define TK_JOIN_KW 89 -#define TK_CONSTRAINT 90 -#define TK_DEFAULT 91 -#define TK_NULL 92 -#define TK_PRIMARY 93 -#define TK_UNIQUE 94 -#define TK_CHECK 95 -#define TK_REFERENCES 96 -#define TK_AUTOINCR 97 -#define TK_ON 98 -#define TK_DELETE 99 -#define TK_UPDATE 100 -#define TK_INSERT 101 -#define TK_SET 102 -#define TK_DEFERRABLE 103 -#define TK_FOREIGN 104 -#define TK_DROP 105 -#define TK_UNION 106 -#define TK_ALL 107 -#define TK_EXCEPT 108 -#define TK_INTERSECT 109 -#define TK_SELECT 110 -#define TK_DISTINCT 111 -#define TK_DOT 112 -#define TK_FROM 113 -#define TK_JOIN 114 -#define TK_USING 115 -#define TK_ORDER 116 -#define TK_BY 117 -#define TK_GROUP 118 -#define TK_HAVING 119 -#define TK_LIMIT 120 -#define TK_WHERE 121 -#define TK_INTO 122 -#define TK_VALUES 123 -#define TK_INTEGER 124 -#define TK_FLOAT 125 -#define TK_BLOB 126 -#define TK_REGISTER 127 -#define TK_VARIABLE 128 -#define TK_CASE 129 -#define TK_WHEN 130 -#define TK_THEN 131 -#define TK_ELSE 132 -#define TK_INDEX 133 -#define TK_ALTER 134 -#define TK_TO 135 -#define TK_ADD 136 -#define TK_COLUMNKW 137 -#define TK_TO_TEXT 138 -#define TK_TO_BLOB 139 -#define TK_TO_NUMERIC 140 -#define TK_TO_INT 141 -#define TK_TO_REAL 142 -#define TK_END_OF_FILE 143 -#define TK_ILLEGAL 144 -#define TK_SPACE 145 -#define TK_UNCLOSED_STRING 146 -#define TK_COMMENT 147 -#define TK_FUNCTION 148 -#define TK_COLUMN 149 -#define TK_AGG_FUNCTION 150 -#define TK_AGG_COLUMN 151 -#define TK_CONST_FUNC 152 +#define TK_SAVEPOINT 13 +#define TK_RELEASE 14 +#define TK_TO 15 +#define TK_CREATE 16 +#define TK_TABLE 17 +#define TK_IF 18 +#define TK_NOT 19 +#define TK_EXISTS 20 +#define TK_TEMP 21 +#define TK_LP 22 +#define TK_RP 23 +#define TK_AS 24 +#define TK_COMMA 25 +#define TK_ID 26 +#define TK_ABORT 27 +#define TK_AFTER 28 +#define TK_ANALYZE 29 +#define TK_ASC 30 +#define TK_ATTACH 31 +#define TK_BEFORE 32 +#define TK_CASCADE 33 +#define TK_CAST 34 +#define TK_CONFLICT 35 +#define TK_DATABASE 36 +#define TK_DESC 37 +#define TK_DETACH 38 +#define TK_EACH 39 +#define TK_FAIL 40 +#define TK_FOR 41 +#define TK_IGNORE 42 +#define TK_INITIALLY 43 +#define TK_INSTEAD 44 +#define TK_LIKE_KW 45 +#define TK_MATCH 46 +#define TK_KEY 47 +#define TK_OF 48 +#define TK_OFFSET 49 +#define TK_PRAGMA 50 +#define TK_RAISE 51 +#define TK_REPLACE 52 +#define TK_RESTRICT 53 +#define TK_ROW 54 +#define TK_TRIGGER 55 +#define TK_VACUUM 56 +#define TK_VIEW 57 +#define TK_VIRTUAL 58 +#define TK_REINDEX 59 +#define TK_RENAME 60 +#define TK_CTIME_KW 61 +#define TK_ANY 62 +#define TK_OR 63 +#define TK_AND 64 +#define TK_IS 65 +#define TK_BETWEEN 66 +#define TK_IN 67 +#define TK_ISNULL 68 +#define TK_NOTNULL 69 +#define TK_NE 70 +#define TK_EQ 71 +#define TK_GT 72 +#define TK_LE 73 +#define TK_LT 74 +#define TK_GE 75 +#define TK_ESCAPE 76 +#define TK_BITAND 77 +#define TK_BITOR 78 +#define TK_LSHIFT 79 +#define TK_RSHIFT 80 +#define TK_PLUS 81 +#define TK_MINUS 82 +#define TK_STAR 83 +#define TK_SLASH 84 +#define TK_REM 85 +#define TK_CONCAT 86 +#define TK_COLLATE 87 +#define TK_UMINUS 88 +#define TK_UPLUS 89 +#define TK_BITNOT 90 +#define TK_STRING 91 +#define TK_JOIN_KW 92 +#define TK_CONSTRAINT 93 +#define TK_DEFAULT 94 +#define TK_NULL 95 +#define TK_PRIMARY 96 +#define TK_UNIQUE 97 +#define TK_CHECK 98 +#define TK_REFERENCES 99 +#define TK_AUTOINCR 100 +#define TK_ON 101 +#define TK_DELETE 102 +#define TK_UPDATE 103 +#define TK_INSERT 104 +#define TK_SET 105 +#define TK_DEFERRABLE 106 +#define TK_FOREIGN 107 +#define TK_DROP 108 +#define TK_UNION 109 +#define TK_ALL 110 +#define TK_EXCEPT 111 +#define TK_INTERSECT 112 +#define TK_SELECT 113 +#define TK_DISTINCT 114 +#define TK_DOT 115 +#define TK_FROM 116 +#define TK_JOIN 117 +#define TK_INDEXED 118 +#define TK_BY 119 +#define TK_USING 120 +#define TK_ORDER 121 +#define TK_GROUP 122 +#define TK_HAVING 123 +#define TK_LIMIT 124 +#define TK_WHERE 125 +#define TK_INTO 126 +#define TK_VALUES 127 +#define TK_INTEGER 128 +#define TK_FLOAT 129 +#define TK_BLOB 130 +#define TK_REGISTER 131 +#define TK_VARIABLE 132 +#define TK_CASE 133 +#define TK_WHEN 134 +#define TK_THEN 135 +#define TK_ELSE 136 +#define TK_INDEX 137 +#define TK_ALTER 138 +#define TK_ADD 139 +#define TK_COLUMNKW 140 +#define TK_TO_TEXT 141 +#define TK_TO_BLOB 142 +#define TK_TO_NUMERIC 143 +#define TK_TO_INT 144 +#define TK_TO_REAL 145 +#define TK_END_OF_FILE 146 +#define TK_ILLEGAL 147 +#define TK_SPACE 148 +#define TK_UNCLOSED_STRING 149 +#define TK_FUNCTION 150 +#define TK_COLUMN 151 +#define TK_AGG_FUNCTION 152 +#define TK_AGG_COLUMN 153 +#define TK_CONST_FUNC 154 diff --git a/pcache.c b/pcache.c new file mode 100644 index 0000000..b513e27 --- /dev/null +++ b/pcache.c @@ -0,0 +1,580 @@ +/* +** 2008 August 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements that page cache. +** +** @(#) $Id: pcache.c,v 1.39 2008/12/04 20:40:10 drh Exp $ +*/ +#include "sqliteInt.h" + +/* +** A complete page cache is an instance of this structure. +*/ +struct PCache { + PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ + PgHdr *pSynced; /* Last synced page in dirty page list */ + int nRef; /* Number of referenced pages */ + int nMax; /* Configured cache size */ + int nMin; /* Configured minimum cache size */ + int szPage; /* Size of every page in this cache */ + int szExtra; /* Size of extra space for each page */ + int bPurgeable; /* True if pages are on backing store */ + int (*xStress)(void*,PgHdr*); /* Call to try make a page clean */ + void *pStress; /* Argument to xStress */ + sqlite3_pcache *pCache; /* Pluggable cache module */ + PgHdr *pPage1; +}; + +/* +** Some of the assert() macros in this code are too expensive to run +** even during normal debugging. Use them only rarely on long-running +** tests. Enable the expensive asserts using the +** -DSQLITE_ENABLE_EXPENSIVE_ASSERT=1 compile-time option. +*/ +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT +# define expensive_assert(X) assert(X) +#else +# define expensive_assert(X) +#endif + +/********************************** Linked List Management ********************/ + +#if !defined(NDEBUG) && defined(SQLITE_ENABLE_EXPENSIVE_ASSERT) +/* +** Check that the pCache->pSynced variable is set correctly. If it +** is not, either fail an assert or return zero. Otherwise, return +** non-zero. This is only used in debugging builds, as follows: +** +** expensive_assert( pcacheCheckSynced(pCache) ); +*/ +static int pcacheCheckSynced(PCache *pCache){ + PgHdr *p; + for(p=pCache->pDirtyTail; p!=pCache->pSynced; p=p->pDirtyPrev){ + assert( p->nRef || (p->flags&PGHDR_NEED_SYNC) ); + } + return (p==0 || p->nRef || (p->flags&PGHDR_NEED_SYNC)==0); +} +#endif /* !NDEBUG && SQLITE_ENABLE_EXPENSIVE_ASSERT */ + +/* +** Remove page pPage from the list of dirty pages. +*/ +static void pcacheRemoveFromDirtyList(PgHdr *pPage){ + PCache *p = pPage->pCache; + + assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); + assert( pPage->pDirtyPrev || pPage==p->pDirty ); + + /* Update the PCache1.pSynced variable if necessary. */ + if( p->pSynced==pPage ){ + PgHdr *pSynced = pPage->pDirtyPrev; + while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){ + pSynced = pSynced->pDirtyPrev; + } + p->pSynced = pSynced; + } + + if( pPage->pDirtyNext ){ + pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; + }else{ + assert( pPage==p->pDirtyTail ); + p->pDirtyTail = pPage->pDirtyPrev; + } + if( pPage->pDirtyPrev ){ + pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; + }else{ + assert( pPage==p->pDirty ); + p->pDirty = pPage->pDirtyNext; + } + pPage->pDirtyNext = 0; + pPage->pDirtyPrev = 0; + + expensive_assert( pcacheCheckSynced(p) ); +} + +/* +** Add page pPage to the head of the dirty list (PCache1.pDirty is set to +** pPage). +*/ +static void pcacheAddToDirtyList(PgHdr *pPage){ + PCache *p = pPage->pCache; + + assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); + + pPage->pDirtyNext = p->pDirty; + if( pPage->pDirtyNext ){ + assert( pPage->pDirtyNext->pDirtyPrev==0 ); + pPage->pDirtyNext->pDirtyPrev = pPage; + } + p->pDirty = pPage; + if( !p->pDirtyTail ){ + p->pDirtyTail = pPage; + } + if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ + p->pSynced = pPage; + } + expensive_assert( pcacheCheckSynced(p) ); +} + +/* +** Wrapper around the pluggable caches xUnpin method. If the cache is +** being used for an in-memory database, this function is a no-op. +*/ +static void pcacheUnpin(PgHdr *p){ + PCache *pCache = p->pCache; + if( pCache->bPurgeable ){ + if( p->pgno==1 ){ + pCache->pPage1 = 0; + } + sqlite3GlobalConfig.pcache.xUnpin(pCache->pCache, p, 0); + } +} + +/*************************************************** General Interfaces ****** +** +** Initialize and shutdown the page cache subsystem. Neither of these +** functions are threadsafe. +*/ +int sqlite3PcacheInitialize(void){ + if( sqlite3GlobalConfig.pcache.xInit==0 ){ + sqlite3PCacheSetDefault(); + } + return sqlite3GlobalConfig.pcache.xInit(sqlite3GlobalConfig.pcache.pArg); +} +void sqlite3PcacheShutdown(void){ + if( sqlite3GlobalConfig.pcache.xShutdown ){ + sqlite3GlobalConfig.pcache.xShutdown(sqlite3GlobalConfig.pcache.pArg); + } +} + +/* +** Return the size in bytes of a PCache object. +*/ +int sqlite3PcacheSize(void){ return sizeof(PCache); } + +/* +** Create a new PCache object. Storage space to hold the object +** has already been allocated and is passed in as the p pointer. +** The caller discovers how much space needs to be allocated by +** calling sqlite3PcacheSize(). +*/ +void sqlite3PcacheOpen( + int szPage, /* Size of every page */ + int szExtra, /* Extra space associated with each page */ + int bPurgeable, /* True if pages are on backing store */ + int (*xStress)(void*,PgHdr*),/* Call to try to make pages clean */ + void *pStress, /* Argument to xStress */ + PCache *p /* Preallocated space for the PCache */ +){ + memset(p, 0, sizeof(PCache)); + p->szPage = szPage; + p->szExtra = szExtra; + p->bPurgeable = bPurgeable; + p->xStress = xStress; + p->pStress = pStress; + p->nMax = 100; + p->nMin = 10; +} + +/* +** Change the page size for PCache object. The caller must ensure that there +** are no outstanding page references when this function is called. +*/ +void sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ + assert( pCache->nRef==0 && pCache->pDirty==0 ); + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache.xDestroy(pCache->pCache); + pCache->pCache = 0; + } + pCache->szPage = szPage; +} + +/* +** Try to obtain a page from the cache. +*/ +int sqlite3PcacheFetch( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number to obtain */ + int createFlag, /* If true, create page if it does not exist already */ + PgHdr **ppPage /* Write the page here */ +){ + PgHdr *pPage = 0; + int eCreate; + + assert( pCache!=0 ); + assert( pgno>0 ); + + /* If the pluggable cache (sqlite3_pcache*) has not been allocated, + ** allocate it now. + */ + if( !pCache->pCache && createFlag ){ + sqlite3_pcache *p; + int nByte; + nByte = pCache->szPage + pCache->szExtra + sizeof(PgHdr); + p = sqlite3GlobalConfig.pcache.xCreate(nByte, pCache->bPurgeable); + if( !p ){ + return SQLITE_NOMEM; + } + sqlite3GlobalConfig.pcache.xCachesize(p, pCache->nMax); + pCache->pCache = p; + } + + eCreate = createFlag ? 1 : 0; + if( eCreate && (!pCache->bPurgeable || !pCache->pDirty) ){ + eCreate = 2; + } + if( pCache->pCache ){ + pPage = sqlite3GlobalConfig.pcache.xFetch(pCache->pCache, pgno, eCreate); + } + + if( !pPage && eCreate==1 ){ + PgHdr *pPg; + + /* Find a dirty page to write-out and recycle. First try to find a + ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC + ** cleared), but if that is not possible settle for any other + ** unreferenced dirty page. + */ + expensive_assert( pcacheCheckSynced(pCache) ); + for(pPg=pCache->pSynced; + pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); + pPg=pPg->pDirtyPrev + ); + if( !pPg ){ + for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); + } + if( pPg ){ + int rc; + rc = pCache->xStress(pCache->pStress, pPg); + if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ + return rc; + } + } + + pPage = sqlite3GlobalConfig.pcache.xFetch(pCache->pCache, pgno, 2); + } + + if( pPage ){ + if( 0==pPage->nRef ){ + pCache->nRef++; + } + pPage->nRef++; + pPage->pData = (void*)&pPage[1]; + pPage->pExtra = (void*)&((char*)pPage->pData)[pCache->szPage]; + pPage->pCache = pCache; + pPage->pgno = pgno; + if( pgno==1 ){ + pCache->pPage1 = pPage; + } + } + *ppPage = pPage; + return (pPage==0 && eCreate) ? SQLITE_NOMEM : SQLITE_OK; +} + +/* +** Decrement the reference count on a page. If the page is clean and the +** reference count drops to 0, then it is made elible for recycling. +*/ +void sqlite3PcacheRelease(PgHdr *p){ + assert( p->nRef>0 ); + p->nRef--; + if( p->nRef==0 ){ + PCache *pCache = p->pCache; + pCache->nRef--; + if( (p->flags&PGHDR_DIRTY)==0 ){ + pcacheUnpin(p); + }else{ + /* Move the page to the head of the dirty list. */ + pcacheRemoveFromDirtyList(p); + pcacheAddToDirtyList(p); + } + } +} + +/* +** Increase the reference count of a supplied page by 1. +*/ +void sqlite3PcacheRef(PgHdr *p){ + assert(p->nRef>0); + p->nRef++; +} + +/* +** Drop a page from the cache. There must be exactly one reference to the +** page. This function deletes that reference, so after it returns the +** page pointed to by p is invalid. +*/ +void sqlite3PcacheDrop(PgHdr *p){ + PCache *pCache; + assert( p->nRef==1 ); + if( p->flags&PGHDR_DIRTY ){ + pcacheRemoveFromDirtyList(p); + } + pCache = p->pCache; + pCache->nRef--; + if( p->pgno==1 ){ + pCache->pPage1 = 0; + } + sqlite3GlobalConfig.pcache.xUnpin(pCache->pCache, p, 1); +} + +/* +** Make sure the page is marked as dirty. If it isn't dirty already, +** make it so. +*/ +void sqlite3PcacheMakeDirty(PgHdr *p){ + PCache *pCache; + p->flags &= ~PGHDR_DONT_WRITE; + assert( p->nRef>0 ); + if( 0==(p->flags & PGHDR_DIRTY) ){ + pCache = p->pCache; + p->flags |= PGHDR_DIRTY; + pcacheAddToDirtyList( p); + } +} + +/* +** Make sure the page is marked as clean. If it isn't clean already, +** make it so. +*/ +void sqlite3PcacheMakeClean(PgHdr *p){ + if( (p->flags & PGHDR_DIRTY) ){ + pcacheRemoveFromDirtyList(p); + p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC); + if( p->nRef==0 ){ + pcacheUnpin(p); + } + } +} + +/* +** Make every page in the cache clean. +*/ +void sqlite3PcacheCleanAll(PCache *pCache){ + PgHdr *p; + while( (p = pCache->pDirty)!=0 ){ + sqlite3PcacheMakeClean(p); + } +} + +/* +** Clear the PGHDR_NEED_SYNC flag from all dirty pages. +*/ +void sqlite3PcacheClearSyncFlags(PCache *pCache){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~PGHDR_NEED_SYNC; + } + pCache->pSynced = pCache->pDirtyTail; +} + +/* +** Change the page number of page p to newPgno. +*/ +void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ + PCache *pCache = p->pCache; + assert( p->nRef>0 ); + assert( newPgno>0 ); + sqlite3GlobalConfig.pcache.xRekey(pCache->pCache, p, p->pgno, newPgno); + p->pgno = newPgno; + if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ + pcacheRemoveFromDirtyList(p); + pcacheAddToDirtyList(p); + } +} + +/* +** Drop every cache entry whose page number is greater than "pgno". The +** caller must ensure that there are no outstanding references to any pages +** other than page 1 with a page number greater than pgno. +** +** If there is a reference to page 1 and the pgno parameter passed to this +** function is 0, then the data area associated with page 1 is zeroed, but +** the page object is not dropped. +*/ +void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ + if( pCache->pCache ){ + PgHdr *p; + PgHdr *pNext; + for(p=pCache->pDirty; p; p=pNext){ + pNext = p->pDirtyNext; + if( p->pgno>pgno ){ + assert( p->flags&PGHDR_DIRTY ); + sqlite3PcacheMakeClean(p); + } + } + if( pgno==0 && pCache->pPage1 ){ + memset(pCache->pPage1->pData, 0, pCache->szPage); + pgno = 1; + } + sqlite3GlobalConfig.pcache.xTruncate(pCache->pCache, pgno+1); + } +} + +/* +** Close a cache. +*/ +void sqlite3PcacheClose(PCache *pCache){ + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache.xDestroy(pCache->pCache); + } +} + +/* +** Discard the contents of the cache. +*/ +int sqlite3PcacheClear(PCache *pCache){ + sqlite3PcacheTruncate(pCache, 0); + return SQLITE_OK; +} + +/* +** Merge two lists of pages connected by pDirty and in pgno order. +** Do not both fixing the pDirtyPrev pointers. +*/ +static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ + PgHdr result, *pTail; + pTail = &result; + while( pA && pB ){ + if( pA->pgnopgno ){ + pTail->pDirty = pA; + pTail = pA; + pA = pA->pDirty; + }else{ + pTail->pDirty = pB; + pTail = pB; + pB = pB->pDirty; + } + } + if( pA ){ + pTail->pDirty = pA; + }else if( pB ){ + pTail->pDirty = pB; + }else{ + pTail->pDirty = 0; + } + return result.pDirty; +} + +/* +** Sort the list of pages in accending order by pgno. Pages are +** connected by pDirty pointers. The pDirtyPrev pointers are +** corrupted by this sort. +*/ +#define N_SORT_BUCKET_ALLOC 25 +#define N_SORT_BUCKET 25 +#ifdef SQLITE_TEST + int sqlite3_pager_n_sort_bucket = 0; + #undef N_SORT_BUCKET + #define N_SORT_BUCKET \ + (sqlite3_pager_n_sort_bucket?sqlite3_pager_n_sort_bucket:N_SORT_BUCKET_ALLOC) +#endif +static PgHdr *pcacheSortDirtyList(PgHdr *pIn){ + PgHdr *a[N_SORT_BUCKET_ALLOC], *p; + int i; + memset(a, 0, sizeof(a)); + while( pIn ){ + p = pIn; + pIn = p->pDirty; + p->pDirty = 0; + for(i=0; ipDirty; p; p=p->pDirtyNext){ + p->pDirty = p->pDirtyNext; + } + return pcacheSortDirtyList(pCache->pDirty); +} + +/* +** Return the total number of referenced pages held by the cache. +*/ +int sqlite3PcacheRefCount(PCache *pCache){ + return pCache->nRef; +} + +/* +** Return the number of references to the page supplied as an argument. +*/ +int sqlite3PcachePageRefcount(PgHdr *p){ + return p->nRef; +} + +/* +** Return the total number of pages in the cache. +*/ +int sqlite3PcachePagecount(PCache *pCache){ + int nPage = 0; + if( pCache->pCache ){ + nPage = sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache); + } + return nPage; +} + +#ifdef SQLITE_TEST +/* +** Get the suggested cache-size value. +*/ +int sqlite3PcacheGetCachesize(PCache *pCache){ + return pCache->nMax; +} +#endif + +/* +** Set the suggested cache-size value. +*/ +void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){ + pCache->nMax = mxPage; + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache.xCachesize(pCache->pCache, mxPage); + } +} + +#ifdef SQLITE_CHECK_PAGES +/* +** For all dirty pages currently in the cache, invoke the specified +** callback. This is only used if the SQLITE_CHECK_PAGES macro is +** defined. +*/ +void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)){ + PgHdr *pDirty; + for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext){ + xIter(pDirty); + } +} +#endif diff --git a/pcache.h b/pcache.h new file mode 100644 index 0000000..8c44287 --- /dev/null +++ b/pcache.h @@ -0,0 +1,157 @@ +/* +** 2008 August 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the sqlite page cache +** subsystem. +** +** @(#) $Id: pcache.h,v 1.16 2008/11/19 16:52:44 danielk1977 Exp $ +*/ + +#ifndef _PCACHE_H_ + +typedef struct PgHdr PgHdr; +typedef struct PCache PCache; + +/* +** Every page in the cache is controlled by an instance of the following +** structure. +*/ +struct PgHdr { + void *pData; /* Content of this page */ + void *pExtra; /* Extra content */ + PgHdr *pDirty; /* Transient list of dirty pages */ + Pgno pgno; /* Page number for this page */ + Pager *pPager; /* The pager this page is part of */ +#ifdef SQLITE_CHECK_PAGES + u32 pageHash; /* Hash of page content */ +#endif + u16 flags; /* PGHDR flags defined below */ + + /********************************************************************** + ** Elements above are public. All that follows is private to pcache.c + ** and should not be accessed by other modules. + */ + i16 nRef; /* Number of users of this page */ + PCache *pCache; /* Cache that owns this page */ + + PgHdr *pDirtyNext; /* Next element in list of dirty pages */ + PgHdr *pDirtyPrev; /* Previous element in list of dirty pages */ +}; + +/* Bit values for PgHdr.flags */ +#define PGHDR_DIRTY 0x002 /* Page has changed */ +#define PGHDR_NEED_SYNC 0x004 /* Fsync the rollback journal before + ** writing this page to the database */ +#define PGHDR_NEED_READ 0x008 /* Content is unread */ +#define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ +#define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ + +/* Initialize and shutdown the page cache subsystem */ +int sqlite3PcacheInitialize(void); +void sqlite3PcacheShutdown(void); + +/* Page cache buffer management: +** These routines implement SQLITE_CONFIG_PAGECACHE. +*/ +void sqlite3PCacheBufferSetup(void *, int sz, int n); + +/* Create a new pager cache. +** Under memory stress, invoke xStress to try to make pages clean. +** Only clean and unpinned pages can be reclaimed. +*/ +void sqlite3PcacheOpen( + int szPage, /* Size of every page */ + int szExtra, /* Extra space associated with each page */ + int bPurgeable, /* True if pages are on backing store */ + int (*xStress)(void*, PgHdr*), /* Call to try to make pages clean */ + void *pStress, /* Argument to xStress */ + PCache *pToInit /* Preallocated space for the PCache */ +); + +/* Modify the page-size after the cache has been created. */ +void sqlite3PcacheSetPageSize(PCache *, int); + +/* Return the size in bytes of a PCache object. Used to preallocate +** storage space. +*/ +int sqlite3PcacheSize(void); + +/* One release per successful fetch. Page is pinned until released. +** Reference counted. +*/ +int sqlite3PcacheFetch(PCache*, Pgno, int createFlag, PgHdr**); +void sqlite3PcacheRelease(PgHdr*); + +void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ +void sqlite3PcacheMakeDirty(PgHdr*); /* Make sure page is marked dirty */ +void sqlite3PcacheMakeClean(PgHdr*); /* Mark a single page as clean */ +void sqlite3PcacheCleanAll(PCache*); /* Mark all dirty list pages as clean */ + +/* Change a page number. Used by incr-vacuum. */ +void sqlite3PcacheMove(PgHdr*, Pgno); + +/* Remove all pages with pgno>x. Reset the cache if x==0 */ +void sqlite3PcacheTruncate(PCache*, Pgno x); + +/* Get a list of all dirty pages in the cache, sorted by page number */ +PgHdr *sqlite3PcacheDirtyList(PCache*); + +/* Reset and close the cache object */ +void sqlite3PcacheClose(PCache*); + +/* Clear flags from pages of the page cache */ +void sqlite3PcacheClearSyncFlags(PCache *); + +/* Discard the contents of the cache */ +int sqlite3PcacheClear(PCache*); + +/* Return the total number of outstanding page references */ +int sqlite3PcacheRefCount(PCache*); + +/* Increment the reference count of an existing page */ +void sqlite3PcacheRef(PgHdr*); + +int sqlite3PcachePageRefcount(PgHdr*); + +/* Return the total number of pages stored in the cache */ +int sqlite3PcachePagecount(PCache*); + +#ifdef SQLITE_CHECK_PAGES +/* Iterate through all dirty pages currently stored in the cache. This +** interface is only available if SQLITE_CHECK_PAGES is defined when the +** library is built. +*/ +void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)); +#endif + +/* Set and get the suggested cache-size for the specified pager-cache. +** +** If no global maximum is configured, then the system attempts to limit +** the total number of pages cached by purgeable pager-caches to the sum +** of the suggested cache-sizes. +*/ +void sqlite3PcacheSetCachesize(PCache *, int); +#ifdef SQLITE_TEST +int sqlite3PcacheGetCachesize(PCache *); +#endif + +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +/* Try to return memory used by the pcache module to the main memory heap */ +int sqlite3PcacheReleaseMemory(int); +#endif + +#ifdef SQLITE_TEST +void sqlite3PcacheStats(int*,int*,int*,int*); +#endif + +void sqlite3PCacheSetDefault(void); + +#endif /* _PCACHE_H_ */ diff --git a/pcache1.c b/pcache1.c new file mode 100644 index 0000000..1a8d6aa --- /dev/null +++ b/pcache1.c @@ -0,0 +1,751 @@ +/* +** 2008 November 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements the default page cache implementation (the +** sqlite3_pcache interface). It also contains part of the implementation +** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. +** If the default page cache implementation is overriden, then neither of +** these two features are available. +** +** @(#) $Id: pcache1.c,v 1.7 2009/01/07 15:18:21 danielk1977 Exp $ +*/ + +#include "sqliteInt.h" + +typedef struct PCache1 PCache1; +typedef struct PgHdr1 PgHdr1; +typedef struct PgFreeslot PgFreeslot; + +/* Pointers to structures of this type are cast and returned as +** opaque sqlite3_pcache* handles +*/ +struct PCache1 { + /* Cache configuration parameters. Page size (szPage) and the purgeable + ** flag (bPurgeable) are set when the cache is created. nMax may be + ** modified at any time by a call to the pcache1CacheSize() method. + ** The global mutex must be held when accessing nMax. + */ + int szPage; /* Size of allocated pages in bytes */ + int bPurgeable; /* True if cache is purgeable */ + unsigned int nMin; /* Minimum number of pages reserved */ + unsigned int nMax; /* Configured "cache_size" value */ + + /* Hash table of all pages. The following variables may only be accessed + ** when the accessor is holding the global mutex (see pcache1EnterMutex() + ** and pcache1LeaveMutex()). + */ + unsigned int nRecyclable; /* Number of pages in the LRU list */ + unsigned int nPage; /* Total number of pages in apHash */ + unsigned int nHash; /* Number of slots in apHash[] */ + PgHdr1 **apHash; /* Hash table for fast lookup by key */ + + unsigned int iMaxKey; /* Largest key seen since xTruncate() */ +}; + +/* +** Each cache entry is represented by an instance of the following +** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated +** directly after the structure in memory (see the PGHDR1_TO_PAGE() +** macro below). +*/ +struct PgHdr1 { + unsigned int iKey; /* Key value (page number) */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ +}; + +/* +** Free slots in the allocator used to divide up the buffer provided using +** the SQLITE_CONFIG_PAGECACHE mechanism. +*/ +struct PgFreeslot { + PgFreeslot *pNext; /* Next free slot */ +}; + +/* +** Global data used by this cache. +*/ +static SQLITE_WSD struct PCacheGlobal { + sqlite3_mutex *mutex; /* static mutex MUTEX_STATIC_LRU */ + + int nMaxPage; /* Sum of nMaxPage for purgeable caches */ + int nMinPage; /* Sum of nMinPage for purgeable caches */ + int nCurrentPage; /* Number of purgeable pages allocated */ + PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ + + /* Variables related to SQLITE_CONFIG_PAGECACHE settings. */ + int szSlot; /* Size of each free slot */ + void *pStart, *pEnd; /* Bounds of pagecache malloc range */ + PgFreeslot *pFree; /* Free page blocks */ +} pcache1_g; + +/* +** All code in this file should access the global structure above via the +** alias "pcache1". This ensures that the WSD emulation is used when +** compiling for systems that do not support real WSD. +*/ +#define pcache1 (GLOBAL(struct PCacheGlobal, pcache1_g)) + +/* +** When a PgHdr1 structure is allocated, the associated PCache1.szPage +** bytes of data are located directly after it in memory (i.e. the total +** size of the allocation is sizeof(PgHdr1)+PCache1.szPage byte). The +** PGHDR1_TO_PAGE() macro takes a pointer to a PgHdr1 structure as +** an argument and returns a pointer to the associated block of szPage +** bytes. The PAGE_TO_PGHDR1() macro does the opposite: its argument is +** a pointer to a block of szPage bytes of data and the return value is +** a pointer to the associated PgHdr1 structure. +** +** assert( PGHDR1_TO_PAGE(PAGE_TO_PGHDR1(X))==X ); +*/ +#define PGHDR1_TO_PAGE(p) (void *)(&((unsigned char *)p)[sizeof(PgHdr1)]) +#define PAGE_TO_PGHDR1(p) (PgHdr1 *)(&((unsigned char *)p)[-1*(int)sizeof(PgHdr1)]) + +/* +** Macros to enter and leave the global LRU mutex. +*/ +#define pcache1EnterMutex() sqlite3_mutex_enter(pcache1.mutex) +#define pcache1LeaveMutex() sqlite3_mutex_leave(pcache1.mutex) + +/******************************************************************************/ +/******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ + +/* +** This function is called during initialization if a static buffer is +** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE +** verb to sqlite3_config(). Parameter pBuf points to an allocation large +** enough to contain 'n' buffers of 'sz' bytes each. +*/ +void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ + PgFreeslot *p; + sz &= ~7; + pcache1.szSlot = sz; + pcache1.pStart = pBuf; + pcache1.pFree = 0; + while( n-- ){ + p = (PgFreeslot*)pBuf; + p->pNext = pcache1.pFree; + pcache1.pFree = p; + pBuf = (void*)&((char*)pBuf)[sz]; + } + pcache1.pEnd = pBuf; +} + +/* +** Malloc function used within this file to allocate space from the buffer +** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no +** such buffer exists or there is no space left in it, this function falls +** back to sqlite3Malloc(). +*/ +static void *pcache1Alloc(int nByte){ + void *p; + assert( sqlite3_mutex_held(pcache1.mutex) ); + if( nByte<=pcache1.szSlot && pcache1.pFree ){ + p = (PgHdr1 *)pcache1.pFree; + pcache1.pFree = pcache1.pFree->pNext; + sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1); + }else{ + + /* Allocate a new buffer using sqlite3Malloc. Before doing so, exit the + ** global pcache mutex and unlock the pager-cache object pCache. This is + ** so that if the attempt to allocate a new buffer causes the the + ** configured soft-heap-limit to be breached, it will be possible to + ** reclaim memory from this pager-cache. + */ + pcache1LeaveMutex(); + p = sqlite3Malloc(nByte); + pcache1EnterMutex(); + if( p ){ + int sz = sqlite3MallocSize(p); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); + } + } + return p; +} + +/* +** Free an allocated buffer obtained from pcache1Alloc(). +*/ +static void pcache1Free(void *p){ + assert( sqlite3_mutex_held(pcache1.mutex) ); + if( p==0 ) return; + if( p>=pcache1.pStart && ppNext = pcache1.pFree; + pcache1.pFree = pSlot; + }else{ + int iSize = sqlite3MallocSize(p); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -iSize); + sqlite3_free(p); + } +} + +/* +** Allocate a new page object initially associated with cache pCache. +*/ +static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ + int nByte = sizeof(PgHdr1) + pCache->szPage; + PgHdr1 *p = (PgHdr1 *)pcache1Alloc(nByte); + if( p ){ + memset(p, 0, nByte); + if( pCache->bPurgeable ){ + pcache1.nCurrentPage++; + } + } + return p; +} + +/* +** Free a page object allocated by pcache1AllocPage(). +*/ +static void pcache1FreePage(PgHdr1 *p){ + if( p ){ + if( p->pCache->bPurgeable ){ + pcache1.nCurrentPage--; + } + pcache1Free(p); + } +} + +/* +** Malloc function used by SQLite to obtain space from the buffer configured +** using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no such buffer +** exists, this function falls back to sqlite3Malloc(). +*/ +void *sqlite3PageMalloc(int sz){ + void *p; + pcache1EnterMutex(); + p = pcache1Alloc(sz); + pcache1LeaveMutex(); + return p; +} + +/* +** Free an allocated buffer obtained from sqlite3PageMalloc(). +*/ +void sqlite3PageFree(void *p){ + pcache1EnterMutex(); + pcache1Free(p); + pcache1LeaveMutex(); +} + +/******************************************************************************/ +/******** General Implementation Functions ************************************/ + +/* +** This function is used to resize the hash table used by the cache passed +** as the first argument. +** +** The global mutex must be held when this function is called. +*/ +static int pcache1ResizeHash(PCache1 *p){ + PgHdr1 **apNew; + unsigned int nNew; + unsigned int i; + + assert( sqlite3_mutex_held(pcache1.mutex) ); + + nNew = p->nHash*2; + if( nNew<256 ){ + nNew = 256; + } + + pcache1LeaveMutex(); + if( p->nHash ){ sqlite3BeginBenignMalloc(); } + apNew = (PgHdr1 **)sqlite3_malloc(sizeof(PgHdr1 *)*nNew); + if( p->nHash ){ sqlite3EndBenignMalloc(); } + pcache1EnterMutex(); + if( apNew ){ + memset(apNew, 0, sizeof(PgHdr1 *)*nNew); + for(i=0; inHash; i++){ + PgHdr1 *pPage; + PgHdr1 *pNext = p->apHash[i]; + while( (pPage = pNext)!=0 ){ + unsigned int h = pPage->iKey % nNew; + pNext = pPage->pNext; + pPage->pNext = apNew[h]; + apNew[h] = pPage; + } + } + sqlite3_free(p->apHash); + p->apHash = apNew; + p->nHash = nNew; + } + + return (p->apHash ? SQLITE_OK : SQLITE_NOMEM); +} + +/* +** This function is used internally to remove the page pPage from the +** global LRU list, if is part of it. If pPage is not part of the global +** LRU list, then this function is a no-op. +** +** The global mutex must be held when this function is called. +*/ +static void pcache1PinPage(PgHdr1 *pPage){ + assert( sqlite3_mutex_held(pcache1.mutex) ); + if( pPage && (pPage->pLruNext || pPage==pcache1.pLruTail) ){ + if( pPage->pLruPrev ){ + pPage->pLruPrev->pLruNext = pPage->pLruNext; + } + if( pPage->pLruNext ){ + pPage->pLruNext->pLruPrev = pPage->pLruPrev; + } + if( pcache1.pLruHead==pPage ){ + pcache1.pLruHead = pPage->pLruNext; + } + if( pcache1.pLruTail==pPage ){ + pcache1.pLruTail = pPage->pLruPrev; + } + pPage->pLruNext = 0; + pPage->pLruPrev = 0; + pPage->pCache->nRecyclable--; + } +} + + +/* +** Remove the page supplied as an argument from the hash table +** (PCache1.apHash structure) that it is currently stored in. +** +** The global mutex must be held when this function is called. +*/ +static void pcache1RemoveFromHash(PgHdr1 *pPage){ + unsigned int h; + PCache1 *pCache = pPage->pCache; + PgHdr1 **pp; + + h = pPage->iKey % pCache->nHash; + for(pp=&pCache->apHash[h]; (*pp)!=pPage; pp=&(*pp)->pNext); + *pp = (*pp)->pNext; + + pCache->nPage--; +} + +/* +** If there are currently more than pcache.nMaxPage pages allocated, try +** to recycle pages to reduce the number allocated to pcache.nMaxPage. +*/ +static void pcache1EnforceMaxPage(void){ + assert( sqlite3_mutex_held(pcache1.mutex) ); + while( pcache1.nCurrentPage>pcache1.nMaxPage && pcache1.pLruTail ){ + PgHdr1 *p = pcache1.pLruTail; + pcache1PinPage(p); + pcache1RemoveFromHash(p); + pcache1FreePage(p); + } +} + +/* +** Discard all pages from cache pCache with a page number (key value) +** greater than or equal to iLimit. Any pinned pages that meet this +** criteria are unpinned before they are discarded. +** +** The global mutex must be held when this function is called. +*/ +static void pcache1TruncateUnsafe( + PCache1 *pCache, + unsigned int iLimit +){ + unsigned int h; + assert( sqlite3_mutex_held(pcache1.mutex) ); + for(h=0; hnHash; h++){ + PgHdr1 **pp = &pCache->apHash[h]; + PgHdr1 *pPage; + while( (pPage = *pp)!=0 ){ + if( pPage->iKey>=iLimit ){ + pcache1PinPage(pPage); + *pp = pPage->pNext; + pcache1FreePage(pPage); + }else{ + pp = &pPage->pNext; + } + } + } +} + +/******************************************************************************/ +/******** sqlite3_pcache Methods **********************************************/ + +/* +** Implementation of the sqlite3_pcache.xInit method. +*/ +static int pcache1Init(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + memset(&pcache1, 0, sizeof(pcache1)); + if( sqlite3GlobalConfig.bCoreMutex ){ + pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); + } + return SQLITE_OK; +} + +/* +** Implementation of the sqlite3_pcache.xShutdown method. +*/ +static void pcache1Shutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + /* no-op */ +} + +/* +** Implementation of the sqlite3_pcache.xCreate method. +** +** Allocate a new cache. +*/ +static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){ + PCache1 *pCache; + + pCache = (PCache1 *)sqlite3_malloc(sizeof(PCache1)); + if( pCache ){ + memset(pCache, 0, sizeof(PCache1)); + pCache->szPage = szPage; + pCache->bPurgeable = (bPurgeable ? 1 : 0); + if( bPurgeable ){ + pCache->nMin = 10; + pcache1EnterMutex(); + pcache1.nMinPage += pCache->nMin; + pcache1LeaveMutex(); + } + } + return (sqlite3_pcache *)pCache; +} + +/* +** Implementation of the sqlite3_pcache.xCachesize method. +** +** Configure the cache_size limit for a cache. +*/ +static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ + PCache1 *pCache = (PCache1 *)p; + if( pCache->bPurgeable ){ + pcache1EnterMutex(); + pcache1.nMaxPage += (nMax - pCache->nMax); + pCache->nMax = nMax; + pcache1EnforceMaxPage(); + pcache1LeaveMutex(); + } +} + +/* +** Implementation of the sqlite3_pcache.xPagecount method. +*/ +static int pcache1Pagecount(sqlite3_pcache *p){ + int n; + pcache1EnterMutex(); + n = ((PCache1 *)p)->nPage; + pcache1LeaveMutex(); + return n; +} + +/* +** Implementation of the sqlite3_pcache.xFetch method. +** +** Fetch a page by key value. +** +** Whether or not a new page may be allocated by this function depends on +** the value of the createFlag argument. +** +** There are three different approaches to obtaining space for a page, +** depending on the value of parameter createFlag (which may be 0, 1 or 2). +** +** 1. Regardless of the value of createFlag, the cache is searched for a +** copy of the requested page. If one is found, it is returned. +** +** 2. If createFlag==0 and the page is not already in the cache, NULL is +** returned. +** +** 3. If createFlag is 1, the cache is marked as purgeable and the page is +** not already in the cache, and if either of the following are true, +** return NULL: +** +** (a) the number of pages pinned by the cache is greater than +** PCache1.nMax, or +** (b) the number of pages pinned by the cache is greater than +** the sum of nMax for all purgeable caches, less the sum of +** nMin for all other purgeable caches. +** +** 4. If none of the first three conditions apply and the cache is marked +** as purgeable, and if one of the following is true: +** +** (a) The number of pages allocated for the cache is already +** PCache1.nMax, or +** +** (b) The number of pages allocated for all purgeable caches is +** already equal to or greater than the sum of nMax for all +** purgeable caches, +** +** then attempt to recycle a page from the LRU list. If it is the right +** size, return the recycled buffer. Otherwise, free the buffer and +** proceed to step 5. +** +** 5. Otherwise, allocate and return a new page buffer. +*/ +static void *pcache1Fetch(sqlite3_pcache *p, unsigned int iKey, int createFlag){ + unsigned int nPinned; + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = 0; + + pcache1EnterMutex(); + if( createFlag==1 ) sqlite3BeginBenignMalloc(); + + /* Search the hash table for an existing entry. */ + if( pCache->nHash>0 ){ + unsigned int h = iKey % pCache->nHash; + for(pPage=pCache->apHash[h]; pPage&&pPage->iKey!=iKey; pPage=pPage->pNext); + } + + if( pPage || createFlag==0 ){ + pcache1PinPage(pPage); + goto fetch_out; + } + + /* Step 3 of header comment. */ + nPinned = pCache->nPage - pCache->nRecyclable; + if( createFlag==1 && pCache->bPurgeable && ( + nPinned>=(pcache1.nMaxPage+pCache->nMin-pcache1.nMinPage) + || nPinned>=(pCache->nMax) + )){ + goto fetch_out; + } + + if( pCache->nPage>=pCache->nHash && pcache1ResizeHash(pCache) ){ + goto fetch_out; + } + + /* Step 4. Try to recycle a page buffer if appropriate. */ + if( pCache->bPurgeable && pcache1.pLruTail && ( + pCache->nPage>=pCache->nMax-1 || pcache1.nCurrentPage>=pcache1.nMaxPage + )){ + pPage = pcache1.pLruTail; + pcache1RemoveFromHash(pPage); + pcache1PinPage(pPage); + if( pPage->pCache->szPage!=pCache->szPage ){ + pcache1FreePage(pPage); + pPage = 0; + }else{ + pcache1.nCurrentPage -= (pPage->pCache->bPurgeable - pCache->bPurgeable); + } + } + + /* Step 5. If a usable page buffer has still not been found, + ** attempt to allocate a new one. + */ + if( !pPage ){ + pPage = pcache1AllocPage(pCache); + } + + if( pPage ){ + unsigned int h = iKey % pCache->nHash; + memset(pPage, 0, pCache->szPage + sizeof(PgHdr1)); + pCache->nPage++; + pPage->iKey = iKey; + pPage->pNext = pCache->apHash[h]; + pPage->pCache = pCache; + pCache->apHash[h] = pPage; + } + +fetch_out: + if( pPage && iKey>pCache->iMaxKey ){ + pCache->iMaxKey = iKey; + } + if( createFlag==1 ) sqlite3EndBenignMalloc(); + pcache1LeaveMutex(); + return (pPage ? PGHDR1_TO_PAGE(pPage) : 0); +} + + +/* +** Implementation of the sqlite3_pcache.xUnpin method. +** +** Mark a page as unpinned (eligible for asynchronous recycling). +*/ +static void pcache1Unpin(sqlite3_pcache *p, void *pPg, int reuseUnlikely){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = PAGE_TO_PGHDR1(pPg); + + pcache1EnterMutex(); + + /* It is an error to call this function if the page is already + ** part of the global LRU list. + */ + assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); + assert( pcache1.pLruHead!=pPage && pcache1.pLruTail!=pPage ); + + if( reuseUnlikely || pcache1.nCurrentPage>pcache1.nMaxPage ){ + pcache1RemoveFromHash(pPage); + pcache1FreePage(pPage); + }else{ + /* Add the page to the global LRU list. Normally, the page is added to + ** the head of the list (last page to be recycled). However, if the + ** reuseUnlikely flag passed to this function is true, the page is added + ** to the tail of the list (first page to be recycled). + */ + if( pcache1.pLruHead ){ + pcache1.pLruHead->pLruPrev = pPage; + pPage->pLruNext = pcache1.pLruHead; + pcache1.pLruHead = pPage; + }else{ + pcache1.pLruTail = pPage; + pcache1.pLruHead = pPage; + } + pCache->nRecyclable++; + } + + pcache1LeaveMutex(); +} + +/* +** Implementation of the sqlite3_pcache.xRekey method. +*/ +static void pcache1Rekey( + sqlite3_pcache *p, + void *pPg, + unsigned int iOld, + unsigned int iNew +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = PAGE_TO_PGHDR1(pPg); + PgHdr1 **pp; + unsigned int h; + assert( pPage->iKey==iOld ); + + pcache1EnterMutex(); + + h = iOld%pCache->nHash; + pp = &pCache->apHash[h]; + while( (*pp)!=pPage ){ + pp = &(*pp)->pNext; + } + *pp = pPage->pNext; + + h = iNew%pCache->nHash; + pPage->iKey = iNew; + pPage->pNext = pCache->apHash[h]; + pCache->apHash[h] = pPage; + + if( iNew>pCache->iMaxKey ){ + pCache->iMaxKey = iNew; + } + + pcache1LeaveMutex(); +} + +/* +** Implementation of the sqlite3_pcache.xTruncate method. +** +** Discard all unpinned pages in the cache with a page number equal to +** or greater than parameter iLimit. Any pinned pages with a page number +** equal to or greater than iLimit are implicitly unpinned. +*/ +static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ + PCache1 *pCache = (PCache1 *)p; + pcache1EnterMutex(); + if( iLimit<=pCache->iMaxKey ){ + pcache1TruncateUnsafe(pCache, iLimit); + pCache->iMaxKey = iLimit-1; + } + pcache1LeaveMutex(); +} + +/* +** Implementation of the sqlite3_pcache.xDestroy method. +** +** Destroy a cache allocated using pcache1Create(). +*/ +static void pcache1Destroy(sqlite3_pcache *p){ + PCache1 *pCache = (PCache1 *)p; + pcache1EnterMutex(); + pcache1TruncateUnsafe(pCache, 0); + pcache1.nMaxPage -= pCache->nMax; + pcache1.nMinPage -= pCache->nMin; + pcache1EnforceMaxPage(); + pcache1LeaveMutex(); + sqlite3_free(pCache->apHash); + sqlite3_free(pCache); +} + +/* +** This function is called during initialization (sqlite3_initialize()) to +** install the default pluggable cache module, assuming the user has not +** already provided an alternative. +*/ +void sqlite3PCacheSetDefault(void){ + static sqlite3_pcache_methods defaultMethods = { + 0, /* pArg */ + pcache1Init, /* xInit */ + pcache1Shutdown, /* xShutdown */ + pcache1Create, /* xCreate */ + pcache1Cachesize, /* xCachesize */ + pcache1Pagecount, /* xPagecount */ + pcache1Fetch, /* xFetch */ + pcache1Unpin, /* xUnpin */ + pcache1Rekey, /* xRekey */ + pcache1Truncate, /* xTruncate */ + pcache1Destroy /* xDestroy */ + }; + sqlite3_config(SQLITE_CONFIG_PCACHE, &defaultMethods); +} + +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +/* +** This function is called to free superfluous dynamically allocated memory +** held by the pager system. Memory in use by any SQLite pager allocated +** by the current thread may be sqlite3_free()ed. +** +** nReq is the number of bytes of memory required. Once this much has +** been released, the function returns. The return value is the total number +** of bytes of memory released. +*/ +int sqlite3PcacheReleaseMemory(int nReq){ + int nFree = 0; + if( pcache1.pStart==0 ){ + PgHdr1 *p; + pcache1EnterMutex(); + while( (nReq<0 || nFreepLruNext){ + nRecyclable++; + } + *pnCurrent = pcache1.nCurrentPage; + *pnMax = pcache1.nMaxPage; + *pnMin = pcache1.nMinPage; + *pnRecyclable = nRecyclable; +} +#endif diff --git a/ppport.h b/ppport.h index f48a36d..aecc1af 100644 --- a/ppport.h +++ b/ppport.h @@ -1,579 +1,4894 @@ - -/* ppport.h -- Perl/Pollution/Portability Version 2.003 - * - * Automatically Created by Devel::PPPort on Wed Jul 14 18:43:16 2004 - * - * Do NOT edit this file directly! -- Edit PPPort.pm instead. - * - * Version 2.x, Copyright (C) 2001, Paul Marquess. - * Version 1.x, Copyright (C) 1999, Kenneth Albanowski. - * This code may be used and distributed under the same license as any - * version of Perl. - * - * This version of ppport.h is designed to support operation with Perl - * installations back to 5.004, and has been tested up to 5.8.0. - * - * If this version of ppport.h is failing during the compilation of this - * module, please check if a newer version of Devel::PPPort is available - * on CPAN before sending a bug report. - * - * If you are using the latest version of Devel::PPPort and it is failing - * during compilation of this module, please send a report to perlbug@perl.com - * - * Include all following information: - * - * 1. The complete output from running "perl -V" - * - * 2. This file. - * - * 3. The name & version of the module you were trying to build. - * - * 4. A full log of the build that failed. - * - * 5. Any other information that you think could be relevant. - * - * - * For the latest version of this code, please retreive the Devel::PPPort - * module from CPAN. - * - */ - -/* - * In order for a Perl extension module to be as portable as possible - * across differing versions of Perl itself, certain steps need to be taken. - * Including this header is the first major one, then using dTHR is all the - * appropriate places and using a PL_ prefix to refer to global Perl - * variables is the second. - * - */ - - -/* If you use one of a few functions that were not present in earlier - * versions of Perl, please add a define before the inclusion of ppport.h - * for a static include, or use the GLOBAL request in a single module to - * produce a global definition that can be referenced from the other - * modules. - * - * Function: Static define: Extern define: - * newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL - * - */ - - -/* To verify whether ppport.h is needed for your module, and whether any - * special defines should be used, ppport.h can be run through Perl to check - * your source code. Simply say: - * - * perl -x ppport.h *.c *.h *.xs foo/bar*.c [etc] - * - * The result will be a list of patches suggesting changes that should at - * least be acceptable, if not necessarily the most efficient solution, or a - * fix for all possible problems. It won't catch where dTHR is needed, and - * doesn't attempt to account for global macro or function definitions, - * nested includes, typemaps, etc. - * - * In order to test for the need of dTHR, please try your module under a - * recent version of Perl that has threading compiled-in. - * - */ - - -/* -#!/usr/bin/perl -@ARGV = ("*.xs") if !@ARGV; -%badmacros = %funcs = %macros = (); $replace = 0; -foreach () { - $funcs{$1} = 1 if /Provide:\s+(\S+)/; - $macros{$1} = 1 if /^#\s*define\s+([a-zA-Z0-9_]+)/; - $replace = $1 if /Replace:\s+(\d+)/; - $badmacros{$2}=$1 if $replace and /^#\s*define\s+([a-zA-Z0-9_]+).*?\s+([a-zA-Z0-9_]+)/; - $badmacros{$1}=$2 if /Replace (\S+) with (\S+)/; -} -foreach $filename (map(glob($_),@ARGV)) { - unless (open(IN, "<$filename")) { - warn "Unable to read from $file: $!\n"; - next; - } - print "Scanning $filename...\n"; - $c = ""; while () { $c .= $_; } close(IN); - $need_include = 0; %add_func = (); $changes = 0; - $has_include = ($c =~ /#.*include.*ppport/m); - - foreach $func (keys %funcs) { - if ($c =~ /#.*define.*\bNEED_$func(_GLOBAL)?\b/m) { - if ($c !~ /\b$func\b/m) { - print "If $func isn't needed, you don't need to request it.\n" if - $changes += ($c =~ s/^.*#.*define.*\bNEED_$func\b.*\n//m); - } else { - print "Uses $func\n"; - $need_include = 1; - } - } else { - if ($c =~ /\b$func\b/m) { - $add_func{$func} =1 ; - print "Uses $func\n"; - $need_include = 1; - } - } - } - - if (not $need_include) { - foreach $macro (keys %macros) { - if ($c =~ /\b$macro\b/m) { - print "Uses $macro\n"; - $need_include = 1; - } - } - } - - foreach $badmacro (keys %badmacros) { - if ($c =~ /\b$badmacro\b/m) { - $changes += ($c =~ s/\b$badmacro\b/$badmacros{$badmacro}/gm); - print "Uses $badmacros{$badmacro} (instead of $badmacro)\n"; - $need_include = 1; - } - } - - if (scalar(keys %add_func) or $need_include != $has_include) { - if (!$has_include) { - $inc = join('',map("#define NEED_$_\n", sort keys %add_func)). - "#include \"ppport.h\"\n"; - $c = "$inc$c" unless $c =~ s/#.*include.*XSUB.*\n/$&$inc/m; - } elsif (keys %add_func) { - $inc = join('',map("#define NEED_$_\n", sort keys %add_func)); - $c = "$inc$c" unless $c =~ s/^.*#.*include.*ppport.*$/$inc$&/m; - } - if (!$need_include) { - print "Doesn't seem to need ppport.h.\n"; - $c =~ s/^.*#.*include.*ppport.*\n//m; - } - $changes++; - } - - if ($changes) { - open(OUT,">/tmp/ppport.h.$$"); - print OUT $c; - close(OUT); - open(DIFF, "diff -u $filename /tmp/ppport.h.$$|"); - while () { s!/tmp/ppport\.h\.$$!$filename.patched!; print STDOUT; } - close(DIFF); - unlink("/tmp/ppport.h.$$"); - } else { - print "Looks OK\n"; - } -} -__DATA__ -*/ - -#ifndef _P_P_PORTABILITY_H_ -#define _P_P_PORTABILITY_H_ - -#ifndef PERL_REVISION -# ifndef __PATCHLEVEL_H_INCLUDED__ -# include -# endif -# if !(defined(PERL_VERSION) || (SUBVERSION > 0 && defined(PATCHLEVEL))) -# include -# endif -# ifndef PERL_REVISION -# define PERL_REVISION (5) - /* Replace: 1 */ -# define PERL_VERSION PATCHLEVEL -# define PERL_SUBVERSION SUBVERSION - /* Replace PERL_PATCHLEVEL with PERL_VERSION */ - /* Replace: 0 */ -# endif -#endif - -#define PERL_BCDVERSION ((PERL_REVISION * 0x1000000L) + (PERL_VERSION * 0x1000L) + PERL_SUBVERSION) - -/* It is very unlikely that anyone will try to use this with Perl 6 - (or greater), but who knows. - */ -#if PERL_REVISION != 5 -# error ppport.h only works with Perl version 5 -#endif /* PERL_REVISION != 5 */ - -#ifndef ERRSV -# define ERRSV perl_get_sv("@",FALSE) -#endif - -#if (PERL_VERSION < 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION <= 5)) -/* Replace: 1 */ -# define PL_Sv Sv -# define PL_compiling compiling -# define PL_copline copline -# define PL_curcop curcop -# define PL_curstash curstash -# define PL_defgv defgv -# define PL_dirty dirty -# define PL_dowarn dowarn -# define PL_hints hints -# define PL_na na -# define PL_perldb perldb -# define PL_rsfp_filters rsfp_filters -# define PL_rsfpv rsfp -# define PL_stdingv stdingv -# define PL_sv_no sv_no -# define PL_sv_undef sv_undef -# define PL_sv_yes sv_yes -/* Replace: 0 */ -#endif - -#ifdef HASATTRIBUTE -# if defined(__GNUC__) && defined(__cplusplus) -# define PERL_UNUSED_DECL -# else -# define PERL_UNUSED_DECL __attribute__((unused)) -# endif -#else -# define PERL_UNUSED_DECL -#endif - -#ifndef dNOOP -# define NOOP (void)0 -# define dNOOP extern int Perl___notused PERL_UNUSED_DECL -#endif - -#ifndef dTHR -# define dTHR dNOOP -#endif - -#ifndef dTHX -# define dTHX dNOOP -# define dTHXa(x) dNOOP -# define dTHXoa(x) dNOOP -#endif - -#ifndef pTHX -# define pTHX void -# define pTHX_ -# define aTHX -# define aTHX_ -#endif - -/* IV could also be a quad (say, a long long), but Perls - * capable of those should have IVSIZE already. */ -#if !defined(IVSIZE) && defined(LONGSIZE) -# define IVSIZE LONGSIZE -#endif -#ifndef IVSIZE -# define IVSIZE 4 /* A bold guess, but the best we can make. */ -#endif - -#ifndef UVSIZE -# define UVSIZE IVSIZE -#endif - -#ifndef NVTYPE -# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) -# define NVTYPE long double -# else -# define NVTYPE double -# endif -typedef NVTYPE NV; -#endif - -#ifndef INT2PTR - -#if (IVSIZE == PTRSIZE) && (UVSIZE == PTRSIZE) -# define PTRV UV -# define INT2PTR(any,d) (any)(d) -#else -# if PTRSIZE == LONGSIZE -# define PTRV unsigned long -# else -# define PTRV unsigned -# endif -# define INT2PTR(any,d) (any)(PTRV)(d) -#endif -#define NUM2PTR(any,d) (any)(PTRV)(d) -#define PTR2IV(p) INT2PTR(IV,p) -#define PTR2UV(p) INT2PTR(UV,p) -#define PTR2NV(p) NUM2PTR(NV,p) -#if PTRSIZE == LONGSIZE -# define PTR2ul(p) (unsigned long)(p) -#else -# define PTR2ul(p) INT2PTR(unsigned long,p) -#endif - -#endif /* !INT2PTR */ - -#ifndef boolSV -# define boolSV(b) ((b) ? &PL_sv_yes : &PL_sv_no) -#endif - -#ifndef gv_stashpvn -# define gv_stashpvn(str,len,flags) gv_stashpv(str,flags) -#endif - -#ifndef newSVpvn -# define newSVpvn(data,len) ((len) ? newSVpv ((data), (len)) : newSVpv ("", 0)) -#endif - -#ifndef newRV_inc -/* Replace: 1 */ -# define newRV_inc(sv) newRV(sv) -/* Replace: 0 */ -#endif - -/* DEFSV appears first in 5.004_56 */ -#ifndef DEFSV -# define DEFSV GvSV(PL_defgv) -#endif - -#ifndef SAVE_DEFSV -# define SAVE_DEFSV SAVESPTR(GvSV(PL_defgv)) -#endif - -#ifndef newRV_noinc -# ifdef __GNUC__ -# define newRV_noinc(sv) \ - ({ \ - SV *nsv = (SV*)newRV(sv); \ - SvREFCNT_dec(sv); \ - nsv; \ - }) -# else -# if defined(USE_THREADS) -static SV * newRV_noinc (SV * sv) -{ - SV *nsv = (SV*)newRV(sv); - SvREFCNT_dec(sv); - return nsv; -} -# else -# define newRV_noinc(sv) \ - (PL_Sv=(SV*)newRV(sv), SvREFCNT_dec(sv), (SV*)PL_Sv) -# endif -# endif -#endif - -/* Provide: newCONSTSUB */ - -/* newCONSTSUB from IO.xs is in the core starting with 5.004_63 */ -#if (PERL_VERSION < 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION < 63)) - -#if defined(NEED_newCONSTSUB) -static -#else -extern void newCONSTSUB(HV * stash, char * name, SV *sv); -#endif - -#if defined(NEED_newCONSTSUB) || defined(NEED_newCONSTSUB_GLOBAL) -void -newCONSTSUB(stash,name,sv) -HV *stash; -char *name; -SV *sv; -{ - U32 oldhints = PL_hints; - HV *old_cop_stash = PL_curcop->cop_stash; - HV *old_curstash = PL_curstash; - line_t oldline = PL_curcop->cop_line; - PL_curcop->cop_line = PL_copline; - - PL_hints &= ~HINT_BLOCK_SCOPE; - if (stash) - PL_curstash = PL_curcop->cop_stash = stash; - - newSUB( - -#if (PERL_VERSION < 3) || ((PERL_VERSION == 3) && (PERL_SUBVERSION < 22)) - /* before 5.003_22 */ - start_subparse(), -#else -# if (PERL_VERSION == 3) && (PERL_SUBVERSION == 22) - /* 5.003_22 */ - start_subparse(0), -# else - /* 5.003_23 onwards */ - start_subparse(FALSE, 0), -# endif -#endif - - newSVOP(OP_CONST, 0, newSVpv(name,0)), - newSVOP(OP_CONST, 0, &PL_sv_no), /* SvPV(&PL_sv_no) == "" -- GMB */ - newSTATEOP(0, Nullch, newSVOP(OP_CONST, 0, sv)) - ); - - PL_hints = oldhints; - PL_curcop->cop_stash = old_cop_stash; - PL_curstash = old_curstash; - PL_curcop->cop_line = oldline; -} -#endif - -#endif /* newCONSTSUB */ - -#ifndef START_MY_CXT - -/* - * Boilerplate macros for initializing and accessing interpreter-local - * data from C. All statics in extensions should be reworked to use - * this, if you want to make the extension thread-safe. See ext/re/re.xs - * for an example of the use of these macros. - * - * Code that uses these macros is responsible for the following: - * 1. #define MY_CXT_KEY to a unique string, e.g. "DynaLoader_guts" - * 2. Declare a typedef named my_cxt_t that is a structure that contains - * all the data that needs to be interpreter-local. - * 3. Use the START_MY_CXT macro after the declaration of my_cxt_t. - * 4. Use the MY_CXT_INIT macro such that it is called exactly once - * (typically put in the BOOT: section). - * 5. Use the members of the my_cxt_t structure everywhere as - * MY_CXT.member. - * 6. Use the dMY_CXT macro (a declaration) in all the functions that - * access MY_CXT. - */ - -#if defined(MULTIPLICITY) || defined(PERL_OBJECT) || \ - defined(PERL_CAPI) || defined(PERL_IMPLICIT_CONTEXT) - -/* This must appear in all extensions that define a my_cxt_t structure, - * right after the definition (i.e. at file scope). The non-threads - * case below uses it to declare the data as static. */ -#define START_MY_CXT - -#if (PERL_VERSION < 4 || (PERL_VERSION == 4 && PERL_SUBVERSION < 68 )) -/* Fetches the SV that keeps the per-interpreter data. */ -#define dMY_CXT_SV \ - SV *my_cxt_sv = perl_get_sv(MY_CXT_KEY, FALSE) -#else /* >= perl5.004_68 */ -#define dMY_CXT_SV \ - SV *my_cxt_sv = *hv_fetch(PL_modglobal, MY_CXT_KEY, \ - sizeof(MY_CXT_KEY)-1, TRUE) -#endif /* < perl5.004_68 */ - -/* This declaration should be used within all functions that use the - * interpreter-local data. */ -#define dMY_CXT \ - dMY_CXT_SV; \ - my_cxt_t *my_cxtp = INT2PTR(my_cxt_t*,SvUV(my_cxt_sv)) - -/* Creates and zeroes the per-interpreter data. - * (We allocate my_cxtp in a Perl SV so that it will be released when - * the interpreter goes away.) */ -#define MY_CXT_INIT \ - dMY_CXT_SV; \ - /* newSV() allocates one more than needed */ \ - my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ - Zero(my_cxtp, 1, my_cxt_t); \ - sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) - -/* This macro must be used to access members of the my_cxt_t structure. - * e.g. MYCXT.some_data */ -#define MY_CXT (*my_cxtp) - -/* Judicious use of these macros can reduce the number of times dMY_CXT - * is used. Use is similar to pTHX, aTHX etc. */ -#define pMY_CXT my_cxt_t *my_cxtp -#define pMY_CXT_ pMY_CXT, -#define _pMY_CXT ,pMY_CXT -#define aMY_CXT my_cxtp -#define aMY_CXT_ aMY_CXT, -#define _aMY_CXT ,aMY_CXT - -#else /* single interpreter */ - -#define START_MY_CXT static my_cxt_t my_cxt; -#define dMY_CXT_SV dNOOP -#define dMY_CXT dNOOP -#define MY_CXT_INIT NOOP -#define MY_CXT my_cxt - -#define pMY_CXT void -#define pMY_CXT_ -#define _pMY_CXT -#define aMY_CXT -#define aMY_CXT_ -#define _aMY_CXT - -#endif - -#endif /* START_MY_CXT */ - -#ifndef IVdf -# if IVSIZE == LONGSIZE -# define IVdf "ld" -# define UVuf "lu" -# define UVof "lo" -# define UVxf "lx" -# define UVXf "lX" -# else -# if IVSIZE == INTSIZE -# define IVdf "d" -# define UVuf "u" -# define UVof "o" -# define UVxf "x" -# define UVXf "X" -# endif -# endif -#endif - -#ifndef NVef -# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) && \ - defined(PERL_PRIfldbl) /* Not very likely, but let's try anyway. */ -# define NVef PERL_PRIeldbl -# define NVff PERL_PRIfldbl -# define NVgf PERL_PRIgldbl -# else -# define NVef "e" -# define NVff "f" -# define NVgf "g" -# endif -#endif - -#ifndef AvFILLp /* Older perls (<=5.003) lack AvFILLp */ -# define AvFILLp AvFILL -#endif - -#ifdef SvPVbyte -# if PERL_REVISION == 5 && PERL_VERSION < 7 - /* SvPVbyte does not work in perl-5.6.1, borrowed version for 5.7.3 */ -# undef SvPVbyte -# define SvPVbyte(sv, lp) \ - ((SvFLAGS(sv) & (SVf_POK|SVf_UTF8)) == (SVf_POK) \ - ? ((lp = SvCUR(sv)), SvPVX(sv)) : my_sv_2pvbyte(aTHX_ sv, &lp)) - static char * - my_sv_2pvbyte(pTHX_ register SV *sv, STRLEN *lp) - { - sv_utf8_downgrade(sv,0); - return SvPV(sv,*lp); - } -# endif -#else -# define SvPVbyte SvPV -#endif - -#ifndef SvPV_nolen -# define SvPV_nolen(sv) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? SvPVX(sv) : sv_2pv_nolen(sv)) - static char * - sv_2pv_nolen(pTHX_ register SV *sv) - { - STRLEN n_a; - return sv_2pv(sv, &n_a); - } -#endif - -#ifndef get_cv -# define get_cv(name,create) perl_get_cv(name,create) -#endif - -#ifndef get_sv -# define get_sv(name,create) perl_get_sv(name,create) -#endif - -#ifndef get_av -# define get_av(name,create) perl_get_av(name,create) -#endif - -#ifndef get_hv -# define get_hv(name,create) perl_get_hv(name,create) -#endif - -#endif /* _P_P_PORTABILITY_H_ */ - -/* End of File ppport.h */ +#if 0 +<<'SKIP'; +#endif +/* +---------------------------------------------------------------------- + + ppport.h -- Perl/Pollution/Portability Version 3.06 + + Automatically created by Devel::PPPort running under + perl 5.008007 on Thu Jan 29 10:01:00 2009. + + Do NOT edit this file directly! -- Edit PPPort_pm.PL and the + includes in parts/inc/ instead. + + Use 'perldoc ppport.h' to view the documentation below. + +---------------------------------------------------------------------- + +SKIP + +=pod + +=head1 NAME + +ppport.h - Perl/Pollution/Portability version 3.06 + +=head1 SYNOPSIS + + perl ppport.h [options] [files] + + --help show short help + + --patch=file write one patch file with changes + --copy=suffix write changed copies with suffix + --diff=program use diff program and options + + --compat-version=version provide compatibility with Perl version + --cplusplus accept C++ comments + + --quiet don't output anything except fatal errors + --nodiag don't show diagnostics + --nohints don't show hints + --nochanges don't suggest changes + + --list-provided list provided API + --list-unsupported list unsupported API + --api-info=name show Perl API portability information + +=head1 COMPATIBILITY + +This version of F is designed to support operation with Perl +installations back to 5.003, and has been tested up to 5.9.2. + +=head1 OPTIONS + +=head2 --help + +Display a brief usage summary. + +=head2 --patch=I + +If this option is given, a single patch file will be created if +any changes are suggested. This requires a working diff program +to be installed on your system. + +=head2 --copy=I + +If this option is given, a copy of each file will be saved with +the given suffix that contains the suggested changes. This does +not require any external programs. + +If neither C<--patch> or C<--copy> are given, the default is to +simply print the diffs for each file. This requires either +C or a C program to be installed. + +=head2 --diff=I + +Manually set the diff program and options to use. The default +is to use C, when installed, and output unified +context diffs. + +=head2 --compat-version=I + +Tell F to check for compatibility with the given +Perl version. The default is to check for compatibility with Perl +version 5.003. You can use this option to reduce the output +of F if you intend to be backward compatible only +up to a certain Perl version. + +=head2 --cplusplus + +Usually, F will detect C++ style comments and +replace them with C style comments for portability reasons. +Using this option instructs F to leave C++ +comments untouched. + +=head2 --quiet + +Be quiet. Don't print anything except fatal errors. + +=head2 --nodiag + +Don't output any diagnostic messages. Only portability +alerts will be printed. + +=head2 --nohints + +Don't output any hints. Hints often contain useful portability +notes. + +=head2 --nochanges + +Don't suggest any changes. Only give diagnostic output and hints +unless these are also deactivated. + +=head2 --list-provided + +Lists the API elements for which compatibility is provided by +F. Also lists if it must be explicitly requested, +if it has dependencies, and if there are hints for it. + +=head2 --list-unsupported + +Lists the API elements that are known not to be supported by +F and below which version of Perl they probably +won't be available or work. + +=head2 --api-info=I + +Show portability information for API elements matching I. +If I is surrounded by slashes, it is interpreted as a regular +expression. + +=head1 DESCRIPTION + +In order for a Perl extension (XS) module to be as portable as possible +across differing versions of Perl itself, certain steps need to be taken. + +=over 4 + +=item * + +Including this header is the first major one. This alone will give you +access to a large part of the Perl API that hasn't been available in +earlier Perl releases. Use + + perl ppport.h --list-provided + +to see which API elements are provided by ppport.h. + +=item * + +You should avoid using deprecated parts of the API. For example, using +global Perl variables without the C prefix is deprecated. Also, +some API functions used to have a C prefix. Using this form is +also deprecated. You can safely use the supported API, as F +will provide wrappers for older Perl versions. + +=item * + +If you use one of a few functions that were not present in earlier +versions of Perl, and that can't be provided using a macro, you have +to explicitly request support for these functions by adding one or +more C<#define>s in your source code before the inclusion of F. + +These functions will be marked C in the list shown by +C<--list-provided>. + +Depending on whether you module has a single or multiple files that +use such functions, you want either C or global variants. + +For a C function, use: + + #define NEED_function + +For a global function, use: + + #define NEED_function_GLOBAL + +Note that you mustn't have more than one global request for one +function in your project. + + Function Static Request Global Request + ----------------------------------------------------------------------------------------- + eval_pv() NEED_eval_pv NEED_eval_pv_GLOBAL + grok_bin() NEED_grok_bin NEED_grok_bin_GLOBAL + grok_hex() NEED_grok_hex NEED_grok_hex_GLOBAL + grok_number() NEED_grok_number NEED_grok_number_GLOBAL + grok_numeric_radix() NEED_grok_numeric_radix NEED_grok_numeric_radix_GLOBAL + grok_oct() NEED_grok_oct NEED_grok_oct_GLOBAL + newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL + newRV_noinc() NEED_newRV_noinc NEED_newRV_noinc_GLOBAL + sv_2pv_nolen() NEED_sv_2pv_nolen NEED_sv_2pv_nolen_GLOBAL + sv_2pvbyte() NEED_sv_2pvbyte NEED_sv_2pvbyte_GLOBAL + sv_catpvf_mg() NEED_sv_catpvf_mg NEED_sv_catpvf_mg_GLOBAL + sv_catpvf_mg_nocontext() NEED_sv_catpvf_mg_nocontext NEED_sv_catpvf_mg_nocontext_GLOBAL + sv_setpvf_mg() NEED_sv_setpvf_mg NEED_sv_setpvf_mg_GLOBAL + sv_setpvf_mg_nocontext() NEED_sv_setpvf_mg_nocontext NEED_sv_setpvf_mg_nocontext_GLOBAL + vnewSVpvf() NEED_vnewSVpvf NEED_vnewSVpvf_GLOBAL + +To avoid namespace conflicts, you can change the namespace of the +explicitly exported functions using the C macro. +Just C<#define> the macro before including C: + + #define DPPP_NAMESPACE MyOwnNamespace_ + #include "ppport.h" + +The default namespace is C. + +=back + +The good thing is that most of the above can be checked by running +F on your source code. See the next section for +details. + +=head1 EXAMPLES + +To verify whether F is needed for your module, whether you +should make any changes to your code, and whether any special defines +should be used, F can be run as a Perl script to check your +source code. Simply say: + + perl ppport.h + +The result will usually be a list of patches suggesting changes +that should at least be acceptable, if not necessarily the most +efficient solution, or a fix for all possible problems. + +If you know that your XS module uses features only available in +newer Perl releases, if you're aware that it uses C++ comments, +and if you want all suggestions as a single patch file, you could +use something like this: + + perl ppport.h --compat-version=5.6.0 --cplusplus --patch=test.diff + +If you only want your code to be scanned without any suggestions +for changes, use: + + perl ppport.h --nochanges + +You can specify a different C program or options, using +the C<--diff> option: + + perl ppport.h --diff='diff -C 10' + +This would output context diffs with 10 lines of context. + +To display portability information for the C function, +use: + + perl ppport.h --api-info=newSVpvn + +Since the argument to C<--api-info> can be a regular expression, +you can use + + perl ppport.h --api-info=/_nomg$/ + +to display portability information for all C<_nomg> functions or + + perl ppport.h --api-info=/./ + +to display information for all known API elements. + +=head1 BUGS + +If this version of F is causing failure during +the compilation of this module, please check if newer versions +of either this module or C are available on CPAN +before sending a bug report. + +If F was generated using the latest version of +C and is causing failure of this module, please +file a bug report using the CPAN Request Tracker at L. + +Please include the following information: + +=over 4 + +=item 1. + +The complete output from running "perl -V" + +=item 2. + +This file. + +=item 3. + +The name and version of the module you were trying to build. + +=item 4. + +A full log of the build that failed. + +=item 5. + +Any other information that you think could be relevant. + +=back + +For the latest version of this code, please get the C +module from CPAN. + +=head1 COPYRIGHT + +Version 3.x, Copyright (c) 2004-2005, Marcus Holland-Moritz. + +Version 2.x, Copyright (C) 2001, Paul Marquess. + +Version 1.x, Copyright (C) 1999, Kenneth Albanowski. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +=head1 SEE ALSO + +See L. + +=cut + +use strict; + +my %opt = ( + quiet => 0, + diag => 1, + hints => 1, + changes => 1, + cplusplus => 0, +); + +my($ppport) = $0 =~ /([\w.]+)$/; +my $LF = '(?:\r\n|[\r\n])'; # line feed +my $HS = "[ \t]"; # horizontal whitespace + +eval { + require Getopt::Long; + Getopt::Long::GetOptions(\%opt, qw( + help quiet diag! hints! changes! cplusplus + patch=s copy=s diff=s compat-version=s + list-provided list-unsupported api-info=s + )) or usage(); +}; + +if ($@ and grep /^-/, @ARGV) { + usage() if "@ARGV" =~ /^--?h(?:elp)?$/; + die "Getopt::Long not found. Please don't use any options.\n"; +} + +usage() if $opt{help}; + +if (exists $opt{'compat-version'}) { + my($r,$v,$s) = eval { parse_version($opt{'compat-version'}) }; + if ($@) { + die "Invalid version number format: '$opt{'compat-version'}'\n"; + } + die "Only Perl 5 is supported\n" if $r != 5; + die "Invalid version number: $opt{'compat-version'}\n" if $v >= 1000 || $v >= 1000; + $opt{'compat-version'} = sprintf "%d.%03d%03d", $r, $v, $s; +} +else { + $opt{'compat-version'} = 5; +} + +# Never use C comments in this file!!!!! +my $ccs = '/'.'*'; +my $cce = '*'.'/'; +my $rccs = quotemeta $ccs; +my $rcce = quotemeta $cce; + +my @files; + +if (@ARGV) { + @files = map { glob $_ } @ARGV; +} +else { + eval { + require File::Find; + File::Find::find(sub { + $File::Find::name =~ /\.(xs|c|h|cc)$/i + and push @files, $File::Find::name; + }, '.'); + }; + if ($@) { + @files = map { glob $_ } qw(*.xs *.c *.h *.cc); + } + my %filter = map { /(.*)\.xs$/ ? ("$1.c" => 1) : () } @files; + @files = grep { !/\b\Q$ppport\E$/i && !exists $filter{$_} } @files; +} + +unless (@files) { + die "No input files given!\n"; +} + +my %API = map { /^(\w+)\|([^|]*)\|([^|]*)\|(\w*)$/ + ? ( $1 => { + ($2 ? ( base => $2 ) : ()), + ($3 ? ( todo => $3 ) : ()), + (index($4, 'v') >= 0 ? ( varargs => 1 ) : ()), + (index($4, 'p') >= 0 ? ( provided => 1 ) : ()), + (index($4, 'n') >= 0 ? ( nothxarg => 1 ) : ()), + } ) + : die "invalid spec: $_" } qw( +AvFILLp|5.004050||p +AvFILL||| +CLASS|||n +CX_CURPAD_SAVE||| +CX_CURPAD_SV||| +CopFILEAV|5.006000||p +CopFILEGV_set|5.006000||p +CopFILEGV|5.006000||p +CopFILESV|5.006000||p +CopFILE_set|5.006000||p +CopFILE|5.006000||p +CopSTASHPV_set|5.006000||p +CopSTASHPV|5.006000||p +CopSTASH_eq|5.006000||p +CopSTASH_set|5.006000||p +CopSTASH|5.006000||p +CopyD|5.009002||p +Copy||| +CvPADLIST||| +CvSTASH||| +CvWEAKOUTSIDE||| +DEFSV|5.004050||p +END_EXTERN_C|5.005000||p +ENTER||| +ERRSV|5.004050||p +EXTEND||| +EXTERN_C|5.005000||p +FREETMPS||| +GIMME_V||5.004000|n +GIMME|||n +GROK_NUMERIC_RADIX|5.007002||p +G_ARRAY||| +G_DISCARD||| +G_EVAL||| +G_NOARGS||| +G_SCALAR||| +G_VOID||5.004000| +GetVars||| +GvSV||| +Gv_AMupdate||| +HEf_SVKEY||5.004000| +HeHASH||5.004000| +HeKEY||5.004000| +HeKLEN||5.004000| +HePV||5.004000| +HeSVKEY_force||5.004000| +HeSVKEY_set||5.004000| +HeSVKEY||5.004000| +HeVAL||5.004000| +HvNAME||| +INT2PTR|5.006000||p +IN_LOCALE_COMPILETIME|5.007002||p +IN_LOCALE_RUNTIME|5.007002||p +IN_LOCALE|5.007002||p +IN_PERL_COMPILETIME|5.008001||p +IS_NUMBER_GREATER_THAN_UV_MAX|5.007002||p +IS_NUMBER_INFINITY|5.007002||p +IS_NUMBER_IN_UV|5.007002||p +IS_NUMBER_NAN|5.007003||p +IS_NUMBER_NEG|5.007002||p +IS_NUMBER_NOT_INT|5.007002||p +IVSIZE|5.006000||p +IVTYPE|5.006000||p +IVdf|5.006000||p +LEAVE||| +LVRET||| +MARK||| +MY_CXT_CLONE|5.009002||p +MY_CXT_INIT|5.007003||p +MY_CXT|5.007003||p +MoveD|5.009002||p +Move||| +NEWSV||| +NOOP|5.005000||p +NUM2PTR|5.006000||p +NVTYPE|5.006000||p +NVef|5.006001||p +NVff|5.006001||p +NVgf|5.006001||p +Newc||| +Newz||| +New||| +Nullav||| +Nullch||| +Nullcv||| +Nullhv||| +Nullsv||| +ORIGMARK||| +PAD_BASE_SV||| +PAD_CLONE_VARS||| +PAD_COMPNAME_FLAGS||| +PAD_COMPNAME_GEN||| +PAD_COMPNAME_OURSTASH||| +PAD_COMPNAME_PV||| +PAD_COMPNAME_TYPE||| +PAD_RESTORE_LOCAL||| +PAD_SAVE_LOCAL||| +PAD_SAVE_SETNULLPAD||| +PAD_SETSV||| +PAD_SET_CUR_NOSAVE||| +PAD_SET_CUR||| +PAD_SVl||| +PAD_SV||| +PERL_BCDVERSION|5.009002||p +PERL_GCC_BRACE_GROUPS_FORBIDDEN|5.008001||p +PERL_INT_MAX|5.004000||p +PERL_INT_MIN|5.004000||p +PERL_LONG_MAX|5.004000||p +PERL_LONG_MIN|5.004000||p +PERL_MAGIC_arylen|5.007002||p +PERL_MAGIC_backref|5.007002||p +PERL_MAGIC_bm|5.007002||p +PERL_MAGIC_collxfrm|5.007002||p +PERL_MAGIC_dbfile|5.007002||p +PERL_MAGIC_dbline|5.007002||p +PERL_MAGIC_defelem|5.007002||p +PERL_MAGIC_envelem|5.007002||p +PERL_MAGIC_env|5.007002||p +PERL_MAGIC_ext|5.007002||p +PERL_MAGIC_fm|5.007002||p +PERL_MAGIC_glob|5.007002||p +PERL_MAGIC_isaelem|5.007002||p +PERL_MAGIC_isa|5.007002||p +PERL_MAGIC_mutex|5.007002||p +PERL_MAGIC_nkeys|5.007002||p +PERL_MAGIC_overload_elem|5.007002||p +PERL_MAGIC_overload_table|5.007002||p +PERL_MAGIC_overload|5.007002||p +PERL_MAGIC_pos|5.007002||p +PERL_MAGIC_qr|5.007002||p +PERL_MAGIC_regdata|5.007002||p +PERL_MAGIC_regdatum|5.007002||p +PERL_MAGIC_regex_global|5.007002||p +PERL_MAGIC_shared_scalar|5.007003||p +PERL_MAGIC_shared|5.007003||p +PERL_MAGIC_sigelem|5.007002||p +PERL_MAGIC_sig|5.007002||p +PERL_MAGIC_substr|5.007002||p +PERL_MAGIC_sv|5.007002||p +PERL_MAGIC_taint|5.007002||p +PERL_MAGIC_tiedelem|5.007002||p +PERL_MAGIC_tiedscalar|5.007002||p +PERL_MAGIC_tied|5.007002||p +PERL_MAGIC_utf8|5.008001||p +PERL_MAGIC_uvar_elem|5.007003||p +PERL_MAGIC_uvar|5.007002||p +PERL_MAGIC_vec|5.007002||p +PERL_MAGIC_vstring|5.008001||p +PERL_QUAD_MAX|5.004000||p +PERL_QUAD_MIN|5.004000||p +PERL_REVISION|5.006000||p +PERL_SCAN_ALLOW_UNDERSCORES|5.007003||p +PERL_SCAN_DISALLOW_PREFIX|5.007003||p +PERL_SCAN_GREATER_THAN_UV_MAX|5.007003||p +PERL_SCAN_SILENT_ILLDIGIT|5.008001||p +PERL_SHORT_MAX|5.004000||p +PERL_SHORT_MIN|5.004000||p +PERL_SUBVERSION|5.006000||p +PERL_UCHAR_MAX|5.004000||p +PERL_UCHAR_MIN|5.004000||p +PERL_UINT_MAX|5.004000||p +PERL_UINT_MIN|5.004000||p +PERL_ULONG_MAX|5.004000||p +PERL_ULONG_MIN|5.004000||p +PERL_UNUSED_DECL|5.007002||p +PERL_UQUAD_MAX|5.004000||p +PERL_UQUAD_MIN|5.004000||p +PERL_USHORT_MAX|5.004000||p +PERL_USHORT_MIN|5.004000||p +PERL_VERSION|5.006000||p +PL_DBsingle|||pn +PL_DBsub|||pn +PL_DBtrace|||n +PL_Sv|5.005000||p +PL_compiling|5.004050||p +PL_copline|5.005000||p +PL_curcop|5.004050||p +PL_curstash|5.004050||p +PL_debstash|5.004050||p +PL_defgv|5.004050||p +PL_diehook|5.004050||p +PL_dirty|5.004050||p +PL_dowarn|||pn +PL_errgv|5.004050||p +PL_hexdigit|5.005000||p +PL_hints|5.005000||p +PL_last_in_gv|||n +PL_modglobal||5.005000|n +PL_na|5.004050||pn +PL_no_modify|5.006000||p +PL_ofs_sv|||n +PL_perl_destruct_level|5.004050||p +PL_perldb|5.004050||p +PL_ppaddr|5.006000||p +PL_rsfp_filters|5.004050||p +PL_rsfp|5.004050||p +PL_rs|||n +PL_stack_base|5.004050||p +PL_stack_sp|5.004050||p +PL_stdingv|5.004050||p +PL_sv_arenaroot|5.004050||p +PL_sv_no|5.004050||pn +PL_sv_undef|5.004050||pn +PL_sv_yes|5.004050||pn +PL_tainted|5.004050||p +PL_tainting|5.004050||p +POPi|||n +POPl|||n +POPn|||n +POPpbytex||5.007001|n +POPpx||5.005030|n +POPp|||n +POPs|||n +PTR2IV|5.006000||p +PTR2NV|5.006000||p +PTR2UV|5.006000||p +PTR2ul|5.007001||p +PTRV|5.006000||p +PUSHMARK||| +PUSHi||| +PUSHmortal|5.009002||p +PUSHn||| +PUSHp||| +PUSHs||| +PUSHu|5.004000||p +PUTBACK||| +PerlIO_clearerr||5.007003| +PerlIO_close||5.007003| +PerlIO_eof||5.007003| +PerlIO_error||5.007003| +PerlIO_fileno||5.007003| +PerlIO_fill||5.007003| +PerlIO_flush||5.007003| +PerlIO_get_base||5.007003| +PerlIO_get_bufsiz||5.007003| +PerlIO_get_cnt||5.007003| +PerlIO_get_ptr||5.007003| +PerlIO_read||5.007003| +PerlIO_seek||5.007003| +PerlIO_set_cnt||5.007003| +PerlIO_set_ptrcnt||5.007003| +PerlIO_setlinebuf||5.007003| +PerlIO_stderr||5.007003| +PerlIO_stdin||5.007003| +PerlIO_stdout||5.007003| +PerlIO_tell||5.007003| +PerlIO_unread||5.007003| +PerlIO_write||5.007003| +Poison|5.008000||p +RETVAL|||n +Renewc||| +Renew||| +SAVECLEARSV||| +SAVECOMPPAD||| +SAVEPADSV||| +SAVETMPS||| +SAVE_DEFSV|5.004050||p +SPAGAIN||| +SP||| +START_EXTERN_C|5.005000||p +START_MY_CXT|5.007003||p +STMT_END|||p +STMT_START|||p +ST||| +SVt_IV||| +SVt_NV||| +SVt_PVAV||| +SVt_PVCV||| +SVt_PVHV||| +SVt_PVMG||| +SVt_PV||| +Safefree||| +Slab_Alloc||| +Slab_Free||| +StructCopy||| +SvCUR_set||| +SvCUR||| +SvEND||| +SvGETMAGIC|5.004050||p +SvGROW||| +SvIOK_UV||5.006000| +SvIOK_notUV||5.006000| +SvIOK_off||| +SvIOK_only_UV||5.006000| +SvIOK_only||| +SvIOK_on||| +SvIOKp||| +SvIOK||| +SvIVX||| +SvIV_nomg|5.009001||p +SvIVx||| +SvIV||| +SvIsCOW_shared_hash||5.008003| +SvIsCOW||5.008003| +SvLEN||| +SvLOCK||5.007003| +SvNIOK_off||| +SvNIOKp||| +SvNIOK||| +SvNOK_off||| +SvNOK_only||| +SvNOK_on||| +SvNOKp||| +SvNOK||| +SvNVX||| +SvNVx||| +SvNV||| +SvOK||| +SvOOK||| +SvPOK_off||| +SvPOK_only_UTF8||5.006000| +SvPOK_only||| +SvPOK_on||| +SvPOKp||| +SvPOK||| +SvPVX||| +SvPV_force_nomg|5.007002||p +SvPV_force||| +SvPV_nolen|5.006000||p +SvPV_nomg|5.007002||p +SvPVbyte_force||5.009002| +SvPVbyte_nolen||5.006000| +SvPVbytex_force||5.006000| +SvPVbytex||5.006000| +SvPVbyte|5.006000||p +SvPVutf8_force||5.006000| +SvPVutf8_nolen||5.006000| +SvPVutf8x_force||5.006000| +SvPVutf8x||5.006000| +SvPVutf8||5.006000| +SvPVx||| +SvPV||| +SvREFCNT_dec||| +SvREFCNT_inc||| +SvREFCNT||| +SvROK_off||| +SvROK_on||| +SvROK||| +SvRV||| +SvSETMAGIC||| +SvSHARE||5.007003| +SvSTASH||| +SvSetMagicSV_nosteal||5.004000| +SvSetMagicSV||5.004000| +SvSetSV_nosteal||5.004000| +SvSetSV||| +SvTAINTED_off||5.004000| +SvTAINTED_on||5.004000| +SvTAINTED||5.004000| +SvTAINT||| +SvTRUE||| +SvTYPE||| +SvUNLOCK||5.007003| +SvUOK||5.007001| +SvUPGRADE||| +SvUTF8_off||5.006000| +SvUTF8_on||5.006000| +SvUTF8||5.006000| +SvUVXx|5.004000||p +SvUVX|5.004000||p +SvUV_nomg|5.009001||p +SvUVx|5.004000||p +SvUV|5.004000||p +SvVOK||5.008001| +THIS|||n +UNDERBAR|5.009002||p +UVSIZE|5.006000||p +UVTYPE|5.006000||p +UVXf|5.007001||p +UVof|5.006000||p +UVuf|5.006000||p +UVxf|5.006000||p +XCPT_CATCH|5.009002||p +XCPT_RETHROW|5.009002||p +XCPT_TRY_END|5.009002||p +XCPT_TRY_START|5.009002||p +XPUSHi||| +XPUSHmortal|5.009002||p +XPUSHn||| +XPUSHp||| +XPUSHs||| +XPUSHu|5.004000||p +XSRETURN_EMPTY||| +XSRETURN_IV||| +XSRETURN_NO||| +XSRETURN_NV||| +XSRETURN_PV||| +XSRETURN_UNDEF||| +XSRETURN_UV|5.008001||p +XSRETURN_YES||| +XSRETURN||| +XST_mIV||| +XST_mNO||| +XST_mNV||| +XST_mPV||| +XST_mUNDEF||| +XST_mUV|5.008001||p +XST_mYES||| +XS_VERSION_BOOTCHECK||| +XS_VERSION||| +XS||| +ZeroD|5.009002||p +Zero||| +_aMY_CXT|5.007003||p +_pMY_CXT|5.007003||p +aMY_CXT_|5.007003||p +aMY_CXT|5.007003||p +aTHX_|5.006000||p +aTHX|5.006000||p +add_data||| +allocmy||| +amagic_call||| +any_dup||| +ao||| +append_elem||| +append_list||| +apply_attrs_my||| +apply_attrs_string||5.006001| +apply_attrs||| +apply||| +asIV||| +asUV||| +atfork_lock||5.007003|n +atfork_unlock||5.007003|n +av_clear||| +av_delete||5.006000| +av_exists||5.006000| +av_extend||| +av_fake||| +av_fetch||| +av_fill||| +av_len||| +av_make||| +av_pop||| +av_push||| +av_reify||| +av_shift||| +av_store||| +av_undef||| +av_unshift||| +ax|||n +bad_type||| +bind_match||| +block_end||| +block_gimme||5.004000| +block_start||| +boolSV|5.004000||p +boot_core_PerlIO||| +boot_core_UNIVERSAL||| +boot_core_xsutils||| +bytes_from_utf8||5.007001| +bytes_to_utf8||5.006001| +cache_re||| +call_argv|5.006000||p +call_atexit||5.006000| +call_body||| +call_list_body||| +call_list||5.004000| +call_method|5.006000||p +call_pv|5.006000||p +call_sv|5.006000||p +calloc||5.007002|n +cando||| +cast_i32||5.006000| +cast_iv||5.006000| +cast_ulong||5.006000| +cast_uv||5.006000| +check_uni||| +checkcomma||| +checkposixcc||| +cl_and||| +cl_anything||| +cl_init_zero||| +cl_init||| +cl_is_anything||| +cl_or||| +closest_cop||| +convert||| +cop_free||| +cr_textfilter||| +croak_nocontext|||vn +croak|||v +csighandler||5.007001|n +custom_op_desc||5.007003| +custom_op_name||5.007003| +cv_ckproto||| +cv_clone||| +cv_const_sv||5.004000| +cv_dump||| +cv_undef||| +cx_dump||5.005000| +cx_dup||| +cxinc||| +dAX|5.007002||p +dITEMS|5.007002||p +dMARK||| +dMY_CXT_SV|5.007003||p +dMY_CXT|5.007003||p +dNOOP|5.006000||p +dORIGMARK||| +dSP||| +dTHR|5.004050||p +dTHXa|5.006000||p +dTHXoa|5.006000||p +dTHX|5.006000||p +dUNDERBAR|5.009002||p +dXCPT|5.009002||p +dXSARGS||| +dXSI32||| +dXSTARG|5.006000||p +deb_curcv||| +deb_nocontext|||vn +deb_stack_all||| +deb_stack_n||| +debop||5.005000| +debprofdump||5.005000| +debprof||| +debstackptrs||5.007003| +debstack||5.007003| +deb||5.007003|v +del_he||| +del_sv||| +del_xiv||| +del_xnv||| +del_xpvav||| +del_xpvbm||| +del_xpvcv||| +del_xpvhv||| +del_xpviv||| +del_xpvlv||| +del_xpvmg||| +del_xpvnv||| +del_xpv||| +del_xrv||| +delimcpy||5.004000| +depcom||| +deprecate_old||| +deprecate||| +despatch_signals||5.007001| +die_nocontext|||vn +die_where||| +die|||v +dirp_dup||| +div128||| +djSP||| +do_aexec5||| +do_aexec||| +do_aspawn||| +do_binmode||5.004050| +do_chomp||| +do_chop||| +do_close||| +do_dump_pad||| +do_eof||| +do_exec3||| +do_execfree||| +do_exec||| +do_gv_dump||5.006000| +do_gvgv_dump||5.006000| +do_hv_dump||5.006000| +do_ipcctl||| +do_ipcget||| +do_join||| +do_kv||| +do_magic_dump||5.006000| +do_msgrcv||| +do_msgsnd||| +do_oddball||| +do_op_dump||5.006000| +do_open9||5.006000| +do_openn||5.007001| +do_open||5.004000| +do_pipe||| +do_pmop_dump||5.006000| +do_print||| +do_readline||| +do_seek||| +do_semop||| +do_shmio||| +do_spawn_nowait||| +do_spawn||| +do_sprintf||| +do_sv_dump||5.006000| +do_sysseek||| +do_tell||| +do_trans_complex_utf8||| +do_trans_complex||| +do_trans_count_utf8||| +do_trans_count||| +do_trans_simple_utf8||| +do_trans_simple||| +do_trans||| +do_vecget||| +do_vecset||| +do_vop||| +docatch_body||| +docatch||| +doencodes||| +doeval||| +dofile||| +dofindlabel||| +doform||| +doing_taint||5.008001|n +dooneliner||| +doopen_pm||| +doparseform||| +dopoptoeval||| +dopoptolabel||| +dopoptoloop||| +dopoptosub_at||| +dopoptosub||| +dounwind||| +dowantarray||| +dump_all||5.006000| +dump_eval||5.006000| +dump_fds||| +dump_form||5.006000| +dump_indent||5.006000|v +dump_mstats||| +dump_packsubs||5.006000| +dump_sub||5.006000| +dump_vindent||5.006000| +dumpuntil||| +dup_attrlist||| +emulate_eaccess||| +eval_pv|5.006000||p +eval_sv|5.006000||p +expect_number||| +fbm_compile||5.005000| +fbm_instr||5.005000| +fd_on_nosuid_fs||| +filter_add||| +filter_del||| +filter_gets||| +filter_read||| +find_beginning||| +find_byclass||| +find_in_my_stash||| +find_runcv||| +find_rundefsvoffset||5.009002| +find_script||| +find_uninit_var||| +fold_constants||| +forbid_setid||| +force_ident||| +force_list||| +force_next||| +force_version||| +force_word||| +form_nocontext|||vn +form||5.004000|v +fp_dup||| +fprintf_nocontext|||vn +free_tied_hv_pool||| +free_tmps||| +gen_constant_list||| +get_av|5.006000||p +get_context||5.006000|n +get_cv|5.006000||p +get_db_sub||| +get_debug_opts||| +get_hash_seed||| +get_hv|5.006000||p +get_mstats||| +get_no_modify||| +get_num||| +get_op_descs||5.005000| +get_op_names||5.005000| +get_opargs||| +get_ppaddr||5.006000| +get_sv|5.006000||p +get_vtbl||5.005030| +getcwd_sv||5.007002| +getenv_len||| +gp_dup||| +gp_free||| +gp_ref||| +grok_bin|5.007003||p +grok_hex|5.007003||p +grok_number|5.007002||p +grok_numeric_radix|5.007002||p +grok_oct|5.007003||p +group_end||| +gv_AVadd||| +gv_HVadd||| +gv_IOadd||| +gv_autoload4||5.004000| +gv_check||| +gv_dump||5.006000| +gv_efullname3||5.004000| +gv_efullname4||5.006001| +gv_efullname||| +gv_ename||| +gv_fetchfile||| +gv_fetchmeth_autoload||5.007003| +gv_fetchmethod_autoload||5.004000| +gv_fetchmethod||| +gv_fetchmeth||| +gv_fetchpvn_flags||5.009002| +gv_fetchpv||| +gv_fetchsv||5.009002| +gv_fullname3||5.004000| +gv_fullname4||5.006001| +gv_fullname||| +gv_handler||5.007001| +gv_init_sv||| +gv_init||| +gv_share||| +gv_stashpvn|5.006000||p +gv_stashpv||| +gv_stashsv||| +he_dup||| +hfreeentries||| +hsplit||| +hv_assert||5.009001| +hv_clear_placeholders||5.009001| +hv_clear||| +hv_delayfree_ent||5.004000| +hv_delete_common||| +hv_delete_ent||5.004000| +hv_delete||| +hv_exists_ent||5.004000| +hv_exists||| +hv_fetch_common||| +hv_fetch_ent||5.004000| +hv_fetch||| +hv_free_ent||5.004000| +hv_iterinit||| +hv_iterkeysv||5.004000| +hv_iterkey||| +hv_iternext_flags||5.008000| +hv_iternextsv||| +hv_iternext||| +hv_iterval||| +hv_ksplit||5.004000| +hv_magic_check||| +hv_magic||| +hv_notallowed||| +hv_scalar||5.009001| +hv_store_ent||5.004000| +hv_store_flags||5.008000| +hv_store||| +hv_undef||| +ibcmp_locale||5.004000| +ibcmp_utf8||5.007003| +ibcmp||| +incl_perldb||| +incline||| +incpush||| +ingroup||| +init_argv_symbols||| +init_debugger||| +init_i18nl10n||5.006000| +init_i18nl14n||5.006000| +init_ids||| +init_interp||| +init_lexer||| +init_main_stash||| +init_perllib||| +init_postdump_symbols||| +init_predump_symbols||| +init_stacks||5.005000| +init_tm||5.007002| +instr||| +intro_my||| +intuit_method||| +intuit_more||| +invert||| +io_close||| +isALNUM||| +isALPHA||| +isDIGIT||| +isLOWER||| +isSPACE||| +isUPPER||| +is_an_int||| +is_gv_magical_sv||| +is_gv_magical||| +is_handle_constructor||| +is_lvalue_sub||5.007001| +is_uni_alnum_lc||5.006000| +is_uni_alnumc_lc||5.006000| +is_uni_alnumc||5.006000| +is_uni_alnum||5.006000| +is_uni_alpha_lc||5.006000| +is_uni_alpha||5.006000| +is_uni_ascii_lc||5.006000| +is_uni_ascii||5.006000| +is_uni_cntrl_lc||5.006000| +is_uni_cntrl||5.006000| +is_uni_digit_lc||5.006000| +is_uni_digit||5.006000| +is_uni_graph_lc||5.006000| +is_uni_graph||5.006000| +is_uni_idfirst_lc||5.006000| +is_uni_idfirst||5.006000| +is_uni_lower_lc||5.006000| +is_uni_lower||5.006000| +is_uni_print_lc||5.006000| +is_uni_print||5.006000| +is_uni_punct_lc||5.006000| +is_uni_punct||5.006000| +is_uni_space_lc||5.006000| +is_uni_space||5.006000| +is_uni_upper_lc||5.006000| +is_uni_upper||5.006000| +is_uni_xdigit_lc||5.006000| +is_uni_xdigit||5.006000| +is_utf8_alnumc||5.006000| +is_utf8_alnum||5.006000| +is_utf8_alpha||5.006000| +is_utf8_ascii||5.006000| +is_utf8_char||5.006000| +is_utf8_cntrl||5.006000| +is_utf8_digit||5.006000| +is_utf8_graph||5.006000| +is_utf8_idcont||5.008000| +is_utf8_idfirst||5.006000| +is_utf8_lower||5.006000| +is_utf8_mark||5.006000| +is_utf8_print||5.006000| +is_utf8_punct||5.006000| +is_utf8_space||5.006000| +is_utf8_string_loc||5.008001| +is_utf8_string||5.006001| +is_utf8_upper||5.006000| +is_utf8_xdigit||5.006000| +isa_lookup||| +items|||n +ix|||n +jmaybe||| +keyword||| +leave_scope||| +lex_end||| +lex_start||| +linklist||| +list_assignment||| +listkids||| +list||| +load_module_nocontext|||vn +load_module||5.006000|v +localize||| +looks_like_number||| +lop||| +mPUSHi|5.009002||p +mPUSHn|5.009002||p +mPUSHp|5.009002||p +mPUSHu|5.009002||p +mXPUSHi|5.009002||p +mXPUSHn|5.009002||p +mXPUSHp|5.009002||p +mXPUSHu|5.009002||p +magic_clear_all_env||| +magic_clearenv||| +magic_clearpack||| +magic_clearsig||| +magic_dump||5.006000| +magic_existspack||| +magic_freeovrld||| +magic_freeregexp||| +magic_getarylen||| +magic_getdefelem||| +magic_getglob||| +magic_getnkeys||| +magic_getpack||| +magic_getpos||| +magic_getsig||| +magic_getsubstr||| +magic_gettaint||| +magic_getuvar||| +magic_getvec||| +magic_get||| +magic_killbackrefs||| +magic_len||| +magic_methcall||| +magic_methpack||| +magic_nextpack||| +magic_regdata_cnt||| +magic_regdatum_get||| +magic_regdatum_set||| +magic_scalarpack||| +magic_set_all_env||| +magic_setamagic||| +magic_setarylen||| +magic_setbm||| +magic_setcollxfrm||| +magic_setdbline||| +magic_setdefelem||| +magic_setenv||| +magic_setfm||| +magic_setglob||| +magic_setisa||| +magic_setmglob||| +magic_setnkeys||| +magic_setpack||| +magic_setpos||| +magic_setregexp||| +magic_setsig||| +magic_setsubstr||| +magic_settaint||| +magic_setutf8||| +magic_setuvar||| +magic_setvec||| +magic_set||| +magic_sizepack||| +magic_wipepack||| +magicname||| +malloced_size|||n +malloc||5.007002|n +markstack_grow||| +measure_struct||| +memEQ|5.004000||p +memNE|5.004000||p +mem_collxfrm||| +mess_alloc||| +mess_nocontext|||vn +mess||5.006000|v +method_common||| +mfree||5.007002|n +mg_clear||| +mg_copy||| +mg_dup||| +mg_find||| +mg_free||| +mg_get||| +mg_length||5.005000| +mg_magical||| +mg_set||| +mg_size||5.005000| +mini_mktime||5.007002| +missingterm||| +mode_from_discipline||| +modkids||| +mod||| +more_he||| +more_sv||| +more_xiv||| +more_xnv||| +more_xpvav||| +more_xpvbm||| +more_xpvcv||| +more_xpvhv||| +more_xpviv||| +more_xpvlv||| +more_xpvmg||| +more_xpvnv||| +more_xpv||| +more_xrv||| +moreswitches||| +mul128||| +mulexp10|||n +my_atof2||5.007002| +my_atof||5.006000| +my_attrs||| +my_bcopy|||n +my_betoh16|||n +my_betoh32|||n +my_betoh64|||n +my_betohi|||n +my_betohl|||n +my_betohs|||n +my_bzero|||n +my_chsize||| +my_exit_jump||| +my_exit||| +my_failure_exit||5.004000| +my_fflush_all||5.006000| +my_fork||5.007003|n +my_htobe16|||n +my_htobe32|||n +my_htobe64|||n +my_htobei|||n +my_htobel|||n +my_htobes|||n +my_htole16|||n +my_htole32|||n +my_htole64|||n +my_htolei|||n +my_htolel|||n +my_htoles|||n +my_htonl||| +my_kid||| +my_letoh16|||n +my_letoh32|||n +my_letoh64|||n +my_letohi|||n +my_letohl|||n +my_letohs|||n +my_lstat||| +my_memcmp||5.004000|n +my_memset|||n +my_ntohl||| +my_pclose||5.004000| +my_popen_list||5.007001| +my_popen||5.004000| +my_setenv||| +my_socketpair||5.007003|n +my_stat||| +my_strftime||5.007002| +my_swabn|||n +my_swap||| +my_unexec||| +my||| +newANONATTRSUB||5.006000| +newANONHASH||| +newANONLIST||| +newANONSUB||| +newASSIGNOP||| +newATTRSUB||5.006000| +newAVREF||| +newAV||| +newBINOP||| +newCONDOP||| +newCONSTSUB|5.006000||p +newCVREF||| +newDEFSVOP||| +newFORM||| +newFOROP||| +newGVOP||| +newGVREF||| +newGVgen||| +newHVREF||| +newHVhv||5.005000| +newHV||| +newIO||| +newLISTOP||| +newLOGOP||| +newLOOPEX||| +newLOOPOP||| +newMYSUB||5.006000| +newNULLLIST||| +newOP||| +newPADOP||5.006000| +newPMOP||| +newPROG||| +newPVOP||| +newRANGE||| +newRV_inc|5.004000||p +newRV_noinc|5.006000||p +newRV||| +newSLICEOP||| +newSTATEOP||| +newSUB||| +newSVOP||| +newSVREF||| +newSViv||| +newSVnv||| +newSVpvf_nocontext|||vn +newSVpvf||5.004000|v +newSVpvn_share||5.007001| +newSVpvn|5.006000||p +newSVpv||| +newSVrv||| +newSVsv||| +newSVuv|5.006000||p +newSV||| +newUNOP||| +newWHILEOP||5.004040| +newXSproto||5.006000| +newXS||5.006000| +new_collate||5.006000| +new_constant||| +new_ctype||5.006000| +new_he||| +new_logop||| +new_numeric||5.006000| +new_stackinfo||5.005000| +new_version||5.009000| +new_xiv||| +new_xnv||| +new_xpvav||| +new_xpvbm||| +new_xpvcv||| +new_xpvhv||| +new_xpviv||| +new_xpvlv||| +new_xpvmg||| +new_xpvnv||| +new_xpv||| +new_xrv||| +next_symbol||| +nextargv||| +nextchar||| +ninstr||| +no_bareword_allowed||| +no_fh_allowed||| +no_op||| +not_a_number||| +nothreadhook||5.008000| +nuke_stacks||| +num_overflow|||n +oopsAV||| +oopsCV||| +oopsHV||| +op_clear||| +op_const_sv||| +op_dump||5.006000| +op_free||| +op_null||5.007002| +op_refcnt_lock||5.009002| +op_refcnt_unlock||5.009002| +open_script||| +pMY_CXT_|5.007003||p +pMY_CXT|5.007003||p +pTHX_|5.006000||p +pTHX|5.006000||p +pack_cat||5.007003| +pack_rec||| +package||| +packlist||5.008001| +pad_add_anon||| +pad_add_name||| +pad_alloc||| +pad_block_start||| +pad_check_dup||| +pad_findlex||| +pad_findmy||| +pad_fixup_inner_anons||| +pad_free||| +pad_leavemy||| +pad_new||| +pad_push||| +pad_reset||| +pad_setsv||| +pad_sv||| +pad_swipe||| +pad_tidy||| +pad_undef||| +parse_body||| +parse_unicode_opts||| +path_is_absolute||| +peep||| +pending_ident||| +perl_alloc_using|||n +perl_alloc|||n +perl_clone_using|||n +perl_clone|||n +perl_construct|||n +perl_destruct||5.007003|n +perl_free|||n +perl_parse||5.006000|n +perl_run|||n +pidgone||| +pmflag||| +pmop_dump||5.006000| +pmruntime||| +pmtrans||| +pop_scope||| +pregcomp||| +pregexec||| +pregfree||| +prepend_elem||| +printf_nocontext|||vn +ptr_table_clear||| +ptr_table_fetch||| +ptr_table_free||| +ptr_table_new||| +ptr_table_split||| +ptr_table_store||| +push_scope||| +put_byte||| +pv_display||5.006000| +pv_uni_display||5.007003| +qerror||| +re_croak2||| +re_dup||| +re_intuit_start||5.006000| +re_intuit_string||5.006000| +realloc||5.007002|n +reentrant_free||| +reentrant_init||| +reentrant_retry|||vn +reentrant_size||| +refkids||| +refto||| +ref||| +reg_node||| +reganode||| +regatom||| +regbranch||| +regclass_swash||5.007003| +regclass||| +regcp_set_to||| +regcppop||| +regcppush||| +regcurly||| +regdump||5.005000| +regexec_flags||5.005000| +reghop3||| +reghopmaybe3||| +reghopmaybe||| +reghop||| +reginclass||| +reginitcolors||5.006000| +reginsert||| +regmatch||| +regnext||5.005000| +regoptail||| +regpiece||| +regpposixcc||| +regprop||| +regrepeat_hard||| +regrepeat||| +regtail||| +regtry||| +reguni||| +regwhite||| +reg||| +repeatcpy||| +report_evil_fh||| +report_uninit||| +require_errno||| +require_pv||5.006000| +rninstr||| +rsignal_restore||| +rsignal_save||| +rsignal_state||5.004000| +rsignal||5.004000| +run_body||| +runops_debug||5.005000| +runops_standard||5.005000| +rxres_free||| +rxres_restore||| +rxres_save||| +safesyscalloc||5.006000|n +safesysfree||5.006000|n +safesysmalloc||5.006000|n +safesysrealloc||5.006000|n +same_dirent||| +save_I16||5.004000| +save_I32||| +save_I8||5.006000| +save_aelem||5.004050| +save_alloc||5.006000| +save_aptr||| +save_ary||| +save_bool||5.008001| +save_clearsv||| +save_delete||| +save_destructor_x||5.006000| +save_destructor||5.006000| +save_freeop||| +save_freepv||| +save_freesv||| +save_generic_pvref||5.006001| +save_generic_svref||5.005030| +save_gp||5.004000| +save_hash||| +save_hek_flags||| +save_helem||5.004050| +save_hints||5.005000| +save_hptr||| +save_int||| +save_item||| +save_iv||5.005000| +save_lines||| +save_list||| +save_long||| +save_magic||| +save_mortalizesv||5.007001| +save_nogv||| +save_op||| +save_padsv||5.007001| +save_pptr||| +save_re_context||5.006000| +save_scalar_at||| +save_scalar||| +save_set_svflags||5.009000| +save_shared_pvref||5.007003| +save_sptr||| +save_svref||| +save_threadsv||5.005000| +save_vptr||5.006000| +savepvn||| +savepv||| +savesharedpv||5.007003| +savestack_grow_cnt||5.008001| +savestack_grow||| +savesvpv||5.009002| +sawparens||| +scalar_mod_type||| +scalarboolean||| +scalarkids||| +scalarseq||| +scalarvoid||| +scalar||| +scan_bin||5.006000| +scan_commit||| +scan_const||| +scan_formline||| +scan_heredoc||| +scan_hex||| +scan_ident||| +scan_inputsymbol||| +scan_num||5.007001| +scan_oct||| +scan_pat||| +scan_str||| +scan_subst||| +scan_trans||| +scan_version||5.009001| +scan_vstring||5.008001| +scan_word||| +scope||| +screaminstr||5.005000| +seed||| +set_context||5.006000|n +set_csh||| +set_numeric_local||5.006000| +set_numeric_radix||5.006000| +set_numeric_standard||5.006000| +setdefout||| +setenv_getix||| +share_hek_flags||| +share_hek||| +si_dup||| +sighandler|||n +simplify_sort||| +skipspace||| +sortsv||5.007003| +ss_dup||| +stack_grow||| +start_glob||| +start_subparse||5.004000| +stdize_locale||| +strEQ||| +strGE||| +strGT||| +strLE||| +strLT||| +strNE||| +str_to_version||5.006000| +strnEQ||| +strnNE||| +study_chunk||| +sub_crush_depth||| +sublex_done||| +sublex_push||| +sublex_start||| +sv_2bool||| +sv_2cv||| +sv_2io||| +sv_2iuv_non_preserve||| +sv_2iv_flags||5.009001| +sv_2iv||| +sv_2mortal||| +sv_2nv||| +sv_2pv_flags||5.007002| +sv_2pv_nolen|5.006000||p +sv_2pvbyte_nolen||| +sv_2pvbyte|5.006000||p +sv_2pvutf8_nolen||5.006000| +sv_2pvutf8||5.006000| +sv_2pv||| +sv_2uv_flags||5.009001| +sv_2uv|5.004000||p +sv_add_arena||| +sv_add_backref||| +sv_backoff||| +sv_bless||| +sv_cat_decode||5.008001| +sv_catpv_mg|5.006000||p +sv_catpvf_mg_nocontext|||pvn +sv_catpvf_mg|5.006000|5.004000|pv +sv_catpvf_nocontext|||vn +sv_catpvf||5.004000|v +sv_catpvn_flags||5.007002| +sv_catpvn_mg|5.006000||p +sv_catpvn_nomg|5.007002||p +sv_catpvn||| +sv_catpv||| +sv_catsv_flags||5.007002| +sv_catsv_mg|5.006000||p +sv_catsv_nomg|5.007002||p +sv_catsv||| +sv_chop||| +sv_clean_all||| +sv_clean_objs||| +sv_clear||| +sv_cmp_locale||5.004000| +sv_cmp||| +sv_collxfrm||| +sv_compile_2op||5.008001| +sv_copypv||5.007003| +sv_dec||| +sv_del_backref||| +sv_derived_from||5.004000| +sv_dump||| +sv_dup||| +sv_eq||| +sv_force_normal_flags||5.007001| +sv_force_normal||5.006000| +sv_free2||| +sv_free_arenas||| +sv_free||| +sv_gets||5.004000| +sv_grow||| +sv_inc||| +sv_insert||| +sv_isa||| +sv_isobject||| +sv_iv||5.005000| +sv_len_utf8||5.006000| +sv_len||| +sv_magicext||5.007003| +sv_magic||| +sv_mortalcopy||| +sv_newmortal||| +sv_newref||| +sv_nolocking||5.007003| +sv_nosharing||5.007003| +sv_nounlocking||5.007003| +sv_nv||5.005000| +sv_peek||5.005000| +sv_pos_b2u||5.006000| +sv_pos_u2b||5.006000| +sv_pvbyten_force||5.006000| +sv_pvbyten||5.006000| +sv_pvbyte||5.006000| +sv_pvn_force_flags||5.007002| +sv_pvn_force|||p +sv_pvn_nomg|5.007003||p +sv_pvn|5.006000||p +sv_pvutf8n_force||5.006000| +sv_pvutf8n||5.006000| +sv_pvutf8||5.006000| +sv_pv||5.006000| +sv_recode_to_utf8||5.007003| +sv_reftype||| +sv_release_COW||| +sv_release_IVX||| +sv_replace||| +sv_report_used||| +sv_reset||| +sv_rvweaken||5.006000| +sv_setiv_mg|5.006000||p +sv_setiv||| +sv_setnv_mg|5.006000||p +sv_setnv||| +sv_setpv_mg|5.006000||p +sv_setpvf_mg_nocontext|||pvn +sv_setpvf_mg|5.006000|5.004000|pv +sv_setpvf_nocontext|||vn +sv_setpvf||5.004000|v +sv_setpviv_mg||5.008001| +sv_setpviv||5.008001| +sv_setpvn_mg|5.006000||p +sv_setpvn||| +sv_setpv||| +sv_setref_iv||| +sv_setref_nv||| +sv_setref_pvn||| +sv_setref_pv||| +sv_setref_uv||5.007001| +sv_setsv_cow||| +sv_setsv_flags||5.007002| +sv_setsv_mg|5.006000||p +sv_setsv_nomg|5.007002||p +sv_setsv||| +sv_setuv_mg|5.006000||p +sv_setuv|5.006000||p +sv_tainted||5.004000| +sv_taint||5.004000| +sv_true||5.005000| +sv_unglob||| +sv_uni_display||5.007003| +sv_unmagic||| +sv_unref_flags||5.007001| +sv_unref||| +sv_untaint||5.004000| +sv_upgrade||| +sv_usepvn_mg|5.006000||p +sv_usepvn||| +sv_utf8_decode||5.006000| +sv_utf8_downgrade||5.006000| +sv_utf8_encode||5.006000| +sv_utf8_upgrade_flags||5.007002| +sv_utf8_upgrade||5.007001| +sv_uv|5.006000||p +sv_vcatpvf_mg|5.006000|5.004000|p +sv_vcatpvfn||5.004000| +sv_vcatpvf|5.006000|5.004000|p +sv_vsetpvf_mg|5.006000|5.004000|p +sv_vsetpvfn||5.004000| +sv_vsetpvf|5.006000|5.004000|p +svtype||| +swallow_bom||| +swash_fetch||5.007002| +swash_init||5.006000| +sys_intern_clear||| +sys_intern_dup||| +sys_intern_init||| +taint_env||| +taint_proper||| +tmps_grow||5.006000| +toLOWER||| +toUPPER||| +to_byte_substr||| +to_uni_fold||5.007003| +to_uni_lower_lc||5.006000| +to_uni_lower||5.007003| +to_uni_title_lc||5.006000| +to_uni_title||5.007003| +to_uni_upper_lc||5.006000| +to_uni_upper||5.007003| +to_utf8_case||5.007003| +to_utf8_fold||5.007003| +to_utf8_lower||5.007003| +to_utf8_substr||| +to_utf8_title||5.007003| +to_utf8_upper||5.007003| +tokeq||| +tokereport||| +too_few_arguments||| +too_many_arguments||| +unlnk||| +unpack_rec||| +unpack_str||5.007003| +unpackstring||5.008001| +unshare_hek_or_pvn||| +unshare_hek||| +unsharepvn||5.004000| +upg_version||5.009000| +usage||| +utf16_textfilter||| +utf16_to_utf8_reversed||5.006001| +utf16_to_utf8||5.006001| +utf16rev_textfilter||| +utf8_distance||5.006000| +utf8_hop||5.006000| +utf8_length||5.007001| +utf8_mg_pos_init||| +utf8_mg_pos||| +utf8_to_bytes||5.006001| +utf8_to_uvchr||5.007001| +utf8_to_uvuni||5.007001| +utf8n_to_uvchr||5.007001| +utf8n_to_uvuni||5.007001| +utilize||| +uvchr_to_utf8_flags||5.007003| +uvchr_to_utf8||5.007001| +uvuni_to_utf8_flags||5.007003| +uvuni_to_utf8||5.007001| +validate_suid||| +vcmp||5.009000| +vcroak||5.006000| +vdeb||5.007003| +vdie||| +vform||5.006000| +visit||| +vivify_defelem||| +vivify_ref||| +vload_module||5.006000| +vmess||5.006000| +vnewSVpvf|5.006000|5.004000|p +vnormal||5.009002| +vnumify||5.009000| +vstringify||5.009000| +vwarner||5.006000| +vwarn||5.006000| +wait4pid||| +warn_nocontext|||vn +warner_nocontext|||vn +warner||5.006000|v +warn|||v +watch||| +whichsig||| +write_to_stderr||| +yyerror||| +yylex||| +yyparse||| +yywarn||| +); + +if (exists $opt{'list-unsupported'}) { + my $f; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $API{$f}{todo}; + print "$f ", '.'x(40-length($f)), " ", format_version($API{$f}{todo}), "\n"; + } + exit 0; +} + +# Scan for possible replacement candidates + +my(%replace, %need, %hints, %depends); +my $replace = 0; +my $hint = ''; + +while () { + if ($hint) { + if (m{^\s*\*\s(.*?)\s*$}) { + $hints{$hint} ||= ''; # suppress warning with older perls + $hints{$hint} .= "$1\n"; + } + else { + $hint = ''; + } + } + $hint = $1 if m{^\s*$rccs\sHint:\s+(\w+)\s*$}; + + $replace = $1 if m{^\s*$rccs\s+Replace:\s+(\d+)\s+$rcce\s*$}; + $replace{$2} = $1 if $replace and m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+)}; + $replace{$2} = $1 if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+).*$rccs\s+Replace\s+$rcce}; + $replace{$1} = $2 if m{^\s*$rccs\s+Replace (\w+) with (\w+)\s+$rcce\s*$}; + + if (m{^\s*$rccs\s+(\w+)\s+depends\s+on\s+(\w+(\s*,\s*\w+)*)\s+$rcce\s*$}) { + push @{$depends{$1}}, map { s/\s+//g; $_ } split /,/, $2; + } + + $need{$1} = 1 if m{^#if\s+defined\(NEED_(\w+)(?:_GLOBAL)?\)}; +} + +if (exists $opt{'api-info'}) { + my $f; + my $count = 0; + my $match = $opt{'api-info'} =~ m!^/(.*)/$! ? $1 : "^\Q$opt{'api-info'}\E\$"; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $f =~ /$match/; + print "\n=== $f ===\n\n"; + my $info = 0; + if ($API{$f}{base} || $API{$f}{todo}) { + my $base = format_version($API{$f}{base} || $API{$f}{todo}); + print "Supported at least starting from perl-$base.\n"; + $info++; + } + if ($API{$f}{provided}) { + my $todo = $API{$f}{todo} ? format_version($API{$f}{todo}) : "5.003"; + print "Support by $ppport provided back to perl-$todo.\n"; + print "Support needs to be explicitly requested by NEED_$f.\n" if exists $need{$f}; + print "Depends on: ", join(', ', @{$depends{$f}}), ".\n" if exists $depends{$f}; + print "$hints{$f}" if exists $hints{$f}; + $info++; + } + unless ($info) { + print "No portability information available.\n"; + } + $count++; + } + if ($count > 0) { + print "\n"; + } + else { + print "Found no API matching '$opt{'api-info'}'.\n"; + } + exit 0; +} + +if (exists $opt{'list-provided'}) { + my $f; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $API{$f}{provided}; + my @flags; + push @flags, 'explicit' if exists $need{$f}; + push @flags, 'depend' if exists $depends{$f}; + push @flags, 'hint' if exists $hints{$f}; + my $flags = @flags ? ' ['.join(', ', @flags).']' : ''; + print "$f$flags\n"; + } + exit 0; +} + +my(%files, %global, %revreplace); +%revreplace = reverse %replace; +my $filename; +my $patch_opened = 0; + +for $filename (@files) { + unless (open IN, "<$filename") { + warn "Unable to read from $filename: $!\n"; + next; + } + + info("Scanning $filename ..."); + + my $c = do { local $/; }; + close IN; + + my %file = (orig => $c, changes => 0); + + # temporarily remove C comments from the code + my @ccom; + $c =~ s{ + ( + [^"'/]+ + | + (?:"[^"\\]*(?:\\.[^"\\]*)*" [^"'/]*)+ + | + (?:'[^'\\]*(?:\\.[^'\\]*)*' [^"'/]*)+ + ) + | + (/ (?: + \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / + | + /[^\r\n]* + )) + }{ + defined $2 and push @ccom, $2; + defined $1 ? $1 : "$ccs$#ccom$cce"; + }egsx; + + $file{ccom} = \@ccom; + $file{code} = $c; + $file{has_inc_ppport} = ($c =~ /#.*include.*\Q$ppport\E/); + + my $func; + + for $func (keys %API) { + my $match = $func; + $match .= "|$revreplace{$func}" if exists $revreplace{$func}; + if ($c =~ /\b(?:Perl_)?($match)\b/) { + $file{uses_replace}{$1}++ if exists $revreplace{$func} && $1 eq $revreplace{$func}; + $file{uses_Perl}{$func}++ if $c =~ /\bPerl_$func\b/; + if (exists $API{$func}{provided}) { + if (!exists $API{$func}{base} || $API{$func}{base} > $opt{'compat-version'}) { + $file{uses}{$func}++; + my @deps = rec_depend($func); + if (@deps) { + $file{uses_deps}{$func} = \@deps; + for (@deps) { + $file{uses}{$_} = 0 unless exists $file{uses}{$_}; + } + } + for ($func, @deps) { + if (exists $need{$_}) { + $file{needs}{$_} = 'static'; + } + } + } + } + if (exists $API{$func}{todo} && $API{$func}{todo} > $opt{'compat-version'}) { + if ($c =~ /\b$func\b/) { + $file{uses_todo}{$func}++; + } + } + } + } + + while ($c =~ /^$HS*#$HS*define$HS+(NEED_(\w+?)(_GLOBAL)?)\b/mg) { + if (exists $need{$2}) { + $file{defined $3 ? 'needed_global' : 'needed_static'}{$2}++; + } + else { + warning("Possibly wrong #define $1 in $filename"); + } + } + + for (qw(uses needs uses_todo needed_global needed_static)) { + for $func (keys %{$file{$_}}) { + push @{$global{$_}{$func}}, $filename; + } + } + + $files{$filename} = \%file; +} + +# Globally resolve NEED_'s +my $need; +for $need (keys %{$global{needs}}) { + if (@{$global{needs}{$need}} > 1) { + my @targets = @{$global{needs}{$need}}; + my @t = grep $files{$_}{needed_global}{$need}, @targets; + @targets = @t if @t; + @t = grep /\.xs$/i, @targets; + @targets = @t if @t; + my $target = shift @targets; + $files{$target}{needs}{$need} = 'global'; + for (@{$global{needs}{$need}}) { + $files{$_}{needs}{$need} = 'extern' if $_ ne $target; + } + } +} + +for $filename (@files) { + exists $files{$filename} or next; + + info("=== Analyzing $filename ==="); + + my %file = %{$files{$filename}}; + my $func; + my $c = $file{code}; + + for $func (sort keys %{$file{uses_Perl}}) { + if ($API{$func}{varargs}) { + my $changes = ($c =~ s{\b(Perl_$func\s*\(\s*)(?!aTHX_?)(\)|[^\s)]*\))} + { $1 . ($2 eq ')' ? 'aTHX' : 'aTHX_ ') . $2 }ge); + if ($changes) { + warning("Doesn't pass interpreter argument aTHX to Perl_$func"); + $file{changes} += $changes; + } + } + else { + warning("Uses Perl_$func instead of $func"); + $file{changes} += ($c =~ s{\bPerl_$func(\s*)\((\s*aTHX_?)?\s*} + {$func$1(}g); + } + } + + for $func (sort keys %{$file{uses_replace}}) { + warning("Uses $func instead of $replace{$func}"); + $file{changes} += ($c =~ s/\b$func\b/$replace{$func}/g); + } + + for $func (sort keys %{$file{uses}}) { + next unless $file{uses}{$func}; # if it's only a dependency + if (exists $file{uses_deps}{$func}) { + diag("Uses $func, which depends on ", join(', ', @{$file{uses_deps}{$func}})); + } + elsif (exists $replace{$func}) { + warning("Uses $func instead of $replace{$func}"); + $file{changes} += ($c =~ s/\b$func\b/$replace{$func}/g); + } + else { + diag("Uses $func"); + } + hint($func); + } + + for $func (sort keys %{$file{uses_todo}}) { + warning("Uses $func, which may not be portable below perl ", + format_version($API{$func}{todo})); + } + + for $func (sort keys %{$file{needed_static}}) { + my $message = ''; + if (not exists $file{uses}{$func}) { + $message = "No need to define NEED_$func if $func is never used"; + } + elsif (exists $file{needs}{$func} && $file{needs}{$func} ne 'static') { + $message = "No need to define NEED_$func when already needed globally"; + } + if ($message) { + diag($message); + $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_$func\b.*$LF//mg); + } + } + + for $func (sort keys %{$file{needed_global}}) { + my $message = ''; + if (not exists $global{uses}{$func}) { + $message = "No need to define NEED_${func}_GLOBAL if $func is never used"; + } + elsif (exists $file{needs}{$func}) { + if ($file{needs}{$func} eq 'extern') { + $message = "No need to define NEED_${func}_GLOBAL when already needed globally"; + } + elsif ($file{needs}{$func} eq 'static') { + $message = "No need to define NEED_${func}_GLOBAL when only used in this file"; + } + } + if ($message) { + diag($message); + $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_${func}_GLOBAL\b.*$LF//mg); + } + } + + $file{needs_inc_ppport} = keys %{$file{uses}}; + + if ($file{needs_inc_ppport}) { + my $pp = ''; + + for $func (sort keys %{$file{needs}}) { + my $type = $file{needs}{$func}; + next if $type eq 'extern'; + my $suffix = $type eq 'global' ? '_GLOBAL' : ''; + unless (exists $file{"needed_$type"}{$func}) { + if ($type eq 'global') { + diag("Files [@{$global{needs}{$func}}] need $func, adding global request"); + } + else { + diag("File needs $func, adding static request"); + } + $pp .= "#define NEED_$func$suffix\n"; + } + } + + if ($pp && ($c =~ s/^(?=$HS*#$HS*define$HS+NEED_\w+)/$pp/m)) { + $pp = ''; + $file{changes}++; + } + + unless ($file{has_inc_ppport}) { + diag("Needs to include '$ppport'"); + $pp .= qq(#include "$ppport"\n) + } + + if ($pp) { + $file{changes} += ($c =~ s/^($HS*#$HS*define$HS+NEED_\w+.*?)^/$1$pp/ms) + || ($c =~ s/^(?=$HS*#$HS*include.*\Q$ppport\E)/$pp/m) + || ($c =~ s/^($HS*#$HS*include.*XSUB.*\s*?)^/$1$pp/m) + || ($c =~ s/^/$pp/); + } + } + else { + if ($file{has_inc_ppport}) { + diag("No need to include '$ppport'"); + $file{changes} += ($c =~ s/^$HS*?#$HS*include.*\Q$ppport\E.*?$LF//m); + } + } + + # put back in our C comments + my $ix; + my $cppc = 0; + my @ccom = @{$file{ccom}}; + for $ix (0 .. $#ccom) { + if (!$opt{cplusplus} && $ccom[$ix] =~ s!^//!!) { + $cppc++; + $file{changes} += $c =~ s/$rccs$ix$rcce/$ccs$ccom[$ix] $cce/; + } + else { + $c =~ s/$rccs$ix$rcce/$ccom[$ix]/; + } + } + + if ($cppc) { + my $s = $cppc != 1 ? 's' : ''; + warning("Uses $cppc C++ style comment$s, which is not portable"); + } + + if ($file{changes}) { + if (exists $opt{copy}) { + my $newfile = "$filename$opt{copy}"; + if (-e $newfile) { + error("'$newfile' already exists, refusing to write copy of '$filename'"); + } + else { + local *F; + if (open F, ">$newfile") { + info("Writing copy of '$filename' with changes to '$newfile'"); + print F $c; + close F; + } + else { + error("Cannot open '$newfile' for writing: $!"); + } + } + } + elsif (exists $opt{patch} || $opt{changes}) { + if (exists $opt{patch}) { + unless ($patch_opened) { + if (open PATCH, ">$opt{patch}") { + $patch_opened = 1; + } + else { + error("Cannot open '$opt{patch}' for writing: $!"); + delete $opt{patch}; + $opt{changes} = 1; + goto fallback; + } + } + mydiff(\*PATCH, $filename, $c); + } + else { +fallback: + info("Suggested changes:"); + mydiff(\*STDOUT, $filename, $c); + } + } + else { + my $s = $file{changes} == 1 ? '' : 's'; + info("$file{changes} potentially required change$s detected"); + } + } + else { + info("Looks good"); + } +} + +close PATCH if $patch_opened; + +exit 0; + + +sub mydiff +{ + local *F = shift; + my($file, $str) = @_; + my $diff; + + if (exists $opt{diff}) { + $diff = run_diff($opt{diff}, $file, $str); + } + + if (!defined $diff and can_use('Text::Diff')) { + $diff = Text::Diff::diff($file, \$str, { STYLE => 'Unified' }); + $diff = <
$tmp") { + print F $str; + close F; + + if (open F, "$prog $file $tmp |") { + while () { + s/\Q$tmp\E/$file.patched/; + $diff .= $_; + } + close F; + unlink $tmp; + return $diff; + } + + unlink $tmp; + } + else { + error("Cannot open '$tmp' for writing: $!"); + } + + return undef; +} + +sub can_use +{ + eval "use @_;"; + return $@ eq ''; +} + +sub rec_depend +{ + my $func = shift; + my %seen; + return () unless exists $depends{$func}; + grep !$seen{$_}++, map { ($_, rec_depend($_)) } @{$depends{$func}}; +} + +sub parse_version +{ + my $ver = shift; + + if ($ver =~ /^(\d+)\.(\d+)\.(\d+)$/) { + return ($1, $2, $3); + } + elsif ($ver !~ /^\d+\.[\d_]+$/) { + die "cannot parse version '$ver'\n"; + } + + $ver =~ s/_//g; + $ver =~ s/$/000000/; + + my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; + + $v = int $v; + $s = int $s; + + if ($r < 5 || ($r == 5 && $v < 6)) { + if ($s % 10) { + die "cannot parse version '$ver'\n"; + } + } + + return ($r, $v, $s); +} + +sub format_version +{ + my $ver = shift; + + $ver =~ s/$/000000/; + my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; + + $v = int $v; + $s = int $s; + + if ($r < 5 || ($r == 5 && $v < 6)) { + if ($s % 10) { + die "invalid version '$ver'\n"; + } + $s /= 10; + + $ver = sprintf "%d.%03d", $r, $v; + $s > 0 and $ver .= sprintf "_%02d", $s; + + return $ver; + } + + return sprintf "%d.%d.%d", $r, $v, $s; +} + +sub info +{ + $opt{quiet} and return; + print @_, "\n"; +} + +sub diag +{ + $opt{quiet} and return; + $opt{diag} and print @_, "\n"; +} + +sub warning +{ + $opt{quiet} and return; + print "*** ", @_, "\n"; +} + +sub error +{ + print "*** ERROR: ", @_, "\n"; +} + +my %given_hints; +sub hint +{ + $opt{quiet} and return; + $opt{hints} or return; + my $func = shift; + exists $hints{$func} or return; + $given_hints{$func}++ and return; + my $hint = $hints{$func}; + $hint =~ s/^/ /mg; + print " --- hint for $func ---\n", $hint; +} + +sub usage +{ + my($usage) = do { local(@ARGV,$/)=($0); <> } =~ /^=head\d$HS+SYNOPSIS\s*^(.*?)\s*^=/ms; + my %M = ( 'I' => '*' ); + $usage =~ s/^\s*perl\s+\S+/$^X $0/; + $usage =~ s/([A-Z])<([^>]+)>/$M{$1}$2$M{$1}/g; + + print < +# endif +# if !(defined(PERL_VERSION) || (defined(SUBVERSION) && defined(PATCHLEVEL))) +# include +# endif +# ifndef PERL_REVISION +# define PERL_REVISION (5) + /* Replace: 1 */ +# define PERL_VERSION PATCHLEVEL +# define PERL_SUBVERSION SUBVERSION + /* Replace PERL_PATCHLEVEL with PERL_VERSION */ + /* Replace: 0 */ +# endif +#endif + +#define PERL_BCDVERSION ((PERL_REVISION * 0x1000000L) + (PERL_VERSION * 0x1000L) + PERL_SUBVERSION) + +/* It is very unlikely that anyone will try to use this with Perl 6 + (or greater), but who knows. + */ +#if PERL_REVISION != 5 +# error ppport.h only works with Perl version 5 +#endif /* PERL_REVISION != 5 */ + +#ifdef I_LIMITS +# include +#endif + +#ifndef PERL_UCHAR_MIN +# define PERL_UCHAR_MIN ((unsigned char)0) +#endif + +#ifndef PERL_UCHAR_MAX +# ifdef UCHAR_MAX +# define PERL_UCHAR_MAX ((unsigned char)UCHAR_MAX) +# else +# ifdef MAXUCHAR +# define PERL_UCHAR_MAX ((unsigned char)MAXUCHAR) +# else +# define PERL_UCHAR_MAX ((unsigned char)~(unsigned)0) +# endif +# endif +#endif + +#ifndef PERL_USHORT_MIN +# define PERL_USHORT_MIN ((unsigned short)0) +#endif + +#ifndef PERL_USHORT_MAX +# ifdef USHORT_MAX +# define PERL_USHORT_MAX ((unsigned short)USHORT_MAX) +# else +# ifdef MAXUSHORT +# define PERL_USHORT_MAX ((unsigned short)MAXUSHORT) +# else +# ifdef USHRT_MAX +# define PERL_USHORT_MAX ((unsigned short)USHRT_MAX) +# else +# define PERL_USHORT_MAX ((unsigned short)~(unsigned)0) +# endif +# endif +# endif +#endif + +#ifndef PERL_SHORT_MAX +# ifdef SHORT_MAX +# define PERL_SHORT_MAX ((short)SHORT_MAX) +# else +# ifdef MAXSHORT /* Often used in */ +# define PERL_SHORT_MAX ((short)MAXSHORT) +# else +# ifdef SHRT_MAX +# define PERL_SHORT_MAX ((short)SHRT_MAX) +# else +# define PERL_SHORT_MAX ((short) (PERL_USHORT_MAX >> 1)) +# endif +# endif +# endif +#endif + +#ifndef PERL_SHORT_MIN +# ifdef SHORT_MIN +# define PERL_SHORT_MIN ((short)SHORT_MIN) +# else +# ifdef MINSHORT +# define PERL_SHORT_MIN ((short)MINSHORT) +# else +# ifdef SHRT_MIN +# define PERL_SHORT_MIN ((short)SHRT_MIN) +# else +# define PERL_SHORT_MIN (-PERL_SHORT_MAX - ((3 & -1) == 3)) +# endif +# endif +# endif +#endif + +#ifndef PERL_UINT_MAX +# ifdef UINT_MAX +# define PERL_UINT_MAX ((unsigned int)UINT_MAX) +# else +# ifdef MAXUINT +# define PERL_UINT_MAX ((unsigned int)MAXUINT) +# else +# define PERL_UINT_MAX (~(unsigned int)0) +# endif +# endif +#endif + +#ifndef PERL_UINT_MIN +# define PERL_UINT_MIN ((unsigned int)0) +#endif + +#ifndef PERL_INT_MAX +# ifdef INT_MAX +# define PERL_INT_MAX ((int)INT_MAX) +# else +# ifdef MAXINT /* Often used in */ +# define PERL_INT_MAX ((int)MAXINT) +# else +# define PERL_INT_MAX ((int)(PERL_UINT_MAX >> 1)) +# endif +# endif +#endif + +#ifndef PERL_INT_MIN +# ifdef INT_MIN +# define PERL_INT_MIN ((int)INT_MIN) +# else +# ifdef MININT +# define PERL_INT_MIN ((int)MININT) +# else +# define PERL_INT_MIN (-PERL_INT_MAX - ((3 & -1) == 3)) +# endif +# endif +#endif + +#ifndef PERL_ULONG_MAX +# ifdef ULONG_MAX +# define PERL_ULONG_MAX ((unsigned long)ULONG_MAX) +# else +# ifdef MAXULONG +# define PERL_ULONG_MAX ((unsigned long)MAXULONG) +# else +# define PERL_ULONG_MAX (~(unsigned long)0) +# endif +# endif +#endif + +#ifndef PERL_ULONG_MIN +# define PERL_ULONG_MIN ((unsigned long)0L) +#endif + +#ifndef PERL_LONG_MAX +# ifdef LONG_MAX +# define PERL_LONG_MAX ((long)LONG_MAX) +# else +# ifdef MAXLONG +# define PERL_LONG_MAX ((long)MAXLONG) +# else +# define PERL_LONG_MAX ((long) (PERL_ULONG_MAX >> 1)) +# endif +# endif +#endif + +#ifndef PERL_LONG_MIN +# ifdef LONG_MIN +# define PERL_LONG_MIN ((long)LONG_MIN) +# else +# ifdef MINLONG +# define PERL_LONG_MIN ((long)MINLONG) +# else +# define PERL_LONG_MIN (-PERL_LONG_MAX - ((3 & -1) == 3)) +# endif +# endif +#endif + +#if defined(HAS_QUAD) && (defined(convex) || defined(uts)) +# ifndef PERL_UQUAD_MAX +# ifdef ULONGLONG_MAX +# define PERL_UQUAD_MAX ((unsigned long long)ULONGLONG_MAX) +# else +# ifdef MAXULONGLONG +# define PERL_UQUAD_MAX ((unsigned long long)MAXULONGLONG) +# else +# define PERL_UQUAD_MAX (~(unsigned long long)0) +# endif +# endif +# endif + +# ifndef PERL_UQUAD_MIN +# define PERL_UQUAD_MIN ((unsigned long long)0L) +# endif + +# ifndef PERL_QUAD_MAX +# ifdef LONGLONG_MAX +# define PERL_QUAD_MAX ((long long)LONGLONG_MAX) +# else +# ifdef MAXLONGLONG +# define PERL_QUAD_MAX ((long long)MAXLONGLONG) +# else +# define PERL_QUAD_MAX ((long long) (PERL_UQUAD_MAX >> 1)) +# endif +# endif +# endif + +# ifndef PERL_QUAD_MIN +# ifdef LONGLONG_MIN +# define PERL_QUAD_MIN ((long long)LONGLONG_MIN) +# else +# ifdef MINLONGLONG +# define PERL_QUAD_MIN ((long long)MINLONGLONG) +# else +# define PERL_QUAD_MIN (-PERL_QUAD_MAX - ((3 & -1) == 3)) +# endif +# endif +# endif +#endif + +/* This is based on code from 5.003 perl.h */ +#ifdef HAS_QUAD +# ifdef cray +#ifndef IVTYPE +# define IVTYPE int +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_INT_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_INT_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_UINT_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_UINT_MAX +#endif + +# ifdef INTSIZE +#ifndef IVSIZE +# define IVSIZE INTSIZE +#endif + +# endif +# else +# if defined(convex) || defined(uts) +#ifndef IVTYPE +# define IVTYPE long long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_QUAD_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_QUAD_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_UQUAD_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_UQUAD_MAX +#endif + +# ifdef LONGLONGSIZE +#ifndef IVSIZE +# define IVSIZE LONGLONGSIZE +#endif + +# endif +# else +#ifndef IVTYPE +# define IVTYPE long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_LONG_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_LONG_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_ULONG_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_ULONG_MAX +#endif + +# ifdef LONGSIZE +#ifndef IVSIZE +# define IVSIZE LONGSIZE +#endif + +# endif +# endif +# endif +#ifndef IVSIZE +# define IVSIZE 8 +#endif + +#ifndef PERL_QUAD_MIN +# define PERL_QUAD_MIN IV_MIN +#endif + +#ifndef PERL_QUAD_MAX +# define PERL_QUAD_MAX IV_MAX +#endif + +#ifndef PERL_UQUAD_MIN +# define PERL_UQUAD_MIN UV_MIN +#endif + +#ifndef PERL_UQUAD_MAX +# define PERL_UQUAD_MAX UV_MAX +#endif + +#else +#ifndef IVTYPE +# define IVTYPE long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_LONG_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_LONG_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_ULONG_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_ULONG_MAX +#endif + +#endif + +#ifndef IVSIZE +# ifdef LONGSIZE +# define IVSIZE LONGSIZE +# else +# define IVSIZE 4 /* A bold guess, but the best we can make. */ +# endif +#endif +#ifndef UVTYPE +# define UVTYPE unsigned IVTYPE +#endif + +#ifndef UVSIZE +# define UVSIZE IVSIZE +#endif + +#ifndef sv_setuv +# define sv_setuv(sv, uv) \ + STMT_START { \ + UV TeMpUv = uv; \ + if (TeMpUv <= IV_MAX) \ + sv_setiv(sv, TeMpUv); \ + else \ + sv_setnv(sv, (double)TeMpUv); \ + } STMT_END +#endif + +#ifndef newSVuv +# define newSVuv(uv) ((uv) <= IV_MAX ? newSViv((IV)uv) : newSVnv((NV)uv)) +#endif +#ifndef sv_2uv +# define sv_2uv(sv) ((PL_Sv = (sv)), (UV) (SvNOK(PL_Sv) ? SvNV(PL_Sv) : sv_2nv(PL_Sv))) +#endif + +#ifndef SvUVX +# define SvUVX(sv) ((UV)SvIVX(sv)) +#endif + +#ifndef SvUVXx +# define SvUVXx(sv) SvUVX(sv) +#endif + +#ifndef SvUV +# define SvUV(sv) (SvIOK(sv) ? SvUVX(sv) : sv_2uv(sv)) +#endif + +#ifndef SvUVx +# define SvUVx(sv) ((PL_Sv = (sv)), SvUV(PL_Sv)) +#endif + +/* Hint: sv_uv + * Always use the SvUVx() macro instead of sv_uv(). + */ +#ifndef sv_uv +# define sv_uv(sv) SvUVx(sv) +#endif +#ifndef XST_mUV +# define XST_mUV(i,v) (ST(i) = sv_2mortal(newSVuv(v)) ) +#endif + +#ifndef XSRETURN_UV +# define XSRETURN_UV(v) STMT_START { XST_mUV(0,v); XSRETURN(1); } STMT_END +#endif +#ifndef PUSHu +# define PUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); PUSHTARG; } STMT_END +#endif + +#ifndef XPUSHu +# define XPUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); XPUSHTARG; } STMT_END +#endif + +#if (PERL_VERSION < 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION <= 5)) +/* Replace: 1 */ +# define PL_DBsingle DBsingle +# define PL_DBsub DBsub +# define PL_Sv Sv +# define PL_compiling compiling +# define PL_copline copline +# define PL_curcop curcop +# define PL_curstash curstash +# define PL_debstash debstash +# define PL_defgv defgv +# define PL_diehook diehook +# define PL_dirty dirty +# define PL_dowarn dowarn +# define PL_errgv errgv +# define PL_hexdigit hexdigit +# define PL_hints hints +# define PL_na na +# define PL_no_modify no_modify +# define PL_perl_destruct_level perl_destruct_level +# define PL_perldb perldb +# define PL_ppaddr ppaddr +# define PL_rsfp_filters rsfp_filters +# define PL_rsfp rsfp +# define PL_stack_base stack_base +# define PL_stack_sp stack_sp +# define PL_stdingv stdingv +# define PL_sv_arenaroot sv_arenaroot +# define PL_sv_no sv_no +# define PL_sv_undef sv_undef +# define PL_sv_yes sv_yes +# define PL_tainted tainted +# define PL_tainting tainting +/* Replace: 0 */ +#endif + +#ifdef HASATTRIBUTE +# if (defined(__GNUC__) && defined(__cplusplus)) || defined(__INTEL_COMPILER) +# define PERL_UNUSED_DECL +# else +# define PERL_UNUSED_DECL __attribute__((unused)) +# endif +#else +# define PERL_UNUSED_DECL +#endif +#ifndef NOOP +# define NOOP (void)0 +#endif + +#ifndef dNOOP +# define dNOOP extern int Perl___notused PERL_UNUSED_DECL +#endif + +#ifndef NVTYPE +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) +# define NVTYPE long double +# else +# define NVTYPE double +# endif +typedef NVTYPE NV; +#endif + +#ifndef INT2PTR + +# if (IVSIZE == PTRSIZE) && (UVSIZE == PTRSIZE) +# define PTRV UV +# define INT2PTR(any,d) (any)(d) +# else +# if PTRSIZE == LONGSIZE +# define PTRV unsigned long +# else +# define PTRV unsigned +# endif +# define INT2PTR(any,d) (any)(PTRV)(d) +# endif + +# define NUM2PTR(any,d) (any)(PTRV)(d) +# define PTR2IV(p) INT2PTR(IV,p) +# define PTR2UV(p) INT2PTR(UV,p) +# define PTR2NV(p) NUM2PTR(NV,p) + +# if PTRSIZE == LONGSIZE +# define PTR2ul(p) (unsigned long)(p) +# else +# define PTR2ul(p) INT2PTR(unsigned long,p) +# endif + +#endif /* !INT2PTR */ + +#undef START_EXTERN_C +#undef END_EXTERN_C +#undef EXTERN_C +#ifdef __cplusplus +# define START_EXTERN_C extern "C" { +# define END_EXTERN_C } +# define EXTERN_C extern "C" +#else +# define START_EXTERN_C +# define END_EXTERN_C +# define EXTERN_C extern +#endif + +#ifndef PERL_GCC_BRACE_GROUPS_FORBIDDEN +# if defined(__STRICT_ANSI__) && defined(PERL_GCC_PEDANTIC) +# define PERL_GCC_BRACE_GROUPS_FORBIDDEN +# endif +#endif + +#undef STMT_START +#undef STMT_END +#if defined(__GNUC__) && !defined(PERL_GCC_BRACE_GROUPS_FORBIDDEN) && !defined(__cplusplus) +# define STMT_START (void)( /* gcc supports ``({ STATEMENTS; })'' */ +# define STMT_END ) +#else +# if defined(VOIDFLAGS) && (VOIDFLAGS) && (defined(sun) || defined(__sun__)) && !defined(__GNUC__) +# define STMT_START if (1) +# define STMT_END else (void)0 +# else +# define STMT_START do +# define STMT_END while (0) +# endif +#endif +#ifndef boolSV +# define boolSV(b) ((b) ? &PL_sv_yes : &PL_sv_no) +#endif + +/* DEFSV appears first in 5.004_56 */ +#ifndef DEFSV +# define DEFSV GvSV(PL_defgv) +#endif + +#ifndef SAVE_DEFSV +# define SAVE_DEFSV SAVESPTR(GvSV(PL_defgv)) +#endif + +/* Older perls (<=5.003) lack AvFILLp */ +#ifndef AvFILLp +# define AvFILLp AvFILL +#endif +#ifndef ERRSV +# define ERRSV get_sv("@",FALSE) +#endif +#ifndef newSVpvn +# define newSVpvn(data,len) ((data) \ + ? ((len) ? newSVpv((data), (len)) : newSVpv("", 0)) \ + : newSV(0)) +#endif + +/* Hint: gv_stashpvn + * This function's backport doesn't support the length parameter, but + * rather ignores it. Portability can only be ensured if the length + * parameter is used for speed reasons, but the length can always be + * correctly computed from the string argument. + */ +#ifndef gv_stashpvn +# define gv_stashpvn(str,len,create) gv_stashpv(str,create) +#endif + +/* Replace: 1 */ +#ifndef get_cv +# define get_cv perl_get_cv +#endif + +#ifndef get_sv +# define get_sv perl_get_sv +#endif + +#ifndef get_av +# define get_av perl_get_av +#endif + +#ifndef get_hv +# define get_hv perl_get_hv +#endif + +/* Replace: 0 */ + +#ifdef HAS_MEMCMP +#ifndef memNE +# define memNE(s1,s2,l) (memcmp(s1,s2,l)) +#endif + +#ifndef memEQ +# define memEQ(s1,s2,l) (!memcmp(s1,s2,l)) +#endif + +#else +#ifndef memNE +# define memNE(s1,s2,l) (bcmp(s1,s2,l)) +#endif + +#ifndef memEQ +# define memEQ(s1,s2,l) (!bcmp(s1,s2,l)) +#endif + +#endif +#ifndef MoveD +# define MoveD(s,d,n,t) memmove((char*)(d),(char*)(s), (n) * sizeof(t)) +#endif + +#ifndef CopyD +# define CopyD(s,d,n,t) memcpy((char*)(d),(char*)(s), (n) * sizeof(t)) +#endif + +#ifdef HAS_MEMSET +#ifndef ZeroD +# define ZeroD(d,n,t) memzero((char*)(d), (n) * sizeof(t)) +#endif + +#else +#ifndef ZeroD +# define ZeroD(d,n,t) ((void)memzero((char*)(d), (n) * sizeof(t)),d) +#endif + +#endif +#ifndef Poison +# define Poison(d,n,t) (void)memset((char*)(d), 0xAB, (n) * sizeof(t)) +#endif +#ifndef dUNDERBAR +# define dUNDERBAR dNOOP +#endif + +#ifndef UNDERBAR +# define UNDERBAR DEFSV +#endif +#ifndef dAX +# define dAX I32 ax = MARK - PL_stack_base + 1 +#endif + +#ifndef dITEMS +# define dITEMS I32 items = SP - MARK +#endif +#ifndef dXSTARG +# define dXSTARG SV * targ = sv_newmortal() +#endif +#ifndef dTHR +# define dTHR dNOOP +#endif +#ifndef dTHX +# define dTHX dNOOP +#endif + +#ifndef dTHXa +# define dTHXa(x) dNOOP +#endif +#ifndef pTHX +# define pTHX void +#endif + +#ifndef pTHX_ +# define pTHX_ +#endif + +#ifndef aTHX +# define aTHX +#endif + +#ifndef aTHX_ +# define aTHX_ +#endif +#ifndef dTHXoa +# define dTHXoa(x) dTHXa(x) +#endif +#ifndef PUSHmortal +# define PUSHmortal PUSHs(sv_newmortal()) +#endif + +#ifndef mPUSHp +# define mPUSHp(p,l) sv_setpvn_mg(PUSHmortal, (p), (l)) +#endif + +#ifndef mPUSHn +# define mPUSHn(n) sv_setnv_mg(PUSHmortal, (NV)(n)) +#endif + +#ifndef mPUSHi +# define mPUSHi(i) sv_setiv_mg(PUSHmortal, (IV)(i)) +#endif + +#ifndef mPUSHu +# define mPUSHu(u) sv_setuv_mg(PUSHmortal, (UV)(u)) +#endif +#ifndef XPUSHmortal +# define XPUSHmortal XPUSHs(sv_newmortal()) +#endif + +#ifndef mXPUSHp +# define mXPUSHp(p,l) STMT_START { EXTEND(sp,1); sv_setpvn_mg(PUSHmortal, (p), (l)); } STMT_END +#endif + +#ifndef mXPUSHn +# define mXPUSHn(n) STMT_START { EXTEND(sp,1); sv_setnv_mg(PUSHmortal, (NV)(n)); } STMT_END +#endif + +#ifndef mXPUSHi +# define mXPUSHi(i) STMT_START { EXTEND(sp,1); sv_setiv_mg(PUSHmortal, (IV)(i)); } STMT_END +#endif + +#ifndef mXPUSHu +# define mXPUSHu(u) STMT_START { EXTEND(sp,1); sv_setuv_mg(PUSHmortal, (UV)(u)); } STMT_END +#endif + +/* Replace: 1 */ +#ifndef call_sv +# define call_sv perl_call_sv +#endif + +#ifndef call_pv +# define call_pv perl_call_pv +#endif + +#ifndef call_argv +# define call_argv perl_call_argv +#endif + +#ifndef call_method +# define call_method perl_call_method +#endif +#ifndef eval_sv +# define eval_sv perl_eval_sv +#endif + +/* Replace: 0 */ + +/* Replace perl_eval_pv with eval_pv */ +/* eval_pv depends on eval_sv */ + +#ifndef eval_pv +#if defined(NEED_eval_pv) +static SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); +static +#else +extern SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); +#endif + +#ifdef eval_pv +# undef eval_pv +#endif +#define eval_pv(a,b) DPPP_(my_eval_pv)(aTHX_ a,b) +#define Perl_eval_pv DPPP_(my_eval_pv) + +#if defined(NEED_eval_pv) || defined(NEED_eval_pv_GLOBAL) + +SV* +DPPP_(my_eval_pv)(char *p, I32 croak_on_error) +{ + dSP; + SV* sv = newSVpv(p, 0); + + PUSHMARK(sp); + eval_sv(sv, G_SCALAR); + SvREFCNT_dec(sv); + + SPAGAIN; + sv = POPs; + PUTBACK; + + if (croak_on_error && SvTRUE(GvSV(errgv))) + croak(SvPVx(GvSV(errgv), na)); + + return sv; +} + +#endif +#endif +#ifndef newRV_inc +# define newRV_inc(sv) newRV(sv) /* Replace */ +#endif + +#ifndef newRV_noinc +#if defined(NEED_newRV_noinc) +static SV * DPPP_(my_newRV_noinc)(SV *sv); +static +#else +extern SV * DPPP_(my_newRV_noinc)(SV *sv); +#endif + +#ifdef newRV_noinc +# undef newRV_noinc +#endif +#define newRV_noinc(a) DPPP_(my_newRV_noinc)(aTHX_ a) +#define Perl_newRV_noinc DPPP_(my_newRV_noinc) + +#if defined(NEED_newRV_noinc) || defined(NEED_newRV_noinc_GLOBAL) +SV * +DPPP_(my_newRV_noinc)(SV *sv) +{ + SV *rv = (SV *)newRV(sv); + SvREFCNT_dec(sv); + return rv; +} +#endif +#endif + +/* Hint: newCONSTSUB + * Returns a CV* as of perl-5.7.1. This return value is not supported + * by Devel::PPPort. + */ + +/* newCONSTSUB from IO.xs is in the core starting with 5.004_63 */ +#if ((PERL_VERSION < 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION < 63))) && ((PERL_VERSION != 4) || (PERL_SUBVERSION != 5)) +#if defined(NEED_newCONSTSUB) +static void DPPP_(my_newCONSTSUB)(HV *stash, char *name, SV *sv); +static +#else +extern void DPPP_(my_newCONSTSUB)(HV *stash, char *name, SV *sv); +#endif + +#ifdef newCONSTSUB +# undef newCONSTSUB +#endif +#define newCONSTSUB(a,b,c) DPPP_(my_newCONSTSUB)(aTHX_ a,b,c) +#define Perl_newCONSTSUB DPPP_(my_newCONSTSUB) + +#if defined(NEED_newCONSTSUB) || defined(NEED_newCONSTSUB_GLOBAL) + +void +DPPP_(my_newCONSTSUB)(HV *stash, char *name, SV *sv) +{ + U32 oldhints = PL_hints; + HV *old_cop_stash = PL_curcop->cop_stash; + HV *old_curstash = PL_curstash; + line_t oldline = PL_curcop->cop_line; + PL_curcop->cop_line = PL_copline; + + PL_hints &= ~HINT_BLOCK_SCOPE; + if (stash) + PL_curstash = PL_curcop->cop_stash = stash; + + newSUB( + +#if ((PERL_VERSION < 3) || ((PERL_VERSION == 3) && (PERL_SUBVERSION < 22))) + start_subparse(), +#elif ((PERL_VERSION == 3) && (PERL_SUBVERSION == 22)) + start_subparse(0), +#else /* 5.003_23 onwards */ + start_subparse(FALSE, 0), +#endif + + newSVOP(OP_CONST, 0, newSVpv(name,0)), + newSVOP(OP_CONST, 0, &PL_sv_no), /* SvPV(&PL_sv_no) == "" -- GMB */ + newSTATEOP(0, Nullch, newSVOP(OP_CONST, 0, sv)) + ); + + PL_hints = oldhints; + PL_curcop->cop_stash = old_cop_stash; + PL_curstash = old_curstash; + PL_curcop->cop_line = oldline; +} +#endif +#endif + +/* + * Boilerplate macros for initializing and accessing interpreter-local + * data from C. All statics in extensions should be reworked to use + * this, if you want to make the extension thread-safe. See ext/re/re.xs + * for an example of the use of these macros. + * + * Code that uses these macros is responsible for the following: + * 1. #define MY_CXT_KEY to a unique string, e.g. "DynaLoader_guts" + * 2. Declare a typedef named my_cxt_t that is a structure that contains + * all the data that needs to be interpreter-local. + * 3. Use the START_MY_CXT macro after the declaration of my_cxt_t. + * 4. Use the MY_CXT_INIT macro such that it is called exactly once + * (typically put in the BOOT: section). + * 5. Use the members of the my_cxt_t structure everywhere as + * MY_CXT.member. + * 6. Use the dMY_CXT macro (a declaration) in all the functions that + * access MY_CXT. + */ + +#if defined(MULTIPLICITY) || defined(PERL_OBJECT) || \ + defined(PERL_CAPI) || defined(PERL_IMPLICIT_CONTEXT) + +#ifndef START_MY_CXT + +/* This must appear in all extensions that define a my_cxt_t structure, + * right after the definition (i.e. at file scope). The non-threads + * case below uses it to declare the data as static. */ +#define START_MY_CXT + +#if (PERL_VERSION < 4 || (PERL_VERSION == 4 && PERL_SUBVERSION < 68 )) +/* Fetches the SV that keeps the per-interpreter data. */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = get_sv(MY_CXT_KEY, FALSE) +#else /* >= perl5.004_68 */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = *hv_fetch(PL_modglobal, MY_CXT_KEY, \ + sizeof(MY_CXT_KEY)-1, TRUE) +#endif /* < perl5.004_68 */ + +/* This declaration should be used within all functions that use the + * interpreter-local data. */ +#define dMY_CXT \ + dMY_CXT_SV; \ + my_cxt_t *my_cxtp = INT2PTR(my_cxt_t*,SvUV(my_cxt_sv)) + +/* Creates and zeroes the per-interpreter data. + * (We allocate my_cxtp in a Perl SV so that it will be released when + * the interpreter goes away.) */ +#define MY_CXT_INIT \ + dMY_CXT_SV; \ + /* newSV() allocates one more than needed */ \ + my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ + Zero(my_cxtp, 1, my_cxt_t); \ + sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) + +/* This macro must be used to access members of the my_cxt_t structure. + * e.g. MYCXT.some_data */ +#define MY_CXT (*my_cxtp) + +/* Judicious use of these macros can reduce the number of times dMY_CXT + * is used. Use is similar to pTHX, aTHX etc. */ +#define pMY_CXT my_cxt_t *my_cxtp +#define pMY_CXT_ pMY_CXT, +#define _pMY_CXT ,pMY_CXT +#define aMY_CXT my_cxtp +#define aMY_CXT_ aMY_CXT, +#define _aMY_CXT ,aMY_CXT + +#endif /* START_MY_CXT */ + +#ifndef MY_CXT_CLONE +/* Clones the per-interpreter data. */ +#define MY_CXT_CLONE \ + dMY_CXT_SV; \ + my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ + Copy(INT2PTR(my_cxt_t*, SvUV(my_cxt_sv)), my_cxtp, 1, my_cxt_t);\ + sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) +#endif + +#else /* single interpreter */ + +#ifndef START_MY_CXT + +#define START_MY_CXT static my_cxt_t my_cxt; +#define dMY_CXT_SV dNOOP +#define dMY_CXT dNOOP +#define MY_CXT_INIT NOOP +#define MY_CXT my_cxt + +#define pMY_CXT void +#define pMY_CXT_ +#define _pMY_CXT +#define aMY_CXT +#define aMY_CXT_ +#define _aMY_CXT + +#endif /* START_MY_CXT */ + +#ifndef MY_CXT_CLONE +#define MY_CXT_CLONE NOOP +#endif + +#endif + +#ifndef IVdf +# if IVSIZE == LONGSIZE +# define IVdf "ld" +# define UVuf "lu" +# define UVof "lo" +# define UVxf "lx" +# define UVXf "lX" +# else +# if IVSIZE == INTSIZE +# define IVdf "d" +# define UVuf "u" +# define UVof "o" +# define UVxf "x" +# define UVXf "X" +# endif +# endif +#endif + +#ifndef NVef +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) && \ + defined(PERL_PRIfldbl) /* Not very likely, but let's try anyway. */ +# define NVef PERL_PRIeldbl +# define NVff PERL_PRIfldbl +# define NVgf PERL_PRIgldbl +# else +# define NVef "e" +# define NVff "f" +# define NVgf "g" +# endif +#endif + +#ifndef SvPV_nolen + +#if defined(NEED_sv_2pv_nolen) +static char * DPPP_(my_sv_2pv_nolen)(pTHX_ register SV *sv); +static +#else +extern char * DPPP_(my_sv_2pv_nolen)(pTHX_ register SV *sv); +#endif + +#ifdef sv_2pv_nolen +# undef sv_2pv_nolen +#endif +#define sv_2pv_nolen(a) DPPP_(my_sv_2pv_nolen)(aTHX_ a) +#define Perl_sv_2pv_nolen DPPP_(my_sv_2pv_nolen) + +#if defined(NEED_sv_2pv_nolen) || defined(NEED_sv_2pv_nolen_GLOBAL) + +char * +DPPP_(my_sv_2pv_nolen)(pTHX_ register SV *sv) +{ + STRLEN n_a; + return sv_2pv(sv, &n_a); +} + +#endif + +/* Hint: sv_2pv_nolen + * Use the SvPV_nolen() macro instead of sv_2pv_nolen(). + */ + +/* SvPV_nolen depends on sv_2pv_nolen */ +#define SvPV_nolen(sv) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX(sv) : sv_2pv_nolen(sv)) + +#endif + +#ifdef SvPVbyte + +/* Hint: SvPVbyte + * Does not work in perl-5.6.1, ppport.h implements a version + * borrowed from perl-5.7.3. + */ + +#if ((PERL_VERSION < 7) || ((PERL_VERSION == 7) && (PERL_SUBVERSION < 0))) + +#if defined(NEED_sv_2pvbyte) +static char * DPPP_(my_sv_2pvbyte)(pTHX_ register SV *sv, STRLEN *lp); +static +#else +extern char * DPPP_(my_sv_2pvbyte)(pTHX_ register SV *sv, STRLEN *lp); +#endif + +#ifdef sv_2pvbyte +# undef sv_2pvbyte +#endif +#define sv_2pvbyte(a,b) DPPP_(my_sv_2pvbyte)(aTHX_ a,b) +#define Perl_sv_2pvbyte DPPP_(my_sv_2pvbyte) + +#if defined(NEED_sv_2pvbyte) || defined(NEED_sv_2pvbyte_GLOBAL) + +char * +DPPP_(my_sv_2pvbyte)(pTHX_ register SV *sv, STRLEN *lp) +{ + sv_utf8_downgrade(sv,0); + return SvPV(sv,*lp); +} + +#endif + +/* Hint: sv_2pvbyte + * Use the SvPVbyte() macro instead of sv_2pvbyte(). + */ + +#undef SvPVbyte + +/* SvPVbyte depends on sv_2pvbyte */ +#define SvPVbyte(sv, lp) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_UTF8)) == (SVf_POK) \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pvbyte(sv, &lp)) + +#endif + +#else + +# define SvPVbyte SvPV +# define sv_2pvbyte sv_2pv + +#endif + +/* sv_2pvbyte_nolen depends on sv_2pv_nolen */ +#ifndef sv_2pvbyte_nolen +# define sv_2pvbyte_nolen sv_2pv_nolen +#endif + +/* Hint: sv_pvn + * Always use the SvPV() macro instead of sv_pvn(). + */ +#ifndef sv_pvn +# define sv_pvn(sv, len) SvPV(sv, len) +#endif + +/* Hint: sv_pvn_force + * Always use the SvPV_force() macro instead of sv_pvn_force(). + */ +#ifndef sv_pvn_force +# define sv_pvn_force(sv, len) SvPV_force(sv, len) +#endif + +#if ((PERL_VERSION > 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION >= 0))) && !defined(vnewSVpvf) +#if defined(NEED_vnewSVpvf) +static SV * DPPP_(my_vnewSVpvf)(pTHX_ const char * pat, va_list * args); +static +#else +extern SV * DPPP_(my_vnewSVpvf)(pTHX_ const char * pat, va_list * args); +#endif + +#ifdef vnewSVpvf +# undef vnewSVpvf +#endif +#define vnewSVpvf(a,b) DPPP_(my_vnewSVpvf)(aTHX_ a,b) +#define Perl_vnewSVpvf DPPP_(my_vnewSVpvf) + +#if defined(NEED_vnewSVpvf) || defined(NEED_vnewSVpvf_GLOBAL) + +SV * +DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args) +{ + register SV *sv = newSV(0); + sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); + return sv; +} + +#endif +#endif + +/* sv_vcatpvf depends on sv_vcatpvfn */ +#if ((PERL_VERSION > 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION >= 0))) && !defined(sv_vcatpvf) +# define sv_vcatpvf(sv, pat, args) sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) +#endif + +/* sv_vsetpvf depends on sv_vsetpvfn */ +#if ((PERL_VERSION > 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION >= 0))) && !defined(sv_vsetpvf) +# define sv_vsetpvf(sv, pat, args) sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) +#endif + +/* sv_catpvf_mg depends on sv_vcatpvfn, sv_catpvf_mg_nocontext */ +#if ((PERL_VERSION > 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION >= 0))) && !defined(sv_catpvf_mg) +#if defined(NEED_sv_catpvf_mg) +static void DPPP_(my_sv_catpvf_mg)(pTHX_ SV * sv, const char * pat, ...); +static +#else +extern void DPPP_(my_sv_catpvf_mg)(pTHX_ SV * sv, const char * pat, ...); +#endif + +#define Perl_sv_catpvf_mg DPPP_(my_sv_catpvf_mg) + +#if defined(NEED_sv_catpvf_mg) || defined(NEED_sv_catpvf_mg_GLOBAL) + +void +DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...) +{ + va_list args; + va_start(args, pat); + sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif + +/* sv_catpvf_mg_nocontext depends on sv_vcatpvfn */ +#ifdef PERL_IMPLICIT_CONTEXT +#if ((PERL_VERSION > 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION >= 0))) && !defined(sv_catpvf_mg_nocontext) +#if defined(NEED_sv_catpvf_mg_nocontext) +static void DPPP_(my_sv_catpvf_mg_nocontext)(SV * sv, const char * pat, ...); +static +#else +extern void DPPP_(my_sv_catpvf_mg_nocontext)(SV * sv, const char * pat, ...); +#endif + +#define sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) +#define Perl_sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) + +#if defined(NEED_sv_catpvf_mg_nocontext) || defined(NEED_sv_catpvf_mg_nocontext_GLOBAL) + +void +DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...) +{ + dTHX; + va_list args; + va_start(args, pat); + sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif +#endif + +#ifndef sv_catpvf_mg +# ifdef PERL_IMPLICIT_CONTEXT +# define sv_catpvf_mg Perl_sv_catpvf_mg_nocontext +# else +# define sv_catpvf_mg Perl_sv_catpvf_mg +# endif +#endif + +/* sv_vcatpvf_mg depends on sv_vcatpvfn */ +#if ((PERL_VERSION > 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION >= 0))) && !defined(sv_vcatpvf_mg) +# define sv_vcatpvf_mg(sv, pat, args) \ + STMT_START { \ + sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ + SvSETMAGIC(sv); \ + } STMT_END +#endif + +/* sv_setpvf_mg depends on sv_vsetpvfn, sv_setpvf_mg_nocontext */ +#if ((PERL_VERSION > 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION >= 0))) && !defined(sv_setpvf_mg) +#if defined(NEED_sv_setpvf_mg) +static void DPPP_(my_sv_setpvf_mg)(pTHX_ SV * sv, const char * pat, ...); +static +#else +extern void DPPP_(my_sv_setpvf_mg)(pTHX_ SV * sv, const char * pat, ...); +#endif + +#define Perl_sv_setpvf_mg DPPP_(my_sv_setpvf_mg) + +#if defined(NEED_sv_setpvf_mg) || defined(NEED_sv_setpvf_mg_GLOBAL) + +void +DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...) +{ + va_list args; + va_start(args, pat); + sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif + +/* sv_setpvf_mg_nocontext depends on sv_vsetpvfn */ +#ifdef PERL_IMPLICIT_CONTEXT +#if ((PERL_VERSION > 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION >= 0))) && !defined(sv_setpvf_mg_nocontext) +#if defined(NEED_sv_setpvf_mg_nocontext) +static void DPPP_(my_sv_setpvf_mg_nocontext)(SV * sv, const char * pat, ...); +static +#else +extern void DPPP_(my_sv_setpvf_mg_nocontext)(SV * sv, const char * pat, ...); +#endif + +#define sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) +#define Perl_sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) + +#if defined(NEED_sv_setpvf_mg_nocontext) || defined(NEED_sv_setpvf_mg_nocontext_GLOBAL) + +void +DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...) +{ + dTHX; + va_list args; + va_start(args, pat); + sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif +#endif + +#ifndef sv_setpvf_mg +# ifdef PERL_IMPLICIT_CONTEXT +# define sv_setpvf_mg Perl_sv_setpvf_mg_nocontext +# else +# define sv_setpvf_mg Perl_sv_setpvf_mg +# endif +#endif + +/* sv_vsetpvf_mg depends on sv_vsetpvfn */ +#if ((PERL_VERSION > 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION >= 0))) && !defined(sv_vsetpvf_mg) +# define sv_vsetpvf_mg(sv, pat, args) \ + STMT_START { \ + sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ + SvSETMAGIC(sv); \ + } STMT_END +#endif +#ifndef SvGETMAGIC +# define SvGETMAGIC(x) STMT_START { if (SvGMAGICAL(x)) mg_get(x); } STMT_END +#endif +#ifndef PERL_MAGIC_sv +# define PERL_MAGIC_sv '\0' +#endif + +#ifndef PERL_MAGIC_overload +# define PERL_MAGIC_overload 'A' +#endif + +#ifndef PERL_MAGIC_overload_elem +# define PERL_MAGIC_overload_elem 'a' +#endif + +#ifndef PERL_MAGIC_overload_table +# define PERL_MAGIC_overload_table 'c' +#endif + +#ifndef PERL_MAGIC_bm +# define PERL_MAGIC_bm 'B' +#endif + +#ifndef PERL_MAGIC_regdata +# define PERL_MAGIC_regdata 'D' +#endif + +#ifndef PERL_MAGIC_regdatum +# define PERL_MAGIC_regdatum 'd' +#endif + +#ifndef PERL_MAGIC_env +# define PERL_MAGIC_env 'E' +#endif + +#ifndef PERL_MAGIC_envelem +# define PERL_MAGIC_envelem 'e' +#endif + +#ifndef PERL_MAGIC_fm +# define PERL_MAGIC_fm 'f' +#endif + +#ifndef PERL_MAGIC_regex_global +# define PERL_MAGIC_regex_global 'g' +#endif + +#ifndef PERL_MAGIC_isa +# define PERL_MAGIC_isa 'I' +#endif + +#ifndef PERL_MAGIC_isaelem +# define PERL_MAGIC_isaelem 'i' +#endif + +#ifndef PERL_MAGIC_nkeys +# define PERL_MAGIC_nkeys 'k' +#endif + +#ifndef PERL_MAGIC_dbfile +# define PERL_MAGIC_dbfile 'L' +#endif + +#ifndef PERL_MAGIC_dbline +# define PERL_MAGIC_dbline 'l' +#endif + +#ifndef PERL_MAGIC_mutex +# define PERL_MAGIC_mutex 'm' +#endif + +#ifndef PERL_MAGIC_shared +# define PERL_MAGIC_shared 'N' +#endif + +#ifndef PERL_MAGIC_shared_scalar +# define PERL_MAGIC_shared_scalar 'n' +#endif + +#ifndef PERL_MAGIC_collxfrm +# define PERL_MAGIC_collxfrm 'o' +#endif + +#ifndef PERL_MAGIC_tied +# define PERL_MAGIC_tied 'P' +#endif + +#ifndef PERL_MAGIC_tiedelem +# define PERL_MAGIC_tiedelem 'p' +#endif + +#ifndef PERL_MAGIC_tiedscalar +# define PERL_MAGIC_tiedscalar 'q' +#endif + +#ifndef PERL_MAGIC_qr +# define PERL_MAGIC_qr 'r' +#endif + +#ifndef PERL_MAGIC_sig +# define PERL_MAGIC_sig 'S' +#endif + +#ifndef PERL_MAGIC_sigelem +# define PERL_MAGIC_sigelem 's' +#endif + +#ifndef PERL_MAGIC_taint +# define PERL_MAGIC_taint 't' +#endif + +#ifndef PERL_MAGIC_uvar +# define PERL_MAGIC_uvar 'U' +#endif + +#ifndef PERL_MAGIC_uvar_elem +# define PERL_MAGIC_uvar_elem 'u' +#endif + +#ifndef PERL_MAGIC_vstring +# define PERL_MAGIC_vstring 'V' +#endif + +#ifndef PERL_MAGIC_vec +# define PERL_MAGIC_vec 'v' +#endif + +#ifndef PERL_MAGIC_utf8 +# define PERL_MAGIC_utf8 'w' +#endif + +#ifndef PERL_MAGIC_substr +# define PERL_MAGIC_substr 'x' +#endif + +#ifndef PERL_MAGIC_defelem +# define PERL_MAGIC_defelem 'y' +#endif + +#ifndef PERL_MAGIC_glob +# define PERL_MAGIC_glob '*' +#endif + +#ifndef PERL_MAGIC_arylen +# define PERL_MAGIC_arylen '#' +#endif + +#ifndef PERL_MAGIC_pos +# define PERL_MAGIC_pos '.' +#endif + +#ifndef PERL_MAGIC_backref +# define PERL_MAGIC_backref '<' +#endif + +#ifndef PERL_MAGIC_ext +# define PERL_MAGIC_ext '~' +#endif + +/* That's the best we can do... */ +#ifndef SvPV_force_nomg +# define SvPV_force_nomg SvPV_force +#endif + +#ifndef SvPV_nomg +# define SvPV_nomg SvPV +#endif + +#ifndef sv_catpvn_nomg +# define sv_catpvn_nomg sv_catpvn +#endif + +#ifndef sv_catsv_nomg +# define sv_catsv_nomg sv_catsv +#endif + +#ifndef sv_setsv_nomg +# define sv_setsv_nomg sv_setsv +#endif + +#ifndef sv_pvn_nomg +# define sv_pvn_nomg sv_pvn +#endif + +#ifndef SvIV_nomg +# define SvIV_nomg SvIV +#endif + +#ifndef SvUV_nomg +# define SvUV_nomg SvUV +#endif + +#ifndef sv_catpv_mg +# define sv_catpv_mg(sv, ptr) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_catpv(TeMpSv,ptr); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_catpvn_mg +# define sv_catpvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_catpvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_catsv_mg +# define sv_catsv_mg(dsv, ssv) \ + STMT_START { \ + SV *TeMpSv = dsv; \ + sv_catsv(TeMpSv,ssv); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setiv_mg +# define sv_setiv_mg(sv, i) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setiv(TeMpSv,i); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setnv_mg +# define sv_setnv_mg(sv, num) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setnv(TeMpSv,num); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setpv_mg +# define sv_setpv_mg(sv, ptr) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setpv(TeMpSv,ptr); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setpvn_mg +# define sv_setpvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setpvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setsv_mg +# define sv_setsv_mg(dsv, ssv) \ + STMT_START { \ + SV *TeMpSv = dsv; \ + sv_setsv(TeMpSv,ssv); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setuv_mg +# define sv_setuv_mg(sv, i) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setuv(TeMpSv,i); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_usepvn_mg +# define sv_usepvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_usepvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifdef USE_ITHREADS +#ifndef CopFILE +# define CopFILE(c) ((c)->cop_file) +#endif + +#ifndef CopFILEGV +# define CopFILEGV(c) (CopFILE(c) ? gv_fetchfile(CopFILE(c)) : Nullgv) +#endif + +#ifndef CopFILE_set +# define CopFILE_set(c,pv) ((c)->cop_file = savepv(pv)) +#endif + +#ifndef CopFILESV +# define CopFILESV(c) (CopFILE(c) ? GvSV(gv_fetchfile(CopFILE(c))) : Nullsv) +#endif + +#ifndef CopFILEAV +# define CopFILEAV(c) (CopFILE(c) ? GvAV(gv_fetchfile(CopFILE(c))) : Nullav) +#endif + +#ifndef CopSTASHPV +# define CopSTASHPV(c) ((c)->cop_stashpv) +#endif + +#ifndef CopSTASHPV_set +# define CopSTASHPV_set(c,pv) ((c)->cop_stashpv = ((pv) ? savepv(pv) : Nullch)) +#endif + +#ifndef CopSTASH +# define CopSTASH(c) (CopSTASHPV(c) ? gv_stashpv(CopSTASHPV(c),GV_ADD) : Nullhv) +#endif + +#ifndef CopSTASH_set +# define CopSTASH_set(c,hv) CopSTASHPV_set(c, (hv) ? HvNAME(hv) : Nullch) +#endif + +#ifndef CopSTASH_eq +# define CopSTASH_eq(c,hv) ((hv) && (CopSTASHPV(c) == HvNAME(hv) \ + || (CopSTASHPV(c) && HvNAME(hv) \ + && strEQ(CopSTASHPV(c), HvNAME(hv))))) +#endif + +#else +#ifndef CopFILEGV +# define CopFILEGV(c) ((c)->cop_filegv) +#endif + +#ifndef CopFILEGV_set +# define CopFILEGV_set(c,gv) ((c)->cop_filegv = (GV*)SvREFCNT_inc(gv)) +#endif + +#ifndef CopFILE_set +# define CopFILE_set(c,pv) CopFILEGV_set((c), gv_fetchfile(pv)) +#endif + +#ifndef CopFILESV +# define CopFILESV(c) (CopFILEGV(c) ? GvSV(CopFILEGV(c)) : Nullsv) +#endif + +#ifndef CopFILEAV +# define CopFILEAV(c) (CopFILEGV(c) ? GvAV(CopFILEGV(c)) : Nullav) +#endif + +#ifndef CopFILE +# define CopFILE(c) (CopFILESV(c) ? SvPVX(CopFILESV(c)) : Nullch) +#endif + +#ifndef CopSTASH +# define CopSTASH(c) ((c)->cop_stash) +#endif + +#ifndef CopSTASH_set +# define CopSTASH_set(c,hv) ((c)->cop_stash = (hv)) +#endif + +#ifndef CopSTASHPV +# define CopSTASHPV(c) (CopSTASH(c) ? HvNAME(CopSTASH(c)) : Nullch) +#endif + +#ifndef CopSTASHPV_set +# define CopSTASHPV_set(c,pv) CopSTASH_set((c), gv_stashpv(pv,GV_ADD)) +#endif + +#ifndef CopSTASH_eq +# define CopSTASH_eq(c,hv) (CopSTASH(c) == (hv)) +#endif + +#endif /* USE_ITHREADS */ +#ifndef IN_PERL_COMPILETIME +# define IN_PERL_COMPILETIME (PL_curcop == &PL_compiling) +#endif + +#ifndef IN_LOCALE_RUNTIME +# define IN_LOCALE_RUNTIME (PL_curcop->op_private & HINT_LOCALE) +#endif + +#ifndef IN_LOCALE_COMPILETIME +# define IN_LOCALE_COMPILETIME (PL_hints & HINT_LOCALE) +#endif + +#ifndef IN_LOCALE +# define IN_LOCALE (IN_PERL_COMPILETIME ? IN_LOCALE_COMPILETIME : IN_LOCALE_RUNTIME) +#endif +#ifndef IS_NUMBER_IN_UV +# define IS_NUMBER_IN_UV 0x01 +#endif + +#ifndef IS_NUMBER_GREATER_THAN_UV_MAX +# define IS_NUMBER_GREATER_THAN_UV_MAX 0x02 +#endif + +#ifndef IS_NUMBER_NOT_INT +# define IS_NUMBER_NOT_INT 0x04 +#endif + +#ifndef IS_NUMBER_NEG +# define IS_NUMBER_NEG 0x08 +#endif + +#ifndef IS_NUMBER_INFINITY +# define IS_NUMBER_INFINITY 0x10 +#endif + +#ifndef IS_NUMBER_NAN +# define IS_NUMBER_NAN 0x20 +#endif + +/* GROK_NUMERIC_RADIX depends on grok_numeric_radix */ +#ifndef GROK_NUMERIC_RADIX +# define GROK_NUMERIC_RADIX(sp, send) grok_numeric_radix(sp, send) +#endif +#ifndef PERL_SCAN_GREATER_THAN_UV_MAX +# define PERL_SCAN_GREATER_THAN_UV_MAX 0x02 +#endif + +#ifndef PERL_SCAN_SILENT_ILLDIGIT +# define PERL_SCAN_SILENT_ILLDIGIT 0x04 +#endif + +#ifndef PERL_SCAN_ALLOW_UNDERSCORES +# define PERL_SCAN_ALLOW_UNDERSCORES 0x01 +#endif + +#ifndef PERL_SCAN_DISALLOW_PREFIX +# define PERL_SCAN_DISALLOW_PREFIX 0x02 +#endif + +#ifndef grok_numeric_radix +#if defined(NEED_grok_numeric_radix) +static bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); +static +#else +extern bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); +#endif + +#ifdef grok_numeric_radix +# undef grok_numeric_radix +#endif +#define grok_numeric_radix(a,b) DPPP_(my_grok_numeric_radix)(aTHX_ a,b) +#define Perl_grok_numeric_radix DPPP_(my_grok_numeric_radix) + +#if defined(NEED_grok_numeric_radix) || defined(NEED_grok_numeric_radix_GLOBAL) +bool +DPPP_(my_grok_numeric_radix)(pTHX_ const char **sp, const char *send) +{ +#ifdef USE_LOCALE_NUMERIC +#ifdef PL_numeric_radix_sv + if (PL_numeric_radix_sv && IN_LOCALE) { + STRLEN len; + char* radix = SvPV(PL_numeric_radix_sv, len); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#else + /* older perls don't have PL_numeric_radix_sv so the radix + * must manually be requested from locale.h + */ +#include + dTHR; /* needed for older threaded perls */ + struct lconv *lc = localeconv(); + char *radix = lc->decimal_point; + if (radix && IN_LOCALE) { + STRLEN len = strlen(radix); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#endif /* PERL_VERSION */ +#endif /* USE_LOCALE_NUMERIC */ + /* always try "." if numeric radix didn't match because + * we may have data from different locales mixed */ + if (*sp < send && **sp == '.') { + ++*sp; + return TRUE; + } + return FALSE; +} +#endif +#endif + +/* grok_number depends on grok_numeric_radix */ + +#ifndef grok_number +#if defined(NEED_grok_number) +static int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); +static +#else +extern int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); +#endif + +#ifdef grok_number +# undef grok_number +#endif +#define grok_number(a,b,c) DPPP_(my_grok_number)(aTHX_ a,b,c) +#define Perl_grok_number DPPP_(my_grok_number) + +#if defined(NEED_grok_number) || defined(NEED_grok_number_GLOBAL) +int +DPPP_(my_grok_number)(pTHX_ const char *pv, STRLEN len, UV *valuep) +{ + const char *s = pv; + const char *send = pv + len; + const UV max_div_10 = UV_MAX / 10; + const char max_mod_10 = UV_MAX % 10; + int numtype = 0; + int sawinf = 0; + int sawnan = 0; + + while (s < send && isSPACE(*s)) + s++; + if (s == send) { + return 0; + } else if (*s == '-') { + s++; + numtype = IS_NUMBER_NEG; + } + else if (*s == '+') + s++; + + if (s == send) + return 0; + + /* next must be digit or the radix separator or beginning of infinity */ + if (isDIGIT(*s)) { + /* UVs are at least 32 bits, so the first 9 decimal digits cannot + overflow. */ + UV value = *s - '0'; + /* This construction seems to be more optimiser friendly. + (without it gcc does the isDIGIT test and the *s - '0' separately) + With it gcc on arm is managing 6 instructions (6 cycles) per digit. + In theory the optimiser could deduce how far to unroll the loop + before checking for overflow. */ + if (++s < send) { + int digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + /* Now got 9 digits, so need to check + each time for overflow. */ + digit = *s - '0'; + while (digit >= 0 && digit <= 9 + && (value < max_div_10 + || (value == max_div_10 + && digit <= max_mod_10))) { + value = value * 10 + digit; + if (++s < send) + digit = *s - '0'; + else + break; + } + if (digit >= 0 && digit <= 9 + && (s < send)) { + /* value overflowed. + skip the remaining digits, don't + worry about setting *valuep. */ + do { + s++; + } while (s < send && isDIGIT(*s)); + numtype |= + IS_NUMBER_GREATER_THAN_UV_MAX; + goto skip_value; + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + numtype |= IS_NUMBER_IN_UV; + if (valuep) + *valuep = value; + + skip_value: + if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT; + while (s < send && isDIGIT(*s)) /* optional digits after the radix */ + s++; + } + } + else if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT | IS_NUMBER_IN_UV; /* valuep assigned below */ + /* no digits before the radix means we need digits after it */ + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + if (valuep) { + /* integer approximation is valid - it's 0. */ + *valuep = 0; + } + } + else + return 0; + } else if (*s == 'I' || *s == 'i') { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'F' && *s != 'f')) return 0; + s++; if (s < send && (*s == 'I' || *s == 'i')) { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'I' && *s != 'i')) return 0; + s++; if (s == send || (*s != 'T' && *s != 't')) return 0; + s++; if (s == send || (*s != 'Y' && *s != 'y')) return 0; + s++; + } + sawinf = 1; + } else if (*s == 'N' || *s == 'n') { + /* XXX TODO: There are signaling NaNs and quiet NaNs. */ + s++; if (s == send || (*s != 'A' && *s != 'a')) return 0; + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; + sawnan = 1; + } else + return 0; + + if (sawinf) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_INFINITY | IS_NUMBER_NOT_INT; + } else if (sawnan) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_NAN | IS_NUMBER_NOT_INT; + } else if (s < send) { + /* we can have an optional exponent part */ + if (*s == 'e' || *s == 'E') { + /* The only flag we keep is sign. Blow away any "it's UV" */ + numtype &= IS_NUMBER_NEG; + numtype |= IS_NUMBER_NOT_INT; + s++; + if (s < send && (*s == '-' || *s == '+')) + s++; + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + } + else + return 0; + } + } + while (s < send && isSPACE(*s)) + s++; + if (s >= send) + return numtype; + if (len == 10 && memEQ(pv, "0 but true", 10)) { + if (valuep) + *valuep = 0; + return IS_NUMBER_IN_UV; + } + return 0; +} +#endif +#endif + +/* + * The grok_* routines have been modified to use warn() instead of + * Perl_warner(). Also, 'hexdigit' was the former name of PL_hexdigit, + * which is why the stack variable has been renamed to 'xdigit'. + */ + +#ifndef grok_bin +#if defined(NEED_grok_bin) +static UV DPPP_(my_grok_bin)(pTHX_ char *start, STRLEN *len_p, I32 *flags, NV *result); +static +#else +extern UV DPPP_(my_grok_bin)(pTHX_ char *start, STRLEN *len_p, I32 *flags, NV *result); +#endif + +#ifdef grok_bin +# undef grok_bin +#endif +#define grok_bin(a,b,c,d) DPPP_(my_grok_bin)(aTHX_ a,b,c,d) +#define Perl_grok_bin DPPP_(my_grok_bin) + +#if defined(NEED_grok_bin) || defined(NEED_grok_bin_GLOBAL) +UV +DPPP_(my_grok_bin)(pTHX_ char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_2 = UV_MAX / 2; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + + if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { + /* strip off leading b or 0b. + for compatibility silently suffer "b" and "0b" as valid binary + numbers. */ + if (len >= 1) { + if (s[0] == 'b') { + s++; + len--; + } + else if (len >= 2 && s[0] == '0' && s[1] == 'b') { + s+=2; + len-=2; + } + } + } + + for (; len-- && *s; s++) { + char bit = *s; + if (bit == '0' || bit == '1') { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + With gcc seems to be much straighter code than old scan_bin. */ + redo: + if (!overflowed) { + if (value <= max_div_2) { + value = (value << 1) | (bit - '0'); + continue; + } + /* Bah. We're just overflowed. */ + warn("Integer overflow in binary number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 2.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount. */ + value_nv += (NV)(bit - '0'); + continue; + } + if (bit == '_' && len && allow_underscores && (bit = s[1]) + && (bit == '0' || bit == '1')) + { + --len; + ++s; + goto redo; + } + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal binary digit '%c' ignored", *s); + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Binary number > 0b11111111111111111111111111111111 non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#ifndef grok_hex +#if defined(NEED_grok_hex) +static UV DPPP_(my_grok_hex)(pTHX_ char *start, STRLEN *len_p, I32 *flags, NV *result); +static +#else +extern UV DPPP_(my_grok_hex)(pTHX_ char *start, STRLEN *len_p, I32 *flags, NV *result); +#endif + +#ifdef grok_hex +# undef grok_hex +#endif +#define grok_hex(a,b,c,d) DPPP_(my_grok_hex)(aTHX_ a,b,c,d) +#define Perl_grok_hex DPPP_(my_grok_hex) + +#if defined(NEED_grok_hex) || defined(NEED_grok_hex_GLOBAL) +UV +DPPP_(my_grok_hex)(pTHX_ char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_16 = UV_MAX / 16; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + const char *xdigit; + + if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { + /* strip off leading x or 0x. + for compatibility silently suffer "x" and "0x" as valid hex numbers. + */ + if (len >= 1) { + if (s[0] == 'x') { + s++; + len--; + } + else if (len >= 2 && s[0] == '0' && s[1] == 'x') { + s+=2; + len-=2; + } + } + } + + for (; len-- && *s; s++) { + xdigit = strchr((char *) PL_hexdigit, *s); + if (xdigit) { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + With gcc seems to be much straighter code than old scan_hex. */ + redo: + if (!overflowed) { + if (value <= max_div_16) { + value = (value << 4) | ((xdigit - PL_hexdigit) & 15); + continue; + } + warn("Integer overflow in hexadecimal number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 16.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount of 16-tuples. */ + value_nv += (NV)((xdigit - PL_hexdigit) & 15); + continue; + } + if (*s == '_' && len && allow_underscores && s[1] + && (xdigit = strchr((char *) PL_hexdigit, s[1]))) + { + --len; + ++s; + goto redo; + } + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal hexadecimal digit '%c' ignored", *s); + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Hexadecimal number > 0xffffffff non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#ifndef grok_oct +#if defined(NEED_grok_oct) +static UV DPPP_(my_grok_oct)(pTHX_ char *start, STRLEN *len_p, I32 *flags, NV *result); +static +#else +extern UV DPPP_(my_grok_oct)(pTHX_ char *start, STRLEN *len_p, I32 *flags, NV *result); +#endif + +#ifdef grok_oct +# undef grok_oct +#endif +#define grok_oct(a,b,c,d) DPPP_(my_grok_oct)(aTHX_ a,b,c,d) +#define Perl_grok_oct DPPP_(my_grok_oct) + +#if defined(NEED_grok_oct) || defined(NEED_grok_oct_GLOBAL) +UV +DPPP_(my_grok_oct)(pTHX_ char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_8 = UV_MAX / 8; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + + for (; len-- && *s; s++) { + /* gcc 2.95 optimiser not smart enough to figure that this subtraction + out front allows slicker code. */ + int digit = *s - '0'; + if (digit >= 0 && digit <= 7) { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + */ + redo: + if (!overflowed) { + if (value <= max_div_8) { + value = (value << 3) | digit; + continue; + } + /* Bah. We're just overflowed. */ + warn("Integer overflow in octal number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 8.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount of 8-tuples. */ + value_nv += (NV)digit; + continue; + } + if (digit == ('_' - '0') && len && allow_underscores + && (digit = s[1] - '0') && (digit >= 0 && digit <= 7)) + { + --len; + ++s; + goto redo; + } + /* Allow \octal to work the DWIM way (that is, stop scanning + * as soon as non-octal characters are seen, complain only iff + * someone seems to want to use the digits eight and nine). */ + if (digit == 8 || digit == 9) { + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal octal digit '%c' ignored", *s); + } + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Octal number > 037777777777 non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#ifdef NO_XSLOCKS +# ifdef dJMPENV +# define dXCPT dJMPENV; int rEtV = 0 +# define XCPT_TRY_START JMPENV_PUSH(rEtV); if (rEtV == 0) +# define XCPT_TRY_END JMPENV_POP; +# define XCPT_CATCH if (rEtV != 0) +# define XCPT_RETHROW JMPENV_JUMP(rEtV) +# else +# define dXCPT Sigjmp_buf oldTOP; int rEtV = 0 +# define XCPT_TRY_START Copy(top_env, oldTOP, 1, Sigjmp_buf); rEtV = Sigsetjmp(top_env, 1); if (rEtV == 0) +# define XCPT_TRY_END Copy(oldTOP, top_env, 1, Sigjmp_buf); +# define XCPT_CATCH if (rEtV != 0) +# define XCPT_RETHROW Siglongjmp(top_env, rEtV) +# endif +#endif + +#endif /* _P_P_PORTABILITY_H_ */ + +/* End of File ppport.h */ diff --git a/pragma.c b/pragma.c index 9a9dd30..4d30fa4 100644 --- a/pragma.c +++ b/pragma.c @@ -11,21 +11,15 @@ ************************************************************************* ** This file contains code used to implement the PRAGMA command. ** -** $Id: pragma.c,v 1.142 2007/06/26 10:38:55 danielk1977 Exp $ +** $Id: pragma.c,v 1.201 2009/01/13 20:14:16 drh Exp $ */ #include "sqliteInt.h" -#include "os.h" #include /* Ignore this whole file if pragmas are disabled */ #if !defined(SQLITE_OMIT_PRAGMA) && !defined(SQLITE_OMIT_PARSER) -#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) -# include "pager.h" -# include "btree.h" -#endif - /* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or @@ -36,7 +30,7 @@ ** to support legacy SQL code. The safety level used to be boolean ** and older scripts may have used numbers 0 for OFF and 1 for ON. */ -static int getSafetyLevel(const char *z){ +static u8 getSafetyLevel(const char *z){ /* 123456789 123456789 */ static const char zText[] = "onoffalseyestruefull"; static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; @@ -44,10 +38,10 @@ static int getSafetyLevel(const char *z){ static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 2}; int i, n; if( isdigit(*z) ){ - return atoi(z); + return (u8)atoi(z); } - n = strlen(z); - for(i=0; i=0&&i<=2)?i:0); + return (u8)((i>=0&&i<=2)?i:0); } #endif /* ifndef SQLITE_OMIT_AUTOVACUUM */ @@ -117,7 +111,7 @@ static int getTempStore(const char *z){ static int invalidateTempStorage(Parse *pParse){ sqlite3 *db = pParse->db; if( db->aDb[1].pBt!=0 ){ - if( !db->autoCommit ){ + if( !db->autoCommit || sqlite3BtreeIsInReadTrans(db->aDb[1].pBt) ){ sqlite3ErrorMsg(pParse, "temporary storage cannot be changed " "from within a transaction"); return SQLITE_ERROR; @@ -133,7 +127,7 @@ static int invalidateTempStorage(Parse *pParse){ #ifndef SQLITE_OMIT_PAGER_PRAGMAS /* ** If the TEMP database is open, close it and mark the database schema -** as needing reloading. This must be done when using the TEMP_STORE +** as needing reloading. This must be done when using the SQLITE_TEMP_STORE ** or DEFAULT_TEMP_STORE pragmas. */ static int changeTempStorage(Parse *pParse, const char *zStorageType){ @@ -143,7 +137,7 @@ static int changeTempStorage(Parse *pParse, const char *zStorageType){ if( invalidateTempStorage( pParse ) != SQLITE_OK ){ return SQLITE_ERROR; } - db->temp_store = ts; + db->temp_store = (u8)ts; return SQLITE_OK; } #endif /* SQLITE_PAGER_PRAGMAS */ @@ -153,12 +147,13 @@ static int changeTempStorage(Parse *pParse, const char *zStorageType){ */ static void returnSingleInt(Parse *pParse, const char *zLabel, int value){ Vdbe *v = sqlite3GetVdbe(pParse); - sqlite3VdbeAddOp(v, OP_Integer, value, 0); + int mem = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, value, mem); if( pParse->explain==0 ){ sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, P3_STATIC); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); } - sqlite3VdbeAddOp(v, OP_Callback, 1, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, mem, 1); } #ifndef SQLITE_OMIT_FLAG_PRAGMAS @@ -196,12 +191,13 @@ static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ }; int i; const struct sPragmaType *p; - for(i=0, p=aPragma; izName)==0 ){ sqlite3 *db = pParse->db; Vdbe *v; v = sqlite3GetVdbe(pParse); - if( v ){ + assert( v!=0 ); /* Already allocated by sqlite3Pragma() */ + if( ALWAYS(v) ){ if( zRight==0 ){ returnSingleInt(pParse, p->zName, (db->flags & p->mask)!=0 ); }else{ @@ -210,8 +206,15 @@ static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ }else{ db->flags &= ~p->mask; } + + /* Many of the flag-pragmas modify the code generated by the SQL + ** compiler (eg. count_changes). So add an opcode to expire all + ** compiled SQL statements after modifying a pragma value. + */ + sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); } } + return 1; } } @@ -219,6 +222,21 @@ static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ } #endif /* SQLITE_OMIT_FLAG_PRAGMAS */ +/* +** Return a human-readable name for a constraint resolution action. +*/ +static const char *actionName(u8 action){ + const char *zName; + switch( action ){ + case OE_SetNull: zName = "SET NULL"; break; + case OE_SetDflt: zName = "SET DEFAULT"; break; + case OE_Cascade: zName = "CASCADE"; break; + default: zName = "RESTRICT"; + assert( action==OE_Restrict ); break; + } + return zName; +} + /* ** Process a pragma statement. ** @@ -248,8 +266,9 @@ void sqlite3Pragma( int iDb; /* Database index for */ sqlite3 *db = pParse->db; Db *pDb; - Vdbe *v = sqlite3GetVdbe(pParse); + Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); if( v==0 ) return; + pParse->nMem = 2; /* Interpret the [database.] part of the pragma statement. iDb is the ** index of the database this pragma is being applied to in db.aDb[]. */ @@ -264,15 +283,16 @@ void sqlite3Pragma( return; } - zLeft = sqlite3NameFromToken(pId); + zLeft = sqlite3NameFromToken(db, pId); if( !zLeft ) return; if( minusFlag ){ - zRight = sqlite3MPrintf("-%T", pValue); + zRight = sqlite3MPrintf(db, "-%T", pValue); }else{ - zRight = sqlite3NameFromToken(pValue); + zRight = sqlite3NameFromToken(db, pValue); } - zDb = ((iDb>0)?pDb->zName:0); + assert( pId2 ); + zDb = pId2->n>0 ? pDb->zName : 0; if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ goto pragma_out; } @@ -296,19 +316,21 @@ void sqlite3Pragma( */ if( sqlite3StrICmp(zLeft,"default_cache_size")==0 ){ static const VdbeOpList getCacheSize[] = { - { OP_ReadCookie, 0, 2, 0}, /* 0 */ - { OP_AbsValue, 0, 0, 0}, - { OP_Dup, 0, 0, 0}, - { OP_Integer, 0, 0, 0}, - { OP_Ne, 0, 6, 0}, - { OP_Integer, 0, 0, 0}, /* 5 */ - { OP_Callback, 1, 0, 0}, + { OP_ReadCookie, 0, 1, 2}, /* 0 */ + { OP_IfPos, 1, 6, 0}, + { OP_Integer, 0, 2, 0}, + { OP_Subtract, 1, 2, 1}, + { OP_IfPos, 1, 6, 0}, + { OP_Integer, 0, 1, 0}, /* 5 */ + { OP_ResultRow, 1, 1, 0}, }; int addr; if( sqlite3ReadSchema(pParse) ) goto pragma_out; + sqlite3VdbeUsesBtree(v, iDb); if( !zRight ){ sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cache_size", P3_STATIC); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cache_size", SQLITE_STATIC); + pParse->nMem += 2; addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize); sqlite3VdbeChangeP1(v, addr, iDb); sqlite3VdbeChangeP1(v, addr+5, SQLITE_DEFAULT_CACHE_SIZE); @@ -316,12 +338,12 @@ void sqlite3Pragma( int size = atoi(zRight); if( size<0 ) size = -size; sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3VdbeAddOp(v, OP_Integer, size, 0); - sqlite3VdbeAddOp(v, OP_ReadCookie, iDb, 2); - addr = sqlite3VdbeAddOp(v, OP_Integer, 0, 0); - sqlite3VdbeAddOp(v, OP_Ge, 0, addr+3); - sqlite3VdbeAddOp(v, OP_Negative, 0, 0); - sqlite3VdbeAddOp(v, OP_SetCookie, iDb, 2); + sqlite3VdbeAddOp2(v, OP_Integer, size, 1); + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, 2, 2); + addr = sqlite3VdbeAddOp2(v, OP_IfPos, 2, 0); + sqlite3VdbeAddOp2(v, OP_Integer, -size, 1); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, 2, 1); pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); } @@ -338,11 +360,18 @@ void sqlite3Pragma( */ if( sqlite3StrICmp(zLeft,"page_size")==0 ){ Btree *pBt = pDb->pBt; + assert( pBt!=0 ); if( !zRight ){ - int size = pBt ? sqlite3BtreeGetPageSize(pBt) : 0; + int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; returnSingleInt(pParse, "page_size", size); }else{ - sqlite3BtreeSetPageSize(pBt, atoi(zRight), -1); + /* Malloc may fail when setting the page-size, as there is an internal + ** buffer that the pager module resizes using sqlite3_realloc(). + */ + db->nextPagesize = atoi(zRight); + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1) ){ + db->mallocFailed = 1; + } } }else @@ -358,15 +387,32 @@ void sqlite3Pragma( if( sqlite3StrICmp(zLeft,"max_page_count")==0 ){ Btree *pBt = pDb->pBt; int newMax = 0; + assert( pBt!=0 ); if( zRight ){ newMax = atoi(zRight); } - if( pBt ){ + if( ALWAYS(pBt) ){ newMax = sqlite3BtreeMaxPageCount(pBt, newMax); } returnSingleInt(pParse, "max_page_count", newMax); }else + /* + ** PRAGMA [database.]page_count + ** + ** Return the number of pages in the specified database. + */ + if( sqlite3StrICmp(zLeft,"page_count")==0 ){ + int iReg; + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + sqlite3CodeVerifySchema(pParse, iDb); + iReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); + sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "page_count", SQLITE_STATIC); + }else + /* ** PRAGMA [database.]locking_mode ** PRAGMA [database.]locking_mode = (normal|exclusive) @@ -398,7 +444,7 @@ void sqlite3Pragma( pPager = sqlite3BtreePager(db->aDb[ii].pBt); sqlite3PagerLockingMode(pPager, eMode); } - db->dfltLockMode = eMode; + db->dfltLockMode = (u8)eMode; } pPager = sqlite3BtreePager(pDb->pBt); eMode = sqlite3PagerLockingMode(pPager, eMode); @@ -409,31 +455,121 @@ void sqlite3Pragma( zRet = "exclusive"; } sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "locking_mode", P3_STATIC); - sqlite3VdbeOp3(v, OP_String8, 0, 0, zRet, 0); - sqlite3VdbeAddOp(v, OP_Callback, 1, 0); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "locking_mode", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zRet, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); }else + + /* + ** PRAGMA [database.]journal_mode + ** PRAGMA [database.]journal_mode = (delete|persist|off|truncate|memory) + */ + if( sqlite3StrICmp(zLeft,"journal_mode")==0 ){ + int eMode; + static char * const azModeName[] = { + "delete", "persist", "off", "truncate", "memory" + }; + + if( zRight==0 ){ + eMode = PAGER_JOURNALMODE_QUERY; + }else{ + int n = sqlite3Strlen30(zRight); + eMode = sizeof(azModeName)/sizeof(azModeName[0]) - 1; + while( eMode>=0 && sqlite3StrNICmp(zRight, azModeName[eMode], n)!=0 ){ + eMode--; + } + } + if( pId2->n==0 && eMode==PAGER_JOURNALMODE_QUERY ){ + /* Simple "PRAGMA journal_mode;" statement. This is a query for + ** the current default journal mode (which may be different to + ** the journal-mode of the main database). + */ + eMode = db->dfltJournalMode; + }else{ + Pager *pPager; + if( pId2->n==0 ){ + /* This indicates that no database name was specified as part + ** of the PRAGMA command. In this case the journal-mode must be + ** set on all attached databases, as well as the main db file. + ** + ** Also, the sqlite3.dfltJournalMode variable is set so that + ** any subsequently attached databases also use the specified + ** journal mode. + */ + int ii; + assert(pDb==&db->aDb[0]); + for(ii=1; iinDb; ii++){ + if( db->aDb[ii].pBt ){ + pPager = sqlite3BtreePager(db->aDb[ii].pBt); + sqlite3PagerJournalMode(pPager, eMode); + } + } + db->dfltJournalMode = (u8)eMode; + } + pPager = sqlite3BtreePager(pDb->pBt); + eMode = sqlite3PagerJournalMode(pPager, eMode); + } + assert( eMode==PAGER_JOURNALMODE_DELETE + || eMode==PAGER_JOURNALMODE_TRUNCATE + || eMode==PAGER_JOURNALMODE_PERSIST + || eMode==PAGER_JOURNALMODE_OFF + || eMode==PAGER_JOURNALMODE_MEMORY ); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, + azModeName[eMode], P4_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + }else + + /* + ** PRAGMA [database.]journal_size_limit + ** PRAGMA [database.]journal_size_limit=N + ** + ** Get or set the size limit on rollback journal files. + */ + if( sqlite3StrICmp(zLeft,"journal_size_limit")==0 ){ + Pager *pPager = sqlite3BtreePager(pDb->pBt); + i64 iLimit = -2; + if( zRight ){ + int iLimit32 = atoi(zRight); + if( iLimit32<-1 ){ + iLimit32 = -1; + } + iLimit = iLimit32; + } + iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); + returnSingleInt(pParse, "journal_size_limit", (int)iLimit); + }else + #endif /* SQLITE_OMIT_PAGER_PRAGMAS */ /* ** PRAGMA [database.]auto_vacuum ** PRAGMA [database.]auto_vacuum=N ** - ** Get or set the (boolean) value of the database 'auto-vacuum' parameter. + ** Get or set the value of the database 'auto-vacuum' parameter. + ** The value is one of: 0 NONE 1 FULL 2 INCREMENTAL */ #ifndef SQLITE_OMIT_AUTOVACUUM if( sqlite3StrICmp(zLeft,"auto_vacuum")==0 ){ Btree *pBt = pDb->pBt; + assert( pBt!=0 ); if( sqlite3ReadSchema(pParse) ){ goto pragma_out; } if( !zRight ){ - int auto_vacuum = - pBt ? sqlite3BtreeGetAutoVacuum(pBt) : SQLITE_DEFAULT_AUTOVACUUM; + int auto_vacuum; + if( ALWAYS(pBt) ){ + auto_vacuum = sqlite3BtreeGetAutoVacuum(pBt); + }else{ + auto_vacuum = SQLITE_DEFAULT_AUTOVACUUM; + } returnSingleInt(pParse, "auto_vacuum", auto_vacuum); }else{ int eAuto = getAutoVacuum(zRight); - if( eAuto>=0 ){ + assert( eAuto>=0 && eAuto<=2 ); + db->nextAutovac = (u8)eAuto; + if( ALWAYS(eAuto>=0) ){ /* Call SetAutoVacuum() to set initialize the internal auto and ** incr-vacuum flags. This is required in case this connection ** creates the database file. It is important that it is created @@ -448,11 +584,11 @@ void sqlite3Pragma( */ static const VdbeOpList setMeta6[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ - { OP_ReadCookie, 0, 3, 0}, /* 1 */ - { OP_If, 0, 0, 0}, /* 2 */ + { OP_ReadCookie, 0, 1, 3}, /* 1 */ + { OP_If, 1, 0, 0}, /* 2 */ { OP_Halt, SQLITE_OK, OE_Abort, 0}, /* 3 */ - { OP_Integer, 0, 0, 0}, /* 4 */ - { OP_SetCookie, 0, 6, 0}, /* 5 */ + { OP_Integer, 0, 1, 0}, /* 4 */ + { OP_SetCookie, 0, 6, 1}, /* 5 */ }; int iAddr; iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6); @@ -461,6 +597,7 @@ void sqlite3Pragma( sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4); sqlite3VdbeChangeP1(v, iAddr+4, eAuto-1); sqlite3VdbeChangeP1(v, iAddr+5, iDb); + sqlite3VdbeUsesBtree(v, iDb); } } } @@ -482,11 +619,11 @@ void sqlite3Pragma( iLimit = 0x7fffffff; } sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3VdbeAddOp(v, OP_MemInt, iLimit, 0); - addr = sqlite3VdbeAddOp(v, OP_IncrVacuum, iDb, 0); - sqlite3VdbeAddOp(v, OP_Callback, 0, 0); - sqlite3VdbeAddOp(v, OP_MemIncr, -1, 0); - sqlite3VdbeAddOp(v, OP_IfMemPos, 0, addr); + sqlite3VdbeAddOp2(v, OP_Integer, iLimit, 1); + addr = sqlite3VdbeAddOp1(v, OP_IncrVacuum, iDb); + sqlite3VdbeAddOp1(v, OP_ResultRow, 1); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); + sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr); sqlite3VdbeJumpHere(v, addr); }else #endif @@ -552,31 +689,87 @@ void sqlite3Pragma( if( sqlite3_temp_directory ){ sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, - "temp_store_directory", P3_STATIC); - sqlite3VdbeOp3(v, OP_String8, 0, 0, sqlite3_temp_directory, 0); - sqlite3VdbeAddOp(v, OP_Callback, 1, 0); + "temp_store_directory", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, sqlite3_temp_directory, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } }else{ - if( zRight[0] && !sqlite3OsIsDirWritable(zRight) ){ - sqlite3ErrorMsg(pParse, "not a writable directory"); - goto pragma_out; +#ifndef SQLITE_OMIT_WSD + if( zRight[0] ){ + int rc; + int res; + rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); + if( rc!=SQLITE_OK || res==0 ){ + sqlite3ErrorMsg(pParse, "not a writable directory"); + goto pragma_out; + } } - if( TEMP_STORE==0 - || (TEMP_STORE==1 && db->temp_store<=1) - || (TEMP_STORE==2 && db->temp_store==1) + if( SQLITE_TEMP_STORE==0 + || (SQLITE_TEMP_STORE==1 && db->temp_store<=1) + || (SQLITE_TEMP_STORE==2 && db->temp_store==1) ){ invalidateTempStorage(pParse); } - sqliteFree(sqlite3_temp_directory); + sqlite3_free(sqlite3_temp_directory); if( zRight[0] ){ - sqlite3_temp_directory = zRight; - zRight = 0; + sqlite3_temp_directory = sqlite3DbStrDup(0, zRight); }else{ sqlite3_temp_directory = 0; } +#endif /* SQLITE_OMIT_WSD */ } }else +#if !defined(SQLITE_ENABLE_LOCKING_STYLE) +# if defined(__APPLE__) +# define SQLITE_ENABLE_LOCKING_STYLE 1 +# else +# define SQLITE_ENABLE_LOCKING_STYLE 0 +# endif +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + /* + ** PRAGMA [database.]lock_proxy_file + ** PRAGMA [database.]lock_proxy_file = ":auto:"|"lock_file_path" + ** + ** Return or set the value of the lock_proxy_file flag. Changing + ** the value sets a specific file to be used for database access locks. + ** + */ + if( sqlite3StrICmp(zLeft, "lock_proxy_file")==0 ){ + if( !zRight ){ + Pager *pPager = sqlite3BtreePager(pDb->pBt); + char *proxy_file_path = NULL; + sqlite3_file *pFile = sqlite3PagerFile(pPager); + sqlite3OsFileControl(pFile, SQLITE_GET_LOCKPROXYFILE, + &proxy_file_path); + + if( proxy_file_path ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, + "lock_proxy_file", SQLITE_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, proxy_file_path, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } + }else{ + Pager *pPager = sqlite3BtreePager(pDb->pBt); + sqlite3_file *pFile = sqlite3PagerFile(pPager); + int res; + if( zRight[0] ){ + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + zRight); + } else { + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + NULL); + } + if( res!=SQLITE_OK ){ + sqlite3ErrorMsg(pParse, "failed to set lock proxy file"); + goto pragma_out; + } + } + }else +#endif /* SQLITE_ENABLE_LOCKING_STYLE */ + /* ** PRAGMA [database.]synchronous ** PRAGMA [database.]synchronous=OFF|ON|NORMAL|FULL @@ -630,12 +823,13 @@ void sqlite3Pragma( int nHidden = 0; Column *pCol; sqlite3VdbeSetNumCols(v, 6); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", P3_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", P3_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", P3_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "notnull", P3_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "dflt_value", P3_STATIC); - sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "pk", P3_STATIC); + pParse->nMem = 6; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "notnull", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "dflt_value", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "pk", SQLITE_STATIC); sqlite3ViewGetColumnNames(pParse, pTab); for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ const Token *pDflt; @@ -643,18 +837,20 @@ void sqlite3Pragma( nHidden++; continue; } - sqlite3VdbeAddOp(v, OP_Integer, i-nHidden, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, pCol->zName, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, + sqlite3VdbeAddOp2(v, OP_Integer, i-nHidden, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pCol->zName, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pCol->zType ? pCol->zType : "", 0); - sqlite3VdbeAddOp(v, OP_Integer, pCol->notNull, 0); - if( pCol->pDflt && (pDflt = &pCol->pDflt->span)->z ){ - sqlite3VdbeOp3(v, OP_String8, 0, 0, (char*)pDflt->z, pDflt->n); + sqlite3VdbeAddOp2(v, OP_Integer, (pCol->notNull ? 1 : 0), 4); + if( pCol->pDflt ){ + pDflt = &pCol->pDflt->span; + assert( pDflt->z ); + sqlite3VdbeAddOp4(v, OP_String8, 0, 5, 0, (char*)pDflt->z, pDflt->n); }else{ - sqlite3VdbeAddOp(v, OP_Null, 0, 0); + sqlite3VdbeAddOp2(v, OP_Null, 0, 5); } - sqlite3VdbeAddOp(v, OP_Integer, pCol->isPrimKey, 0); - sqlite3VdbeAddOp(v, OP_Callback, 6, 0); + sqlite3VdbeAddOp2(v, OP_Integer, pCol->isPrimKey, 6); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); } } }else @@ -668,16 +864,17 @@ void sqlite3Pragma( int i; pTab = pIdx->pTable; sqlite3VdbeSetNumCols(v, 3); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", P3_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", P3_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", P3_STATIC); + pParse->nMem = 3; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC); for(i=0; inColumn; i++){ int cnum = pIdx->aiColumn[i]; - sqlite3VdbeAddOp(v, OP_Integer, i, 0); - sqlite3VdbeAddOp(v, OP_Integer, cnum, 0); + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp2(v, OP_Integer, cnum, 2); assert( pTab->nCol>cnum ); - sqlite3VdbeOp3(v, OP_String8, 0, 0, pTab->aCol[cnum].zName, 0); - sqlite3VdbeAddOp(v, OP_Callback, 3, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pTab->aCol[cnum].zName, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } } }else @@ -693,14 +890,15 @@ void sqlite3Pragma( if( pIdx ){ int i = 0; sqlite3VdbeSetNumCols(v, 3); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", P3_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", P3_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", P3_STATIC); + pParse->nMem = 3; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC); while(pIdx){ - sqlite3VdbeAddOp(v, OP_Integer, i, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, pIdx->zName, 0); - sqlite3VdbeAddOp(v, OP_Integer, pIdx->onError!=OE_None, 0); - sqlite3VdbeAddOp(v, OP_Callback, 3, 0); + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->onError!=OE_None, 3); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); ++i; pIdx = pIdx->pNext; } @@ -712,17 +910,18 @@ void sqlite3Pragma( int i; if( sqlite3ReadSchema(pParse) ) goto pragma_out; sqlite3VdbeSetNumCols(v, 3); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", P3_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", P3_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "file", P3_STATIC); + pParse->nMem = 3; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "file", SQLITE_STATIC); for(i=0; inDb; i++){ if( db->aDb[i].pBt==0 ) continue; assert( db->aDb[i].zName!=0 ); - sqlite3VdbeAddOp(v, OP_Integer, i, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, db->aDb[i].zName, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, db->aDb[i].zName, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, sqlite3BtreeGetFilename(db->aDb[i].pBt), 0); - sqlite3VdbeAddOp(v, OP_Callback, 3, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } }else @@ -730,13 +929,14 @@ void sqlite3Pragma( int i = 0; HashElem *p; sqlite3VdbeSetNumCols(v, 2); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", P3_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", P3_STATIC); + pParse->nMem = 2; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC); for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ CollSeq *pColl = (CollSeq *)sqliteHashData(p); - sqlite3VdbeAddOp(v, OP_Integer, i++, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, pColl->zName, 0); - sqlite3VdbeAddOp(v, OP_Callback, 2, 0); + sqlite3VdbeAddOp2(v, OP_Integer, i++, 1); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pColl->zName, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } }else #endif /* SQLITE_OMIT_SCHEMA_PRAGMAS */ @@ -752,23 +952,32 @@ void sqlite3Pragma( pFK = pTab->pFKey; if( pFK ){ int i = 0; - sqlite3VdbeSetNumCols(v, 5); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", P3_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", P3_STATIC); - sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", P3_STATIC); - sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "from", P3_STATIC); - sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "to", P3_STATIC); + sqlite3VdbeSetNumCols(v, 8); + pParse->nMem = 8; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "from", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 4, COLNAME_NAME, "to", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 5, COLNAME_NAME, "on_update", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 6, COLNAME_NAME, "on_delete", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 7, COLNAME_NAME, "match", SQLITE_STATIC); while(pFK){ int j; for(j=0; jnCol; j++){ char *zCol = pFK->aCol[j].zCol; - sqlite3VdbeAddOp(v, OP_Integer, i, 0); - sqlite3VdbeAddOp(v, OP_Integer, j, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, pFK->zTo, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, - pTab->aCol[pFK->aCol[j].iFrom].zName, 0); - sqlite3VdbeOp3(v, zCol ? OP_String8 : OP_Null, 0, 0, zCol, 0); - sqlite3VdbeAddOp(v, OP_Callback, 5, 0); + char *zOnUpdate = (char *)actionName(pFK->updateConf); + char *zOnDelete = (char *)actionName(pFK->deleteConf); + sqlite3VdbeAddOp2(v, OP_Integer, i, 1); + sqlite3VdbeAddOp2(v, OP_Integer, j, 2); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pFK->zTo, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + pTab->aCol[pFK->aCol[j].iFrom].zName, 0); + sqlite3VdbeAddOp4(v, zCol ? OP_String8 : OP_Null, 0, 5, 0, zCol, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 6, 0, zOnUpdate, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 7, 0, zOnDelete, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 8, 0, "NONE", 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 8); } ++i; pFK = pFK->pNextFrom; @@ -804,7 +1013,13 @@ void sqlite3Pragma( #endif #ifndef SQLITE_OMIT_INTEGRITY_CHECK - if( sqlite3StrICmp(zLeft, "integrity_check")==0 ){ + /* Pragma "quick_check" is an experimental reduced version of + ** integrity_check designed to detect most database corruption + ** without most of the overhead of a full integrity-check. + */ + if( sqlite3StrICmp(zLeft, "integrity_check")==0 + || sqlite3StrICmp(zLeft, "quick_check")==0 + ){ int i, j, addr, mxErr; /* Code that appears at the end of the integrity check. If no error @@ -812,17 +1027,19 @@ void sqlite3Pragma( ** error message */ static const VdbeOpList endCode[] = { - { OP_MemLoad, 0, 0, 0}, - { OP_Integer, 0, 0, 0}, - { OP_Ne, 0, 0, 0}, /* 2 */ - { OP_String8, 0, 0, "ok"}, - { OP_Callback, 1, 0, 0}, + { OP_AddImm, 1, 0, 0}, /* 0 */ + { OP_IfNeg, 1, 0, 0}, /* 1 */ + { OP_String8, 0, 3, 0}, /* 2 */ + { OP_ResultRow, 3, 1, 0}, }; + int isQuick = (zLeft[0]=='q'); + /* Initialize the VDBE program */ if( sqlite3ReadSchema(pParse) ) goto pragma_out; + pParse->nMem = 6; sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "integrity_check", P3_STATIC); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "integrity_check", SQLITE_STATIC); /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; @@ -832,7 +1049,7 @@ void sqlite3Pragma( mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } } - sqlite3VdbeAddOp(v, OP_MemInt, mxErr, 0); + sqlite3VdbeAddOp2(v, OP_Integer, mxErr, 1); /* reg[1] holds errors left */ /* Do an integrity check on each database file */ for(i=0; inDb; i++){ @@ -843,100 +1060,119 @@ void sqlite3Pragma( if( OMIT_TEMPDB && i==1 ) continue; sqlite3CodeVerifySchema(pParse, i); - addr = sqlite3VdbeAddOp(v, OP_IfMemPos, 0, 0); - sqlite3VdbeAddOp(v, OP_Halt, 0, 0); + addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Halt if out of errors */ + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeJumpHere(v, addr); /* Do an integrity check of the B-Tree + ** + ** Begin by filling registers 2, 3, ... with the root pages numbers + ** for all tables and indices in the database. */ pTbls = &db->aDb[i].pSchema->tblHash; for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx; - sqlite3VdbeAddOp(v, OP_Integer, pTab->tnum, 0); + sqlite3VdbeAddOp2(v, OP_Integer, pTab->tnum, 2+cnt); cnt++; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeAddOp(v, OP_Integer, pIdx->tnum, 0); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->tnum, 2+cnt); cnt++; } } if( cnt==0 ) continue; - sqlite3VdbeAddOp(v, OP_IntegrityCk, 0, i); - addr = sqlite3VdbeAddOp(v, OP_IsNull, -1, 0); - sqlite3VdbeOp3(v, OP_String8, 0, 0, - sqlite3MPrintf("*** in database %s ***\n", db->aDb[i].zName), - P3_DYNAMIC); - sqlite3VdbeAddOp(v, OP_Pull, 1, 0); - sqlite3VdbeAddOp(v, OP_Concat, 0, 0); - sqlite3VdbeAddOp(v, OP_Callback, 1, 0); + + /* Make sure sufficient number of registers have been allocated */ + if( pParse->nMem < cnt+4 ){ + pParse->nMem = cnt+4; + } + + /* Do the b-tree integrity checks */ + sqlite3VdbeAddOp3(v, OP_IntegrityCk, 2, cnt, 1); + sqlite3VdbeChangeP5(v, (u8)i); + addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), + P4_DYNAMIC); + sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); + sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); sqlite3VdbeJumpHere(v, addr); /* Make sure all the indices are constructed correctly. */ - for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + for(x=sqliteHashFirst(pTbls); x && !isQuick; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx; int loopTop; if( pTab->pIndex==0 ) continue; - addr = sqlite3VdbeAddOp(v, OP_IfMemPos, 0, 0); - sqlite3VdbeAddOp(v, OP_Halt, 0, 0); + addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeJumpHere(v, addr); sqlite3OpenTableAndIndices(pParse, pTab, 1, OP_OpenRead); - sqlite3VdbeAddOp(v, OP_MemInt, 0, 1); - loopTop = sqlite3VdbeAddOp(v, OP_Rewind, 1, 0); - sqlite3VdbeAddOp(v, OP_MemIncr, 1, 1); + sqlite3VdbeAddOp2(v, OP_Integer, 0, 2); /* reg(2) will count entries */ + loopTop = sqlite3VdbeAddOp2(v, OP_Rewind, 1, 0); + sqlite3VdbeAddOp2(v, OP_AddImm, 2, 1); /* increment entry count */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int jmp2; static const VdbeOpList idxErr[] = { - { OP_MemIncr, -1, 0, 0}, - { OP_String8, 0, 0, "rowid "}, - { OP_Rowid, 1, 0, 0}, - { OP_String8, 0, 0, " missing from index "}, - { OP_String8, 0, 0, 0}, /* 4 */ - { OP_Concat, 2, 0, 0}, - { OP_Callback, 1, 0, 0}, + { OP_AddImm, 1, -1, 0}, + { OP_String8, 0, 3, 0}, /* 1 */ + { OP_Rowid, 1, 4, 0}, + { OP_String8, 0, 5, 0}, /* 3 */ + { OP_String8, 0, 6, 0}, /* 4 */ + { OP_Concat, 4, 3, 3}, + { OP_Concat, 5, 3, 3}, + { OP_Concat, 6, 3, 3}, + { OP_ResultRow, 3, 1, 0}, + { OP_IfPos, 1, 0, 0}, /* 9 */ + { OP_Halt, 0, 0, 0}, }; - sqlite3GenerateIndexKey(v, pIdx, 1); - jmp2 = sqlite3VdbeAddOp(v, OP_Found, j+2, 0); + sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 1); + jmp2 = sqlite3VdbeAddOp3(v, OP_Found, j+2, 0, 3); addr = sqlite3VdbeAddOpList(v, ArraySize(idxErr), idxErr); - sqlite3VdbeChangeP3(v, addr+4, pIdx->zName, P3_STATIC); + sqlite3VdbeChangeP4(v, addr+1, "rowid ", P4_STATIC); + sqlite3VdbeChangeP4(v, addr+3, " missing from index ", P4_STATIC); + sqlite3VdbeChangeP4(v, addr+4, pIdx->zName, P4_STATIC); + sqlite3VdbeJumpHere(v, addr+9); sqlite3VdbeJumpHere(v, jmp2); } - sqlite3VdbeAddOp(v, OP_Next, 1, loopTop+1); + sqlite3VdbeAddOp2(v, OP_Next, 1, loopTop+1); sqlite3VdbeJumpHere(v, loopTop); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ static const VdbeOpList cntIdx[] = { - { OP_MemInt, 0, 2, 0}, + { OP_Integer, 0, 3, 0}, { OP_Rewind, 0, 0, 0}, /* 1 */ - { OP_MemIncr, 1, 2, 0}, + { OP_AddImm, 3, 1, 0}, { OP_Next, 0, 0, 0}, /* 3 */ - { OP_MemLoad, 1, 0, 0}, - { OP_MemLoad, 2, 0, 0}, - { OP_Eq, 0, 0, 0}, /* 6 */ - { OP_MemIncr, -1, 0, 0}, - { OP_String8, 0, 0, "wrong # of entries in index "}, - { OP_String8, 0, 0, 0}, /* 9 */ - { OP_Concat, 0, 0, 0}, - { OP_Callback, 1, 0, 0}, + { OP_Eq, 2, 0, 3}, /* 4 */ + { OP_AddImm, 1, -1, 0}, + { OP_String8, 0, 2, 0}, /* 6 */ + { OP_String8, 0, 3, 0}, /* 7 */ + { OP_Concat, 3, 2, 2}, + { OP_ResultRow, 2, 1, 0}, }; if( pIdx->tnum==0 ) continue; - addr = sqlite3VdbeAddOp(v, OP_IfMemPos, 0, 0); - sqlite3VdbeAddOp(v, OP_Halt, 0, 0); + addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeJumpHere(v, addr); addr = sqlite3VdbeAddOpList(v, ArraySize(cntIdx), cntIdx); sqlite3VdbeChangeP1(v, addr+1, j+2); sqlite3VdbeChangeP2(v, addr+1, addr+4); sqlite3VdbeChangeP1(v, addr+3, j+2); sqlite3VdbeChangeP2(v, addr+3, addr+2); - sqlite3VdbeJumpHere(v, addr+6); - sqlite3VdbeChangeP3(v, addr+9, pIdx->zName, P3_STATIC); + sqlite3VdbeJumpHere(v, addr+4); + sqlite3VdbeChangeP4(v, addr+6, + "wrong # of entries in index ", P4_STATIC); + sqlite3VdbeChangeP4(v, addr+7, pIdx->zName, P4_STATIC); } } } addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode); - sqlite3VdbeChangeP1(v, addr+1, mxErr); - sqlite3VdbeJumpHere(v, addr+2); + sqlite3VdbeChangeP2(v, addr, -mxErr); + sqlite3VdbeJumpHere(v, addr+1); + sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC); }else #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -945,7 +1181,7 @@ void sqlite3Pragma( ** PRAGMA encoding ** PRAGMA encoding = "utf-8"|"utf-16"|"utf-16le"|"utf-16be" ** - ** In it's first form, this pragma returns the encoding of the main + ** In its first form, this pragma returns the encoding of the main ** database. If the database is not initialized, it is initialized now. ** ** The second form of this pragma is a no-op if the main database file @@ -968,11 +1204,11 @@ void sqlite3Pragma( char *zName; u8 enc; } encnames[] = { - { "UTF-8", SQLITE_UTF8 }, { "UTF8", SQLITE_UTF8 }, - { "UTF-16le", SQLITE_UTF16LE }, + { "UTF-8", SQLITE_UTF8 }, /* Must be element [1] */ + { "UTF-16le", SQLITE_UTF16LE }, /* Must be element [2] */ + { "UTF-16be", SQLITE_UTF16BE }, /* Must be element [3] */ { "UTF16le", SQLITE_UTF16LE }, - { "UTF-16be", SQLITE_UTF16BE }, { "UTF16be", SQLITE_UTF16BE }, { "UTF-16", 0 }, /* SQLITE_UTF16NATIVE */ { "UTF16", 0 }, /* SQLITE_UTF16NATIVE */ @@ -982,15 +1218,13 @@ void sqlite3Pragma( if( !zRight ){ /* "PRAGMA encoding" */ if( sqlite3ReadSchema(pParse) ) goto pragma_out; sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "encoding", P3_STATIC); - sqlite3VdbeAddOp(v, OP_String8, 0, 0); - for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ - if( pEnc->enc==ENC(pParse->db) ){ - sqlite3VdbeChangeP3(v, -1, pEnc->zName, P3_STATIC); - break; - } - } - sqlite3VdbeAddOp(v, OP_Callback, 1, 0); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "encoding", SQLITE_STATIC); + sqlite3VdbeAddOp2(v, OP_String8, 0, 1); + assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); + assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); + assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); + sqlite3VdbeChangeP4(v, -1, encnames[ENC(pParse->db)].zName, P4_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); }else{ /* "PRAGMA encoding = XXX" */ /* Only change the value of sqlite.enc if the database handle is not ** initialized. If the main database exists, the new sqlite.enc value @@ -1045,8 +1279,8 @@ void sqlite3Pragma( || sqlite3StrICmp(zLeft, "user_version")==0 || sqlite3StrICmp(zLeft, "freelist_count")==0 ){ - int iCookie; /* Cookie index. 0 for schema-cookie, 6 for user-cookie. */ + sqlite3VdbeUsesBtree(v, iDb); switch( zLeft[0] ){ case 's': case 'S': iCookie = 0; @@ -1065,8 +1299,8 @@ void sqlite3Pragma( /* Write the specified cookie value */ static const VdbeOpList setCookie[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ - { OP_Integer, 0, 0, 0}, /* 1 */ - { OP_SetCookie, 0, 0, 0}, /* 2 */ + { OP_Integer, 0, 1, 0}, /* 1 */ + { OP_SetCookie, 0, 0, 1}, /* 2 */ }; int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie); sqlite3VdbeChangeP1(v, addr, iDb); @@ -1076,14 +1310,14 @@ void sqlite3Pragma( }else{ /* Read the specified cookie value */ static const VdbeOpList readCookie[] = { - { OP_ReadCookie, 0, 0, 0}, /* 0 */ - { OP_Callback, 1, 0, 0} + { OP_ReadCookie, 0, 1, 0}, /* 0 */ + { OP_ResultRow, 1, 1, 0} }; int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie); sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP2(v, addr, iCookie); + sqlite3VdbeChangeP3(v, addr, iCookie); sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, P3_TRANSIENT); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); } }else #endif /* SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS */ @@ -1097,25 +1331,28 @@ void sqlite3Pragma( "unlocked", "shared", "reserved", "pending", "exclusive" }; int i; - Vdbe *v = sqlite3GetVdbe(pParse); sqlite3VdbeSetNumCols(v, 2); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "database", P3_STATIC); - sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", P3_STATIC); + pParse->nMem = 2; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "database", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "status", SQLITE_STATIC); for(i=0; inDb; i++){ Btree *pBt; Pager *pPager; + const char *zState = "unknown"; + int j; if( db->aDb[i].zName==0 ) continue; - sqlite3VdbeOp3(v, OP_String8, 0, 0, db->aDb[i].zName, P3_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, db->aDb[i].zName, P4_STATIC); pBt = db->aDb[i].pBt; if( pBt==0 || (pPager = sqlite3BtreePager(pBt))==0 ){ - sqlite3VdbeOp3(v, OP_String8, 0, 0, "closed", P3_STATIC); - }else{ - int j = sqlite3PagerLockstate(pPager); - sqlite3VdbeOp3(v, OP_String8, 0, 0, - (j>=0 && j<=4) ? azLockName[j] : "unknown", P3_STATIC); + zState = "closed"; + }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, + SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ + zState = azLockName[j]; } - sqlite3VdbeAddOp(v, OP_Callback, 2, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, zState, P4_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } + }else #endif @@ -1131,8 +1368,26 @@ void sqlite3Pragma( #endif #if SQLITE_HAS_CODEC - if( sqlite3StrICmp(zLeft, "key")==0 ){ - sqlite3_key(db, zRight, strlen(zRight)); + if( sqlite3StrICmp(zLeft, "key")==0 && zRight ){ + sqlite3_key(db, zRight, sqlite3Strlen30(zRight)); + }else + if( sqlite3StrICmp(zLeft, "rekey")==0 && zRight ){ + sqlite3_rekey(db, zRight, sqlite3Strlen30(zRight)); + }else + if( zRight && (sqlite3StrICmp(zLeft, "hexkey")==0 || + sqlite3StrICmp(zLeft, "hexrekey")==0) ){ + int i, h1, h2; + char zKey[40]; + for(i=0; (h1 = zRight[i])!=0 && (h2 = zRight[i+1])!=0; i+=2){ + h1 += 9*(1&(h1>>6)); + h2 += 9*(1&(h2>>6)); + zKey[i/2] = (h2 & 0x0f) | ((h1 & 0xf)<<4); + } + if( (zLeft[3] & 0xf)==0xb ){ + sqlite3_key(db, zKey, i/2); + }else{ + sqlite3_rekey(db, zKey, i/2); + } }else #endif #if SQLITE_HAS_CODEC || defined(SQLITE_ENABLE_CEROD) @@ -1149,32 +1404,31 @@ void sqlite3Pragma( sqlite3_activate_cerod(&zRight[6]); } #endif - } + }else #endif - {} + + {/* Empty ELSE clause */} - if( v ){ - /* Code an OP_Expire at the end of each PRAGMA program to cause - ** the VDBE implementing the pragma to expire. Most (all?) pragmas - ** are only valid for a single execution. - */ - sqlite3VdbeAddOp(v, OP_Expire, 1, 0); + /* Code an OP_Expire at the end of each PRAGMA program to cause + ** the VDBE implementing the pragma to expire. Most (all?) pragmas + ** are only valid for a single execution. + */ + sqlite3VdbeAddOp2(v, OP_Expire, 1, 0); - /* - ** Reset the safety level, in case the fullfsync flag or synchronous - ** setting changed. - */ + /* + ** Reset the safety level, in case the fullfsync flag or synchronous + ** setting changed. + */ #ifndef SQLITE_OMIT_PAGER_PRAGMAS - if( db->autoCommit ){ - sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level, - (db->flags&SQLITE_FullFSync)!=0); - } -#endif + if( db->autoCommit ){ + sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level, + (db->flags&SQLITE_FullFSync)!=0); } +#endif pragma_out: - sqliteFree(zLeft); - sqliteFree(zRight); + sqlite3DbFree(db, zLeft); + sqlite3DbFree(db, zRight); } #endif /* SQLITE_OMIT_PRAGMA || SQLITE_OMIT_PARSER */ diff --git a/prepare.c b/prepare.c index 4a040eb..38faaf7 100644 --- a/prepare.c +++ b/prepare.c @@ -13,20 +13,29 @@ ** interface, and routines that contribute to loading the database schema ** from disk. ** -** $Id: prepare.c,v 1.52 2007/08/13 14:41:19 danielk1977 Exp $ +** $Id: prepare.c,v 1.104 2009/01/09 02:49:32 drh Exp $ */ #include "sqliteInt.h" -#include "os.h" #include /* ** Fill the InitData structure with an error message that indicates ** that the database is corrupt. */ -static void corruptSchema(InitData *pData, const char *zExtra){ - if( !sqlite3MallocFailed() ){ - sqlite3SetString(pData->pzErrMsg, "malformed database schema", - zExtra!=0 && zExtra[0]!=0 ? " - " : (char*)0, zExtra, (char*)0); +static void corruptSchema( + InitData *pData, /* Initialization context */ + const char *zObj, /* Object being parsed at the point of error */ + const char *zExtra /* Error information */ +){ + sqlite3 *db = pData->db; + if( !db->mallocFailed && (db->flags & SQLITE_RecoveryMode)==0 ){ + if( zObj==0 ) zObj = "?"; + sqlite3SetString(pData->pzErrMsg, pData->db, + "malformed database schema (%s)", zObj); + if( zExtra && zExtra[0] ){ + *pData->pzErrMsg = sqlite3MAppendf(pData->db, *pData->pzErrMsg, "%s - %s", + *pData->pzErrMsg, zExtra); + } } pData->rc = SQLITE_CORRUPT; } @@ -43,26 +52,25 @@ static void corruptSchema(InitData *pData, const char *zExtra){ ** argv[2] = SQL text for the CREATE statement. ** */ -int sqlite3InitCallback(void *pInit, int argc, char **argv, char **azColName){ +int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ InitData *pData = (InitData*)pInit; sqlite3 *db = pData->db; int iDb = pData->iDb; - pData->rc = SQLITE_OK; + assert( argc==3 ); + UNUSED_PARAMETER2(NotUsed, argc); + assert( sqlite3_mutex_held(db->mutex) ); DbClearProperty(db, iDb, DB_Empty); - if( sqlite3MallocFailed() ){ - corruptSchema(pData, 0); + if( db->mallocFailed ){ + corruptSchema(pData, argv[0], 0); return SQLITE_NOMEM; } - assert( argc==3 ); + assert( iDb>=0 && iDbnDb ); if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ if( argv[1]==0 ){ - corruptSchema(pData, 0); - return 1; - } - assert( iDb>=0 && iDbnDb ); - if( argv[2] && argv[2][0] ){ + corruptSchema(pData, argv[0], 0); + }else if( argv[2] && argv[2][0] ){ /* Call the parser to process a CREATE TABLE, INDEX or VIEW. ** But because db->init.busy is set to 1, no VDBE code is generated ** or executed. All the parser does is build the internal data @@ -70,22 +78,27 @@ int sqlite3InitCallback(void *pInit, int argc, char **argv, char **azColName){ */ char *zErr; int rc; + u8 lookasideEnabled; assert( db->init.busy ); db->init.iDb = iDb; db->init.newTnum = atoi(argv[1]); + lookasideEnabled = db->lookaside.bEnabled; + db->lookaside.bEnabled = 0; rc = sqlite3_exec(db, argv[2], 0, 0, &zErr); db->init.iDb = 0; + db->lookaside.bEnabled = lookasideEnabled; assert( rc!=SQLITE_OK || zErr==0 ); if( SQLITE_OK!=rc ){ pData->rc = rc; if( rc==SQLITE_NOMEM ){ - sqlite3FailedMalloc(); + db->mallocFailed = 1; }else if( rc!=SQLITE_INTERRUPT ){ - corruptSchema(pData, zErr); + corruptSchema(pData, argv[0], zErr); } - sqlite3_free(zErr); - return 1; + sqlite3DbFree(db, zErr); } + }else if( argv[0]==0 ){ + corruptSchema(pData, 0, 0); }else{ /* If the SQL column is blank it means this is an index that ** was created to be the PRIMARY KEY or to fulfill a UNIQUE @@ -157,6 +170,8 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pSchema ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); /* zMasterSchema and zInitScript are set to point at the master schema ** and initialisation script appropriate for the database being @@ -170,24 +185,25 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ zMasterName = SCHEMA_TABLE(iDb); /* Construct the schema tables. */ - sqlite3SafetyOff(db); azArg[0] = zMasterName; azArg[1] = "1"; azArg[2] = zMasterSchema; azArg[3] = 0; initData.db = db; initData.iDb = iDb; + initData.rc = SQLITE_OK; initData.pzErrMsg = pzErrMsg; - rc = sqlite3InitCallback(&initData, 3, (char **)azArg, 0); - if( rc ){ - sqlite3SafetyOn(db); - return initData.rc; + (void)sqlite3SafetyOff(db); + sqlite3InitCallback(&initData, 3, (char **)azArg, 0); + (void)sqlite3SafetyOn(db); + if( initData.rc ){ + rc = initData.rc; + goto error_out; } pTab = sqlite3FindTable(db, zMasterName, db->aDb[iDb].zName); if( pTab ){ - pTab->readOnly = 1; + pTab->tabFlags |= TF_Readonly; } - sqlite3SafetyOn(db); /* Create a cursor to hold the database open */ @@ -198,10 +214,16 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ } return SQLITE_OK; } - rc = sqlite3BtreeCursor(pDb->pBt, MASTER_ROOT, 0, 0, 0, &curMain); + curMain = sqlite3MallocZero(sqlite3BtreeCursorSize()); + if( !curMain ){ + rc = SQLITE_NOMEM; + goto error_out; + } + sqlite3BtreeEnter(pDb->pBt); + rc = sqlite3BtreeCursor(pDb->pBt, MASTER_ROOT, 0, 0, curMain); if( rc!=SQLITE_OK && rc!=SQLITE_EMPTY ){ - sqlite3SetString(pzErrMsg, sqlite3ErrStr(rc), (char*)0); - return rc; + sqlite3SetString(pzErrMsg, db, "%s", sqlite3ErrStr(rc)); + goto initone_error_out; } /* Get the database meta information. @@ -223,13 +245,12 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ */ if( rc==SQLITE_OK ){ int i; - for(i=0; rc==SQLITE_OK && ipBt, i+1, (u32 *)&meta[i]); - } - if( rc ){ - sqlite3SetString(pzErrMsg, sqlite3ErrStr(rc), (char*)0); - sqlite3BtreeCloseCursor(curMain); - return rc; + if( rc ){ + sqlite3SetString(pzErrMsg, db, "%s", sqlite3ErrStr(rc)); + goto initone_error_out; + } } }else{ memset(meta, 0, sizeof(meta)); @@ -249,10 +270,10 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ }else{ /* If opening an attached database, the encoding much match ENC(db) */ if( meta[4]!=ENC(db) ){ - sqlite3BtreeCloseCursor(curMain); - sqlite3SetString(pzErrMsg, "attached databases must use the same" - " text encoding as main database", (char*)0); - return SQLITE_ERROR; + sqlite3SetString(pzErrMsg, db, "attached databases must use the same" + " text encoding as main database"); + rc = SQLITE_ERROR; + goto initone_error_out; } } }else{ @@ -260,10 +281,13 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ } pDb->pSchema->enc = ENC(db); - size = meta[2]; - if( size==0 ){ size = SQLITE_DEFAULT_CACHE_SIZE; } - pDb->pSchema->cache_size = size; - sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + if( pDb->pSchema->cache_size==0 ){ + size = meta[2]; + if( size==0 ){ size = SQLITE_DEFAULT_CACHE_SIZE; } + if( size<0 ) size = -size; + pDb->pSchema->cache_size = size; + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } /* ** file_format==1 Version 3.0.0. @@ -271,16 +295,24 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ ** file_format==3 Version 3.1.4. // ditto but with non-NULL defaults ** file_format==4 Version 3.3.0. // DESC indices. Boolean constants */ - pDb->pSchema->file_format = meta[1]; + pDb->pSchema->file_format = (u8)meta[1]; if( pDb->pSchema->file_format==0 ){ pDb->pSchema->file_format = 1; } if( pDb->pSchema->file_format>SQLITE_MAX_FILE_FORMAT ){ - sqlite3BtreeCloseCursor(curMain); - sqlite3SetString(pzErrMsg, "unsupported file format", (char*)0); - return SQLITE_ERROR; + sqlite3SetString(pzErrMsg, db, "unsupported file format"); + rc = SQLITE_ERROR; + goto initone_error_out; } + /* Ticket #2804: When we open a database in the newer file format, + ** clear the legacy_file_format pragma flag so that a VACUUM will + ** not downgrade the database and thus invalidate any descending + ** indices that the user might have created. + */ + if( iDb==0 && meta[1]>=4 ){ + db->flags &= ~SQLITE_LegacyFileFmt; + } /* Read the schema information out of the schema tables */ @@ -290,23 +322,31 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ rc = SQLITE_OK; }else{ char *zSql; - zSql = sqlite3MPrintf( + zSql = sqlite3MPrintf(db, "SELECT name, rootpage, sql FROM '%q'.%s", db->aDb[iDb].zName, zMasterName); - sqlite3SafetyOff(db); - rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0); - if( rc==SQLITE_ABORT ) rc = initData.rc; - sqlite3SafetyOn(db); - sqliteFree(zSql); + (void)sqlite3SafetyOff(db); +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + xAuth = db->xAuth; + db->xAuth = 0; +#endif + rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; + } +#endif + if( rc==SQLITE_OK ) rc = initData.rc; + (void)sqlite3SafetyOn(db); + sqlite3DbFree(db, zSql); #ifndef SQLITE_OMIT_ANALYZE if( rc==SQLITE_OK ){ sqlite3AnalysisLoad(db, iDb); } #endif - sqlite3BtreeCloseCursor(curMain); } - if( sqlite3MallocFailed() ){ - /* sqlite3SetString(pzErrMsg, "out of memory", (char*)0); */ + if( db->mallocFailed ){ rc = SQLITE_NOMEM; sqlite3ResetInternalSchema(db, 0); } @@ -317,11 +357,25 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ ** will attempt to compile the supplied statement against whatever subset ** of the schema was loaded before the error occured. The primary ** purpose of this is to allow access to the sqlite_master table - ** even when it's contents have been corrupted. + ** even when its contents have been corrupted. */ DbSetProperty(db, iDb, DB_SchemaLoaded); rc = SQLITE_OK; } + + /* Jump here for an error that occurs after successfully allocating + ** curMain and calling sqlite3BtreeEnter(). For an error that occurs + ** before that point, jump to error_out. + */ +initone_error_out: + sqlite3BtreeCloseCursor(curMain); + sqlite3_free(curMain); + sqlite3BtreeLeave(pDb->pBt); + +error_out: + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + db->mallocFailed = 1; + } return rc; } @@ -339,6 +393,7 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ int i, rc; int commit_internal = !(db->flags&SQLITE_InternChanges); + assert( sqlite3_mutex_held(db->mutex) ); if( db->init.busy ) return SQLITE_OK; rc = SQLITE_OK; db->init.busy = 1; @@ -378,6 +433,7 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ int sqlite3ReadSchema(Parse *pParse){ int rc = SQLITE_OK; sqlite3 *db = pParse->db; + assert( sqlite3_mutex_held(db->mutex) ); if( !db->init.busy ){ rc = sqlite3Init(db, &pParse->zErrMsg); } @@ -400,19 +456,32 @@ static int schemaIsValid(sqlite3 *db){ int cookie; int allOk = 1; - for(iDb=0; allOk && iDbnDb; iDb++){ - Btree *pBt; - pBt = db->aDb[iDb].pBt; - if( pBt==0 ) continue; - rc = sqlite3BtreeCursor(pBt, MASTER_ROOT, 0, 0, 0, &curTemp); - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeGetMeta(pBt, 1, (u32 *)&cookie); - if( rc==SQLITE_OK && cookie!=db->aDb[iDb].pSchema->schema_cookie ){ - allOk = 0; + curTemp = (BtCursor *)sqlite3Malloc(sqlite3BtreeCursorSize()); + if( curTemp ){ + assert( sqlite3_mutex_held(db->mutex) ); + for(iDb=0; allOk && iDbnDb; iDb++){ + Btree *pBt; + pBt = db->aDb[iDb].pBt; + if( pBt==0 ) continue; + memset(curTemp, 0, sqlite3BtreeCursorSize()); + rc = sqlite3BtreeCursor(pBt, MASTER_ROOT, 0, 0, curTemp); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeGetMeta(pBt, 1, (u32 *)&cookie); + if( rc==SQLITE_OK && cookie!=db->aDb[iDb].pSchema->schema_cookie ){ + allOk = 0; + } + sqlite3BtreeCloseCursor(curTemp); + } + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + db->mallocFailed = 1; } - sqlite3BtreeCloseCursor(curTemp); } + sqlite3_free(curTemp); + }else{ + allOk = 0; + db->mallocFailed = 1; } + return allOk; } @@ -432,17 +501,18 @@ int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ ** function should never be used. ** ** We return -1000000 instead of the more usual -1 simply because using - ** -1000000 as incorrectly using -1000000 index into db->aDb[] is much + ** -1000000 as the incorrect index into db->aDb[] is much ** more likely to cause a segfault than -1 (of course there are assert() ** statements too, but it never hurts to play the odds). */ + assert( sqlite3_mutex_held(db->mutex) ); if( pSchema ){ - for(i=0; inDb; i++){ + for(i=0; ALWAYS(inDb); i++){ if( db->aDb[i].pSchema==pSchema ){ break; } } - assert( i>=0 &&i>=0 && inDb ); + assert( i>=0 && inDb ); } return i; } @@ -450,7 +520,7 @@ int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ /* ** Compile the UTF-8 encoded SQL statement zSql into a statement handle. */ -int sqlite3Prepare( +static int sqlite3Prepare( sqlite3 *db, /* Database handle. */ const char *zSql, /* UTF-8 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ @@ -463,46 +533,53 @@ int sqlite3Prepare( int rc = SQLITE_OK; int i; - /* Assert that malloc() has not failed */ - assert( !sqlite3MallocFailed() ); - assert( ppStmt ); *ppStmt = 0; if( sqlite3SafetyOn(db) ){ return SQLITE_MISUSE; } + assert( !db->mallocFailed ); + assert( sqlite3_mutex_held(db->mutex) ); /* If any attached database schemas are locked, do not proceed with ** compilation. Instead return SQLITE_LOCKED immediately. */ for(i=0; inDb; i++) { Btree *pBt = db->aDb[i].pBt; - if( pBt && sqlite3BtreeSchemaLocked(pBt) ){ - const char *zDb = db->aDb[i].zName; - sqlite3Error(db, SQLITE_LOCKED, "database schema is locked: %s", zDb); - sqlite3SafetyOff(db); - return SQLITE_LOCKED; + if( pBt ){ + rc = sqlite3BtreeSchemaLocked(pBt); + if( rc ){ + const char *zDb = db->aDb[i].zName; + sqlite3Error(db, SQLITE_LOCKED, "database schema is locked: %s", zDb); + (void)sqlite3SafetyOff(db); + return sqlite3ApiExit(db, SQLITE_LOCKED); + } } } memset(&sParse, 0, sizeof(sParse)); sParse.db = db; - if( nBytes>=0 && zSql[nBytes]!=0 ){ + if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; - if( nBytes>SQLITE_MAX_SQL_LENGTH ){ - return SQLITE_TOOBIG; + int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; + if( nBytes>mxLen ){ + sqlite3Error(db, SQLITE_TOOBIG, "statement too long"); + (void)sqlite3SafetyOff(db); + return sqlite3ApiExit(db, SQLITE_TOOBIG); } - zSqlCopy = sqlite3StrNDup(zSql, nBytes); + zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); if( zSqlCopy ){ sqlite3RunParser(&sParse, zSqlCopy, &zErrMsg); - sqliteFree(zSqlCopy); + sqlite3DbFree(db, zSqlCopy); + sParse.zTail = &zSql[sParse.zTail-zSqlCopy]; + }else{ + sParse.zTail = &zSql[nBytes]; } - sParse.zTail = &zSql[nBytes]; }else{ sqlite3RunParser(&sParse, zSql, &zErrMsg); } - if( sqlite3MallocFailed() ){ + if( db->mallocFailed ){ sParse.rc = SQLITE_NOMEM; } if( sParse.rc==SQLITE_DONE ) sParse.rc = SQLITE_OK; @@ -512,7 +589,7 @@ int sqlite3Prepare( if( sParse.rc==SQLITE_SCHEMA ){ sqlite3ResetInternalSchema(db, 0); } - if( sqlite3MallocFailed() ){ + if( db->mallocFailed ){ sParse.rc = SQLITE_NOMEM; } if( pzTail ){ @@ -524,16 +601,19 @@ int sqlite3Prepare( if( rc==SQLITE_OK && sParse.pVdbe && sParse.explain ){ if( sParse.explain==2 ){ sqlite3VdbeSetNumCols(sParse.pVdbe, 3); - sqlite3VdbeSetColName(sParse.pVdbe, 0, COLNAME_NAME, "order", P3_STATIC); - sqlite3VdbeSetColName(sParse.pVdbe, 1, COLNAME_NAME, "from", P3_STATIC); - sqlite3VdbeSetColName(sParse.pVdbe, 2, COLNAME_NAME, "detail", P3_STATIC); + sqlite3VdbeSetColName(sParse.pVdbe, 0, COLNAME_NAME, "order", SQLITE_STATIC); + sqlite3VdbeSetColName(sParse.pVdbe, 1, COLNAME_NAME, "from", SQLITE_STATIC); + sqlite3VdbeSetColName(sParse.pVdbe, 2, COLNAME_NAME, "detail", SQLITE_STATIC); }else{ - sqlite3VdbeSetNumCols(sParse.pVdbe, 5); - sqlite3VdbeSetColName(sParse.pVdbe, 0, COLNAME_NAME, "addr", P3_STATIC); - sqlite3VdbeSetColName(sParse.pVdbe, 1, COLNAME_NAME, "opcode", P3_STATIC); - sqlite3VdbeSetColName(sParse.pVdbe, 2, COLNAME_NAME, "p1", P3_STATIC); - sqlite3VdbeSetColName(sParse.pVdbe, 3, COLNAME_NAME, "p2", P3_STATIC); - sqlite3VdbeSetColName(sParse.pVdbe, 4, COLNAME_NAME, "p3", P3_STATIC); + sqlite3VdbeSetNumCols(sParse.pVdbe, 8); + sqlite3VdbeSetColName(sParse.pVdbe, 0, COLNAME_NAME, "addr", SQLITE_STATIC); + sqlite3VdbeSetColName(sParse.pVdbe, 1, COLNAME_NAME, "opcode", SQLITE_STATIC); + sqlite3VdbeSetColName(sParse.pVdbe, 2, COLNAME_NAME, "p1", SQLITE_STATIC); + sqlite3VdbeSetColName(sParse.pVdbe, 3, COLNAME_NAME, "p2", SQLITE_STATIC); + sqlite3VdbeSetColName(sParse.pVdbe, 4, COLNAME_NAME, "p3", SQLITE_STATIC); + sqlite3VdbeSetColName(sParse.pVdbe, 5, COLNAME_NAME, "p4", SQLITE_STATIC); + sqlite3VdbeSetColName(sParse.pVdbe, 6, COLNAME_NAME, "p5", SQLITE_STATIC); + sqlite3VdbeSetColName(sParse.pVdbe, 7, COLNAME_NAME, "comment", SQLITE_STATIC); } } #endif @@ -543,9 +623,9 @@ int sqlite3Prepare( } if( saveSqlFlag ){ - sqlite3VdbeSetSql(sParse.pVdbe, zSql, sParse.zTail - zSql); + sqlite3VdbeSetSql(sParse.pVdbe, zSql, (int)(sParse.zTail - zSql)); } - if( rc!=SQLITE_OK || sqlite3MallocFailed() ){ + if( rc!=SQLITE_OK || db->mallocFailed ){ sqlite3_finalize((sqlite3_stmt*)sParse.pVdbe); assert(!(*ppStmt)); }else{ @@ -554,16 +634,34 @@ int sqlite3Prepare( if( zErrMsg ){ sqlite3Error(db, rc, "%s", zErrMsg); - sqliteFree(zErrMsg); + sqlite3DbFree(db, zErrMsg); }else{ sqlite3Error(db, rc, 0); } rc = sqlite3ApiExit(db, rc); - sqlite3ReleaseThreadData(); assert( (rc&db->errMask)==rc ); return rc; } +static int sqlite3LockAndPrepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + int saveSqlFlag, /* True to copy SQL text into the sqlite3_stmt */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE; + } + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, ppStmt, pzTail); + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return rc; +} /* ** Rerun the compilation of a statement after a schema change. @@ -575,21 +673,24 @@ int sqlite3Reprepare(Vdbe *p){ sqlite3_stmt *pNew; const char *zSql; sqlite3 *db; - - zSql = sqlite3VdbeGetSql(p); - if( zSql==0 ){ - return 0; - } + + assert( sqlite3_mutex_held(sqlite3VdbeDb(p)->mutex) ); + zSql = sqlite3_sql((sqlite3_stmt *)p); + assert( zSql!=0 ); /* Reprepare only called for prepare_v2() statements */ db = sqlite3VdbeDb(p); - rc = sqlite3Prepare(db, zSql, -1, 0, &pNew, 0); + assert( sqlite3_mutex_held(db->mutex) ); + rc = sqlite3LockAndPrepare(db, zSql, -1, 0, &pNew, 0); if( rc ){ + if( rc==SQLITE_NOMEM ){ + db->mallocFailed = 1; + } assert( pNew==0 ); return 0; }else{ assert( pNew!=0 ); } sqlite3VdbeSwap((Vdbe*)pNew, p); - sqlite3_transfer_bindings(pNew, (sqlite3_stmt*)p); + sqlite3TransferBindings(pNew, (sqlite3_stmt*)p); sqlite3VdbeResetStepResult((Vdbe*)pNew); sqlite3VdbeFinalize((Vdbe*)pNew); return 1; @@ -611,7 +712,10 @@ int sqlite3_prepare( sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const char **pzTail /* OUT: End of parsed string */ ){ - return sqlite3Prepare(db,zSql,nBytes,0,ppStmt,pzTail); + int rc; + rc = sqlite3LockAndPrepare(db,zSql,nBytes,0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; } int sqlite3_prepare_v2( sqlite3 *db, /* Database handle. */ @@ -620,7 +724,10 @@ int sqlite3_prepare_v2( sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const char **pzTail /* OUT: End of parsed string */ ){ - return sqlite3Prepare(db,zSql,nBytes,1,ppStmt,pzTail); + int rc; + rc = sqlite3LockAndPrepare(db,zSql,nBytes,1,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; } @@ -644,12 +751,13 @@ static int sqlite3Prepare16( const char *zTail8 = 0; int rc = SQLITE_OK; - if( sqlite3SafetyCheck(db) ){ + if( !sqlite3SafetyCheckOk(db) ){ return SQLITE_MISUSE; } - zSql8 = sqlite3Utf16to8(zSql, nBytes); + sqlite3_mutex_enter(db->mutex); + zSql8 = sqlite3Utf16to8(db, zSql, nBytes); if( zSql8 ){ - rc = sqlite3Prepare(db, zSql8, -1, saveSqlFlag, ppStmt, &zTail8); + rc = sqlite3LockAndPrepare(db, zSql8, -1, saveSqlFlag, ppStmt, &zTail8); } if( zTail8 && pzTail ){ @@ -658,11 +766,13 @@ static int sqlite3Prepare16( ** characters between zSql8 and zTail8, and then returning a pointer ** the same number of characters into the UTF-16 string. */ - int chars_parsed = sqlite3Utf8CharLen(zSql8, zTail8-zSql8); + int chars_parsed = sqlite3Utf8CharLen(zSql8, (int)(zTail8-zSql8)); *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, chars_parsed); } - sqliteFree(zSql8); - return sqlite3ApiExit(db, rc); + sqlite3DbFree(db, zSql8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } /* @@ -680,7 +790,10 @@ int sqlite3_prepare16( sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const void **pzTail /* OUT: End of parsed string */ ){ - return sqlite3Prepare16(db,zSql,nBytes,0,ppStmt,pzTail); + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes,0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; } int sqlite3_prepare16_v2( sqlite3 *db, /* Database handle. */ @@ -689,7 +802,10 @@ int sqlite3_prepare16_v2( sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const void **pzTail /* OUT: End of parsed string */ ){ - return sqlite3Prepare16(db,zSql,nBytes,1,ppStmt,pzTail); + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes,1,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; } #endif /* SQLITE_OMIT_UTF16 */ diff --git a/printf.c b/printf.c index 01ed7a1..58e575b 100644 --- a/printf.c +++ b/printf.c @@ -5,6 +5,8 @@ ** an historical reference. Most of the "enhancements" have been backed ** out so that the functionality is now the same as standard printf(). ** +** $Id: printf.c,v 1.99 2008/12/10 19:26:24 drh Exp $ +** ************************************************************************** ** ** The following modules is an enhanced replacement for the "printf" subroutines @@ -51,7 +53,6 @@ ** */ #include "sqliteInt.h" -#include /* ** Conversion types fall into various categories as defined by the @@ -67,14 +68,14 @@ #define etPERCENT 8 /* Percent symbol. %% */ #define etCHARX 9 /* Characters. %c */ /* The rest are extensions, not normally found in printf() */ -#define etCHARLIT 10 /* Literal characters. %' */ -#define etSQLESCAPE 11 /* Strings with '\'' doubled. %q */ -#define etSQLESCAPE2 12 /* Strings with '\'' doubled and enclosed in '', +#define etSQLESCAPE 10 /* Strings with '\'' doubled. %q */ +#define etSQLESCAPE2 11 /* Strings with '\'' doubled and enclosed in '', NULL pointers replaced by SQL NULL. %Q */ -#define etTOKEN 13 /* a pointer to a Token structure */ -#define etSRCLIST 14 /* a pointer to a SrcList */ -#define etPOINTER 15 /* The %p conversion */ -#define etSQLESCAPE3 16 /* %w -> Strings with '\"' doubled */ +#define etTOKEN 12 /* a pointer to a Token structure */ +#define etSRCLIST 13 /* a pointer to a SrcList */ +#define etPOINTER 14 /* The %p conversion */ +#define etSQLESCAPE3 15 /* %w -> Strings with '\"' doubled */ +#define etORDINAL 16 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ /* @@ -113,7 +114,7 @@ static const et_info fmtinfo[] = { { 'd', 10, 1, etRADIX, 0, 0 }, { 's', 0, 4, etSTRING, 0, 0 }, { 'g', 0, 1, etGENERIC, 30, 0 }, - { 'z', 0, 6, etDYNSTRING, 0, 0 }, + { 'z', 0, 4, etDYNSTRING, 0, 0 }, { 'q', 0, 4, etSQLESCAPE, 0, 0 }, { 'Q', 0, 4, etSQLESCAPE2, 0, 0 }, { 'w', 0, 4, etSQLESCAPE3, 0, 0 }, @@ -134,8 +135,8 @@ static const et_info fmtinfo[] = { { 'p', 16, 0, etPOINTER, 0, 1 }, { 'T', 0, 2, etTOKEN, 0, 0 }, { 'S', 0, 2, etSRCLIST, 0, 0 }, + { 'r', 10, 3, etORDINAL, 0, 0 }, }; -#define etNINFO (sizeof(fmtinfo)/sizeof(fmtinfo[0])) /* ** If SQLITE_OMIT_FLOATING_POINT is defined, then none of the floating point @@ -155,7 +156,7 @@ static const et_info fmtinfo[] = { ** 16 (the number of significant digits in a 64-bit float) '0' is ** always returned. */ -static int et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ +static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ int digit; LONGDOUBLE_TYPE d; if( (*cnt)++ >= 16 ) return '0'; @@ -163,10 +164,24 @@ static int et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ d = digit; digit += '0'; *val = (*val - d)*10.0; - return digit; + return (char)digit; } #endif /* SQLITE_OMIT_FLOATING_POINT */ +/* +** Append N space characters to the given string buffer. +*/ +static void appendSpace(StrAccum *pAccum, int N){ + static const char zSpaces[] = " "; + while( N>=(int)sizeof(zSpaces)-1 ){ + sqlite3StrAccumAppend(pAccum, zSpaces, sizeof(zSpaces)-1); + N -= sizeof(zSpaces)-1; + } + if( N>0 ){ + sqlite3StrAccumAppend(pAccum, zSpaces, N); + } +} + /* ** On machines with a small stack size, you can redefine the ** SQLITE_PRINT_BUF_SIZE to be less than 350. But beware - for @@ -204,9 +219,8 @@ static int et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ ** seems to make a big difference in determining how fast this beast ** will run. */ -static int vxprintf( - void (*func)(void*,const char*,int), /* Consumer of text */ - void *arg, /* First argument to the consumer */ +void sqlite3VXPrintf( + StrAccum *pAccum, /* Accumulate results here */ int useExtended, /* Allow extended %-conversions */ const char *fmt, /* Format string */ va_list ap /* arguments */ @@ -216,7 +230,6 @@ static int vxprintf( int precision; /* Precision of the current field */ int length; /* Length of the field */ int idx; /* A general purpose loop counter */ - int count; /* Total number of characters output */ int width; /* Width of the current field */ etByte flag_leftjustify; /* True if "-" flag is present */ etByte flag_plussign; /* True if "+" flag is present */ @@ -232,12 +245,8 @@ static int vxprintf( const et_info *infop; /* Pointer to the appropriate info structure */ char buf[etBUFSIZE]; /* Conversion buffer */ char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ - etByte errorflag = 0; /* True if an error is encountered */ - etByte xtype; /* Conversion paradigm */ + etByte xtype = 0; /* Conversion paradigm */ char *zExtra; /* Extra memory used for etTCLESCAPE conversions */ - static const char spaces[] = - " "; -#define etSPACESIZE (sizeof(spaces)-1) #ifndef SQLITE_OMIT_FLOATING_POINT int exp, e2; /* exponent of real numbers */ double rounder; /* Used for rounding floating point values */ @@ -247,8 +256,7 @@ static int vxprintf( int nsd; /* Number of significant digits returned */ #endif - func(arg,"",0); - count = length = 0; + length = 0; bufpt = 0; for(; (c=(*fmt))!=0; ++fmt){ if( c!='%' ){ @@ -256,14 +264,11 @@ static int vxprintf( bufpt = (char *)fmt; amt = 1; while( (c=(*++fmt))!='%' && c!=0 ) amt++; - (*func)(arg,bufpt,amt); - count += amt; + sqlite3StrAccumAppend(pAccum, bufpt, amt); if( c==0 ) break; } if( (c=(*++fmt))==0 ){ - errorflag = 1; - (*func)(arg,"%",1); - count++; + sqlite3StrAccumAppend(pAccum, "%", 1); break; } /* Find out what flags are present */ @@ -331,20 +336,20 @@ static int vxprintf( } /* Fetch the info entry for the field */ infop = 0; - for(idx=0; idxflags & FLAG_INTERN)==0 ){ xtype = infop->type; }else{ - return -1; + return; } break; } } zExtra = 0; if( infop==0 ){ - return -1; + return; } @@ -379,6 +384,7 @@ static int vxprintf( flag_longlong = sizeof(char*)==sizeof(i64); flag_long = sizeof(char*)==sizeof(long int); /* Fall through into the next case */ + case etORDINAL: case etRADIX: if( infop->flags & FLAG_SIGNED ){ i64 v; @@ -405,6 +411,16 @@ static int vxprintf( precision = width-(prefix!=0); } bufpt = &buf[etBUFSIZE-1]; + if( xtype==etORDINAL ){ + static const char zOrd[] = "thstndrd"; + int x = (int)(longvalue % 10); + if( x>=4 || (longvalue/10)%10==1 ){ + x = 0; + } + buf[etBUFSIZE-3] = zOrd[x*2]; + buf[etBUFSIZE-2] = zOrd[x*2+1]; + bufpt -= 2; + } { register const char *cset; /* Use registers for speed */ register int base; @@ -415,7 +431,7 @@ static int vxprintf( longvalue = longvalue/base; }while( longvalue>0 ); } - length = &buf[etBUFSIZE-1]-bufpt; + length = (int)(&buf[etBUFSIZE-1]-bufpt); for(idx=precision-length; idx>0; idx--){ *(--bufpt) = '0'; /* Zero pad */ } @@ -424,11 +440,9 @@ static int vxprintf( const char *pre; char x; pre = &aPrefix[infop->prefix]; - if( *bufpt!=pre[0] ){ - for(; (x=(*pre))!=0; pre++) *(--bufpt) = x; - } + for(; (x=(*pre))!=0; pre++) *(--bufpt) = x; } - length = &buf[etBUFSIZE-1]-bufpt; + length = (int)(&buf[etBUFSIZE-1]-bufpt); break; case etFLOAT: case etEXP: @@ -456,7 +470,7 @@ static int vxprintf( if( xtype==etFLOAT ) realvalue += rounder; /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ exp = 0; - if( sqlite3_isnan(realvalue) ){ + if( sqlite3IsNaN((double)realvalue) ){ bufpt = "NaN"; length = 3; break; @@ -465,9 +479,9 @@ static int vxprintf( while( realvalue>=1e32 && exp<=350 ){ realvalue *= 1e-32; exp+=32; } while( realvalue>=1e8 && exp<=350 ){ realvalue *= 1e-8; exp+=8; } while( realvalue>=10.0 && exp<=350 ){ realvalue *= 0.1; exp++; } - while( realvalue<1e-8 && exp>=-350 ){ realvalue *= 1e8; exp-=8; } - while( realvalue<1.0 && exp>=-350 ){ realvalue *= 10.0; exp--; } - if( exp>350 || exp<-350 ){ + while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } + while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } + if( exp>350 ){ if( prefix=='-' ){ bufpt = "-Inf"; }else if( prefix=='+' ){ @@ -475,7 +489,7 @@ static int vxprintf( }else{ bufpt = "Inf"; } - length = strlen(bufpt); + length = sqlite3Strlen30(bufpt); break; } } @@ -506,7 +520,7 @@ static int vxprintf( e2 = exp; } nsd = 0; - flag_dp = (precision>0) | flag_alternateform | flag_altform2; + flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; /* The sign in front of the number */ if( prefix ){ *(bufpt++) = prefix; @@ -525,7 +539,8 @@ static int vxprintf( } /* "0" digits after the decimal point but before the first ** significant digit of the number */ - for(e2++; e2<0 && precision>0; precision--, e2++){ + for(e2++; e2<0; precision--, e2++){ + assert( precision>0 ); *(bufpt++) = '0'; } /* Significant digits after the decimal point */ @@ -545,7 +560,7 @@ static int vxprintf( } } /* Add the "eNNN" suffix */ - if( flag_exp || (xtype==etEXP && exp) ){ + if( flag_exp || xtype==etEXP ){ *(bufpt++) = aDigits[infop->charset]; if( exp<0 ){ *(bufpt++) = '-'; exp = -exp; @@ -553,18 +568,18 @@ static int vxprintf( *(bufpt++) = '+'; } if( exp>=100 ){ - *(bufpt++) = (exp/100)+'0'; /* 100's digit */ + *(bufpt++) = (char)((exp/100)+'0'); /* 100's digit */ exp %= 100; } - *(bufpt++) = exp/10+'0'; /* 10's digit */ - *(bufpt++) = exp%10+'0'; /* 1's digit */ + *(bufpt++) = (char)(exp/10+'0'); /* 10's digit */ + *(bufpt++) = (char)(exp%10+'0'); /* 1's digit */ } *bufpt = 0; /* The converted number is in buf[] and zero terminated. Output it. ** Note that the number is in the usual order, not reversed as with ** integer conversions. */ - length = bufpt-buf; + length = (int)(bufpt-buf); bufpt = buf; /* Special case: Add leading zeros if the flag_zeropad flag is @@ -582,7 +597,7 @@ static int vxprintf( #endif break; case etSIZE: - *(va_arg(ap,int*)) = count; + *(va_arg(ap,int*)) = pAccum->nChar; length = width = 0; break; case etPERCENT: @@ -590,11 +605,11 @@ static int vxprintf( bufpt = buf; length = 1; break; - case etCHARLIT: case etCHARX: - c = buf[0] = (xtype==etCHARX ? va_arg(ap,int) : *++fmt); + c = va_arg(ap,int); + buf[0] = (char)c; if( precision>=0 ){ - for(idx=1; idx=0 && precision=0 ){ + for(length=0; lengthetBUFSIZE ){ - bufpt = zExtra = sqliteMalloc( n ); - if( bufpt==0 ) return -1; + bufpt = zExtra = sqlite3Malloc( n ); + if( bufpt==0 ){ + pAccum->mallocFailed = 1; + return; + } }else{ bufpt = buf; } @@ -647,8 +669,8 @@ static int vxprintf( } case etTOKEN: { Token *pToken = va_arg(ap, Token*); - if( pToken && pToken->z ){ - (*func)(arg, (char*)pToken->z, pToken->n); + if( pToken ){ + sqlite3StrAccumAppend(pAccum, (const char*)pToken->z, pToken->n); } length = width = 0; break; @@ -658,11 +680,11 @@ static int vxprintf( int k = va_arg(ap, int); struct SrcList_item *pItem = &pSrc->a[k]; assert( k>=0 && knSrc ); - if( pItem->zDatabase && pItem->zDatabase[0] ){ - (*func)(arg, pItem->zDatabase, strlen(pItem->zDatabase)); - (*func)(arg, ".", 1); + if( pItem->zDatabase ){ + sqlite3StrAccumAppend(pAccum, pItem->zDatabase, -1); + sqlite3StrAccumAppend(pAccum, ".", 1); } - (*func)(arg, pItem->zName, strlen(pItem->zName)); + sqlite3StrAccumAppend(pAccum, pItem->zName, -1); length = width = 0; break; } @@ -676,161 +698,181 @@ static int vxprintf( register int nspace; nspace = width-length; if( nspace>0 ){ - count += nspace; - while( nspace>=etSPACESIZE ){ - (*func)(arg,spaces,etSPACESIZE); - nspace -= etSPACESIZE; - } - if( nspace>0 ) (*func)(arg,spaces,nspace); + appendSpace(pAccum, nspace); } } if( length>0 ){ - (*func)(arg,bufpt,length); - count += length; + sqlite3StrAccumAppend(pAccum, bufpt, length); } if( flag_leftjustify ){ register int nspace; nspace = width-length; if( nspace>0 ){ - count += nspace; - while( nspace>=etSPACESIZE ){ - (*func)(arg,spaces,etSPACESIZE); - nspace -= etSPACESIZE; - } - if( nspace>0 ) (*func)(arg,spaces,nspace); + appendSpace(pAccum, nspace); } } if( zExtra ){ - sqliteFree(zExtra); + sqlite3_free(zExtra); } }/* End for loop over the format string */ - return errorflag ? -1 : count; } /* End of function */ - -/* This structure is used to store state information about the -** write to memory that is currently in progress. +/* +** Append N bytes of text from z to the StrAccum object. */ -struct sgMprintf { - char *zBase; /* A base allocation */ - char *zText; /* The string collected so far */ - int nChar; /* Length of the string so far */ - int nTotal; /* Output size if unconstrained */ - int nAlloc; /* Amount of space allocated in zText */ - void *(*xRealloc)(void*,int); /* Function used to realloc memory */ -}; - -/* -** This function implements the callback from vxprintf. -** -** This routine add nNewChar characters of text in zNewText to -** the sgMprintf structure pointed to by "arg". -*/ -static void mout(void *arg, const char *zNewText, int nNewChar){ - struct sgMprintf *pM = (struct sgMprintf*)arg; - pM->nTotal += nNewChar; - if( pM->nChar + nNewChar + 1 > pM->nAlloc ){ - if( pM->xRealloc==0 ){ - nNewChar = pM->nAlloc - pM->nChar - 1; +void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ + if( p->tooBig | p->mallocFailed ){ + return; + } + if( N<0 ){ + N = sqlite3Strlen30(z); + } + if( N==0 || z==0 ){ + return; + } + if( p->nChar+N >= p->nAlloc ){ + char *zNew; + if( !p->useMalloc ){ + p->tooBig = 1; + N = p->nAlloc - p->nChar - 1; + if( N<=0 ){ + return; + } }else{ - int nAlloc = pM->nChar + nNewChar*2 + 1; - if( pM->zText==pM->zBase ){ - pM->zText = pM->xRealloc(0, nAlloc); - if( pM->zText && pM->nChar ){ - memcpy(pM->zText, pM->zBase, pM->nChar); - } + i64 szNew = p->nChar; + szNew += N + 1; + if( szNew > p->mxAlloc ){ + sqlite3StrAccumReset(p); + p->tooBig = 1; + return; }else{ - char *zNew; - zNew = pM->xRealloc(pM->zText, nAlloc); - if( zNew ){ - pM->zText = zNew; - }else{ - return; - } + p->nAlloc = (int)szNew; } - pM->nAlloc = nAlloc; - } - } - if( pM->zText ){ - if( nNewChar>0 ){ - memcpy(&pM->zText[pM->nChar], zNewText, nNewChar); - pM->nChar += nNewChar; - } - pM->zText[pM->nChar] = 0; - } -} - -/* -** This routine is a wrapper around xprintf() that invokes mout() as -** the consumer. -*/ -static char *base_vprintf( - void *(*xRealloc)(void*,int), /* Routine to realloc memory. May be NULL */ - int useInternal, /* Use internal %-conversions if true */ - char *zInitBuf, /* Initially write here, before mallocing */ - int nInitBuf, /* Size of zInitBuf[] */ - const char *zFormat, /* format string */ - va_list ap /* arguments */ -){ - struct sgMprintf sM; - sM.zBase = sM.zText = zInitBuf; - sM.nChar = sM.nTotal = 0; - sM.nAlloc = nInitBuf; - sM.xRealloc = xRealloc; - vxprintf(mout, &sM, useInternal, zFormat, ap); - if( xRealloc ){ - if( sM.zText==sM.zBase ){ - sM.zText = xRealloc(0, sM.nChar+1); - if( sM.zText ){ - memcpy(sM.zText, sM.zBase, sM.nChar+1); - } - }else if( sM.nAlloc>sM.nChar+10 ){ - char *zNew = xRealloc(sM.zText, sM.nChar+1); + zNew = sqlite3DbMallocRaw(p->db, p->nAlloc ); if( zNew ){ - sM.zText = zNew; + memcpy(zNew, p->zText, p->nChar); + sqlite3StrAccumReset(p); + p->zText = zNew; + }else{ + p->mallocFailed = 1; + sqlite3StrAccumReset(p); + return; } } } - return sM.zText; + memcpy(&p->zText[p->nChar], z, N); + p->nChar += N; } /* -** Realloc that is a real function, not a macro. +** Finish off a string by making sure it is zero-terminated. +** Return a pointer to the resulting string. Return a NULL +** pointer if any kind of error was encountered. */ -static void *printf_realloc(void *old, int size){ - return sqliteRealloc(old,size); +char *sqlite3StrAccumFinish(StrAccum *p){ + if( p->zText ){ + p->zText[p->nChar] = 0; + if( p->useMalloc && p->zText==p->zBase ){ + p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + if( p->zText ){ + memcpy(p->zText, p->zBase, p->nChar+1); + }else{ + p->mallocFailed = 1; + } + } + } + return p->zText; +} + +/* +** Reset an StrAccum string. Reclaim all malloced memory. +*/ +void sqlite3StrAccumReset(StrAccum *p){ + if( p->zText!=p->zBase ){ + sqlite3DbFree(p->db, p->zText); + } + p->zText = 0; +} + +/* +** Initialize a string accumulator +*/ +void sqlite3StrAccumInit(StrAccum *p, char *zBase, int n, int mx){ + p->zText = p->zBase = zBase; + p->db = 0; + p->nChar = 0; + p->nAlloc = n; + p->mxAlloc = mx; + p->useMalloc = 1; + p->tooBig = 0; + p->mallocFailed = 0; } /* ** Print into memory obtained from sqliteMalloc(). Use the internal ** %-conversion extensions. */ -char *sqlite3VMPrintf(const char *zFormat, va_list ap){ - char zBase[SQLITE_PRINT_BUF_SIZE]; - return base_vprintf(printf_realloc, 1, zBase, sizeof(zBase), zFormat, ap); -} - -/* -** Print into memory obtained from sqliteMalloc(). Use the internal -** %-conversion extensions. -*/ -char *sqlite3MPrintf(const char *zFormat, ...){ - va_list ap; +char *sqlite3VMPrintf(sqlite3 *db, const char *zFormat, va_list ap){ char *z; char zBase[SQLITE_PRINT_BUF_SIZE]; + StrAccum acc; + sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), + db ? db->aLimit[SQLITE_LIMIT_LENGTH] : SQLITE_MAX_LENGTH); + acc.db = db; + sqlite3VXPrintf(&acc, 1, zFormat, ap); + z = sqlite3StrAccumFinish(&acc); + if( acc.mallocFailed && db ){ + db->mallocFailed = 1; + } + return z; +} + +/* +** Print into memory obtained from sqliteMalloc(). Use the internal +** %-conversion extensions. +*/ +char *sqlite3MPrintf(sqlite3 *db, const char *zFormat, ...){ + va_list ap; + char *z; va_start(ap, zFormat); - z = base_vprintf(printf_realloc, 1, zBase, sizeof(zBase), zFormat, ap); + z = sqlite3VMPrintf(db, zFormat, ap); va_end(ap); return z; } +/* +** Like sqlite3MPrintf(), but call sqlite3DbFree() on zStr after formatting +** the string and before returnning. This routine is intended to be used +** to modify an existing string. For example: +** +** x = sqlite3MPrintf(db, x, "prefix %s suffix", x); +** +*/ +char *sqlite3MAppendf(sqlite3 *db, char *zStr, const char *zFormat, ...){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + sqlite3DbFree(db, zStr); + return z; +} + /* ** Print into memory obtained from sqlite3_malloc(). Omit the internal ** %-conversion extensions. */ char *sqlite3_vmprintf(const char *zFormat, va_list ap){ + char *z; char zBase[SQLITE_PRINT_BUF_SIZE]; - return base_vprintf(sqlite3_realloc, 0, zBase, sizeof(zBase), zFormat, ap); + StrAccum acc; +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); + sqlite3VXPrintf(&acc, 0, zFormat, ap); + z = sqlite3StrAccumFinish(&acc); + return z; } /* @@ -840,6 +882,9 @@ char *sqlite3_vmprintf(const char *zFormat, va_list ap){ char *sqlite3_mprintf(const char *zFormat, ...){ va_list ap; char *z; +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); va_end(ap); @@ -855,30 +900,36 @@ char *sqlite3_mprintf(const char *zFormat, ...){ char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ char *z; va_list ap; + StrAccum acc; if( n<=0 ){ return zBuf; } - zBuf[0] = 0; + sqlite3StrAccumInit(&acc, zBuf, n, 0); + acc.useMalloc = 0; va_start(ap,zFormat); - z = base_vprintf(0, 0, zBuf, n, zFormat, ap); + sqlite3VXPrintf(&acc, 0, zFormat, ap); va_end(ap); + z = sqlite3StrAccumFinish(&acc); return z; } -#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) || defined(SQLITE_MEMDEBUG) +#if defined(SQLITE_DEBUG) /* ** A version of printf() that understands %lld. Used for debugging. ** The printf() built into some versions of windows does not understand %lld ** and segfaults if you give it a long long int. */ void sqlite3DebugPrintf(const char *zFormat, ...){ - extern int getpid(void); va_list ap; + StrAccum acc; char zBuf[500]; - va_start(ap, zFormat); - base_vprintf(0, 0, zBuf, sizeof(zBuf), zFormat, ap); + sqlite3StrAccumInit(&acc, zBuf, sizeof(zBuf), 0); + acc.useMalloc = 0; + va_start(ap,zFormat); + sqlite3VXPrintf(&acc, 0, zFormat, ap); va_end(ap); + sqlite3StrAccumFinish(&acc); fprintf(stdout,"%s", zBuf); fflush(stdout); } diff --git a/random.c b/random.c index 03ed7a4..f90a34e 100644 --- a/random.c +++ b/random.c @@ -15,12 +15,20 @@ ** Random numbers are used by some of the database backends in order ** to generate random integer keys for tables or random filenames. ** -** $Id: random.c,v 1.16 2007/01/05 14:38:56 drh Exp $ +** $Id: random.c,v 1.29 2008/12/10 19:26:24 drh Exp $ */ #include "sqliteInt.h" -#include "os.h" +/* All threads share a single random number generator. +** This structure is the current state of the generator. +*/ +static SQLITE_WSD struct sqlite3PrngType { + unsigned char isInit; /* True if initialized */ + unsigned char i, j; /* State variables */ + unsigned char s[256]; /* State variables */ +} sqlite3Prng; + /* ** Get a single 8-bit random value from the RC4 PRNG. The Mutex ** must be held while executing this routine. @@ -37,17 +45,23 @@ ** (Later): Actually, OP_NewRowid does not depend on a good source of ** randomness any more. But we will leave this code in all the same. */ -static int randomByte(void){ +static u8 randomByte(void){ unsigned char t; - /* All threads share a single random number generator. - ** This structure is the current state of the generator. + + /* The "wsdPrng" macro will resolve to the pseudo-random number generator + ** state vector. If writable static data is unsupported on the target, + ** we have to locate the state vector at run-time. In the more common + ** case where writable static data is supported, wsdPrng can refer directly + ** to the "sqlite3Prng" state vector declared above. */ - static struct { - unsigned char isInit; /* True if initialized */ - unsigned char i, j; /* State variables */ - unsigned char s[256]; /* State variables */ - } prng; +#ifdef SQLITE_OMIT_WSD + struct sqlite3PrngType *p = &GLOBAL(struct sqlite3PrngType, sqlite3Prng); +# define wsdPrng p[0] +#else +# define wsdPrng sqlite3Prng +#endif + /* Initialize the state of the random number generator once, ** the first time this routine is called. The seed value does @@ -58,43 +72,76 @@ static int randomByte(void){ ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random ** number generator) not as an encryption device. */ - if( !prng.isInit ){ + if( !wsdPrng.isInit ){ int i; char k[256]; - prng.j = 0; - prng.i = 0; - sqlite3OsRandomSeed(k); + wsdPrng.j = 0; + wsdPrng.i = 0; + sqlite3OsRandomness(sqlite3_vfs_find(0), 256, k); for(i=0; i<256; i++){ - prng.s[i] = i; + wsdPrng.s[i] = (u8)i; } for(i=0; i<256; i++){ - prng.j += prng.s[i] + k[i]; - t = prng.s[prng.j]; - prng.s[prng.j] = prng.s[i]; - prng.s[i] = t; + wsdPrng.j += wsdPrng.s[i] + k[i]; + t = wsdPrng.s[wsdPrng.j]; + wsdPrng.s[wsdPrng.j] = wsdPrng.s[i]; + wsdPrng.s[i] = t; } - prng.isInit = 1; + wsdPrng.isInit = 1; } /* Generate and return single random byte */ - prng.i++; - t = prng.s[prng.i]; - prng.j += t; - prng.s[prng.i] = prng.s[prng.j]; - prng.s[prng.j] = t; - t += prng.s[prng.i]; - return prng.s[t]; + wsdPrng.i++; + t = wsdPrng.s[wsdPrng.i]; + wsdPrng.j += t; + wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; + wsdPrng.s[wsdPrng.j] = t; + t += wsdPrng.s[wsdPrng.i]; + return wsdPrng.s[t]; } /* ** Return N random bytes. */ -void sqlite3Randomness(int N, void *pBuf){ +void sqlite3_randomness(int N, void *pBuf){ unsigned char *zBuf = pBuf; - sqlite3OsEnterMutex(); +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); +#endif + sqlite3_mutex_enter(mutex); while( N-- ){ *(zBuf++) = randomByte(); } - sqlite3OsLeaveMutex(); + sqlite3_mutex_leave(mutex); } + +#ifndef SQLITE_OMIT_BUILTIN_TEST +/* +** For testing purposes, we sometimes want to preserve the state of +** PRNG and restore the PRNG to its saved state at a later time, or +** to reset the PRNG to its initial state. These routines accomplish +** those tasks. +** +** The sqlite3_test_control() interface calls these routines to +** control the PRNG. +*/ +static SQLITE_WSD struct sqlite3PrngType sqlite3SavedPrng; +void sqlite3PrngSaveState(void){ + memcpy( + &GLOBAL(struct sqlite3PrngType, sqlite3SavedPrng), + &GLOBAL(struct sqlite3PrngType, sqlite3Prng), + sizeof(sqlite3Prng) + ); +} +void sqlite3PrngRestoreState(void){ + memcpy( + &GLOBAL(struct sqlite3PrngType, sqlite3Prng), + &GLOBAL(struct sqlite3PrngType, sqlite3SavedPrng), + sizeof(sqlite3Prng) + ); +} +void sqlite3PrngResetState(void){ + GLOBAL(struct sqlite3PrngType, sqlite3Prng).isInit = 0; +} +#endif /* SQLITE_OMIT_BUILTIN_TEST */ diff --git a/resolve.c b/resolve.c new file mode 100644 index 0000000..d6f58fc --- /dev/null +++ b/resolve.c @@ -0,0 +1,1169 @@ +/* +** 2008 August 18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains routines used for walking the parser tree and +** resolve all identifiers by associating them with a particular +** table and column. +** +** $Id: resolve.c,v 1.15 2008/12/10 19:26:24 drh Exp $ +*/ +#include "sqliteInt.h" +#include +#include + +/* +** Turn the pExpr expression into an alias for the iCol-th column of the +** result set in pEList. +** +** If the result set column is a simple column reference, then this routine +** makes an exact copy. But for any other kind of expression, this +** routine make a copy of the result set column as the argument to the +** TK_AS operator. The TK_AS operator causes the expression to be +** evaluated just once and then reused for each alias. +** +** The reason for suppressing the TK_AS term when the expression is a simple +** column reference is so that the column reference will be recognized as +** usable by indices within the WHERE clause processing logic. +** +** Hack: The TK_AS operator is inhibited if zType[0]=='G'. This means +** that in a GROUP BY clause, the expression is evaluated twice. Hence: +** +** SELECT random()%5 AS x, count(*) FROM tab GROUP BY x +** +** Is equivalent to: +** +** SELECT random()%5 AS x, count(*) FROM tab GROUP BY random()%5 +** +** The result of random()%5 in the GROUP BY clause is probably different +** from the result in the result-set. We might fix this someday. Or +** then again, we might not... +*/ +static void resolveAlias( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* A result set */ + int iCol, /* A column in the result set. 0..pEList->nExpr-1 */ + Expr *pExpr, /* Transform this into an alias to the result set */ + const char *zType /* "GROUP" or "ORDER" or "" */ +){ + Expr *pOrig; /* The iCol-th column of the result set */ + Expr *pDup; /* Copy of pOrig */ + sqlite3 *db; /* The database connection */ + + assert( iCol>=0 && iColnExpr ); + pOrig = pEList->a[iCol].pExpr; + assert( pOrig!=0 ); + assert( pOrig->flags & EP_Resolved ); + db = pParse->db; + pDup = sqlite3ExprDup(db, pOrig); + if( pDup==0 ) return; + if( pDup->op!=TK_COLUMN && zType[0]!='G' ){ + pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); + if( pDup==0 ) return; + if( pEList->a[iCol].iAlias==0 ){ + pEList->a[iCol].iAlias = (u16)(++pParse->nAlias); + } + pDup->iTable = pEList->a[iCol].iAlias; + } + if( pExpr->flags & EP_ExpCollate ){ + pDup->pColl = pExpr->pColl; + pDup->flags |= EP_ExpCollate; + } + sqlite3ExprClear(db, pExpr); + memcpy(pExpr, pDup, sizeof(*pExpr)); + sqlite3DbFree(db, pDup); +} + +/* +** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up +** that name in the set of source tables in pSrcList and make the pExpr +** expression node refer back to that source column. The following changes +** are made to pExpr: +** +** pExpr->iDb Set the index in db->aDb[] of the database X +** (even if X is implied). +** pExpr->iTable Set to the cursor number for the table obtained +** from pSrcList. +** pExpr->pTab Points to the Table structure of X.Y (even if +** X and/or Y are implied.) +** pExpr->iColumn Set to the column number within the table. +** pExpr->op Set to TK_COLUMN. +** pExpr->pLeft Any expression this points to is deleted +** pExpr->pRight Any expression this points to is deleted. +** +** The pDbToken is the name of the database (the "X"). This value may be +** NULL meaning that name is of the form Y.Z or Z. Any available database +** can be used. The pTableToken is the name of the table (the "Y"). This +** value can be NULL if pDbToken is also NULL. If pTableToken is NULL it +** means that the form of the name is Z and that columns from any table +** can be used. +** +** If the name cannot be resolved unambiguously, leave an error message +** in pParse and return non-zero. Return zero on success. +*/ +static int lookupName( + Parse *pParse, /* The parsing context */ + Token *pDbToken, /* Name of the database containing table, or NULL */ + Token *pTableToken, /* Name of table containing column, or NULL */ + Token *pColumnToken, /* Name of the column. */ + NameContext *pNC, /* The name context used to resolve the name */ + Expr *pExpr /* Make this EXPR node point to the selected column */ +){ + char *zDb = 0; /* Name of the database. The "X" in X.Y.Z */ + char *zTab = 0; /* Name of the table. The "Y" in X.Y.Z or Y.Z */ + char *zCol = 0; /* Name of the column. The "Z" */ + int i, j; /* Loop counters */ + int cnt = 0; /* Number of matching column names */ + int cntTab = 0; /* Number of matching table names */ + sqlite3 *db = pParse->db; /* The database connection */ + struct SrcList_item *pItem; /* Use for looping over pSrcList items */ + struct SrcList_item *pMatch = 0; /* The matching pSrcList item */ + NameContext *pTopNC = pNC; /* First namecontext in the list */ + Schema *pSchema = 0; /* Schema of the expression */ + + assert( pColumnToken && pColumnToken->z ); /* The Z in X.Y.Z cannot be NULL */ + + /* Dequote and zero-terminate the names */ + zDb = sqlite3NameFromToken(db, pDbToken); + zTab = sqlite3NameFromToken(db, pTableToken); + zCol = sqlite3NameFromToken(db, pColumnToken); + if( db->mallocFailed ){ + goto lookupname_end; + } + + /* Initialize the node to no-match */ + pExpr->iTable = -1; + pExpr->pTab = 0; + + /* Start at the inner-most context and move outward until a match is found */ + while( pNC && cnt==0 ){ + ExprList *pEList; + SrcList *pSrcList = pNC->pSrcList; + + if( pSrcList ){ + for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ + Table *pTab; + int iDb; + Column *pCol; + + pTab = pItem->pTab; + assert( pTab!=0 && pTab->zName!=0 ); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( pTab->nCol>0 ); + if( zTab ){ + if( pItem->zAlias ){ + char *zTabName = pItem->zAlias; + if( sqlite3StrICmp(zTabName, zTab)!=0 ) continue; + }else{ + char *zTabName = pTab->zName; + if( zTabName==0 || sqlite3StrICmp(zTabName, zTab)!=0 ) continue; + if( zDb!=0 && sqlite3StrICmp(db->aDb[iDb].zName, zDb)!=0 ){ + continue; + } + } + } + if( 0==(cntTab++) ){ + pExpr->iTable = pItem->iCursor; + pExpr->pTab = pTab; + pSchema = pTab->pSchema; + pMatch = pItem; + } + for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ + if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ + IdList *pUsing; + cnt++; + pExpr->iTable = pItem->iCursor; + pExpr->pTab = pTab; + pMatch = pItem; + pSchema = pTab->pSchema; + /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ + pExpr->iColumn = j==pTab->iPKey ? -1 : j; + if( inSrc-1 ){ + if( pItem[1].jointype & JT_NATURAL ){ + /* If this match occurred in the left table of a natural join, + ** then skip the right table to avoid a duplicate match */ + pItem++; + i++; + }else if( (pUsing = pItem[1].pUsing)!=0 ){ + /* If this match occurs on a column that is in the USING clause + ** of a join, skip the search of the right table of the join + ** to avoid a duplicate match there. */ + int k; + for(k=0; knId; k++){ + if( sqlite3StrICmp(pUsing->a[k].zName, zCol)==0 ){ + pItem++; + i++; + break; + } + } + } + } + break; + } + } + } + } + +#ifndef SQLITE_OMIT_TRIGGER + /* If we have not already resolved the name, then maybe + ** it is a new.* or old.* trigger argument reference + */ + if( zDb==0 && zTab!=0 && cnt==0 && pParse->trigStack!=0 ){ + TriggerStack *pTriggerStack = pParse->trigStack; + Table *pTab = 0; + u32 *piColMask = 0; + if( pTriggerStack->newIdx != -1 && sqlite3StrICmp("new", zTab) == 0 ){ + pExpr->iTable = pTriggerStack->newIdx; + assert( pTriggerStack->pTab ); + pTab = pTriggerStack->pTab; + piColMask = &(pTriggerStack->newColMask); + }else if( pTriggerStack->oldIdx != -1 && sqlite3StrICmp("old", zTab)==0 ){ + pExpr->iTable = pTriggerStack->oldIdx; + assert( pTriggerStack->pTab ); + pTab = pTriggerStack->pTab; + piColMask = &(pTriggerStack->oldColMask); + } + + if( pTab ){ + int iCol; + Column *pCol = pTab->aCol; + + pSchema = pTab->pSchema; + cntTab++; + for(iCol=0; iCol < pTab->nCol; iCol++, pCol++) { + if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ + cnt++; + pExpr->iColumn = iCol==pTab->iPKey ? -1 : iCol; + pExpr->pTab = pTab; + if( iCol>=0 ){ + testcase( iCol==31 ); + testcase( iCol==32 ); + *piColMask |= ((u32)1<=32?0xffffffff:0); + } + break; + } + } + } + } +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ + + /* + ** Perhaps the name is a reference to the ROWID + */ + if( cnt==0 && cntTab==1 && sqlite3IsRowid(zCol) ){ + cnt = 1; + pExpr->iColumn = -1; + pExpr->affinity = SQLITE_AFF_INTEGER; + } + + /* + ** If the input is of the form Z (not Y.Z or X.Y.Z) then the name Z + ** might refer to an result-set alias. This happens, for example, when + ** we are resolving names in the WHERE clause of the following command: + ** + ** SELECT a+b AS x FROM table WHERE x<10; + ** + ** In cases like this, replace pExpr with a copy of the expression that + ** forms the result set entry ("a+b" in the example) and return immediately. + ** Note that the expression in the result set should have already been + ** resolved by the time the WHERE clause is resolved. + */ + if( cnt==0 && (pEList = pNC->pEList)!=0 && zTab==0 ){ + for(j=0; jnExpr; j++){ + char *zAs = pEList->a[j].zName; + if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ + Expr *pOrig; + assert( pExpr->pLeft==0 && pExpr->pRight==0 ); + assert( pExpr->pList==0 ); + assert( pExpr->pSelect==0 ); + pOrig = pEList->a[j].pExpr; + if( !pNC->allowAgg && ExprHasProperty(pOrig, EP_Agg) ){ + sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); + sqlite3DbFree(db, zCol); + return 2; + } + resolveAlias(pParse, pEList, j, pExpr, ""); + cnt = 1; + pMatch = 0; + assert( zTab==0 && zDb==0 ); + goto lookupname_end_2; + } + } + } + + /* Advance to the next name context. The loop will exit when either + ** we have a match (cnt>0) or when we run out of name contexts. + */ + if( cnt==0 ){ + pNC = pNC->pNext; + } + } + + /* + ** If X and Y are NULL (in other words if only the column name Z is + ** supplied) and the value of Z is enclosed in double-quotes, then + ** Z is a string literal if it doesn't match any column names. In that + ** case, we need to return right away and not make any changes to + ** pExpr. + ** + ** Because no reference was made to outer contexts, the pNC->nRef + ** fields are not changed in any context. + */ + if( cnt==0 && zTab==0 && pColumnToken->z[0]=='"' ){ + sqlite3DbFree(db, zCol); + pExpr->op = TK_STRING; + pExpr->pTab = 0; + return 0; + } + + /* + ** cnt==0 means there was not match. cnt>1 means there were two or + ** more matches. Either way, we have an error. + */ + if( cnt!=1 ){ + const char *zErr; + zErr = cnt==0 ? "no such column" : "ambiguous column name"; + if( zDb ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); + }else if( zTab ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); + } + pTopNC->nErr++; + } + + /* If a column from a table in pSrcList is referenced, then record + ** this fact in the pSrcList.a[].colUsed bitmask. Column 0 causes + ** bit 0 to be set. Column 1 sets bit 1. And so forth. If the + ** column number is greater than the number of bits in the bitmask + ** then set the high-order bit of the bitmask. + */ + if( pExpr->iColumn>=0 && pMatch!=0 ){ + int n = pExpr->iColumn; + testcase( n==BMS-1 ); + if( n>=BMS ){ + n = BMS-1; + } + assert( pMatch->iCursor==pExpr->iTable ); + pMatch->colUsed |= ((Bitmask)1)<pLeft); + pExpr->pLeft = 0; + sqlite3ExprDelete(db, pExpr->pRight); + pExpr->pRight = 0; + pExpr->op = TK_COLUMN; +lookupname_end_2: + sqlite3DbFree(db, zCol); + if( cnt==1 ){ + assert( pNC!=0 ); + sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + /* Increment the nRef value on all name contexts from TopNC up to + ** the point where the name matched. */ + for(;;){ + assert( pTopNC!=0 ); + pTopNC->nRef++; + if( pTopNC==pNC ) break; + pTopNC = pTopNC->pNext; + } + return 0; + } else { + return 1; + } +} + +/* +** This routine is callback for sqlite3WalkExpr(). +** +** Resolve symbolic names into TK_COLUMN operators for the current +** node in the expression tree. Return 0 to continue the search down +** the tree or 2 to abort the tree walk. +** +** This routine also does error checking and name resolution for +** function names. The operator for aggregate functions is changed +** to TK_AGG_FUNCTION. +*/ +static int resolveExprStep(Walker *pWalker, Expr *pExpr){ + NameContext *pNC; + Parse *pParse; + + pNC = pWalker->u.pNC; + assert( pNC!=0 ); + pParse = pNC->pParse; + assert( pParse==pWalker->pParse ); + + if( ExprHasAnyProperty(pExpr, EP_Resolved) ) return WRC_Prune; + ExprSetProperty(pExpr, EP_Resolved); +#ifndef NDEBUG + if( pNC->pSrcList && pNC->pSrcList->nAlloc>0 ){ + SrcList *pSrcList = pNC->pSrcList; + int i; + for(i=0; ipSrcList->nSrc; i++){ + assert( pSrcList->a[i].iCursor>=0 && pSrcList->a[i].iCursornTab); + } + } +#endif + switch( pExpr->op ){ + +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) + /* The special operator TK_ROW means use the rowid for the first + ** column in the FROM clause. This is used by the LIMIT and ORDER BY + ** clause processing on UPDATE and DELETE statements. + */ + case TK_ROW: { + SrcList *pSrcList = pNC->pSrcList; + struct SrcList_item *pItem; + assert( pSrcList && pSrcList->nSrc==1 ); + pItem = pSrcList->a; + pExpr->op = TK_COLUMN; + pExpr->pTab = pItem->pTab; + pExpr->iTable = pItem->iCursor; + pExpr->iColumn = -1; + pExpr->affinity = SQLITE_AFF_INTEGER; + break; + } +#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ + + /* A lone identifier is the name of a column. + */ + case TK_ID: { + lookupName(pParse, 0, 0, &pExpr->token, pNC, pExpr); + return WRC_Prune; + } + + /* A table name and column name: ID.ID + ** Or a database, table and column: ID.ID.ID + */ + case TK_DOT: { + Token *pColumn; + Token *pTable; + Token *pDb; + Expr *pRight; + + /* if( pSrcList==0 ) break; */ + pRight = pExpr->pRight; + if( pRight->op==TK_ID ){ + pDb = 0; + pTable = &pExpr->pLeft->token; + pColumn = &pRight->token; + }else{ + assert( pRight->op==TK_DOT ); + pDb = &pExpr->pLeft->token; + pTable = &pRight->pLeft->token; + pColumn = &pRight->pRight->token; + } + lookupName(pParse, pDb, pTable, pColumn, pNC, pExpr); + return WRC_Prune; + } + + /* Resolve function names + */ + case TK_CONST_FUNC: + case TK_FUNCTION: { + ExprList *pList = pExpr->pList; /* The argument list */ + int n = pList ? pList->nExpr : 0; /* Number of arguments */ + int no_such_func = 0; /* True if no such function exists */ + int wrong_num_args = 0; /* True if wrong number of arguments */ + int is_agg = 0; /* True if is an aggregate function */ + int auth; /* Authorization to use the function */ + int nId; /* Number of characters in function name */ + const char *zId; /* The function name. */ + FuncDef *pDef; /* Information about the function */ + u8 enc = ENC(pParse->db); /* The database encoding */ + + zId = (char*)pExpr->token.z; + nId = pExpr->token.n; + pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); + if( pDef==0 ){ + pDef = sqlite3FindFunction(pParse->db, zId, nId, -1, enc, 0); + if( pDef==0 ){ + no_such_func = 1; + }else{ + wrong_num_args = 1; + } + }else{ + is_agg = pDef->xFunc==0; + } +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pDef ){ + auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0); + if( auth!=SQLITE_OK ){ + if( auth==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized to use function: %s", + pDef->zName); + pNC->nErr++; + } + pExpr->op = TK_NULL; + return WRC_Prune; + } + } +#endif + if( is_agg && !pNC->allowAgg ){ + sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); + pNC->nErr++; + is_agg = 0; + }else if( no_such_func ){ + sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); + pNC->nErr++; + }else if( wrong_num_args ){ + sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", + nId, zId); + pNC->nErr++; + } + if( is_agg ){ + pExpr->op = TK_AGG_FUNCTION; + pNC->hasAgg = 1; + } + if( is_agg ) pNC->allowAgg = 0; + sqlite3WalkExprList(pWalker, pList); + if( is_agg ) pNC->allowAgg = 1; + /* FIX ME: Compute pExpr->affinity based on the expected return + ** type of the function + */ + return WRC_Prune; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_SELECT: + case TK_EXISTS: +#endif + case TK_IN: { + if( pExpr->pSelect ){ + int nRef = pNC->nRef; +#ifndef SQLITE_OMIT_CHECK + if( pNC->isCheck ){ + sqlite3ErrorMsg(pParse,"subqueries prohibited in CHECK constraints"); + } +#endif + sqlite3WalkSelect(pWalker, pExpr->pSelect); + assert( pNC->nRef>=nRef ); + if( nRef!=pNC->nRef ){ + ExprSetProperty(pExpr, EP_VarSelect); + } + } + break; + } +#ifndef SQLITE_OMIT_CHECK + case TK_VARIABLE: { + if( pNC->isCheck ){ + sqlite3ErrorMsg(pParse,"parameters prohibited in CHECK constraints"); + } + break; + } +#endif + } + return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; +} + +/* +** pEList is a list of expressions which are really the result set of the +** a SELECT statement. pE is a term in an ORDER BY or GROUP BY clause. +** This routine checks to see if pE is a simple identifier which corresponds +** to the AS-name of one of the terms of the expression list. If it is, +** this routine return an integer between 1 and N where N is the number of +** elements in pEList, corresponding to the matching entry. If there is +** no match, or if pE is not a simple identifier, then this routine +** return 0. +** +** pEList has been resolved. pE has not. +*/ +static int resolveAsName( + Parse *pParse, /* Parsing context for error messages */ + ExprList *pEList, /* List of expressions to scan */ + Expr *pE /* Expression we are trying to match */ +){ + int i; /* Loop counter */ + + if( pE->op==TK_ID || (pE->op==TK_STRING && pE->token.z[0]!='\'') ){ + sqlite3 *db = pParse->db; + char *zCol = sqlite3NameFromToken(db, &pE->token); + if( zCol==0 ){ + return -1; + } + for(i=0; inExpr; i++){ + char *zAs = pEList->a[i].zName; + if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ + sqlite3DbFree(db, zCol); + return i+1; + } + } + sqlite3DbFree(db, zCol); + } + return 0; +} + +/* +** pE is a pointer to an expression which is a single term in the +** ORDER BY of a compound SELECT. The expression has not been +** name resolved. +** +** At the point this routine is called, we already know that the +** ORDER BY term is not an integer index into the result set. That +** case is handled by the calling routine. +** +** Attempt to match pE against result set columns in the left-most +** SELECT statement. Return the index i of the matching column, +** as an indication to the caller that it should sort by the i-th column. +** The left-most column is 1. In other words, the value returned is the +** same integer value that would be used in the SQL statement to indicate +** the column. +** +** If there is no match, return 0. Return -1 if an error occurs. +*/ +static int resolveOrderByTermToExprList( + Parse *pParse, /* Parsing context for error messages */ + Select *pSelect, /* The SELECT statement with the ORDER BY clause */ + Expr *pE /* The specific ORDER BY term */ +){ + int i; /* Loop counter */ + ExprList *pEList; /* The columns of the result set */ + NameContext nc; /* Name context for resolving pE */ + + assert( sqlite3ExprIsInteger(pE, &i)==0 ); + pEList = pSelect->pEList; + + /* Resolve all names in the ORDER BY term expression + */ + memset(&nc, 0, sizeof(nc)); + nc.pParse = pParse; + nc.pSrcList = pSelect->pSrc; + nc.pEList = pEList; + nc.allowAgg = 1; + nc.nErr = 0; + if( sqlite3ResolveExprNames(&nc, pE) ){ + sqlite3ErrorClear(pParse); + return 0; + } + + /* Try to match the ORDER BY expression against an expression + ** in the result set. Return an 1-based index of the matching + ** result-set entry. + */ + for(i=0; inExpr; i++){ + if( sqlite3ExprCompare(pEList->a[i].pExpr, pE) ){ + return i+1; + } + } + + /* If no match, return 0. */ + return 0; +} + +/* +** Generate an ORDER BY or GROUP BY term out-of-range error. +*/ +static void resolveOutOfRangeError( + Parse *pParse, /* The error context into which to write the error */ + const char *zType, /* "ORDER" or "GROUP" */ + int i, /* The index (1-based) of the term out of range */ + int mx /* Largest permissible value of i */ +){ + sqlite3ErrorMsg(pParse, + "%r %s BY term out of range - should be " + "between 1 and %d", i, zType, mx); +} + +/* +** Analyze the ORDER BY clause in a compound SELECT statement. Modify +** each term of the ORDER BY clause is a constant integer between 1 +** and N where N is the number of columns in the compound SELECT. +** +** ORDER BY terms that are already an integer between 1 and N are +** unmodified. ORDER BY terms that are integers outside the range of +** 1 through N generate an error. ORDER BY terms that are expressions +** are matched against result set expressions of compound SELECT +** beginning with the left-most SELECT and working toward the right. +** At the first match, the ORDER BY expression is transformed into +** the integer column number. +** +** Return the number of errors seen. +*/ +static int resolveCompoundOrderBy( + Parse *pParse, /* Parsing context. Leave error messages here */ + Select *pSelect /* The SELECT statement containing the ORDER BY */ +){ + int i; + ExprList *pOrderBy; + ExprList *pEList; + sqlite3 *db; + int moreToDo = 1; + + pOrderBy = pSelect->pOrderBy; + if( pOrderBy==0 ) return 0; + db = pParse->db; +#if SQLITE_MAX_COLUMN + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause"); + return 1; + } +#endif + for(i=0; inExpr; i++){ + pOrderBy->a[i].done = 0; + } + pSelect->pNext = 0; + while( pSelect->pPrior ){ + pSelect->pPrior->pNext = pSelect; + pSelect = pSelect->pPrior; + } + while( pSelect && moreToDo ){ + struct ExprList_item *pItem; + moreToDo = 0; + pEList = pSelect->pEList; + assert( pEList!=0 ); + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + int iCol = -1; + Expr *pE, *pDup; + if( pItem->done ) continue; + pE = pItem->pExpr; + if( sqlite3ExprIsInteger(pE, &iCol) ){ + if( iCol<0 || iCol>pEList->nExpr ){ + resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); + return 1; + } + }else{ + iCol = resolveAsName(pParse, pEList, pE); + if( iCol==0 ){ + pDup = sqlite3ExprDup(db, pE); + if( !db->mallocFailed ){ + assert(pDup); + iCol = resolveOrderByTermToExprList(pParse, pSelect, pDup); + } + sqlite3ExprDelete(db, pDup); + } + if( iCol<0 ){ + return 1; + } + } + if( iCol>0 ){ + CollSeq *pColl = pE->pColl; + int flags = pE->flags & EP_ExpCollate; + sqlite3ExprDelete(db, pE); + pItem->pExpr = pE = sqlite3Expr(db, TK_INTEGER, 0, 0, 0); + if( pE==0 ) return 1; + pE->pColl = pColl; + pE->flags |= EP_IntValue | flags; + pE->iTable = iCol; + pItem->iCol = (u16)iCol; + pItem->done = 1; + }else{ + moreToDo = 1; + } + } + pSelect = pSelect->pNext; + } + for(i=0; inExpr; i++){ + if( pOrderBy->a[i].done==0 ){ + sqlite3ErrorMsg(pParse, "%r ORDER BY term does not match any " + "column in the result set", i+1); + return 1; + } + } + return 0; +} + +/* +** Check every term in the ORDER BY or GROUP BY clause pOrderBy of +** the SELECT statement pSelect. If any term is reference to a +** result set expression (as determined by the ExprList.a.iCol field) +** then convert that term into a copy of the corresponding result set +** column. +** +** If any errors are detected, add an error message to pParse and +** return non-zero. Return zero if no errors are seen. +*/ +int sqlite3ResolveOrderGroupBy( + Parse *pParse, /* Parsing context. Leave error messages here */ + Select *pSelect, /* The SELECT statement containing the clause */ + ExprList *pOrderBy, /* The ORDER BY or GROUP BY clause to be processed */ + const char *zType /* "ORDER" or "GROUP" */ +){ + int i; + sqlite3 *db = pParse->db; + ExprList *pEList; + struct ExprList_item *pItem; + + if( pOrderBy==0 || pParse->db->mallocFailed ) return 0; +#if SQLITE_MAX_COLUMN + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType); + return 1; + } +#endif + pEList = pSelect->pEList; + assert( pEList!=0 ); /* sqlite3SelectNew() guarantees this */ + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + if( pItem->iCol ){ + if( pItem->iCol>pEList->nExpr ){ + resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); + return 1; + } + resolveAlias(pParse, pEList, pItem->iCol-1, pItem->pExpr, zType); + } + } + return 0; +} + +/* +** pOrderBy is an ORDER BY or GROUP BY clause in SELECT statement pSelect. +** The Name context of the SELECT statement is pNC. zType is either +** "ORDER" or "GROUP" depending on which type of clause pOrderBy is. +** +** This routine resolves each term of the clause into an expression. +** If the order-by term is an integer I between 1 and N (where N is the +** number of columns in the result set of the SELECT) then the expression +** in the resolution is a copy of the I-th result-set expression. If +** the order-by term is an identify that corresponds to the AS-name of +** a result-set expression, then the term resolves to a copy of the +** result-set expression. Otherwise, the expression is resolved in +** the usual way - using sqlite3ResolveExprNames(). +** +** This routine returns the number of errors. If errors occur, then +** an appropriate error message might be left in pParse. (OOM errors +** excepted.) +*/ +static int resolveOrderGroupBy( + NameContext *pNC, /* The name context of the SELECT statement */ + Select *pSelect, /* The SELECT statement holding pOrderBy */ + ExprList *pOrderBy, /* An ORDER BY or GROUP BY clause to resolve */ + const char *zType /* Either "ORDER" or "GROUP", as appropriate */ +){ + int i; /* Loop counter */ + int iCol; /* Column number */ + struct ExprList_item *pItem; /* A term of the ORDER BY clause */ + Parse *pParse; /* Parsing context */ + int nResult; /* Number of terms in the result set */ + + if( pOrderBy==0 ) return 0; + nResult = pSelect->pEList->nExpr; + pParse = pNC->pParse; + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + Expr *pE = pItem->pExpr; + iCol = resolveAsName(pParse, pSelect->pEList, pE); + if( iCol<0 ){ + return 1; /* OOM error */ + } + if( iCol>0 ){ + /* If an AS-name match is found, mark this ORDER BY column as being + ** a copy of the iCol-th result-set column. The subsequent call to + ** sqlite3ResolveOrderGroupBy() will convert the expression to a + ** copy of the iCol-th result-set expression. */ + pItem->iCol = (u16)iCol; + continue; + } + if( sqlite3ExprIsInteger(pE, &iCol) ){ + /* The ORDER BY term is an integer constant. Again, set the column + ** number so that sqlite3ResolveOrderGroupBy() will convert the + ** order-by term to a copy of the result-set expression */ + if( iCol<1 ){ + resolveOutOfRangeError(pParse, zType, i+1, nResult); + return 1; + } + pItem->iCol = (u16)iCol; + continue; + } + + /* Otherwise, treat the ORDER BY term as an ordinary expression */ + pItem->iCol = 0; + if( sqlite3ResolveExprNames(pNC, pE) ){ + return 1; + } + } + return sqlite3ResolveOrderGroupBy(pParse, pSelect, pOrderBy, zType); +} + +/* +** Resolve names in the SELECT statement p and all of its descendents. +*/ +static int resolveSelectStep(Walker *pWalker, Select *p){ + NameContext *pOuterNC; /* Context that contains this SELECT */ + NameContext sNC; /* Name context of this SELECT */ + int isCompound; /* True if p is a compound select */ + int nCompound; /* Number of compound terms processed so far */ + Parse *pParse; /* Parsing context */ + ExprList *pEList; /* Result set expression list */ + int i; /* Loop counter */ + ExprList *pGroupBy; /* The GROUP BY clause */ + Select *pLeftmost; /* Left-most of SELECT of a compound */ + sqlite3 *db; /* Database connection */ + + + assert( p!=0 ); + if( p->selFlags & SF_Resolved ){ + return WRC_Prune; + } + pOuterNC = pWalker->u.pNC; + pParse = pWalker->pParse; + db = pParse->db; + + /* Normally sqlite3SelectExpand() will be called first and will have + ** already expanded this SELECT. However, if this is a subquery within + ** an expression, sqlite3ResolveExprNames() will be called without a + ** prior call to sqlite3SelectExpand(). When that happens, let + ** sqlite3SelectPrep() do all of the processing for this SELECT. + ** sqlite3SelectPrep() will invoke both sqlite3SelectExpand() and + ** this routine in the correct order. + */ + if( (p->selFlags & SF_Expanded)==0 ){ + sqlite3SelectPrep(pParse, p, pOuterNC); + return (pParse->nErr || db->mallocFailed) ? WRC_Abort : WRC_Prune; + } + + isCompound = p->pPrior!=0; + nCompound = 0; + pLeftmost = p; + while( p ){ + assert( (p->selFlags & SF_Expanded)!=0 ); + assert( (p->selFlags & SF_Resolved)==0 ); + p->selFlags |= SF_Resolved; + + /* Resolve the expressions in the LIMIT and OFFSET clauses. These + ** are not allowed to refer to any names, so pass an empty NameContext. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + if( sqlite3ResolveExprNames(&sNC, p->pLimit) || + sqlite3ResolveExprNames(&sNC, p->pOffset) ){ + return WRC_Abort; + } + + /* Set up the local name-context to pass to sqlite3ResolveExprNames() to + ** resolve the result-set expression list. + */ + sNC.allowAgg = 1; + sNC.pSrcList = p->pSrc; + sNC.pNext = pOuterNC; + + /* Resolve names in the result set. */ + pEList = p->pEList; + assert( pEList!=0 ); + for(i=0; inExpr; i++){ + Expr *pX = pEList->a[i].pExpr; + if( sqlite3ResolveExprNames(&sNC, pX) ){ + return WRC_Abort; + } + } + + /* Recursively resolve names in all subqueries + */ + for(i=0; ipSrc->nSrc; i++){ + struct SrcList_item *pItem = &p->pSrc->a[i]; + if( pItem->pSelect ){ + const char *zSavedContext = pParse->zAuthContext; + if( pItem->zName ) pParse->zAuthContext = pItem->zName; + sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); + pParse->zAuthContext = zSavedContext; + if( pParse->nErr || db->mallocFailed ) return WRC_Abort; + } + } + + /* If there are no aggregate functions in the result-set, and no GROUP BY + ** expression, do not allow aggregates in any of the other expressions. + */ + assert( (p->selFlags & SF_Aggregate)==0 ); + pGroupBy = p->pGroupBy; + if( pGroupBy || sNC.hasAgg ){ + p->selFlags |= SF_Aggregate; + }else{ + sNC.allowAgg = 0; + } + + /* If a HAVING clause is present, then there must be a GROUP BY clause. + */ + if( p->pHaving && !pGroupBy ){ + sqlite3ErrorMsg(pParse, "a GROUP BY clause is required before HAVING"); + return WRC_Abort; + } + + /* Add the expression list to the name-context before parsing the + ** other expressions in the SELECT statement. This is so that + ** expressions in the WHERE clause (etc.) can refer to expressions by + ** aliases in the result set. + ** + ** Minor point: If this is the case, then the expression will be + ** re-evaluated for each reference to it. + */ + sNC.pEList = p->pEList; + if( sqlite3ResolveExprNames(&sNC, p->pWhere) || + sqlite3ResolveExprNames(&sNC, p->pHaving) + ){ + return WRC_Abort; + } + + /* The ORDER BY and GROUP BY clauses may not refer to terms in + ** outer queries + */ + sNC.pNext = 0; + sNC.allowAgg = 1; + + /* Process the ORDER BY clause for singleton SELECT statements. + ** The ORDER BY clause for compounds SELECT statements is handled + ** below, after all of the result-sets for all of the elements of + ** the compound have been resolved. + */ + if( !isCompound && resolveOrderGroupBy(&sNC, p, p->pOrderBy, "ORDER") ){ + return WRC_Abort; + } + if( db->mallocFailed ){ + return WRC_Abort; + } + + /* Resolve the GROUP BY clause. At the same time, make sure + ** the GROUP BY clause does not contain aggregate functions. + */ + if( pGroupBy ){ + struct ExprList_item *pItem; + + if( resolveOrderGroupBy(&sNC, p, pGroupBy, "GROUP") || db->mallocFailed ){ + return WRC_Abort; + } + for(i=0, pItem=pGroupBy->a; inExpr; i++, pItem++){ + if( ExprHasProperty(pItem->pExpr, EP_Agg) ){ + sqlite3ErrorMsg(pParse, "aggregate functions are not allowed in " + "the GROUP BY clause"); + return WRC_Abort; + } + } + } + + /* Advance to the next term of the compound + */ + p = p->pPrior; + nCompound++; + } + + /* Resolve the ORDER BY on a compound SELECT after all terms of + ** the compound have been resolved. + */ + if( isCompound && resolveCompoundOrderBy(pParse, pLeftmost) ){ + return WRC_Abort; + } + + return WRC_Prune; +} + +/* +** This routine walks an expression tree and resolves references to +** table columns and result-set columns. At the same time, do error +** checking on function usage and set a flag if any aggregate functions +** are seen. +** +** To resolve table columns references we look for nodes (or subtrees) of the +** form X.Y.Z or Y.Z or just Z where +** +** X: The name of a database. Ex: "main" or "temp" or +** the symbolic name assigned to an ATTACH-ed database. +** +** Y: The name of a table in a FROM clause. Or in a trigger +** one of the special names "old" or "new". +** +** Z: The name of a column in table Y. +** +** The node at the root of the subtree is modified as follows: +** +** Expr.op Changed to TK_COLUMN +** Expr.pTab Points to the Table object for X.Y +** Expr.iColumn The column index in X.Y. -1 for the rowid. +** Expr.iTable The VDBE cursor number for X.Y +** +** +** To resolve result-set references, look for expression nodes of the +** form Z (with no X and Y prefix) where the Z matches the right-hand +** size of an AS clause in the result-set of a SELECT. The Z expression +** is replaced by a copy of the left-hand side of the result-set expression. +** Table-name and function resolution occurs on the substituted expression +** tree. For example, in: +** +** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY x; +** +** The "x" term of the order by is replaced by "a+b" to render: +** +** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY a+b; +** +** Function calls are checked to make sure that the function is +** defined and that the correct number of arguments are specified. +** If the function is an aggregate function, then the pNC->hasAgg is +** set and the opcode is changed from TK_FUNCTION to TK_AGG_FUNCTION. +** If an expression contains aggregate functions then the EP_Agg +** property on the expression is set. +** +** An error message is left in pParse if anything is amiss. The number +** if errors is returned. +*/ +int sqlite3ResolveExprNames( + NameContext *pNC, /* Namespace to resolve expressions in. */ + Expr *pExpr /* The expression to be analyzed. */ +){ + int savedHasAgg; + Walker w; + + if( pExpr==0 ) return 0; +#if SQLITE_MAX_EXPR_DEPTH>0 + { + Parse *pParse = pNC->pParse; + if( sqlite3ExprCheckHeight(pParse, pExpr->nHeight+pNC->pParse->nHeight) ){ + return 1; + } + pParse->nHeight += pExpr->nHeight; + } +#endif + savedHasAgg = pNC->hasAgg; + pNC->hasAgg = 0; + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.pParse = pNC->pParse; + w.u.pNC = pNC; + sqlite3WalkExpr(&w, pExpr); +#if SQLITE_MAX_EXPR_DEPTH>0 + pNC->pParse->nHeight -= pExpr->nHeight; +#endif + if( pNC->nErr>0 ){ + ExprSetProperty(pExpr, EP_Error); + } + if( pNC->hasAgg ){ + ExprSetProperty(pExpr, EP_Agg); + }else if( savedHasAgg ){ + pNC->hasAgg = 1; + } + return ExprHasProperty(pExpr, EP_Error); +} + + +/* +** Resolve all names in all expressions of a SELECT and in all +** decendents of the SELECT, including compounds off of p->pPrior, +** subqueries in expressions, and subqueries used as FROM clause +** terms. +** +** See sqlite3ResolveExprNames() for a description of the kinds of +** transformations that occur. +** +** All SELECT statements should have been expanded using +** sqlite3SelectExpand() prior to invoking this routine. +*/ +void sqlite3ResolveSelectNames( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + NameContext *pOuterNC /* Name context for parent SELECT statement */ +){ + Walker w; + + assert( p!=0 ); + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.pParse = pParse; + w.u.pNC = pOuterNC; + sqlite3WalkSelect(&w, p); +} diff --git a/rowset.c b/rowset.c new file mode 100644 index 0000000..08ac492 --- /dev/null +++ b/rowset.c @@ -0,0 +1,238 @@ +/* +** 2008 December 3 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module implements an object we call a "Row Set". +** +** The RowSet object is a bag of rowids. Rowids +** are inserted into the bag in an arbitrary order. Then they are +** pulled from the bag in sorted order. Rowids only appear in the +** bag once. If the same rowid is inserted multiple times, the +** second and subsequent inserts make no difference on the output. +** +** This implementation accumulates rowids in a linked list. For +** output, it first sorts the linked list (removing duplicates during +** the sort) then returns elements one by one by walking the list. +** +** Big chunks of rowid/next-ptr pairs are allocated at a time, to +** reduce the malloc overhead. +** +** $Id: rowset.c,v 1.3 2009/01/13 20:14:16 drh Exp $ +*/ +#include "sqliteInt.h" + +/* +** The number of rowset entries per allocation chunk. +*/ +#define ROWSET_ENTRY_PER_CHUNK 63 + +/* +** Each entry in a RowSet is an instance of the following +** structure: +*/ +struct RowSetEntry { + i64 v; /* ROWID value for this entry */ + struct RowSetEntry *pNext; /* Next entry on a list of all entries */ +}; + +/* +** Index entries are allocated in large chunks (instances of the +** following structure) to reduce memory allocation overhead. The +** chunks are kept on a linked list so that they can be deallocated +** when the RowSet is destroyed. +*/ +struct RowSetChunk { + struct RowSetChunk *pNext; /* Next chunk on list of them all */ + struct RowSetEntry aEntry[ROWSET_ENTRY_PER_CHUNK]; /* Allocated entries */ +}; + +/* +** A RowSet in an instance of the following structure. +** +** A typedef of this structure if found in sqliteInt.h. +*/ +struct RowSet { + struct RowSetChunk *pChunk; /* List of all chunk allocations */ + sqlite3 *db; /* The database connection */ + struct RowSetEntry *pEntry; /* List of entries in the rowset */ + struct RowSetEntry *pLast; /* Last entry on the pEntry list */ + struct RowSetEntry *pFresh; /* Source of new entry objects */ + u16 nFresh; /* Number of objects on pFresh */ + u8 isSorted; /* True if content is sorted */ +}; + +/* +** Turn bulk memory into a RowSet object. N bytes of memory +** are available at pSpace. The db pointer is used as a memory context +** for any subsequent allocations that need to occur. +** Return a pointer to the new RowSet object. +** +** It must be the case that N is sufficient to make a Rowset. If not +** an assertion fault occurs. +** +** If N is larger than the minimum, use the surplus as an initial +** allocation of entries available to be filled. +*/ +RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int N){ + RowSet *p; + assert( N >= sizeof(*p) ); + p = pSpace; + p->pChunk = 0; + p->db = db; + p->pEntry = 0; + p->pLast = 0; + p->pFresh = (struct RowSetEntry*)&p[1]; + p->nFresh = (u16)((N - sizeof(*p))/sizeof(struct RowSetEntry)); + p->isSorted = 1; + return p; +} + +/* +** Deallocate all chunks from a RowSet. +*/ +void sqlite3RowSetClear(RowSet *p){ + struct RowSetChunk *pChunk, *pNextChunk; + for(pChunk=p->pChunk; pChunk; pChunk = pNextChunk){ + pNextChunk = pChunk->pNext; + sqlite3DbFree(p->db, pChunk); + } + p->pChunk = 0; + p->nFresh = 0; + p->pEntry = 0; + p->pLast = 0; + p->isSorted = 1; +} + +/* +** Insert a new value into a RowSet. +** +** The mallocFailed flag of the database connection is set if a +** memory allocation fails. +*/ +void sqlite3RowSetInsert(RowSet *p, i64 rowid){ + struct RowSetEntry *pEntry; + struct RowSetEntry *pLast; + if( p==0 ) return; /* Must have been a malloc failure */ + if( p->nFresh==0 ){ + struct RowSetChunk *pNew; + pNew = sqlite3DbMallocRaw(p->db, sizeof(*pNew)); + if( pNew==0 ){ + return; + } + pNew->pNext = p->pChunk; + p->pChunk = pNew; + p->pFresh = pNew->aEntry; + p->nFresh = ROWSET_ENTRY_PER_CHUNK; + } + pEntry = p->pFresh++; + p->nFresh--; + pEntry->v = rowid; + pEntry->pNext = 0; + pLast = p->pLast; + if( pLast ){ + if( p->isSorted && rowid<=pLast->v ){ + p->isSorted = 0; + } + pLast->pNext = pEntry; + }else{ + assert( p->pEntry==0 ); + p->pEntry = pEntry; + } + p->pLast = pEntry; +} + +/* +** Merge two lists of RowSet entries. Remove duplicates. +** +** The input lists are assumed to be in sorted order. +*/ +static struct RowSetEntry *boolidxMerge( + struct RowSetEntry *pA, /* First sorted list to be merged */ + struct RowSetEntry *pB /* Second sorted list to be merged */ +){ + struct RowSetEntry head; + struct RowSetEntry *pTail; + + pTail = &head; + while( pA && pB ){ + assert( pA->pNext==0 || pA->v<=pA->pNext->v ); + assert( pB->pNext==0 || pB->v<=pB->pNext->v ); + if( pA->vv ){ + pTail->pNext = pA; + pA = pA->pNext; + pTail = pTail->pNext; + }else if( pB->vv ){ + pTail->pNext = pB; + pB = pB->pNext; + pTail = pTail->pNext; + }else{ + pA = pA->pNext; + } + } + if( pA ){ + assert( pA->pNext==0 || pA->v<=pA->pNext->v ); + pTail->pNext = pA; + }else{ + assert( pB==0 || pB->pNext==0 || pB->v<=pB->pNext->v ); + pTail->pNext = pB; + } + return head.pNext; +} + +/* +** Sort all elements of the RowSet into ascending order. +*/ +static void sqlite3RowSetSort(RowSet *p){ + unsigned int i; + struct RowSetEntry *pEntry; + struct RowSetEntry *aBucket[40]; + + assert( p->isSorted==0 ); + memset(aBucket, 0, sizeof(aBucket)); + while( p->pEntry ){ + pEntry = p->pEntry; + p->pEntry = pEntry->pNext; + pEntry->pNext = 0; + for(i=0; aBucket[i]; i++){ + pEntry = boolidxMerge(aBucket[i],pEntry); + aBucket[i] = 0; + } + aBucket[i] = pEntry; + } + pEntry = 0; + for(i=0; ipEntry = pEntry; + p->pLast = 0; + p->isSorted = 1; +} + +/* +** Extract the next (smallest) element from the RowSet. +** Write the element into *pRowid. Return 1 on success. Return +** 0 if the RowSet is already empty. +*/ +int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ + if( !p->isSorted ){ + sqlite3RowSetSort(p); + } + if( p->pEntry ){ + *pRowid = p->pEntry->v; + p->pEntry = p->pEntry->pNext; + if( p->pEntry==0 ){ + sqlite3RowSetClear(p); + } + return 1; + }else{ + return 0; + } +} diff --git a/select.c b/select.c index 8c29b57..432e550 100644 --- a/select.c +++ b/select.c @@ -12,7 +12,7 @@ ** This file contains C code routines that are called by the parser ** to handle SELECT statements in SQLite. ** -** $Id: select.c,v 1.354 2007/07/18 18:17:12 drh Exp $ +** $Id: select.c,v 1.498 2009/01/09 02:49:32 drh Exp $ */ #include "sqliteInt.h" @@ -21,16 +21,27 @@ ** Delete all the content of a Select structure but do not deallocate ** the select structure itself. */ -static void clearSelect(Select *p){ - sqlite3ExprListDelete(p->pEList); - sqlite3SrcListDelete(p->pSrc); - sqlite3ExprDelete(p->pWhere); - sqlite3ExprListDelete(p->pGroupBy); - sqlite3ExprDelete(p->pHaving); - sqlite3ExprListDelete(p->pOrderBy); - sqlite3SelectDelete(p->pPrior); - sqlite3ExprDelete(p->pLimit); - sqlite3ExprDelete(p->pOffset); +static void clearSelect(sqlite3 *db, Select *p){ + sqlite3ExprListDelete(db, p->pEList); + sqlite3SrcListDelete(db, p->pSrc); + sqlite3ExprDelete(db, p->pWhere); + sqlite3ExprListDelete(db, p->pGroupBy); + sqlite3ExprDelete(db, p->pHaving); + sqlite3ExprListDelete(db, p->pOrderBy); + sqlite3SelectDelete(db, p->pPrior); + sqlite3ExprDelete(db, p->pLimit); + sqlite3ExprDelete(db, p->pOffset); +} + +/* +** Initialize a SelectDest structure. +*/ +void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ + pDest->eDest = (u8)eDest; + pDest->iParm = iParm; + pDest->affinity = 0; + pDest->iMem = 0; + pDest->nMem = 0; } @@ -39,6 +50,7 @@ static void clearSelect(Select *p){ ** structure. */ Select *sqlite3SelectNew( + Parse *pParse, /* Parsing context */ ExprList *pEList, /* which columns to include in the result */ SrcList *pSrc, /* the FROM clause -- which tables to scan */ Expr *pWhere, /* the WHERE clause */ @@ -51,14 +63,15 @@ Select *sqlite3SelectNew( ){ Select *pNew; Select standin; - pNew = sqliteMalloc( sizeof(*pNew) ); - assert( !pOffset || pLimit ); /* Can't have OFFSET without LIMIT. */ + sqlite3 *db = pParse->db; + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + assert( db->mallocFailed || !pOffset || pLimit ); /* OFFSET implies LIMIT */ if( pNew==0 ){ pNew = &standin; memset(pNew, 0, sizeof(*pNew)); } if( pEList==0 ){ - pEList = sqlite3ExprListAppend(0, sqlite3Expr(TK_ALL,0,0,0), 0); + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ALL,0,0,0), 0); } pNew->pEList = pEList; pNew->pSrc = pSrc; @@ -66,18 +79,16 @@ Select *sqlite3SelectNew( pNew->pGroupBy = pGroupBy; pNew->pHaving = pHaving; pNew->pOrderBy = pOrderBy; - pNew->isDistinct = isDistinct; + pNew->selFlags = isDistinct ? SF_Distinct : 0; pNew->op = TK_SELECT; - assert( pOffset==0 || pLimit!=0 ); pNew->pLimit = pLimit; pNew->pOffset = pOffset; - pNew->iLimit = -1; - pNew->iOffset = -1; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->addrOpenEphm[2] = -1; - if( pNew==&standin) { - clearSelect(pNew); + if( db->mallocFailed ) { + clearSelect(db, pNew); + if( pNew!=&standin ) sqlite3DbFree(db, pNew); pNew = 0; } return pNew; @@ -86,10 +97,10 @@ Select *sqlite3SelectNew( /* ** Delete the given Select structure and all of its substructures. */ -void sqlite3SelectDelete(Select *p){ +void sqlite3SelectDelete(sqlite3 *db, Select *p){ if( p ){ - clearSelect(p); - sqliteFree(p); + clearSelect(db, p); + sqlite3DbFree(db, p); } } @@ -133,14 +144,14 @@ int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){ apAll[2] = pC; for(i=0; i<3 && apAll[i]; i++){ p = apAll[i]; - for(j=0; jn==keywords[j].nChar && sqlite3StrNICmp((char*)p->z, keywords[j].zKeyword, p->n)==0 ){ jointype |= keywords[j].code; break; } } - if( j>=sizeof(keywords)/sizeof(keywords[0]) ){ + if( j>=ArraySize(keywords) ){ jointype |= JT_ERROR; break; } @@ -149,12 +160,11 @@ int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){ (jointype & (JT_INNER|JT_OUTER))==(JT_INNER|JT_OUTER) || (jointype & JT_ERROR)!=0 ){ - const char *zSp1 = " "; - const char *zSp2 = " "; - if( pB==0 ){ zSp1++; } - if( pC==0 ){ zSp2++; } + const char *zSp = " "; + assert( pB!=0 ); + if( pC==0 ){ zSp++; } sqlite3ErrorMsg(pParse, "unknown or unsupported join type: " - "%T%s%T%s%T", pA, zSp1, pB, zSp2, pC); + "%T %T%s%T", pA, pB, zSp, pC); jointype = JT_INNER; }else if( jointype & JT_RIGHT ){ sqlite3ErrorMsg(pParse, @@ -181,7 +191,7 @@ static int columnIndex(Table *pTab, const char *zCol){ */ static void setToken(Token *p, const char *z){ p->z = (u8*)z; - p->n = z ? strlen(z) : 0; + p->n = z ? sqlite3Strlen30(z) : 0; p->dyn = 0; } @@ -191,62 +201,82 @@ static void setToken(Token *p, const char *z){ ** ** {a"bc} -> {"a""bc"} */ -static void setQuotedToken(Token *p, const char *z){ - p->z = (u8 *)sqlite3MPrintf("\"%w\"", z); - p->dyn = 1; - if( p->z ){ - p->n = strlen((char *)p->z); +static void setQuotedToken(Parse *pParse, Token *p, const char *z){ + + /* Check if the string appears to be quoted using "..." or `...` + ** or [...] or '...' or if the string contains any " characters. + ** If it does, then record a version of the string with the special + ** characters escaped. + */ + const char *z2 = z; + if( *z2!='[' && *z2!='`' && *z2!='\'' ){ + while( *z2 ){ + if( *z2=='"' ) break; + z2++; + } + } + + if( *z2 ){ + /* String contains " characters - copy and quote the string. */ + p->z = (u8 *)sqlite3MPrintf(pParse->db, "\"%w\"", z); + if( p->z ){ + p->n = sqlite3Strlen30((char *)p->z); + p->dyn = 1; + } + }else{ + /* String contains no " characters - copy the pointer. */ + p->z = (u8*)z; + p->n = (int)(z2 - z); + p->dyn = 0; } } /* ** Create an expression node for an identifier with the name of zName */ -Expr *sqlite3CreateIdExpr(const char *zName){ +Expr *sqlite3CreateIdExpr(Parse *pParse, const char *zName){ Token dummy; setToken(&dummy, zName); - return sqlite3Expr(TK_ID, 0, 0, &dummy); + return sqlite3PExpr(pParse, TK_ID, 0, 0, &dummy); } - /* ** Add a term to the WHERE expression in *ppExpr that requires the ** zCol column to be equal in the two tables pTab1 and pTab2. */ static void addWhereTerm( + Parse *pParse, /* Parsing context */ const char *zCol, /* Name of the column */ const Table *pTab1, /* First table */ const char *zAlias1, /* Alias for first table. May be NULL */ const Table *pTab2, /* Second table */ const char *zAlias2, /* Alias for second table. May be NULL */ int iRightJoinTable, /* VDBE cursor for the right table */ - Expr **ppExpr /* Add the equality term to this expression */ + Expr **ppExpr, /* Add the equality term to this expression */ + int isOuterJoin /* True if dealing with an OUTER join */ ){ Expr *pE1a, *pE1b, *pE1c; Expr *pE2a, *pE2b, *pE2c; Expr *pE; - pE1a = sqlite3CreateIdExpr(zCol); - pE2a = sqlite3CreateIdExpr(zCol); + pE1a = sqlite3CreateIdExpr(pParse, zCol); + pE2a = sqlite3CreateIdExpr(pParse, zCol); if( zAlias1==0 ){ zAlias1 = pTab1->zName; } - pE1b = sqlite3CreateIdExpr(zAlias1); + pE1b = sqlite3CreateIdExpr(pParse, zAlias1); if( zAlias2==0 ){ zAlias2 = pTab2->zName; } - pE2b = sqlite3CreateIdExpr(zAlias2); - pE1c = sqlite3ExprOrFree(TK_DOT, pE1b, pE1a, 0); - pE2c = sqlite3ExprOrFree(TK_DOT, pE2b, pE2a, 0); - pE = sqlite3ExprOrFree(TK_EQ, pE1c, pE2c, 0); - if( pE ){ + pE2b = sqlite3CreateIdExpr(pParse, zAlias2); + pE1c = sqlite3PExpr(pParse, TK_DOT, pE1b, pE1a, 0); + pE2c = sqlite3PExpr(pParse, TK_DOT, pE2b, pE2a, 0); + pE = sqlite3PExpr(pParse, TK_EQ, pE1c, pE2c, 0); + if( pE && isOuterJoin ){ ExprSetProperty(pE, EP_FromJoin); pE->iRightJoinTable = iRightJoinTable; } - pE = sqlite3ExprAnd(*ppExpr, pE); - if( pE ){ - *ppExpr = pE; - } + *ppExpr = sqlite3ExprAnd(pParse->db,*ppExpr, pE); } /* @@ -310,8 +340,10 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ for(i=0; inSrc-1; i++, pRight++, pLeft++){ Table *pLeftTab = pLeft->pTab; Table *pRightTab = pRight->pTab; + int isOuter; - if( pLeftTab==0 || pRightTab==0 ) continue; + if( NEVER(pLeftTab==0 || pRightTab==0) ) continue; + isOuter = (pRight->jointype & JT_OUTER)!=0; /* When the NATURAL keyword is present, add WHERE clause terms for ** every column that the two tables have in common. @@ -325,9 +357,9 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ for(j=0; jnCol; j++){ char *zName = pLeftTab->aCol[j].zName; if( columnIndex(pRightTab, zName)>=0 ){ - addWhereTerm(zName, pLeftTab, pLeft->zAlias, + addWhereTerm(pParse, zName, pLeftTab, pLeft->zAlias, pRightTab, pRight->zAlias, - pRight->iCursor, &p->pWhere); + pRight->iCursor, &p->pWhere, isOuter); } } @@ -345,8 +377,8 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ ** an AND operator. */ if( pRight->pOn ){ - setJoinExpr(pRight->pOn, pRight->iCursor); - p->pWhere = sqlite3ExprAnd(p->pWhere, pRight->pOn); + if( isOuter ) setJoinExpr(pRight->pOn, pRight->iCursor); + p->pWhere = sqlite3ExprAnd(pParse->db, p->pWhere, pRight->pOn); pRight->pOn = 0; } @@ -366,9 +398,9 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ "not present in both tables", zName); return 1; } - addWhereTerm(zName, pLeftTab, pLeft->zAlias, + addWhereTerm(pParse, zName, pLeftTab, pLeft->zAlias, pRightTab, pRight->zAlias, - pRight->iCursor, &p->pWhere); + pRight->iCursor, &p->pWhere, isOuter); } } } @@ -382,24 +414,36 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ static void pushOntoSorter( Parse *pParse, /* Parser context */ ExprList *pOrderBy, /* The ORDER BY clause */ - Select *pSelect /* The whole SELECT statement */ + Select *pSelect, /* The whole SELECT statement */ + int regData /* Register holding data to be sorted */ ){ Vdbe *v = pParse->pVdbe; - sqlite3ExprCodeExprList(pParse, pOrderBy); - sqlite3VdbeAddOp(v, OP_Sequence, pOrderBy->iECursor, 0); - sqlite3VdbeAddOp(v, OP_Pull, pOrderBy->nExpr + 1, 0); - sqlite3VdbeAddOp(v, OP_MakeRecord, pOrderBy->nExpr + 2, 0); - sqlite3VdbeAddOp(v, OP_IdxInsert, pOrderBy->iECursor, 0); - if( pSelect->iLimit>=0 ){ + int nExpr = pOrderBy->nExpr; + int regBase = sqlite3GetTempRange(pParse, nExpr+2); + int regRecord = sqlite3GetTempReg(pParse); + sqlite3ExprCodeExprList(pParse, pOrderBy, regBase, 0); + sqlite3VdbeAddOp2(v, OP_Sequence, pOrderBy->iECursor, regBase+nExpr); + sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nExpr + 2, regRecord); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pOrderBy->iECursor, regRecord); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); + if( pSelect->iLimit ){ int addr1, addr2; - addr1 = sqlite3VdbeAddOp(v, OP_IfMemZero, pSelect->iLimit+1, 0); - sqlite3VdbeAddOp(v, OP_MemIncr, -1, pSelect->iLimit+1); - addr2 = sqlite3VdbeAddOp(v, OP_Goto, 0, 0); + int iLimit; + if( pSelect->iOffset ){ + iLimit = pSelect->iOffset+1; + }else{ + iLimit = pSelect->iLimit; + } + addr1 = sqlite3VdbeAddOp1(v, OP_IfZero, iLimit); + sqlite3VdbeAddOp2(v, OP_AddImm, iLimit, -1); + addr2 = sqlite3VdbeAddOp0(v, OP_Goto); sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp(v, OP_Last, pOrderBy->iECursor, 0); - sqlite3VdbeAddOp(v, OP_Delete, pOrderBy->iECursor, 0); + sqlite3VdbeAddOp1(v, OP_Last, pOrderBy->iECursor); + sqlite3VdbeAddOp1(v, OP_Delete, pOrderBy->iECursor); sqlite3VdbeJumpHere(v, addr2); - pSelect->iLimit = -1; + pSelect->iLimit = 0; } } @@ -409,25 +453,21 @@ static void pushOntoSorter( static void codeOffset( Vdbe *v, /* Generate code into this VM */ Select *p, /* The SELECT statement being coded */ - int iContinue, /* Jump here to skip the current record */ - int nPop /* Number of times to pop stack when jumping */ + int iContinue /* Jump here to skip the current record */ ){ - if( p->iOffset>=0 && iContinue!=0 ){ + if( p->iOffset && iContinue!=0 ){ int addr; - sqlite3VdbeAddOp(v, OP_MemIncr, -1, p->iOffset); - addr = sqlite3VdbeAddOp(v, OP_IfMemNeg, p->iOffset, 0); - if( nPop>0 ){ - sqlite3VdbeAddOp(v, OP_Pop, nPop, 0); - } - sqlite3VdbeAddOp(v, OP_Goto, 0, iContinue); - VdbeComment((v, "# skip OFFSET records")); + sqlite3VdbeAddOp2(v, OP_AddImm, p->iOffset, -1); + addr = sqlite3VdbeAddOp1(v, OP_IfNeg, p->iOffset); + sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); + VdbeComment((v, "skip OFFSET records")); sqlite3VdbeJumpHere(v, addr); } } /* -** Add code that will check to make sure the top N elements of the -** stack are distinct. iTab is a sorting index that holds previously +** Add code that will check to make sure the N registers starting at iMem +** form a distinct entry. iTab is a sorting index that holds previously ** seen combinations of the N values. A new entry is made in iTab ** if the current N values are new. ** @@ -435,17 +475,21 @@ static void codeOffset( ** stack if the top N elements are not distinct. */ static void codeDistinct( - Vdbe *v, /* Generate code into this VM */ + Parse *pParse, /* Parsing and code generating context */ int iTab, /* A sorting index used to test for distinctness */ int addrRepeat, /* Jump to here if not distinct */ - int N /* The top N elements of the stack must be distinct */ + int N, /* Number of elements */ + int iMem /* First element */ ){ - sqlite3VdbeAddOp(v, OP_MakeRecord, -N, 0); - sqlite3VdbeAddOp(v, OP_Distinct, iTab, sqlite3VdbeCurrentAddr(v)+3); - sqlite3VdbeAddOp(v, OP_Pop, N+1, 0); - sqlite3VdbeAddOp(v, OP_Goto, 0, addrRepeat); - VdbeComment((v, "# skip indistinct records")); - sqlite3VdbeAddOp(v, OP_IdxInsert, iTab, 0); + Vdbe *v; + int r1; + + v = pParse->pVdbe; + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); + sqlite3VdbeAddOp3(v, OP_Found, iTab, addrRepeat, r1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1); + sqlite3ReleaseTempReg(pParse, r1); } /* @@ -454,7 +498,12 @@ static void codeDistinct( ** column. We do this in a subroutine because the error occurs in multiple ** places. */ -static int checkForMultiColumnSelectError(Parse *pParse, int eDest, int nExpr){ +static int checkForMultiColumnSelectError( + Parse *pParse, /* Parse context. */ + SelectDest *pDest, /* Destination of SELECT results */ + int nExpr /* Number of result columns returned by SELECT */ +){ + int eDest = pDest->eDest; if( nExpr>1 && (eDest==SRT_Mem || eDest==SRT_Set) ){ sqlite3ErrorMsg(pParse, "only a single result allowed for " "a SELECT that is part of an expression"); @@ -473,7 +522,7 @@ static int checkForMultiColumnSelectError(Parse *pParse, int eDest, int nExpr){ ** then data is pulled from srcTab and pEList is used only to get the ** datatypes for each column. */ -static int selectInnerLoop( +static void selectInnerLoop( Parse *pParse, /* The parser context */ Select *p, /* The complete select statement being coded */ ExprList *pEList, /* List of values being extracted */ @@ -481,37 +530,52 @@ static int selectInnerLoop( int nColumn, /* Number of columns in the source table */ ExprList *pOrderBy, /* If not NULL, sort results using this key */ int distinct, /* If >=0, make sure results are distinct */ - int eDest, /* How to dispose of the results */ - int iParm, /* An argument to the disposal method */ + SelectDest *pDest, /* How to dispose of the results */ int iContinue, /* Jump here to continue with next row */ - int iBreak, /* Jump here to break out of the inner loop */ - char *aff /* affinity string if eDest is SRT_Union */ + int iBreak /* Jump here to break out of the inner loop */ ){ Vdbe *v = pParse->pVdbe; int i; int hasDistinct; /* True if the DISTINCT keyword is present */ + int regResult; /* Start of memory holding result set */ + int eDest = pDest->eDest; /* How to dispose of results */ + int iParm = pDest->iParm; /* First argument to disposal method */ + int nResultCol; /* Number of result columns */ - if( v==0 ) return 0; + assert( v ); + if( NEVER(v==0) ) return; assert( pEList!=0 ); - - /* If there was a LIMIT clause on the SELECT statement, then do the check - ** to see if this row should be output. - */ - hasDistinct = distinct>=0 && pEList->nExpr>0; + hasDistinct = distinct>=0; if( pOrderBy==0 && !hasDistinct ){ - codeOffset(v, p, iContinue, 0); + codeOffset(v, p, iContinue); } /* Pull the requested columns. */ if( nColumn>0 ){ - for(i=0; inExpr; - sqlite3ExprCodeExprList(pParse, pEList); + nResultCol = pEList->nExpr; } + if( pDest->iMem==0 ){ + pDest->iMem = pParse->nMem+1; + pDest->nMem = nResultCol; + pParse->nMem += nResultCol; + }else{ + assert( pDest->nMem==nResultCol ); + } + regResult = pDest->iMem; + if( nColumn>0 ){ + for(i=0; inExpr==nColumn ); - codeDistinct(v, distinct, iContinue, nColumn); + codeDistinct(pParse, distinct, iContinue, nColumn, regResult); if( pOrderBy==0 ){ - codeOffset(v, p, iContinue, nColumn); + codeOffset(v, p, iContinue); } } - if( checkForMultiColumnSelectError(pParse, eDest, pEList->nExpr) ){ - return 0; + if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ + return; } switch( eDest ){ @@ -536,11 +600,11 @@ static int selectInnerLoop( */ #ifndef SQLITE_OMIT_COMPOUND_SELECT case SRT_Union: { - sqlite3VdbeAddOp(v, OP_MakeRecord, nColumn, 0); - if( aff ){ - sqlite3VdbeChangeP3(v, -1, aff, P3_STATIC); - } - sqlite3VdbeAddOp(v, OP_IdxInsert, iParm, 0); + int r1; + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + sqlite3ReleaseTempReg(pParse, r1); break; } @@ -549,11 +613,7 @@ static int selectInnerLoop( ** the temporary table iParm. */ case SRT_Except: { - int addr; - addr = sqlite3VdbeAddOp(v, OP_MakeRecord, nColumn, 0); - sqlite3VdbeChangeP3(v, -1, aff, P3_STATIC); - sqlite3VdbeAddOp(v, OP_NotFound, iParm, addr+3); - sqlite3VdbeAddOp(v, OP_Delete, iParm, 0); + sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nColumn); break; } #endif @@ -562,14 +622,18 @@ static int selectInnerLoop( */ case SRT_Table: case SRT_EphemTab: { - sqlite3VdbeAddOp(v, OP_MakeRecord, nColumn, 0); + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); if( pOrderBy ){ - pushOntoSorter(pParse, pOrderBy, p); + pushOntoSorter(pParse, pOrderBy, p, r1); }else{ - sqlite3VdbeAddOp(v, OP_NewRowid, iParm, 0); - sqlite3VdbeAddOp(v, OP_Pull, 1, 0); - sqlite3VdbeAddOp(v, OP_Insert, iParm, OPFLAG_APPEND); + int r2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3ReleaseTempReg(pParse, r2); } + sqlite3ReleaseTempReg(pParse, r1); break; } @@ -579,33 +643,28 @@ static int selectInnerLoop( ** item into the set table with bogus data. */ case SRT_Set: { - int addr1 = sqlite3VdbeCurrentAddr(v); - int addr2; - assert( nColumn==1 ); - sqlite3VdbeAddOp(v, OP_NotNull, -1, addr1+3); - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); - addr2 = sqlite3VdbeAddOp(v, OP_Goto, 0, 0); - p->affinity = sqlite3CompareAffinity(pEList->a[0].pExpr,(iParm>>16)&0xff); + p->affinity = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affinity); if( pOrderBy ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pOrderBy, p); + pushOntoSorter(pParse, pOrderBy, p, regResult); }else{ - sqlite3VdbeOp3(v, OP_MakeRecord, 1, 0, &p->affinity, 1); - sqlite3VdbeAddOp(v, OP_IdxInsert, (iParm&0x0000FFFF), 0); + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, 1, r1, &p->affinity, 1); + sqlite3ExprCacheAffinityChange(pParse, regResult, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + sqlite3ReleaseTempReg(pParse, r1); } - sqlite3VdbeJumpHere(v, addr2); break; } /* If any row exist in the result set, record that fact and abort. */ case SRT_Exists: { - sqlite3VdbeAddOp(v, OP_MemInt, 1, iParm); - sqlite3VdbeAddOp(v, OP_Pop, nColumn, 0); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iParm); /* The LIMIT clause will terminate the loop for us */ break; } @@ -617,9 +676,9 @@ static int selectInnerLoop( case SRT_Mem: { assert( nColumn==1 ); if( pOrderBy ){ - pushOntoSorter(pParse, pOrderBy, p); + pushOntoSorter(pParse, pOrderBy, p, regResult); }else{ - sqlite3VdbeAddOp(v, OP_MemStore, iParm, 1); + sqlite3ExprCodeMove(pParse, regResult, iParm, 1); /* The LIMIT clause will jump out of the loop for us */ } break; @@ -630,15 +689,18 @@ static int selectInnerLoop( ** case of a subroutine, the subroutine itself is responsible for ** popping the data from the stack. */ - case SRT_Subroutine: - case SRT_Callback: { + case SRT_Coroutine: + case SRT_Output: { if( pOrderBy ){ - sqlite3VdbeAddOp(v, OP_MakeRecord, nColumn, 0); - pushOntoSorter(pParse, pOrderBy, p); - }else if( eDest==SRT_Subroutine ){ - sqlite3VdbeAddOp(v, OP_Gosub, 0, iParm); + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + pushOntoSorter(pParse, pOrderBy, p, r1); + sqlite3ReleaseTempReg(pParse, r1); + }else if( eDest==SRT_Coroutine ){ + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); }else{ - sqlite3VdbeAddOp(v, OP_Callback, nColumn, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nColumn); + sqlite3ExprCacheAffinityChange(pParse, regResult, nColumn); } break; } @@ -651,7 +713,6 @@ static int selectInnerLoop( */ default: { assert( eDest==SRT_Discard ); - sqlite3VdbeAddOp(v, OP_Pop, nColumn, 0); break; } #endif @@ -659,11 +720,12 @@ static int selectInnerLoop( /* Jump to the end of the loop if the LIMIT is reached. */ - if( p->iLimit>=0 && pOrderBy==0 ){ - sqlite3VdbeAddOp(v, OP_MemIncr, -1, p->iLimit); - sqlite3VdbeAddOp(v, OP_IfMemZero, p->iLimit, iBreak); + if( p->iLimit ){ + assert( pOrderBy==0 ); /* If there is an ORDER BY, the call to + ** pushOntoSorter() would have cleared p->iLimit */ + sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1); + sqlite3VdbeAddOp2(v, OP_IfZero, p->iLimit, iBreak); } - return 0; } /* @@ -678,8 +740,8 @@ static int selectInnerLoop( ** ** Space to hold the KeyInfo structure is obtain from malloc. The calling ** function is responsible for seeing that this structure is eventually -** freed. Add the KeyInfo structure to the P3 field of an opcode using -** P3_KEYINFO_HANDOFF is the usual way of dealing with this. +** freed. Add the KeyInfo structure to the P4 field of an opcode using +** P4_KEYINFO_HANDOFF is the usual way of dealing with this. */ static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ sqlite3 *db = pParse->db; @@ -689,11 +751,12 @@ static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ int i; nExpr = pList->nExpr; - pInfo = sqliteMalloc( sizeof(*pInfo) + nExpr*(sizeof(CollSeq*)+1) ); + pInfo = sqlite3DbMallocZero(db, sizeof(*pInfo) + nExpr*(sizeof(CollSeq*)+1) ); if( pInfo ){ pInfo->aSortOrder = (u8*)&pInfo->aColl[nExpr]; - pInfo->nField = nExpr; + pInfo->nField = (u16)nExpr; pInfo->enc = ENC(db); + pInfo->db = db; for(i=0, pItem=pList->a; ipExpr); @@ -715,68 +778,77 @@ static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ ** routine generates the code needed to do that. */ static void generateSortTail( - Parse *pParse, /* Parsing context */ - Select *p, /* The SELECT statement */ - Vdbe *v, /* Generate code into this VDBE */ - int nColumn, /* Number of columns of data */ - int eDest, /* Write the sorted results here */ - int iParm /* Optional parameter associated with eDest */ + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement */ + Vdbe *v, /* Generate code into this VDBE */ + int nColumn, /* Number of columns of data */ + SelectDest *pDest /* Write the sorted results here */ ){ - int brk = sqlite3VdbeMakeLabel(v); - int cont = sqlite3VdbeMakeLabel(v); + int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ + int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; int iTab; int pseudoTab = 0; ExprList *pOrderBy = p->pOrderBy; + int eDest = pDest->eDest; + int iParm = pDest->iParm; + + int regRow; + int regRowid; + iTab = pOrderBy->iECursor; - if( eDest==SRT_Callback || eDest==SRT_Subroutine ){ + if( eDest==SRT_Output || eDest==SRT_Coroutine ){ pseudoTab = pParse->nTab++; - sqlite3VdbeAddOp(v, OP_OpenPseudo, pseudoTab, 0); - sqlite3VdbeAddOp(v, OP_SetNumColumns, pseudoTab, nColumn); + sqlite3VdbeAddOp2(v, OP_SetNumColumns, 0, nColumn); + sqlite3VdbeAddOp2(v, OP_OpenPseudo, pseudoTab, eDest==SRT_Output); } - addr = 1 + sqlite3VdbeAddOp(v, OP_Sort, iTab, brk); - codeOffset(v, p, cont, 0); - if( eDest==SRT_Callback || eDest==SRT_Subroutine ){ - sqlite3VdbeAddOp(v, OP_Integer, 1, 0); - } - sqlite3VdbeAddOp(v, OP_Column, iTab, pOrderBy->nExpr + 1); + addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); + codeOffset(v, p, addrContinue); + regRow = sqlite3GetTempReg(pParse); + regRowid = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr + 1, regRow); switch( eDest ){ case SRT_Table: case SRT_EphemTab: { - sqlite3VdbeAddOp(v, OP_NewRowid, iParm, 0); - sqlite3VdbeAddOp(v, OP_Pull, 1, 0); - sqlite3VdbeAddOp(v, OP_Insert, iParm, OPFLAG_APPEND); + testcase( eDest==SRT_Table ); + testcase( eDest==SRT_EphemTab ); + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); break; } #ifndef SQLITE_OMIT_SUBQUERY case SRT_Set: { assert( nColumn==1 ); - sqlite3VdbeAddOp(v, OP_NotNull, -1, sqlite3VdbeCurrentAddr(v)+3); - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); - sqlite3VdbeAddOp(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+3); - sqlite3VdbeOp3(v, OP_MakeRecord, 1, 0, &p->affinity, 1); - sqlite3VdbeAddOp(v, OP_IdxInsert, (iParm&0x0000FFFF), 0); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, &p->affinity, 1); + sqlite3ExprCacheAffinityChange(pParse, regRow, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); break; } case SRT_Mem: { assert( nColumn==1 ); - sqlite3VdbeAddOp(v, OP_MemStore, iParm, 1); + sqlite3ExprCodeMove(pParse, regRow, iParm, 1); /* The LIMIT clause will terminate the loop for us */ break; } #endif - case SRT_Callback: - case SRT_Subroutine: { + case SRT_Output: + case SRT_Coroutine: { int i; - sqlite3VdbeAddOp(v, OP_Insert, pseudoTab, 0); + testcase( eDest==SRT_Output ); + testcase( eDest==SRT_Coroutine ); + sqlite3VdbeAddOp2(v, OP_Integer, 1, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, pseudoTab, regRow, regRowid); for(i=0; iiMem+i ); + sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iMem+i); } - if( eDest==SRT_Callback ){ - sqlite3VdbeAddOp(v, OP_Callback, nColumn, 0); + if( eDest==SRT_Output ){ + sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iMem, nColumn); + sqlite3ExprCacheAffinityChange(pParse, pDest->iMem, nColumn); }else{ - sqlite3VdbeAddOp(v, OP_Gosub, 0, iParm); + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); } break; } @@ -785,23 +857,21 @@ static void generateSortTail( break; } } + sqlite3ReleaseTempReg(pParse, regRow); + sqlite3ReleaseTempReg(pParse, regRowid); - /* Jump to the end of the loop when the LIMIT is reached + /* LIMIT has been implemented by the pushOntoSorter() routine. */ - if( p->iLimit>=0 ){ - sqlite3VdbeAddOp(v, OP_MemIncr, -1, p->iLimit); - sqlite3VdbeAddOp(v, OP_IfMemZero, p->iLimit, brk); - } + assert( p->iLimit==0 ); /* The bottom of the loop */ - sqlite3VdbeResolveLabel(v, cont); - sqlite3VdbeAddOp(v, OP_Next, iTab, addr); - sqlite3VdbeResolveLabel(v, brk); - if( eDest==SRT_Callback || eDest==SRT_Subroutine ){ - sqlite3VdbeAddOp(v, OP_Close, pseudoTab, 0); + sqlite3VdbeResolveLabel(v, addrContinue); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); + sqlite3VdbeResolveLabel(v, addrBreak); + if( eDest==SRT_Output || eDest==SRT_Coroutine ){ + sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); } - } /* @@ -878,7 +948,7 @@ static const char *columnType( ** of the SELECT statement. Return the declaration type and origin ** data for the result-set column of the sub-select. */ - if( iCol>=0 && iColpEList->nExpr ){ + if( ALWAYS(iCol>=0 && iColpEList->nExpr) ){ /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see ** test case misc2.2.2) - it always evaluates to NULL. @@ -890,7 +960,7 @@ static const char *columnType( sNC.pParse = pNC->pParse; zType = columnType(&sNC, p, &zOriginDb, &zOriginTab, &zOriginCol); } - }else if( pTab->pSchema ){ + }else if( ALWAYS(pTab->pSchema) ){ /* A real table */ assert( !pS ); if( iCol<0 ) iCol = pTab->iPKey; @@ -946,6 +1016,7 @@ static void generateColumnTypes( SrcList *pTabList, /* List of tables */ ExprList *pEList /* Expressions defining the result set */ ){ +#ifndef SQLITE_OMIT_DECLTYPE Vdbe *v = pParse->pVdbe; int i; NameContext sNC; @@ -953,20 +1024,26 @@ static void generateColumnTypes( sNC.pParse = pParse; for(i=0; inExpr; i++){ Expr *p = pEList->a[i].pExpr; + const char *zType; +#ifdef SQLITE_ENABLE_COLUMN_METADATA const char *zOrigDb = 0; const char *zOrigTab = 0; const char *zOrigCol = 0; - const char *zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); - /* The vdbe must make it's own copy of the column-type and other + /* The vdbe must make its own copy of the column-type and other ** column specific strings, in case the schema is reset before this ** virtual machine is deleted. */ - sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, P3_TRANSIENT); - sqlite3VdbeSetColName(v, i, COLNAME_DATABASE, zOrigDb, P3_TRANSIENT); - sqlite3VdbeSetColName(v, i, COLNAME_TABLE, zOrigTab, P3_TRANSIENT); - sqlite3VdbeSetColName(v, i, COLNAME_COLUMN, zOrigCol, P3_TRANSIENT); + sqlite3VdbeSetColName(v, i, COLNAME_DATABASE, zOrigDb, SQLITE_TRANSIENT); + sqlite3VdbeSetColName(v, i, COLNAME_TABLE, zOrigTab, SQLITE_TRANSIENT); + sqlite3VdbeSetColName(v, i, COLNAME_COLUMN, zOrigCol, SQLITE_TRANSIENT); +#else + zType = columnType(&sNC, p, 0, 0, 0); +#endif + sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); } +#endif /* SQLITE_OMIT_DECLTYPE */ } /* @@ -992,7 +1069,7 @@ static void generateColumnNames( #endif assert( v!=0 ); - if( pParse->colNamesSet || v==0 || sqlite3MallocFailed() ) return; + if( pParse->colNamesSet || NEVER(v==0) || db->mallocFailed ) return; pParse->colNamesSet = 1; fullNames = (db->flags & SQLITE_FullColNames)!=0; shortNames = (db->flags & SQLITE_ShortColNames)!=0; @@ -1003,14 +1080,14 @@ static void generateColumnNames( if( p==0 ) continue; if( pEList->a[i].zName ){ char *zName = pEList->a[i].zName; - sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, strlen(zName)); - continue; - } - if( p->op==TK_COLUMN && pTabList ){ + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); + }else if( (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) && pTabList ){ Table *pTab; char *zCol; int iCol = p->iColumn; - for(j=0; jnSrc && pTabList->a[j].iCursor!=p->iTable; j++){} + for(j=0; ALWAYS(jnSrc); j++){ + if( pTabList->a[j].iCursor==p->iTable ) break; + } assert( jnSrc ); pTab = pTabList->a[j].pTab; if( iCol<0 ) iCol = pTab->iPKey; @@ -1020,27 +1097,19 @@ static void generateColumnNames( }else{ zCol = pTab->aCol[iCol].zName; } - if( !shortNames && !fullNames && p->span.z && p->span.z[0] ){ - sqlite3VdbeSetColName(v, i, COLNAME_NAME, (char*)p->span.z, p->span.n); - }else if( fullNames || (!shortNames && pTabList->nSrc>1) ){ + if( !shortNames && !fullNames ){ + sqlite3VdbeSetColName(v, i, COLNAME_NAME, + sqlite3DbStrNDup(db, (char*)p->span.z, p->span.n), SQLITE_DYNAMIC); + }else if( fullNames ){ char *zName = 0; - char *zTab; - - zTab = pTabList->a[j].zAlias; - if( fullNames || zTab==0 ) zTab = pTab->zName; - sqlite3SetString(&zName, zTab, ".", zCol, (char*)0); - sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, P3_DYNAMIC); + zName = sqlite3MPrintf(db, "%s.%s", pTab->zName, zCol); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_DYNAMIC); }else{ - sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, strlen(zCol)); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT); } - }else if( p->span.z && p->span.z[0] ){ - sqlite3VdbeSetColName(v, i, COLNAME_NAME, (char*)p->span.z, p->span.n); - /* sqlite3VdbeCompressSpace(v, addr); */ }else{ - char zName[30]; - assert( p->op!=TK_COLUMN || pTabList==0 ); - sqlite3_snprintf(sizeof(zName), zName, "column%d", i+1); - sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, 0); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, + sqlite3DbStrNDup(db, (char*)p->span.z, p->span.n), SQLITE_DYNAMIC); } } generateColumnTypes(pParse, pTabList, pEList); @@ -1063,440 +1132,171 @@ static const char *selectOpName(int id){ #endif /* SQLITE_OMIT_COMPOUND_SELECT */ /* -** Forward declaration +** Given a an expression list (which is really the list of expressions +** that form the result set of a SELECT statement) compute appropriate +** column names for a table that would hold the expression list. +** +** All column names will be unique. +** +** Only the column names are computed. Column.zType, Column.zColl, +** and other fields of Column are zeroed. +** +** Return SQLITE_OK on success. If a memory allocation error occurs, +** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. */ -static int prepSelectStmt(Parse*, Select*); +static int selectColumnsFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* Expr list from which to derive column names */ + int *pnCol, /* Write the number of columns here */ + Column **paCol /* Write the new column list here */ +){ + sqlite3 *db = pParse->db; /* Database connection */ + int i, j; /* Loop counters */ + int cnt; /* Index added to make the name unique */ + Column *aCol, *pCol; /* For looping over result columns */ + int nCol; /* Number of columns in the result set */ + Expr *p; /* Expression for a single result column */ + char *zName; /* Column name */ + int nName; /* Size of name in zName[] */ -/* -** Given a SELECT statement, generate a Table structure that describes -** the result set of that SELECT. -*/ -Table *sqlite3ResultSetOfSelect(Parse *pParse, char *zTabName, Select *pSelect){ - Table *pTab; - int i, j; - ExprList *pEList; - Column *aCol, *pCol; - - while( pSelect->pPrior ) pSelect = pSelect->pPrior; - if( prepSelectStmt(pParse, pSelect) ){ - return 0; - } - if( sqlite3SelectResolve(pParse, pSelect, 0) ){ - return 0; - } - pTab = sqliteMalloc( sizeof(Table) ); - if( pTab==0 ){ - return 0; - } - pTab->nRef = 1; - pTab->zName = zTabName ? sqliteStrDup(zTabName) : 0; - pEList = pSelect->pEList; - pTab->nCol = pEList->nExpr; - assert( pTab->nCol>0 ); - pTab->aCol = aCol = sqliteMalloc( sizeof(pTab->aCol[0])*pTab->nCol ); - for(i=0, pCol=aCol; inCol; i++, pCol++){ - Expr *p, *pR; - char *zType; - char *zName; - int nName; - CollSeq *pColl; - int cnt; - NameContext sNC; - + *pnCol = nCol = pEList->nExpr; + aCol = *paCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); + if( aCol==0 ) return SQLITE_NOMEM; + for(i=0, pCol=aCol; ia[i].pExpr; assert( p->pRight==0 || p->pRight->token.z==0 || p->pRight->token.z[0]!=0 ); if( (zName = pEList->a[i].zName)!=0 ){ /* If the column contains an "AS " phrase, use as the name */ - zName = sqliteStrDup(zName); - }else if( p->op==TK_DOT - && (pR=p->pRight)!=0 && pR->token.z && pR->token.z[0] ){ - /* For columns of the from A.B use B as the name */ - zName = sqlite3MPrintf("%T", &pR->token); - }else if( p->span.z && p->span.z[0] ){ - /* Use the original text of the column expression as its name */ - zName = sqlite3MPrintf("%T", &p->span); + zName = sqlite3DbStrDup(db, zName); }else{ - /* If all else fails, make up a name */ - zName = sqlite3MPrintf("column%d", i+1); + Expr *pColExpr = p; /* The expression that is the result column name */ + Table *pTab; /* Table associated with this expression */ + while( pColExpr->op==TK_DOT ) pColExpr = pColExpr->pRight; + if( pColExpr->op==TK_COLUMN && (pTab = pColExpr->pTab)!=0 ){ + /* For columns use the column name name */ + int iCol = pColExpr->iColumn; + if( iCol<0 ) iCol = pTab->iPKey; + zName = sqlite3MPrintf(db, "%s", + iCol>=0 ? pTab->aCol[iCol].zName : "rowid"); + }else{ + /* Use the original text of the column expression as its name */ + Token *pToken = (pColExpr->span.z?&pColExpr->span:&pColExpr->token); + zName = sqlite3MPrintf(db, "%T", pToken); + } + } + if( db->mallocFailed ){ + sqlite3DbFree(db, zName); + break; } sqlite3Dequote(zName); - if( sqlite3MallocFailed() ){ - sqliteFree(zName); - sqlite3DeleteTable(pTab); - return 0; - } /* Make sure the column name is unique. If the name is not unique, ** append a integer to the name so that it becomes unique. */ - nName = strlen(zName); + nName = sqlite3Strlen30(zName); for(j=cnt=0; jzName = zName; + } + if( db->mallocFailed ){ + for(j=0; jpSrc; - zType = sqliteStrDup(columnType(&sNC, p, 0, 0, 0)); - pCol->zType = zType; +/* +** Add type and collation information to a column list based on +** a SELECT statement. +** +** The column list presumably came from selectColumnNamesFromExprList(). +** The column list has only names, not types or collations. This +** routine goes through and adds the types and collations. +** +** This routine requires that all indentifiers in the SELECT +** statement be resolved. +*/ +static void selectAddColumnTypeAndCollation( + Parse *pParse, /* Parsing contexts */ + int nCol, /* Number of columns */ + Column *aCol, /* List of columns */ + Select *pSelect /* SELECT used to determine types and collations */ +){ + sqlite3 *db = pParse->db; + NameContext sNC; + Column *pCol; + CollSeq *pColl; + int i; + Expr *p; + struct ExprList_item *a; + + assert( pSelect!=0 ); + assert( (pSelect->selFlags & SF_Resolved)!=0 ); + assert( nCol==pSelect->pEList->nExpr || db->mallocFailed ); + if( db->mallocFailed ) return; + memset(&sNC, 0, sizeof(sNC)); + sNC.pSrcList = pSelect->pSrc; + a = pSelect->pEList->a; + for(i=0, pCol=aCol; izType = sqlite3DbStrDup(db, columnType(&sNC, p, 0, 0, 0)); pCol->affinity = sqlite3ExprAffinity(p); pColl = sqlite3ExprCollSeq(pParse, p); if( pColl ){ - pCol->zColl = sqliteStrDup(pColl->zName); + pCol->zColl = sqlite3DbStrDup(db, pColl->zName); } } +} + +/* +** Given a SELECT statement, generate a Table structure that describes +** the result set of that SELECT. +*/ +Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ + Table *pTab; + sqlite3 *db = pParse->db; + int savedFlags; + + savedFlags = db->flags; + db->flags &= ~SQLITE_FullColNames; + db->flags |= SQLITE_ShortColNames; + sqlite3SelectPrep(pParse, pSelect, 0); + if( pParse->nErr ) return 0; + while( pSelect->pPrior ) pSelect = pSelect->pPrior; + db->flags = savedFlags; + pTab = sqlite3DbMallocZero(db, sizeof(Table) ); + if( pTab==0 ){ + return 0; + } + pTab->db = db; + pTab->nRef = 1; + pTab->zName = 0; + selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); + selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSelect); pTab->iPKey = -1; + if( db->mallocFailed ){ + sqlite3DeleteTable(pTab); + return 0; + } return pTab; } -/* -** Prepare a SELECT statement for processing by doing the following -** things: -** -** (1) Make sure VDBE cursor numbers have been assigned to every -** element of the FROM clause. -** -** (2) Fill in the pTabList->a[].pTab fields in the SrcList that -** defines FROM clause. When views appear in the FROM clause, -** fill pTabList->a[].pSelect with a copy of the SELECT statement -** that implements the view. A copy is made of the view's SELECT -** statement so that we can freely modify or delete that statement -** without worrying about messing up the presistent representation -** of the view. -** -** (3) Add terms to the WHERE clause to accomodate the NATURAL keyword -** on joins and the ON and USING clause of joins. -** -** (4) Scan the list of columns in the result set (pEList) looking -** for instances of the "*" operator or the TABLE.* operator. -** If found, expand each "*" to be every column in every table -** and TABLE.* to be every column in TABLE. -** -** Return 0 on success. If there are problems, leave an error message -** in pParse and return non-zero. -*/ -static int prepSelectStmt(Parse *pParse, Select *p){ - int i, j, k, rc; - SrcList *pTabList; - ExprList *pEList; - struct SrcList_item *pFrom; - - if( p==0 || p->pSrc==0 || sqlite3MallocFailed() ){ - return 1; - } - pTabList = p->pSrc; - pEList = p->pEList; - - /* Make sure cursor numbers have been assigned to all entries in - ** the FROM clause of the SELECT statement. - */ - sqlite3SrcListAssignCursors(pParse, p->pSrc); - - /* Look up every table named in the FROM clause of the select. If - ** an entry of the FROM clause is a subquery instead of a table or view, - ** then create a transient table structure to describe the subquery. - */ - for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab; - if( pFrom->pTab!=0 ){ - /* This statement has already been prepared. There is no need - ** to go further. */ - assert( i==0 ); - return 0; - } - if( pFrom->zName==0 ){ -#ifndef SQLITE_OMIT_SUBQUERY - /* A sub-query in the FROM clause of a SELECT */ - assert( pFrom->pSelect!=0 ); - if( pFrom->zAlias==0 ){ - pFrom->zAlias = - sqlite3MPrintf("sqlite_subquery_%p_", (void*)pFrom->pSelect); - } - assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = - sqlite3ResultSetOfSelect(pParse, pFrom->zAlias, pFrom->pSelect); - if( pTab==0 ){ - return 1; - } - /* The isEphem flag indicates that the Table structure has been - ** dynamically allocated and may be freed at any time. In other words, - ** pTab is not pointing to a persistent table structure that defines - ** part of the schema. */ - pTab->isEphem = 1; -#endif - }else{ - /* An ordinary table or view name in the FROM clause */ - assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = - sqlite3LocateTable(pParse,pFrom->zName,pFrom->zDatabase); - if( pTab==0 ){ - return 1; - } - pTab->nRef++; -#if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) - if( pTab->pSelect || IsVirtual(pTab) ){ - /* We reach here if the named table is a really a view */ - if( sqlite3ViewGetColumnNames(pParse, pTab) ){ - return 1; - } - /* If pFrom->pSelect!=0 it means we are dealing with a - ** view within a view. The SELECT structure has already been - ** copied by the outer view so we can skip the copy step here - ** in the inner view. - */ - if( pFrom->pSelect==0 ){ - pFrom->pSelect = sqlite3SelectDup(pTab->pSelect); - } - } -#endif - } - } - - /* Process NATURAL keywords, and ON and USING clauses of joins. - */ - if( sqliteProcessJoin(pParse, p) ) return 1; - - /* For every "*" that occurs in the column list, insert the names of - ** all columns in all tables. And for every TABLE.* insert the names - ** of all columns in TABLE. The parser inserted a special expression - ** with the TK_ALL operator for each "*" that it found in the column list. - ** The following code just has to locate the TK_ALL expressions and expand - ** each one to the list of all columns in all tables. - ** - ** The first loop just checks to see if there are any "*" operators - ** that need expanding. - */ - for(k=0; knExpr; k++){ - Expr *pE = pEList->a[k].pExpr; - if( pE->op==TK_ALL ) break; - if( pE->op==TK_DOT && pE->pRight && pE->pRight->op==TK_ALL - && pE->pLeft && pE->pLeft->op==TK_ID ) break; - } - rc = 0; - if( knExpr ){ - /* - ** If we get here it means the result set contains one or more "*" - ** operators that need to be expanded. Loop through each expression - ** in the result set and expand them one by one. - */ - struct ExprList_item *a = pEList->a; - ExprList *pNew = 0; - int flags = pParse->db->flags; - int longNames = (flags & SQLITE_FullColNames)!=0 && - (flags & SQLITE_ShortColNames)==0; - - for(k=0; knExpr; k++){ - Expr *pE = a[k].pExpr; - if( pE->op!=TK_ALL && - (pE->op!=TK_DOT || pE->pRight==0 || pE->pRight->op!=TK_ALL) ){ - /* This particular expression does not need to be expanded. - */ - pNew = sqlite3ExprListAppend(pNew, a[k].pExpr, 0); - if( pNew ){ - pNew->a[pNew->nExpr-1].zName = a[k].zName; - }else{ - rc = 1; - } - a[k].pExpr = 0; - a[k].zName = 0; - }else{ - /* This expression is a "*" or a "TABLE.*" and needs to be - ** expanded. */ - int tableSeen = 0; /* Set to 1 when TABLE matches */ - char *zTName; /* text of name of TABLE */ - if( pE->op==TK_DOT && pE->pLeft ){ - zTName = sqlite3NameFromToken(&pE->pLeft->token); - }else{ - zTName = 0; - } - for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; - char *zTabName = pFrom->zAlias; - if( zTabName==0 || zTabName[0]==0 ){ - zTabName = pTab->zName; - } - if( zTName && (zTabName==0 || zTabName[0]==0 || - sqlite3StrICmp(zTName, zTabName)!=0) ){ - continue; - } - tableSeen = 1; - for(j=0; jnCol; j++){ - Expr *pExpr, *pRight; - char *zName = pTab->aCol[j].zName; - - /* If a column is marked as 'hidden' (currently only possible - ** for virtual tables), do not include it in the expanded - ** result-set list. - */ - if( IsHiddenColumn(&pTab->aCol[j]) ){ - assert(IsVirtual(pTab)); - continue; - } - - if( i>0 ){ - struct SrcList_item *pLeft = &pTabList->a[i-1]; - if( (pLeft[1].jointype & JT_NATURAL)!=0 && - columnIndex(pLeft->pTab, zName)>=0 ){ - /* In a NATURAL join, omit the join columns from the - ** table on the right */ - continue; - } - if( sqlite3IdListIndex(pLeft[1].pUsing, zName)>=0 ){ - /* In a join with a USING clause, omit columns in the - ** using clause from the table on the right. */ - continue; - } - } - pRight = sqlite3Expr(TK_ID, 0, 0, 0); - if( pRight==0 ) break; - setQuotedToken(&pRight->token, zName); - if( zTabName && (longNames || pTabList->nSrc>1) ){ - Expr *pLeft = sqlite3Expr(TK_ID, 0, 0, 0); - pExpr = sqlite3Expr(TK_DOT, pLeft, pRight, 0); - if( pExpr==0 ) break; - setQuotedToken(&pLeft->token, zTabName); - setToken(&pExpr->span, sqlite3MPrintf("%s.%s", zTabName, zName)); - pExpr->span.dyn = 1; - pExpr->token.z = 0; - pExpr->token.n = 0; - pExpr->token.dyn = 0; - }else{ - pExpr = pRight; - pExpr->span = pExpr->token; - pExpr->span.dyn = 0; - } - if( longNames ){ - pNew = sqlite3ExprListAppend(pNew, pExpr, &pExpr->span); - }else{ - pNew = sqlite3ExprListAppend(pNew, pExpr, &pRight->token); - } - } - } - if( !tableSeen ){ - if( zTName ){ - sqlite3ErrorMsg(pParse, "no such table: %s", zTName); - }else{ - sqlite3ErrorMsg(pParse, "no tables specified"); - } - rc = 1; - } - sqliteFree(zTName); - } - } - sqlite3ExprListDelete(pEList); - p->pEList = pNew; - } - if( p->pEList && p->pEList->nExpr>SQLITE_MAX_COLUMN ){ - sqlite3ErrorMsg(pParse, "too many columns in result set"); - rc = SQLITE_ERROR; - } - if( sqlite3MallocFailed() ){ - rc = SQLITE_NOMEM; - } - return rc; -} - -#ifndef SQLITE_OMIT_COMPOUND_SELECT -/* -** This routine associates entries in an ORDER BY expression list with -** columns in a result. For each ORDER BY expression, the opcode of -** the top-level node is changed to TK_COLUMN and the iColumn value of -** the top-level node is filled in with column number and the iTable -** value of the top-level node is filled with iTable parameter. -** -** If there are prior SELECT clauses, they are processed first. A match -** in an earlier SELECT takes precedence over a later SELECT. -** -** Any entry that does not match is flagged as an error. The number -** of errors is returned. -*/ -static int matchOrderbyToColumn( - Parse *pParse, /* A place to leave error messages */ - Select *pSelect, /* Match to result columns of this SELECT */ - ExprList *pOrderBy, /* The ORDER BY values to match against columns */ - int iTable, /* Insert this value in iTable */ - int mustComplete /* If TRUE all ORDER BYs must match */ -){ - int nErr = 0; - int i, j; - ExprList *pEList; - - if( pSelect==0 || pOrderBy==0 ) return 1; - if( mustComplete ){ - for(i=0; inExpr; i++){ pOrderBy->a[i].done = 0; } - } - if( prepSelectStmt(pParse, pSelect) ){ - return 1; - } - if( pSelect->pPrior ){ - if( matchOrderbyToColumn(pParse, pSelect->pPrior, pOrderBy, iTable, 0) ){ - return 1; - } - } - pEList = pSelect->pEList; - for(i=0; inExpr; i++){ - struct ExprList_item *pItem; - Expr *pE = pOrderBy->a[i].pExpr; - int iCol = -1; - char *zLabel; - - if( pOrderBy->a[i].done ) continue; - if( sqlite3ExprIsInteger(pE, &iCol) ){ - if( iCol<=0 || iCol>pEList->nExpr ){ - sqlite3ErrorMsg(pParse, - "ORDER BY position %d should be between 1 and %d", - iCol, pEList->nExpr); - nErr++; - break; - } - if( !mustComplete ) continue; - iCol--; - } - if( iCol<0 && (zLabel = sqlite3NameFromToken(&pE->token))!=0 ){ - for(j=0, pItem=pEList->a; jnExpr; j++, pItem++){ - char *zName; - int isMatch; - if( pItem->zName ){ - zName = sqlite3StrDup(pItem->zName); - }else{ - zName = sqlite3NameFromToken(&pItem->pExpr->token); - } - isMatch = zName && sqlite3StrICmp(zName, zLabel)==0; - sqliteFree(zName); - if( isMatch ){ - iCol = j; - break; - } - } - sqliteFree(zLabel); - } - if( iCol>=0 ){ - pE->op = TK_COLUMN; - pE->iColumn = iCol; - pE->iTable = iTable; - pE->iAgg = -1; - pOrderBy->a[i].done = 1; - }else if( mustComplete ){ - sqlite3ErrorMsg(pParse, - "ORDER BY term number %d does not match any result column", i+1); - nErr++; - break; - } - } - return nErr; -} -#endif /* #ifndef SQLITE_OMIT_COMPOUND_SELECT */ - /* ** Get a VDBE for the given parser context. Create a new one if necessary. ** If an error occurs, return NULL and leave a message in pParse. @@ -1505,6 +1305,11 @@ Vdbe *sqlite3GetVdbe(Parse *pParse){ Vdbe *v = pParse->pVdbe; if( v==0 ){ v = pParse->pVdbe = sqlite3VdbeCreate(pParse->db); +#ifndef SQLITE_OMIT_TRACE + if( v ){ + sqlite3VdbeAddOp0(v, OP_Trace); + } +#endif } return v; } @@ -1532,7 +1337,8 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ Vdbe *v = 0; int iLimit = 0; int iOffset; - int addr1, addr2; + int addr1; + if( p->iLimit ) return; /* ** "LIMIT -1" always shows all rows. There is some @@ -1541,58 +1347,35 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ ** no rows. */ if( p->pLimit ){ - p->iLimit = iLimit = pParse->nMem; - pParse->nMem += 2; + p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); if( v==0 ) return; - sqlite3ExprCode(pParse, p->pLimit); - sqlite3VdbeAddOp(v, OP_MustBeInt, 0, 0); - sqlite3VdbeAddOp(v, OP_MemStore, iLimit, 1); - VdbeComment((v, "# LIMIT counter")); - sqlite3VdbeAddOp(v, OP_IfMemZero, iLimit, iBreak); - sqlite3VdbeAddOp(v, OP_MemLoad, iLimit, 0); + sqlite3ExprCode(pParse, p->pLimit, iLimit); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); + VdbeComment((v, "LIMIT counter")); + sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak); } if( p->pOffset ){ - p->iOffset = iOffset = pParse->nMem++; + p->iOffset = iOffset = ++pParse->nMem; + if( p->pLimit ){ + pParse->nMem++; /* Allocate an extra register for limit+offset */ + } v = sqlite3GetVdbe(pParse); if( v==0 ) return; - sqlite3ExprCode(pParse, p->pOffset); - sqlite3VdbeAddOp(v, OP_MustBeInt, 0, 0); - sqlite3VdbeAddOp(v, OP_MemStore, iOffset, p->pLimit==0); - VdbeComment((v, "# OFFSET counter")); - addr1 = sqlite3VdbeAddOp(v, OP_IfMemPos, iOffset, 0); - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); - sqlite3VdbeAddOp(v, OP_Integer, 0, 0); + sqlite3ExprCode(pParse, p->pOffset, iOffset); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); + VdbeComment((v, "OFFSET counter")); + addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset); sqlite3VdbeJumpHere(v, addr1); if( p->pLimit ){ - sqlite3VdbeAddOp(v, OP_Add, 0, 0); + sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); + VdbeComment((v, "LIMIT+OFFSET")); + addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); + sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1); + sqlite3VdbeJumpHere(v, addr1); } } - if( p->pLimit ){ - addr1 = sqlite3VdbeAddOp(v, OP_IfMemPos, iLimit, 0); - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); - sqlite3VdbeAddOp(v, OP_MemInt, -1, iLimit+1); - addr2 = sqlite3VdbeAddOp(v, OP_Goto, 0, 0); - sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp(v, OP_MemStore, iLimit+1, 1); - VdbeComment((v, "# LIMIT+OFFSET")); - sqlite3VdbeJumpHere(v, addr2); - } -} - -/* -** Allocate a virtual index to use for sorting. -*/ -static void createSortingIndex(Parse *pParse, Select *p, ExprList *pOrderBy){ - if( pOrderBy ){ - int addr; - assert( pOrderBy->iECursor==0 ); - pOrderBy->iECursor = pParse->nTab++; - addr = sqlite3VdbeAddOp(pParse->pVdbe, OP_OpenEphemeral, - pOrderBy->iECursor, pOrderBy->nExpr+1); - assert( p->addrOpenEphm[2] == -1 ); - p->addrOpenEphm[2] = addr; - } } #ifndef SQLITE_OMIT_COMPOUND_SELECT @@ -1618,10 +1401,19 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ } #endif /* SQLITE_OMIT_COMPOUND_SELECT */ +/* Forward reference */ +static int multiSelectOrderBy( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +); + + #ifndef SQLITE_OMIT_COMPOUND_SELECT /* -** This routine is called to process a query that is really the union -** or intersection of two or more separate queries. +** This routine is called to process a compound query form from +** two or more separate queries using UNION, UNION ALL, EXCEPT, or +** INTERSECT ** ** "p" points to the right-most of the two queries. the query on the ** left is p->pPrior. The left query could also be a compound query @@ -1652,28 +1444,24 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ static int multiSelect( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ - int eDest, /* \___ Store query results as specified */ - int iParm, /* / by these two parameters. */ - char *aff /* If eDest is SRT_Union, the affinity string */ + SelectDest *pDest /* What to do with query results */ ){ int rc = SQLITE_OK; /* Success code from a subroutine */ Select *pPrior; /* Another SELECT immediately to our left */ Vdbe *v; /* Generate code to this VDBE */ - int nCol; /* Number of columns in the result set */ - ExprList *pOrderBy; /* The ORDER BY clause on p */ - int aSetP2[2]; /* Set P2 value of these op to number of columns */ - int nSetP2 = 0; /* Number of slots in aSetP2[] used */ + SelectDest dest; /* Alternative data destination */ + Select *pDelete = 0; /* Chain of simple selects to delete */ + sqlite3 *db; /* Database connection */ /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. */ - if( p==0 || p->pPrior==0 ){ - rc = 1; - goto multi_select_end; - } + assert( p && p->pPrior ); /* Calling function guarantees this much */ + db = pParse->db; pPrior = p->pPrior; assert( pPrior->pRightmost!=pPrior ); assert( pPrior->pRightmost==p->pRightmost ); + dest = *pDest; if( pPrior->pOrderBy ){ sqlite3ErrorMsg(pParse,"ORDER BY clause should come after %s not before", selectOpName(p->op)); @@ -1687,241 +1475,15 @@ static int multiSelect( goto multi_select_end; } - /* Make sure we have a valid query engine. If not, create a new one. - */ v = sqlite3GetVdbe(pParse); - if( v==0 ){ - rc = 1; - goto multi_select_end; - } + assert( v!=0 ); /* The VDBE already created by calling function */ /* Create the destination temporary table if necessary */ - if( eDest==SRT_EphemTab ){ + if( dest.eDest==SRT_EphemTab ){ assert( p->pEList ); - assert( nSetP2pOrderBy; - switch( p->op ){ - case TK_ALL: { - if( pOrderBy==0 ){ - int addr = 0; - assert( !pPrior->pLimit ); - pPrior->pLimit = p->pLimit; - pPrior->pOffset = p->pOffset; - rc = sqlite3Select(pParse, pPrior, eDest, iParm, 0, 0, 0, aff); - p->pLimit = 0; - p->pOffset = 0; - if( rc ){ - goto multi_select_end; - } - p->pPrior = 0; - p->iLimit = pPrior->iLimit; - p->iOffset = pPrior->iOffset; - if( p->iLimit>=0 ){ - addr = sqlite3VdbeAddOp(v, OP_IfMemZero, p->iLimit, 0); - VdbeComment((v, "# Jump ahead if LIMIT reached")); - } - rc = sqlite3Select(pParse, p, eDest, iParm, 0, 0, 0, aff); - p->pPrior = pPrior; - if( rc ){ - goto multi_select_end; - } - if( addr ){ - sqlite3VdbeJumpHere(v, addr); - } - break; - } - /* For UNION ALL ... ORDER BY fall through to the next case */ - } - case TK_EXCEPT: - case TK_UNION: { - int unionTab; /* Cursor number of the temporary table holding result */ - int op = 0; /* One of the SRT_ operations to apply to self */ - int priorOp; /* The SRT_ operation to apply to prior selects */ - Expr *pLimit, *pOffset; /* Saved values of p->nLimit and p->nOffset */ - int addr; - - priorOp = p->op==TK_ALL ? SRT_Table : SRT_Union; - if( eDest==priorOp && pOrderBy==0 && !p->pLimit && !p->pOffset ){ - /* We can reuse a temporary table generated by a SELECT to our - ** right. - */ - unionTab = iParm; - }else{ - /* We will need to create our own temporary table to hold the - ** intermediate results. - */ - unionTab = pParse->nTab++; - if( pOrderBy && matchOrderbyToColumn(pParse, p, pOrderBy, unionTab,1) ){ - rc = 1; - goto multi_select_end; - } - addr = sqlite3VdbeAddOp(v, OP_OpenEphemeral, unionTab, 0); - if( priorOp==SRT_Table ){ - assert( nSetP2addrOpenEphm[0] == -1 ); - p->addrOpenEphm[0] = addr; - p->pRightmost->usesEphm = 1; - } - createSortingIndex(pParse, p, pOrderBy); - assert( p->pEList ); - } - - /* Code the SELECT statements to our left - */ - assert( !pPrior->pOrderBy ); - rc = sqlite3Select(pParse, pPrior, priorOp, unionTab, 0, 0, 0, aff); - if( rc ){ - goto multi_select_end; - } - - /* Code the current SELECT statement - */ - switch( p->op ){ - case TK_EXCEPT: op = SRT_Except; break; - case TK_UNION: op = SRT_Union; break; - case TK_ALL: op = SRT_Table; break; - } - p->pPrior = 0; - p->pOrderBy = 0; - p->disallowOrderBy = pOrderBy!=0; - pLimit = p->pLimit; - p->pLimit = 0; - pOffset = p->pOffset; - p->pOffset = 0; - rc = sqlite3Select(pParse, p, op, unionTab, 0, 0, 0, aff); - /* Query flattening in sqlite3Select() might refill p->pOrderBy. - ** Be sure to delete p->pOrderBy, therefore, to avoid a memory leak. */ - sqlite3ExprListDelete(p->pOrderBy); - p->pPrior = pPrior; - p->pOrderBy = pOrderBy; - sqlite3ExprDelete(p->pLimit); - p->pLimit = pLimit; - p->pOffset = pOffset; - p->iLimit = -1; - p->iOffset = -1; - if( rc ){ - goto multi_select_end; - } - - - /* Convert the data in the temporary table into whatever form - ** it is that we currently need. - */ - if( eDest!=priorOp || unionTab!=iParm ){ - int iCont, iBreak, iStart; - assert( p->pEList ); - if( eDest==SRT_Callback ){ - Select *pFirst = p; - while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, 0, pFirst->pEList); - } - iBreak = sqlite3VdbeMakeLabel(v); - iCont = sqlite3VdbeMakeLabel(v); - computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp(v, OP_Rewind, unionTab, iBreak); - iStart = sqlite3VdbeCurrentAddr(v); - rc = selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, - pOrderBy, -1, eDest, iParm, - iCont, iBreak, 0); - if( rc ){ - rc = 1; - goto multi_select_end; - } - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp(v, OP_Next, unionTab, iStart); - sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp(v, OP_Close, unionTab, 0); - } - break; - } - case TK_INTERSECT: { - int tab1, tab2; - int iCont, iBreak, iStart; - Expr *pLimit, *pOffset; - int addr; - - /* INTERSECT is different from the others since it requires - ** two temporary tables. Hence it has its own case. Begin - ** by allocating the tables we will need. - */ - tab1 = pParse->nTab++; - tab2 = pParse->nTab++; - if( pOrderBy && matchOrderbyToColumn(pParse,p,pOrderBy,tab1,1) ){ - rc = 1; - goto multi_select_end; - } - createSortingIndex(pParse, p, pOrderBy); - - addr = sqlite3VdbeAddOp(v, OP_OpenEphemeral, tab1, 0); - assert( p->addrOpenEphm[0] == -1 ); - p->addrOpenEphm[0] = addr; - p->pRightmost->usesEphm = 1; - assert( p->pEList ); - - /* Code the SELECTs to our left into temporary table "tab1". - */ - rc = sqlite3Select(pParse, pPrior, SRT_Union, tab1, 0, 0, 0, aff); - if( rc ){ - goto multi_select_end; - } - - /* Code the current SELECT into temporary table "tab2" - */ - addr = sqlite3VdbeAddOp(v, OP_OpenEphemeral, tab2, 0); - assert( p->addrOpenEphm[1] == -1 ); - p->addrOpenEphm[1] = addr; - p->pPrior = 0; - pLimit = p->pLimit; - p->pLimit = 0; - pOffset = p->pOffset; - p->pOffset = 0; - rc = sqlite3Select(pParse, p, SRT_Union, tab2, 0, 0, 0, aff); - p->pPrior = pPrior; - sqlite3ExprDelete(p->pLimit); - p->pLimit = pLimit; - p->pOffset = pOffset; - if( rc ){ - goto multi_select_end; - } - - /* Generate code to take the intersection of the two temporary - ** tables. - */ - assert( p->pEList ); - if( eDest==SRT_Callback ){ - Select *pFirst = p; - while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, 0, pFirst->pEList); - } - iBreak = sqlite3VdbeMakeLabel(v); - iCont = sqlite3VdbeMakeLabel(v); - computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp(v, OP_Rewind, tab1, iBreak); - iStart = sqlite3VdbeAddOp(v, OP_RowKey, tab1, 0); - sqlite3VdbeAddOp(v, OP_NotFound, tab2, iCont); - rc = selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, - pOrderBy, -1, eDest, iParm, - iCont, iBreak, 0); - if( rc ){ - rc = 1; - goto multi_select_end; - } - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp(v, OP_Next, tab1, iStart); - sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp(v, OP_Close, tab2, 0); - sqlite3VdbeAddOp(v, OP_Close, tab1, 0); - break; - } + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iParm, p->pEList->nExpr); + dest.eDest = SRT_Table; } /* Make sure all SELECTs in the statement have the same number of elements @@ -1935,46 +1497,252 @@ static int multiSelect( goto multi_select_end; } - /* Set the number of columns in temporary tables + /* Compound SELECTs that have an ORDER BY clause are handled separately. */ - nCol = p->pEList->nExpr; - while( nSetP2 ){ - sqlite3VdbeChangeP2(v, aSetP2[--nSetP2], nCol); + if( p->pOrderBy ){ + return multiSelectOrderBy(pParse, p, pDest); } - /* Compute collating sequences used by either the ORDER BY clause or - ** by any temporary tables needed to implement the compound select. - ** Attach the KeyInfo structure to all temporary tables. Invoke the - ** ORDER BY processing if there is an ORDER BY clause. + /* Generate code for the left and right SELECT statements. + */ + switch( p->op ){ + case TK_ALL: { + int addr = 0; + assert( !pPrior->pLimit ); + pPrior->pLimit = p->pLimit; + pPrior->pOffset = p->pOffset; + rc = sqlite3Select(pParse, pPrior, &dest); + p->pLimit = 0; + p->pOffset = 0; + if( rc ){ + goto multi_select_end; + } + p->pPrior = 0; + p->iLimit = pPrior->iLimit; + p->iOffset = pPrior->iOffset; + if( p->iLimit ){ + addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit); + VdbeComment((v, "Jump ahead if LIMIT reached")); + } + rc = sqlite3Select(pParse, p, &dest); + pDelete = p->pPrior; + p->pPrior = pPrior; + if( rc ){ + goto multi_select_end; + } + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + } + break; + } + case TK_EXCEPT: + case TK_UNION: { + int unionTab; /* Cursor number of the temporary table holding result */ + u8 op = 0; /* One of the SRT_ operations to apply to self */ + int priorOp; /* The SRT_ operation to apply to prior selects */ + Expr *pLimit, *pOffset; /* Saved values of p->nLimit and p->nOffset */ + int addr; + SelectDest uniondest; + + priorOp = SRT_Union; + if( dest.eDest==priorOp && ALWAYS(!p->pLimit &&!p->pOffset) ){ + /* We can reuse a temporary table generated by a SELECT to our + ** right. + */ + assert( p->pRightmost!=p ); /* Can only happen for leftward elements + ** of a 3-way or more compound */ + assert( p->pLimit==0 ); /* Not allowed on leftward elements */ + assert( p->pOffset==0 ); /* Not allowed on leftward elements */ + unionTab = dest.iParm; + }else{ + /* We will need to create our own temporary table to hold the + ** intermediate results. + */ + unionTab = pParse->nTab++; + assert( p->pOrderBy==0 ); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + p->pRightmost->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); + } + + /* Code the SELECT statements to our left + */ + assert( !pPrior->pOrderBy ); + sqlite3SelectDestInit(&uniondest, priorOp, unionTab); + rc = sqlite3Select(pParse, pPrior, &uniondest); + if( rc ){ + goto multi_select_end; + } + + /* Code the current SELECT statement + */ + if( p->op==TK_EXCEPT ){ + op = SRT_Except; + }else{ + assert( p->op==TK_UNION ); + op = SRT_Union; + } + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + pOffset = p->pOffset; + p->pOffset = 0; + uniondest.eDest = op; + rc = sqlite3Select(pParse, p, &uniondest); + /* Query flattening in sqlite3Select() might refill p->pOrderBy. + ** Be sure to delete p->pOrderBy, therefore, to avoid a memory leak. */ + sqlite3ExprListDelete(db, p->pOrderBy); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->pOrderBy = 0; + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + p->pOffset = pOffset; + p->iLimit = 0; + p->iOffset = 0; + if( rc ){ + goto multi_select_end; + } + + + /* Convert the data in the temporary table into whatever form + ** it is that we currently need. + */ + if( dest.eDest!=priorOp || unionTab!=dest.iParm ){ + int iCont, iBreak, iStart; + assert( p->pEList ); + if( dest.eDest==SRT_Output ){ + Select *pFirst = p; + while( pFirst->pPrior ) pFirst = pFirst->pPrior; + generateColumnNames(pParse, 0, pFirst->pEList); + } + iBreak = sqlite3VdbeMakeLabel(v); + iCont = sqlite3VdbeMakeLabel(v); + computeLimitRegisters(pParse, p, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); + iStart = sqlite3VdbeCurrentAddr(v); + selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, + 0, -1, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); + } + break; + } + case TK_INTERSECT: { + int tab1, tab2; + int iCont, iBreak, iStart; + Expr *pLimit, *pOffset; + int addr; + SelectDest intersectdest; + int r1; + + /* INTERSECT is different from the others since it requires + ** two temporary tables. Hence it has its own case. Begin + ** by allocating the tables we will need. + */ + tab1 = pParse->nTab++; + tab2 = pParse->nTab++; + assert( p->pOrderBy==0 ); + + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + p->pRightmost->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); + + /* Code the SELECTs to our left into temporary table "tab1". + */ + sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); + rc = sqlite3Select(pParse, pPrior, &intersectdest); + if( rc ){ + goto multi_select_end; + } + + /* Code the current SELECT into temporary table "tab2" + */ + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); + assert( p->addrOpenEphm[1] == -1 ); + p->addrOpenEphm[1] = addr; + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + pOffset = p->pOffset; + p->pOffset = 0; + intersectdest.iParm = tab2; + rc = sqlite3Select(pParse, p, &intersectdest); + pDelete = p->pPrior; + p->pPrior = pPrior; + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + p->pOffset = pOffset; + if( rc ){ + goto multi_select_end; + } + + /* Generate code to take the intersection of the two temporary + ** tables. + */ + assert( p->pEList ); + if( dest.eDest==SRT_Output ){ + Select *pFirst = p; + while( pFirst->pPrior ) pFirst = pFirst->pPrior; + generateColumnNames(pParse, 0, pFirst->pEList); + } + iBreak = sqlite3VdbeMakeLabel(v); + iCont = sqlite3VdbeMakeLabel(v); + computeLimitRegisters(pParse, p, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); + r1 = sqlite3GetTempReg(pParse); + iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); + sqlite3VdbeAddOp3(v, OP_NotFound, tab2, iCont, r1); + sqlite3ReleaseTempReg(pParse, r1); + selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, + 0, -1, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); + sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); + break; + } + } + + /* Compute collating sequences used by + ** temporary tables needed to implement the compound select. + ** Attach the KeyInfo structure to all temporary tables. ** ** This section is run by the right-most SELECT statement only. ** SELECT statements to the left always skip this part. The right-most ** SELECT might also skip this part if it has no ORDER BY clause and ** no temp tables are required. */ - if( pOrderBy || p->usesEphm ){ + if( p->selFlags & SF_UsesEphemeral ){ int i; /* Loop counter */ KeyInfo *pKeyInfo; /* Collating sequence for the result set */ Select *pLoop; /* For looping through SELECT statements */ - int nKeyCol; /* Number of entries in pKeyInfo->aCol[] */ CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ - CollSeq **aCopy; /* A copy of pKeyInfo->aColl[] */ + int nCol; /* Number of columns in result set */ assert( p->pRightmost==p ); - nKeyCol = nCol + (pOrderBy ? pOrderBy->nExpr : 0); - pKeyInfo = sqliteMalloc(sizeof(*pKeyInfo)+nKeyCol*(sizeof(CollSeq*) + 1)); + nCol = p->pEList->nExpr; + pKeyInfo = sqlite3DbMallocZero(db, + sizeof(*pKeyInfo)+nCol*(sizeof(CollSeq*) + 1)); if( !pKeyInfo ){ rc = SQLITE_NOMEM; goto multi_select_end; } - pKeyInfo->enc = ENC(pParse->db); - pKeyInfo->nField = nCol; + pKeyInfo->enc = ENC(db); + pKeyInfo->nField = (u16)nCol; for(i=0, apColl=pKeyInfo->aColl; idb->pDfltColl; + *apColl = db->pDfltColl; } } @@ -1988,64 +1756,616 @@ static int multiSelect( break; } sqlite3VdbeChangeP2(v, addr, nCol); - sqlite3VdbeChangeP3(v, addr, (char*)pKeyInfo, P3_KEYINFO); + sqlite3VdbeChangeP4(v, addr, (char*)pKeyInfo, P4_KEYINFO); pLoop->addrOpenEphm[i] = -1; } } - - if( pOrderBy ){ - struct ExprList_item *pOTerm = pOrderBy->a; - int nOrderByExpr = pOrderBy->nExpr; - int addr; - u8 *pSortOrder; - - /* Reuse the same pKeyInfo for the ORDER BY as was used above for - ** the compound select statements. Except we have to change out the - ** pKeyInfo->aColl[] values. Some of the aColl[] values will be - ** reused when constructing the pKeyInfo for the ORDER BY, so make - ** a copy. Sufficient space to hold both the nCol entries for - ** the compound select and the nOrderbyExpr entries for the ORDER BY - ** was allocated above. But we need to move the compound select - ** entries out of the way before constructing the ORDER BY entries. - ** Move the compound select entries into aCopy[] where they can be - ** accessed and reused when constructing the ORDER BY entries. - ** Because nCol might be greater than or less than nOrderByExpr - ** we have to use memmove() when doing the copy. - */ - aCopy = &pKeyInfo->aColl[nOrderByExpr]; - pSortOrder = pKeyInfo->aSortOrder = (u8*)&aCopy[nCol]; - memmove(aCopy, pKeyInfo->aColl, nCol*sizeof(CollSeq*)); - - apColl = pKeyInfo->aColl; - for(i=0; ipExpr; - if( (pExpr->flags & EP_ExpCollate) ){ - assert( pExpr->pColl!=0 ); - *apColl = pExpr->pColl; - }else{ - *apColl = aCopy[pExpr->iColumn]; - } - *pSortOrder = pOTerm->sortOrder; - } - assert( p->pRightmost==p ); - assert( p->addrOpenEphm[2]>=0 ); - addr = p->addrOpenEphm[2]; - sqlite3VdbeChangeP2(v, addr, p->pOrderBy->nExpr+2); - pKeyInfo->nField = nOrderByExpr; - sqlite3VdbeChangeP3(v, addr, (char*)pKeyInfo, P3_KEYINFO_HANDOFF); - pKeyInfo = 0; - generateSortTail(pParse, p, v, p->pEList->nExpr, eDest, iParm); - } - - sqliteFree(pKeyInfo); + sqlite3DbFree(db, pKeyInfo); } multi_select_end: + pDest->iMem = dest.iMem; + pDest->nMem = dest.nMem; + sqlite3SelectDelete(db, pDelete); return rc; } #endif /* SQLITE_OMIT_COMPOUND_SELECT */ -#ifndef SQLITE_OMIT_VIEW +/* +** Code an output subroutine for a coroutine implementation of a +** SELECT statment. +** +** The data to be output is contained in pIn->iMem. There are +** pIn->nMem columns to be output. pDest is where the output should +** be sent. +** +** regReturn is the number of the register holding the subroutine +** return address. +** +** If regPrev>0 then it is a the first register in a vector that +** records the previous output. mem[regPrev] is a flag that is false +** if there has been no previous output. If regPrev>0 then code is +** generated to suppress duplicates. pKeyInfo is used for comparing +** keys. +** +** If the LIMIT found in p->iLimit is reached, jump immediately to +** iBreak. +*/ +static int generateOutputSubroutine( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement */ + SelectDest *pIn, /* Coroutine supplying data */ + SelectDest *pDest, /* Where to send the data */ + int regReturn, /* The return address register */ + int regPrev, /* Previous result register. No uniqueness if 0 */ + KeyInfo *pKeyInfo, /* For comparing with previous entry */ + int p4type, /* The p4 type for pKeyInfo */ + int iBreak /* Jump here if we hit the LIMIT */ +){ + Vdbe *v = pParse->pVdbe; + int iContinue; + int addr; + + addr = sqlite3VdbeCurrentAddr(v); + iContinue = sqlite3VdbeMakeLabel(v); + + /* Suppress duplicates for UNION, EXCEPT, and INTERSECT + */ + if( regPrev ){ + int j1, j2; + j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); + j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iMem, regPrev+1, pIn->nMem, + (char*)pKeyInfo, p4type); + sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); + sqlite3VdbeJumpHere(v, j1); + sqlite3ExprCodeCopy(pParse, pIn->iMem, regPrev+1, pIn->nMem); + sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); + } + if( pParse->db->mallocFailed ) return 0; + + /* Suppress the the first OFFSET entries if there is an OFFSET clause + */ + codeOffset(v, p, iContinue); + + switch( pDest->eDest ){ + /* Store the result as data using a unique key. + */ + case SRT_Table: + case SRT_EphemTab: { + int r1 = sqlite3GetTempReg(pParse); + int r2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iMem, pIn->nMem, r1); + sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, pDest->iParm, r1, r2); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3ReleaseTempReg(pParse, r2); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + +#ifndef SQLITE_OMIT_SUBQUERY + /* If we are creating a set for an "expr IN (SELECT ...)" construct, + ** then there should be a single item on the stack. Write this + ** item into the set table with bogus data. + */ + case SRT_Set: { + int r1; + assert( pIn->nMem==1 ); + p->affinity = + sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affinity); + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iMem, 1, r1, &p->affinity, 1); + sqlite3ExprCacheAffinityChange(pParse, pIn->iMem, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iParm, r1); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + +#if 0 /* Never occurs on an ORDER BY query */ + /* If any row exist in the result set, record that fact and abort. + */ + case SRT_Exists: { + sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iParm); + /* The LIMIT clause will terminate the loop for us */ + break; + } +#endif + + /* If this is a scalar select that is part of an expression, then + ** store the results in the appropriate memory cell and break out + ** of the scan loop. + */ + case SRT_Mem: { + assert( pIn->nMem==1 ); + sqlite3ExprCodeMove(pParse, pIn->iMem, pDest->iParm, 1); + /* The LIMIT clause will jump out of the loop for us */ + break; + } +#endif /* #ifndef SQLITE_OMIT_SUBQUERY */ + + /* The results are stored in a sequence of registers + ** starting at pDest->iMem. Then the co-routine yields. + */ + case SRT_Coroutine: { + if( pDest->iMem==0 ){ + pDest->iMem = sqlite3GetTempRange(pParse, pIn->nMem); + pDest->nMem = pIn->nMem; + } + sqlite3ExprCodeMove(pParse, pIn->iMem, pDest->iMem, pDest->nMem); + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); + break; + } + + /* Results are stored in a sequence of registers. Then the + ** OP_ResultRow opcode is used to cause sqlite3_step() to return + ** the next row of result. + */ + case SRT_Output: { + sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iMem, pIn->nMem); + sqlite3ExprCacheAffinityChange(pParse, pIn->iMem, pIn->nMem); + break; + } + +#if !defined(SQLITE_OMIT_TRIGGER) + /* Discard the results. This is used for SELECT statements inside + ** the body of a TRIGGER. The purpose of such selects is to call + ** user-defined functions that have side effects. We do not care + ** about the actual results of the select. + */ + default: { + break; + } +#endif + } + + /* Jump to the end of the loop if the LIMIT is reached. + */ + if( p->iLimit ){ + sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1); + sqlite3VdbeAddOp2(v, OP_IfZero, p->iLimit, iBreak); + } + + /* Generate the subroutine return + */ + sqlite3VdbeResolveLabel(v, iContinue); + sqlite3VdbeAddOp1(v, OP_Return, regReturn); + + return addr; +} + +/* +** Alternative compound select code generator for cases when there +** is an ORDER BY clause. +** +** We assume a query of the following form: +** +** ORDER BY +** +** is one of UNION ALL, UNION, EXCEPT, or INTERSECT. The idea +** is to code both and with the ORDER BY clause as +** co-routines. Then run the co-routines in parallel and merge the results +** into the output. In addition to the two coroutines (called selectA and +** selectB) there are 7 subroutines: +** +** outA: Move the output of the selectA coroutine into the output +** of the compound query. +** +** outB: Move the output of the selectB coroutine into the output +** of the compound query. (Only generated for UNION and +** UNION ALL. EXCEPT and INSERTSECT never output a row that +** appears only in B.) +** +** AltB: Called when there is data from both coroutines and AB. +** +** EofA: Called when data is exhausted from selectA. +** +** EofB: Called when data is exhausted from selectB. +** +** The implementation of the latter five subroutines depend on which +** is used: +** +** +** UNION ALL UNION EXCEPT INTERSECT +** ------------- ----------------- -------------- ----------------- +** AltB: outA, nextA outA, nextA outA, nextA nextA +** +** AeqB: outA, nextA nextA nextA outA, nextA +** +** AgtB: outB, nextB outB, nextB nextB nextB +** +** EofA: outB, nextB outB, nextB halt halt +** +** EofB: outA, nextA outA, nextA outA, nextA halt +** +** In the AltB, AeqB, and AgtB subroutines, an EOF on A following nextA +** causes an immediate jump to EofA and an EOF on B following nextB causes +** an immediate jump to EofB. Within EofA and EofB, and EOF on entry or +** following nextX causes a jump to the end of the select processing. +** +** Duplicate removal in the UNION, EXCEPT, and INTERSECT cases is handled +** within the output subroutine. The regPrev register set holds the previously +** output value. A comparison is made against this value and the output +** is skipped if the next results would be the same as the previous. +** +** The implementation plan is to implement the two coroutines and seven +** subroutines first, then put the control logic at the bottom. Like this: +** +** goto Init +** coA: coroutine for left query (A) +** coB: coroutine for right query (B) +** outA: output one row of A +** outB: output one row of B (UNION and UNION ALL only) +** EofA: ... +** EofB: ... +** AltB: ... +** AeqB: ... +** AgtB: ... +** Init: initialize coroutine registers +** yield coA +** if eof(A) goto EofA +** yield coB +** if eof(B) goto EofB +** Cmpr: Compare A, B +** Jump AltB, AeqB, AgtB +** End: ... +** +** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not +** actually called using Gosub and they do not Return. EofA and EofB loop +** until all data is exhausted then jump to the "end" labe. AltB, AeqB, +** and AgtB jump to either L2 or to one of EofA or EofB. +*/ +#ifndef SQLITE_OMIT_COMPOUND_SELECT +static int multiSelectOrderBy( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + int i, j; /* Loop counters */ + Select *pPrior; /* Another SELECT immediately to our left */ + Vdbe *v; /* Generate code to this VDBE */ + SelectDest destA; /* Destination for coroutine A */ + SelectDest destB; /* Destination for coroutine B */ + int regAddrA; /* Address register for select-A coroutine */ + int regEofA; /* Flag to indicate when select-A is complete */ + int regAddrB; /* Address register for select-B coroutine */ + int regEofB; /* Flag to indicate when select-B is complete */ + int addrSelectA; /* Address of the select-A coroutine */ + int addrSelectB; /* Address of the select-B coroutine */ + int regOutA; /* Address register for the output-A subroutine */ + int regOutB; /* Address register for the output-B subroutine */ + int addrOutA; /* Address of the output-A subroutine */ + int addrOutB = 0; /* Address of the output-B subroutine */ + int addrEofA; /* Address of the select-A-exhausted subroutine */ + int addrEofB; /* Address of the select-B-exhausted subroutine */ + int addrAltB; /* Address of the AB subroutine */ + int regLimitA; /* Limit register for select-A */ + int regLimitB; /* Limit register for select-A */ + int regPrev; /* A range of registers to hold previous output */ + int savedLimit; /* Saved value of p->iLimit */ + int savedOffset; /* Saved value of p->iOffset */ + int labelCmpr; /* Label for the start of the merge algorithm */ + int labelEnd; /* Label for the end of the overall SELECT stmt */ + int j1; /* Jump instructions that get retargetted */ + int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ + KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ + KeyInfo *pKeyMerge; /* Comparison information for merging rows */ + sqlite3 *db; /* Database connection */ + ExprList *pOrderBy; /* The ORDER BY clause */ + int nOrderBy; /* Number of terms in the ORDER BY clause */ + int *aPermute; /* Mapping from ORDER BY terms to result set columns */ + + assert( p->pOrderBy!=0 ); + assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */ + db = pParse->db; + v = pParse->pVdbe; + if( v==0 ) return SQLITE_NOMEM; + labelEnd = sqlite3VdbeMakeLabel(v); + labelCmpr = sqlite3VdbeMakeLabel(v); + + + /* Patch up the ORDER BY clause + */ + op = p->op; + pPrior = p->pPrior; + assert( pPrior->pOrderBy==0 ); + pOrderBy = p->pOrderBy; + assert( pOrderBy ); + nOrderBy = pOrderBy->nExpr; + + /* For operators other than UNION ALL we have to make sure that + ** the ORDER BY clause covers every term of the result set. Add + ** terms to the ORDER BY clause as necessary. + */ + if( op!=TK_ALL ){ + for(i=1; db->mallocFailed==0 && i<=p->pEList->nExpr; i++){ + struct ExprList_item *pItem; + for(j=0, pItem=pOrderBy->a; jiCol>0 ); + if( pItem->iCol==i ) break; + } + if( j==nOrderBy ){ + Expr *pNew = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, 0); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->flags |= EP_IntValue; + pNew->iTable = i; + pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew, 0); + pOrderBy->a[nOrderBy++].iCol = (u16)i; + } + } + } + + /* Compute the comparison permutation and keyinfo that is used with + ** the permutation in order to comparisons to determine if the next + ** row of results comes from selectA or selectB. Also add explicit + ** collations to the ORDER BY clause terms so that when the subqueries + ** to the right and the left are evaluated, they use the correct + ** collation. + */ + aPermute = sqlite3DbMallocRaw(db, sizeof(int)*nOrderBy); + if( aPermute ){ + struct ExprList_item *pItem; + for(i=0, pItem=pOrderBy->a; iiCol>0 && pItem->iCol<=p->pEList->nExpr ); + aPermute[i] = pItem->iCol - 1; + } + pKeyMerge = + sqlite3DbMallocRaw(db, sizeof(*pKeyMerge)+nOrderBy*(sizeof(CollSeq*)+1)); + if( pKeyMerge ){ + pKeyMerge->aSortOrder = (u8*)&pKeyMerge->aColl[nOrderBy]; + pKeyMerge->nField = (u16)nOrderBy; + pKeyMerge->enc = ENC(db); + for(i=0; ia[i].pExpr; + if( pTerm->flags & EP_ExpCollate ){ + pColl = pTerm->pColl; + }else{ + pColl = multiSelectCollSeq(pParse, p, aPermute[i]); + pTerm->flags |= EP_ExpCollate; + pTerm->pColl = pColl; + } + pKeyMerge->aColl[i] = pColl; + pKeyMerge->aSortOrder[i] = pOrderBy->a[i].sortOrder; + } + } + }else{ + pKeyMerge = 0; + } + + /* Reattach the ORDER BY clause to the query. + */ + p->pOrderBy = pOrderBy; + pPrior->pOrderBy = sqlite3ExprListDup(pParse->db, pOrderBy); + + /* Allocate a range of temporary registers and the KeyInfo needed + ** for the logic that removes duplicate result rows when the + ** operator is UNION, EXCEPT, or INTERSECT (but not UNION ALL). + */ + if( op==TK_ALL ){ + regPrev = 0; + }else{ + int nExpr = p->pEList->nExpr; + assert( nOrderBy>=nExpr || db->mallocFailed ); + regPrev = sqlite3GetTempRange(pParse, nExpr+1); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev); + pKeyDup = sqlite3DbMallocZero(db, + sizeof(*pKeyDup) + nExpr*(sizeof(CollSeq*)+1) ); + if( pKeyDup ){ + pKeyDup->aSortOrder = (u8*)&pKeyDup->aColl[nExpr]; + pKeyDup->nField = (u16)nExpr; + pKeyDup->enc = ENC(db); + for(i=0; iaColl[i] = multiSelectCollSeq(pParse, p, i); + pKeyDup->aSortOrder[i] = 0; + } + } + } + + /* Separate the left and the right query from one another + */ + p->pPrior = 0; + pPrior->pRightmost = 0; + sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); + if( pPrior->pPrior==0 ){ + sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); + } + + /* Compute the limit registers */ + computeLimitRegisters(pParse, p, labelEnd); + if( p->iLimit && op==TK_ALL ){ + regLimitA = ++pParse->nMem; + regLimitB = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Copy, p->iOffset ? p->iOffset+1 : p->iLimit, + regLimitA); + sqlite3VdbeAddOp2(v, OP_Copy, regLimitA, regLimitB); + }else{ + regLimitA = regLimitB = 0; + } + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = 0; + sqlite3ExprDelete(db, p->pOffset); + p->pOffset = 0; + + regAddrA = ++pParse->nMem; + regEofA = ++pParse->nMem; + regAddrB = ++pParse->nMem; + regEofB = ++pParse->nMem; + regOutA = ++pParse->nMem; + regOutB = ++pParse->nMem; + sqlite3SelectDestInit(&destA, SRT_Coroutine, regAddrA); + sqlite3SelectDestInit(&destB, SRT_Coroutine, regAddrB); + + /* Jump past the various subroutines and coroutines to the main + ** merge loop + */ + j1 = sqlite3VdbeAddOp0(v, OP_Goto); + addrSelectA = sqlite3VdbeCurrentAddr(v); + + + /* Generate a coroutine to evaluate the SELECT statement to the + ** left of the compound operator - the "A" select. + */ + VdbeNoopComment((v, "Begin coroutine for left SELECT")); + pPrior->iLimit = regLimitA; + sqlite3Select(pParse, pPrior, &destA); + sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofA); + sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); + VdbeNoopComment((v, "End coroutine for left SELECT")); + + /* Generate a coroutine to evaluate the SELECT statement on + ** the right - the "B" select + */ + addrSelectB = sqlite3VdbeCurrentAddr(v); + VdbeNoopComment((v, "Begin coroutine for right SELECT")); + savedLimit = p->iLimit; + savedOffset = p->iOffset; + p->iLimit = regLimitB; + p->iOffset = 0; + sqlite3Select(pParse, p, &destB); + p->iLimit = savedLimit; + p->iOffset = savedOffset; + sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofB); + sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); + VdbeNoopComment((v, "End coroutine for right SELECT")); + + /* Generate a subroutine that outputs the current row of the A + ** select as the next output row of the compound select. + */ + VdbeNoopComment((v, "Output routine for A")); + addrOutA = generateOutputSubroutine(pParse, + p, &destA, pDest, regOutA, + regPrev, pKeyDup, P4_KEYINFO_HANDOFF, labelEnd); + + /* Generate a subroutine that outputs the current row of the B + ** select as the next output row of the compound select. + */ + if( op==TK_ALL || op==TK_UNION ){ + VdbeNoopComment((v, "Output routine for B")); + addrOutB = generateOutputSubroutine(pParse, + p, &destB, pDest, regOutB, + regPrev, pKeyDup, P4_KEYINFO_STATIC, labelEnd); + } + + /* Generate a subroutine to run when the results from select A + ** are exhausted and only data in select B remains. + */ + VdbeNoopComment((v, "eof-A subroutine")); + if( op==TK_EXCEPT || op==TK_INTERSECT ){ + addrEofA = sqlite3VdbeAddOp2(v, OP_Goto, 0, labelEnd); + }else{ + addrEofA = sqlite3VdbeAddOp2(v, OP_If, regEofB, labelEnd); + sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); + } + + /* Generate a subroutine to run when the results from select B + ** are exhausted and only data in select A remains. + */ + if( op==TK_INTERSECT ){ + addrEofB = addrEofA; + }else{ + VdbeNoopComment((v, "eof-B subroutine")); + addrEofB = sqlite3VdbeAddOp2(v, OP_If, regEofA, labelEnd); + sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); + sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofB); + } + + /* Generate code to handle the case of AB + */ + VdbeNoopComment((v, "A-gt-B subroutine")); + addrAgtB = sqlite3VdbeCurrentAddr(v); + if( op==TK_ALL || op==TK_UNION ){ + sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + } + sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); + sqlite3VdbeAddOp2(v, OP_If, regEofB, addrEofB); + sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); + + /* This code runs once to initialize everything. + */ + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regEofA); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regEofB); + sqlite3VdbeAddOp2(v, OP_Gosub, regAddrA, addrSelectA); + sqlite3VdbeAddOp2(v, OP_Gosub, regAddrB, addrSelectB); + sqlite3VdbeAddOp2(v, OP_If, regEofA, addrEofA); + sqlite3VdbeAddOp2(v, OP_If, regEofB, addrEofB); + + /* Implement the main merge loop + */ + sqlite3VdbeResolveLabel(v, labelCmpr); + sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); + sqlite3VdbeAddOp4(v, OP_Compare, destA.iMem, destB.iMem, nOrderBy, + (char*)pKeyMerge, P4_KEYINFO_HANDOFF); + sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); + + /* Release temporary registers + */ + if( regPrev ){ + sqlite3ReleaseTempRange(pParse, regPrev, nOrderBy+1); + } + + /* Jump to the this point in order to terminate the query. + */ + sqlite3VdbeResolveLabel(v, labelEnd); + + /* Set the number of output columns + */ + if( pDest->eDest==SRT_Output ){ + Select *pFirst = pPrior; + while( pFirst->pPrior ) pFirst = pFirst->pPrior; + generateColumnNames(pParse, 0, pFirst->pEList); + } + + /* Reassembly the compound query so that it will be freed correctly + ** by the calling function */ + if( p->pPrior ){ + sqlite3SelectDelete(db, p->pPrior); + } + p->pPrior = pPrior; + + /*** TBD: Insert subroutine calls to close cursors on incomplete + **** subqueries ****/ + return SQLITE_OK; +} +#endif + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* Forward Declarations */ +static void substExprList(sqlite3*, ExprList*, int, ExprList*); +static void substSelect(sqlite3*, Select *, int, ExprList *); + /* ** Scan through the expression pExpr. Replace every reference to ** a column in table number iTable with a copy of the iColumn-th @@ -2059,9 +2379,12 @@ multi_select_end: ** changes to pExpr so that it refers directly to the source table ** of the subquery rather the result set of the subquery. */ -static void substExprList(ExprList*,int,ExprList*); /* Forward Decl */ -static void substSelect(Select *, int, ExprList *); /* Forward Decl */ -static void substExpr(Expr *pExpr, int iTable, ExprList *pEList){ +static void substExpr( + sqlite3 *db, /* Report malloc errors to this connection */ + Expr *pExpr, /* Expr in which substitution occurs */ + int iTable, /* Table to be substituted */ + ExprList *pEList /* Substitute expressions */ +){ if( pExpr==0 ) return; if( pExpr->op==TK_COLUMN && pExpr->iTable==iTable ){ if( pExpr->iColumn<0 ){ @@ -2074,46 +2397,68 @@ static void substExpr(Expr *pExpr, int iTable, ExprList *pEList){ assert( pNew!=0 ); pExpr->op = pNew->op; assert( pExpr->pLeft==0 ); - pExpr->pLeft = sqlite3ExprDup(pNew->pLeft); + pExpr->pLeft = sqlite3ExprDup(db, pNew->pLeft); assert( pExpr->pRight==0 ); - pExpr->pRight = sqlite3ExprDup(pNew->pRight); + pExpr->pRight = sqlite3ExprDup(db, pNew->pRight); assert( pExpr->pList==0 ); - pExpr->pList = sqlite3ExprListDup(pNew->pList); + pExpr->pList = sqlite3ExprListDup(db, pNew->pList); pExpr->iTable = pNew->iTable; pExpr->pTab = pNew->pTab; pExpr->iColumn = pNew->iColumn; pExpr->iAgg = pNew->iAgg; - sqlite3TokenCopy(&pExpr->token, &pNew->token); - sqlite3TokenCopy(&pExpr->span, &pNew->span); - pExpr->pSelect = sqlite3SelectDup(pNew->pSelect); + sqlite3TokenCopy(db, &pExpr->token, &pNew->token); + sqlite3TokenCopy(db, &pExpr->span, &pNew->span); + pExpr->pSelect = sqlite3SelectDup(db, pNew->pSelect); pExpr->flags = pNew->flags; + pExpr->pAggInfo = pNew->pAggInfo; + pNew->pAggInfo = 0; } }else{ - substExpr(pExpr->pLeft, iTable, pEList); - substExpr(pExpr->pRight, iTable, pEList); - substSelect(pExpr->pSelect, iTable, pEList); - substExprList(pExpr->pList, iTable, pEList); + substExpr(db, pExpr->pLeft, iTable, pEList); + substExpr(db, pExpr->pRight, iTable, pEList); + substSelect(db, pExpr->pSelect, iTable, pEList); + substExprList(db, pExpr->pList, iTable, pEList); } } -static void substExprList(ExprList *pList, int iTable, ExprList *pEList){ +static void substExprList( + sqlite3 *db, /* Report malloc errors here */ + ExprList *pList, /* List to scan and in which to make substitutes */ + int iTable, /* Table to be substituted */ + ExprList *pEList /* Substitute values */ +){ int i; if( pList==0 ) return; for(i=0; inExpr; i++){ - substExpr(pList->a[i].pExpr, iTable, pEList); + substExpr(db, pList->a[i].pExpr, iTable, pEList); } } -static void substSelect(Select *p, int iTable, ExprList *pEList){ +static void substSelect( + sqlite3 *db, /* Report malloc errors here */ + Select *p, /* SELECT statement in which to make substitutions */ + int iTable, /* Table to be replaced */ + ExprList *pEList /* Substitute values */ +){ + SrcList *pSrc; + struct SrcList_item *pItem; + int i; if( !p ) return; - substExprList(p->pEList, iTable, pEList); - substExprList(p->pGroupBy, iTable, pEList); - substExprList(p->pOrderBy, iTable, pEList); - substExpr(p->pHaving, iTable, pEList); - substExpr(p->pWhere, iTable, pEList); - substSelect(p->pPrior, iTable, pEList); + substExprList(db, p->pEList, iTable, pEList); + substExprList(db, p->pGroupBy, iTable, pEList); + substExprList(db, p->pOrderBy, iTable, pEList); + substExpr(db, p->pHaving, iTable, pEList); + substExpr(db, p->pWhere, iTable, pEList); + substSelect(db, p->pPrior, iTable, pEList); + pSrc = p->pSrc; + assert( pSrc ); /* Even for (SELECT 1) we have: pSrc!=0 but pSrc->nSrc==0 */ + if( ALWAYS(pSrc) ){ + for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ + substSelect(db, pItem->pSelect, iTable, pEList); + } + } } -#endif /* !defined(SQLITE_OMIT_VIEW) */ +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ -#ifndef SQLITE_OMIT_VIEW +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** This routine attempts to flatten subqueries in order to speed ** execution. It returns 1 if it makes changes and 0 if no flattening @@ -2147,8 +2492,8 @@ static void substSelect(Select *p, int iTable, ExprList *pEList){ ** ** (2) The subquery is not an aggregate or the outer query is not a join. ** -** (3) The subquery is not the right operand of a left outer join, or -** the subquery is not itself a join. (Ticket #306) +** (3) The subquery is not the right operand of a left outer join +** (Originally ticket #306. Strenghtened by ticket #3300) ** ** (4) The subquery is not DISTINCT or the outer query is not a join. ** @@ -2170,8 +2515,8 @@ static void substSelect(Select *p, int iTable, ExprList *pEList){ ** ** (11) The subquery and the outer query do not both have ORDER BY clauses. ** -** (12) The subquery is not the right term of a LEFT OUTER JOIN or the -** subquery has no WHERE clause. (added by ticket #350) +** (12) Not implemented. Subsumed into restriction (3). Was previously +** a separate restriction deriving from ticket #350. ** ** (13) The subquery and outer query do not both use LIMIT ** @@ -2181,6 +2526,29 @@ static void substSelect(Select *p, int iTable, ExprList *pEList){ ** subquery does not have both an ORDER BY and a LIMIT clause. ** (See ticket #2339) ** +** (16) The outer query is not an aggregate or the subquery does +** not contain ORDER BY. (Ticket #2942) This used to not matter +** until we introduced the group_concat() function. +** +** (17) The sub-query is not a compound select, or it is a UNION ALL +** compound clause made up entirely of non-aggregate queries, and +** the parent query: +** +** * is not itself part of a compound select, +** * is not an aggregate or DISTINCT query, and +** * has no other tables or sub-selects in the FROM clause. +** +** The parent and sub-query may contain WHERE clauses. Subject to +** rules (11), (13) and (14), they may also contain ORDER BY, +** LIMIT and OFFSET clauses. +** +** (18) If the sub-query is a compound select, then all terms of the +** ORDER by clause of the parent must be simple references to +** columns of the sub-query. +** +** (19) The subquery does not use LIMIT or the outer query does not +** have a WHERE clause. +** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query ** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates. @@ -2192,12 +2560,16 @@ static void substSelect(Select *p, int iTable, ExprList *pEList){ ** the subquery before this routine runs. */ static int flattenSubquery( + Parse *pParse, /* Parsing context */ Select *p, /* The parent or outer SELECT statement */ int iFrom, /* Index in p->pSrc->a[] of the inner subquery */ int isAgg, /* True if outer SELECT uses aggregate functions */ int subqueryIsAgg /* True if the subquery uses aggregate functions */ ){ + const char *zSavedAuthContext = pParse->zAuthContext; + Select *pParent; Select *pSub; /* The inner query or "subquery" */ + Select *pSub1; /* Pointer to the rightmost select in sub-query */ SrcList *pSrc; /* The FROM clause of the outer query */ SrcList *pSubSrc; /* The FROM clause of the subquery */ ExprList *pList; /* The result set of the outer query */ @@ -2205,13 +2577,16 @@ static int flattenSubquery( int i; /* Loop counter */ Expr *pWhere; /* The WHERE clause */ struct SrcList_item *pSubitem; /* The subquery */ + sqlite3 *db = pParse->db; /* Check to see if flattening is permitted. Return 0 if not. */ - if( p==0 ) return 0; + assert( p!=0 ); + assert( p->pPrior==0 ); /* Unable to flatten compound queries */ pSrc = p->pSrc; assert( pSrc && iFrom>=0 && iFromnSrc ); pSubitem = &pSrc->a[iFrom]; + iParent = pSubitem->iCursor; pSub = pSubitem->pSelect; assert( pSub!=0 ); if( isAgg && subqueryIsAgg ) return 0; /* Restriction (1) */ @@ -2229,16 +2604,21 @@ static int flattenSubquery( return 0; /* Restriction (15) */ } if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ - if( (pSub->isDistinct || pSub->pLimit) + if( ((pSub->selFlags & SF_Distinct)!=0 || pSub->pLimit) && (pSrc->nSrc>1 || isAgg) ){ /* Restrictions (4)(5)(8)(9) */ return 0; } - if( p->isDistinct && subqueryIsAgg ) return 0; /* Restriction (6) */ - if( (p->disallowOrderBy || p->pOrderBy) && pSub->pOrderBy ){ + if( (p->selFlags & SF_Distinct)!=0 && subqueryIsAgg ){ + return 0; /* Restriction (6) */ + } + if( p->pOrderBy && pSub->pOrderBy ){ return 0; /* Restriction (11) */ } + if( isAgg && pSub->pOrderBy ) return 0; /* Restriction (16) */ + if( pSub->pLimit && p->pWhere ) return 0; /* Restriction (19) */ - /* Restriction 3: If the subquery is a join, make sure the subquery is + /* OBSOLETE COMMENT 1: + ** Restriction 3: If the subquery is a join, make sure the subquery is ** not used as the right operand of an outer join. Examples of why this ** is not allowed: ** @@ -2249,12 +2629,9 @@ static int flattenSubquery( ** (t1 LEFT OUTER JOIN t2) JOIN t3 ** ** which is not at all the same thing. - */ - if( pSubSrc->nSrc>1 && (pSubitem->jointype & JT_OUTER)!=0 ){ - return 0; - } - - /* Restriction 12: If the subquery is the right operand of a left outer + ** + ** OBSOLETE COMMENT 2: + ** Restriction 12: If the subquery is the right operand of a left outer ** join, make sure the subquery has no WHERE clause. ** An examples of why this is not allowed: ** @@ -2266,16 +2643,146 @@ static int flattenSubquery( ** ** But the t2.x>0 test will always fail on a NULL row of t2, which ** effectively converts the OUTER JOIN into an INNER JOIN. + ** + ** THIS OVERRIDES OBSOLETE COMMENTS 1 AND 2 ABOVE: + ** Ticket #3300 shows that flattening the right term of a LEFT JOIN + ** is fraught with danger. Best to avoid the whole thing. If the + ** subquery is the right term of a LEFT JOIN, then do not flatten. */ - if( (pSubitem->jointype & JT_OUTER)!=0 && pSub->pWhere!=0 ){ + if( (pSubitem->jointype & JT_OUTER)!=0 ){ return 0; } - /* If we reach this point, it means flattening is permitted for the - ** iFrom-th entry of the FROM clause in the outer query. + /* Restriction 17: If the sub-query is a compound SELECT, then it must + ** use only the UNION ALL operator. And none of the simple select queries + ** that make up the compound SELECT are allowed to be aggregate or distinct + ** queries. */ + if( pSub->pPrior ){ + if( isAgg || (p->selFlags & SF_Distinct)!=0 || pSrc->nSrc!=1 ){ + return 0; + } + for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){ + if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 + || (pSub1->pPrior && pSub1->op!=TK_ALL) + || !pSub1->pSrc || pSub1->pSrc->nSrc!=1 + ){ + return 0; + } + } - /* Move all of the FROM elements of the subquery into the + /* Restriction 18. */ + if( p->pOrderBy ){ + int ii; + for(ii=0; iipOrderBy->nExpr; ii++){ + if( p->pOrderBy->a[ii].iCol==0 ) return 0; + } + } + } + + /***** If we reach this point, flattening is permitted. *****/ + + /* Authorize the subquery */ + pParse->zAuthContext = pSubitem->zName; + sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); + pParse->zAuthContext = zSavedAuthContext; + + /* If the sub-query is a compound SELECT statement, then (by restrictions + ** 17 and 18 above) it must be a UNION ALL and the parent query must + ** be of the form: + ** + ** SELECT FROM () + ** + ** followed by any ORDER BY, LIMIT and/or OFFSET clauses. This block + ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or + ** OFFSET clauses and joins them to the left-hand-side of the original + ** using UNION ALL operators. In this case N is the number of simple + ** select statements in the compound sub-query. + ** + ** Example: + ** + ** SELECT a+1 FROM ( + ** SELECT x FROM tab + ** UNION ALL + ** SELECT y FROM tab + ** UNION ALL + ** SELECT abs(z*2) FROM tab2 + ** ) WHERE a!=5 ORDER BY 1 + ** + ** Transformed into: + ** + ** SELECT x+1 FROM tab WHERE x+1!=5 + ** UNION ALL + ** SELECT y+1 FROM tab WHERE y+1!=5 + ** UNION ALL + ** SELECT abs(z*2)+1 FROM tab2 WHERE abs(z*2)+1!=5 + ** ORDER BY 1 + ** + ** We call this the "compound-subquery flattening". + */ + for(pSub=pSub->pPrior; pSub; pSub=pSub->pPrior){ + Select *pNew; + ExprList *pOrderBy = p->pOrderBy; + Expr *pLimit = p->pLimit; + Select *pPrior = p->pPrior; + p->pOrderBy = 0; + p->pSrc = 0; + p->pPrior = 0; + p->pLimit = 0; + pNew = sqlite3SelectDup(db, p); + p->pLimit = pLimit; + p->pOrderBy = pOrderBy; + p->pSrc = pSrc; + p->op = TK_ALL; + p->pRightmost = 0; + if( pNew==0 ){ + pNew = pPrior; + }else{ + pNew->pPrior = pPrior; + pNew->pRightmost = 0; + } + p->pPrior = pNew; + if( db->mallocFailed ) return 1; + } + + /* Begin flattening the iFrom-th entry of the FROM clause + ** in the outer query. + */ + pSub = pSub1 = pSubitem->pSelect; + + /* Delete the transient table structure associated with the + ** subquery + */ + sqlite3DbFree(db, pSubitem->zDatabase); + sqlite3DbFree(db, pSubitem->zName); + sqlite3DbFree(db, pSubitem->zAlias); + pSubitem->zDatabase = 0; + pSubitem->zName = 0; + pSubitem->zAlias = 0; + pSubitem->pSelect = 0; + + /* Defer deleting the Table object associated with the + ** subquery until code generation is + ** complete, since there may still exist Expr.pTab entries that + ** refer to the subquery even after flattening. Ticket #3346. + */ + if( pSubitem->pTab!=0 ){ + Table *pTabToDel = pSubitem->pTab; + if( pTabToDel->nRef==1 ){ + pTabToDel->pNextZombie = pParse->pZombieTab; + pParse->pZombieTab = pTabToDel; + }else{ + pTabToDel->nRef--; + } + pSubitem->pTab = 0; + } + + /* The following loop runs once for each term in a compound-subquery + ** flattening (as described above). If we are doing a different kind + ** of flattening - a flattening other than a compound-subquery flattening - + ** then this loop only runs once. + ** + ** This loop moves all of the FROM elements of the subquery into the ** the FROM clause of the outer query. Before doing this, remember ** the cursor number for the original outer query FROM element in ** iParent. The iParent cursor will never be used. Subsequent code @@ -2283,439 +2790,557 @@ static int flattenSubquery( ** those references with expressions that resolve to the subquery FROM ** elements we are now copying in. */ - iParent = pSubitem->iCursor; - { - int nSubSrc = pSubSrc->nSrc; - int jointype = pSubitem->jointype; + for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ + int nSubSrc; + u8 jointype = 0; + pSubSrc = pSub->pSrc; /* FROM clause of subquery */ + nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ + pSrc = pParent->pSrc; /* FROM clause of the outer query */ - sqlite3DeleteTable(pSubitem->pTab); - sqliteFree(pSubitem->zDatabase); - sqliteFree(pSubitem->zName); - sqliteFree(pSubitem->zAlias); - if( nSubSrc>1 ){ - int extra = nSubSrc - 1; - for(i=1; ipSrc = pSrc; - for(i=pSrc->nSrc-1; i-extra>=iFrom; i--){ - pSrc->a[i] = pSrc->a[i-extra]; + if( pSrc ){ + assert( pParent==p ); /* First time through the loop */ + jointype = pSubitem->jointype; + }else{ + assert( pParent!=p ); /* 2nd and subsequent times through the loop */ + pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); + if( pSrc==0 ){ + assert( db->mallocFailed ); + break; } } + + /* The subquery uses a single slot of the FROM clause of the outer + ** query. If the subquery has more than one element in its FROM clause, + ** then expand the outer query to make space for it to hold all elements + ** of the subquery. + ** + ** Example: + ** + ** SELECT * FROM tabA, (SELECT * FROM sub1, sub2), tabB; + ** + ** The outer query has 3 slots in its FROM clause. One slot of the + ** outer query (the middle slot) is used by the subquery. The next + ** block of code will expand the out query to 4 slots. The middle + ** slot is expanded to two slots in order to make space for the + ** two elements in the FROM clause of the subquery. + */ + if( nSubSrc>1 ){ + pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); + if( db->mallocFailed ){ + break; + } + } + + /* Transfer the FROM clause terms from the subquery into the + ** outer query. + */ for(i=0; ia[i+iFrom] = pSubSrc->a[i]; memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } pSrc->a[iFrom].jointype = jointype; - } - - /* Now begin substituting subquery result set expressions for - ** references to the iParent in the outer query. - ** - ** Example: - ** - ** SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b; - ** \ \_____________ subquery __________/ / - ** \_____________________ outer query ______________________________/ - ** - ** We look at every expression in the outer query and every place we see - ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". - */ - pList = p->pEList; - for(i=0; inExpr; i++){ - Expr *pExpr; - if( pList->a[i].zName==0 && (pExpr = pList->a[i].pExpr)->span.z!=0 ){ - pList->a[i].zName = sqliteStrNDup((char*)pExpr->span.z, pExpr->span.n); + + /* Now begin substituting subquery result set expressions for + ** references to the iParent in the outer query. + ** + ** Example: + ** + ** SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b; + ** \ \_____________ subquery __________/ / + ** \_____________________ outer query ______________________________/ + ** + ** We look at every expression in the outer query and every place we see + ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". + */ + pList = pParent->pEList; + for(i=0; inExpr; i++){ + Expr *pExpr; + if( pList->a[i].zName==0 && (pExpr = pList->a[i].pExpr)->span.z!=0 ){ + pList->a[i].zName = + sqlite3DbStrNDup(db, (char*)pExpr->span.z, pExpr->span.n); + } + } + substExprList(db, pParent->pEList, iParent, pSub->pEList); + if( isAgg ){ + substExprList(db, pParent->pGroupBy, iParent, pSub->pEList); + substExpr(db, pParent->pHaving, iParent, pSub->pEList); + } + if( pSub->pOrderBy ){ + assert( pParent->pOrderBy==0 ); + pParent->pOrderBy = pSub->pOrderBy; + pSub->pOrderBy = 0; + }else if( pParent->pOrderBy ){ + substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); + } + if( pSub->pWhere ){ + pWhere = sqlite3ExprDup(db, pSub->pWhere); + }else{ + pWhere = 0; + } + if( subqueryIsAgg ){ + assert( pParent->pHaving==0 ); + pParent->pHaving = pParent->pWhere; + pParent->pWhere = pWhere; + substExpr(db, pParent->pHaving, iParent, pSub->pEList); + pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, + sqlite3ExprDup(db, pSub->pHaving)); + assert( pParent->pGroupBy==0 ); + pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy); + }else{ + substExpr(db, pParent->pWhere, iParent, pSub->pEList); + pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); + } + + /* The flattened query is distinct if either the inner or the + ** outer query is distinct. + */ + pParent->selFlags |= pSub->selFlags & SF_Distinct; + + /* + ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y; + ** + ** One is tempted to try to add a and b to combine the limits. But this + ** does not work if either limit is negative. + */ + if( pSub->pLimit ){ + pParent->pLimit = pSub->pLimit; + pSub->pLimit = 0; } - } - substExprList(p->pEList, iParent, pSub->pEList); - if( isAgg ){ - substExprList(p->pGroupBy, iParent, pSub->pEList); - substExpr(p->pHaving, iParent, pSub->pEList); - } - if( pSub->pOrderBy ){ - assert( p->pOrderBy==0 ); - p->pOrderBy = pSub->pOrderBy; - pSub->pOrderBy = 0; - }else if( p->pOrderBy ){ - substExprList(p->pOrderBy, iParent, pSub->pEList); - } - if( pSub->pWhere ){ - pWhere = sqlite3ExprDup(pSub->pWhere); - }else{ - pWhere = 0; - } - if( subqueryIsAgg ){ - assert( p->pHaving==0 ); - p->pHaving = p->pWhere; - p->pWhere = pWhere; - substExpr(p->pHaving, iParent, pSub->pEList); - p->pHaving = sqlite3ExprAnd(p->pHaving, sqlite3ExprDup(pSub->pHaving)); - assert( p->pGroupBy==0 ); - p->pGroupBy = sqlite3ExprListDup(pSub->pGroupBy); - }else{ - substExpr(p->pWhere, iParent, pSub->pEList); - p->pWhere = sqlite3ExprAnd(p->pWhere, pWhere); - } - - /* The flattened query is distinct if either the inner or the - ** outer query is distinct. - */ - p->isDistinct = p->isDistinct || pSub->isDistinct; - - /* - ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y; - ** - ** One is tempted to try to add a and b to combine the limits. But this - ** does not work if either limit is negative. - */ - if( pSub->pLimit ){ - p->pLimit = pSub->pLimit; - pSub->pLimit = 0; } /* Finially, delete what is left of the subquery and return ** success. */ - sqlite3SelectDelete(pSub); + sqlite3SelectDelete(db, pSub1); + return 1; } -#endif /* SQLITE_OMIT_VIEW */ +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ /* -** Analyze the SELECT statement passed in as an argument to see if it -** is a simple min() or max() query. If it is and this query can be -** satisfied using a single seek to the beginning or end of an index, -** then generate the code for this SELECT and return 1. If this is not a -** simple min() or max() query, then return 0; +** Analyze the SELECT statement passed as an argument to see if it +** is a min() or max() query. Return WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX if +** it is, or 0 otherwise. At present, a query is considered to be +** a min()/max() query if: ** -** A simply min() or max() query looks like this: +** 1. There is a single object in the FROM clause. ** -** SELECT min(a) FROM table; -** SELECT max(a) FROM table; -** -** The query may have only a single table in its FROM argument. There -** can be no GROUP BY or HAVING or WHERE clauses. The result set must -** be the min() or max() of a single column of the table. The column -** in the min() or max() function must be indexed. -** -** The parameters to this routine are the same as for sqlite3Select(). -** See the header comment on that routine for additional information. +** 2. There is a single expression in the result set, and it is +** either min(x) or max(x), where x is a column reference. */ -static int simpleMinMaxQuery(Parse *pParse, Select *p, int eDest, int iParm){ +static u8 minMaxQuery(Select *p){ Expr *pExpr; - int iCol; - Table *pTab; - Index *pIdx; - int base; - Vdbe *v; - int seekOp; - ExprList *pEList, *pList, eList; - struct ExprList_item eListItem; - SrcList *pSrc; - int brk; - int iDb; + ExprList *pEList = p->pEList; - /* Check to see if this query is a simple min() or max() query. Return - ** zero if it is not. - */ - if( p->pGroupBy || p->pHaving || p->pWhere ) return 0; - pSrc = p->pSrc; - if( pSrc->nSrc!=1 ) return 0; - pEList = p->pEList; - if( pEList->nExpr!=1 ) return 0; + if( pEList->nExpr!=1 ) return WHERE_ORDERBY_NORMAL; pExpr = pEList->a[0].pExpr; - if( pExpr->op!=TK_AGG_FUNCTION ) return 0; - pList = pExpr->pList; - if( pList==0 || pList->nExpr!=1 ) return 0; - if( pExpr->token.n!=3 ) return 0; + pEList = pExpr->pList; + if( pExpr->op!=TK_AGG_FUNCTION || pEList==0 || pEList->nExpr!=1 ) return 0; + if( pEList->a[0].pExpr->op!=TK_AGG_COLUMN ) return WHERE_ORDERBY_NORMAL; + if( pExpr->token.n!=3 ) return WHERE_ORDERBY_NORMAL; if( sqlite3StrNICmp((char*)pExpr->token.z,"min",3)==0 ){ - seekOp = OP_Rewind; + return WHERE_ORDERBY_MIN; }else if( sqlite3StrNICmp((char*)pExpr->token.z,"max",3)==0 ){ - seekOp = OP_Last; - }else{ - return 0; + return WHERE_ORDERBY_MAX; } - pExpr = pList->a[0].pExpr; - if( pExpr->op!=TK_COLUMN ) return 0; - iCol = pExpr->iColumn; - pTab = pSrc->a[0].pTab; - - /* This optimization cannot be used with virtual tables. */ - if( IsVirtual(pTab) ) return 0; - - /* If we get to here, it means the query is of the correct form. - ** Check to make sure we have an index and make pIdx point to the - ** appropriate index. If the min() or max() is on an INTEGER PRIMARY - ** key column, no index is necessary so set pIdx to NULL. If no - ** usable index is found, return 0. - */ - if( iCol<0 ){ - pIdx = 0; - }else{ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr); - if( pColl==0 ) return 0; - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - assert( pIdx->nColumn>=1 ); - if( pIdx->aiColumn[0]==iCol && - 0==sqlite3StrICmp(pIdx->azColl[0], pColl->zName) ){ - break; - } - } - if( pIdx==0 ) return 0; - } - - /* Identify column types if we will be using the callback. This - ** step is skipped if the output is going to a table or a memory cell. - ** The column names have already been generated in the calling function. - */ - v = sqlite3GetVdbe(pParse); - if( v==0 ) return 0; - - /* If the output is destined for a temporary table, open that table. - */ - if( eDest==SRT_EphemTab ){ - sqlite3VdbeAddOp(v, OP_OpenEphemeral, iParm, 1); - } - - /* Generating code to find the min or the max. Basically all we have - ** to do is find the first or the last entry in the chosen index. If - ** the min() or max() is on the INTEGER PRIMARY KEY, then find the first - ** or last entry in the main table. - */ - iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - assert( iDb>=0 || pTab->isEphem ); - sqlite3CodeVerifySchema(pParse, iDb); - sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - base = pSrc->a[0].iCursor; - brk = sqlite3VdbeMakeLabel(v); - computeLimitRegisters(pParse, p, brk); - if( pSrc->a[0].pSelect==0 ){ - sqlite3OpenTable(pParse, base, iDb, pTab, OP_OpenRead); - } - if( pIdx==0 ){ - sqlite3VdbeAddOp(v, seekOp, base, 0); - }else{ - /* Even though the cursor used to open the index here is closed - ** as soon as a single value has been read from it, allocate it - ** using (pParse->nTab++) to prevent the cursor id from being - ** reused. This is important for statements of the form - ** "INSERT INTO x SELECT max() FROM x". - */ - int iIdx; - KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); - iIdx = pParse->nTab++; - assert( pIdx->pSchema==pTab->pSchema ); - sqlite3VdbeAddOp(v, OP_Integer, iDb, 0); - sqlite3VdbeOp3(v, OP_OpenRead, iIdx, pIdx->tnum, - (char*)pKey, P3_KEYINFO_HANDOFF); - if( seekOp==OP_Rewind ){ - sqlite3VdbeAddOp(v, OP_Null, 0, 0); - sqlite3VdbeAddOp(v, OP_MakeRecord, 1, 0); - seekOp = OP_MoveGt; - } - if( pIdx->aSortOrder[0]==SQLITE_SO_DESC ){ - /* Ticket #2514: invert the seek operator if we are using - ** a descending index. */ - if( seekOp==OP_Last ){ - seekOp = OP_Rewind; - }else{ - assert( seekOp==OP_MoveGt ); - seekOp = OP_MoveLt; - } - } - sqlite3VdbeAddOp(v, seekOp, iIdx, 0); - sqlite3VdbeAddOp(v, OP_IdxRowid, iIdx, 0); - sqlite3VdbeAddOp(v, OP_Close, iIdx, 0); - sqlite3VdbeAddOp(v, OP_MoveGe, base, 0); - } - eList.nExpr = 1; - memset(&eListItem, 0, sizeof(eListItem)); - eList.a = &eListItem; - eList.a[0].pExpr = pExpr; - selectInnerLoop(pParse, p, &eList, 0, 0, 0, -1, eDest, iParm, brk, brk, 0); - sqlite3VdbeResolveLabel(v, brk); - sqlite3VdbeAddOp(v, OP_Close, base, 0); - - return 1; + return WHERE_ORDERBY_NORMAL; } /* -** Analyze and ORDER BY or GROUP BY clause in a SELECT statement. Return -** the number of errors seen. +** If the source-list item passed as an argument was augmented with an +** INDEXED BY clause, then try to locate the specified index. If there +** was such a clause and the named index cannot be found, return +** SQLITE_ERROR and leave an error in pParse. Otherwise, populate +** pFrom->pIndex and return SQLITE_OK. +*/ +int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ + if( pFrom->pTab && pFrom->zIndex ){ + Table *pTab = pFrom->pTab; + char *zIndex = pFrom->zIndex; + Index *pIdx; + for(pIdx=pTab->pIndex; + pIdx && sqlite3StrICmp(pIdx->zName, zIndex); + pIdx=pIdx->pNext + ); + if( !pIdx ){ + sqlite3ErrorMsg(pParse, "no such index: %s", zIndex, 0); + return SQLITE_ERROR; + } + pFrom->pIndex = pIdx; + } + return SQLITE_OK; +} + +/* +** This routine is a Walker callback for "expanding" a SELECT statement. +** "Expanding" means to do the following: +** +** (1) Make sure VDBE cursor numbers have been assigned to every +** element of the FROM clause. +** +** (2) Fill in the pTabList->a[].pTab fields in the SrcList that +** defines FROM clause. When views appear in the FROM clause, +** fill pTabList->a[].pSelect with a copy of the SELECT statement +** that implements the view. A copy is made of the view's SELECT +** statement so that we can freely modify or delete that statement +** without worrying about messing up the presistent representation +** of the view. +** +** (3) Add terms to the WHERE clause to accomodate the NATURAL keyword +** on joins and the ON and USING clause of joins. +** +** (4) Scan the list of columns in the result set (pEList) looking +** for instances of the "*" operator or the TABLE.* operator. +** If found, expand each "*" to be every column in every table +** and TABLE.* to be every column in TABLE. ** -** An ORDER BY or GROUP BY is a list of expressions. If any expression -** is an integer constant, then that expression is replaced by the -** corresponding entry in the result set. */ -static int processOrderGroupBy( - NameContext *pNC, /* Name context of the SELECT statement. */ - ExprList *pOrderBy, /* The ORDER BY or GROUP BY clause to be processed */ - const char *zType /* Either "ORDER" or "GROUP", as appropriate */ -){ - int i; - ExprList *pEList = pNC->pEList; /* The result set of the SELECT */ - Parse *pParse = pNC->pParse; /* The result set of the SELECT */ - assert( pEList ); +static int selectExpander(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + int i, j, k; + SrcList *pTabList; + ExprList *pEList; + struct SrcList_item *pFrom; + sqlite3 *db = pParse->db; - if( pOrderBy==0 ) return 0; - if( pOrderBy->nExpr>SQLITE_MAX_COLUMN ){ - sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType); - return 1; + if( db->mallocFailed ){ + return WRC_Abort; } - for(i=0; inExpr; i++){ - int iCol; - Expr *pE = pOrderBy->a[i].pExpr; - if( sqlite3ExprIsInteger(pE, &iCol) ){ - if( iCol>0 && iCol<=pEList->nExpr ){ - CollSeq *pColl = pE->pColl; - int flags = pE->flags & EP_ExpCollate; - sqlite3ExprDelete(pE); - pE = pOrderBy->a[i].pExpr = sqlite3ExprDup(pEList->a[iCol-1].pExpr); - if( pColl && flags ){ - pE->pColl = pColl; - pE->flags |= flags; + if( p->pSrc==0 || (p->selFlags & SF_Expanded)!=0 ){ + return WRC_Prune; + } + p->selFlags |= SF_Expanded; + pTabList = p->pSrc; + pEList = p->pEList; + + /* Make sure cursor numbers have been assigned to all entries in + ** the FROM clause of the SELECT statement. + */ + sqlite3SrcListAssignCursors(pParse, pTabList); + + /* Look up every table named in the FROM clause of the select. If + ** an entry of the FROM clause is a subquery instead of a table or view, + ** then create a transient table structure to describe the subquery. + */ + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab; + if( pFrom->pTab!=0 ){ + /* This statement has already been prepared. There is no need + ** to go further. */ + assert( i==0 ); + return WRC_Prune; + } + if( pFrom->zName==0 ){ +#ifndef SQLITE_OMIT_SUBQUERY + Select *pSel = pFrom->pSelect; + /* A sub-query in the FROM clause of a SELECT */ + assert( pSel!=0 ); + assert( pFrom->pTab==0 ); + sqlite3WalkSelect(pWalker, pSel); + pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return WRC_Abort; + pTab->db = db; + pTab->nRef = 1; + pTab->zName = sqlite3MPrintf(db, "sqlite_subquery_%p_", (void*)pTab); + while( pSel->pPrior ){ pSel = pSel->pPrior; } + selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); + pTab->iPKey = -1; + pTab->tabFlags |= TF_Ephemeral; +#endif + }else{ + /* An ordinary table or view name in the FROM clause */ + assert( pFrom->pTab==0 ); + pFrom->pTab = pTab = + sqlite3LocateTable(pParse,0,pFrom->zName,pFrom->zDatabase); + if( pTab==0 ) return WRC_Abort; + pTab->nRef++; +#if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) + if( pTab->pSelect || IsVirtual(pTab) ){ + /* We reach here if the named table is a really a view */ + if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; + + /* If pFrom->pSelect!=0 it means we are dealing with a + ** view within a view. The SELECT structure has already been + ** copied by the outer view so we can skip the copy step here + ** in the inner view. + */ + if( pFrom->pSelect==0 ){ + pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect); + sqlite3WalkSelect(pWalker, pFrom->pSelect); } + } +#endif + } + + /* Locate the index named by the INDEXED BY clause, if any. */ + if( sqlite3IndexedByLookup(pParse, pFrom) ){ + return WRC_Abort; + } + } + + /* Process NATURAL keywords, and ON and USING clauses of joins. + */ + if( db->mallocFailed || sqliteProcessJoin(pParse, p) ){ + return WRC_Abort; + } + + /* For every "*" that occurs in the column list, insert the names of + ** all columns in all tables. And for every TABLE.* insert the names + ** of all columns in TABLE. The parser inserted a special expression + ** with the TK_ALL operator for each "*" that it found in the column list. + ** The following code just has to locate the TK_ALL expressions and expand + ** each one to the list of all columns in all tables. + ** + ** The first loop just checks to see if there are any "*" operators + ** that need expanding. + */ + for(k=0; knExpr; k++){ + Expr *pE = pEList->a[k].pExpr; + if( pE->op==TK_ALL ) break; + if( pE->op==TK_DOT && pE->pRight && pE->pRight->op==TK_ALL + && pE->pLeft && pE->pLeft->op==TK_ID ) break; + } + if( knExpr ){ + /* + ** If we get here it means the result set contains one or more "*" + ** operators that need to be expanded. Loop through each expression + ** in the result set and expand them one by one. + */ + struct ExprList_item *a = pEList->a; + ExprList *pNew = 0; + int flags = pParse->db->flags; + int longNames = (flags & SQLITE_FullColNames)!=0 + && (flags & SQLITE_ShortColNames)==0; + + for(k=0; knExpr; k++){ + Expr *pE = a[k].pExpr; + if( pE->op!=TK_ALL && + (pE->op!=TK_DOT || pE->pRight==0 || pE->pRight->op!=TK_ALL) ){ + /* This particular expression does not need to be expanded. + */ + pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr, 0); + if( pNew ){ + pNew->a[pNew->nExpr-1].zName = a[k].zName; + } + a[k].pExpr = 0; + a[k].zName = 0; }else{ - sqlite3ErrorMsg(pParse, - "%s BY column number %d out of range - should be " - "between 1 and %d", zType, iCol, pEList->nExpr); - return 1; + /* This expression is a "*" or a "TABLE.*" and needs to be + ** expanded. */ + int tableSeen = 0; /* Set to 1 when TABLE matches */ + char *zTName; /* text of name of TABLE */ + if( pE->op==TK_DOT && pE->pLeft ){ + zTName = sqlite3NameFromToken(db, &pE->pLeft->token); + }else{ + zTName = 0; + } + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + char *zTabName = pFrom->zAlias; + if( zTabName==0 || zTabName[0]==0 ){ + zTabName = pTab->zName; + } + if( db->mallocFailed ) break; + if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ + continue; + } + tableSeen = 1; + for(j=0; jnCol; j++){ + Expr *pExpr, *pRight; + char *zName = pTab->aCol[j].zName; + + /* If a column is marked as 'hidden' (currently only possible + ** for virtual tables), do not include it in the expanded + ** result-set list. + */ + if( IsHiddenColumn(&pTab->aCol[j]) ){ + assert(IsVirtual(pTab)); + continue; + } + + if( i>0 && zTName==0 ){ + struct SrcList_item *pLeft = &pTabList->a[i-1]; + if( (pLeft[1].jointype & JT_NATURAL)!=0 && + columnIndex(pLeft->pTab, zName)>=0 ){ + /* In a NATURAL join, omit the join columns from the + ** table on the right */ + continue; + } + if( sqlite3IdListIndex(pLeft[1].pUsing, zName)>=0 ){ + /* In a join with a USING clause, omit columns in the + ** using clause from the table on the right. */ + continue; + } + } + pRight = sqlite3PExpr(pParse, TK_ID, 0, 0, 0); + if( pRight==0 ) break; + setQuotedToken(pParse, &pRight->token, zName); + if( longNames || pTabList->nSrc>1 ){ + Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, 0); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + if( pExpr==0 ) break; + setQuotedToken(pParse, &pLeft->token, zTabName); + setToken(&pExpr->span, + sqlite3MPrintf(db, "%s.%s", zTabName, zName)); + pExpr->span.dyn = 1; + pExpr->token.z = 0; + pExpr->token.n = 0; + pExpr->token.dyn = 0; + }else{ + pExpr = pRight; + pExpr->span = pExpr->token; + pExpr->span.dyn = 0; + } + if( longNames ){ + pNew = sqlite3ExprListAppend(pParse, pNew, pExpr, &pExpr->span); + }else{ + pNew = sqlite3ExprListAppend(pParse, pNew, pExpr, &pRight->token); + } + } + } + if( !tableSeen ){ + if( zTName ){ + sqlite3ErrorMsg(pParse, "no such table: %s", zTName); + }else{ + sqlite3ErrorMsg(pParse, "no tables specified"); + } + } + sqlite3DbFree(db, zTName); } } - if( sqlite3ExprResolveNames(pNC, pE) ){ - return 1; - } + sqlite3ExprListDelete(db, pEList); + p->pEList = pNew; } - return 0; +#if SQLITE_MAX_COLUMN + if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns in result set"); + } +#endif + return WRC_Continue; } /* -** This routine resolves any names used in the result set of the -** supplied SELECT statement. If the SELECT statement being resolved -** is a sub-select, then pOuterNC is a pointer to the NameContext -** of the parent SELECT. +** No-op routine for the parse-tree walker. +** +** When this routine is the Walker.xExprCallback then expression trees +** are walked without any actions being taken at each node. Presumably, +** when this routine is used for Walker.xExprCallback then +** Walker.xSelectCallback is set to do something useful for every +** subquery in the parser tree. */ -int sqlite3SelectResolve( +static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return WRC_Continue; +} + +/* +** This routine "expands" a SELECT statement and all of its subqueries. +** For additional information on what it means to "expand" a SELECT +** statement, see the comment on the selectExpand worker callback above. +** +** Expanding a SELECT statement is the first step in processing a +** SELECT statement. The SELECT statement must be expanded before +** name resolution is performed. +** +** If anything goes wrong, an error message is written into pParse. +** The calling function can detect the problem by looking at pParse->nErr +** and/or pParse->db->mallocFailed. +*/ +static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ + Walker w; + w.xSelectCallback = selectExpander; + w.xExprCallback = exprWalkNoop; + w.pParse = pParse; + sqlite3WalkSelect(&w, pSelect); +} + + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo() +** interface. +** +** For each FROM-clause subquery, add Column.zType and Column.zColl +** information to the Table structure that represents the result set +** of that subquery. +** +** The Table structure that represents the result set was constructed +** by selectExpander() but the type and collation information was omitted +** at that point because identifiers had not yet been resolved. This +** routine is called after identifier resolution. +*/ +static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ + Parse *pParse; + int i; + SrcList *pTabList; + struct SrcList_item *pFrom; + + assert( p->selFlags & SF_Resolved ); + if( (p->selFlags & SF_HasTypeInfo)==0 ){ + p->selFlags |= SF_HasTypeInfo; + pParse = pWalker->pParse; + pTabList = p->pSrc; + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + if( pTab && (pTab->tabFlags & TF_Ephemeral)!=0 ){ + /* A sub-query in the FROM clause of a SELECT */ + Select *pSel = pFrom->pSelect; + assert( pSel ); + while( pSel->pPrior ) pSel = pSel->pPrior; + selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSel); + } + } + } + return WRC_Continue; +} +#endif + + +/* +** This routine adds datatype and collating sequence information to +** the Table structures of all FROM-clause subqueries in a +** SELECT statement. +** +** Use this routine after name resolution. +*/ +static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ +#ifndef SQLITE_OMIT_SUBQUERY + Walker w; + w.xSelectCallback = selectAddSubqueryTypeInfo; + w.xExprCallback = exprWalkNoop; + w.pParse = pParse; + sqlite3WalkSelect(&w, pSelect); +#endif +} + + +/* +** This routine sets of a SELECT statement for processing. The +** following is accomplished: +** +** * VDBE Cursor numbers are assigned to all FROM-clause terms. +** * Ephemeral Table objects are created for all FROM-clause subqueries. +** * ON and USING clauses are shifted into WHERE statements +** * Wildcards "*" and "TABLE.*" in result sets are expanded. +** * Identifiers in expression are matched to tables. +** +** This routine acts recursively on all subqueries within the SELECT. +*/ +void sqlite3SelectPrep( Parse *pParse, /* The parser context */ Select *p, /* The SELECT statement being coded. */ - NameContext *pOuterNC /* The outer name context. May be NULL. */ + NameContext *pOuterNC /* Name context for container */ ){ - ExprList *pEList; /* Result set. */ - int i; /* For-loop variable used in multiple places */ - NameContext sNC; /* Local name-context */ - ExprList *pGroupBy; /* The group by clause */ - - /* If this routine has run before, return immediately. */ - if( p->isResolved ){ - assert( !pOuterNC ); - return SQLITE_OK; - } - p->isResolved = 1; - - /* If there have already been errors, do nothing. */ - if( pParse->nErr>0 ){ - return SQLITE_ERROR; - } - - /* Prepare the select statement. This call will allocate all cursors - ** required to handle the tables and subqueries in the FROM clause. - */ - if( prepSelectStmt(pParse, p) ){ - return SQLITE_ERROR; - } - - /* Resolve the expressions in the LIMIT and OFFSET clauses. These - ** are not allowed to refer to any names, so pass an empty NameContext. - */ - memset(&sNC, 0, sizeof(sNC)); - sNC.pParse = pParse; - if( sqlite3ExprResolveNames(&sNC, p->pLimit) || - sqlite3ExprResolveNames(&sNC, p->pOffset) ){ - return SQLITE_ERROR; - } - - /* Set up the local name-context to pass to ExprResolveNames() to - ** resolve the expression-list. - */ - sNC.allowAgg = 1; - sNC.pSrcList = p->pSrc; - sNC.pNext = pOuterNC; - - /* Resolve names in the result set. */ - pEList = p->pEList; - if( !pEList ) return SQLITE_ERROR; - for(i=0; inExpr; i++){ - Expr *pX = pEList->a[i].pExpr; - if( sqlite3ExprResolveNames(&sNC, pX) ){ - return SQLITE_ERROR; - } - } - - /* If there are no aggregate functions in the result-set, and no GROUP BY - ** expression, do not allow aggregates in any of the other expressions. - */ - assert( !p->isAgg ); - pGroupBy = p->pGroupBy; - if( pGroupBy || sNC.hasAgg ){ - p->isAgg = 1; - }else{ - sNC.allowAgg = 0; - } - - /* If a HAVING clause is present, then there must be a GROUP BY clause. - */ - if( p->pHaving && !pGroupBy ){ - sqlite3ErrorMsg(pParse, "a GROUP BY clause is required before HAVING"); - return SQLITE_ERROR; - } - - /* Add the expression list to the name-context before parsing the - ** other expressions in the SELECT statement. This is so that - ** expressions in the WHERE clause (etc.) can refer to expressions by - ** aliases in the result set. - ** - ** Minor point: If this is the case, then the expression will be - ** re-evaluated for each reference to it. - */ - sNC.pEList = p->pEList; - if( sqlite3ExprResolveNames(&sNC, p->pWhere) || - sqlite3ExprResolveNames(&sNC, p->pHaving) ){ - return SQLITE_ERROR; - } - if( p->pPrior==0 ){ - if( processOrderGroupBy(&sNC, p->pOrderBy, "ORDER") || - processOrderGroupBy(&sNC, pGroupBy, "GROUP") ){ - return SQLITE_ERROR; - } - } - - if( sqlite3MallocFailed() ){ - return SQLITE_NOMEM; - } - - /* Make sure the GROUP BY clause does not contain aggregate functions. - */ - if( pGroupBy ){ - struct ExprList_item *pItem; - - for(i=0, pItem=pGroupBy->a; inExpr; i++, pItem++){ - if( ExprHasProperty(pItem->pExpr, EP_Agg) ){ - sqlite3ErrorMsg(pParse, "aggregate functions are not allowed in " - "the GROUP BY clause"); - return SQLITE_ERROR; - } - } - } - - /* If this is one SELECT of a compound, be sure to resolve names - ** in the other SELECTs. - */ - if( p->pPrior ){ - return sqlite3SelectResolve(pParse, p->pPrior, pOuterNC); - }else{ - return SQLITE_OK; - } + sqlite3 *db; + if( p==0 ) return; + db = pParse->db; + if( p->selFlags & SF_HasTypeInfo ) return; + if( pParse->nErr || db->mallocFailed ) return; + sqlite3SelectExpand(pParse, p); + if( pParse->nErr || db->mallocFailed ) return; + sqlite3ResolveSelectNames(pParse, p, pOuterNC); + if( pParse->nErr || db->mallocFailed ) return; + sqlite3SelectAddTypeInfo(pParse, p); } /* @@ -2733,10 +3358,10 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ return; } for(i=0; inColumn; i++){ - sqlite3VdbeAddOp(v, OP_MemNull, pAggInfo->aCol[i].iMem, 0); + sqlite3VdbeAddOp2(v, OP_Null, 0, pAggInfo->aCol[i].iMem); } for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ - sqlite3VdbeAddOp(v, OP_MemNull, pFunc->iMem, 0); + sqlite3VdbeAddOp2(v, OP_Null, 0, pFunc->iMem); if( pFunc->iDistinct>=0 ){ Expr *pE = pFunc->pExpr; if( pE->pList==0 || pE->pList->nExpr!=1 ){ @@ -2745,8 +3370,8 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ pFunc->iDistinct = -1; }else{ KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->pList); - sqlite3VdbeOp3(v, OP_OpenEphemeral, pFunc->iDistinct, 0, - (char*)pKeyInfo, P3_KEYINFO_HANDOFF); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, + (char*)pKeyInfo, P4_KEYINFO_HANDOFF); } } } @@ -2762,8 +3387,8 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ struct AggInfo_func *pF; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ ExprList *pList = pF->pExpr->pList; - sqlite3VdbeOp3(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0, - (void*)pF->pFunc, P3_FUNCDEF); + sqlite3VdbeAddOp4(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0, 0, + (void*)pF->pFunc, P4_FUNCDEF); } } @@ -2781,67 +3406,95 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ int nArg; int addrNext = 0; + int regAgg; ExprList *pList = pF->pExpr->pList; if( pList ){ nArg = pList->nExpr; - sqlite3ExprCodeExprList(pParse, pList); + regAgg = sqlite3GetTempRange(pParse, nArg); + sqlite3ExprCodeExprList(pParse, pList, regAgg, 0); }else{ nArg = 0; + regAgg = 0; } if( pF->iDistinct>=0 ){ addrNext = sqlite3VdbeMakeLabel(v); assert( nArg==1 ); - codeDistinct(v, pF->iDistinct, addrNext, 1); + codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg); } - if( pF->pFunc->needCollSeq ){ + if( pF->pFunc->flags & SQLITE_FUNC_NEEDCOLL ){ CollSeq *pColl = 0; struct ExprList_item *pItem; int j; - assert( pList!=0 ); /* pList!=0 if pF->pFunc->needCollSeq is true */ + assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ for(j=0, pItem=pList->a; !pColl && jpExpr); } if( !pColl ){ pColl = pParse->db->pDfltColl; } - sqlite3VdbeOp3(v, OP_CollSeq, 0, 0, (char *)pColl, P3_COLLSEQ); + sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeOp3(v, OP_AggStep, pF->iMem, nArg, (void*)pF->pFunc, P3_FUNCDEF); + sqlite3VdbeAddOp4(v, OP_AggStep, 0, regAgg, pF->iMem, + (void*)pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); + sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); } } for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ - sqlite3ExprCode(pParse, pC->pExpr); - sqlite3VdbeAddOp(v, OP_MemStore, pC->iMem, 1); + sqlite3ExprCode(pParse, pC->pExpr, pC->iMem); } pAggInfo->directMode = 0; } - /* -** Generate code for the given SELECT statement. +** Generate code for the SELECT statement given in the p argument. ** ** The results are distributed in various ways depending on the -** value of eDest and iParm. +** contents of the SelectDest structure pointed to by argument pDest +** as follows: ** -** eDest Value Result +** pDest->eDest Result ** ------------ ------------------------------------------- -** SRT_Callback Invoke the callback for each row of the result. +** SRT_Output Generate a row of output (using the OP_ResultRow +** opcode) for each row in the result set. ** -** SRT_Mem Store first result in memory cell iParm +** SRT_Mem Only valid if the result is a single column. +** Store the first column of the first result row +** in register pDest->iParm then abandon the rest +** of the query. This destination implies "LIMIT 1". ** -** SRT_Set Store results as keys of table iParm. +** SRT_Set The result must be a single column. Store each +** row of result as the key in table pDest->iParm. +** Apply the affinity pDest->affinity before storing +** results. Used to implement "IN (SELECT ...)". ** -** SRT_Union Store results as a key in a temporary table iParm +** SRT_Union Store results as a key in a temporary table pDest->iParm. ** -** SRT_Except Remove results from the temporary table iParm. +** SRT_Except Remove results from the temporary table pDest->iParm. ** -** SRT_Table Store results in temporary table iParm +** SRT_Table Store results in temporary table pDest->iParm. +** This is like SRT_EphemTab except that the table +** is assumed to already be open. ** -** The table above is incomplete. Additional eDist value have be added -** since this comment was written. See the selectInnerLoop() function for -** a complete listing of the allowed values of eDest and their meanings. +** SRT_EphemTab Create an temporary table pDest->iParm and store +** the result there. The cursor is left open after +** returning. This is like SRT_Table except that +** this destination uses OP_OpenEphemeral to create +** the table first. +** +** SRT_Coroutine Generate a co-routine that returns a new row of +** results each time it is invoked. The entry point +** of the co-routine is stored in register pDest->iParm. +** +** SRT_Exists Store a 1 in memory cell pDest->iParm if the result +** set is not empty. +** +** SRT_Discard Throw the results away. This is used by SELECT +** statements within triggers whose only purpose is +** the side-effects of functions. ** ** This routine returns the number of errors. If any errors are ** encountered, then an appropriate error message is left in @@ -2849,37 +3502,11 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ ** ** This routine does NOT free the Select structure passed in. The ** calling function needs to do that. -** -** The pParent, parentTab, and *pParentAgg fields are filled in if this -** SELECT is a subquery. This routine may try to combine this SELECT -** with its parent to form a single flat query. In so doing, it might -** change the parent query from a non-aggregate to an aggregate query. -** For that reason, the pParentAgg flag is passed as a pointer, so it -** can be changed. -** -** Example 1: The meaning of the pParent parameter. -** -** SELECT * FROM t1 JOIN (SELECT x, count(*) FROM t2) JOIN t3; -** \ \_______ subquery _______/ / -** \ / -** \____________________ outer query ___________________/ -** -** This routine is called for the outer query first. For that call, -** pParent will be NULL. During the processing of the outer query, this -** routine is called recursively to handle the subquery. For the recursive -** call, pParent will point to the outer query. Because the subquery is -** the second element in a three-way join, the parentTab parameter will -** be 1 (the 2nd value of a 0-indexed array.) */ int sqlite3Select( Parse *pParse, /* The parser context */ Select *p, /* The SELECT statement being coded. */ - int eDest, /* How to dispose of the results */ - int iParm, /* A parameter used by the eDest disposal method */ - Select *pParent, /* Another SELECT for which this is a sub-query */ - int parentTab, /* Index in pParent->pSrc of this query */ - int *pParentAgg, /* True if pParent uses aggregate functions */ - char *aff /* If eDest is SRT_Union, the affinity string */ + SelectDest *pDest /* What to do with the query results */ ){ int i, j; /* Loop counters */ WhereInfo *pWInfo; /* Return from sqlite3WhereBegin() */ @@ -2897,50 +3524,34 @@ int sqlite3Select( int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ + sqlite3 *db; /* The database connection */ - if( p==0 || sqlite3MallocFailed() || pParse->nErr ){ + db = pParse->db; + if( p==0 || db->mallocFailed || pParse->nErr ){ return 1; } if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; memset(&sAggInfo, 0, sizeof(sAggInfo)); -#ifndef SQLITE_OMIT_COMPOUND_SELECT - /* If there is are a sequence of queries, do the earlier ones first. - */ - if( p->pPrior ){ - if( p->pRightmost==0 ){ - Select *pLoop; - int cnt = 0; - for(pLoop=p; pLoop; pLoop=pLoop->pPrior, cnt++){ - pLoop->pRightmost = p; - } - if( SQLITE_MAX_COMPOUND_SELECT>0 && cnt>SQLITE_MAX_COMPOUND_SELECT ){ - sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); - return 1; - } - } - return multiSelect(pParse, p, eDest, iParm, aff); - } -#endif - pOrderBy = p->pOrderBy; - if( IgnorableOrderby(eDest) ){ + if( IgnorableOrderby(pDest) ){ p->pOrderBy = 0; + + /* In these cases the DISTINCT operator makes no difference to the + ** results, so remove it if it were specified. + */ + assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || + pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard); + p->selFlags &= ~SF_Distinct; } - if( sqlite3SelectResolve(pParse, p, 0) ){ + sqlite3SelectPrep(pParse, p, 0); + pTabList = p->pSrc; + pEList = p->pEList; + if( pParse->nErr || db->mallocFailed ){ goto select_end; } p->pOrderBy = pOrderBy; - - /* Make local copies of the parameters for this query. - */ - pTabList = p->pSrc; - pWhere = p->pWhere; - pGroupBy = p->pGroupBy; - pHaving = p->pHaving; - isAgg = p->isAgg; - isDistinct = p->isDistinct; - pEList = p->pEList; + isAgg = (p->selFlags & SF_Aggregate)!=0; if( pEList==0 ) goto select_end; /* @@ -2949,18 +3560,9 @@ int sqlite3Select( */ if( pParse->nErr>0 ) goto select_end; - /* If writing to memory or generating a set - ** only a single column may be output. - */ -#ifndef SQLITE_OMIT_SUBQUERY - if( checkForMultiColumnSelectError(pParse, eDest, pEList->nExpr) ){ - goto select_end; - } -#endif - /* ORDER BY is ignored for some destinations. */ - if( IgnorableOrderby(eDest) ){ + if( IgnorableOrderby(pDest) ){ pOrderBy = 0; } @@ -2972,20 +3574,14 @@ int sqlite3Select( /* Generate code for all sub-queries in the FROM clause */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) - for(i=0; inSrc; i++){ - const char *zSavedAuthContext = 0; - int needRestoreContext; + for(i=0; !p->pPrior && inSrc; i++){ struct SrcList_item *pItem = &pTabList->a[i]; + SelectDest dest; + Select *pSub = pItem->pSelect; + int isAggSub; + + if( pSub==0 || pItem->isPopulated ) continue; - if( pItem->pSelect==0 || pItem->isPopulated ) continue; - if( pItem->zName!=0 ){ - zSavedAuthContext = pParse->zAuthContext; - pParse->zAuthContext = pItem->zName; - needRestoreContext = 1; - }else{ - needRestoreContext = 0; - } -#if SQLITE_MAX_EXPR_DEPTH>0 /* Increment Parse.nHeight by the height of the largest expression ** tree refered to by this, the parent select. The child select ** may contain expression trees of at most @@ -2994,44 +3590,78 @@ int sqlite3Select( ** an exact limit. */ pParse->nHeight += sqlite3SelectExprHeight(p); -#endif - sqlite3Select(pParse, pItem->pSelect, SRT_EphemTab, - pItem->iCursor, p, i, &isAgg, 0); -#if SQLITE_MAX_EXPR_DEPTH>0 - pParse->nHeight -= sqlite3SelectExprHeight(p); -#endif - if( needRestoreContext ){ - pParse->zAuthContext = zSavedAuthContext; + + /* Check to see if the subquery can be absorbed into the parent. */ + isAggSub = (pSub->selFlags & SF_Aggregate)!=0; + if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ + if( isAggSub ){ + isAgg = 1; + p->selFlags |= SF_Aggregate; + } + i = -1; + }else{ + sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); + assert( pItem->isPopulated==0 ); + sqlite3Select(pParse, pSub, &dest); + pItem->isPopulated = 1; } + if( pParse->nErr || db->mallocFailed ){ + goto select_end; + } + pParse->nHeight -= sqlite3SelectExprHeight(p); pTabList = p->pSrc; - pWhere = p->pWhere; - if( !IgnorableOrderby(eDest) ){ + if( !IgnorableOrderby(pDest) ){ pOrderBy = p->pOrderBy; } + } + pEList = p->pEList; +#endif + pWhere = p->pWhere; + pGroupBy = p->pGroupBy; + pHaving = p->pHaving; + isDistinct = (p->selFlags & SF_Distinct)!=0; + +#ifndef SQLITE_OMIT_COMPOUND_SELECT + /* If there is are a sequence of queries, do the earlier ones first. + */ + if( p->pPrior ){ + if( p->pRightmost==0 ){ + Select *pLoop, *pRight = 0; + int cnt = 0; + int mxSelect; + for(pLoop=p; pLoop; pLoop=pLoop->pPrior, cnt++){ + pLoop->pRightmost = p; + pLoop->pNext = pRight; + pRight = pLoop; + } + mxSelect = db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT]; + if( mxSelect && cnt>mxSelect ){ + sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); + return 1; + } + } + return multiSelect(pParse, p, pDest); + } +#endif + + /* If writing to memory or generating a set + ** only a single column may be output. + */ +#ifndef SQLITE_OMIT_SUBQUERY + if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ + goto select_end; + } +#endif + + /* If possible, rewrite the query to use GROUP BY instead of DISTINCT. + ** GROUP BY might use an index, DISTINCT never does. + */ + if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct && !p->pGroupBy ){ + p->pGroupBy = sqlite3ExprListDup(db, p->pEList); pGroupBy = p->pGroupBy; - pHaving = p->pHaving; - isDistinct = p->isDistinct; + p->selFlags &= ~SF_Distinct; + isDistinct = 0; } -#endif - - /* Check for the special case of a min() or max() function by itself - ** in the result set. - */ - if( simpleMinMaxQuery(pParse, p, eDest, iParm) ){ - rc = 0; - goto select_end; - } - - /* Check to see if this is a subquery that can be "flattened" into its parent. - ** If flattening is a possiblity, do so and return immediately. - */ -#ifndef SQLITE_OMIT_VIEW - if( pParent && pParentAgg && - flattenSubquery(pParent, parentTab, *pParentAgg, isAgg) ){ - if( isAgg ) *pParentAgg = 1; - goto select_end; - } -#endif /* If there is an ORDER BY clause, then this sorting ** index might end up being unused if the data can be @@ -3042,21 +3672,20 @@ int sqlite3Select( */ if( pOrderBy ){ KeyInfo *pKeyInfo; - if( pParse->nErr ){ - goto select_end; - } pKeyInfo = keyInfoFromExprList(pParse, pOrderBy); pOrderBy->iECursor = pParse->nTab++; p->addrOpenEphm[2] = addrSortIndex = - sqlite3VdbeOp3(v, OP_OpenEphemeral, pOrderBy->iECursor, pOrderBy->nExpr+2, (char*)pKeyInfo, P3_KEYINFO_HANDOFF); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + pOrderBy->iECursor, pOrderBy->nExpr+2, 0, + (char*)pKeyInfo, P4_KEYINFO_HANDOFF); }else{ addrSortIndex = -1; } /* If the output is destined for a temporary table, open that table. */ - if( eDest==SRT_EphemTab ){ - sqlite3VdbeAddOp(v, OP_OpenEphemeral, iParm, pEList->nExpr); + if( pDest->eDest==SRT_EphemTab ){ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iParm, pEList->nExpr); } /* Set the limiter. @@ -3068,10 +3697,11 @@ int sqlite3Select( */ if( isDistinct ){ KeyInfo *pKeyInfo; + assert( isAgg || pGroupBy ); distinct = pParse->nTab++; pKeyInfo = keyInfoFromExprList(pParse, p->pEList); - sqlite3VdbeOp3(v, OP_OpenEphemeral, distinct, 0, - (char*)pKeyInfo, P3_KEYINFO_HANDOFF); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, + (char*)pKeyInfo, P4_KEYINFO_HANDOFF); }else{ distinct = -1; } @@ -3081,7 +3711,7 @@ int sqlite3Select( /* This case is for non-aggregate queries ** Begin the database scan */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0, 0); if( pWInfo==0 ) goto select_end; /* If sorting index that was created by a prior OP_OpenEphemeral @@ -3095,10 +3725,9 @@ int sqlite3Select( /* Use the standard inner loop */ - if( selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, eDest, - iParm, pWInfo->iContinue, pWInfo->iBreak, aff) ){ - goto select_end; - } + assert(!isDistinct); + selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, -1, pDest, + pWInfo->iContinue, pWInfo->iBreak); /* End the database scan loop. */ @@ -3113,20 +3742,25 @@ int sqlite3Select( ** processed */ int iAbortFlag; /* Mem address which causes query abort if positive */ int groupBySort; /* Rows come from source in GROUP BY order */ + int addrEnd; /* End of processing for this SELECT */ + /* Remove any and all aliases between the result set and the + ** GROUP BY clause. + */ + if( pGroupBy ){ + int k; /* Loop counter */ + struct ExprList_item *pItem; /* For looping over expression in a list */ - /* The following variables hold addresses or labels for parts of the - ** virtual machine program we are putting together */ - int addrOutputRow; /* Start of subroutine that outputs a result row */ - int addrSetAbort; /* Set the abort flag and return */ - int addrInitializeLoop; /* Start of code that initializes the input loop */ - int addrTopOfLoop; /* Top of the input loop */ - int addrGroupByChange; /* Code that runs when any GROUP BY term changes */ - int addrProcessRow; /* Code to process a single input row */ - int addrEnd; /* End of all processing */ - int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */ - int addrReset; /* Subroutine for resetting the accumulator */ + for(k=p->pEList->nExpr, pItem=p->pEList->a; k>0; k--, pItem++){ + pItem->iAlias = 0; + } + for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ + pItem->iAlias = 0; + } + } + + /* Create a label to jump to when we want to abort the query */ addrEnd = sqlite3VdbeMakeLabel(v); /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in @@ -3139,35 +3773,30 @@ int sqlite3Select( sNC.pAggInfo = &sAggInfo; sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; sAggInfo.pGroupBy = pGroupBy; - if( sqlite3ExprAnalyzeAggList(&sNC, pEList) ){ - goto select_end; - } - if( sqlite3ExprAnalyzeAggList(&sNC, pOrderBy) ){ - goto select_end; - } - if( pHaving && sqlite3ExprAnalyzeAggregates(&sNC, pHaving) ){ - goto select_end; + sqlite3ExprAnalyzeAggList(&sNC, pEList); + sqlite3ExprAnalyzeAggList(&sNC, pOrderBy); + if( pHaving ){ + sqlite3ExprAnalyzeAggregates(&sNC, pHaving); } sAggInfo.nAccumulator = sAggInfo.nColumn; for(i=0; ipList) ){ - goto select_end; - } + sqlite3ExprAnalyzeAggList(&sNC, sAggInfo.aFunc[i].pExpr->pList); } - if( sqlite3MallocFailed() ) goto select_end; + if( db->mallocFailed ) goto select_end; /* Processing for aggregates with GROUP BY is very different and - ** much more complex tha aggregates without a GROUP BY. + ** much more complex than aggregates without a GROUP BY. */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ - - /* Create labels that we will be needing - */ - - addrInitializeLoop = sqlite3VdbeMakeLabel(v); - addrGroupByChange = sqlite3VdbeMakeLabel(v); - addrProcessRow = sqlite3VdbeMakeLabel(v); + int j1; /* A-vs-B comparision jump */ + int addrOutputRow; /* Start of subroutine that outputs a result row */ + int regOutputRow; /* Return address register for output subroutine */ + int addrSetAbort; /* Set the abort flag and return */ + int addrTopOfLoop; /* Top of the input loop */ + int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */ + int addrReset; /* Subroutine for resetting the accumulator */ + int regReset; /* Return address register for reset subroutine */ /* If there is a GROUP BY clause we might need a sorting index to ** implement it. Allocate that sorting index now. If it turns out @@ -3176,67 +3805,34 @@ int sqlite3Select( */ sAggInfo.sortingIdx = pParse->nTab++; pKeyInfo = keyInfoFromExprList(pParse, pGroupBy); - addrSortingIdx = - sqlite3VdbeOp3(v, OP_OpenEphemeral, sAggInfo.sortingIdx, - sAggInfo.nSortingColumn, - (char*)pKeyInfo, P3_KEYINFO_HANDOFF); + addrSortingIdx = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + sAggInfo.sortingIdx, sAggInfo.nSortingColumn, + 0, (char*)pKeyInfo, P4_KEYINFO_HANDOFF); /* Initialize memory locations used by GROUP BY aggregate processing */ - iUseFlag = pParse->nMem++; - iAbortFlag = pParse->nMem++; - iAMem = pParse->nMem; + iUseFlag = ++pParse->nMem; + iAbortFlag = ++pParse->nMem; + regOutputRow = ++pParse->nMem; + addrOutputRow = sqlite3VdbeMakeLabel(v); + regReset = ++pParse->nMem; + addrReset = sqlite3VdbeMakeLabel(v); + iAMem = pParse->nMem + 1; pParse->nMem += pGroupBy->nExpr; - iBMem = pParse->nMem; + iBMem = pParse->nMem + 1; pParse->nMem += pGroupBy->nExpr; - sqlite3VdbeAddOp(v, OP_MemInt, 0, iAbortFlag); - VdbeComment((v, "# clear abort flag")); - sqlite3VdbeAddOp(v, OP_MemInt, 0, iUseFlag); - VdbeComment((v, "# indicate accumulator empty")); - sqlite3VdbeAddOp(v, OP_Goto, 0, addrInitializeLoop); - - /* Generate a subroutine that outputs a single row of the result - ** set. This subroutine first looks at the iUseFlag. If iUseFlag - ** is less than or equal to zero, the subroutine is a no-op. If - ** the processing calls for the query to abort, this subroutine - ** increments the iAbortFlag memory location before returning in - ** order to signal the caller to abort. - */ - addrSetAbort = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp(v, OP_MemInt, 1, iAbortFlag); - VdbeComment((v, "# set abort flag")); - sqlite3VdbeAddOp(v, OP_Return, 0, 0); - addrOutputRow = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp(v, OP_IfMemPos, iUseFlag, addrOutputRow+2); - VdbeComment((v, "# Groupby result generator entry point")); - sqlite3VdbeAddOp(v, OP_Return, 0, 0); - finalizeAggFunctions(pParse, &sAggInfo); - if( pHaving ){ - sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, 1); - } - rc = selectInnerLoop(pParse, p, p->pEList, 0, 0, pOrderBy, - distinct, eDest, iParm, - addrOutputRow+1, addrSetAbort, aff); - if( rc ){ - goto select_end; - } - sqlite3VdbeAddOp(v, OP_Return, 0, 0); - VdbeComment((v, "# end groupby result generator")); - - /* Generate a subroutine that will reset the group-by accumulator - */ - addrReset = sqlite3VdbeCurrentAddr(v); - resetAccumulator(pParse, &sAggInfo); - sqlite3VdbeAddOp(v, OP_Return, 0, 0); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); + VdbeComment((v, "clear abort flag")); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); + VdbeComment((v, "indicate accumulator empty")); /* Begin a loop that will extract all source rows in GROUP BY order. ** This might involve two separate loops with an OP_Sort in between, or ** it might be a single loop that uses an index to extract information ** in the right order to begin with. */ - sqlite3VdbeResolveLabel(v, addrInitializeLoop); - sqlite3VdbeAddOp(v, OP_Gosub, 0, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy); + sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0); if( pWInfo==0 ) goto select_end; if( pGroupBy==0 ){ /* The optimizer is able to deliver rows in group by order so @@ -3251,21 +3847,47 @@ int sqlite3Select( ** then loop over the sorting index in order to get the output ** in sorted order */ + int regBase; + int regRecord; + int nCol; + int nGroupBy; + groupBySort = 1; - sqlite3ExprCodeExprList(pParse, pGroupBy); - sqlite3VdbeAddOp(v, OP_Sequence, sAggInfo.sortingIdx, 0); - j = pGroupBy->nExpr+1; + nGroupBy = pGroupBy->nExpr; + nCol = nGroupBy + 1; + j = nGroupBy+1; + for(i=0; i=j ){ + nCol++; + j++; + } + } + regBase = sqlite3GetTempRange(pParse, nCol); + sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); + sqlite3VdbeAddOp2(v, OP_Sequence, sAggInfo.sortingIdx,regBase+nGroupBy); + j = nGroupBy+1; for(i=0; iiSorterColumnpTab, pCol->iColumn, pCol->iTable); - j++; + if( pCol->iSorterColumn>=j ){ + int r1 = j + regBase; + int r2; + + r2 = sqlite3ExprCodeGetColumn(pParse, + pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); + if( r1!=r2 ){ + sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); + } + j++; + } } - sqlite3VdbeAddOp(v, OP_MakeRecord, j, 0); - sqlite3VdbeAddOp(v, OP_IdxInsert, sAggInfo.sortingIdx, 0); + regRecord = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); + sqlite3VdbeAddOp2(v, OP_IdxInsert, sAggInfo.sortingIdx, regRecord); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ReleaseTempRange(pParse, regBase, nCol); sqlite3WhereEnd(pWInfo); - sqlite3VdbeAddOp(v, OP_Sort, sAggInfo.sortingIdx, addrEnd); - VdbeComment((v, "# GROUP BY sort")); + sqlite3VdbeAddOp2(v, OP_Sort, sAggInfo.sortingIdx, addrEnd); + VdbeComment((v, "GROUP BY sort")); sAggInfo.useSortingIdx = 1; } @@ -3277,28 +3899,19 @@ int sqlite3Select( addrTopOfLoop = sqlite3VdbeCurrentAddr(v); for(j=0; jnExpr; j++){ if( groupBySort ){ - sqlite3VdbeAddOp(v, OP_Column, sAggInfo.sortingIdx, j); + sqlite3VdbeAddOp3(v, OP_Column, sAggInfo.sortingIdx, j, iBMem+j); }else{ sAggInfo.directMode = 1; - sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr); + sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); } - sqlite3VdbeAddOp(v, OP_MemStore, iBMem+j, jnExpr-1); - } - for(j=pGroupBy->nExpr-1; j>=0; j--){ - if( jnExpr-1 ){ - sqlite3VdbeAddOp(v, OP_MemLoad, iBMem+j, 0); - } - sqlite3VdbeAddOp(v, OP_MemLoad, iAMem+j, 0); - if( j==0 ){ - sqlite3VdbeAddOp(v, OP_Eq, 0x200, addrProcessRow); - }else{ - sqlite3VdbeAddOp(v, OP_Ne, 0x200, addrGroupByChange); - } - sqlite3VdbeChangeP3(v, -1, (void*)pKeyInfo->aColl[j], P3_COLLSEQ); } + sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, + (char*)pKeyInfo, P4_KEYINFO); + j1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); /* Generate code that runs whenever the GROUP BY changes. - ** Change in the GROUP BY are detected by the previous code + ** Changes in the GROUP BY are detected by the previous code ** block. If there were no changes, this block is skipped. ** ** This code copies current group by terms in b0,b1,b2,... @@ -3306,29 +3919,26 @@ int sqlite3Select( ** and resets the aggregate accumulator registers in preparation ** for the next GROUP BY batch. */ - sqlite3VdbeResolveLabel(v, addrGroupByChange); - for(j=0; jnExpr; j++){ - sqlite3VdbeAddOp(v, OP_MemMove, iAMem+j, iBMem+j); - } - sqlite3VdbeAddOp(v, OP_Gosub, 0, addrOutputRow); - VdbeComment((v, "# output one row")); - sqlite3VdbeAddOp(v, OP_IfMemPos, iAbortFlag, addrEnd); - VdbeComment((v, "# check abort flag")); - sqlite3VdbeAddOp(v, OP_Gosub, 0, addrReset); - VdbeComment((v, "# reset accumulator")); + sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); + sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); + VdbeComment((v, "output one row")); + sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); + VdbeComment((v, "check abort flag")); + sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); + VdbeComment((v, "reset accumulator")); /* Update the aggregate accumulators based on the content of ** the current row */ - sqlite3VdbeResolveLabel(v, addrProcessRow); + sqlite3VdbeJumpHere(v, j1); updateAccumulator(pParse, &sAggInfo); - sqlite3VdbeAddOp(v, OP_MemInt, 1, iUseFlag); - VdbeComment((v, "# indicate data in accumulator")); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); + VdbeComment((v, "indicate data in accumulator")); /* End of the loop */ if( groupBySort ){ - sqlite3VdbeAddOp(v, OP_Next, sAggInfo.sortingIdx, addrTopOfLoop); + sqlite3VdbeAddOp2(v, OP_Next, sAggInfo.sortingIdx, addrTopOfLoop); }else{ sqlite3WhereEnd(pWInfo); sqlite3VdbeChangeToNoop(v, addrSortingIdx, 1); @@ -3336,27 +3946,110 @@ int sqlite3Select( /* Output the final row of result */ - sqlite3VdbeAddOp(v, OP_Gosub, 0, addrOutputRow); - VdbeComment((v, "# output final row")); - + sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); + VdbeComment((v, "output final row")); + + /* Jump over the subroutines + */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEnd); + + /* Generate a subroutine that outputs a single row of the result + ** set. This subroutine first looks at the iUseFlag. If iUseFlag + ** is less than or equal to zero, the subroutine is a no-op. If + ** the processing calls for the query to abort, this subroutine + ** increments the iAbortFlag memory location before returning in + ** order to signal the caller to abort. + */ + addrSetAbort = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iAbortFlag); + VdbeComment((v, "set abort flag")); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + sqlite3VdbeResolveLabel(v, addrOutputRow); + addrOutputRow = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); + VdbeComment((v, "Groupby result generator entry point")); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + finalizeAggFunctions(pParse, &sAggInfo); + if( pHaving ){ + sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); + } + selectInnerLoop(pParse, p, p->pEList, 0, 0, pOrderBy, + distinct, pDest, + addrOutputRow+1, addrSetAbort); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + VdbeComment((v, "end groupby result generator")); + + /* Generate a subroutine that will reset the group-by accumulator + */ + sqlite3VdbeResolveLabel(v, addrReset); + resetAccumulator(pParse, &sAggInfo); + sqlite3VdbeAddOp1(v, OP_Return, regReset); + } /* endif pGroupBy */ else { + ExprList *pMinMax = 0; + ExprList *pDel = 0; + u8 flag; + + /* Check if the query is of one of the following forms: + ** + ** SELECT min(x) FROM ... + ** SELECT max(x) FROM ... + ** + ** If it is, then ask the code in where.c to attempt to sort results + ** as if there was an "ORDER ON x" or "ORDER ON x DESC" clause. + ** If where.c is able to produce results sorted in this order, then + ** add vdbe code to break out of the processing loop after the + ** first iteration (since the first iteration of the loop is + ** guaranteed to operate on the row with the minimum or maximum + ** value of x, the only row required). + ** + ** A special flag must be passed to sqlite3WhereBegin() to slightly + ** modify behaviour as follows: + ** + ** + If the query is a "SELECT min(x)", then the loop coded by + ** where.c should not iterate over any values with a NULL value + ** for x. + ** + ** + The optimizer code in where.c (the thing that decides which + ** index or indices to use) should place a different priority on + ** satisfying the 'ORDER BY' clause than it does in other cases. + ** Refer to code and comments in where.c for details. + */ + flag = minMaxQuery(p); + if( flag ){ + pDel = pMinMax = sqlite3ExprListDup(db, p->pEList->a[0].pExpr->pList); + if( pMinMax && !db->mallocFailed ){ + pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; + pMinMax->a[0].pExpr->op = TK_COLUMN; + } + } + /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0); - if( pWInfo==0 ) goto select_end; + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag, 0); + if( pWInfo==0 ){ + sqlite3ExprListDelete(db, pDel); + goto select_end; + } updateAccumulator(pParse, &sAggInfo); + if( !pMinMax && flag ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak); + VdbeComment((v, "%s() by index",(flag==WHERE_ORDERBY_MIN?"min":"max"))); + } sqlite3WhereEnd(pWInfo); finalizeAggFunctions(pParse, &sAggInfo); pOrderBy = 0; if( pHaving ){ - sqlite3ExprIfFalse(pParse, pHaving, addrEnd, 1); + sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); } selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, -1, - eDest, iParm, addrEnd, addrEnd, aff); + pDest, addrEnd, addrEnd); + + sqlite3ExprListDelete(db, pDel); } sqlite3VdbeResolveLabel(v, addrEnd); @@ -3366,22 +4059,9 @@ int sqlite3Select( ** and send them to the callback one by one. */ if( pOrderBy ){ - generateSortTail(pParse, p, v, pEList->nExpr, eDest, iParm); + generateSortTail(pParse, p, v, pEList->nExpr, pDest); } -#ifndef SQLITE_OMIT_SUBQUERY - /* If this was a subquery, we have now converted the subquery into a - ** temporary table. So set the SrcList_item.isPopulated flag to prevent - ** this subquery from being evaluated again and to force the use of - ** the temporary table. - */ - if( pParent ){ - assert( pParent->pSrc->nSrc>parentTab ); - assert( pParent->pSrc->a[parentTab].pSelect==p ); - pParent->pSrc->a[parentTab].isPopulated = 1; - } -#endif - /* Jump here to skip this query */ sqlite3VdbeResolveLabel(v, iEnd); @@ -3396,15 +4076,14 @@ int sqlite3Select( */ select_end: - /* Identify column names if we will be using them in a callback. This - ** step is skipped if the output is going to some other destination. + /* Identify column names if results of the SELECT are to be output. */ - if( rc==SQLITE_OK && eDest==SRT_Callback ){ + if( rc==SQLITE_OK && pDest->eDest==SRT_Output ){ generateColumnNames(pParse, pTabList, pEList); } - sqliteFree(sAggInfo.aCol); - sqliteFree(sAggInfo.aFunc); + sqlite3DbFree(db, sAggInfo.aCol); + sqlite3DbFree(db, sAggInfo.aFunc); return rc; } diff --git a/shell.c b/shell.c new file mode 100644 index 0000000..10a165b --- /dev/null +++ b/shell.c @@ -0,0 +1,2148 @@ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code to implement the "sqlite" command line +** utility for accessing SQLite databases. +** +** $Id: shell.c,v 1.198 2009/01/14 23:38:03 drh Exp $ +*/ +#if defined(_WIN32) || defined(WIN32) +/* This needs to come before any includes for MSVC compiler */ +#define _CRT_SECURE_NO_WARNINGS +#endif + +#include +#include +#include +#include +#include "sqlite3.h" +#include +#include + +#if !defined(_WIN32) && !defined(WIN32) && !defined(__OS2__) +# include +# if !defined(__RTP__) && !defined(_WRS_KERNEL) +# include +# endif +# include +# include +#endif + +#ifdef __OS2__ +# include +#endif + +#if defined(HAVE_READLINE) && HAVE_READLINE==1 +# include +# include +#else +# define readline(p) local_getline(p,stdin) +# define add_history(X) +# define read_history(X) +# define write_history(X) +# define stifle_history(X) +#endif + +#if defined(_WIN32) || defined(WIN32) +# include +#define isatty(h) _isatty(h) +#define access(f,m) _access((f),(m)) +#else +/* Make sure isatty() has a prototype. +*/ +extern int isatty(); +#endif + +#if defined(_WIN32_WCE) +/* Windows CE (arm-wince-mingw32ce-gcc) does not provide isatty() + * thus we always assume that we have a console. That can be + * overridden with the -batch command line option. + */ +#define isatty(x) 1 +#endif + +#if !defined(_WIN32) && !defined(WIN32) && !defined(__OS2__) && !defined(__RTP__) && !defined(_WRS_KERNEL) +#include +#include + +/* Saved resource information for the beginning of an operation */ +static struct rusage sBegin; + +/* True if the timer is enabled */ +static int enableTimer = 0; + +/* +** Begin timing an operation +*/ +static void beginTimer(void){ + if( enableTimer ){ + getrusage(RUSAGE_SELF, &sBegin); + } +} + +/* Return the difference of two time_structs in seconds */ +static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ + return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + + (double)(pEnd->tv_sec - pStart->tv_sec); +} + +/* +** Print the timing results. +*/ +static void endTimer(void){ + if( enableTimer ){ + struct rusage sEnd; + getrusage(RUSAGE_SELF, &sEnd); + printf("CPU Time: user %f sys %f\n", + timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), + timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); + } +} +#define BEGIN_TIMER beginTimer() +#define END_TIMER endTimer() +#define HAS_TIMER 1 +#else +#define BEGIN_TIMER +#define END_TIMER +#define HAS_TIMER 0 +#endif + +/* +** Used to prevent warnings about unused parameters +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** If the following flag is set, then command execution stops +** at an error if we are not interactive. +*/ +static int bail_on_error = 0; + +/* +** Threat stdin as an interactive input if the following variable +** is true. Otherwise, assume stdin is connected to a file or pipe. +*/ +static int stdin_is_interactive = 1; + +/* +** The following is the open SQLite database. We make a pointer +** to this database a static variable so that it can be accessed +** by the SIGINT handler to interrupt database processing. +*/ +static sqlite3 *db = 0; + +/* +** True if an interrupt (Control-C) has been received. +*/ +static volatile int seenInterrupt = 0; + +/* +** This is the name of our program. It is set in main(), used +** in a number of other places, mostly for error messages. +*/ +static char *Argv0; + +/* +** Prompt strings. Initialized in main. Settable with +** .prompt main continue +*/ +static char mainPrompt[20]; /* First line prompt. default: "sqlite> "*/ +static char continuePrompt[20]; /* Continuation prompt. default: " ...> " */ + +/* +** Write I/O traces to the following stream. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static FILE *iotrace = 0; +#endif + +/* +** This routine works like printf in that its first argument is a +** format string and subsequent arguments are values to be substituted +** in place of % fields. The result of formatting this string +** is written to iotrace. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static void iotracePrintf(const char *zFormat, ...){ + va_list ap; + char *z; + if( iotrace==0 ) return; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + fprintf(iotrace, "%s", z); + sqlite3_free(z); +} +#endif + + +/* +** Determines if a string is a number of not. +*/ +static int isNumber(const char *z, int *realnum){ + if( *z=='-' || *z=='+' ) z++; + if( !isdigit(*z) ){ + return 0; + } + z++; + if( realnum ) *realnum = 0; + while( isdigit(*z) ){ z++; } + if( *z=='.' ){ + z++; + if( !isdigit(*z) ) return 0; + while( isdigit(*z) ){ z++; } + if( realnum ) *realnum = 1; + } + if( *z=='e' || *z=='E' ){ + z++; + if( *z=='+' || *z=='-' ) z++; + if( !isdigit(*z) ) return 0; + while( isdigit(*z) ){ z++; } + if( realnum ) *realnum = 1; + } + return *z==0; +} + +/* +** A global char* and an SQL function to access its current value +** from within an SQL statement. This program used to use the +** sqlite_exec_printf() API to substitue a string into an SQL statement. +** The correct way to do this with sqlite3 is to use the bind API, but +** since the shell is built around the callback paradigm it would be a lot +** of work. Instead just use this hack, which is quite harmless. +*/ +static const char *zShellStatic = 0; +static void shellstaticFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( 0==argc ); + assert( zShellStatic ); + UNUSED_PARAMETER(argv); + sqlite3_result_text(context, zShellStatic, -1, SQLITE_STATIC); +} + + +/* +** This routine reads a line of text from FILE in, stores +** the text in memory obtained from malloc() and returns a pointer +** to the text. NULL is returned at end of file, or if malloc() +** fails. +** +** The interface is like "readline" but no command-line editing +** is done. +*/ +static char *local_getline(char *zPrompt, FILE *in){ + char *zLine; + int nLine; + int n; + int eol; + + if( zPrompt && *zPrompt ){ + printf("%s",zPrompt); + fflush(stdout); + } + nLine = 100; + zLine = malloc( nLine ); + if( zLine==0 ) return 0; + n = 0; + eol = 0; + while( !eol ){ + if( n+100>nLine ){ + nLine = nLine*2 + 100; + zLine = realloc(zLine, nLine); + if( zLine==0 ) return 0; + } + if( fgets(&zLine[n], nLine - n, in)==0 ){ + if( n==0 ){ + free(zLine); + return 0; + } + zLine[n] = 0; + eol = 1; + break; + } + while( zLine[n] ){ n++; } + if( n>0 && zLine[n-1]=='\n' ){ + n--; + zLine[n] = 0; + eol = 1; + } + } + zLine = realloc( zLine, n+1 ); + return zLine; +} + +/* +** Retrieve a single line of input text. +** +** zPrior is a string of prior text retrieved. If not the empty +** string, then issue a continuation prompt. +*/ +static char *one_input_line(const char *zPrior, FILE *in){ + char *zPrompt; + char *zResult; + if( in!=0 ){ + return local_getline(0, in); + } + if( zPrior && zPrior[0] ){ + zPrompt = continuePrompt; + }else{ + zPrompt = mainPrompt; + } + zResult = readline(zPrompt); +#if defined(HAVE_READLINE) && HAVE_READLINE==1 + if( zResult && *zResult ) add_history(zResult); +#endif + return zResult; +} + +struct previous_mode_data { + int valid; /* Is there legit data in here? */ + int mode; + int showHeader; + int colWidth[100]; +}; + +/* +** An pointer to an instance of this structure is passed from +** the main program to the callback. This is used to communicate +** state and mode information. +*/ +struct callback_data { + sqlite3 *db; /* The database */ + int echoOn; /* True to echo input commands */ + int cnt; /* Number of records displayed so far */ + FILE *out; /* Write results here */ + int mode; /* An output mode setting */ + int writableSchema; /* True if PRAGMA writable_schema=ON */ + int showHeader; /* True to show column names in List or Column mode */ + char *zDestTable; /* Name of destination table when MODE_Insert */ + char separator[20]; /* Separator character for MODE_List */ + int colWidth[100]; /* Requested width of each column when in column mode*/ + int actualWidth[100]; /* Actual width of each column */ + char nullvalue[20]; /* The text to print when a NULL comes back from + ** the database */ + struct previous_mode_data explainPrev; + /* Holds the mode information just before + ** .explain ON */ + char outfile[FILENAME_MAX]; /* Filename for *out */ + const char *zDbFilename; /* name of the database file */ +}; + +/* +** These are the allowed modes. +*/ +#define MODE_Line 0 /* One column per line. Blank line between records */ +#define MODE_Column 1 /* One record per line in neat columns */ +#define MODE_List 2 /* One record per line with a separator */ +#define MODE_Semi 3 /* Same as MODE_List but append ";" to each line */ +#define MODE_Html 4 /* Generate an XHTML table */ +#define MODE_Insert 5 /* Generate SQL "insert" statements */ +#define MODE_Tcl 6 /* Generate ANSI-C or TCL quoted elements */ +#define MODE_Csv 7 /* Quote strings, numbers are plain */ +#define MODE_Explain 8 /* Like MODE_Column, but do not truncate data */ + +static const char *modeDescr[] = { + "line", + "column", + "list", + "semi", + "html", + "insert", + "tcl", + "csv", + "explain", +}; + +/* +** Number of elements in an array +*/ +#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) + +/* +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. +*/ +static int strlen30(const char *z){ + const char *z2 = z; + while( *z2 ){ z2++; } + return 0x3fffffff & (int)(z2 - z); +} + +/* +** Output the given string as a quoted string using SQL quoting conventions. +*/ +static void output_quoted_string(FILE *out, const char *z){ + int i; + int nSingle = 0; + for(i=0; z[i]; i++){ + if( z[i]=='\'' ) nSingle++; + } + if( nSingle==0 ){ + fprintf(out,"'%s'",z); + }else{ + fprintf(out,"'"); + while( *z ){ + for(i=0; z[i] && z[i]!='\''; i++){} + if( i==0 ){ + fprintf(out,"''"); + z++; + }else if( z[i]=='\'' ){ + fprintf(out,"%.*s''",i,z); + z += i+1; + }else{ + fprintf(out,"%s",z); + break; + } + } + fprintf(out,"'"); + } +} + +/* +** Output the given string as a quoted according to C or TCL quoting rules. +*/ +static void output_c_string(FILE *out, const char *z){ + unsigned int c; + fputc('"', out); + while( (c = *(z++))!=0 ){ + if( c=='\\' ){ + fputc(c, out); + fputc(c, out); + }else if( c=='\t' ){ + fputc('\\', out); + fputc('t', out); + }else if( c=='\n' ){ + fputc('\\', out); + fputc('n', out); + }else if( c=='\r' ){ + fputc('\\', out); + fputc('r', out); + }else if( !isprint(c) ){ + fprintf(out, "\\%03o", c&0xff); + }else{ + fputc(c, out); + } + } + fputc('"', out); +} + +/* +** Output the given string with characters that are special to +** HTML escaped. +*/ +static void output_html_string(FILE *out, const char *z){ + int i; + while( *z ){ + for(i=0; z[i] && z[i]!='<' && z[i]!='&'; i++){} + if( i>0 ){ + fprintf(out,"%.*s",i,z); + } + if( z[i]=='<' ){ + fprintf(out,"<"); + }else if( z[i]=='&' ){ + fprintf(out,"&"); + }else{ + break; + } + z += i + 1; + } +} + +/* +** If a field contains any character identified by a 1 in the following +** array, then the string must be quoted for CSV. +*/ +static const char needCsvQuote[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +}; + +/* +** Output a single term of CSV. Actually, p->separator is used for +** the separator, which may or may not be a comma. p->nullvalue is +** the null value. Strings are quoted using ANSI-C rules. Numbers +** appear outside of quotes. +*/ +static void output_csv(struct callback_data *p, const char *z, int bSep){ + FILE *out = p->out; + if( z==0 ){ + fprintf(out,"%s",p->nullvalue); + }else{ + int i; + int nSep = strlen30(p->separator); + for(i=0; z[i]; i++){ + if( needCsvQuote[((unsigned char*)z)[i]] + || (z[i]==p->separator[0] && + (nSep==1 || memcmp(z, p->separator, nSep)==0)) ){ + i = 0; + break; + } + } + if( i==0 ){ + putc('"', out); + for(i=0; z[i]; i++){ + if( z[i]=='"' ) putc('"', out); + putc(z[i], out); + } + putc('"', out); + }else{ + fprintf(out, "%s", z); + } + } + if( bSep ){ + fprintf(p->out, "%s", p->separator); + } +} + +#ifdef SIGINT +/* +** This routine runs when the user presses Ctrl-C +*/ +static void interrupt_handler(int NotUsed){ + UNUSED_PARAMETER(NotUsed); + seenInterrupt = 1; + if( db ) sqlite3_interrupt(db); +} +#endif + +/* +** This is the callback routine that the SQLite library +** invokes for each row of a query result. +*/ +static int callback(void *pArg, int nArg, char **azArg, char **azCol){ + int i; + struct callback_data *p = (struct callback_data*)pArg; + switch( p->mode ){ + case MODE_Line: { + int w = 5; + if( azArg==0 ) break; + for(i=0; iw ) w = len; + } + if( p->cnt++>0 ) fprintf(p->out,"\n"); + for(i=0; iout,"%*s = %s\n", w, azCol[i], + azArg[i] ? azArg[i] : p->nullvalue); + } + break; + } + case MODE_Explain: + case MODE_Column: { + if( p->cnt++==0 ){ + for(i=0; icolWidth) ){ + w = p->colWidth[i]; + }else{ + w = 0; + } + if( w<=0 ){ + w = strlen30(azCol[i] ? azCol[i] : ""); + if( w<10 ) w = 10; + n = strlen30(azArg && azArg[i] ? azArg[i] : p->nullvalue); + if( wactualWidth) ){ + p->actualWidth[i] = w; + } + if( p->showHeader ){ + fprintf(p->out,"%-*.*s%s",w,w,azCol[i], i==nArg-1 ? "\n": " "); + } + } + if( p->showHeader ){ + for(i=0; iactualWidth) ){ + w = p->actualWidth[i]; + }else{ + w = 10; + } + fprintf(p->out,"%-*.*s%s",w,w,"-----------------------------------" + "----------------------------------------------------------", + i==nArg-1 ? "\n": " "); + } + } + } + if( azArg==0 ) break; + for(i=0; iactualWidth) ){ + w = p->actualWidth[i]; + }else{ + w = 10; + } + if( p->mode==MODE_Explain && azArg[i] && + strlen30(azArg[i])>w ){ + w = strlen30(azArg[i]); + } + fprintf(p->out,"%-*.*s%s",w,w, + azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); + } + break; + } + case MODE_Semi: + case MODE_List: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; iout,"%s%s",azCol[i], i==nArg-1 ? "\n" : p->separator); + } + } + if( azArg==0 ) break; + for(i=0; inullvalue; + fprintf(p->out, "%s", z); + if( iout, "%s", p->separator); + }else if( p->mode==MODE_Semi ){ + fprintf(p->out, ";\n"); + }else{ + fprintf(p->out, "\n"); + } + } + break; + } + case MODE_Html: { + if( p->cnt++==0 && p->showHeader ){ + fprintf(p->out,"
"); + for(i=0; iout,"",azCol[i]); + } + fprintf(p->out,"\n"); + } + if( azArg==0 ) break; + fprintf(p->out,""); + for(i=0; iout,"\n"); + } + fprintf(p->out,"\n"); + break; + } + case MODE_Tcl: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; iout,azCol[i] ? azCol[i] : ""); + fprintf(p->out, "%s", p->separator); + } + fprintf(p->out,"\n"); + } + if( azArg==0 ) break; + for(i=0; iout, azArg[i] ? azArg[i] : p->nullvalue); + fprintf(p->out, "%s", p->separator); + } + fprintf(p->out,"\n"); + break; + } + case MODE_Csv: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; iout,"\n"); + } + if( azArg==0 ) break; + for(i=0; iout,"\n"); + break; + } + case MODE_Insert: { + if( azArg==0 ) break; + fprintf(p->out,"INSERT INTO %s VALUES(",p->zDestTable); + for(i=0; i0 ? ",": ""; + if( azArg[i]==0 ){ + fprintf(p->out,"%sNULL",zSep); + }else if( isNumber(azArg[i], 0) ){ + fprintf(p->out,"%s%s",zSep, azArg[i]); + }else{ + if( zSep[0] ) fprintf(p->out,"%s",zSep); + output_quoted_string(p->out, azArg[i]); + } + } + fprintf(p->out,");\n"); + break; + } + } + return 0; +} + +/* +** Set the destination table field of the callback_data structure to +** the name of the table given. Escape any quote characters in the +** table name. +*/ +static void set_table_name(struct callback_data *p, const char *zName){ + int i, n; + int needQuote; + char *z; + + if( p->zDestTable ){ + free(p->zDestTable); + p->zDestTable = 0; + } + if( zName==0 ) return; + needQuote = !isalpha((unsigned char)*zName) && *zName!='_'; + for(i=n=0; zName[i]; i++, n++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ){ + needQuote = 1; + if( zName[i]=='\'' ) n++; + } + } + if( needQuote ) n += 2; + z = p->zDestTable = malloc( n+1 ); + if( z==0 ){ + fprintf(stderr,"Out of memory!\n"); + exit(1); + } + n = 0; + if( needQuote ) z[n++] = '\''; + for(i=0; zName[i]; i++){ + z[n++] = zName[i]; + if( zName[i]=='\'' ) z[n++] = '\''; + } + if( needQuote ) z[n++] = '\''; + z[n] = 0; +} + +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static char *appendText(char *zIn, char const *zAppend, char quote){ + int len; + int i; + int nAppend = strlen30(zAppend); + int nIn = (zIn?strlen30(zIn):0); + + len = nAppend+nIn+1; + if( quote ){ + len += 2; + for(i=0; iout, "DELETE FROM sqlite_sequence;\n"); + }else if( strcmp(zTable, "sqlite_stat1")==0 ){ + fprintf(p->out, "ANALYZE sqlite_master;\n"); + }else if( strncmp(zTable, "sqlite_", 7)==0 ){ + return 0; + }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ + char *zIns; + if( !p->writableSchema ){ + fprintf(p->out, "PRAGMA writable_schema=ON;\n"); + p->writableSchema = 1; + } + zIns = sqlite3_mprintf( + "INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)" + "VALUES('table','%q','%q',0,'%q');", + zTable, zTable, zSql); + fprintf(p->out, "%s\n", zIns); + sqlite3_free(zIns); + return 0; + }else{ + fprintf(p->out, "%s;\n", zSql); + } + + if( strcmp(zType, "table")==0 ){ + sqlite3_stmt *pTableInfo = 0; + char *zSelect = 0; + char *zTableInfo = 0; + char *zTmp = 0; + + zTableInfo = appendText(zTableInfo, "PRAGMA table_info(", 0); + zTableInfo = appendText(zTableInfo, zTable, '"'); + zTableInfo = appendText(zTableInfo, ");", 0); + + rc = sqlite3_prepare(p->db, zTableInfo, -1, &pTableInfo, 0); + if( zTableInfo ) free(zTableInfo); + if( rc!=SQLITE_OK || !pTableInfo ){ + return 1; + } + + zSelect = appendText(zSelect, "SELECT 'INSERT INTO ' || ", 0); + zTmp = appendText(zTmp, zTable, '"'); + if( zTmp ){ + zSelect = appendText(zSelect, zTmp, '\''); + } + zSelect = appendText(zSelect, " || ' VALUES(' || ", 0); + rc = sqlite3_step(pTableInfo); + while( rc==SQLITE_ROW ){ + const char *zText = (const char *)sqlite3_column_text(pTableInfo, 1); + zSelect = appendText(zSelect, "quote(", 0); + zSelect = appendText(zSelect, zText, '"'); + rc = sqlite3_step(pTableInfo); + if( rc==SQLITE_ROW ){ + zSelect = appendText(zSelect, ") || ',' || ", 0); + }else{ + zSelect = appendText(zSelect, ") ", 0); + } + } + rc = sqlite3_finalize(pTableInfo); + if( rc!=SQLITE_OK ){ + if( zSelect ) free(zSelect); + return 1; + } + zSelect = appendText(zSelect, "|| ')' FROM ", 0); + zSelect = appendText(zSelect, zTable, '"'); + + rc = run_table_dump_query(p->out, p->db, zSelect); + if( rc==SQLITE_CORRUPT ){ + zSelect = appendText(zSelect, " ORDER BY rowid DESC", 0); + rc = run_table_dump_query(p->out, p->db, zSelect); + } + if( zSelect ) free(zSelect); + } + return 0; +} + +/* +** Run zQuery. Use dump_callback() as the callback routine so that +** the contents of the query are output as SQL statements. +** +** If we get a SQLITE_CORRUPT error, rerun the query after appending +** "ORDER BY rowid DESC" to the end. +*/ +static int run_schema_dump_query( + struct callback_data *p, + const char *zQuery, + char **pzErrMsg +){ + int rc; + rc = sqlite3_exec(p->db, zQuery, dump_callback, p, pzErrMsg); + if( rc==SQLITE_CORRUPT ){ + char *zQ2; + int len = strlen30(zQuery); + if( pzErrMsg ) sqlite3_free(*pzErrMsg); + zQ2 = malloc( len+100 ); + if( zQ2==0 ) return rc; + sqlite3_snprintf(sizeof(zQ2), zQ2, "%s ORDER BY rowid DESC", zQuery); + rc = sqlite3_exec(p->db, zQ2, dump_callback, p, pzErrMsg); + free(zQ2); + } + return rc; +} + +/* +** Text of a help message +*/ +static char zHelp[] = + ".bail ON|OFF Stop after hitting an error. Default OFF\n" + ".databases List names and files of attached databases\n" + ".dump ?TABLE? ... Dump the database in an SQL text format\n" + ".echo ON|OFF Turn command echo on or off\n" + ".exit Exit this program\n" + ".explain ON|OFF Turn output mode suitable for EXPLAIN on or off.\n" + ".header(s) ON|OFF Turn display of headers on or off\n" + ".help Show this message\n" + ".import FILE TABLE Import data from FILE into TABLE\n" + ".indices TABLE Show names of all indices on TABLE\n" +#ifdef SQLITE_ENABLE_IOTRACE + ".iotrace FILE Enable I/O diagnostic logging to FILE\n" +#endif +#ifndef SQLITE_OMIT_LOAD_EXTENSION + ".load FILE ?ENTRY? Load an extension library\n" +#endif + ".mode MODE ?TABLE? Set output mode where MODE is one of:\n" + " csv Comma-separated values\n" + " column Left-aligned columns. (See .width)\n" + " html HTML
%s
"); + output_html_string(p->out, azArg[i] ? azArg[i] : p->nullvalue); + fprintf(p->out,"
code\n" + " insert SQL insert statements for TABLE\n" + " line One value per line\n" + " list Values delimited by .separator string\n" + " tabs Tab-separated values\n" + " tcl TCL list elements\n" + ".nullvalue STRING Print STRING in place of NULL values\n" + ".output FILENAME Send output to FILENAME\n" + ".output stdout Send output to the screen\n" + ".prompt MAIN CONTINUE Replace the standard prompts\n" + ".quit Exit this program\n" + ".read FILENAME Execute SQL in FILENAME\n" + ".schema ?TABLE? Show the CREATE statements\n" + ".separator STRING Change separator used by output mode and .import\n" + ".show Show the current values for various settings\n" + ".tables ?PATTERN? List names of tables matching a LIKE pattern\n" + ".timeout MS Try opening locked tables for MS milliseconds\n" +#if HAS_TIMER + ".timer ON|OFF Turn the CPU timer measurement on or off\n" +#endif + ".width NUM NUM ... Set column widths for \"column\" mode\n" +; + +/* Forward reference */ +static int process_input(struct callback_data *p, FILE *in); + +/* +** Make sure the database is open. If it is not, then open it. If +** the database fails to open, print an error message and exit. +*/ +static void open_db(struct callback_data *p){ + if( p->db==0 ){ + sqlite3_open(p->zDbFilename, &p->db); + db = p->db; + if( db && sqlite3_errcode(db)==SQLITE_OK ){ + sqlite3_create_function(db, "shellstatic", 0, SQLITE_UTF8, 0, + shellstaticFunc, 0, 0); + } + if( db==0 || SQLITE_OK!=sqlite3_errcode(db) ){ + fprintf(stderr,"Unable to open database \"%s\": %s\n", + p->zDbFilename, sqlite3_errmsg(db)); + exit(1); + } +#ifndef SQLITE_OMIT_LOAD_EXTENSION + sqlite3_enable_load_extension(p->db, 1); +#endif + } +} + +/* +** Do C-language style dequoting. +** +** \t -> tab +** \n -> newline +** \r -> carriage return +** \NNN -> ascii character NNN in octal +** \\ -> backslash +*/ +static void resolve_backslashes(char *z){ + int i, j; + char c; + for(i=j=0; (c = z[i])!=0; i++, j++){ + if( c=='\\' ){ + c = z[++i]; + if( c=='n' ){ + c = '\n'; + }else if( c=='t' ){ + c = '\t'; + }else if( c=='r' ){ + c = '\r'; + }else if( c>='0' && c<='7' ){ + c -= '0'; + if( z[i+1]>='0' && z[i+1]<='7' ){ + i++; + c = (c<<3) + z[i] - '0'; + if( z[i+1]>='0' && z[i+1]<='7' ){ + i++; + c = (c<<3) + z[i] - '0'; + } + } + } + } + z[j] = c; + } + z[j] = 0; +} + +/* +** Interpret zArg as a boolean value. Return either 0 or 1. +*/ +static int booleanValue(char *zArg){ + int val = atoi(zArg); + int j; + for(j=0; zArg[j]; j++){ + zArg[j] = (char)tolower(zArg[j]); + } + if( strcmp(zArg,"on")==0 ){ + val = 1; + }else if( strcmp(zArg,"yes")==0 ){ + val = 1; + } + return val; +} + +/* +** If an input line begins with "." then invoke this routine to +** process that line. +** +** Return 1 on error, 2 to exit, and 0 otherwise. +*/ +static int do_meta_command(char *zLine, struct callback_data *p){ + int i = 1; + int nArg = 0; + int n, c; + int rc = 0; + char *azArg[50]; + + /* Parse the input line into tokens. + */ + while( zLine[i] && nArg1 && strncmp(azArg[0], "bail", n)==0 && nArg>1 ){ + bail_on_error = booleanValue(azArg[1]); + }else + + if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 ){ + struct callback_data data; + char *zErrMsg = 0; + open_db(p); + memcpy(&data, p, sizeof(data)); + data.showHeader = 1; + data.mode = MODE_Column; + data.colWidth[0] = 3; + data.colWidth[1] = 15; + data.colWidth[2] = 58; + data.cnt = 0; + sqlite3_exec(p->db, "PRAGMA database_list; ", callback, &data, &zErrMsg); + if( zErrMsg ){ + fprintf(stderr,"Error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + } + }else + + if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ + char *zErrMsg = 0; + open_db(p); + fprintf(p->out, "BEGIN TRANSACTION;\n"); + p->writableSchema = 0; + sqlite3_exec(p->db, "PRAGMA writable_schema=ON", 0, 0, 0); + if( nArg==1 ){ + run_schema_dump_query(p, + "SELECT name, type, sql FROM sqlite_master " + "WHERE sql NOT NULL AND type=='table'", 0 + ); + run_table_dump_query(p->out, p->db, + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL AND type IN ('index','trigger','view')" + ); + }else{ + int i; + for(i=1; iout, p->db, + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL" + " AND type IN ('index','trigger','view')" + " AND tbl_name LIKE shellstatic()" + ); + zShellStatic = 0; + } + } + if( p->writableSchema ){ + fprintf(p->out, "PRAGMA writable_schema=OFF;\n"); + p->writableSchema = 0; + } + sqlite3_exec(p->db, "PRAGMA writable_schema=OFF", 0, 0, 0); + if( zErrMsg ){ + fprintf(stderr,"Error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + }else{ + fprintf(p->out, "COMMIT;\n"); + } + }else + + if( c=='e' && strncmp(azArg[0], "echo", n)==0 && nArg>1 ){ + p->echoOn = booleanValue(azArg[1]); + }else + + if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ + rc = 2; + }else + + if( c=='e' && strncmp(azArg[0], "explain", n)==0 ){ + int val = nArg>=2 ? booleanValue(azArg[1]) : 1; + if(val == 1) { + if(!p->explainPrev.valid) { + p->explainPrev.valid = 1; + p->explainPrev.mode = p->mode; + p->explainPrev.showHeader = p->showHeader; + memcpy(p->explainPrev.colWidth,p->colWidth,sizeof(p->colWidth)); + } + /* We could put this code under the !p->explainValid + ** condition so that it does not execute if we are already in + ** explain mode. However, always executing it allows us an easy + ** was to reset to explain mode in case the user previously + ** did an .explain followed by a .width, .mode or .header + ** command. + */ + p->mode = MODE_Explain; + p->showHeader = 1; + memset(p->colWidth,0,ArraySize(p->colWidth)); + p->colWidth[0] = 4; /* addr */ + p->colWidth[1] = 13; /* opcode */ + p->colWidth[2] = 4; /* P1 */ + p->colWidth[3] = 4; /* P2 */ + p->colWidth[4] = 4; /* P3 */ + p->colWidth[5] = 13; /* P4 */ + p->colWidth[6] = 2; /* P5 */ + p->colWidth[7] = 13; /* Comment */ + }else if (p->explainPrev.valid) { + p->explainPrev.valid = 0; + p->mode = p->explainPrev.mode; + p->showHeader = p->explainPrev.showHeader; + memcpy(p->colWidth,p->explainPrev.colWidth,sizeof(p->colWidth)); + } + }else + + if( c=='h' && (strncmp(azArg[0], "header", n)==0 || + strncmp(azArg[0], "headers", n)==0 )&& nArg>1 ){ + p->showHeader = booleanValue(azArg[1]); + }else + + if( c=='h' && strncmp(azArg[0], "help", n)==0 ){ + fprintf(stderr,"%s",zHelp); + }else + + if( c=='i' && strncmp(azArg[0], "import", n)==0 && nArg>=3 ){ + char *zTable = azArg[2]; /* Insert data into this table */ + char *zFile = azArg[1]; /* The file from which to extract data */ + sqlite3_stmt *pStmt; /* A statement */ + int rc; /* Result code */ + int nCol; /* Number of columns in the table */ + int nByte; /* Number of bytes in an SQL string */ + int i, j; /* Loop counters */ + int nSep; /* Number of bytes in p->separator[] */ + char *zSql; /* An SQL statement */ + char *zLine; /* A single line of input from the file */ + char **azCol; /* zLine[] broken up into columns */ + char *zCommit; /* How to commit changes */ + FILE *in; /* The input file */ + int lineno = 0; /* Line number of input file */ + + open_db(p); + nSep = strlen30(p->separator); + if( nSep==0 ){ + fprintf(stderr, "non-null separator required for import\n"); + return 0; + } + zSql = sqlite3_mprintf("SELECT * FROM '%q'", zTable); + if( zSql==0 ) return 0; + nByte = strlen30(zSql); + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + fprintf(stderr,"Error: %s\n", sqlite3_errmsg(db)); + nCol = 0; + rc = 1; + }else{ + nCol = sqlite3_column_count(pStmt); + } + sqlite3_finalize(pStmt); + if( nCol==0 ) return 0; + zSql = malloc( nByte + 20 + nCol*2 ); + if( zSql==0 ) return 0; + sqlite3_snprintf(nByte+20, zSql, "INSERT INTO '%q' VALUES(?", zTable); + j = strlen30(zSql); + for(i=1; idb, zSql, -1, &pStmt, 0); + free(zSql); + if( rc ){ + fprintf(stderr, "Error: %s\n", sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + return 1; + } + in = fopen(zFile, "rb"); + if( in==0 ){ + fprintf(stderr, "cannot open file: %s\n", zFile); + sqlite3_finalize(pStmt); + return 0; + } + azCol = malloc( sizeof(azCol[0])*(nCol+1) ); + if( azCol==0 ){ + fclose(in); + return 0; + } + sqlite3_exec(p->db, "BEGIN", 0, 0, 0); + zCommit = "COMMIT"; + while( (zLine = local_getline(0, in))!=0 ){ + char *z; + i = 0; + lineno++; + azCol[0] = zLine; + for(i=0, z=zLine; *z && *z!='\n' && *z!='\r'; z++){ + if( *z==p->separator[0] && strncmp(z, p->separator, nSep)==0 ){ + *z = 0; + i++; + if( idb, zCommit, 0, 0, 0); + }else + + if( c=='i' && strncmp(azArg[0], "indices", n)==0 && nArg>1 ){ + struct callback_data data; + char *zErrMsg = 0; + open_db(p); + memcpy(&data, p, sizeof(data)); + data.showHeader = 0; + data.mode = MODE_List; + zShellStatic = azArg[1]; + sqlite3_exec(p->db, + "SELECT name FROM sqlite_master " + "WHERE type='index' AND tbl_name LIKE shellstatic() " + "UNION ALL " + "SELECT name FROM sqlite_temp_master " + "WHERE type='index' AND tbl_name LIKE shellstatic() " + "ORDER BY 1", + callback, &data, &zErrMsg + ); + zShellStatic = 0; + if( zErrMsg ){ + fprintf(stderr,"Error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + } + }else + +#ifdef SQLITE_ENABLE_IOTRACE + if( c=='i' && strncmp(azArg[0], "iotrace", n)==0 ){ + extern void (*sqlite3IoTrace)(const char*, ...); + if( iotrace && iotrace!=stdout ) fclose(iotrace); + iotrace = 0; + if( nArg<2 ){ + sqlite3IoTrace = 0; + }else if( strcmp(azArg[1], "-")==0 ){ + sqlite3IoTrace = iotracePrintf; + iotrace = stdout; + }else{ + iotrace = fopen(azArg[1], "w"); + if( iotrace==0 ){ + fprintf(stderr, "cannot open \"%s\"\n", azArg[1]); + sqlite3IoTrace = 0; + }else{ + sqlite3IoTrace = iotracePrintf; + } + } + }else +#endif + +#ifndef SQLITE_OMIT_LOAD_EXTENSION + if( c=='l' && strncmp(azArg[0], "load", n)==0 && nArg>=2 ){ + const char *zFile, *zProc; + char *zErrMsg = 0; + int rc; + zFile = azArg[1]; + zProc = nArg>=3 ? azArg[2] : 0; + open_db(p); + rc = sqlite3_load_extension(p->db, zFile, zProc, &zErrMsg); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "%s\n", zErrMsg); + sqlite3_free(zErrMsg); + rc = 1; + } + }else +#endif + + if( c=='m' && strncmp(azArg[0], "mode", n)==0 && nArg>=2 ){ + int n2 = strlen30(azArg[1]); + if( strncmp(azArg[1],"line",n2)==0 + || + strncmp(azArg[1],"lines",n2)==0 ){ + p->mode = MODE_Line; + }else if( strncmp(azArg[1],"column",n2)==0 + || + strncmp(azArg[1],"columns",n2)==0 ){ + p->mode = MODE_Column; + }else if( strncmp(azArg[1],"list",n2)==0 ){ + p->mode = MODE_List; + }else if( strncmp(azArg[1],"html",n2)==0 ){ + p->mode = MODE_Html; + }else if( strncmp(azArg[1],"tcl",n2)==0 ){ + p->mode = MODE_Tcl; + }else if( strncmp(azArg[1],"csv",n2)==0 ){ + p->mode = MODE_Csv; + sqlite3_snprintf(sizeof(p->separator), p->separator, ","); + }else if( strncmp(azArg[1],"tabs",n2)==0 ){ + p->mode = MODE_List; + sqlite3_snprintf(sizeof(p->separator), p->separator, "\t"); + }else if( strncmp(azArg[1],"insert",n2)==0 ){ + p->mode = MODE_Insert; + if( nArg>=3 ){ + set_table_name(p, azArg[2]); + }else{ + set_table_name(p, "table"); + } + }else { + fprintf(stderr,"mode should be one of: " + "column csv html insert line list tabs tcl\n"); + } + }else + + if( c=='n' && strncmp(azArg[0], "nullvalue", n)==0 && nArg==2 ) { + sqlite3_snprintf(sizeof(p->nullvalue), p->nullvalue, + "%.*s", (int)ArraySize(p->nullvalue)-1, azArg[1]); + }else + + if( c=='o' && strncmp(azArg[0], "output", n)==0 && nArg==2 ){ + if( p->out!=stdout ){ + fclose(p->out); + } + if( strcmp(azArg[1],"stdout")==0 ){ + p->out = stdout; + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "stdout"); + }else{ + p->out = fopen(azArg[1], "wb"); + if( p->out==0 ){ + fprintf(stderr,"can't write to \"%s\"\n", azArg[1]); + p->out = stdout; + } else { + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", azArg[1]); + } + } + }else + + if( c=='p' && strncmp(azArg[0], "prompt", n)==0 && (nArg==2 || nArg==3)){ + if( nArg >= 2) { + strncpy(mainPrompt,azArg[1],(int)ArraySize(mainPrompt)-1); + } + if( nArg >= 3) { + strncpy(continuePrompt,azArg[2],(int)ArraySize(continuePrompt)-1); + } + }else + + if( c=='q' && strncmp(azArg[0], "quit", n)==0 ){ + rc = 2; + }else + + if( c=='r' && strncmp(azArg[0], "read", n)==0 && nArg==2 ){ + FILE *alt = fopen(azArg[1], "rb"); + if( alt==0 ){ + fprintf(stderr,"can't open \"%s\"\n", azArg[1]); + }else{ + process_input(p, alt); + fclose(alt); + } + }else + + if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){ + struct callback_data data; + char *zErrMsg = 0; + open_db(p); + memcpy(&data, p, sizeof(data)); + data.showHeader = 0; + data.mode = MODE_Semi; + if( nArg>1 ){ + int i; + for(i=0; azArg[1][i]; i++) azArg[1][i] = (char)tolower(azArg[1][i]); + if( strcmp(azArg[1],"sqlite_master")==0 ){ + char *new_argv[2], *new_colv[2]; + new_argv[0] = "CREATE TABLE sqlite_master (\n" + " type text,\n" + " name text,\n" + " tbl_name text,\n" + " rootpage integer,\n" + " sql text\n" + ")"; + new_argv[1] = 0; + new_colv[0] = "sql"; + new_colv[1] = 0; + callback(&data, 1, new_argv, new_colv); + }else if( strcmp(azArg[1],"sqlite_temp_master")==0 ){ + char *new_argv[2], *new_colv[2]; + new_argv[0] = "CREATE TEMP TABLE sqlite_temp_master (\n" + " type text,\n" + " name text,\n" + " tbl_name text,\n" + " rootpage integer,\n" + " sql text\n" + ")"; + new_argv[1] = 0; + new_colv[0] = "sql"; + new_colv[1] = 0; + callback(&data, 1, new_argv, new_colv); + }else{ + zShellStatic = azArg[1]; + sqlite3_exec(p->db, + "SELECT sql FROM " + " (SELECT sql sql, type type, tbl_name tbl_name, name name" + " FROM sqlite_master UNION ALL" + " SELECT sql, type, tbl_name, name FROM sqlite_temp_master) " + "WHERE tbl_name LIKE shellstatic() AND type!='meta' AND sql NOTNULL " + "ORDER BY substr(type,2,1), name", + callback, &data, &zErrMsg); + zShellStatic = 0; + } + }else{ + sqlite3_exec(p->db, + "SELECT sql FROM " + " (SELECT sql sql, type type, tbl_name tbl_name, name name" + " FROM sqlite_master UNION ALL" + " SELECT sql, type, tbl_name, name FROM sqlite_temp_master) " + "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%'" + "ORDER BY substr(type,2,1), name", + callback, &data, &zErrMsg + ); + } + if( zErrMsg ){ + fprintf(stderr,"Error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + } + }else + + if( c=='s' && strncmp(azArg[0], "separator", n)==0 && nArg==2 ){ + sqlite3_snprintf(sizeof(p->separator), p->separator, + "%.*s", (int)sizeof(p->separator)-1, azArg[1]); + }else + + if( c=='s' && strncmp(azArg[0], "show", n)==0){ + int i; + fprintf(p->out,"%9.9s: %s\n","echo", p->echoOn ? "on" : "off"); + fprintf(p->out,"%9.9s: %s\n","explain", p->explainPrev.valid ? "on" :"off"); + fprintf(p->out,"%9.9s: %s\n","headers", p->showHeader ? "on" : "off"); + fprintf(p->out,"%9.9s: %s\n","mode", modeDescr[p->mode]); + fprintf(p->out,"%9.9s: ", "nullvalue"); + output_c_string(p->out, p->nullvalue); + fprintf(p->out, "\n"); + fprintf(p->out,"%9.9s: %s\n","output", + strlen30(p->outfile) ? p->outfile : "stdout"); + fprintf(p->out,"%9.9s: ", "separator"); + output_c_string(p->out, p->separator); + fprintf(p->out, "\n"); + fprintf(p->out,"%9.9s: ","width"); + for (i=0;i<(int)ArraySize(p->colWidth) && p->colWidth[i] != 0;i++) { + fprintf(p->out,"%d ",p->colWidth[i]); + } + fprintf(p->out,"\n"); + }else + + if( c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0 ){ + char **azResult; + int nRow, rc; + char *zErrMsg; + open_db(p); + if( nArg==1 ){ + rc = sqlite3_get_table(p->db, + "SELECT name FROM sqlite_master " + "WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%'" + "UNION ALL " + "SELECT name FROM sqlite_temp_master " + "WHERE type IN ('table','view') " + "ORDER BY 1", + &azResult, &nRow, 0, &zErrMsg + ); + }else{ + zShellStatic = azArg[1]; + rc = sqlite3_get_table(p->db, + "SELECT name FROM sqlite_master " + "WHERE type IN ('table','view') AND name LIKE '%'||shellstatic()||'%' " + "UNION ALL " + "SELECT name FROM sqlite_temp_master " + "WHERE type IN ('table','view') AND name LIKE '%'||shellstatic()||'%' " + "ORDER BY 1", + &azResult, &nRow, 0, &zErrMsg + ); + zShellStatic = 0; + } + if( zErrMsg ){ + fprintf(stderr,"Error: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + } + if( rc==SQLITE_OK ){ + int len, maxlen = 0; + int i, j; + int nPrintCol, nPrintRow; + for(i=1; i<=nRow; i++){ + if( azResult[i]==0 ) continue; + len = strlen30(azResult[i]); + if( len>maxlen ) maxlen = len; + } + nPrintCol = 80/(maxlen+2); + if( nPrintCol<1 ) nPrintCol = 1; + nPrintRow = (nRow + nPrintCol - 1)/nPrintCol; + for(i=0; i4 && strncmp(azArg[0], "timeout", n)==0 && nArg>=2 ){ + open_db(p); + sqlite3_busy_timeout(p->db, atoi(azArg[1])); + }else + +#if HAS_TIMER + if( c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 && nArg>1 ){ + enableTimer = booleanValue(azArg[1]); + }else +#endif + + if( c=='w' && strncmp(azArg[0], "width", n)==0 ){ + int j; + assert( nArg<=ArraySize(azArg) ); + for(j=1; jcolWidth); j++){ + p->colWidth[j-1] = atoi(azArg[j]); + } + }else + + + { + fprintf(stderr, "unknown command or invalid arguments: " + " \"%s\". Enter \".help\" for help\n", azArg[0]); + } + + return rc; +} + +/* +** Return TRUE if a semicolon occurs anywhere in the first N characters +** of string z[]. +*/ +static int _contains_semicolon(const char *z, int N){ + int i; + for(i=0; iout); + free(zLine); + zLine = one_input_line(zSql, in); + if( zLine==0 ){ + break; /* We have reached EOF */ + } + if( seenInterrupt ){ + if( in!=0 ) break; + seenInterrupt = 0; + } + lineno++; + if( p->echoOn ) printf("%s\n", zLine); + if( (zSql==0 || zSql[0]==0) && _all_whitespace(zLine) ) continue; + if( zLine && zLine[0]=='.' && nSql==0 ){ + rc = do_meta_command(zLine, p); + if( rc==2 ){ + break; + }else if( rc ){ + errCnt++; + } + continue; + } + if( _is_command_terminator(zLine) && _is_complete(zSql, nSql) ){ + memcpy(zLine,";",2); + } + nSqlPrior = nSql; + if( zSql==0 ){ + int i; + for(i=0; zLine[i] && isspace((unsigned char)zLine[i]); i++){} + if( zLine[i]!=0 ){ + nSql = strlen30(zLine); + zSql = malloc( nSql+3 ); + if( zSql==0 ){ + fprintf(stderr, "out of memory\n"); + exit(1); + } + memcpy(zSql, zLine, nSql+1); + startline = lineno; + } + }else{ + int len = strlen30(zLine); + zSql = realloc( zSql, nSql + len + 4 ); + if( zSql==0 ){ + fprintf(stderr,"%s: out of memory!\n", Argv0); + exit(1); + } + zSql[nSql++] = '\n'; + memcpy(&zSql[nSql], zLine, len+1); + nSql += len; + } + if( zSql && _contains_semicolon(&zSql[nSqlPrior], nSql-nSqlPrior) + && sqlite3_complete(zSql) ){ + p->cnt = 0; + open_db(p); + BEGIN_TIMER; + rc = sqlite3_exec(p->db, zSql, callback, p, &zErrMsg); + END_TIMER; + if( rc || zErrMsg ){ + char zPrefix[100]; + if( in!=0 || !stdin_is_interactive ){ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "SQL error near line %d:", startline); + }else{ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, "SQL error:"); + } + if( zErrMsg!=0 ){ + printf("%s %s\n", zPrefix, zErrMsg); + sqlite3_free(zErrMsg); + zErrMsg = 0; + }else{ + printf("%s %s\n", zPrefix, sqlite3_errmsg(p->db)); + } + errCnt++; + } + free(zSql); + zSql = 0; + nSql = 0; + } + } + if( zSql ){ + if( !_all_whitespace(zSql) ) fprintf(stderr, "Incomplete SQL: %s\n", zSql); + free(zSql); + } + free(zLine); + return errCnt; +} + +/* +** Return a pathname which is the user's home directory. A +** 0 return indicates an error of some kind. Space to hold the +** resulting string is obtained from malloc(). The calling +** function should free the result. +*/ +static char *find_home_dir(void){ + char *home_dir = NULL; + +#if !defined(_WIN32) && !defined(WIN32) && !defined(__OS2__) && !defined(_WIN32_WCE) && !defined(__RTP__) && !defined(_WRS_KERNEL) + struct passwd *pwent; + uid_t uid = getuid(); + if( (pwent=getpwuid(uid)) != NULL) { + home_dir = pwent->pw_dir; + } +#endif + +#if defined(_WIN32_WCE) + /* Windows CE (arm-wince-mingw32ce-gcc) does not provide getenv() + */ + home_dir = strdup("/"); +#else + +#if defined(_WIN32) || defined(WIN32) || defined(__OS2__) + if (!home_dir) { + home_dir = getenv("USERPROFILE"); + } +#endif + + if (!home_dir) { + home_dir = getenv("HOME"); + } + +#if defined(_WIN32) || defined(WIN32) || defined(__OS2__) + if (!home_dir) { + char *zDrive, *zPath; + int n; + zDrive = getenv("HOMEDRIVE"); + zPath = getenv("HOMEPATH"); + if( zDrive && zPath ){ + n = strlen30(zDrive) + strlen30(zPath) + 1; + home_dir = malloc( n ); + if( home_dir==0 ) return 0; + sqlite3_snprintf(n, home_dir, "%s%s", zDrive, zPath); + return home_dir; + } + home_dir = "c:\\"; + } +#endif + +#endif /* !_WIN32_WCE */ + + if( home_dir ){ + int n = strlen30(home_dir) + 1; + char *z = malloc( n ); + if( z ) memcpy(z, home_dir, n); + home_dir = z; + } + + return home_dir; +} + +/* +** Read input from the file given by sqliterc_override. Or if that +** parameter is NULL, take input from ~/.sqliterc +*/ +static void process_sqliterc( + struct callback_data *p, /* Configuration data */ + const char *sqliterc_override /* Name of config file. NULL to use default */ +){ + char *home_dir = NULL; + const char *sqliterc = sqliterc_override; + char *zBuf = 0; + FILE *in = NULL; + int nBuf; + + if (sqliterc == NULL) { + home_dir = find_home_dir(); + if( home_dir==0 ){ +#if !defined(__RTP__) && !defined(_WRS_KERNEL) + fprintf(stderr,"%s: cannot locate your home directory!\n", Argv0); +#endif + return; + } + nBuf = strlen30(home_dir) + 16; + zBuf = malloc( nBuf ); + if( zBuf==0 ){ + fprintf(stderr,"%s: out of memory!\n", Argv0); + exit(1); + } + sqlite3_snprintf(nBuf, zBuf,"%s/.sqliterc",home_dir); + free(home_dir); + sqliterc = (const char*)zBuf; + } + in = fopen(sqliterc,"rb"); + if( in ){ + if( stdin_is_interactive ){ + printf("-- Loading resources from %s\n",sqliterc); + } + process_input(p,in); + fclose(in); + } + free(zBuf); + return; +} + +/* +** Show available command line options +*/ +static const char zOptions[] = + " -init filename read/process named file\n" + " -echo print commands before execution\n" + " -[no]header turn headers on or off\n" + " -bail stop after hitting an error\n" + " -interactive force interactive I/O\n" + " -batch force batch I/O\n" + " -column set output mode to 'column'\n" + " -csv set output mode to 'csv'\n" + " -html set output mode to HTML\n" + " -line set output mode to 'line'\n" + " -list set output mode to 'list'\n" + " -separator 'x' set output field separator (|)\n" + " -nullvalue 'text' set text string for NULL values\n" + " -version show SQLite version\n" +; +static void usage(int showDetail){ + fprintf(stderr, + "Usage: %s [OPTIONS] FILENAME [SQL]\n" + "FILENAME is the name of an SQLite database. A new database is created\n" + "if the file does not previously exist.\n", Argv0); + if( showDetail ){ + fprintf(stderr, "OPTIONS include:\n%s", zOptions); + }else{ + fprintf(stderr, "Use the -help option for additional information\n"); + } + exit(1); +} + +/* +** Initialize the state information in data +*/ +static void main_init(struct callback_data *data) { + memset(data, 0, sizeof(*data)); + data->mode = MODE_List; + memcpy(data->separator,"|", 2); + data->showHeader = 0; + sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); + sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); +} + +int main(int argc, char **argv){ + char *zErrMsg = 0; + struct callback_data data; + const char *zInitFile = 0; + char *zFirstCmd = 0; + int i; + int rc = 0; + + Argv0 = argv[0]; + main_init(&data); + stdin_is_interactive = isatty(0); + + /* Make sure we have a valid signal handler early, before anything + ** else is done. + */ +#ifdef SIGINT + signal(SIGINT, interrupt_handler); +#endif + + /* Do an initial pass through the command-line argument to locate + ** the name of the database file, the name of the initialization file, + ** and the first command to execute. + */ + for(i=1; i ** -** The version of the SQLite library is contained in the sqlite3.h -** header file in a #define named SQLITE_VERSION. The SQLITE_VERSION -** macro resolves to a string constant. +** The SQLITE_VERSION and SQLITE_VERSION_NUMBER #defines in +** the sqlite3.h file specify the version of SQLite with which +** that header file is associated. ** -** The format of the version string is "X.Y.Z", where -** X is the major version number, Y is the minor version number and Z -** is the release number. The X.Y.Z might be followed by "alpha" or "beta". -** For example "3.1.1beta". -** -** The X value is always 3 in SQLite. The X value only changes when -** backwards compatibility is broken and we intend to never break -** backwards compatibility. The Y value only changes when +** The "version" of SQLite is a string of the form "X.Y.Z". +** The phrase "alpha" or "beta" might be appended after the Z. +** The X value is major version number always 3 in SQLite3. +** The X value only changes when backwards compatibility is +** broken and we intend to never break backwards compatibility. +** The Y value is the minor version number and only changes when ** there are major feature enhancements that are forwards compatible -** but not backwards compatible. The Z value is incremented with -** each release but resets back to 0 when Y is incremented. -** -** The SQLITE_VERSION_NUMBER is an integer with the value -** (X*1000000 + Y*1000 + Z). For example, for version "3.1.1beta", -** SQLITE_VERSION_NUMBER is set to 3001001. To detect if they are using -** version 3.1.1 or greater at compile time, programs may use the test -** (SQLITE_VERSION_NUMBER>=3001001). +** but not backwards compatible. +** The Z value is the release number and is incremented with +** each release but resets back to 0 whenever Y is incremented. ** ** See also: [sqlite3_libversion()] and [sqlite3_libversion_number()]. +** +** INVARIANTS: +** +** {H10011} The SQLITE_VERSION #define in the sqlite3.h header file shall +** evaluate to a string literal that is the SQLite version +** with which the header file is associated. +** +** {H10014} The SQLITE_VERSION_NUMBER #define shall resolve to an integer +** with the value (X*1000000 + Y*1000 + Z) where X, Y, and Z +** are the major version, minor version, and release number. */ -#define SQLITE_VERSION "3.4.2" -#define SQLITE_VERSION_NUMBER 3004002 +#define SQLITE_VERSION "3.6.10" +#define SQLITE_VERSION_NUMBER 3006010 /* -** CAPI3REF: Run-Time Library Version Numbers +** CAPI3REF: Run-Time Library Version Numbers {H10020} +** KEYWORDS: sqlite3_version ** -** These routines return values equivalent to the header constants -** [SQLITE_VERSION] and [SQLITE_VERSION_NUMBER]. The values returned -** by this routines should only be different from the header values -** if you compile your program using an sqlite3.h header from a -** different version of SQLite that the version of the library you -** link against. +** These features provide the same information as the [SQLITE_VERSION] +** and [SQLITE_VERSION_NUMBER] #defines in the header, but are associated +** with the library instead of the header file. Cautious programmers might +** include a check in their application to verify that +** sqlite3_libversion_number() always returns the value +** [SQLITE_VERSION_NUMBER]. ** -** The sqlite3_version[] string constant contains the text of the -** [SQLITE_VERSION] string. The sqlite3_libversion() function returns -** a poiner to the sqlite3_version[] string constant. The function -** is provided for DLL users who can only access functions and not +** The sqlite3_libversion() function returns the same information as is +** in the sqlite3_version[] string constant. The function is provided +** for use in DLLs since DLL users usually do not have direct access to string ** constants within the DLL. +** +** INVARIANTS: +** +** {H10021} The [sqlite3_libversion_number()] interface shall return +** an integer equal to [SQLITE_VERSION_NUMBER]. +** +** {H10022} The [sqlite3_version] string constant shall contain +** the text of the [SQLITE_VERSION] string. +** +** {H10023} The [sqlite3_libversion()] function shall return +** a pointer to the [sqlite3_version] string constant. */ SQLITE_EXTERN const char sqlite3_version[]; const char *sqlite3_libversion(void); int sqlite3_libversion_number(void); /* -** CAPI3REF: Database Connection Handle +** CAPI3REF: Test To See If The Library Is Threadsafe {H10100} ** -** Each open SQLite database is represented by pointer to an instance of the -** opaque structure named "sqlite3". It is useful to think of an sqlite3 -** pointer as an object. The [sqlite3_open] interface is its constructor -** and [sqlite3_close] is its destructor. There are many other interfaces -** (such as [sqlite3_prepare_v2], [sqlite3_create_function], and -** [sqlite3_busy_timeout] to name but three) that are methods on this -** object. +** SQLite can be compiled with or without mutexes. When +** the [SQLITE_THREADSAFE] C preprocessor macro 1 or 2, mutexes +** are enabled and SQLite is threadsafe. When the +** [SQLITE_THREADSAFE] macro is 0, +** the mutexes are omitted. Without the mutexes, it is not safe +** to use SQLite concurrently from more than one thread. +** +** Enabling mutexes incurs a measurable performance penalty. +** So if speed is of utmost importance, it makes sense to disable +** the mutexes. But for maximum safety, mutexes should be enabled. +** The default behavior is for mutexes to be enabled. +** +** This interface can be used by a program to make sure that the +** version of SQLite that it is linking against was compiled with +** the desired setting of the [SQLITE_THREADSAFE] macro. +** +** This interface only reports on the compile-time mutex setting +** of the [SQLITE_THREADSAFE] flag. If SQLite is compiled with +** SQLITE_THREADSAFE=1 then mutexes are enabled by default but +** can be fully or partially disabled using a call to [sqlite3_config()] +** with the verbs [SQLITE_CONFIG_SINGLETHREAD], [SQLITE_CONFIG_MULTITHREAD], +** or [SQLITE_CONFIG_MUTEX]. The return value of this function shows +** only the default compile-time setting, not any run-time changes +** to that setting. +** +** See the [threading mode] documentation for additional information. +** +** INVARIANTS: +** +** {H10101} The [sqlite3_threadsafe()] function shall return zero if +** and only if SQLite was compiled with mutexing code omitted. +** +** {H10102} The value returned by the [sqlite3_threadsafe()] function +** shall remain the same across calls to [sqlite3_config()]. +*/ +int sqlite3_threadsafe(void); + +/* +** CAPI3REF: Database Connection Handle {H12000} +** KEYWORDS: {database connection} {database connections} +** +** Each open SQLite database is represented by a pointer to an instance of +** the opaque structure named "sqlite3". It is useful to think of an sqlite3 +** pointer as an object. The [sqlite3_open()], [sqlite3_open16()], and +** [sqlite3_open_v2()] interfaces are its constructors, and [sqlite3_close()] +** is its destructor. There are many other interfaces (such as +** [sqlite3_prepare_v2()], [sqlite3_create_function()], and +** [sqlite3_busy_timeout()] to name but three) that are methods on an +** sqlite3 object. */ typedef struct sqlite3 sqlite3; - /* -** CAPI3REF: 64-Bit Integer Types +** CAPI3REF: 64-Bit Integer Types {H10200} +** KEYWORDS: sqlite_int64 sqlite_uint64 ** -** Some compilers do not support the "long long" datatype. So we have -** to do compiler-specific typedefs for 64-bit signed and unsigned integers. +** Because there is no cross-platform way to specify 64-bit integer types +** SQLite includes typedefs for 64-bit signed and unsigned integers. ** -** Many SQLite interface functions require a 64-bit integer arguments. -** Those interfaces are declared using this typedef. +** The sqlite3_int64 and sqlite3_uint64 are the preferred type definitions. +** The sqlite_int64 and sqlite_uint64 types are supported for backwards +** compatibility only. +** +** INVARIANTS: +** +** {H10201} The [sqlite_int64] and [sqlite3_int64] type shall specify +** a 64-bit signed integer. +** +** {H10202} The [sqlite_uint64] and [sqlite3_uint64] type shall specify +** a 64-bit unsigned integer. */ #ifdef SQLITE_INT64_TYPE typedef SQLITE_INT64_TYPE sqlite_int64; @@ -144,26 +225,67 @@ typedef struct sqlite3 sqlite3; typedef long long int sqlite_int64; typedef unsigned long long int sqlite_uint64; #endif +typedef sqlite_int64 sqlite3_int64; +typedef sqlite_uint64 sqlite3_uint64; /* ** If compiling for a processor that lacks floating point support, -** substitute integer for floating-point +** substitute integer for floating-point. */ #ifdef SQLITE_OMIT_FLOATING_POINT -# define double sqlite_int64 +# define double sqlite3_int64 #endif /* -** CAPI3REF: Closing A Database Connection +** CAPI3REF: Closing A Database Connection {H12010} ** -** Call this function with a pointer to a structure that was previously -** returned from [sqlite3_open()] and the corresponding database will by -** closed. +** This routine is the destructor for the [sqlite3] object. ** -** All SQL statements prepared using [sqlite3_prepare_v2()] or -** [sqlite3_prepare16_v2()] must be destroyed using [sqlite3_finalize()] -** before this routine is called. Otherwise, SQLITE_BUSY is returned and the -** database connection remains open. +** Applications should [sqlite3_finalize | finalize] all [prepared statements] +** and [sqlite3_blob_close | close] all [BLOB handles] associated with +** the [sqlite3] object prior to attempting to close the object. +** The [sqlite3_next_stmt()] interface can be used to locate all +** [prepared statements] associated with a [database connection] if desired. +** Typical code might look like this: +** +** 
+** sqlite3_stmt *pStmt;
+** while( (pStmt = sqlite3_next_stmt(db, 0))!=0 ){
+**     sqlite3_finalize(pStmt);
+** }
+**
+** +** If [sqlite3_close()] is invoked while a transaction is open, +** the transaction is automatically rolled back. +** +** INVARIANTS: +** +** {H12011} A successful call to [sqlite3_close(C)] shall destroy the +** [database connection] object C. +** +** {H12012} A successful call to [sqlite3_close(C)] shall return SQLITE_OK. +** +** {H12013} A successful call to [sqlite3_close(C)] shall release all +** memory and system resources associated with [database connection] +** C. +** +** {H12014} A call to [sqlite3_close(C)] on a [database connection] C that +** has one or more open [prepared statements] shall fail with +** an [SQLITE_BUSY] error code. +** +** {H12015} A call to [sqlite3_close(C)] where C is a NULL pointer shall +** be a harmless no-op returning SQLITE_OK. +** +** {H12019} When [sqlite3_close(C)] is invoked on a [database connection] C +** that has a pending transaction, the transaction shall be +** rolled back. +** +** ASSUMPTIONS: +** +** {A12016} The C parameter to [sqlite3_close(C)] must be either a NULL +** pointer or an [sqlite3] object pointer obtained +** from [sqlite3_open()], [sqlite3_open16()], or +** [sqlite3_open_v2()], and not previously closed. */ int sqlite3_close(sqlite3 *); @@ -175,76 +297,137 @@ int sqlite3_close(sqlite3 *); typedef int (*sqlite3_callback)(void*,int,char**, char**); /* -** CAPI3REF: One-Step Query Execution Interface +** CAPI3REF: One-Step Query Execution Interface {H12100} ** -** This interface is used to do a one-time evaluatation of zero -** or more SQL statements. UTF-8 text of the SQL statements to -** be evaluted is passed in as the second parameter. The statements -** are prepared one by one using [sqlite3_prepare()], evaluated -** using [sqlite3_step()], then destroyed using [sqlite3_finalize()]. +** The sqlite3_exec() interface is a convenient way of running one or more +** SQL statements without having to write a lot of C code. The UTF-8 encoded +** SQL statements are passed in as the second parameter to sqlite3_exec(). +** The statements are evaluated one by one until either an error or +** an interrupt is encountered, or until they are all done. The 3rd parameter +** is an optional callback that is invoked once for each row of any query +** results produced by the SQL statements. The 5th parameter tells where +** to write any error messages. ** -** If one or more of the SQL statements are queries, then -** the callback function specified by the 3rd parameter is -** invoked once for each row of the query result. This callback -** should normally return 0. If the callback returns a non-zero -** value then the query is aborted, all subsequent SQL statements -** are skipped and the sqlite3_exec() function returns the SQLITE_ABORT. +** The error message passed back through the 5th parameter is held +** in memory obtained from [sqlite3_malloc()]. To avoid a memory leak, +** the calling application should call [sqlite3_free()] on any error +** message returned through the 5th parameter when it has finished using +** the error message. ** -** The 4th parameter to this interface is an arbitrary pointer that is -** passed through to the callback function as its first parameter. +** If the SQL statement in the 2nd parameter is NULL or an empty string +** or a string containing only whitespace and comments, then no SQL +** statements are evaluated and the database is not changed. ** -** The 2nd parameter to the callback function is the number of -** columns in the query result. The 3rd parameter to the callback -** is an array of strings holding the values for each column -** as extracted using [sqlite3_column_text()]. -** The 4th parameter to the callback is an array of strings -** obtained using [sqlite3_column_name()] and holding -** the names of each column. +** The sqlite3_exec() interface is implemented in terms of +** [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()]. +** The sqlite3_exec() routine does nothing to the database that cannot be done +** by [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()]. ** -** The callback function may be NULL, even for queries. A NULL -** callback is not an error. It just means that no callback -** will be invoked. +** INVARIANTS: ** -** If an error occurs while parsing or evaluating the SQL (but -** not while executing the callback) then an appropriate error -** message is written into memory obtained from [sqlite3_malloc()] and -** *errmsg is made to point to that message. The calling function -** is responsible for freeing the memory that holds the error -** message. Use [sqlite3_free()] for this. If errmsg==NULL, -** then no error message is ever written. +** {H12101} A successful invocation of [sqlite3_exec(D,S,C,A,E)] +** shall sequentially evaluate all of the UTF-8 encoded, +** semicolon-separated SQL statements in the zero-terminated +** string S within the context of the [database connection] D. ** -** The return value is is SQLITE_OK if there are no errors and -** some other [SQLITE_OK | return code] if there is an error. -** The particular return value depends on the type of error. +** {H12102} If the S parameter to [sqlite3_exec(D,S,C,A,E)] is NULL then +** the actions of the interface shall be the same as if the +** S parameter were an empty string. ** +** {H12104} The return value of [sqlite3_exec()] shall be [SQLITE_OK] if all +** SQL statements run successfully and to completion. +** +** {H12105} The return value of [sqlite3_exec()] shall be an appropriate +** non-zero [error code] if any SQL statement fails. +** +** {H12107} If one or more of the SQL statements handed to [sqlite3_exec()] +** return results and the 3rd parameter is not NULL, then +** the callback function specified by the 3rd parameter shall be +** invoked once for each row of result. +** +** {H12110} If the callback returns a non-zero value then [sqlite3_exec()] +** shall abort the SQL statement it is currently evaluating, +** skip all subsequent SQL statements, and return [SQLITE_ABORT]. +** +** {H12113} The [sqlite3_exec()] routine shall pass its 4th parameter through +** as the 1st parameter of the callback. +** +** {H12116} The [sqlite3_exec()] routine shall set the 2nd parameter of its +** callback to be the number of columns in the current row of +** result. +** +** {H12119} The [sqlite3_exec()] routine shall set the 3rd parameter of its +** callback to be an array of pointers to strings holding the +** values for each column in the current result set row as +** obtained from [sqlite3_column_text()]. +** +** {H12122} The [sqlite3_exec()] routine shall set the 4th parameter of its +** callback to be an array of pointers to strings holding the +** names of result columns as obtained from [sqlite3_column_name()]. +** +** {H12125} If the 3rd parameter to [sqlite3_exec()] is NULL then +** [sqlite3_exec()] shall silently discard query results. +** +** {H12131} If an error occurs while parsing or evaluating any of the SQL +** statements in the S parameter of [sqlite3_exec(D,S,C,A,E)] and if +** the E parameter is not NULL, then [sqlite3_exec()] shall store +** in *E an appropriate error message written into memory obtained +** from [sqlite3_malloc()]. +** +** {H12134} The [sqlite3_exec(D,S,C,A,E)] routine shall set the value of +** *E to NULL if E is not NULL and there are no errors. +** +** {H12137} The [sqlite3_exec(D,S,C,A,E)] function shall set the [error code] +** and message accessible via [sqlite3_errcode()], +** [sqlite3_extended_errcode()], +** [sqlite3_errmsg()], and [sqlite3_errmsg16()]. +** +** {H12138} If the S parameter to [sqlite3_exec(D,S,C,A,E)] is NULL or an +** empty string or contains nothing other than whitespace, comments, +** and/or semicolons, then results of [sqlite3_errcode()], +** [sqlite3_extended_errcode()], +** [sqlite3_errmsg()], and [sqlite3_errmsg16()] +** shall reset to indicate no errors. +** +** ASSUMPTIONS: +** +** {A12141} The first parameter to [sqlite3_exec()] must be an valid and open +** [database connection]. +** +** {A12142} The database connection must not be closed while +** [sqlite3_exec()] is running. +** +** {A12143} The calling function should use [sqlite3_free()] to free +** the memory that *errmsg is left pointing at once the error +** message is no longer needed. +** +** {A12145} The SQL statement text in the 2nd parameter to [sqlite3_exec()] +** must remain unchanged while [sqlite3_exec()] is running. */ int sqlite3_exec( sqlite3*, /* An open database */ - const char *sql, /* SQL to be evaluted */ + const char *sql, /* SQL to be evaluated */ int (*callback)(void*,int,char**,char**), /* Callback function */ void *, /* 1st argument to callback */ char **errmsg /* Error msg written here */ ); /* -** CAPI3REF: Result Codes -** KEYWORDS: SQLITE_OK +** CAPI3REF: Result Codes {H10210} +** KEYWORDS: SQLITE_OK {error code} {error codes} +** KEYWORDS: {result code} {result codes} ** ** Many SQLite functions return an integer result code from the set shown -** above in order to indicates success or failure. +** here in order to indicates success or failure. ** -** The result codes above are the only ones returned by SQLite in its -** default configuration. However, the [sqlite3_extended_result_codes()] -** API can be used to set a database connectoin to return more detailed -** result codes. +** New error codes may be added in future versions of SQLite. ** ** See also: [SQLITE_IOERR_READ | extended result codes] -** */ #define SQLITE_OK 0 /* Successful result */ /* beginning-of-error-codes */ #define SQLITE_ERROR 1 /* SQL error or missing database */ -#define SQLITE_INTERNAL 2 /* NOT USED. Internal logic error in SQLite */ +#define SQLITE_INTERNAL 2 /* Internal logic error in SQLite */ #define SQLITE_PERM 3 /* Access permission denied */ #define SQLITE_ABORT 4 /* Callback routine requested an abort */ #define SQLITE_BUSY 5 /* The database file is locked */ @@ -261,7 +444,7 @@ int sqlite3_exec( #define SQLITE_EMPTY 16 /* Database is empty */ #define SQLITE_SCHEMA 17 /* The database schema changed */ #define SQLITE_TOOBIG 18 /* String or BLOB exceeds size limit */ -#define SQLITE_CONSTRAINT 19 /* Abort due to contraint violation */ +#define SQLITE_CONSTRAINT 19 /* Abort due to constraint violation */ #define SQLITE_MISMATCH 20 /* Data type mismatch */ #define SQLITE_MISUSE 21 /* Library used incorrectly */ #define SQLITE_NOLFS 22 /* Uses OS features not supported on host */ @@ -274,202 +457,1234 @@ int sqlite3_exec( /* end-of-error-codes */ /* -** CAPI3REF: Extended Result Codes +** CAPI3REF: Extended Result Codes {H10220} +** KEYWORDS: {extended error code} {extended error codes} +** KEYWORDS: {extended result code} {extended result codes} ** ** In its default configuration, SQLite API routines return one of 26 integer -** result codes described at result-codes. However, experience has shown that -** many of these result codes are too course-grained. They do not provide as -** much information about problems as users might like. In an effort to +** [SQLITE_OK | result codes]. However, experience has shown that many of +** these result codes are too coarse-grained. They do not provide as +** much information about problems as programmers might like. In an effort to ** address this, newer versions of SQLite (version 3.3.8 and later) include ** support for additional result codes that provide more detailed information -** about errors. The extended result codes are enabled (or disabled) for -** each database -** connection using the [sqlite3_extended_result_codes()] API. -** -** Some of the available extended result codes are listed above. -** We expect the number of extended result codes will be expand +** about errors. The extended result codes are enabled or disabled +** on a per database connection basis using the +** [sqlite3_extended_result_codes()] API. +** +** Some of the available extended result codes are listed here. +** One may expect the number of extended result codes will be expand ** over time. Software that uses extended result codes should expect ** to see new result codes in future releases of SQLite. -** -** The symbolic name for an extended result code always contains a related -** primary result code as a prefix. Primary result codes contain a single -** "_" character. Extended result codes contain two or more "_" characters. -** The numeric value of an extended result code can be converted to its -** corresponding primary result code by masking off the lower 8 bytes. ** ** The SQLITE_OK result code will never be extended. It will always ** be exactly zero. +** +** INVARIANTS: +** +** {H10223} The symbolic name for an extended result code shall contains +** a related primary result code as a prefix. +** +** {H10224} Primary result code names shall contain a single "_" character. +** +** {H10225} Extended result code names shall contain two or more "_" characters. +** +** {H10226} The numeric value of an extended result code shall contain the +** numeric value of its corresponding primary result code in +** its least significant 8 bits. */ -#define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) -#define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) -#define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) -#define SQLITE_IOERR_FSYNC (SQLITE_IOERR | (4<<8)) -#define SQLITE_IOERR_DIR_FSYNC (SQLITE_IOERR | (5<<8)) -#define SQLITE_IOERR_TRUNCATE (SQLITE_IOERR | (6<<8)) -#define SQLITE_IOERR_FSTAT (SQLITE_IOERR | (7<<8)) -#define SQLITE_IOERR_UNLOCK (SQLITE_IOERR | (8<<8)) -#define SQLITE_IOERR_RDLOCK (SQLITE_IOERR | (9<<8)) -#define SQLITE_IOERR_DELETE (SQLITE_IOERR | (10<<8)) -#define SQLITE_IOERR_BLOCKED (SQLITE_IOERR | (11<<8)) +#define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) +#define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) +#define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) +#define SQLITE_IOERR_FSYNC (SQLITE_IOERR | (4<<8)) +#define SQLITE_IOERR_DIR_FSYNC (SQLITE_IOERR | (5<<8)) +#define SQLITE_IOERR_TRUNCATE (SQLITE_IOERR | (6<<8)) +#define SQLITE_IOERR_FSTAT (SQLITE_IOERR | (7<<8)) +#define SQLITE_IOERR_UNLOCK (SQLITE_IOERR | (8<<8)) +#define SQLITE_IOERR_RDLOCK (SQLITE_IOERR | (9<<8)) +#define SQLITE_IOERR_DELETE (SQLITE_IOERR | (10<<8)) +#define SQLITE_IOERR_BLOCKED (SQLITE_IOERR | (11<<8)) +#define SQLITE_IOERR_NOMEM (SQLITE_IOERR | (12<<8)) +#define SQLITE_IOERR_ACCESS (SQLITE_IOERR | (13<<8)) +#define SQLITE_IOERR_CHECKRESERVEDLOCK (SQLITE_IOERR | (14<<8)) +#define SQLITE_IOERR_LOCK (SQLITE_IOERR | (15<<8)) +#define SQLITE_IOERR_CLOSE (SQLITE_IOERR | (16<<8)) +#define SQLITE_IOERR_DIR_CLOSE (SQLITE_IOERR | (17<<8)) /* -** CAPI3REF: Enable Or Disable Extended Result Codes +** CAPI3REF: Flags For File Open Operations {H10230} ** -** This routine enables or disables the -** [SQLITE_IOERR_READ | extended result codes] feature. -** By default, SQLite API routines return one of only 26 integer -** [SQLITE_OK | result codes]. When extended result codes -** are enabled by this routine, the repetoire of result codes can be -** much larger and can (hopefully) provide more detailed information -** about the cause of an error. +** These bit values are intended for use in the +** 3rd parameter to the [sqlite3_open_v2()] interface and +** in the 4th parameter to the xOpen method of the +** [sqlite3_vfs] object. +*/ +#define SQLITE_OPEN_READONLY 0x00000001 +#define SQLITE_OPEN_READWRITE 0x00000002 +#define SQLITE_OPEN_CREATE 0x00000004 +#define SQLITE_OPEN_DELETEONCLOSE 0x00000008 +#define SQLITE_OPEN_EXCLUSIVE 0x00000010 +#define SQLITE_OPEN_MAIN_DB 0x00000100 +#define SQLITE_OPEN_TEMP_DB 0x00000200 +#define SQLITE_OPEN_TRANSIENT_DB 0x00000400 +#define SQLITE_OPEN_MAIN_JOURNAL 0x00000800 +#define SQLITE_OPEN_TEMP_JOURNAL 0x00001000 +#define SQLITE_OPEN_SUBJOURNAL 0x00002000 +#define SQLITE_OPEN_MASTER_JOURNAL 0x00004000 +#define SQLITE_OPEN_NOMUTEX 0x00008000 +#define SQLITE_OPEN_FULLMUTEX 0x00010000 + +/* +** CAPI3REF: Device Characteristics {H10240} ** -** The second argument is a boolean value that turns extended result -** codes on and off. Extended result codes are off by default for -** backwards compatibility with older versions of SQLite. +** The xDeviceCapabilities method of the [sqlite3_io_methods] +** object returns an integer which is a vector of the these +** bit values expressing I/O characteristics of the mass storage +** device that holds the file that the [sqlite3_io_methods] +** refers to. +** +** The SQLITE_IOCAP_ATOMIC property means that all writes of +** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values +** mean that writes of blocks that are nnn bytes in size and +** are aligned to an address which is an integer multiple of +** nnn are atomic. The SQLITE_IOCAP_SAFE_APPEND value means +** that when data is appended to a file, the data is appended +** first then the size of the file is extended, never the other +** way around. The SQLITE_IOCAP_SEQUENTIAL property means that +** information is written to disk in the same order as calls +** to xWrite(). +*/ +#define SQLITE_IOCAP_ATOMIC 0x00000001 +#define SQLITE_IOCAP_ATOMIC512 0x00000002 +#define SQLITE_IOCAP_ATOMIC1K 0x00000004 +#define SQLITE_IOCAP_ATOMIC2K 0x00000008 +#define SQLITE_IOCAP_ATOMIC4K 0x00000010 +#define SQLITE_IOCAP_ATOMIC8K 0x00000020 +#define SQLITE_IOCAP_ATOMIC16K 0x00000040 +#define SQLITE_IOCAP_ATOMIC32K 0x00000080 +#define SQLITE_IOCAP_ATOMIC64K 0x00000100 +#define SQLITE_IOCAP_SAFE_APPEND 0x00000200 +#define SQLITE_IOCAP_SEQUENTIAL 0x00000400 + +/* +** CAPI3REF: File Locking Levels {H10250} +** +** SQLite uses one of these integer values as the second +** argument to calls it makes to the xLock() and xUnlock() methods +** of an [sqlite3_io_methods] object. +*/ +#define SQLITE_LOCK_NONE 0 +#define SQLITE_LOCK_SHARED 1 +#define SQLITE_LOCK_RESERVED 2 +#define SQLITE_LOCK_PENDING 3 +#define SQLITE_LOCK_EXCLUSIVE 4 + +/* +** CAPI3REF: Synchronization Type Flags {H10260} +** +** When SQLite invokes the xSync() method of an +** [sqlite3_io_methods] object it uses a combination of +** these integer values as the second argument. +** +** When the SQLITE_SYNC_DATAONLY flag is used, it means that the +** sync operation only needs to flush data to mass storage. Inode +** information need not be flushed. The SQLITE_SYNC_NORMAL flag means +** to use normal fsync() semantics. The SQLITE_SYNC_FULL flag means +** to use Mac OS X style fullsync instead of fsync(). +*/ +#define SQLITE_SYNC_NORMAL 0x00002 +#define SQLITE_SYNC_FULL 0x00003 +#define SQLITE_SYNC_DATAONLY 0x00010 + +/* +** CAPI3REF: OS Interface Open File Handle {H11110} +** +** An [sqlite3_file] object represents an open file in the OS +** interface layer. Individual OS interface implementations will +** want to subclass this object by appending additional fields +** for their own use. The pMethods entry is a pointer to an +** [sqlite3_io_methods] object that defines methods for performing +** I/O operations on the open file. +*/ +typedef struct sqlite3_file sqlite3_file; +struct sqlite3_file { + const struct sqlite3_io_methods *pMethods; /* Methods for an open file */ +}; + +/* +** CAPI3REF: OS Interface File Virtual Methods Object {H11120} +** +** Every file opened by the [sqlite3_vfs] xOpen method populates an +** [sqlite3_file] object (or, more commonly, a subclass of the +** [sqlite3_file] object) with a pointer to an instance of this object. +** This object defines the methods used to perform various operations +** against the open file represented by the [sqlite3_file] object. +** +** The flags argument to xSync may be one of [SQLITE_SYNC_NORMAL] or +** [SQLITE_SYNC_FULL]. The first choice is the normal fsync(). +** The second choice is a Mac OS X style fullsync. The [SQLITE_SYNC_DATAONLY] +** flag may be ORed in to indicate that only the data of the file +** and not its inode needs to be synced. +** +** The integer values to xLock() and xUnlock() are one of +**
    +**
  • [SQLITE_LOCK_NONE], +**
  • [SQLITE_LOCK_SHARED], +**
  • [SQLITE_LOCK_RESERVED], +**
  • [SQLITE_LOCK_PENDING], or +**
  • [SQLITE_LOCK_EXCLUSIVE]. +**
+** xLock() increases the lock. xUnlock() decreases the lock. +** The xCheckReservedLock() method checks whether any database connection, +** either in this process or in some other process, is holding a RESERVED, +** PENDING, or EXCLUSIVE lock on the file. It returns true +** if such a lock exists and false otherwise. +** +** The xFileControl() method is a generic interface that allows custom +** VFS implementations to directly control an open file using the +** [sqlite3_file_control()] interface. The second "op" argument is an +** integer opcode. The third argument is a generic pointer intended to +** point to a structure that may contain arguments or space in which to +** write return values. Potential uses for xFileControl() might be +** functions to enable blocking locks with timeouts, to change the +** locking strategy (for example to use dot-file locks), to inquire +** about the status of a lock, or to break stale locks. The SQLite +** core reserves all opcodes less than 100 for its own use. +** A [SQLITE_FCNTL_LOCKSTATE | list of opcodes] less than 100 is available. +** Applications that define a custom xFileControl method should use opcodes +** greater than 100 to avoid conflicts. +** +** The xSectorSize() method returns the sector size of the +** device that underlies the file. The sector size is the +** minimum write that can be performed without disturbing +** other bytes in the file. The xDeviceCharacteristics() +** method returns a bit vector describing behaviors of the +** underlying device: +** +**
    +**
  • [SQLITE_IOCAP_ATOMIC] +**
  • [SQLITE_IOCAP_ATOMIC512] +**
  • [SQLITE_IOCAP_ATOMIC1K] +**
  • [SQLITE_IOCAP_ATOMIC2K] +**
  • [SQLITE_IOCAP_ATOMIC4K] +**
  • [SQLITE_IOCAP_ATOMIC8K] +**
  • [SQLITE_IOCAP_ATOMIC16K] +**
  • [SQLITE_IOCAP_ATOMIC32K] +**
  • [SQLITE_IOCAP_ATOMIC64K] +**
  • [SQLITE_IOCAP_SAFE_APPEND] +**
  • [SQLITE_IOCAP_SEQUENTIAL] +**
+** +** The SQLITE_IOCAP_ATOMIC property means that all writes of +** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values +** mean that writes of blocks that are nnn bytes in size and +** are aligned to an address which is an integer multiple of +** nnn are atomic. The SQLITE_IOCAP_SAFE_APPEND value means +** that when data is appended to a file, the data is appended +** first then the size of the file is extended, never the other +** way around. The SQLITE_IOCAP_SEQUENTIAL property means that +** information is written to disk in the same order as calls +** to xWrite(). +** +** If xRead() returns SQLITE_IOERR_SHORT_READ it must also fill +** in the unread portions of the buffer with zeros. A VFS that +** fails to zero-fill short reads might seem to work. However, +** failure to zero-fill short reads will eventually lead to +** database corruption. +*/ +typedef struct sqlite3_io_methods sqlite3_io_methods; +struct sqlite3_io_methods { + int iVersion; + int (*xClose)(sqlite3_file*); + int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); + int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst); + int (*xTruncate)(sqlite3_file*, sqlite3_int64 size); + int (*xSync)(sqlite3_file*, int flags); + int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize); + int (*xLock)(sqlite3_file*, int); + int (*xUnlock)(sqlite3_file*, int); + int (*xCheckReservedLock)(sqlite3_file*, int *pResOut); + int (*xFileControl)(sqlite3_file*, int op, void *pArg); + int (*xSectorSize)(sqlite3_file*); + int (*xDeviceCharacteristics)(sqlite3_file*); + /* Additional methods may be added in future releases */ +}; + +/* +** CAPI3REF: Standard File Control Opcodes {H11310} +** +** These integer constants are opcodes for the xFileControl method +** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] +** interface. +** +** The [SQLITE_FCNTL_LOCKSTATE] opcode is used for debugging. This +** opcode causes the xFileControl method to write the current state of +** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], +** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) +** into an integer that the pArg argument points to. This capability +** is used during testing and only needs to be supported when SQLITE_TEST +** is defined. +*/ +#define SQLITE_FCNTL_LOCKSTATE 1 +#define SQLITE_GET_LOCKPROXYFILE 2 +#define SQLITE_SET_LOCKPROXYFILE 3 +#define SQLITE_LAST_ERRNO 4 + +/* +** CAPI3REF: Mutex Handle {H17110} +** +** The mutex module within SQLite defines [sqlite3_mutex] to be an +** abstract type for a mutex object. The SQLite core never looks +** at the internal representation of an [sqlite3_mutex]. It only +** deals with pointers to the [sqlite3_mutex] object. +** +** Mutexes are created using [sqlite3_mutex_alloc()]. +*/ +typedef struct sqlite3_mutex sqlite3_mutex; + +/* +** CAPI3REF: OS Interface Object {H11140} +** +** An instance of the sqlite3_vfs object defines the interface between +** the SQLite core and the underlying operating system. The "vfs" +** in the name of the object stands for "virtual file system". +** +** The value of the iVersion field is initially 1 but may be larger in +** future versions of SQLite. Additional fields may be appended to this +** object when the iVersion value is increased. Note that the structure +** of the sqlite3_vfs object changes in the transaction between +** SQLite version 3.5.9 and 3.6.0 and yet the iVersion field was not +** modified. +** +** The szOsFile field is the size of the subclassed [sqlite3_file] +** structure used by this VFS. mxPathname is the maximum length of +** a pathname in this VFS. +** +** Registered sqlite3_vfs objects are kept on a linked list formed by +** the pNext pointer. The [sqlite3_vfs_register()] +** and [sqlite3_vfs_unregister()] interfaces manage this list +** in a thread-safe way. The [sqlite3_vfs_find()] interface +** searches the list. Neither the application code nor the VFS +** implementation should use the pNext pointer. +** +** The pNext field is the only field in the sqlite3_vfs +** structure that SQLite will ever modify. SQLite will only access +** or modify this field while holding a particular static mutex. +** The application should never modify anything within the sqlite3_vfs +** object once the object has been registered. +** +** The zName field holds the name of the VFS module. The name must +** be unique across all VFS modules. +** +** SQLite will guarantee that the zFilename parameter to xOpen +** is either a NULL pointer or string obtained +** from xFullPathname(). SQLite further guarantees that +** the string will be valid and unchanged until xClose() is +** called. Because of the previous sentense, +** the [sqlite3_file] can safely store a pointer to the +** filename if it needs to remember the filename for some reason. +** If the zFilename parameter is xOpen is a NULL pointer then xOpen +** must invite its own temporary name for the file. Whenever the +** xFilename parameter is NULL it will also be the case that the +** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. +** +** The flags argument to xOpen() includes all bits set in +** the flags argument to [sqlite3_open_v2()]. Or if [sqlite3_open()] +** or [sqlite3_open16()] is used, then flags includes at least +** [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]. +** If xOpen() opens a file read-only then it sets *pOutFlags to +** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. +** +** SQLite will also add one of the following flags to the xOpen() +** call, depending on the object being opened: +** +**
    +**
  • [SQLITE_OPEN_MAIN_DB] +**
  • [SQLITE_OPEN_MAIN_JOURNAL] +**
  • [SQLITE_OPEN_TEMP_DB] +**
  • [SQLITE_OPEN_TEMP_JOURNAL] +**
  • [SQLITE_OPEN_TRANSIENT_DB] +**
  • [SQLITE_OPEN_SUBJOURNAL] +**
  • [SQLITE_OPEN_MASTER_JOURNAL] +**
+** +** The file I/O implementation can use the object type flags to +** change the way it deals with files. For example, an application +** that does not care about crash recovery or rollback might make +** the open of a journal file a no-op. Writes to this journal would +** also be no-ops, and any attempt to read the journal would return +** SQLITE_IOERR. Or the implementation might recognize that a database +** file will be doing page-aligned sector reads and writes in a random +** order and set up its I/O subsystem accordingly. +** +** SQLite might also add one of the following flags to the xOpen method: +** +**
    +**
  • [SQLITE_OPEN_DELETEONCLOSE] +**
  • [SQLITE_OPEN_EXCLUSIVE] +**
+** +** The [SQLITE_OPEN_DELETEONCLOSE] flag means the file should be +** deleted when it is closed. The [SQLITE_OPEN_DELETEONCLOSE] +** will be set for TEMP databases, journals and for subjournals. +** +** The [SQLITE_OPEN_EXCLUSIVE] flag means the file should be opened +** for exclusive access. This flag is set for all files except +** for the main database file. +** +** At least szOsFile bytes of memory are allocated by SQLite +** to hold the [sqlite3_file] structure passed as the third +** argument to xOpen. The xOpen method does not have to +** allocate the structure; it should just fill it in. +** +** The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] +** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to +** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] +** to test whether a file is at least readable. The file can be a +** directory. +** +** SQLite will always allocate at least mxPathname+1 bytes for the +** output buffer xFullPathname. The exact size of the output buffer +** is also passed as a parameter to both methods. If the output buffer +** is not large enough, [SQLITE_CANTOPEN] should be returned. Since this is +** handled as a fatal error by SQLite, vfs implementations should endeavor +** to prevent this by setting mxPathname to a sufficiently large value. +** +** The xRandomness(), xSleep(), and xCurrentTime() interfaces +** are not strictly a part of the filesystem, but they are +** included in the VFS structure for completeness. +** The xRandomness() function attempts to return nBytes bytes +** of good-quality randomness into zOut. The return value is +** the actual number of bytes of randomness obtained. +** The xSleep() method causes the calling thread to sleep for at +** least the number of microseconds given. The xCurrentTime() +** method returns a Julian Day Number for the current date and time. +** +*/ +typedef struct sqlite3_vfs sqlite3_vfs; +struct sqlite3_vfs { + int iVersion; /* Structure version number */ + int szOsFile; /* Size of subclassed sqlite3_file */ + int mxPathname; /* Maximum file pathname length */ + sqlite3_vfs *pNext; /* Next registered VFS */ + const char *zName; /* Name of this virtual file system */ + void *pAppData; /* Pointer to application-specific data */ + int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*, + int flags, int *pOutFlags); + int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); + int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); + int (*xFullPathname)(sqlite3_vfs*, const char *zName, int nOut, char *zOut); + void *(*xDlOpen)(sqlite3_vfs*, const char *zFilename); + void (*xDlError)(sqlite3_vfs*, int nByte, char *zErrMsg); + void (*(*xDlSym)(sqlite3_vfs*,void*, const char *zSymbol))(void); + void (*xDlClose)(sqlite3_vfs*, void*); + int (*xRandomness)(sqlite3_vfs*, int nByte, char *zOut); + int (*xSleep)(sqlite3_vfs*, int microseconds); + int (*xCurrentTime)(sqlite3_vfs*, double*); + int (*xGetLastError)(sqlite3_vfs*, int, char *); + /* New fields may be appended in figure versions. The iVersion + ** value will increment whenever this happens. */ +}; + +/* +** CAPI3REF: Flags for the xAccess VFS method {H11190} +** +** These integer constants can be used as the third parameter to +** the xAccess method of an [sqlite3_vfs] object. {END} They determine +** what kind of permissions the xAccess method is looking for. +** With SQLITE_ACCESS_EXISTS, the xAccess method +** simply checks whether the file exists. +** With SQLITE_ACCESS_READWRITE, the xAccess method +** checks whether the file is both readable and writable. +** With SQLITE_ACCESS_READ, the xAccess method +** checks whether the file is readable. +*/ +#define SQLITE_ACCESS_EXISTS 0 +#define SQLITE_ACCESS_READWRITE 1 +#define SQLITE_ACCESS_READ 2 + +/* +** CAPI3REF: Initialize The SQLite Library {H10130} +** +** The sqlite3_initialize() routine initializes the +** SQLite library. The sqlite3_shutdown() routine +** deallocates any resources that were allocated by sqlite3_initialize(). +** +** A call to sqlite3_initialize() is an "effective" call if it is +** the first time sqlite3_initialize() is invoked during the lifetime of +** the process, or if it is the first time sqlite3_initialize() is invoked +** following a call to sqlite3_shutdown(). Only an effective call +** of sqlite3_initialize() does any initialization. All other calls +** are harmless no-ops. +** +** Among other things, sqlite3_initialize() shall invoke +** sqlite3_os_init(). Similarly, sqlite3_shutdown() +** shall invoke sqlite3_os_end(). +** +** The sqlite3_initialize() routine returns [SQLITE_OK] on success. +** If for some reason, sqlite3_initialize() is unable to initialize +** the library (perhaps it is unable to allocate a needed resource such +** as a mutex) it returns an [error code] other than [SQLITE_OK]. +** +** The sqlite3_initialize() routine is called internally by many other +** SQLite interfaces so that an application usually does not need to +** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] +** calls sqlite3_initialize() so the SQLite library will be automatically +** initialized when [sqlite3_open()] is called if it has not be initialized +** already. However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] +** compile-time option, then the automatic calls to sqlite3_initialize() +** are omitted and the application must call sqlite3_initialize() directly +** prior to using any other SQLite interface. For maximum portability, +** it is recommended that applications always invoke sqlite3_initialize() +** directly prior to using any other SQLite interface. Future releases +** of SQLite may require this. In other words, the behavior exhibited +** when SQLite is compiled with [SQLITE_OMIT_AUTOINIT] might become the +** default behavior in some future release of SQLite. +** +** The sqlite3_os_init() routine does operating-system specific +** initialization of the SQLite library. The sqlite3_os_end() +** routine undoes the effect of sqlite3_os_init(). Typical tasks +** performed by these routines include allocation or deallocation +** of static resources, initialization of global variables, +** setting up a default [sqlite3_vfs] module, or setting up +** a default configuration using [sqlite3_config()]. +** +** The application should never invoke either sqlite3_os_init() +** or sqlite3_os_end() directly. The application should only invoke +** sqlite3_initialize() and sqlite3_shutdown(). The sqlite3_os_init() +** interface is called automatically by sqlite3_initialize() and +** sqlite3_os_end() is called by sqlite3_shutdown(). Appropriate +** implementations for sqlite3_os_init() and sqlite3_os_end() +** are built into SQLite when it is compiled for unix, windows, or os/2. +** When built for other platforms (using the [SQLITE_OS_OTHER=1] compile-time +** option) the application must supply a suitable implementation for +** sqlite3_os_init() and sqlite3_os_end(). An application-supplied +** implementation of sqlite3_os_init() or sqlite3_os_end() +** must return [SQLITE_OK] on success and some other [error code] upon +** failure. +*/ +int sqlite3_initialize(void); +int sqlite3_shutdown(void); +int sqlite3_os_init(void); +int sqlite3_os_end(void); + +/* +** CAPI3REF: Configuring The SQLite Library {H14100} +** EXPERIMENTAL +** +** The sqlite3_config() interface is used to make global configuration +** changes to SQLite in order to tune SQLite to the specific needs of +** the application. The default configuration is recommended for most +** applications and so this routine is usually not necessary. It is +** provided to support rare applications with unusual needs. +** +** The sqlite3_config() interface is not threadsafe. The application +** must insure that no other SQLite interfaces are invoked by other +** threads while sqlite3_config() is running. Furthermore, sqlite3_config() +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** Note, however, that sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** +** The first argument to sqlite3_config() is an integer +** [SQLITE_CONFIG_SINGLETHREAD | configuration option] that determines +** what property of SQLite is to be configured. Subsequent arguments +** vary depending on the [SQLITE_CONFIG_SINGLETHREAD | configuration option] +** in the first argument. +** +** When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. +** If the option is unknown or SQLite is unable to set the option +** then this routine returns a non-zero [error code]. +** +** INVARIANTS: +** +** {H14103} A successful invocation of [sqlite3_config()] shall return +** [SQLITE_OK]. +** +** {H14106} The [sqlite3_config()] interface shall return [SQLITE_MISUSE] +** if it is invoked in between calls to [sqlite3_initialize()] and +** [sqlite3_shutdown()]. +** +** {H14120} A successful call to [sqlite3_config]([SQLITE_CONFIG_SINGLETHREAD]) +** shall set the default [threading mode] to Single-thread. +** +** {H14123} A successful call to [sqlite3_config]([SQLITE_CONFIG_MULTITHREAD]) +** shall set the default [threading mode] to Multi-thread. +** +** {H14126} A successful call to [sqlite3_config]([SQLITE_CONFIG_SERIALIZED]) +** shall set the default [threading mode] to Serialized. +** +** {H14129} A successful call to [sqlite3_config]([SQLITE_CONFIG_MUTEX],X) +** where X is a pointer to an initialized [sqlite3_mutex_methods] +** object shall cause all subsequent mutex operations performed +** by SQLite to use the mutex methods that were present in X +** during the call to [sqlite3_config()]. +** +** {H14132} A successful call to [sqlite3_config]([SQLITE_CONFIG_GETMUTEX],X) +** where X is a pointer to an [sqlite3_mutex_methods] object +** shall overwrite the content of [sqlite3_mutex_methods] object +** with the mutex methods currently in use by SQLite. +** +** {H14135} A successful call to [sqlite3_config]([SQLITE_CONFIG_MALLOC],M) +** where M is a pointer to an initialized [sqlite3_mem_methods] +** object shall cause all subsequent memory allocation operations +** performed by SQLite to use the methods that were present in +** M during the call to [sqlite3_config()]. +** +** {H14138} A successful call to [sqlite3_config]([SQLITE_CONFIG_GETMALLOC],M) +** where M is a pointer to an [sqlite3_mem_methods] object shall +** overwrite the content of [sqlite3_mem_methods] object with +** the memory allocation methods currently in use by +** SQLite. +** +** {H14141} A successful call to [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],1) +** shall enable the memory allocation status collection logic. +** +** {H14144} A successful call to [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],0) +** shall disable the memory allocation status collection logic. +** +** {H14147} The memory allocation status collection logic shall be +** enabled by default. +** +** {H14150} A successful call to [sqlite3_config]([SQLITE_CONFIG_SCRATCH],S,Z,N) +** where Z and N are non-negative integers and +** S is a pointer to an aligned memory buffer not less than +** Z*N bytes in size shall cause S to be used by the +** [scratch memory allocator] for as many as N simulataneous +** allocations each of size (Z & ~7). +** +** {H14153} A successful call to [sqlite3_config]([SQLITE_CONFIG_SCRATCH],S,Z,N) +** where S is a NULL pointer shall disable the +** [scratch memory allocator]. +** +** {H14156} A successful call to +** [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],S,Z,N) +** where Z and N are non-negative integers and +** S is a pointer to an aligned memory buffer not less than +** Z*N bytes in size shall cause S to be used by the +** [pagecache memory allocator] for as many as N simulataneous +** allocations each of size (Z & ~7). +** +** {H14159} A successful call to +** [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],S,Z,N) +** where S is a NULL pointer shall disable the +** [pagecache memory allocator]. +** +** {H14162} A successful call to [sqlite3_config]([SQLITE_CONFIG_HEAP],H,Z,N) +** where Z and N are non-negative integers and +** H is a pointer to an aligned memory buffer not less than +** Z bytes in size shall enable the [memsys5] memory allocator +** and cause it to use buffer S as its memory source and to use +** a minimum allocation size of N. +** +** {H14165} A successful call to [sqlite3_config]([SQLITE_CONFIG_HEAP],H,Z,N) +** where H is a NULL pointer shall disable the +** [memsys5] memory allocator. +** +** {H14168} A successful call to [sqlite3_config]([SQLITE_CONFIG_LOOKASIDE],Z,N) +** shall cause the default [lookaside memory allocator] configuration +** for new [database connections] to be N slots of Z bytes each. +*/ +SQLITE_EXPERIMENTAL int sqlite3_config(int, ...); + +/* +** CAPI3REF: Configure database connections {H14200} +** EXPERIMENTAL +** +** The sqlite3_db_config() interface is used to make configuration +** changes to a [database connection]. The interface is similar to +** [sqlite3_config()] except that the changes apply to a single +** [database connection] (specified in the first argument). The +** sqlite3_db_config() interface can only be used immediately after +** the database connection is created using [sqlite3_open()], +** [sqlite3_open16()], or [sqlite3_open_v2()]. +** +** The second argument to sqlite3_db_config(D,V,...) is the +** configuration verb - an integer code that indicates what +** aspect of the [database connection] is being configured. +** The only choice for this value is [SQLITE_DBCONFIG_LOOKASIDE]. +** New verbs are likely to be added in future releases of SQLite. +** Additional arguments depend on the verb. +** +** INVARIANTS: +** +** {H14203} A call to [sqlite3_db_config(D,V,...)] shall return [SQLITE_OK] +** if and only if the call is successful. +** +** {H14206} If one or more slots of the [lookaside memory allocator] for +** [database connection] D are in use, then a call to +** [sqlite3_db_config](D,[SQLITE_DBCONFIG_LOOKASIDE],...) shall +** fail with an [SQLITE_BUSY] return code. +** +** {H14209} A successful call to +** [sqlite3_db_config](D,[SQLITE_DBCONFIG_LOOKASIDE],B,Z,N) where +** D is an open [database connection] and Z and N are positive +** integers and B is an aligned buffer at least Z*N bytes in size +** shall cause the [lookaside memory allocator] for D to use buffer B +** with N slots of Z bytes each. +** +** {H14212} A successful call to +** [sqlite3_db_config](D,[SQLITE_DBCONFIG_LOOKASIDE],B,Z,N) where +** D is an open [database connection] and Z and N are positive +** integers and B is NULL pointer shall cause the +** [lookaside memory allocator] for D to a obtain Z*N byte buffer +** from the primary memory allocator and use that buffer +** with N lookaside slots of Z bytes each. +** +** {H14215} A successful call to +** [sqlite3_db_config](D,[SQLITE_DBCONFIG_LOOKASIDE],B,Z,N) where +** D is an open [database connection] and Z and N are zero shall +** disable the [lookaside memory allocator] for D. +** +** +*/ +SQLITE_EXPERIMENTAL int sqlite3_db_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Memory Allocation Routines {H10155} +** EXPERIMENTAL +** +** An instance of this object defines the interface between SQLite +** and low-level memory allocation routines. +** +** This object is used in only one place in the SQLite interface. +** A pointer to an instance of this object is the argument to +** [sqlite3_config()] when the configuration option is +** [SQLITE_CONFIG_MALLOC]. By creating an instance of this object +** and passing it to [sqlite3_config()] during configuration, an +** application can specify an alternative memory allocation subsystem +** for SQLite to use for all of its dynamic memory needs. +** +** Note that SQLite comes with a built-in memory allocator that is +** perfectly adequate for the overwhelming majority of applications +** and that this object is only useful to a tiny minority of applications +** with specialized memory allocation requirements. This object is +** also used during testing of SQLite in order to specify an alternative +** memory allocator that simulates memory out-of-memory conditions in +** order to verify that SQLite recovers gracefully from such +** conditions. +** +** The xMalloc, xFree, and xRealloc methods must work like the +** malloc(), free(), and realloc() functions from the standard library. +** +** xSize should return the allocated size of a memory allocation +** previously obtained from xMalloc or xRealloc. The allocated size +** is always at least as big as the requested size but may be larger. +** +** The xRoundup method returns what would be the allocated size of +** a memory allocation given a particular requested size. Most memory +** allocators round up memory allocations at least to the next multiple +** of 8. Some allocators round up to a larger multiple or to a power of 2. +** +** The xInit method initializes the memory allocator. (For example, +** it might allocate any require mutexes or initialize internal data +** structures. The xShutdown method is invoked (indirectly) by +** [sqlite3_shutdown()] and should deallocate any resources acquired +** by xInit. The pAppData pointer is used as the only parameter to +** xInit and xShutdown. +*/ +typedef struct sqlite3_mem_methods sqlite3_mem_methods; +struct sqlite3_mem_methods { + void *(*xMalloc)(int); /* Memory allocation function */ + void (*xFree)(void*); /* Free a prior allocation */ + void *(*xRealloc)(void*,int); /* Resize an allocation */ + int (*xSize)(void*); /* Return the size of an allocation */ + int (*xRoundup)(int); /* Round up request size to allocation size */ + int (*xInit)(void*); /* Initialize the memory allocator */ + void (*xShutdown)(void*); /* Deinitialize the memory allocator */ + void *pAppData; /* Argument to xInit() and xShutdown() */ +}; + +/* +** CAPI3REF: Configuration Options {H10160} +** EXPERIMENTAL +** +** These constants are the available integer configuration options that +** can be passed as the first argument to the [sqlite3_config()] interface. +** +** New configuration options may be added in future releases of SQLite. +** Existing configuration options might be discontinued. Applications +** should check the return code from [sqlite3_config()] to make sure that +** the call worked. The [sqlite3_config()] interface will return a +** non-zero [error code] if a discontinued or unsupported configuration option +** is invoked. +** +**
+**
SQLITE_CONFIG_SINGLETHREAD
+**
There are no arguments to this option. This option disables +** all mutexing and puts SQLite into a mode where it can only be used +** by a single thread.
+** +**
SQLITE_CONFIG_MULTITHREAD
+**
There are no arguments to this option. This option disables +** mutexing on [database connection] and [prepared statement] objects. +** The application is responsible for serializing access to +** [database connections] and [prepared statements]. But other mutexes +** are enabled so that SQLite will be safe to use in a multi-threaded +** environment as long as no two threads attempt to use the same +** [database connection] at the same time. See the [threading mode] +** documentation for additional information.
+** +**
SQLITE_CONFIG_SERIALIZED
+**
There are no arguments to this option. This option enables +** all mutexes including the recursive +** mutexes on [database connection] and [prepared statement] objects. +** In this mode (which is the default when SQLite is compiled with +** [SQLITE_THREADSAFE=1]) the SQLite library will itself serialize access +** to [database connections] and [prepared statements] so that the +** application is free to use the same [database connection] or the +** same [prepared statement] in different threads at the same time. +** See the [threading mode] documentation for additional information.
+** +**
SQLITE_CONFIG_MALLOC
+**
This option takes a single argument which is a pointer to an +** instance of the [sqlite3_mem_methods] structure. The argument specifies +** alternative low-level memory allocation routines to be used in place of +** the memory allocation routines built into SQLite.
+** +**
SQLITE_CONFIG_GETMALLOC
+**
This option takes a single argument which is a pointer to an +** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] +** structure is filled with the currently defined memory allocation routines. +** This option can be used to overload the default memory allocation +** routines with a wrapper that simulations memory allocation failure or +** tracks memory usage, for example.
+** +**
SQLITE_CONFIG_MEMSTATUS
+**
This option takes single argument of type int, interpreted as a +** boolean, which enables or disables the collection of memory allocation +** statistics. When disabled, the following SQLite interfaces become +** non-operational: +**
    +**
  • [sqlite3_memory_used()] +**
  • [sqlite3_memory_highwater()] +**
  • [sqlite3_soft_heap_limit()] +**
  • [sqlite3_status()] +**
+**
+** +**
SQLITE_CONFIG_SCRATCH
+**
This option specifies a static memory buffer that SQLite can use for +** scratch memory. There are three arguments: A pointer to the memory, the +** size of each scratch buffer (sz), and the number of buffers (N). The sz +** argument must be a multiple of 16. The sz parameter should be a few bytes +** larger than the actual scratch space required due internal overhead. +** The first +** argument should point to an allocation of at least sz*N bytes of memory. +** SQLite will use no more than one scratch buffer at once per thread, so +** N should be set to the expected maximum number of threads. The sz +** parameter should be 6 times the size of the largest database page size. +** Scratch buffers are used as part of the btree balance operation. If +** The btree balancer needs additional memory beyond what is provided by +** scratch buffers or if no scratch buffer space is specified, then SQLite +** goes to [sqlite3_malloc()] to obtain the memory it needs.
+** +**
SQLITE_CONFIG_PAGECACHE
+**
This option specifies a static memory buffer that SQLite can use for +** the database page cache with the default page cache implemenation. +** This configuration should not be used if an application-define page +** cache implementation is loaded using the SQLITE_CONFIG_PCACHE option. +** There are three arguments to this option: A pointer to the +** memory, the size of each page buffer (sz), and the number of pages (N). +** The sz argument must be a power of two between 512 and 32768. The first +** argument should point to an allocation of at least sz*N bytes of memory. +** SQLite will use the memory provided by the first argument to satisfy its +** memory needs for the first N pages that it adds to cache. If additional +** page cache memory is needed beyond what is provided by this option, then +** SQLite goes to [sqlite3_malloc()] for the additional storage space. +** The implementation might use one or more of the N buffers to hold +** memory accounting information.
+** +**
SQLITE_CONFIG_HEAP
+**
This option specifies a static memory buffer that SQLite will use +** for all of its dynamic memory allocation needs beyond those provided +** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. +** There are three arguments: A pointer to the memory, the number of +** bytes in the memory buffer, and the minimum allocation size. If +** the first pointer (the memory pointer) is NULL, then SQLite reverts +** to using its default memory allocator (the system malloc() implementation), +** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. If the +** memory pointer is not NULL and either [SQLITE_ENABLE_MEMSYS3] or +** [SQLITE_ENABLE_MEMSYS5] are defined, then the alternative memory +** allocator is engaged to handle all of SQLites memory allocation needs.
+** +**
SQLITE_CONFIG_MUTEX
+**
This option takes a single argument which is a pointer to an +** instance of the [sqlite3_mutex_methods] structure. The argument specifies +** alternative low-level mutex routines to be used in place +** the mutex routines built into SQLite.
+** +**
SQLITE_CONFIG_GETMUTEX
+**
This option takes a single argument which is a pointer to an +** instance of the [sqlite3_mutex_methods] structure. The +** [sqlite3_mutex_methods] +** structure is filled with the currently defined mutex routines. +** This option can be used to overload the default mutex allocation +** routines with a wrapper used to track mutex usage for performance +** profiling or testing, for example.
+** +**
SQLITE_CONFIG_LOOKASIDE
+**
This option takes two arguments that determine the default +** memory allcation lookaside optimization. The first argument is the +** size of each lookaside buffer slot and the second is the number of +** slots allocated to each database connection.
+** +**
SQLITE_CONFIG_PCACHE
+**
This option takes a single argument which is a pointer to +** an [sqlite3_pcache_methods] object. This object specifies the interface +** to a custom page cache implementation. SQLite makes a copy of the +** object and uses it for page cache memory allocations.
+** +**
SQLITE_CONFIG_GETPCACHE
+**
This option takes a single argument which is a pointer to an +** [sqlite3_pcache_methods] object. SQLite copies of the current +** page cache implementation into that object.
+** +**
+*/ +#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ +#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ +#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_SCRATCH 6 /* void*, int sz, int N */ +#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ +#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ +#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ +#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ +#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ +#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ +#define SQLITE_CONFIG_PCACHE 14 /* sqlite3_pcache_methods* */ +#define SQLITE_CONFIG_GETPCACHE 15 /* sqlite3_pcache_methods* */ + +/* +** CAPI3REF: Configuration Options {H10170} +** EXPERIMENTAL +** +** These constants are the available integer configuration options that +** can be passed as the second argument to the [sqlite3_db_config()] interface. +** +** New configuration options may be added in future releases of SQLite. +** Existing configuration options might be discontinued. Applications +** should check the return code from [sqlite3_db_config()] to make sure that +** the call worked. The [sqlite3_db_config()] interface will return a +** non-zero [error code] if a discontinued or unsupported configuration option +** is invoked. +** +**
+**
SQLITE_DBCONFIG_LOOKASIDE
+**
This option takes three additional arguments that determine the +** [lookaside memory allocator] configuration for the [database connection]. +** The first argument (the third parameter to [sqlite3_db_config()] is a +** pointer to a memory buffer to use for lookaside memory. The first +** argument may be NULL in which case SQLite will allocate the lookaside +** buffer itself using [sqlite3_malloc()]. The second argument is the +** size of each lookaside buffer slot and the third argument is the number of +** slots. The size of the buffer in the first argument must be greater than +** or equal to the product of the second and third arguments.
+** +**
+*/ +#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ + + +/* +** CAPI3REF: Enable Or Disable Extended Result Codes {H12200} +** +** The sqlite3_extended_result_codes() routine enables or disables the +** [extended result codes] feature of SQLite. The extended result +** codes are disabled by default for historical compatibility considerations. +** +** INVARIANTS: +** +** {H12201} Each new [database connection] shall have the +** [extended result codes] feature disabled by default. +** +** {H12202} The [sqlite3_extended_result_codes(D,F)] interface shall enable +** [extended result codes] for the [database connection] D +** if the F parameter is true, or disable them if F is false. */ int sqlite3_extended_result_codes(sqlite3*, int onoff); /* -** CAPI3REF: Last Insert Rowid +** CAPI3REF: Last Insert Rowid {H12220} ** -** Each entry in an SQLite table has a unique 64-bit signed integer key -** called the "rowid". The rowid is always available as an undeclared -** column named ROWID, OID, or _ROWID_. If the table has a column of -** type INTEGER PRIMARY KEY then that column is another an alias for the -** rowid. +** Each entry in an SQLite table has a unique 64-bit signed +** integer key called the [ROWID | "rowid"]. The rowid is always available +** as an undeclared column named ROWID, OID, or _ROWID_ as long as those +** names are not also used by explicitly declared columns. If +** the table has a column of type [INTEGER PRIMARY KEY] then that column +** is another alias for the rowid. ** -** This routine returns the rowid of the most recent INSERT into -** the database from the database connection given in the first -** argument. If no inserts have ever occurred on this database -** connection, zero is returned. +** This routine returns the [rowid] of the most recent +** successful [INSERT] into the database from the [database connection] +** in the first argument. If no successful [INSERT]s +** have ever occurred on that database connection, zero is returned. ** -** If an INSERT occurs within a trigger, then the rowid of the -** inserted row is returned by this routine as long as the trigger -** is running. But once the trigger terminates, the value returned -** by this routine reverts to the last value inserted before the -** trigger fired. +** If an [INSERT] occurs within a trigger, then the [rowid] of the inserted +** row is returned by this routine as long as the trigger is running. +** But once the trigger terminates, the value returned by this routine +** reverts to the last value inserted before the trigger fired. +** +** An [INSERT] that fails due to a constraint violation is not a +** successful [INSERT] and does not change the value returned by this +** routine. Thus INSERT OR FAIL, INSERT OR IGNORE, INSERT OR ROLLBACK, +** and INSERT OR ABORT make no changes to the return value of this +** routine when their insertion fails. When INSERT OR REPLACE +** encounters a constraint violation, it does not fail. The +** INSERT continues to completion after deleting rows that caused +** the constraint problem so INSERT OR REPLACE will always change +** the return value of this interface. +** +** For the purposes of this routine, an [INSERT] is considered to +** be successful even if it is subsequently rolled back. +** +** INVARIANTS: +** +** {H12221} The [sqlite3_last_insert_rowid()] function shall return +** the [rowid] +** of the most recent successful [INSERT] performed on the same +** [database connection] and within the same or higher level +** trigger context, or zero if there have been no qualifying +** [INSERT] statements. +** +** {H12223} The [sqlite3_last_insert_rowid()] function shall return the +** same value when called from the same trigger context +** immediately before and after a [ROLLBACK]. +** +** ASSUMPTIONS: +** +** {A12232} If a separate thread performs a new [INSERT] on the same +** database connection while the [sqlite3_last_insert_rowid()] +** function is running and thus changes the last insert [rowid], +** then the value returned by [sqlite3_last_insert_rowid()] is +** unpredictable and might not equal either the old or the new +** last insert [rowid]. */ -sqlite_int64 sqlite3_last_insert_rowid(sqlite3*); +sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); /* -** CAPI3REF: Count The Number Of Rows Modified +** CAPI3REF: Count The Number Of Rows Modified {H12240} ** ** This function returns the number of database rows that were changed -** (or inserted or deleted) by the most recent SQL statement. Only -** changes that are directly specified by the INSERT, UPDATE, or -** DELETE statement are counted. Auxiliary changes caused by -** triggers are not counted. Use the [sqlite3_total_changes()] function +** or inserted or deleted by the most recently completed SQL statement +** on the [database connection] specified by the first parameter. +** Only changes that are directly specified by the [INSERT], [UPDATE], +** or [DELETE] statement are counted. Auxiliary changes caused by +** triggers are not counted. Use the [sqlite3_total_changes()] function ** to find the total number of changes including changes caused by triggers. ** -** Within the body of a trigger, the sqlite3_changes() interface can be -** called to find the number of +** A "row change" is a change to a single row of a single table +** caused by an INSERT, DELETE, or UPDATE statement. Rows that +** are changed as side effects of REPLACE constraint resolution, +** rollback, ABORT processing, DROP TABLE, or by any other +** mechanisms do not count as direct row changes. +** +** A "trigger context" is a scope of execution that begins and +** ends with the script of a trigger. Most SQL statements are +** evaluated outside of any trigger. This is the "top level" +** trigger context. If a trigger fires from the top level, a +** new trigger context is entered for the duration of that one +** trigger. Subtriggers create subcontexts for their duration. +** +** Calling [sqlite3_exec()] or [sqlite3_step()] recursively does +** not create a new trigger context. +** +** This function returns the number of direct row changes in the +** most recent INSERT, UPDATE, or DELETE statement within the same +** trigger context. +** +** Thus, when called from the top level, this function returns the +** number of changes in the most recent INSERT, UPDATE, or DELETE +** that also occurred at the top level. Within the body of a trigger, +** the sqlite3_changes() interface can be called to find the number of ** changes in the most recently completed INSERT, UPDATE, or DELETE -** statement within the body of the trigger. -** -** All changes are counted, even if they were later undone by a -** ROLLBACK or ABORT. Except, changes associated with creating and -** dropping tables are not counted. -** -** If a callback invokes [sqlite3_exec()] or [sqlite3_step()] recursively, -** then the changes in the inner, recursive call are counted together -** with the changes in the outer call. +** statement within the body of the same trigger. +** However, the number returned does not include changes +** caused by subtriggers since those have their own context. ** ** SQLite implements the command "DELETE FROM table" without a WHERE clause -** by dropping and recreating the table. (This is much faster than going -** through and deleting individual elements from the table.) Because of -** this optimization, the change count for "DELETE FROM table" will be -** zero regardless of the number of elements that were originally in the -** table. To get an accurate count of the number of rows deleted, use -** "DELETE FROM table WHERE 1" instead. +** by dropping and recreating the table. Doing so is much faster than going +** through and deleting individual elements from the table. Because of this +** optimization, the deletions in "DELETE FROM table" are not row changes and +** will not be counted by the sqlite3_changes() or [sqlite3_total_changes()] +** functions, regardless of the number of elements that were originally +** in the table. To get an accurate count of the number of rows deleted, use +** "DELETE FROM table WHERE 1" instead. Or recompile using the +** [SQLITE_OMIT_TRUNCATE_OPTIMIZATION] compile-time option to disable the +** optimization on all queries. +** +** INVARIANTS: +** +** {H12241} The [sqlite3_changes()] function shall return the number of +** row changes caused by the most recent INSERT, UPDATE, +** or DELETE statement on the same database connection and +** within the same or higher trigger context, or zero if there have +** not been any qualifying row changes. +** +** {H12243} Statements of the form "DELETE FROM tablename" with no +** WHERE clause shall cause subsequent calls to +** [sqlite3_changes()] to return zero, regardless of the +** number of rows originally in the table. +** +** ASSUMPTIONS: +** +** {A12252} If a separate thread makes changes on the same database connection +** while [sqlite3_changes()] is running then the value returned +** is unpredictable and not meaningful. */ int sqlite3_changes(sqlite3*); /* -** CAPI3REF: Total Number Of Rows Modified -*** -** This function returns the number of database rows that have been -** modified by INSERT, UPDATE or DELETE statements since the database handle -** was opened. This includes UPDATE, INSERT and DELETE statements executed -** as part of trigger programs. All changes are counted as soon as the -** statement that makes them is completed (when the statement handle is -** passed to [sqlite3_reset()] or [sqlite_finalise()]). +** CAPI3REF: Total Number Of Rows Modified {H12260} ** -** See also the [sqlite3_change()] interface. +** This function returns the number of row changes caused by INSERT, +** UPDATE or DELETE statements since the [database connection] was opened. +** The count includes all changes from all trigger contexts. However, +** the count does not include changes used to implement REPLACE constraints, +** do rollbacks or ABORT processing, or DROP table processing. +** The changes are counted as soon as the statement that makes them is +** completed (when the statement handle is passed to [sqlite3_reset()] or +** [sqlite3_finalize()]). ** ** SQLite implements the command "DELETE FROM table" without a WHERE clause ** by dropping and recreating the table. (This is much faster than going -** through and deleting individual elements form the table.) Because of -** this optimization, the change count for "DELETE FROM table" will be -** zero regardless of the number of elements that were originally in the -** table. To get an accurate count of the number of rows deleted, use -** "DELETE FROM table WHERE 1" instead. +** through and deleting individual elements from the table.) Because of this +** optimization, the deletions in "DELETE FROM table" are not row changes and +** will not be counted by the sqlite3_changes() or [sqlite3_total_changes()] +** functions, regardless of the number of elements that were originally +** in the table. To get an accurate count of the number of rows deleted, use +** "DELETE FROM table WHERE 1" instead. Or recompile using the +** [SQLITE_OMIT_TRUNCATE_OPTIMIZATION] compile-time option to disable the +** optimization on all queries. +** +** See also the [sqlite3_changes()] interface. +** +** INVARIANTS: +** +** {H12261} The [sqlite3_total_changes()] returns the total number +** of row changes caused by INSERT, UPDATE, and/or DELETE +** statements on the same [database connection], in any +** trigger context, since the database connection was created. +** +** {H12263} Statements of the form "DELETE FROM tablename" with no +** WHERE clause shall not change the value returned +** by [sqlite3_total_changes()]. +** +** ASSUMPTIONS: +** +** {A12264} If a separate thread makes changes on the same database connection +** while [sqlite3_total_changes()] is running then the value +** returned is unpredictable and not meaningful. */ int sqlite3_total_changes(sqlite3*); /* -** CAPI3REF: Interrupt A Long-Running Query +** CAPI3REF: Interrupt A Long-Running Query {H12270} ** ** This function causes any pending database operation to abort and -** return at its earliest opportunity. This routine is typically +** return at its earliest opportunity. This routine is typically ** called in response to a user action such as pressing "Cancel" ** or Ctrl-C where the user wants a long query operation to halt ** immediately. ** ** It is safe to call this routine from a thread different from the -** thread that is currently running the database operation. +** thread that is currently running the database operation. But it +** is not safe to call this routine with a [database connection] that +** is closed or might close before sqlite3_interrupt() returns. ** -** The SQL operation that is interrupted will return [SQLITE_INTERRUPT]. -** If an interrupted operation was an update that is inside an -** explicit transaction, then the entire transaction will be rolled -** back automatically. +** If an SQL operation is very nearly finished at the time when +** sqlite3_interrupt() is called, then it might not have an opportunity +** to be interrupted and might continue to completion. +** +** An SQL operation that is interrupted will return [SQLITE_INTERRUPT]. +** If the interrupted SQL operation is an INSERT, UPDATE, or DELETE +** that is inside an explicit transaction, then the entire transaction +** will be rolled back automatically. +** +** A call to sqlite3_interrupt() has no effect on SQL statements +** that are started after sqlite3_interrupt() returns. +** +** INVARIANTS: +** +** {H12271} The [sqlite3_interrupt()] interface will force all running +** SQL statements associated with the same database connection +** to halt after processing at most one additional row of data. +** +** {H12272} Any SQL statement that is interrupted by [sqlite3_interrupt()] +** will return [SQLITE_INTERRUPT]. +** +** ASSUMPTIONS: +** +** {A12279} If the database connection closes while [sqlite3_interrupt()] +** is running then bad things will likely happen. */ void sqlite3_interrupt(sqlite3*); /* -** CAPI3REF: Determine If An SQL Statement Is Complete -** -** These functions return true if the given input string comprises -** one or more complete SQL statements. For the sqlite3_complete() call, -** the parameter must be a nul-terminated UTF-8 string. For -** sqlite3_complete16(), a nul-terminated machine byte order UTF-16 string -** is required. +** CAPI3REF: Determine If An SQL Statement Is Complete {H10510} ** ** These routines are useful for command-line input to determine if the -** currently entered text forms one or more complete SQL statements or -** if additional input is needed before sending the statements into -** SQLite for parsing. The algorithm is simple. If the -** last token other than spaces and comments is a semicolon, then return -** true. Actually, the algorithm is a little more complicated than that -** in order to deal with triggers, but the basic idea is the same: the -** statement is not complete unless it ends in a semicolon. +** currently entered text seems to form complete a SQL statement or +** if additional input is needed before sending the text into +** SQLite for parsing. These routines return true if the input string +** appears to be a complete SQL statement. A statement is judged to be +** complete if it ends with a semicolon token and is not a fragment of a +** CREATE TRIGGER statement. Semicolons that are embedded within +** string literals or quoted identifier names or comments are not +** independent tokens (they are part of the token in which they are +** embedded) and thus do not count as a statement terminator. +** +** These routines do not parse the SQL statements thus +** will not detect syntactically incorrect SQL. +** +** INVARIANTS: +** +** {H10511} A successful evaluation of [sqlite3_complete()] or +** [sqlite3_complete16()] functions shall +** return a numeric 1 if and only if the last non-whitespace +** token in their input is a semicolon that is not in between +** the BEGIN and END of a CREATE TRIGGER statement. +** +** {H10512} If a memory allocation error occurs during an invocation +** of [sqlite3_complete()] or [sqlite3_complete16()] then the +** routine shall return [SQLITE_NOMEM]. +** +** ASSUMPTIONS: +** +** {A10512} The input to [sqlite3_complete()] must be a zero-terminated +** UTF-8 string. +** +** {A10513} The input to [sqlite3_complete16()] must be a zero-terminated +** UTF-16 string in native byte order. */ int sqlite3_complete(const char *sql); int sqlite3_complete16(const void *sql); /* -** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors +** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors {H12310} ** -** This routine identifies a callback function that might be invoked -** whenever an attempt is made to open a database table -** that another thread or process has locked. -** If the busy callback is NULL, then [SQLITE_BUSY] -** (or sometimes [SQLITE_IOERR_BLOCKED]) -** is returned immediately upon encountering the lock. -** If the busy callback is not NULL, then the -** callback will be invoked with two arguments. The -** first argument to the handler is a copy of the void* pointer which -** is the third argument to this routine. The second argument to -** the handler is the number of times that the busy handler has -** been invoked for this locking event. If the +** This routine sets a callback function that might be invoked whenever +** an attempt is made to open a database table that another thread +** or process has locked. +** +** If the busy callback is NULL, then [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] +** is returned immediately upon encountering the lock. If the busy callback +** is not NULL, then the callback will be invoked with two arguments. +** +** The first argument to the handler is a copy of the void* pointer which +** is the third argument to sqlite3_busy_handler(). The second argument to +** the handler callback is the number of times that the busy handler has +** been invoked for this locking event. If the ** busy callback returns 0, then no additional attempts are made to ** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned. -** If the callback returns non-zero, then another attempt is made to open the -** database for reading and the cycle repeats. +** If the callback returns non-zero, then another attempt +** is made to open the database for reading and the cycle repeats. ** -** The presence of a busy handler does not guarantee that -** it will be invoked when there is lock contention. -** If SQLite determines that invoking the busy handler could result in -** a deadlock, it will return [SQLITE_BUSY] instead. +** The presence of a busy handler does not guarantee that it will be invoked +** when there is lock contention. If SQLite determines that invoking the busy +** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] +** or [SQLITE_IOERR_BLOCKED] instead of invoking the busy handler. ** Consider a scenario where one process is holding a read lock that ** it is trying to promote to a reserved lock and ** a second process is holding a reserved lock that it is trying @@ -483,8 +1698,8 @@ int sqlite3_complete16(const void *sql); ** ** The default busy callback is NULL. ** -** The [SQLITE_BUSY] error is converted to [SQLITE_IOERR_BLOCKED] when -** SQLite is in the middle of a large transaction where all the +** The [SQLITE_BUSY] error is converted to [SQLITE_IOERR_BLOCKED] +** when SQLite is in the middle of a large transaction where all the ** changes will not fit into the in-memory cache. SQLite will ** already hold a RESERVED lock on the database file, but it needs ** to promote this lock to EXCLUSIVE so that it can spill cache @@ -493,109 +1708,204 @@ int sqlite3_complete16(const void *sql); ** cache will be left in an inconsistent state and so the error ** code is promoted from the relatively benign [SQLITE_BUSY] to ** the more severe [SQLITE_IOERR_BLOCKED]. This error code promotion -** forces an automatic rollback of the changes. See the -** +** forces an automatic rollback of the changes. See the +** ** CorruptionFollowingBusyError wiki page for a discussion of why ** this is important. -** -** Sqlite is re-entrant, so the busy handler may start a new query. -** (It is not clear why anyone would every want to do this, but it -** is allowed, in theory.) But the busy handler may not close the -** database. Closing the database from a busy handler will delete -** data structures out from under the executing query and will -** probably result in a segmentation fault or other runtime error. ** -** There can only be a single busy handler defined for each database -** connection. Setting a new busy handler clears any previous one. -** Note that calling [sqlite3_busy_timeout()] will also set or clear -** the busy handler. +** There can only be a single busy handler defined for each +** [database connection]. Setting a new busy handler clears any +** previously set handler. Note that calling [sqlite3_busy_timeout()] +** will also set or clear the busy handler. +** +** The busy callback should not take any actions which modify the +** database connection that invoked the busy handler. Any such actions +** result in undefined behavior. +** +** INVARIANTS: +** +** {H12311} The [sqlite3_busy_handler(D,C,A)] function shall replace +** busy callback in the [database connection] D with a new +** a new busy handler C and application data pointer A. +** +** {H12312} Newly created [database connections] shall have a busy +** handler of NULL. +** +** {H12314} When two or more [database connections] share a +** [sqlite3_enable_shared_cache | common cache], +** the busy handler for the database connection currently using +** the cache shall be invoked when the cache encounters a lock. +** +** {H12316} If a busy handler callback returns zero, then the SQLite interface +** that provoked the locking event shall return [SQLITE_BUSY]. +** +** {H12318} SQLite shall invokes the busy handler with two arguments which +** are a copy of the pointer supplied by the 3rd parameter to +** [sqlite3_busy_handler()] and a count of the number of prior +** invocations of the busy handler for the same locking event. +** +** ASSUMPTIONS: +** +** {A12319} A busy handler must not close the database connection +** or [prepared statement] that invoked the busy handler. */ int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); /* -** CAPI3REF: Set A Busy Timeout +** CAPI3REF: Set A Busy Timeout {H12340} ** -** This routine sets a busy handler that sleeps for a while when a -** table is locked. The handler will sleep multiple times until -** at least "ms" milliseconds of sleeping have been done. After -** "ms" milliseconds of sleeping, the handler returns 0 which -** causes [sqlite3_step()] to return [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED]. +** This routine sets a [sqlite3_busy_handler | busy handler] that sleeps +** for a specified amount of time when a table is locked. The handler +** will sleep multiple times until at least "ms" milliseconds of sleeping +** have accumulated. {H12343} After "ms" milliseconds of sleeping, +** the handler returns 0 which causes [sqlite3_step()] to return +** [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED]. ** ** Calling this routine with an argument less than or equal to zero ** turns off all busy handlers. ** -** There can only be a single busy handler for a particular database -** connection. If another busy handler was defined -** (using [sqlite3_busy_handler()]) prior to calling +** There can only be a single busy handler for a particular +** [database connection] any any given moment. If another busy handler +** was defined (using [sqlite3_busy_handler()]) prior to calling ** this routine, that other busy handler is cleared. +** +** INVARIANTS: +** +** {H12341} The [sqlite3_busy_timeout()] function shall override any prior +** [sqlite3_busy_timeout()] or [sqlite3_busy_handler()] setting +** on the same [database connection]. +** +** {H12343} If the 2nd parameter to [sqlite3_busy_timeout()] is less than +** or equal to zero, then the busy handler shall be cleared so that +** all subsequent locking events immediately return [SQLITE_BUSY]. +** +** {H12344} If the 2nd parameter to [sqlite3_busy_timeout()] is a positive +** number N, then a busy handler shall be set that repeatedly calls +** the xSleep() method in the [sqlite3_vfs | VFS interface] until +** either the lock clears or until the cumulative sleep time +** reported back by xSleep() exceeds N milliseconds. */ int sqlite3_busy_timeout(sqlite3*, int ms); /* -** CAPI3REF: Convenience Routines For Running Queries +** CAPI3REF: Convenience Routines For Running Queries {H12370} ** -** This next routine is a convenience wrapper around [sqlite3_exec()]. -** Instead of invoking a user-supplied callback for each row of the -** result, this routine remembers each row of the result in memory -** obtained from [sqlite3_malloc()], then returns all of the result after the -** query has finished. +** Definition: A result table is memory data structure created by the +** [sqlite3_get_table()] interface. A result table records the +** complete query results from one or more queries. ** -** As an example, suppose the query result where this table: +** The table conceptually has a number of rows and columns. But +** these numbers are not part of the result table itself. These +** numbers are obtained separately. Let N be the number of rows +** and M be the number of columns. ** -** 
+** A result table is an array of pointers to zero-terminated UTF-8 strings.
+** There are (N+1)*M elements in the array.  The first M pointers point
+** to zero-terminated strings that  contain the names of the columns.
+** The remaining entries all point to query results.  NULL values result
+** in NULL pointers.  All other values are in their UTF-8 zero-terminated
+** string representation as returned by [sqlite3_column_text()].
+**
+** A result table might consist of one or more memory allocations.
+** It is not safe to pass a result table directly to [sqlite3_free()].
+** A result table should be deallocated using [sqlite3_free_table()].
+**
+** As an example of the result table format, suppose a query result
+** is as follows:
+**
+** 
 **        Name        | Age
 **        -----------------------
 **        Alice       | 43
 **        Bob         | 28
 **        Cindy       | 21
-** 

+**
** -** If the 3rd argument were &azResult then after the function returns -** azResult will contain the following data: +** There are two column (M==2) and three rows (N==3). Thus the +** result table has 8 entries. Suppose the result table is stored +** in an array names azResult. Then azResult holds this content: ** -** 
-**        azResult[0] = "Name";
-**        azResult[1] = "Age";
-**        azResult[2] = "Alice";
-**        azResult[3] = "43";
-**        azResult[4] = "Bob";
-**        azResult[5] = "28";
-**        azResult[6] = "Cindy";
-**        azResult[7] = "21";
-**
+** 
+**        azResult[0] = "Name";
+**        azResult[1] = "Age";
+**        azResult[2] = "Alice";
+**        azResult[3] = "43";
+**        azResult[4] = "Bob";
+**        azResult[5] = "28";
+**        azResult[6] = "Cindy";
+**        azResult[7] = "21";
+**
** -** Notice that there is an extra row of data containing the column -** headers. But the *nrow return value is still 3. *ncolumn is -** set to 2. In general, the number of values inserted into azResult -** will be ((*nrow) + 1)*(*ncolumn). +** The sqlite3_get_table() function evaluates one or more +** semicolon-separated SQL statements in the zero-terminated UTF-8 +** string of its 2nd parameter. It returns a result table to the +** pointer given in its 3rd parameter. ** -** After the calling function has finished using the result, it should -** pass the result data pointer to sqlite3_free_table() in order to -** release the memory that was malloc-ed. Because of the way the -** [sqlite3_malloc()] happens, the calling function must not try to call -** [sqlite3_free()] directly. Only [sqlite3_free_table()] is able to release -** the memory properly and safely. +** After the calling function has finished using the result, it should +** pass the pointer to the result table to sqlite3_free_table() in order to +** release the memory that was malloced. Because of the way the +** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling +** function must not try to call [sqlite3_free()] directly. Only +** [sqlite3_free_table()] is able to release the memory properly and safely. ** -** The return value of this routine is the same as from [sqlite3_exec()]. +** The sqlite3_get_table() interface is implemented as a wrapper around +** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access +** to any internal data structures of SQLite. It uses only the public +** interface defined here. As a consequence, errors that occur in the +** wrapper layer outside of the internal [sqlite3_exec()] call are not +** reflected in subsequent calls to [sqlite3_errcode()] or [sqlite3_errmsg()]. +** +** INVARIANTS: +** +** {H12371} If a [sqlite3_get_table()] fails a memory allocation, then +** it shall free the result table under construction, abort the +** query in process, skip any subsequent queries, set the +** *pazResult output pointer to NULL and return [SQLITE_NOMEM]. +** +** {H12373} If the pnColumn parameter to [sqlite3_get_table()] is not NULL +** then a successful invocation of [sqlite3_get_table()] shall +** write the number of columns in the +** result set of the query into *pnColumn. +** +** {H12374} If the pnRow parameter to [sqlite3_get_table()] is not NULL +** then a successful invocation of [sqlite3_get_table()] shall +** writes the number of rows in the +** result set of the query into *pnRow. +** +** {H12376} A successful invocation of [sqlite3_get_table()] that computes +** N rows of result with C columns per row shall make *pazResult +** point to an array of pointers to (N+1)*C strings where the first +** C strings are column names as obtained from +** [sqlite3_column_name()] and the rest are column result values +** obtained from [sqlite3_column_text()]. +** +** {H12379} The values in the pazResult array returned by [sqlite3_get_table()] +** shall remain valid until cleared by [sqlite3_free_table()]. +** +** {H12382} When an error occurs during evaluation of [sqlite3_get_table()] +** the function shall set *pazResult to NULL, write an error message +** into memory obtained from [sqlite3_malloc()], make +** **pzErrmsg point to that error message, and return a +** appropriate [error code]. */ int sqlite3_get_table( - sqlite3*, /* An open database */ - const char *sql, /* SQL to be executed */ - char ***resultp, /* Result written to a char *[] that this points to */ - int *nrow, /* Number of result rows written here */ - int *ncolumn, /* Number of result columns written here */ - char **errmsg /* Error msg written here */ + sqlite3 *db, /* An open database */ + const char *zSql, /* SQL to be evaluated */ + char ***pazResult, /* Results of the query */ + int *pnRow, /* Number of result rows written here */ + int *pnColumn, /* Number of result columns written here */ + char **pzErrmsg /* Error msg written here */ ); void sqlite3_free_table(char **result); /* -** CAPI3REF: Formatted String Printing Functions +** CAPI3REF: Formatted String Printing Functions {H17400} ** ** These routines are workalikes of the "printf()" family of functions ** from the standard C library. ** ** The sqlite3_mprintf() and sqlite3_vmprintf() routines write their -** results into memory obtained from [sqlite_malloc()]. +** results into memory obtained from [sqlite3_malloc()]. ** The strings returned by these two routines should be ** released by [sqlite3_free()]. Both routines return a ** NULL pointer if [sqlite3_malloc()] is unable to allocate enough @@ -604,7 +1914,7 @@ void sqlite3_free_table(char **result); ** In sqlite3_snprintf() routine is similar to "snprintf()" from ** the standard C library. The result is written into the ** buffer supplied as the second parameter whose size is given by -** the first parameter. Note that the order of the +** the first parameter. Note that the order of the ** first two parameters is reversed from snprintf(). This is an ** historical accident that cannot be fixed without breaking ** backwards compatibility. Note also that sqlite3_snprintf() @@ -622,8 +1932,8 @@ void sqlite3_free_table(char **result); ** ** These routines all implement some additional formatting ** options that are useful for constructing SQL statements. -** All of the usual printf formatting options apply. In addition, there -** is are "%q" and "%Q" options. +** All of the usual printf() formatting options apply. In addition, there +** is are "%q", "%Q", and "%z" options. ** ** The %q option works like %s in that it substitutes a null-terminated ** string from the argument list. But %q also doubles every '\'' character. @@ -631,7 +1941,7 @@ void sqlite3_free_table(char **result); ** character it escapes that character and allows it to be inserted into ** the string. ** -** For example, so some string variable contains text as follows: +** For example, assume the string variable zText contains text as follows: ** ** 
 **  char *zText = "It's a happy day!";
@@ -659,14 +1969,13 @@ void sqlite3_free_table(char **result);
 **  INSERT INTO table1 VALUES('It's a happy day!');
 **
** -** This second example is an SQL syntax error. As a general rule you -** should always use %q instead of %s when inserting text into a string -** literal. +** This second example is an SQL syntax error. As a general rule you should +** always use %q instead of %s when inserting text into a string literal. ** ** The %Q option works like %q except it also adds single quotes around -** the outside of the total string. Or if the parameter in the argument -** list is a NULL pointer, %Q substitutes the text "NULL" (without single -** quotes) in place of the %Q option. So, for example, one could say: +** the outside of the total string. Additionally, if the parameter in the +** argument list is a NULL pointer, %Q substitutes the text "NULL" (without +** single quotes) in place of the %Q option. So, for example, one could say: ** ** 
 **  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES(%Q)", zText);
@@ -676,76 +1985,336 @@ void sqlite3_free_table(char **result);
 **
 ** The code above will render a correct SQL statement in the zSQL
 ** variable even if the zText variable is a NULL pointer.
+**
+** The "%z" formatting option works exactly like "%s" with the
+** addition that after the string has been read and copied into
+** the result, [sqlite3_free()] is called on the input string. {END}
+**
+** INVARIANTS:
+**
+** {H17403}  The [sqlite3_mprintf()] and [sqlite3_vmprintf()] interfaces
+**           return either pointers to zero-terminated UTF-8 strings held in
+**           memory obtained from [sqlite3_malloc()] or NULL pointers if
+**           a call to [sqlite3_malloc()] fails.
+**
+** {H17406}  The [sqlite3_snprintf()] interface writes a zero-terminated
+**           UTF-8 string into the buffer pointed to by the second parameter
+**           provided that the first parameter is greater than zero.
+**
+** {H17407}  The [sqlite3_snprintf()] interface does not write slots of
+**           its output buffer (the second parameter) outside the range
+**           of 0 through N-1 (where N is the first parameter)
+**           regardless of the length of the string
+**           requested by the format specification.
 */
 char *sqlite3_mprintf(const char*,...);
 char *sqlite3_vmprintf(const char*, va_list);
 char *sqlite3_snprintf(int,char*,const char*, ...);
 
 /*
-** CAPI3REF: Memory Allocation Functions
+** CAPI3REF: Memory Allocation Subsystem {H17300} 
 **
-** SQLite uses its own memory allocator.  On some installations, this
-** memory allocator is identical to the standard malloc()/realloc()/free()
-** and can be used interchangable.  On others, the implementations are
-** different.  For maximum portability, it is best not to mix calls
-** to the standard malloc/realloc/free with the sqlite versions.
+** The SQLite core  uses these three routines for all of its own
+** internal memory allocation needs. "Core" in the previous sentence
+** does not include operating-system specific VFS implementation.  The
+** Windows VFS uses native malloc() and free() for some operations.
+**
+** The sqlite3_malloc() routine returns a pointer to a block
+** of memory at least N bytes in length, where N is the parameter.
+** If sqlite3_malloc() is unable to obtain sufficient free
+** memory, it returns a NULL pointer.  If the parameter N to
+** sqlite3_malloc() is zero or negative then sqlite3_malloc() returns
+** a NULL pointer.
+**
+** Calling sqlite3_free() with a pointer previously returned
+** by sqlite3_malloc() or sqlite3_realloc() releases that memory so
+** that it might be reused.  The sqlite3_free() routine is
+** a no-op if is called with a NULL pointer.  Passing a NULL pointer
+** to sqlite3_free() is harmless.  After being freed, memory
+** should neither be read nor written.  Even reading previously freed
+** memory might result in a segmentation fault or other severe error.
+** Memory corruption, a segmentation fault, or other severe error
+** might result if sqlite3_free() is called with a non-NULL pointer that
+** was not obtained from sqlite3_malloc() or sqlite3_realloc().
+**
+** The sqlite3_realloc() interface attempts to resize a
+** prior memory allocation to be at least N bytes, where N is the
+** second parameter.  The memory allocation to be resized is the first
+** parameter.  If the first parameter to sqlite3_realloc()
+** is a NULL pointer then its behavior is identical to calling
+** sqlite3_malloc(N) where N is the second parameter to sqlite3_realloc().
+** If the second parameter to sqlite3_realloc() is zero or
+** negative then the behavior is exactly the same as calling
+** sqlite3_free(P) where P is the first parameter to sqlite3_realloc().
+** sqlite3_realloc() returns a pointer to a memory allocation
+** of at least N bytes in size or NULL if sufficient memory is unavailable.
+** If M is the size of the prior allocation, then min(N,M) bytes
+** of the prior allocation are copied into the beginning of buffer returned
+** by sqlite3_realloc() and the prior allocation is freed.
+** If sqlite3_realloc() returns NULL, then the prior allocation
+** is not freed.
+**
+** The memory returned by sqlite3_malloc() and sqlite3_realloc()
+** is always aligned to at least an 8 byte boundary. {END}
+**
+** The default implementation of the memory allocation subsystem uses
+** the malloc(), realloc() and free() provided by the standard C library.
+** {H17382} However, if SQLite is compiled with the
+** SQLITE_MEMORY_SIZE=NNN C preprocessor macro (where NNN
+** is an integer), then SQLite create a static array of at least
+** NNN bytes in size and uses that array for all of its dynamic
+** memory allocation needs. {END}  Additional memory allocator options
+** may be added in future releases.
+**
+** In SQLite version 3.5.0 and 3.5.1, it was possible to define
+** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in
+** implementation of these routines to be omitted.  That capability
+** is no longer provided.  Only built-in memory allocators can be used.
+**
+** The Windows OS interface layer calls
+** the system malloc() and free() directly when converting
+** filenames between the UTF-8 encoding used by SQLite
+** and whatever filename encoding is used by the particular Windows
+** installation.  Memory allocation errors are detected, but
+** they are reported back as [SQLITE_CANTOPEN] or
+** [SQLITE_IOERR] rather than [SQLITE_NOMEM].
+**
+** INVARIANTS:
+**
+** {H17303}  The [sqlite3_malloc(N)] interface returns either a pointer to
+**           a newly checked-out block of at least N bytes of memory
+**           that is 8-byte aligned, or it returns NULL if it is unable
+**           to fulfill the request.
+**
+** {H17304}  The [sqlite3_malloc(N)] interface returns a NULL pointer if
+**           N is less than or equal to zero.
+**
+** {H17305}  The [sqlite3_free(P)] interface releases memory previously
+**           returned from [sqlite3_malloc()] or [sqlite3_realloc()],
+**           making it available for reuse.
+**
+** {H17306}  A call to [sqlite3_free(NULL)] is a harmless no-op.
+**
+** {H17310}  A call to [sqlite3_realloc(0,N)] is equivalent to a call
+**           to [sqlite3_malloc(N)].
+**
+** {H17312}  A call to [sqlite3_realloc(P,0)] is equivalent to a call
+**           to [sqlite3_free(P)].
+**
+** {H17315}  The SQLite core uses [sqlite3_malloc()], [sqlite3_realloc()],
+**           and [sqlite3_free()] for all of its memory allocation and
+**           deallocation needs.
+**
+** {H17318}  The [sqlite3_realloc(P,N)] interface returns either a pointer
+**           to a block of checked-out memory of at least N bytes in size
+**           that is 8-byte aligned, or a NULL pointer.
+**
+** {H17321}  When [sqlite3_realloc(P,N)] returns a non-NULL pointer, it first
+**           copies the first K bytes of content from P into the newly
+**           allocated block, where K is the lesser of N and the size of
+**           the buffer P.
+**
+** {H17322}  When [sqlite3_realloc(P,N)] returns a non-NULL pointer, it first
+**           releases the buffer P.
+**
+** {H17323}  When [sqlite3_realloc(P,N)] returns NULL, the buffer P is
+**           not modified or released.
+**
+** ASSUMPTIONS:
+**
+** {A17350}  The pointer arguments to [sqlite3_free()] and [sqlite3_realloc()]
+**           must be either NULL or else pointers obtained from a prior
+**           invocation of [sqlite3_malloc()] or [sqlite3_realloc()] that have
+**           not yet been released.
+**
+** {A17351}  The application must not read or write any part of
+**           a block of memory after it has been released using
+**           [sqlite3_free()] or [sqlite3_realloc()].
 */
 void *sqlite3_malloc(int);
 void *sqlite3_realloc(void*, int);
 void sqlite3_free(void*);
 
 /*
-** CAPI3REF: Compile-Time Authorization Callbacks
-***
-** This routine registers a authorizer callback with the SQLite library.  
+** CAPI3REF: Memory Allocator Statistics {H17370} 
+**
+** SQLite provides these two interfaces for reporting on the status
+** of the [sqlite3_malloc()], [sqlite3_free()], and [sqlite3_realloc()]
+** routines, which form the built-in memory allocation subsystem.
+**
+** INVARIANTS:
+**
+** {H17371} The [sqlite3_memory_used()] routine returns the number of bytes
+**          of memory currently outstanding (malloced but not freed).
+**
+** {H17373} The [sqlite3_memory_highwater()] routine returns the maximum
+**          value of [sqlite3_memory_used()] since the high-water mark
+**          was last reset.
+**
+** {H17374} The values returned by [sqlite3_memory_used()] and
+**          [sqlite3_memory_highwater()] include any overhead
+**          added by SQLite in its implementation of [sqlite3_malloc()],
+**          but not overhead added by the any underlying system library
+**          routines that [sqlite3_malloc()] may call.
+**
+** {H17375} The memory high-water mark is reset to the current value of
+**          [sqlite3_memory_used()] if and only if the parameter to
+**          [sqlite3_memory_highwater()] is true.  The value returned
+**          by [sqlite3_memory_highwater(1)] is the high-water mark
+**          prior to the reset.
+*/
+sqlite3_int64 sqlite3_memory_used(void);
+sqlite3_int64 sqlite3_memory_highwater(int resetFlag);
+
+/*
+** CAPI3REF: Pseudo-Random Number Generator {H17390} 
+**
+** SQLite contains a high-quality pseudo-random number generator (PRNG) used to
+** select random [ROWID | ROWIDs] when inserting new records into a table that
+** already uses the largest possible [ROWID].  The PRNG is also used for
+** the build-in random() and randomblob() SQL functions.  This interface allows
+** applications to access the same PRNG for other purposes.
+**
+** A call to this routine stores N bytes of randomness into buffer P.
+**
+** The first time this routine is invoked (either internally or by
+** the application) the PRNG is seeded using randomness obtained
+** from the xRandomness method of the default [sqlite3_vfs] object.
+** On all subsequent invocations, the pseudo-randomness is generated
+** internally and without recourse to the [sqlite3_vfs] xRandomness
+** method.
+**
+** INVARIANTS:
+**
+** {H17392} The [sqlite3_randomness(N,P)] interface writes N bytes of
+**          high-quality pseudo-randomness into buffer P.
+*/
+void sqlite3_randomness(int N, void *P);
+
+/*
+** CAPI3REF: Compile-Time Authorization Callbacks {H12500} 
+**
+** This routine registers a authorizer callback with a particular
+** [database connection], supplied in the first argument.
 ** The authorizer callback is invoked as SQL statements are being compiled
 ** by [sqlite3_prepare()] or its variants [sqlite3_prepare_v2()],
 ** [sqlite3_prepare16()] and [sqlite3_prepare16_v2()].  At various
 ** points during the compilation process, as logic is being created
 ** to perform various actions, the authorizer callback is invoked to
 ** see if those actions are allowed.  The authorizer callback should
-** return SQLITE_OK to allow the action, [SQLITE_IGNORE] to disallow the
+** return [SQLITE_OK] to allow the action, [SQLITE_IGNORE] to disallow the
 ** specific action but allow the SQL statement to continue to be
 ** compiled, or [SQLITE_DENY] to cause the entire SQL statement to be
-** rejected with an error.  
+** rejected with an error.  If the authorizer callback returns
+** any value other than [SQLITE_IGNORE], [SQLITE_OK], or [SQLITE_DENY]
+** then the [sqlite3_prepare_v2()] or equivalent call that triggered
+** the authorizer will fail with an error message.
 **
-** Depending on the action, the [SQLITE_IGNORE] and [SQLITE_DENY] return
-** codes might mean something different or they might mean the same
-** thing.  If the action is, for example, to perform a delete opertion,
-** then [SQLITE_IGNORE] and [SQLITE_DENY] both cause the statement compilation
-** to fail with an error.  But if the action is to read a specific column
-** from a specific table, then [SQLITE_DENY] will cause the entire
-** statement to fail but [SQLITE_IGNORE] will cause a NULL value to be
-** read instead of the actual column value.
+** When the callback returns [SQLITE_OK], that means the operation
+** requested is ok.  When the callback returns [SQLITE_DENY], the
+** [sqlite3_prepare_v2()] or equivalent call that triggered the
+** authorizer will fail with an error message explaining that
+** access is denied.  If the authorizer code is [SQLITE_READ]
+** and the callback returns [SQLITE_IGNORE] then the
+** [prepared statement] statement is constructed to substitute
+** a NULL value in place of the table column that would have
+** been read if [SQLITE_OK] had been returned.  The [SQLITE_IGNORE]
+** return can be used to deny an untrusted user access to individual
+** columns of a table.
 **
-** The first parameter to the authorizer callback is a copy of
-** the third parameter to the sqlite3_set_authorizer() interface.
-** The second parameter to the callback is an integer 
-** [SQLITE_COPY | action code] that specifies the particular action
-** to be authorized.  The available action codes are
-** [SQLITE_COPY | documented separately].  The third through sixth
-** parameters to the callback are strings that contain additional
+** The first parameter to the authorizer callback is a copy of the third
+** parameter to the sqlite3_set_authorizer() interface. The second parameter
+** to the callback is an integer [SQLITE_COPY | action code] that specifies
+** the particular action to be authorized. The third through sixth parameters
+** to the callback are zero-terminated strings that contain additional
 ** details about the action to be authorized.
 **
-** An authorizer is used when preparing SQL statements from an untrusted
-** source, to ensure that the SQL statements do not try to access data
-** that they are not allowed to see, or that they do not try to
-** execute malicious statements that damage the database.  For
+** An authorizer is used when [sqlite3_prepare | preparing]
+** SQL statements from an untrusted source, to ensure that the SQL statements
+** do not try to access data they are not allowed to see, or that they do not
+** try to execute malicious statements that damage the database.  For
 ** example, an application may allow a user to enter arbitrary
 ** SQL queries for evaluation by a database.  But the application does
 ** not want the user to be able to make arbitrary changes to the
 ** database.  An authorizer could then be put in place while the
-** user-entered SQL is being prepared that disallows everything
-** except SELECT statements.  
+** user-entered SQL is being [sqlite3_prepare | prepared] that
+** disallows everything except [SELECT] statements.
+**
+** Applications that need to process SQL from untrusted sources
+** might also consider lowering resource limits using [sqlite3_limit()]
+** and limiting database size using the [max_page_count] [PRAGMA]
+** in addition to using an authorizer.
 **
 ** Only a single authorizer can be in place on a database connection
 ** at a time.  Each call to sqlite3_set_authorizer overrides the
-** previous call.  A NULL authorizer means that no authorization
-** callback is invoked.  The default authorizer is NULL.
+** previous call.  Disable the authorizer by installing a NULL callback.
+** The authorizer is disabled by default.
 **
-** Note that the authorizer callback is invoked only during 
+** The authorizer callback must not do anything that will modify
+** the database connection that invoked the authorizer callback.
+** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their
+** database connections for the meaning of "modify" in this paragraph.
+**
+** When [sqlite3_prepare_v2()] is used to prepare a statement, the
+** statement might be reprepared during [sqlite3_step()] due to a 
+** schema change.  Hence, the application should ensure that the
+** correct authorizer callback remains in place during the [sqlite3_step()].
+**
+** Note that the authorizer callback is invoked only during
 ** [sqlite3_prepare()] or its variants.  Authorization is not
 ** performed during statement evaluation in [sqlite3_step()].
+**
+** INVARIANTS:
+**
+** {H12501} The [sqlite3_set_authorizer(D,...)] interface registers a
+**          authorizer callback with database connection D.
+**
+** {H12502} The authorizer callback is invoked as SQL statements are
+**          being parseed and compiled.
+**
+** {H12503} If the authorizer callback returns any value other than
+**          [SQLITE_IGNORE], [SQLITE_OK], or [SQLITE_DENY], then
+**          the application interface call that caused
+**          the authorizer callback to run shall fail with an
+**          [SQLITE_ERROR] error code and an appropriate error message.
+**
+** {H12504} When the authorizer callback returns [SQLITE_OK], the operation
+**          described is processed normally.
+**
+** {H12505} When the authorizer callback returns [SQLITE_DENY], the
+**          application interface call that caused the
+**          authorizer callback to run shall fail
+**          with an [SQLITE_ERROR] error code and an error message
+**          explaining that access is denied.
+**
+** {H12506} If the authorizer code (the 2nd parameter to the authorizer
+**          callback) is [SQLITE_READ] and the authorizer callback returns
+**          [SQLITE_IGNORE], then the prepared statement is constructed to
+**          insert a NULL value in place of the table column that would have
+**          been read if [SQLITE_OK] had been returned.
+**
+** {H12507} If the authorizer code (the 2nd parameter to the authorizer
+**          callback) is anything other than [SQLITE_READ], then
+**          a return of [SQLITE_IGNORE] has the same effect as [SQLITE_DENY].
+**
+** {H12510} The first parameter to the authorizer callback is a copy of
+**          the third parameter to the [sqlite3_set_authorizer()] interface.
+**
+** {H12511} The second parameter to the callback is an integer
+**          [SQLITE_COPY | action code] that specifies the particular action
+**          to be authorized.
+**
+** {H12512} The third through sixth parameters to the callback are
+**          zero-terminated strings that contain
+**          additional details about the action to be authorized.
+**
+** {H12520} Each call to [sqlite3_set_authorizer()] overrides
+**          any previously installed authorizer.
+**
+** {H12521} A NULL authorizer means that no authorization
+**          callback is invoked.
+**
+** {H12522} The default authorizer is NULL.
 */
 int sqlite3_set_authorizer(
   sqlite3*,
@@ -754,7 +2323,7 @@ int sqlite3_set_authorizer(
 );
 
 /*
-** CAPI3REF: Authorizer Return Codes
+** CAPI3REF: Authorizer Return Codes {H12590} 
 **
 ** The [sqlite3_set_authorizer | authorizer callback function] must
 ** return either [SQLITE_OK] or one of these two constants in order
@@ -766,23 +2335,45 @@ int sqlite3_set_authorizer(
 #define SQLITE_IGNORE 2   /* Don't allow access, but don't generate an error */
 
 /*
-** CAPI3REF: Authorizer Action Codes
+** CAPI3REF: Authorizer Action Codes {H12550} 
 **
 ** The [sqlite3_set_authorizer()] interface registers a callback function
-** that is invoked to authorizer certain SQL statement actions.  The
+** that is invoked to authorize certain SQL statement actions.  The
 ** second parameter to the callback is an integer code that specifies
 ** what action is being authorized.  These are the integer action codes that
 ** the authorizer callback may be passed.
 **
-** These action code values signify what kind of operation is to be 
-** authorized.  The 3rd and 4th parameters to the authorization callback
-** function will be parameters or NULL depending on which of these
+** These action code values signify what kind of operation is to be
+** authorized.  The 3rd and 4th parameters to the authorization
+** callback function will be parameters or NULL depending on which of these
 ** codes is used as the second parameter.  The 5th parameter to the
-** authorizer callback is the name of the database ("main", "temp", 
+** authorizer callback is the name of the database ("main", "temp",
 ** etc.) if applicable.  The 6th parameter to the authorizer callback
 ** is the name of the inner-most trigger or view that is responsible for
-** the access attempt or NULL if this access attempt is directly from 
+** the access attempt or NULL if this access attempt is directly from
 ** top-level SQL code.
+**
+** INVARIANTS:
+**
+** {H12551} The second parameter to an
+**          [sqlite3_set_authorizer | authorizer callback] shall be an integer
+**          [SQLITE_COPY | authorizer code] that specifies what action
+**          is being authorized.
+**
+** {H12552} The 3rd and 4th parameters to the
+**          [sqlite3_set_authorizer | authorization callback]
+**          shall be parameters or NULL depending on which
+**          [SQLITE_COPY | authorizer code] is used as the second parameter.
+**
+** {H12553} The 5th parameter to the
+**          [sqlite3_set_authorizer | authorizer callback] shall be the name
+**          of the database (example: "main", "temp", etc.) if applicable.
+**
+** {H12554} The 6th parameter to the
+**          [sqlite3_set_authorizer | authorizer callback] shall be the name
+**          of the inner-most trigger or view that is responsible for
+**          the access attempt or NULL if this access attempt is directly from
+**          top-level SQL code.
 */
 /******************************************* 3rd ************ 4th ***********/
 #define SQLITE_CREATE_INDEX          1   /* Index Name      Table Name      */
@@ -806,7 +2397,7 @@ int sqlite3_set_authorizer(
 #define SQLITE_PRAGMA               19   /* Pragma Name     1st arg or NULL */
 #define SQLITE_READ                 20   /* Table Name      Column Name     */
 #define SQLITE_SELECT               21   /* NULL            NULL            */
-#define SQLITE_TRANSACTION          22   /* NULL            NULL            */
+#define SQLITE_TRANSACTION          22   /* Operation       NULL            */
 #define SQLITE_UPDATE               23   /* Table Name      Column Name     */
 #define SQLITE_ATTACH               24   /* Filename        NULL            */
 #define SQLITE_DETACH               25   /* Database Name   NULL            */
@@ -815,83 +2406,267 @@ int sqlite3_set_authorizer(
 #define SQLITE_ANALYZE              28   /* Table Name      NULL            */
 #define SQLITE_CREATE_VTABLE        29   /* Table Name      Module Name     */
 #define SQLITE_DROP_VTABLE          30   /* Table Name      Module Name     */
-#define SQLITE_FUNCTION             31   /* Function Name   NULL            */
+#define SQLITE_FUNCTION             31   /* NULL            Function Name   */
+#define SQLITE_SAVEPOINT            32   /* Operation       Savepoint Name  */
 #define SQLITE_COPY                  0   /* No longer used */
 
 /*
-** CAPI3REF: Tracing And Profiling Functions
+** CAPI3REF: Tracing And Profiling Functions {H12280} 
+** EXPERIMENTAL
 **
 ** These routines register callback functions that can be used for
 ** tracing and profiling the execution of SQL statements.
-** The callback function registered by sqlite3_trace() is invoked
-** at the first [sqlite3_step()] for the evaluation of an SQL statement.
-** The callback function registered by sqlite3_profile() is invoked
-** as each SQL statement finishes and includes
-** information on how long that statement ran.
 **
-** The sqlite3_profile() API is currently considered experimental and
-** is subject to change.
+** The callback function registered by sqlite3_trace() is invoked at
+** various times when an SQL statement is being run by [sqlite3_step()].
+** The callback returns a UTF-8 rendering of the SQL statement text
+** as the statement first begins executing.  Additional callbacks occur
+** as each triggered subprogram is entered.  The callbacks for triggers
+** contain a UTF-8 SQL comment that identifies the trigger.
+**
+** The callback function registered by sqlite3_profile() is invoked
+** as each SQL statement finishes.  The profile callback contains
+** the original statement text and an estimate of wall-clock time
+** of how long that statement took to run.
+**
+** INVARIANTS:
+**
+** {H12281} The callback function registered by [sqlite3_trace()] 
+**          shall be invoked
+**          whenever an SQL statement first begins to execute and
+**          whenever a trigger subprogram first begins to run.
+**
+** {H12282} Each call to [sqlite3_trace()] shall override the previously
+**          registered trace callback.
+**
+** {H12283} A NULL trace callback shall disable tracing.
+**
+** {H12284} The first argument to the trace callback shall be a copy of
+**          the pointer which was the 3rd argument to [sqlite3_trace()].
+**
+** {H12285} The second argument to the trace callback is a
+**          zero-terminated UTF-8 string containing the original text
+**          of the SQL statement as it was passed into [sqlite3_prepare_v2()]
+**          or the equivalent, or an SQL comment indicating the beginning
+**          of a trigger subprogram.
+**
+** {H12287} The callback function registered by [sqlite3_profile()] is invoked
+**          as each SQL statement finishes.
+**
+** {H12288} The first parameter to the profile callback is a copy of
+**          the 3rd parameter to [sqlite3_profile()].
+**
+** {H12289} The second parameter to the profile callback is a
+**          zero-terminated UTF-8 string that contains the complete text of
+**          the SQL statement as it was processed by [sqlite3_prepare_v2()]
+**          or the equivalent.
+**
+** {H12290} The third parameter to the profile callback is an estimate
+**          of the number of nanoseconds of wall-clock time required to
+**          run the SQL statement from start to finish.
 */
-void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*);
-void *sqlite3_profile(sqlite3*,
-   void(*xProfile)(void*,const char*,sqlite_uint64), void*);
+SQLITE_EXPERIMENTAL void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*);
+SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*,
+   void(*xProfile)(void*,const char*,sqlite3_uint64), void*);
 
 /*
-** CAPI3REF: Query Progress Callbacks
+** CAPI3REF: Query Progress Callbacks {H12910} 
 **
-** This routine configures a callback function - the progress callback - that
-** is invoked periodically during long running calls to [sqlite3_exec()],
-** [sqlite3_step()] and [sqlite3_get_table()].  An example use for this 
+** This routine configures a callback function - the
+** progress callback - that is invoked periodically during long
+** running calls to [sqlite3_exec()], [sqlite3_step()] and
+** [sqlite3_get_table()].  An example use for this
 ** interface is to keep a GUI updated during a large query.
 **
-** The progress callback is invoked once for every N virtual machine opcodes,
-** where N is the second argument to this function. The progress callback
-** itself is identified by the third argument to this function. The fourth
-** argument to this function is a void pointer passed to the progress callback
-** function each time it is invoked.
+** If the progress callback returns non-zero, the operation is
+** interrupted.  This feature can be used to implement a
+** "Cancel" button on a GUI progress dialog box.
 **
-** If a call to [sqlite3_exec()], [sqlite3_step()], or [sqlite3_get_table()]
-** results in fewer than N opcodes being executed, then the progress 
-** callback is never invoked.
-** 
-** Only a single progress callback function may be registered for each
-** open database connection.  Every call to sqlite3_progress_handler()
-** overwrites the results of the previous call.
-** To remove the progress callback altogether, pass NULL as the third
-** argument to this function.
+** The progress handler must not do anything that will modify
+** the database connection that invoked the progress handler.
+** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their
+** database connections for the meaning of "modify" in this paragraph.
 **
-** If the progress callback returns a result other than 0, then the current 
-** query is immediately terminated and any database changes rolled back.
-** The containing [sqlite3_exec()], [sqlite3_step()], or
-** [sqlite3_get_table()] call returns SQLITE_INTERRUPT.   This feature
-** can be used, for example, to implement the "Cancel" button on a
-** progress dialog box in a GUI.
+** INVARIANTS:
+**
+** {H12911} The callback function registered by sqlite3_progress_handler()
+**          is invoked periodically during long running calls to
+**          [sqlite3_step()].
+**
+** {H12912} The progress callback is invoked once for every N virtual
+**          machine opcodes, where N is the second argument to
+**          the [sqlite3_progress_handler()] call that registered
+**          the callback.  If N is less than 1, sqlite3_progress_handler()
+**          acts as if a NULL progress handler had been specified.
+**
+** {H12913} The progress callback itself is identified by the third
+**          argument to sqlite3_progress_handler().
+**
+** {H12914} The fourth argument to sqlite3_progress_handler() is a
+**          void pointer passed to the progress callback
+**          function each time it is invoked.
+**
+** {H12915} If a call to [sqlite3_step()] results in fewer than N opcodes
+**          being executed, then the progress callback is never invoked.
+**
+** {H12916} Every call to [sqlite3_progress_handler()]
+**          overwrites any previously registered progress handler.
+**
+** {H12917} If the progress handler callback is NULL then no progress
+**          handler is invoked.
+**
+** {H12918} If the progress callback returns a result other than 0, then
+**          the behavior is a if [sqlite3_interrupt()] had been called.
+**          
 */
 void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 
 /*
-** CAPI3REF: Opening A New Database Connection
+** CAPI3REF: Opening A New Database Connection {H12700} 
 **
-** Open the sqlite database file "filename".  The "filename" is UTF-8
-** encoded for sqlite3_open() and UTF-16 encoded in the native byte order
-** for sqlite3_open16().  An [sqlite3*] handle is returned in *ppDb, even
-** if an error occurs. If the database is opened (or created) successfully,
-** then SQLITE_OK is returned. Otherwise an error code is returned. The
-** sqlite3_errmsg() or sqlite3_errmsg16()  routines can be used to obtain
+** These routines open an SQLite database file whose name is given by the
+** filename argument. The filename argument is interpreted as UTF-8 for
+** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte
+** order for sqlite3_open16(). A [database connection] handle is usually
+** returned in *ppDb, even if an error occurs.  The only exception is that
+** if SQLite is unable to allocate memory to hold the [sqlite3] object,
+** a NULL will be written into *ppDb instead of a pointer to the [sqlite3]
+** object. If the database is opened (and/or created) successfully, then
+** [SQLITE_OK] is returned.  Otherwise an [error code] is returned.  The
+** [sqlite3_errmsg()] or [sqlite3_errmsg16()] routines can be used to obtain
 ** an English language description of the error.
 **
-** If the database file does not exist, then a new database will be created
-** as needed.  The default encoding for the database will be UTF-8 if
-** sqlite3_open() is called and UTF-16 if sqlite3_open16 is used.
+** The default encoding for the database will be UTF-8 if
+** sqlite3_open() or sqlite3_open_v2() is called and
+** UTF-16 in the native byte order if sqlite3_open16() is used.
 **
-** Whether or not an error occurs when it is opened, resources associated
-** with the [sqlite3*] handle should be released by passing it to
-** sqlite3_close() when it is no longer required.
+** Whether or not an error occurs when it is opened, resources
+** associated with the [database connection] handle should be released by
+** passing it to [sqlite3_close()] when it is no longer required.
 **
-** Note to windows users:  The encoding used for the filename argument
-** of sqlite3_open() must be UTF-8, not whatever codepage is currently
-** defined.  Filenames containing international characters must be converted
-** to UTF-8 prior to passing them into sqlite3_open().
+** The sqlite3_open_v2() interface works like sqlite3_open()
+** except that it accepts two additional parameters for additional control
+** over the new database connection.  The flags parameter can take one of
+** the following three values, optionally combined with the 
+** [SQLITE_OPEN_NOMUTEX] or [SQLITE_OPEN_FULLMUTEX] flags:
+**
+** 

+** 
[SQLITE_OPEN_READONLY]

+** 
The database is opened in read-only mode.  If the database does not
+** already exist, an error is returned.

+**
+** 
[SQLITE_OPEN_READWRITE]

+** 
The database is opened for reading and writing if possible, or reading
+** only if the file is write protected by the operating system.  In either
+** case the database must already exist, otherwise an error is returned.

+**
+** 
[SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]

+** 
The database is opened for reading and writing, and is creates it if
+** it does not already exist. This is the behavior that is always used for
+** sqlite3_open() and sqlite3_open16().

+** 

+**
+** If the 3rd parameter to sqlite3_open_v2() is not one of the
+** combinations shown above or one of the combinations shown above combined
+** with the [SQLITE_OPEN_NOMUTEX] or [SQLITE_OPEN_FULLMUTEX] flags,
+** then the behavior is undefined.
+**
+** If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection
+** opens in the multi-thread [threading mode] as long as the single-thread
+** mode has not been set at compile-time or start-time.  If the
+** [SQLITE_OPEN_FULLMUTEX] flag is set then the database connection opens
+** in the serialized [threading mode] unless single-thread was
+** previously selected at compile-time or start-time.
+**
+** If the filename is ":memory:", then a private, temporary in-memory database
+** is created for the connection.  This in-memory database will vanish when
+** the database connection is closed.  Future versions of SQLite might
+** make use of additional special filenames that begin with the ":" character.
+** It is recommended that when a database filename actually does begin with
+** a ":" character you should prefix the filename with a pathname such as
+** "./" to avoid ambiguity.
+**
+** If the filename is an empty string, then a private, temporary
+** on-disk database will be created.  This private database will be
+** automatically deleted as soon as the database connection is closed.
+**
+** The fourth parameter to sqlite3_open_v2() is the name of the
+** [sqlite3_vfs] object that defines the operating system interface that
+** the new database connection should use.  If the fourth parameter is
+** a NULL pointer then the default [sqlite3_vfs] object is used.
+**
+** Note to Windows users:  The encoding used for the filename argument
+** of sqlite3_open() and sqlite3_open_v2() must be UTF-8, not whatever
+** codepage is currently defined.  Filenames containing international
+** characters must be converted to UTF-8 prior to passing them into
+** sqlite3_open() or sqlite3_open_v2().
+**
+** INVARIANTS:
+**
+** {H12701} The [sqlite3_open()], [sqlite3_open16()], and
+**          [sqlite3_open_v2()] interfaces create a new
+**          [database connection] associated with
+**          the database file given in their first parameter.
+**
+** {H12702} The filename argument is interpreted as UTF-8
+**          for [sqlite3_open()] and [sqlite3_open_v2()] and as UTF-16
+**          in the native byte order for [sqlite3_open16()].
+**
+** {H12703} A successful invocation of [sqlite3_open()], [sqlite3_open16()],
+**          or [sqlite3_open_v2()] writes a pointer to a new
+**          [database connection] into *ppDb.
+**
+** {H12704} The [sqlite3_open()], [sqlite3_open16()], and
+**          [sqlite3_open_v2()] interfaces return [SQLITE_OK] upon success,
+**          or an appropriate [error code] on failure.
+**
+** {H12706} The default text encoding for a new database created using
+**          [sqlite3_open()] or [sqlite3_open_v2()] will be UTF-8.
+**
+** {H12707} The default text encoding for a new database created using
+**          [sqlite3_open16()] will be UTF-16.
+**
+** {H12709} The [sqlite3_open(F,D)] interface is equivalent to
+**          [sqlite3_open_v2(F,D,G,0)] where the G parameter is
+**          [SQLITE_OPEN_READWRITE]|[SQLITE_OPEN_CREATE].
+**
+** {H12711} If the G parameter to [sqlite3_open_v2(F,D,G,V)] contains the
+**          bit value [SQLITE_OPEN_READONLY] then the database is opened
+**          for reading only.
+**
+** {H12712} If the G parameter to [sqlite3_open_v2(F,D,G,V)] contains the
+**          bit value [SQLITE_OPEN_READWRITE] then the database is opened
+**          reading and writing if possible, or for reading only if the
+**          file is write protected by the operating system.
+**
+** {H12713} If the G parameter to [sqlite3_open_v2(F,D,G,V)] omits the
+**          bit value [SQLITE_OPEN_CREATE] and the database does not
+**          previously exist, an error is returned.
+**
+** {H12714} If the G parameter to [sqlite3_open_v2(F,D,G,V)] contains the
+**          bit value [SQLITE_OPEN_CREATE] and the database does not
+**          previously exist, then an attempt is made to create and
+**          initialize the database.
+**
+** {H12717} If the filename argument to [sqlite3_open()], [sqlite3_open16()],
+**          or [sqlite3_open_v2()] is ":memory:", then an private,
+**          ephemeral, in-memory database is created for the connection.
+**          Is SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE required
+**          in sqlite3_open_v2()?
+**
+** {H12719} If the filename is NULL or an empty string, then a private,
+**          ephemeral on-disk database will be created.
+**          Is SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE required
+**          in sqlite3_open_v2()?
+**
+** {H12721} The [database connection] created by [sqlite3_open_v2(F,D,G,V)]
+**          will use the [sqlite3_vfs] object identified by the V parameter,
+**          or the default [sqlite3_vfs] object if V is a NULL pointer.
+**
+** {H12723} Two [database connections] will share a common cache if both were
+**          opened with the same VFS while [shared cache mode] was enabled and
+**          if both filenames compare equal using memcmp() after having been
+**          processed by the [sqlite3_vfs | xFullPathname] method of the VFS.
 */
 int sqlite3_open(
   const char *filename,   /* Database filename (UTF-8) */
@@ -901,53 +2676,97 @@ int sqlite3_open16(
   const void *filename,   /* Database filename (UTF-16) */
   sqlite3 **ppDb          /* OUT: SQLite db handle */
 );
+int sqlite3_open_v2(
+  const char *filename,   /* Database filename (UTF-8) */
+  sqlite3 **ppDb,         /* OUT: SQLite db handle */
+  int flags,              /* Flags */
+  const char *zVfs        /* Name of VFS module to use */
+);
 
 /*
-** CAPI3REF: Error Codes And Messages
+** CAPI3REF: Error Codes And Messages {H12800} 
 **
-** The sqlite3_errcode() interface returns the numeric
-** [SQLITE_OK | result code] or [SQLITE_IOERR_READ | extended result code]
-** for the most recent failed sqlite3_* API call associated
-** with [sqlite3] handle 'db'.  If a prior API call failed but the
-** most recent API call succeeded, the return value from sqlite3_errcode()
-** is undefined. 
+** The sqlite3_errcode() interface returns the numeric [result code] or
+** [extended result code] for the most recent failed sqlite3_* API call
+** associated with a [database connection]. If a prior API call failed
+** but the most recent API call succeeded, the return value from
+** sqlite3_errcode() is undefined.  The sqlite3_extended_errcode()
+** interface is the same except that it always returns the 
+** [extended result code] even when extended result codes are
+** disabled.
 **
-** The sqlite3_errmsg() and sqlite3_errmsg16() return English-langauge
-** text that describes the error, as either UTF8 or UTF16 respectively.
-** Memory to hold the error message string is managed internally.  The 
-** string may be overwritten or deallocated by subsequent calls to SQLite
-** interface functions.
+** The sqlite3_errmsg() and sqlite3_errmsg16() return English-language
+** text that describes the error, as either UTF-8 or UTF-16 respectively.
+** Memory to hold the error message string is managed internally.
+** The application does not need to worry about freeing the result.
+** However, the error string might be overwritten or deallocated by
+** subsequent calls to other SQLite interface functions.
 **
-** Calls to many sqlite3_* functions set the error code and string returned
-** by [sqlite3_errcode()], [sqlite3_errmsg()], and [sqlite3_errmsg16()]
-** (overwriting the previous values). Note that calls to [sqlite3_errcode()],
-** [sqlite3_errmsg()], and [sqlite3_errmsg16()] themselves do not affect the
-** results of future invocations.  Calls to API routines that do not return
-** an error code (examples: [sqlite3_data_count()] or [sqlite3_mprintf()]) do
-** not change the error code returned by this routine.
+** When the serialized [threading mode] is in use, it might be the
+** case that a second error occurs on a separate thread in between
+** the time of the first error and the call to these interfaces.
+** When that happens, the second error will be reported since these
+** interfaces always report the most recent result.  To avoid
+** this, each thread can obtain exclusive use of the [database connection] D
+** by invoking [sqlite3_mutex_enter]([sqlite3_db_mutex](D)) before beginning
+** to use D and invoking [sqlite3_mutex_leave]([sqlite3_db_mutex](D)) after
+** all calls to the interfaces listed here are completed.
 **
-** Assuming no other intervening sqlite3_* API calls are made, the error
-** code returned by this function is associated with the same error as
-** the strings returned by [sqlite3_errmsg()] and [sqlite3_errmsg16()].
+** If an interface fails with SQLITE_MISUSE, that means the interface
+** was invoked incorrectly by the application.  In that case, the
+** error code and message may or may not be set.
+**
+** INVARIANTS:
+**
+** {H12801} The [sqlite3_errcode(D)] interface returns the numeric
+**          [result code] or [extended result code] for the most recently
+**          failed interface call associated with the [database connection] D.
+**
+** {H12802} The [sqlite3_extended_errcode(D)] interface returns the numeric
+**          [extended result code] for the most recently
+**          failed interface call associated with the [database connection] D.
+**
+** {H12803} The [sqlite3_errmsg(D)] and [sqlite3_errmsg16(D)]
+**          interfaces return English-language text that describes
+**          the error in the mostly recently failed interface call,
+**          encoded as either UTF-8 or UTF-16 respectively.
+**
+** {H12807} The strings returned by [sqlite3_errmsg()] and [sqlite3_errmsg16()]
+**          are valid until the next SQLite interface call.
+**
+** {H12808} Calls to API routines that do not return an error code
+**          (example: [sqlite3_data_count()]) do not
+**          change the error code or message returned by
+**          [sqlite3_errcode()], [sqlite3_extended_errcode()],
+**          [sqlite3_errmsg()], or [sqlite3_errmsg16()].
+**
+** {H12809} Interfaces that are not associated with a specific
+**          [database connection] (examples:
+**          [sqlite3_mprintf()] or [sqlite3_enable_shared_cache()]
+**          do not change the values returned by
+**          [sqlite3_errcode()], [sqlite3_extended_errcode()],
+**          [sqlite3_errmsg()], or [sqlite3_errmsg16()].
 */
 int sqlite3_errcode(sqlite3 *db);
+int sqlite3_extended_errcode(sqlite3 *db);
 const char *sqlite3_errmsg(sqlite3*);
 const void *sqlite3_errmsg16(sqlite3*);
 
 /*
-** CAPI3REF: SQL Statement Object
+** CAPI3REF: SQL Statement Object {H13000} 
+** KEYWORDS: {prepared statement} {prepared statements}
 **
-** Instance of this object represent single SQL statements.  This
-** is variously known as a "prepared statement" or a 
+** An instance of this object represents a single SQL statement.
+** This object is variously known as a "prepared statement" or a
 ** "compiled SQL statement" or simply as a "statement".
-** 
+**
 ** The life of a statement object goes something like this:
 **
 ** 
    
 ** 
  1.  Create the object using [sqlite3_prepare_v2()] or a related
 **      function.
-** 
  2.  Bind values to host parameters using
-**      [sqlite3_bind_blob | sqlite3_bind_* interfaces].
+** 
  3.  Bind values to [host parameters] using the sqlite3_bind_*()
+**      interfaces.
 ** 
  4.  Run the SQL by calling [sqlite3_step()] one or more times.
 ** 
  5.  Reset the statement using [sqlite3_reset()] then go back
 **      to step 2.  Do this zero or more times.
@@ -960,45 +2779,153 @@ const void *sqlite3_errmsg16(sqlite3*);
 typedef struct sqlite3_stmt sqlite3_stmt;
 
 /*
-** CAPI3REF: Compiling An SQL Statement
+** CAPI3REF: Run-time Limits {H12760} 
+**
+** This interface allows the size of various constructs to be limited
+** on a connection by connection basis.  The first parameter is the
+** [database connection] whose limit is to be set or queried.  The
+** second parameter is one of the [limit categories] that define a
+** class of constructs to be size limited.  The third parameter is the
+** new limit for that construct.  The function returns the old limit.
+**
+** If the new limit is a negative number, the limit is unchanged.
+** For the limit category of SQLITE_LIMIT_XYZ there is a hard upper
+** bound set by a compile-time C preprocessor macro named SQLITE_MAX_XYZ.
+** (The "_LIMIT_" in the name is changed to "_MAX_".)
+** Attempts to increase a limit above its hard upper bound are
+** silently truncated to the hard upper limit.
+**
+** Run time limits are intended for use in applications that manage
+** both their own internal database and also databases that are controlled
+** by untrusted external sources.  An example application might be a
+** webbrowser that has its own databases for storing history and
+** separate databases controlled by JavaScript applications downloaded
+** off the Internet.  The internal databases can be given the
+** large, default limits.  Databases managed by external sources can
+** be given much smaller limits designed to prevent a denial of service
+** attack.  Developers might also want to use the [sqlite3_set_authorizer()]
+** interface to further control untrusted SQL.  The size of the database
+** created by an untrusted script can be contained using the
+** [max_page_count] [PRAGMA].
+**
+** New run-time limit categories may be added in future releases.
+**
+** INVARIANTS:
+**
+** {H12762} A successful call to [sqlite3_limit(D,C,V)] where V is
+**          positive changes the limit on the size of construct C in the
+**          [database connection] D to the lesser of V and the hard upper
+**          bound on the size of C that is set at compile-time.
+**
+** {H12766} A successful call to [sqlite3_limit(D,C,V)] where V is negative
+**          leaves the state of the [database connection] D unchanged.
+**
+** {H12769} A successful call to [sqlite3_limit(D,C,V)] returns the
+**          value of the limit on the size of construct C in the
+**          [database connection] D as it was prior to the call.
+*/
+int sqlite3_limit(sqlite3*, int id, int newVal);
+
+/*
+** CAPI3REF: Run-Time Limit Categories {H12790} 
+** KEYWORDS: {limit category} {limit categories}
+**
+** These constants define various aspects of a [database connection]
+** that can be limited in size by calls to [sqlite3_limit()].
+** The meanings of the various limits are as follows:
+**
+** 
    
+** 
    SQLITE_LIMIT_LENGTH
    
+** 
    The maximum size of any string or BLOB or table row.
    
+**
+** 
    SQLITE_LIMIT_SQL_LENGTH
    
+** 
    The maximum length of an SQL statement.
    
+**
+** 
    SQLITE_LIMIT_COLUMN
    
+** 
    The maximum number of columns in a table definition or in the
+** result set of a SELECT or the maximum number of columns in an index
+** or in an ORDER BY or GROUP BY clause.
    
+**
+** 
    SQLITE_LIMIT_EXPR_DEPTH
    
+** 
    The maximum depth of the parse tree on any expression.
    
+**
+** 
    SQLITE_LIMIT_COMPOUND_SELECT
    
+** 
    The maximum number of terms in a compound SELECT statement.
    
+**
+** 
    SQLITE_LIMIT_VDBE_OP
    
+** 
    The maximum number of instructions in a virtual machine program
+** used to implement an SQL statement.
    
+**
+** 
    SQLITE_LIMIT_FUNCTION_ARG
    
+** 
    The maximum number of arguments on a function.
    
+**
+** 
    SQLITE_LIMIT_ATTACHED
    
+** 
    The maximum number of attached databases.
    
+**
+** 
    SQLITE_LIMIT_LIKE_PATTERN_LENGTH
    
+** 
    The maximum length of the pattern argument to the LIKE or
+** GLOB operators.
    
+**
+** 
    SQLITE_LIMIT_VARIABLE_NUMBER
    
+** 
    The maximum number of variables in an SQL statement that can
+** be bound.
    
+** 
    
+*/
+#define SQLITE_LIMIT_LENGTH                    0
+#define SQLITE_LIMIT_SQL_LENGTH                1
+#define SQLITE_LIMIT_COLUMN                    2
+#define SQLITE_LIMIT_EXPR_DEPTH                3
+#define SQLITE_LIMIT_COMPOUND_SELECT           4
+#define SQLITE_LIMIT_VDBE_OP                   5
+#define SQLITE_LIMIT_FUNCTION_ARG              6
+#define SQLITE_LIMIT_ATTACHED                  7
+#define SQLITE_LIMIT_LIKE_PATTERN_LENGTH       8
+#define SQLITE_LIMIT_VARIABLE_NUMBER           9
+
+/*
+** CAPI3REF: Compiling An SQL Statement {H13010} 
+** KEYWORDS: {SQL statement compiler}
 **
 ** To execute an SQL query, it must first be compiled into a byte-code
-** program using one of these routines. 
+** program using one of these routines.
 **
-** The first argument "db" is an [sqlite3 | SQLite database handle] 
-** obtained from a prior call to [sqlite3_open()] or [sqlite3_open16()].
-** The second argument "zSql" is the statement to be compiled, encoded
+** The first argument, "db", is a [database connection] obtained from a
+** prior call to [sqlite3_open()], [sqlite3_open_v2()] or [sqlite3_open16()].
+**
+** The second argument, "zSql", is the statement to be compiled, encoded
 ** as either UTF-8 or UTF-16.  The sqlite3_prepare() and sqlite3_prepare_v2()
-** interfaces uses UTF-8 and sqlite3_prepare16() and sqlite3_prepare16_v2()
+** interfaces use UTF-8, and sqlite3_prepare16() and sqlite3_prepare16_v2()
 ** use UTF-16.
 **
-** If the nByte argument is less
-** than zero, then zSql is read up to the first zero terminator.  If
-** nByte is non-negative, then it is the maximum number of 
-** bytes read from zSql.  When nByte is non-negative, the
-** zSql string ends at either the first '\000' character or 
-** until the nByte-th byte, whichever comes first.
+** If the nByte argument is less than zero, then zSql is read up to the
+** first zero terminator. If nByte is non-negative, then it is the maximum
+** number of  bytes read from zSql.  When nByte is non-negative, the
+** zSql string ends at either the first '\000' or '\u0000' character or
+** the nByte-th byte, whichever comes first. If the caller knows
+** that the supplied string is nul-terminated, then there is a small
+** performance advantage to be gained by passing an nByte parameter that
+** is equal to the number of bytes in the input string including
+** the nul-terminator bytes.
 **
-** *pzTail is made to point to the first byte past the end of the first
-** SQL statement in zSql.  This routine only compiles the first statement
-** in zSql, so *pzTail is left pointing to what remains uncompiled.
+** *pzTail is made to point to the first byte past the end of the
+** first SQL statement in zSql.  These routines only compile the first
+** statement in zSql, so *pzTail is left pointing to what remains
+** uncompiled.
 **
-** *ppStmt is left pointing to a compiled 
-** [sqlite3_stmt | SQL statement structure] that can be
-** executed using [sqlite3_step()].  Or if there is an error, *ppStmt may be
-** set to NULL.  If the input text contained no SQL (if the input is and
-** empty string or a comment) then *ppStmt is set to NULL.  The calling
-** procedure is responsible for deleting the compiled SQL statement
-** using [sqlite3_finalize()] after it has finished with it.
+** *ppStmt is left pointing to a compiled [prepared statement] that can be
+** executed using [sqlite3_step()].  If there is an error, *ppStmt is set
+** to NULL.  If the input text contains no SQL (if the input is an empty
+** string or a comment) then *ppStmt is set to NULL.
+** {A13018} The calling procedure is responsible for deleting the compiled
+** SQL statement using [sqlite3_finalize()] after it has finished with it.
 **
-** On success, [SQLITE_OK] is returned.  Otherwise an 
-** [SQLITE_ERROR | error code] is returned.
+** On success, [SQLITE_OK] is returned, otherwise an [error code] is returned.
 **
 ** The sqlite3_prepare_v2() and sqlite3_prepare16_v2() interfaces are
 ** recommended for all new programs. The two older interfaces are retained
 ** for backwards compatibility, but their use is discouraged.
 ** In the "v2" interfaces, the prepared statement
-** that is returned (the [sqlite3_stmt] object) contains a copy of the 
+** that is returned (the [sqlite3_stmt] object) contains a copy of the
 ** original SQL text. This causes the [sqlite3_step()] interface to
 ** behave a differently in two ways:
 **
@@ -1006,26 +2933,59 @@ typedef struct sqlite3_stmt sqlite3_stmt;
 ** 
  6. 
 ** If the database schema changes, instead of returning [SQLITE_SCHEMA] as it
 ** always used to do, [sqlite3_step()] will automatically recompile the SQL
-** statement and try to run it again.  If the schema has changed in a way
-** that makes the statement no longer valid, [sqlite3_step()] will still
+** statement and try to run it again.  If the schema has changed in
+** a way that makes the statement no longer valid, [sqlite3_step()] will still
 ** return [SQLITE_SCHEMA].  But unlike the legacy behavior, [SQLITE_SCHEMA] is
 ** now a fatal error.  Calling [sqlite3_prepare_v2()] again will not make the
-** error go away.  Note: use [sqlite3_errmsg()] to find the text of the parsing
-** error that results in an [SQLITE_SCHEMA] return.
+** error go away.  Note: use [sqlite3_errmsg()] to find the text
+** of the parsing error that results in an [SQLITE_SCHEMA] return.
 ** 
    7. 
 **
 ** 
  7. 
-** When an error occurs, 
-** [sqlite3_step()] will return one of the detailed 
-** [SQLITE_ERROR | result codes] or
-** [SQLITE_IOERR_READ | extended result codes] such as directly.
-** The legacy behavior was that [sqlite3_step()] would only return a generic
-** [SQLITE_ERROR] result code and you would have to make a second call to
-** [sqlite3_reset()] in order to find the underlying cause of the problem.
-** With the "v2" prepare interfaces, the underlying reason for the error is
-** returned immediately.
+** When an error occurs, [sqlite3_step()] will return one of the detailed
+** [error codes] or [extended error codes].  The legacy behavior was that
+** [sqlite3_step()] would only return a generic [SQLITE_ERROR] result code
+** and you would have to make a second call to [sqlite3_reset()] in order
+** to find the underlying cause of the problem. With the "v2" prepare
+** interfaces, the underlying reason for the error is returned immediately.
 ** 
    8. 
 ** 

+**
+** INVARIANTS:
+**
+** {H13011} The [sqlite3_prepare(db,zSql,...)] and
+**          [sqlite3_prepare_v2(db,zSql,...)] interfaces interpret the
+**          text in their zSql parameter as UTF-8.
+**
+** {H13012} The [sqlite3_prepare16(db,zSql,...)] and
+**          [sqlite3_prepare16_v2(db,zSql,...)] interfaces interpret the
+**          text in their zSql parameter as UTF-16 in the native byte order.
+**
+** {H13013} If the nByte argument to [sqlite3_prepare_v2(db,zSql,nByte,...)]
+**          and its variants is less than zero, the SQL text is
+**          read from zSql is read up to the first zero terminator.
+**
+** {H13014} If the nByte argument to [sqlite3_prepare_v2(db,zSql,nByte,...)]
+**          and its variants is non-negative, then at most nBytes bytes of
+**          SQL text is read from zSql.
+**
+** {H13015} In [sqlite3_prepare_v2(db,zSql,N,P,pzTail)] and its variants
+**          if the zSql input text contains more than one SQL statement
+**          and pzTail is not NULL, then *pzTail is made to point to the
+**          first byte past the end of the first SQL statement in zSql.
+**          What does *pzTail point to if there is one statement?
+**
+** {H13016} A successful call to [sqlite3_prepare_v2(db,zSql,N,ppStmt,...)]
+**          or one of its variants writes into *ppStmt a pointer to a new
+**          [prepared statement] or a pointer to NULL if zSql contains
+**          nothing other than whitespace or comments.
+**
+** {H13019} The [sqlite3_prepare_v2()] interface and its variants return
+**          [SQLITE_OK] or an appropriate [error code] upon failure.
+**
+** {H13021} Before [sqlite3_prepare(db,zSql,nByte,ppStmt,pzTail)] or its
+**          variants returns an error (any value other than [SQLITE_OK]),
+**          they first set *ppStmt to NULL.
 */
 int sqlite3_prepare(
   sqlite3 *db,            /* Database handle */
@@ -1057,84 +3017,141 @@ int sqlite3_prepare16_v2(
 );
 
 /*
-** CAPI3REF:  Dynamically Typed Value Object
+** CAPI3REF: Retrieving Statement SQL {H13100} 
 **
-** SQLite uses dynamic typing for the values it stores.  Values can 
-** be integers, floating point values, strings, BLOBs, or NULL.  When
-** passing around values internally, each value is represented as
-** an instance of the sqlite3_value object.
+** This interface can be used to retrieve a saved copy of the original
+** SQL text used to create a [prepared statement] if that statement was
+** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()].
+**
+** INVARIANTS:
+**
+** {H13101} If the [prepared statement] passed as the argument to
+**          [sqlite3_sql()] was compiled using either [sqlite3_prepare_v2()] or
+**          [sqlite3_prepare16_v2()], then [sqlite3_sql()] returns
+**          a pointer to a zero-terminated string containing a UTF-8 rendering
+**          of the original SQL statement.
+**
+** {H13102} If the [prepared statement] passed as the argument to
+**          [sqlite3_sql()] was compiled using either [sqlite3_prepare()] or
+**          [sqlite3_prepare16()], then [sqlite3_sql()] returns a NULL pointer.
+**
+** {H13103} The string returned by [sqlite3_sql(S)] is valid until the
+**          [prepared statement] S is deleted using [sqlite3_finalize(S)].
+*/
+const char *sqlite3_sql(sqlite3_stmt *pStmt);
+
+/*
+** CAPI3REF: Dynamically Typed Value Object {H15000} 
+** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value}
+**
+** SQLite uses the sqlite3_value object to represent all values
+** that can be stored in a database table. SQLite uses dynamic typing
+** for the values it stores. Values stored in sqlite3_value objects
+** can be integers, floating point values, strings, BLOBs, or NULL.
+**
+** An sqlite3_value object may be either "protected" or "unprotected".
+** Some interfaces require a protected sqlite3_value.  Other interfaces
+** will accept either a protected or an unprotected sqlite3_value.
+** Every interface that accepts sqlite3_value arguments specifies
+** whether or not it requires a protected sqlite3_value.
+**
+** The terms "protected" and "unprotected" refer to whether or not
+** a mutex is held.  A internal mutex is held for a protected
+** sqlite3_value object but no mutex is held for an unprotected
+** sqlite3_value object.  If SQLite is compiled to be single-threaded
+** (with [SQLITE_THREADSAFE=0] and with [sqlite3_threadsafe()] returning 0)
+** or if SQLite is run in one of reduced mutex modes 
+** [SQLITE_CONFIG_SINGLETHREAD] or [SQLITE_CONFIG_MULTITHREAD]
+** then there is no distinction between protected and unprotected
+** sqlite3_value objects and they can be used interchangeably.  However,
+** for maximum code portability it is recommended that applications
+** still make the distinction between between protected and unprotected
+** sqlite3_value objects even when not strictly required.
+**
+** The sqlite3_value objects that are passed as parameters into the
+** implementation of [application-defined SQL functions] are protected.
+** The sqlite3_value object returned by
+** [sqlite3_column_value()] is unprotected.
+** Unprotected sqlite3_value objects may only be used with
+** [sqlite3_result_value()] and [sqlite3_bind_value()].
+** The [sqlite3_value_blob | sqlite3_value_type()] family of
+** interfaces require protected sqlite3_value objects.
 */
 typedef struct Mem sqlite3_value;
 
 /*
-** CAPI3REF:  SQL Function Context Object
+** CAPI3REF: SQL Function Context Object {H16001} 
 **
 ** The context in which an SQL function executes is stored in an
-** sqlite3_context object.  A pointer to such an object is the
-** first parameter to user-defined SQL functions.
+** sqlite3_context object.  A pointer to an sqlite3_context object
+** is always first parameter to [application-defined SQL functions].
+** The application-defined SQL function implementation will pass this
+** pointer through into calls to [sqlite3_result_int | sqlite3_result()],
+** [sqlite3_aggregate_context()], [sqlite3_user_data()],
+** [sqlite3_context_db_handle()], [sqlite3_get_auxdata()],
+** and/or [sqlite3_set_auxdata()].
 */
 typedef struct sqlite3_context sqlite3_context;
 
 /*
-** CAPI3REF:  Binding Values To Prepared Statements
+** CAPI3REF: Binding Values To Prepared Statements {H13500} 
+** KEYWORDS: {host parameter} {host parameters} {host parameter name}
+** KEYWORDS: {SQL parameter} {SQL parameters} {parameter binding}
 **
 ** In the SQL strings input to [sqlite3_prepare_v2()] and its variants,
-** one or more literals can be replace by a parameter in one of these
-** forms:
+** literals may be replaced by a parameter in one of these forms:
 **
 ** 
    
 ** 
  •   ?
 ** 
  •   ?NNN
-** 
  •   :AAA
-** 
  •   @AAA
+** 
  •   :VVV
+** 
  •   @VVV
 ** 
  •   $VVV
 ** 

 **
 ** In the parameter forms shown above NNN is an integer literal,
-** AAA is an alphanumeric identifier and VVV is a variable name according
-** to the syntax rules of the TCL programming language.
-** The values of these parameters (also called "host parameter names")
+** and VVV is an alpha-numeric parameter name. The values of these
+** parameters (also called "host parameter names" or "SQL parameters")
 ** can be set using the sqlite3_bind_*() routines defined here.
 **
-** The first argument to the sqlite3_bind_*() routines always is a pointer
-** to the [sqlite3_stmt] object returned from [sqlite3_prepare_v2()] or
-** its variants.  The second
-** argument is the index of the parameter to be set.  The first parameter has
-** an index of 1. When the same named parameter is used more than once, second
-** and subsequent
-** occurrences have the same index as the first occurrence.  The index for
-** named parameters can be looked up using the
-** [sqlite3_bind_parameter_name()] API if desired.  The index for "?NNN"
-** parametes is the value of NNN.
-** The NNN value must be between 1 and the compile-time
-** parameter SQLITE_MAX_VARIABLE_NUMBER (default value: 999).
-** See limits.html for additional information.
+** The first argument to the sqlite3_bind_*() routines is always
+** a pointer to the [sqlite3_stmt] object returned from
+** [sqlite3_prepare_v2()] or its variants.
+**
+** The second argument is the index of the SQL parameter to be set.
+** The leftmost SQL parameter has an index of 1.  When the same named
+** SQL parameter is used more than once, second and subsequent
+** occurrences have the same index as the first occurrence.
+** The index for named parameters can be looked up using the
+** [sqlite3_bind_parameter_index()] API if desired.  The index
+** for "?NNN" parameters is the value of NNN.
+** The NNN value must be between 1 and the [sqlite3_limit()]
+** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999).
 **
 ** The third argument is the value to bind to the parameter.
 **
-** In those
-** routines that have a fourth argument, its value is the number of bytes
-** in the parameter.  To be clear: the value is the number of bytes in the
-** string, not the number of characters.  The number
-** of bytes does not include the zero-terminator at the end of strings.
+** In those routines that have a fourth argument, its value is the
+** number of bytes in the parameter.  To be clear: the value is the
+** number of bytes in the value, not the number of characters.
 ** If the fourth parameter is negative, the length of the string is
-** number of bytes up to the first zero terminator.
+** the number of bytes up to the first zero terminator.
 **
 ** The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and
 ** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or
-** text after SQLite has finished with it.  If the fifth argument is the
-** special value [SQLITE_STATIC], then the library assumes that the information
-** is in static, unmanaged space and does not need to be freed.  If the
-** fifth argument has the value [SQLITE_TRANSIENT], then SQLite makes its
-** own private copy of the data immediately, before the sqlite3_bind_*()
-** routine returns.
+** string after SQLite has finished with it. If the fifth argument is
+** the special value [SQLITE_STATIC], then SQLite assumes that the
+** information is in static, unmanaged space and does not need to be freed.
+** If the fifth argument has the value [SQLITE_TRANSIENT], then
+** SQLite makes its own private copy of the data immediately, before
+** the sqlite3_bind_*() routine returns.
 **
-** The sqlite3_bind_zeroblob() routine binds a BLOB of length n that
-** is filled with zeros.  A zeroblob uses a fixed amount of memory
-** (just an integer to hold it size) while it is being processed.
-** Zeroblobs are intended to serve as place-holders for BLOBs whose
-** content is later written using 
-** [sqlite3_blob_open | increment BLOB I/O] routines.
+** The sqlite3_bind_zeroblob() routine binds a BLOB of length N that
+** is filled with zeroes.  A zeroblob uses a fixed amount of memory
+** (just an integer to hold its size) while it is being processed.
+** Zeroblobs are intended to serve as placeholders for BLOBs whose
+** content is later written using
+** [sqlite3_blob_open | incremental BLOB I/O] routines.
+** A negative value for the zeroblob results in a zero-length BLOB.
 **
 ** The sqlite3_bind_*() routines must be called after
 ** [sqlite3_prepare_v2()] (and its variants) or [sqlite3_reset()] and
@@ -1144,14 +3161,95 @@ typedef struct sqlite3_context sqlite3_context;
 **
 ** These routines return [SQLITE_OK] on success or an error code if
 ** anything goes wrong.  [SQLITE_RANGE] is returned if the parameter
-** index is out of range.  [SQLITE_NOMEM] is returned if malloc fails.
-** [SQLITE_MISUSE] is returned if these routines are called on a virtual
-** machine that is the wrong state or which has already been finalized.
+** index is out of range.  [SQLITE_NOMEM] is returned if malloc() fails.
+** [SQLITE_MISUSE] might be returned if these routines are called on a
+** virtual machine that is the wrong state or which has already been finalized.
+** Detection of misuse is unreliable.  Applications should not depend
+** on SQLITE_MISUSE returns.  SQLITE_MISUSE is intended to indicate a
+** a logic error in the application.  Future versions of SQLite might
+** panic rather than return SQLITE_MISUSE.
+**
+** See also: [sqlite3_bind_parameter_count()],
+** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()].
+**
+** INVARIANTS:
+**
+** {H13506} The [SQL statement compiler] recognizes tokens of the forms
+**          "?", "?NNN", "$VVV", ":VVV", and "@VVV" as SQL parameters,
+**          where NNN is any sequence of one or more digits
+**          and where VVV is any sequence of one or more alphanumeric
+**          characters or "::" optionally followed by a string containing
+**          no spaces and contained within parentheses.
+**
+** {H13509} The initial value of an SQL parameter is NULL.
+**
+** {H13512} The index of an "?" SQL parameter is one larger than the
+**          largest index of SQL parameter to the left, or 1 if
+**          the "?" is the leftmost SQL parameter.
+**
+** {H13515} The index of an "?NNN" SQL parameter is the integer NNN.
+**
+** {H13518} The index of an ":VVV", "$VVV", or "@VVV" SQL parameter is
+**          the same as the index of leftmost occurrences of the same
+**          parameter, or one more than the largest index over all
+**          parameters to the left if this is the first occurrence
+**          of this parameter, or 1 if this is the leftmost parameter.
+**
+** {H13521} The [SQL statement compiler] fails with an [SQLITE_RANGE]
+**          error if the index of an SQL parameter is less than 1
+**          or greater than the compile-time SQLITE_MAX_VARIABLE_NUMBER
+**          parameter.
+**
+** {H13524} Calls to [sqlite3_bind_text | sqlite3_bind(S,N,V,...)]
+**          associate the value V with all SQL parameters having an
+**          index of N in the [prepared statement] S.
+**
+** {H13527} Calls to [sqlite3_bind_text | sqlite3_bind(S,N,...)]
+**          override prior calls with the same values of S and N.
+**
+** {H13530} Bindings established by [sqlite3_bind_text | sqlite3_bind(S,...)]
+**          persist across calls to [sqlite3_reset(S)].
+**
+** {H13533} In calls to [sqlite3_bind_blob(S,N,V,L,D)],
+**          [sqlite3_bind_text(S,N,V,L,D)], or
+**          [sqlite3_bind_text16(S,N,V,L,D)] SQLite binds the first L
+**          bytes of the BLOB or string pointed to by V, when L
+**          is non-negative.
+**
+** {H13536} In calls to [sqlite3_bind_text(S,N,V,L,D)] or
+**          [sqlite3_bind_text16(S,N,V,L,D)] SQLite binds characters
+**          from V through the first zero character when L is negative.
+**
+** {H13539} In calls to [sqlite3_bind_blob(S,N,V,L,D)],
+**          [sqlite3_bind_text(S,N,V,L,D)], or
+**          [sqlite3_bind_text16(S,N,V,L,D)] when D is the special
+**          constant [SQLITE_STATIC], SQLite assumes that the value V
+**          is held in static unmanaged space that will not change
+**          during the lifetime of the binding.
+**
+** {H13542} In calls to [sqlite3_bind_blob(S,N,V,L,D)],
+**          [sqlite3_bind_text(S,N,V,L,D)], or
+**          [sqlite3_bind_text16(S,N,V,L,D)] when D is the special
+**          constant [SQLITE_TRANSIENT], the routine makes a
+**          private copy of the value V before it returns.
+**
+** {H13545} In calls to [sqlite3_bind_blob(S,N,V,L,D)],
+**          [sqlite3_bind_text(S,N,V,L,D)], or
+**          [sqlite3_bind_text16(S,N,V,L,D)] when D is a pointer to
+**          a function, SQLite invokes that function to destroy the
+**          value V after it has finished using the value V.
+**
+** {H13548} In calls to [sqlite3_bind_zeroblob(S,N,V,L)] the value bound
+**          is a BLOB of L bytes, or a zero-length BLOB if L is negative.
+**
+** {H13551} In calls to [sqlite3_bind_value(S,N,V)] the V argument may
+**          be either a [protected sqlite3_value] object or an
+**          [unprotected sqlite3_value] object.
 */
 int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*));
 int sqlite3_bind_double(sqlite3_stmt*, int, double);
 int sqlite3_bind_int(sqlite3_stmt*, int, int);
-int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite_int64);
+int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64);
 int sqlite3_bind_null(sqlite3_stmt*, int);
 int sqlite3_bind_text(sqlite3_stmt*, int, const char*, int n, void(*)(void*));
 int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*));
@@ -1159,122 +3257,263 @@ int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*);
 int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n);
 
 /*
-** CAPI3REF: Number Of Host Parameters
+** CAPI3REF: Number Of SQL Parameters {H13600} 
 **
-** Return the largest host parameter index in the precompiled statement given
-** as the argument.  When the host parameters are of the forms like ":AAA"
-** or "?", then they are assigned sequential increasing numbers beginning
-** with one, so the value returned is the number of parameters.  However
-** if the same host parameter name is used multiple times, each occurrance
-** is given the same number, so the value returned in that case is the number
-** of unique host parameter names.  If host parameters of the form "?NNN"
-** are used (where NNN is an integer) then there might be gaps in the
-** numbering and the value returned by this interface is the index of the
-** host parameter with the largest index value.
+** This routine can be used to find the number of [SQL parameters]
+** in a [prepared statement].  SQL parameters are tokens of the
+** form "?", "?NNN", ":AAA", "$AAA", or "@AAA" that serve as
+** placeholders for values that are [sqlite3_bind_blob | bound]
+** to the parameters at a later time.
+**
+** This routine actually returns the index of the largest (rightmost)
+** parameter. For all forms except ?NNN, this will correspond to the
+** number of unique parameters.  If parameters of the ?NNN are used,
+** there may be gaps in the list.
+**
+** See also: [sqlite3_bind_blob|sqlite3_bind()],
+** [sqlite3_bind_parameter_name()], and
+** [sqlite3_bind_parameter_index()].
+**
+** INVARIANTS:
+**
+** {H13601} The [sqlite3_bind_parameter_count(S)] interface returns
+**          the largest index of all SQL parameters in the
+**          [prepared statement] S, or 0 if S contains no SQL parameters.
 */
 int sqlite3_bind_parameter_count(sqlite3_stmt*);
 
 /*
-** CAPI3REF: Name Of A Host Parameter
+** CAPI3REF: Name Of A Host Parameter {H13620} 
 **
-** This routine returns a pointer to the name of the n-th parameter in a 
-** [sqlite3_stmt | prepared statement].
-** Host parameters of the form ":AAA" or "@AAA" or "$VVV" have a name
-** which is the string ":AAA" or "@AAA" or "$VVV".  
-** In other words, the initial ":" or "$" or "@"
+** This routine returns a pointer to the name of the n-th
+** [SQL parameter] in a [prepared statement].
+** SQL parameters of the form "?NNN" or ":AAA" or "@AAA" or "$AAA"
+** have a name which is the string "?NNN" or ":AAA" or "@AAA" or "$AAA"
+** respectively.
+** In other words, the initial ":" or "$" or "@" or "?"
 ** is included as part of the name.
-** Parameters of the form "?" or "?NNN" have no name.
+** Parameters of the form "?" without a following integer have no name
+** and are also referred to as "anonymous parameters".
 **
-** The first bound parameter has an index of 1, not 0.
+** The first host parameter has an index of 1, not 0.
 **
-** If the value n is out of range or if the n-th parameter is nameless,
-** then NULL is returned.  The returned string is always in the
-** UTF-8 encoding even if the named parameter was originally specified
-** as UTF-16 in [sqlite3_prepare16()] or [sqlite3_prepare16_v2()].
+** If the value n is out of range or if the n-th parameter is
+** nameless, then NULL is returned.  The returned string is
+** always in UTF-8 encoding even if the named parameter was
+** originally specified as UTF-16 in [sqlite3_prepare16()] or
+** [sqlite3_prepare16_v2()].
+**
+** See also: [sqlite3_bind_blob|sqlite3_bind()],
+** [sqlite3_bind_parameter_count()], and
+** [sqlite3_bind_parameter_index()].
+**
+** INVARIANTS:
+**
+** {H13621} The [sqlite3_bind_parameter_name(S,N)] interface returns
+**          a UTF-8 rendering of the name of the SQL parameter in
+**          the [prepared statement] S having index N, or
+**          NULL if there is no SQL parameter with index N or if the
+**          parameter with index N is an anonymous parameter "?".
 */
 const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int);
 
 /*
-** CAPI3REF: Index Of A Parameter With A Given Name
+** CAPI3REF: Index Of A Parameter With A Given Name {H13640} 
 **
-** This routine returns the index of a host parameter with the given name.
-** The name must match exactly.  If no parameter with the given name is 
-** found, return 0.  Parameter names must be UTF8.
+** Return the index of an SQL parameter given its name.  The
+** index value returned is suitable for use as the second
+** parameter to [sqlite3_bind_blob|sqlite3_bind()].  A zero
+** is returned if no matching parameter is found.  The parameter
+** name must be given in UTF-8 even if the original statement
+** was prepared from UTF-16 text using [sqlite3_prepare16_v2()].
+**
+** See also: [sqlite3_bind_blob|sqlite3_bind()],
+** [sqlite3_bind_parameter_count()], and
+** [sqlite3_bind_parameter_index()].
+**
+** INVARIANTS:
+**
+** {H13641} The [sqlite3_bind_parameter_index(S,N)] interface returns
+**          the index of SQL parameter in the [prepared statement]
+**          S whose name matches the UTF-8 string N, or 0 if there is
+**          no match.
 */
 int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName);
 
 /*
-** CAPI3REF: Reset All Bindings On A Prepared Statement
+** CAPI3REF: Reset All Bindings On A Prepared Statement {H13660} 
 **
-** Contrary to the intuition of many, [sqlite3_reset()] does not
-** reset the [sqlite3_bind_blob | bindings] on a 
-** [sqlite3_stmt | prepared statement].  Use this routine to
-** reset all host parameters to NULL.
+** Contrary to the intuition of many, [sqlite3_reset()] does not reset
+** the [sqlite3_bind_blob | bindings] on a [prepared statement].
+** Use this routine to reset all host parameters to NULL.
+**
+** INVARIANTS:
+**
+** {H13661} The [sqlite3_clear_bindings(S)] interface resets all SQL
+**          parameter bindings in the [prepared statement] S back to NULL.
 */
 int sqlite3_clear_bindings(sqlite3_stmt*);
 
 /*
-** CAPI3REF: Number Of Columns In A Result Set
+** CAPI3REF: Number Of Columns In A Result Set {H13710} 
 **
-** Return the number of columns in the result set returned by the 
-** [sqlite3_stmt | compiled SQL statement]. This routine returns 0
-** if pStmt is an SQL statement that does not return data (for 
-** example an UPDATE).
+** Return the number of columns in the result set returned by the
+** [prepared statement]. This routine returns 0 if pStmt is an SQL
+** statement that does not return data (for example an [UPDATE]).
+**
+** INVARIANTS:
+**
+** {H13711} The [sqlite3_column_count(S)] interface returns the number of
+**          columns in the result set generated by the [prepared statement] S,
+**          or 0 if S does not generate a result set.
 */
 int sqlite3_column_count(sqlite3_stmt *pStmt);
 
 /*
-** CAPI3REF: Column Names In A Result Set
+** CAPI3REF: Column Names In A Result Set {H13720} 
 **
 ** These routines return the name assigned to a particular column
-** in the result set of a SELECT statement.  The sqlite3_column_name()
-** interface returns a pointer to a UTF8 string and sqlite3_column_name16()
-** returns a pointer to a UTF16 string.  The first parameter is the
-** [sqlite_stmt | prepared statement] that implements the SELECT statement.
-** The second parameter is the column number.  The left-most column is
-** number 0.
+** in the result set of a [SELECT] statement.  The sqlite3_column_name()
+** interface returns a pointer to a zero-terminated UTF-8 string
+** and sqlite3_column_name16() returns a pointer to a zero-terminated
+** UTF-16 string.  The first parameter is the [prepared statement]
+** that implements the [SELECT] statement. The second parameter is the
+** column number.  The leftmost column is number 0.
 **
-** The returned string pointer is valid until either the 
-** [sqlite_stmt | prepared statement] is destroyed by [sqlite3_finalize()]
-** or until the next call sqlite3_column_name() or sqlite3_column_name16()
-** on the same column.
+** The returned string pointer is valid until either the [prepared statement]
+** is destroyed by [sqlite3_finalize()] or until the next call to
+** sqlite3_column_name() or sqlite3_column_name16() on the same column.
+**
+** If sqlite3_malloc() fails during the processing of either routine
+** (for example during a conversion from UTF-8 to UTF-16) then a
+** NULL pointer is returned.
+**
+** The name of a result column is the value of the "AS" clause for
+** that column, if there is an AS clause.  If there is no AS clause
+** then the name of the column is unspecified and may change from
+** one release of SQLite to the next.
+**
+** INVARIANTS:
+**
+** {H13721} A successful invocation of the [sqlite3_column_name(S,N)]
+**          interface returns the name of the Nth column (where 0 is
+**          the leftmost column) for the result set of the
+**          [prepared statement] S as a zero-terminated UTF-8 string.
+**
+** {H13723} A successful invocation of the [sqlite3_column_name16(S,N)]
+**          interface returns the name of the Nth column (where 0 is
+**          the leftmost column) for the result set of the
+**          [prepared statement] S as a zero-terminated UTF-16 string
+**          in the native byte order.
+**
+** {H13724} The [sqlite3_column_name()] and [sqlite3_column_name16()]
+**          interfaces return a NULL pointer if they are unable to
+**          allocate memory to hold their normal return strings.
+**
+** {H13725} If the N parameter to [sqlite3_column_name(S,N)] or
+**          [sqlite3_column_name16(S,N)] is out of range, then the
+**          interfaces return a NULL pointer.
+**
+** {H13726} The strings returned by [sqlite3_column_name(S,N)] and
+**          [sqlite3_column_name16(S,N)] are valid until the next
+**          call to either routine with the same S and N parameters
+**          or until [sqlite3_finalize(S)] is called.
+**
+** {H13727} When a result column of a [SELECT] statement contains
+**          an AS clause, the name of that column is the identifier
+**          to the right of the AS keyword.
 */
 const char *sqlite3_column_name(sqlite3_stmt*, int N);
 const void *sqlite3_column_name16(sqlite3_stmt*, int N);
 
 /*
-** CAPI3REF: Source Of Data In A Query Result
+** CAPI3REF: Source Of Data In A Query Result {H13740} 
 **
 ** These routines provide a means to determine what column of what
-** table in which database a result of a SELECT statement comes from.
+** table in which database a result of a [SELECT] statement comes from.
 ** The name of the database or table or column can be returned as
-** either a UTF8 or UTF16 string.  The _database_ routines return
+** either a UTF-8 or UTF-16 string.  The _database_ routines return
 ** the database name, the _table_ routines return the table name, and
 ** the origin_ routines return the column name.
-** The returned string is valid until
-** the [sqlite3_stmt | prepared statement] is destroyed using
-** [sqlite3_finalize()] or until the same information is requested
+** The returned string is valid until the [prepared statement] is destroyed
+** using [sqlite3_finalize()] or until the same information is requested
 ** again in a different encoding.
 **
 ** The names returned are the original un-aliased names of the
 ** database, table, and column.
 **
-** The first argument to the following calls is a 
-** [sqlite3_stmt | compiled SQL statement].
-** These functions return information about the Nth column returned by 
+** The first argument to the following calls is a [prepared statement].
+** These functions return information about the Nth column returned by
 ** the statement, where N is the second function argument.
 **
-** If the Nth column returned by the statement is an expression
-** or subquery and is not a column value, then all of these functions
-** return NULL. Otherwise, they return the 
-** name of the attached database, table and column that query result
-** column was extracted from.
+** If the Nth column returned by the statement is an expression or
+** subquery and is not a column value, then all of these functions return
+** NULL.  These routine might also return NULL if a memory allocation error
+** occurs.  Otherwise, they return the name of the attached database, table
+** and column that query result column was extracted from.
 **
-** As with all other SQLite APIs, those postfixed with "16" return UTF-16
-** encoded strings, the other functions return UTF-8.
+** As with all other SQLite APIs, those postfixed with "16" return
+** UTF-16 encoded strings, the other functions return UTF-8. {END}
 **
-** These APIs are only available if the library was compiled with the 
-** SQLITE_ENABLE_COLUMN_METADATA preprocessor symbol defined.
+** These APIs are only available if the library was compiled with the
+** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol defined.
+**
+** {A13751}
+** If two or more threads call one or more of these routines against the same
+** prepared statement and column at the same time then the results are
+** undefined.
+**
+** INVARIANTS:
+**
+** {H13741} The [sqlite3_column_database_name(S,N)] interface returns either
+**          the UTF-8 zero-terminated name of the database from which the
+**          Nth result column of the [prepared statement] S is extracted,
+**          or NULL if the Nth column of S is a general expression
+**          or if unable to allocate memory to store the name.
+**
+** {H13742} The [sqlite3_column_database_name16(S,N)] interface returns either
+**          the UTF-16 native byte order zero-terminated name of the database
+**          from which the Nth result column of the [prepared statement] S is
+**          extracted, or NULL if the Nth column of S is a general expression
+**          or if unable to allocate memory to store the name.
+**
+** {H13743} The [sqlite3_column_table_name(S,N)] interface returns either
+**          the UTF-8 zero-terminated name of the table from which the
+**          Nth result column of the [prepared statement] S is extracted,
+**          or NULL if the Nth column of S is a general expression
+**          or if unable to allocate memory to store the name.
+**
+** {H13744} The [sqlite3_column_table_name16(S,N)] interface returns either
+**          the UTF-16 native byte order zero-terminated name of the table
+**          from which the Nth result column of the [prepared statement] S is
+**          extracted, or NULL if the Nth column of S is a general expression
+**          or if unable to allocate memory to store the name.
+**
+** {H13745} The [sqlite3_column_origin_name(S,N)] interface returns either
+**          the UTF-8 zero-terminated name of the table column from which the
+**          Nth result column of the [prepared statement] S is extracted,
+**          or NULL if the Nth column of S is a general expression
+**          or if unable to allocate memory to store the name.
+**
+** {H13746} The [sqlite3_column_origin_name16(S,N)] interface returns either
+**          the UTF-16 native byte order zero-terminated name of the table
+**          column from which the Nth result column of the
+**          [prepared statement] S is extracted, or NULL if the Nth column
+**          of S is a general expression or if unable to allocate memory
+**          to store the name.
+**
+** {H13748} The return values from
+**          [sqlite3_column_database_name | column metadata interfaces]
+**          are valid for the lifetime of the [prepared statement]
+**          or until the encoding is changed by another metadata
+**          interface call for the same prepared statement and column.
+**
+** ASSUMPTIONS:
+**
+** {A13751} If two or more threads call one or more
+**          [sqlite3_column_database_name | column metadata interfaces]
+**          for the same [prepared statement] and result column
+**          at the same time then the results are undefined.
 */
 const char *sqlite3_column_database_name(sqlite3_stmt*,int);
 const void *sqlite3_column_database_name16(sqlite3_stmt*,int);
@@ -1284,26 +3523,26 @@ const char *sqlite3_column_origin_name(sqlite3_stmt*,int);
 const void *sqlite3_column_origin_name16(sqlite3_stmt*,int);
 
 /*
-** CAPI3REF: Declared Datatype Of A Query Result
+** CAPI3REF: Declared Datatype Of A Query Result {H13760} 
 **
-** The first parameter is a [sqlite3_stmt | compiled SQL statement]. 
-** If this statement is a SELECT statement and the Nth column of the 
-** returned result set  of that SELECT is a table column (not an
+** The first parameter is a [prepared statement].
+** If this statement is a [SELECT] statement and the Nth column of the
+** returned result set of that [SELECT] is a table column (not an
 ** expression or subquery) then the declared type of the table
-** column is returned. If the Nth column of the result set is an
+** column is returned.  If the Nth column of the result set is an
 ** expression or subquery, then a NULL pointer is returned.
-** The returned string is always UTF-8 encoded. For example, in
-** the database schema:
+** The returned string is always UTF-8 encoded. {END}
+**
+** For example, given the database schema:
 **
 ** CREATE TABLE t1(c1 VARIANT);
 **
-** And the following statement compiled:
+** and the following statement to be compiled:
 **
 ** SELECT c1 + 1, c1 FROM t1;
 **
-** Then this routine would return the string "VARIANT" for the second
-** result column (i==1), and a NULL pointer for the first result column
-** (i==0).
+** this routine would return the string "VARIANT" for the second result
+** column (i==1), and a NULL pointer for the first result column (i==0).
 **
 ** SQLite uses dynamic run-time typing.  So just because a column
 ** is declared to contain a particular type does not mean that the
@@ -1311,36 +3550,55 @@ const void *sqlite3_column_origin_name16(sqlite3_stmt*,int);
 ** strongly typed, but the typing is dynamic not static.  Type
 ** is associated with individual values, not with the containers
 ** used to hold those values.
+**
+** INVARIANTS:
+**
+** {H13761}  A successful call to [sqlite3_column_decltype(S,N)] returns a
+**           zero-terminated UTF-8 string containing the declared datatype
+**           of the table column that appears as the Nth column (numbered
+**           from 0) of the result set to the [prepared statement] S.
+**
+** {H13762}  A successful call to [sqlite3_column_decltype16(S,N)]
+**           returns a zero-terminated UTF-16 native byte order string
+**           containing the declared datatype of the table column that appears
+**           as the Nth column (numbered from 0) of the result set to the
+**           [prepared statement] S.
+**
+** {H13763}  If N is less than 0 or N is greater than or equal to
+**           the number of columns in the [prepared statement] S,
+**           or if the Nth column of S is an expression or subquery rather
+**           than a table column, or if a memory allocation failure
+**           occurs during encoding conversions, then
+**           calls to [sqlite3_column_decltype(S,N)] or
+**           [sqlite3_column_decltype16(S,N)] return NULL.
 */
-const char *sqlite3_column_decltype(sqlite3_stmt *, int i);
+const char *sqlite3_column_decltype(sqlite3_stmt*,int);
 const void *sqlite3_column_decltype16(sqlite3_stmt*,int);
 
-/* 
-** CAPI3REF:  Evaluate An SQL Statement
+/*
+** CAPI3REF: Evaluate An SQL Statement {H13200} 
 **
-** After an [sqlite3_stmt | SQL statement] has been prepared with a call
-** to either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] or to one of
-** the legacy interfaces [sqlite3_prepare()] or [sqlite3_prepare16()],
-** then this function must be called one or more times to evaluate the 
-** statement.
+** After a [prepared statement] has been prepared using either
+** [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] or one of the legacy
+** interfaces [sqlite3_prepare()] or [sqlite3_prepare16()], this function
+** must be called one or more times to evaluate the statement.
 **
-** The details of the behavior of this sqlite3_step() interface depend
+** The details of the behavior of the sqlite3_step() interface depend
 ** on whether the statement was prepared using the newer "v2" interface
 ** [sqlite3_prepare_v2()] and [sqlite3_prepare16_v2()] or the older legacy
 ** interface [sqlite3_prepare()] and [sqlite3_prepare16()].  The use of the
 ** new "v2" interface is recommended for new applications but the legacy
 ** interface will continue to be supported.
 **
-** In the lagacy interface, the return value will be either [SQLITE_BUSY], 
+** In the legacy interface, the return value will be either [SQLITE_BUSY],
 ** [SQLITE_DONE], [SQLITE_ROW], [SQLITE_ERROR], or [SQLITE_MISUSE].
-** With the "v2" interface, any of the other [SQLITE_OK | result code]
-** or [SQLITE_IOERR_READ | extended result code] might be returned as
-** well.
+** With the "v2" interface, any of the other [result codes] or
+** [extended result codes] might be returned as well.
 **
 ** [SQLITE_BUSY] means that the database engine was unable to acquire the
-** database locks it needs to do its job.  If the statement is a COMMIT
+** database locks it needs to do its job.  If the statement is a [COMMIT]
 ** or occurs outside of an explicit transaction, then you can retry the
-** statement.  If the statement is not a COMMIT and occurs within a
+** statement.  If the statement is not a [COMMIT] and occurs within a
 ** explicit transaction then you should rollback the transaction before
 ** continuing.
 **
@@ -1349,62 +3607,91 @@ const void *sqlite3_column_decltype16(sqlite3_stmt*,int);
 ** machine without first calling [sqlite3_reset()] to reset the virtual
 ** machine back to its initial state.
 **
-** If the SQL statement being executed returns any data, then 
-** [SQLITE_ROW] is returned each time a new row of data is ready
-** for processing by the caller. The values may be accessed using
-** the [sqlite3_column_int | column access functions].
+** If the SQL statement being executed returns any data, then [SQLITE_ROW]
+** is returned each time a new row of data is ready for processing by the
+** caller. The values may be accessed using the [column access functions].
 ** sqlite3_step() is called again to retrieve the next row of data.
-** 
+**
 ** [SQLITE_ERROR] means that a run-time error (such as a constraint
 ** violation) has occurred.  sqlite3_step() should not be called again on
 ** the VM. More information may be found by calling [sqlite3_errmsg()].
-** With the legacy interface, a more specific error code (example:
+** With the legacy interface, a more specific error code (for example,
 ** [SQLITE_INTERRUPT], [SQLITE_SCHEMA], [SQLITE_CORRUPT], and so forth)
 ** can be obtained by calling [sqlite3_reset()] on the
-** [sqlite_stmt | prepared statement].  In the "v2" interface,
+** [prepared statement].  In the "v2" interface,
 ** the more specific error code is returned directly by sqlite3_step().
 **
 ** [SQLITE_MISUSE] means that the this routine was called inappropriately.
-** Perhaps it was called on a [sqlite_stmt | prepared statement] that has
-** already been [sqlite3_finalize | finalized] or on one that had 
+** Perhaps it was called on a [prepared statement] that has
+** already been [sqlite3_finalize | finalized] or on one that had
 ** previously returned [SQLITE_ERROR] or [SQLITE_DONE].  Or it could
 ** be the case that the same database connection is being used by two or
 ** more threads at the same moment in time.
 **
-** Goofy Interface Alert:
-** In the legacy interface, 
-** the sqlite3_step() API always returns a generic error code,
-** [SQLITE_ERROR], following any error other than [SQLITE_BUSY]
-** and [SQLITE_MISUSE].  You must call [sqlite3_reset()] or
-** [sqlite3_finalize()] in order to find one of the specific
-** [SQLITE_ERROR | result codes] that better describes the error.
+** Goofy Interface Alert: In the legacy interface, the sqlite3_step()
+** API always returns a generic error code, [SQLITE_ERROR], following any
+** error other than [SQLITE_BUSY] and [SQLITE_MISUSE].  You must call
+** [sqlite3_reset()] or [sqlite3_finalize()] in order to find one of the
+** specific [error codes] that better describes the error.
 ** We admit that this is a goofy design.  The problem has been fixed
 ** with the "v2" interface.  If you prepare all of your SQL statements
 ** using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] instead
-** of the legacy [sqlite3_prepare()] and [sqlite3_prepare16()], then the 
-** more specific [SQLITE_ERROR | result codes] are returned directly
+** of the legacy [sqlite3_prepare()] and [sqlite3_prepare16()] interfaces,
+** then the more specific [error codes] are returned directly
 ** by sqlite3_step().  The use of the "v2" interface is recommended.
+**
+** INVARIANTS:
+**
+** {H13202}  If the [prepared statement] S is ready to be run, then
+**           [sqlite3_step(S)] advances that prepared statement until
+**           completion or until it is ready to return another row of the
+**           result set, or until an [sqlite3_interrupt | interrupt]
+**           or a run-time error occurs.
+**
+** {H15304}  When a call to [sqlite3_step(S)] causes the [prepared statement]
+**           S to run to completion, the function returns [SQLITE_DONE].
+**
+** {H15306}  When a call to [sqlite3_step(S)] stops because it is ready to
+**           return another row of the result set, it returns [SQLITE_ROW].
+**
+** {H15308}  If a call to [sqlite3_step(S)] encounters an
+**           [sqlite3_interrupt | interrupt] or a run-time error,
+**           it returns an appropriate error code that is not one of
+**           [SQLITE_OK], [SQLITE_ROW], or [SQLITE_DONE].
+**
+** {H15310}  If an [sqlite3_interrupt | interrupt] or a run-time error
+**           occurs during a call to [sqlite3_step(S)]
+**           for a [prepared statement] S created using
+**           legacy interfaces [sqlite3_prepare()] or
+**           [sqlite3_prepare16()], then the function returns either
+**           [SQLITE_ERROR], [SQLITE_BUSY], or [SQLITE_MISUSE].
 */
 int sqlite3_step(sqlite3_stmt*);
 
 /*
-** CAPI3REF:
+** CAPI3REF: Number of columns in a result set {H13770} 
 **
-** Return the number of values in the current row of the result set.
+** Returns the number of values in the current row of the result set.
 **
-** After a call to [sqlite3_step()] that returns [SQLITE_ROW], this routine
-** will return the same value as the [sqlite3_column_count()] function.
-** After [sqlite3_step()] has returned an [SQLITE_DONE], [SQLITE_BUSY], or
-** a [SQLITE_ERROR | error code], or before [sqlite3_step()] has been 
-** called on the [sqlite_stmt | prepared statement] for the first time,
-** this routine returns zero.
+** INVARIANTS:
+**
+** {H13771}  After a call to [sqlite3_step(S)] that returns [SQLITE_ROW],
+**           the [sqlite3_data_count(S)] routine will return the same value
+**           as the [sqlite3_column_count(S)] function.
+**
+** {H13772}  After [sqlite3_step(S)] has returned any value other than
+**           [SQLITE_ROW] or before [sqlite3_step(S)] has been called on the
+**           [prepared statement] for the first time since it was
+**           [sqlite3_prepare | prepared] or [sqlite3_reset | reset],
+**           the [sqlite3_data_count(S)] routine returns zero.
 */
 int sqlite3_data_count(sqlite3_stmt *pStmt);
 
 /*
-** CAPI3REF: Fundamental Datatypes
+** CAPI3REF: Fundamental Datatypes {H10265} 
+** KEYWORDS: SQLITE_TEXT
 **
-** Every value in SQLite has one of five fundamental datatypes:
+** {H10266} Every value in SQLite has one of five fundamental datatypes:
 **
 ** 
    
 ** 
  •  64-bit signed integer
@@ -1412,13 +3699,13 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** 
  •  string
 ** 
  •  BLOB
 ** 
  •  NULL
-** 

+**  {END}
 **
 ** These constants are codes for each of those types.
 **
 ** Note that the SQLITE_TEXT constant was also used in SQLite version 2
 ** for a completely different meaning.  Software that links against both
-** SQLite version 2 and SQLite version 3 should use SQLITE3_TEXT not
+** SQLite version 2 and SQLite version 3 should use SQLITE3_TEXT, not
 ** SQLITE_TEXT.
 */
 #define SQLITE_INTEGER  1
@@ -1433,21 +3720,31 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 #define SQLITE3_TEXT     3
 
 /*
-** CAPI3REF: Results Values From A Query
+** CAPI3REF: Result Values From A Query {H13800} 
+** KEYWORDS: {column access functions}
 **
-** These routines return information about the information
-** in a single column of the current result row of a query.  In every
-** case the first argument is a pointer to the 
-** [sqlite3_stmt | SQL statement] that is being
-** evaluate (the [sqlite_stmt*] that was returned from 
-** [sqlite3_prepare_v2()] or one of its variants) and
-** the second argument is the index of the column for which information 
-** should be returned.  The left-most column has an index of 0.
+** These routines form the "result set query" interface.
 **
-** If the SQL statement is not currently point to a valid row, or if the
-** the column index is out of range, the result is undefined.
+** These routines return information about a single column of the current
+** result row of a query.  In every case the first argument is a pointer
+** to the [prepared statement] that is being evaluated (the [sqlite3_stmt*]
+** that was returned from [sqlite3_prepare_v2()] or one of its variants)
+** and the second argument is the index of the column for which information
+** should be returned.  The leftmost column of the result set has the index 0.
 **
-** The sqlite3_column_type() routine returns 
+** If the SQL statement does not currently point to a valid row, or if the
+** column index is out of range, the result is undefined.
+** These routines may only be called when the most recent call to
+** [sqlite3_step()] has returned [SQLITE_ROW] and neither
+** [sqlite3_reset()] nor [sqlite3_finalize()] have been called subsequently.
+** If any of these routines are called after [sqlite3_reset()] or
+** [sqlite3_finalize()] or after [sqlite3_step()] has returned
+** something other than [SQLITE_ROW], the results are undefined.
+** If [sqlite3_step()] or [sqlite3_reset()] or [sqlite3_finalize()]
+** are called from a different thread while any of these routines
+** are pending, then the results are undefined.
+**
+** The sqlite3_column_type() routine returns the
 ** [SQLITE_INTEGER | datatype code] for the initial data type
 ** of the result column.  The returned value is one of [SQLITE_INTEGER],
 ** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL].  The value
@@ -1457,7 +3754,7 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** versions of SQLite may change the behavior of sqlite3_column_type()
 ** following a type conversion.
 **
-** If the result is a BLOB or UTF-8 string then the sqlite3_column_bytes() 
+** If the result is a BLOB or UTF-8 string then the sqlite3_column_bytes()
 ** routine returns the number of bytes in that BLOB or string.
 ** If the result is a UTF-16 string, then sqlite3_column_bytes() converts
 ** the string to UTF-8 and then returns the number of bytes.
@@ -1468,20 +3765,32 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** of the string.  For clarity: the value returned is the number of
 ** bytes in the string, not the number of characters.
 **
+** Strings returned by sqlite3_column_text() and sqlite3_column_text16(),
+** even empty strings, are always zero terminated.  The return
+** value from sqlite3_column_blob() for a zero-length BLOB is an arbitrary
+** pointer, possibly even a NULL pointer.
+**
 ** The sqlite3_column_bytes16() routine is similar to sqlite3_column_bytes()
-** but leaves the result in UTF-16 instead of UTF-8.  
+** but leaves the result in UTF-16 in native byte order instead of UTF-8.
 ** The zero terminator is not included in this count.
 **
+** The object returned by [sqlite3_column_value()] is an
+** [unprotected sqlite3_value] object.  An unprotected sqlite3_value object
+** may only be used with [sqlite3_bind_value()] and [sqlite3_result_value()].
+** If the [unprotected sqlite3_value] object returned by
+** [sqlite3_column_value()] is used in any other way, including calls
+** to routines like [sqlite3_value_int()], [sqlite3_value_text()],
+** or [sqlite3_value_bytes()], then the behavior is undefined.
+**
 ** These routines attempt to convert the value where appropriate.  For
 ** example, if the internal representation is FLOAT and a text result
-** is requested, [sqlite3_snprintf()] is used internally to do the conversion
-** automatically.  The following table details the conversions that
-** are applied:
+** is requested, [sqlite3_snprintf()] is used internally to perform the
+** conversion automatically.  The following table details the conversions
+** that are applied:
 **
 ** 

 **
-**
Internal Requested -**
Type Type Conversion +**
Internal
Type 		Requested
Type 		Conversion ** **
NULL INTEGER Result is 0 **
NULL FLOAT Result is 0.0 @@ -1489,7 +3798,7 @@ int sqlite3_data_count(sqlite3_stmt *pStmt); **
NULL BLOB Result is NULL pointer **
INTEGER FLOAT Convert from integer to float **
INTEGER TEXT ASCII rendering of the integer -**
INTEGER BLOB Same as for INTEGER->TEXT +**
INTEGER BLOB Same as INTEGER->TEXT **
FLOAT INTEGER Convert from float to integer **
FLOAT TEXT ASCII rendering of the float **
FLOAT BLOB Same as FLOAT->TEXT @@ -1504,176 +3813,349 @@ int sqlite3_data_count(sqlite3_stmt *pStmt); ** ** The table above makes reference to standard C library functions atoi() ** and atof(). SQLite does not really use these functions. It has its -** on equavalent internal routines. The atoi() and atof() names are +** own equivalent internal routines. The atoi() and atof() names are ** used in the table for brevity and because they are familiar to most ** C programmers. ** ** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or -** sqlite3_column_text16() may be invalidated. +** sqlite3_column_text16() may be invalidated. ** Type conversions and pointer invalidations might occur ** in the following cases: ** **
    -**

  • The initial content is a BLOB and sqlite3_column_text() -** or sqlite3_column_text16() is called. A zero-terminator might -** need to be added to the string.

    • -** -**

  • The initial content is UTF-8 text and sqlite3_column_bytes16() or -** sqlite3_column_text16() is called. The content must be converted -** to UTF-16.

    • -** -**

  • The initial content is UTF-16 text and sqlite3_column_bytes() or -** sqlite3_column_text() is called. The content must be converted -** to UTF-8.

    • +**
  • The initial content is a BLOB and sqlite3_column_text() or +** sqlite3_column_text16() is called. A zero-terminator might +** need to be added to the string.
    • +**
  • The initial content is UTF-8 text and sqlite3_column_bytes16() or +** sqlite3_column_text16() is called. The content must be converted +** to UTF-16.
    • +**
  • The initial content is UTF-16 text and sqlite3_column_bytes() or +** sqlite3_column_text() is called. The content must be converted +** to UTF-8.
    • **
** ** Conversions between UTF-16be and UTF-16le are always done in place and do ** not invalidate a prior pointer, though of course the content of the buffer ** that the prior pointer points to will have been modified. Other kinds -** of conversion are done in place when it is possible, but sometime it is -** not possible and in those cases prior pointers are invalidated. +** of conversion are done in place when it is possible, but sometimes they +** are not possible and in those cases prior pointers are invalidated. ** ** The safest and easiest to remember policy is to invoke these routines ** in one of the following ways: ** -**
    +**
      **
    • sqlite3_column_text() followed by sqlite3_column_bytes()
      • **
    • sqlite3_column_blob() followed by sqlite3_column_bytes()
      • **
    • sqlite3_column_text16() followed by sqlite3_column_bytes16()
      • -**
    +**
** -** In other words, you should call sqlite3_column_text(), sqlite3_column_blob(), -** or sqlite3_column_text16() first to force the result into the desired -** format, then invoke sqlite3_column_bytes() or sqlite3_column_bytes16() to -** find the size of the result. Do not mix call to sqlite3_column_text() or -** sqlite3_column_blob() with calls to sqlite3_column_bytes16(). And do not -** mix calls to sqlite3_column_text16() with calls to sqlite3_column_bytes(). +** In other words, you should call sqlite3_column_text(), +** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result +** into the desired format, then invoke sqlite3_column_bytes() or +** sqlite3_column_bytes16() to find the size of the result. Do not mix calls +** to sqlite3_column_text() or sqlite3_column_blob() with calls to +** sqlite3_column_bytes16(), and do not mix calls to sqlite3_column_text16() +** with calls to sqlite3_column_bytes(). +** +** The pointers returned are valid until a type conversion occurs as +** described above, or until [sqlite3_step()] or [sqlite3_reset()] or +** [sqlite3_finalize()] is called. The memory space used to hold strings +** and BLOBs is freed automatically. Do not pass the pointers returned +** [sqlite3_column_blob()], [sqlite3_column_text()], etc. into +** [sqlite3_free()]. +** +** If a memory allocation error occurs during the evaluation of any +** of these routines, a default value is returned. The default value +** is either the integer 0, the floating point number 0.0, or a NULL +** pointer. Subsequent calls to [sqlite3_errcode()] will return +** [SQLITE_NOMEM]. +** +** INVARIANTS: +** +** {H13803} The [sqlite3_column_blob(S,N)] interface converts the +** Nth column in the current row of the result set for +** the [prepared statement] S into a BLOB and then returns a +** pointer to the converted value. +** +** {H13806} The [sqlite3_column_bytes(S,N)] interface returns the +** number of bytes in the BLOB or string (exclusive of the +** zero terminator on the string) that was returned by the +** most recent call to [sqlite3_column_blob(S,N)] or +** [sqlite3_column_text(S,N)]. +** +** {H13809} The [sqlite3_column_bytes16(S,N)] interface returns the +** number of bytes in the string (exclusive of the +** zero terminator on the string) that was returned by the +** most recent call to [sqlite3_column_text16(S,N)]. +** +** {H13812} The [sqlite3_column_double(S,N)] interface converts the +** Nth column in the current row of the result set for the +** [prepared statement] S into a floating point value and +** returns a copy of that value. +** +** {H13815} The [sqlite3_column_int(S,N)] interface converts the +** Nth column in the current row of the result set for the +** [prepared statement] S into a 64-bit signed integer and +** returns the lower 32 bits of that integer. +** +** {H13818} The [sqlite3_column_int64(S,N)] interface converts the +** Nth column in the current row of the result set for the +** [prepared statement] S into a 64-bit signed integer and +** returns a copy of that integer. +** +** {H13821} The [sqlite3_column_text(S,N)] interface converts the +** Nth column in the current row of the result set for +** the [prepared statement] S into a zero-terminated UTF-8 +** string and returns a pointer to that string. +** +** {H13824} The [sqlite3_column_text16(S,N)] interface converts the +** Nth column in the current row of the result set for the +** [prepared statement] S into a zero-terminated 2-byte +** aligned UTF-16 native byte order string and returns +** a pointer to that string. +** +** {H13827} The [sqlite3_column_type(S,N)] interface returns +** one of [SQLITE_NULL], [SQLITE_INTEGER], [SQLITE_FLOAT], +** [SQLITE_TEXT], or [SQLITE_BLOB] as appropriate for +** the Nth column in the current row of the result set for +** the [prepared statement] S. +** +** {H13830} The [sqlite3_column_value(S,N)] interface returns a +** pointer to an [unprotected sqlite3_value] object for the +** Nth column in the current row of the result set for +** the [prepared statement] S. */ const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); int sqlite3_column_bytes(sqlite3_stmt*, int iCol); int sqlite3_column_bytes16(sqlite3_stmt*, int iCol); double sqlite3_column_double(sqlite3_stmt*, int iCol); int sqlite3_column_int(sqlite3_stmt*, int iCol); -sqlite_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol); +sqlite3_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol); const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); const void *sqlite3_column_text16(sqlite3_stmt*, int iCol); int sqlite3_column_type(sqlite3_stmt*, int iCol); sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); /* -** CAPI3REF: Destroy A Prepared Statement Object +** CAPI3REF: Destroy A Prepared Statement Object {H13300} ** -** The sqlite3_finalize() function is called to delete a -** [sqlite3_stmt | compiled SQL statement]. If the statement was -** executed successfully, or not executed at all, then SQLITE_OK is returned. -** If execution of the statement failed then an -** [SQLITE_ERROR | error code] or [SQLITE_IOERR_READ | extended error code] -** is returned. +** The sqlite3_finalize() function is called to delete a [prepared statement]. +** If the statement was executed successfully or not executed at all, then +** SQLITE_OK is returned. If execution of the statement failed then an +** [error code] or [extended error code] is returned. ** ** This routine can be called at any point during the execution of the -** [sqlite3_stmt | virtual machine]. If the virtual machine has not +** [prepared statement]. If the virtual machine has not ** completed execution when this routine is called, that is like -** encountering an error or an interrupt. (See [sqlite3_interrupt()].) -** Incomplete updates may be rolled back and transactions cancelled, -** depending on the circumstances, and the -** [SQLITE_ERROR | result code] returned will be [SQLITE_ABORT]. +** encountering an error or an [sqlite3_interrupt | interrupt]. +** Incomplete updates may be rolled back and transactions canceled, +** depending on the circumstances, and the +** [error code] returned will be [SQLITE_ABORT]. +** +** INVARIANTS: +** +** {H11302} The [sqlite3_finalize(S)] interface destroys the +** [prepared statement] S and releases all +** memory and file resources held by that object. +** +** {H11304} If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S returned an error, +** then [sqlite3_finalize(S)] returns that same error. */ int sqlite3_finalize(sqlite3_stmt *pStmt); /* -** CAPI3REF: Reset A Prepared Statement Object +** CAPI3REF: Reset A Prepared Statement Object {H13330} ** -** The sqlite3_reset() function is called to reset a -** [sqlite_stmt | compiled SQL statement] object. -** back to it's initial state, ready to be re-executed. +** The sqlite3_reset() function is called to reset a [prepared statement] +** object back to its initial state, ready to be re-executed. ** Any SQL statement variables that had values bound to them using ** the [sqlite3_bind_blob | sqlite3_bind_*() API] retain their values. ** Use [sqlite3_clear_bindings()] to reset the bindings. +** +** {H11332} The [sqlite3_reset(S)] interface resets the [prepared statement] S +** back to the beginning of its program. +** +** {H11334} If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE], +** or if [sqlite3_step(S)] has never before been called on S, +** then [sqlite3_reset(S)] returns [SQLITE_OK]. +** +** {H11336} If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S indicated an error, then +** [sqlite3_reset(S)] returns an appropriate [error code]. +** +** {H11338} The [sqlite3_reset(S)] interface does not change the values +** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. */ int sqlite3_reset(sqlite3_stmt *pStmt); /* -** CAPI3REF: Create Or Redefine SQL Functions +** CAPI3REF: Create Or Redefine SQL Functions {H16100} +** KEYWORDS: {function creation routines} +** KEYWORDS: {application-defined SQL function} +** KEYWORDS: {application-defined SQL functions} ** -** The following two functions are used to add SQL functions or aggregates -** or to redefine the behavior of existing SQL functions or aggregates. The -** difference only between the two is that the second parameter, the -** name of the (scalar) function or aggregate, is encoded in UTF-8 for -** sqlite3_create_function() and UTF-16 for sqlite3_create_function16(). +** These two functions (collectively known as "function creation routines") +** are used to add SQL functions or aggregates or to redefine the behavior +** of existing SQL functions or aggregates. The only difference between the +** two is that the second parameter, the name of the (scalar) function or +** aggregate, is encoded in UTF-8 for sqlite3_create_function() and UTF-16 +** for sqlite3_create_function16(). ** -** The first argument is the [sqlite3 | database handle] that holds the -** SQL function or aggregate is to be added or redefined. If a single -** program uses more than one database handle internally, then SQL -** functions or aggregates must be added individually to each database -** handle with which they will be used. +** The first parameter is the [database connection] to which the SQL +** function is to be added. If a single program uses more than one database +** connection internally, then SQL functions must be added individually to +** each database connection. ** -** The second parameter is the name of the SQL function to be created -** or redefined. -** The length of the name is limited to 255 bytes, exclusive of the -** zero-terminator. Note that the name length limit is in bytes, not +** The second parameter is the name of the SQL function to be created or +** redefined. The length of the name is limited to 255 bytes, exclusive of +** the zero-terminator. Note that the name length limit is in bytes, not ** characters. Any attempt to create a function with a longer name -** will result in an SQLITE_ERROR error. +** will result in [SQLITE_ERROR] being returned. ** -** The third parameter is the number of arguments that the SQL function or +** The third parameter (nArg) +** is the number of arguments that the SQL function or ** aggregate takes. If this parameter is negative, then the SQL function or ** aggregate may take any number of arguments. ** -** The fourth parameter, eTextRep, specifies what +** The fourth parameter, eTextRep, specifies what ** [SQLITE_UTF8 | text encoding] this SQL function prefers for ** its parameters. Any SQL function implementation should be able to work ** work with UTF-8, UTF-16le, or UTF-16be. But some implementations may be ** more efficient with one encoding than another. It is allowed to -** invoke sqlite_create_function() or sqlite3_create_function16() multiple +** invoke sqlite3_create_function() or sqlite3_create_function16() multiple ** times with the same function but with different values of eTextRep. ** When multiple implementations of the same function are available, SQLite ** will pick the one that involves the least amount of data conversion. -** If there is only a single implementation which does not care what -** text encoding is used, then the fourth argument should be -** [SQLITE_ANY]. +** If there is only a single implementation which does not care what text +** encoding is used, then the fourth argument should be [SQLITE_ANY]. ** -** The fifth parameter is an arbitrary pointer. The implementation -** of the function can gain access to this pointer using -** [sqlite_user_data()]. +** The fifth parameter is an arbitrary pointer. The implementation of the +** function can gain access to this pointer using [sqlite3_user_data()]. ** ** The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are -** pointers to C-language functions that implement the SQL -** function or aggregate. A scalar SQL function requires an implementation of -** the xFunc callback only, NULL pointers should be passed as the xStep -** and xFinal parameters. An aggregate SQL function requires an implementation -** of xStep and xFinal and NULL should be passed for xFunc. To delete an -** existing SQL function or aggregate, pass NULL for all three function -** callback. +** pointers to C-language functions that implement the SQL function or +** aggregate. A scalar SQL function requires an implementation of the xFunc +** callback only, NULL pointers should be passed as the xStep and xFinal +** parameters. An aggregate SQL function requires an implementation of xStep +** and xFinal and NULL should be passed for xFunc. To delete an existing +** SQL function or aggregate, pass NULL for all three function callbacks. ** ** It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of -** arguments or differing perferred text encodings. SQLite will use +** arguments or differing preferred text encodings. SQLite will use ** the implementation most closely matches the way in which the -** SQL function is used. +** SQL function is used. A function implementation with a non-negative +** nArg parameter is a better match than a function implementation with +** a negative nArg. A function where the preferred text encoding +** matches the database encoding is a better +** match than a function where the encoding is different. +** A function where the encoding difference is between UTF16le and UTF16be +** is a closer match than a function where the encoding difference is +** between UTF8 and UTF16. +** +** Built-in functions may be overloaded by new application-defined functions. +** The first application-defined function with a given name overrides all +** built-in functions in the same [database connection] with the same name. +** Subsequent application-defined functions of the same name only override +** prior application-defined functions that are an exact match for the +** number of parameters and preferred encoding. +** +** An application-defined function is permitted to call other +** SQLite interfaces. However, such calls must not +** close the database connection nor finalize or reset the prepared +** statement in which the function is running. +** +** INVARIANTS: +** +** {H16103} The [sqlite3_create_function16(D,X,...)] interface shall behave +** as [sqlite3_create_function(D,X,...)] in every way except that it +** interprets the X argument as zero-terminated UTF-16 +** native byte order instead of as zero-terminated UTF-8. +** +** {H16106} A successful invocation of the +** [sqlite3_create_function(D,X,N,E,...)] interface shall register +** or replaces callback functions in the [database connection] D +** used to implement the SQL function named X with N parameters +** and having a preferred text encoding of E. +** +** {H16109} A successful call to [sqlite3_create_function(D,X,N,E,P,F,S,L)] +** shall replace the P, F, S, and L values from any prior calls with +** the same D, X, N, and E values. +** +** {H16112} The [sqlite3_create_function(D,X,...)] interface shall fail +** if the SQL function name X is +** longer than 255 bytes exclusive of the zero terminator. +** +** {H16118} The [sqlite3_create_function(D,X,N,E,P,F,S,L)] interface +** shall fail unless either F is NULL and S and L are non-NULL or +*** F is non-NULL and S and L are NULL. +** +** {H16121} The [sqlite3_create_function(D,...)] interface shall fails with an +** error code of [SQLITE_BUSY] if there exist [prepared statements] +** associated with the [database connection] D. +** +** {H16124} The [sqlite3_create_function(D,X,N,...)] interface shall fail with +** an error code of [SQLITE_ERROR] if parameter N is less +** than -1 or greater than 127. +** +** {H16127} When N is non-negative, the [sqlite3_create_function(D,X,N,...)] +** interface shall register callbacks to be invoked for the +** SQL function +** named X when the number of arguments to the SQL function is +** exactly N. +** +** {H16130} When N is -1, the [sqlite3_create_function(D,X,N,...)] +** interface shall register callbacks to be invoked for the SQL +** function named X with any number of arguments. +** +** {H16133} When calls to [sqlite3_create_function(D,X,N,...)] +** specify multiple implementations of the same function X +** and when one implementation has N>=0 and the other has N=(-1) +** the implementation with a non-zero N shall be preferred. +** +** {H16136} When calls to [sqlite3_create_function(D,X,N,E,...)] +** specify multiple implementations of the same function X with +** the same number of arguments N but with different +** encodings E, then the implementation where E matches the +** database encoding shall preferred. +** +** {H16139} For an aggregate SQL function created using +** [sqlite3_create_function(D,X,N,E,P,0,S,L)] the finalizer +** function L shall always be invoked exactly once if the +** step function S is called one or more times. +** +** {H16142} When SQLite invokes either the xFunc or xStep function of +** an application-defined SQL function or aggregate created +** by [sqlite3_create_function()] or [sqlite3_create_function16()], +** then the array of [sqlite3_value] objects passed as the +** third parameter shall be [protected sqlite3_value] objects. */ int sqlite3_create_function( - sqlite3 *, + sqlite3 *db, const char *zFunctionName, int nArg, int eTextRep, - void*, + void *pApp, void (*xFunc)(sqlite3_context*,int,sqlite3_value**), void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); int sqlite3_create_function16( - sqlite3*, + sqlite3 *db, const void *zFunctionName, int nArg, int eTextRep, - void*, + void *pApp, void (*xFunc)(sqlite3_context*,int,sqlite3_value**), void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); /* -** CAPI3REF: Text Encodings +** CAPI3REF: Text Encodings {H10267} ** ** These constant define integer codes that represent the various ** text encodings supported by SQLite. @@ -1686,22 +4168,26 @@ int sqlite3_create_function16( #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ /* -** CAPI3REF: Obsolete Functions +** CAPI3REF: Deprecated Functions +** DEPRECATED ** -** These functions are all now obsolete. In order to maintain -** backwards compatibility with older code, we continue to support -** these functions. However, new development projects should avoid +** These functions are [deprecated]. In order to maintain +** backwards compatibility with older code, these functions continue +** to be supported. However, new applications should avoid ** the use of these functions. To help encourage people to avoid -** using these functions, we are not going to tell you want they do. +** using these functions, we are not going to tell you what they do. */ -int sqlite3_aggregate_count(sqlite3_context*); -int sqlite3_expired(sqlite3_stmt*); -int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); -int sqlite3_global_recover(void); - +#ifndef SQLITE_OMIT_DEPRECATED +SQLITE_DEPRECATED int sqlite3_aggregate_count(sqlite3_context*); +SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); +SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); +SQLITE_DEPRECATED int sqlite3_global_recover(void); +SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); +SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),void*,sqlite3_int64); +#endif /* -** CAPI3REF: Obtaining SQL Function Parameter Values +** CAPI3REF: Obtaining SQL Function Parameter Values {H15100} ** ** The C-language implementation of SQL functions and aggregates uses ** this set of interface routines to access the parameter values on @@ -1711,40 +4197,108 @@ int sqlite3_global_recover(void); ** to [sqlite3_create_function()] and [sqlite3_create_function16()] ** define callbacks that implement the SQL functions and aggregates. ** The 4th parameter to these callbacks is an array of pointers to -** [sqlite3_value] objects. There is one [sqlite3_value] object for +** [protected sqlite3_value] objects. There is one [sqlite3_value] object for ** each parameter to the SQL function. These routines are used to ** extract values from the [sqlite3_value] objects. ** -** These routines work just like the corresponding -** [sqlite3_column_blob | sqlite3_column_* routines] except that -** these routines take a single [sqlite3_value*] pointer instead -** of an [sqlite3_stmt*] pointer and an integer column number. +** These routines work only with [protected sqlite3_value] objects. +** Any attempt to use these routines on an [unprotected sqlite3_value] +** object results in undefined behavior. ** -** The sqlite3_value_text16() interface extracts a UTF16 string +** These routines work just like the corresponding [column access functions] +** except that these routines take a single [protected sqlite3_value] object +** pointer instead of a [sqlite3_stmt*] pointer and an integer column number. +** +** The sqlite3_value_text16() interface extracts a UTF-16 string ** in the native byte-order of the host machine. The ** sqlite3_value_text16be() and sqlite3_value_text16le() interfaces -** extract UTF16 strings as big-endian and little-endian respectively. +** extract UTF-16 strings as big-endian and little-endian respectively. ** ** The sqlite3_value_numeric_type() interface attempts to apply ** numeric affinity to the value. This means that an attempt is ** made to convert the value to an integer or floating point. If -** such a conversion is possible without loss of information (in order -** words if the value is original a string that looks like a number) -** then it is done. Otherwise no conversion occurs. The -** [SQLITE_INTEGER | datatype] after conversion is returned. +** such a conversion is possible without loss of information (in other +** words, if the value is a string that looks like a number) +** then the conversion is performed. Otherwise no conversion occurs. +** The [SQLITE_INTEGER | datatype] after conversion is returned. ** -** Please pay particular attention to the fact that the pointer that -** is returned from [sqlite3_value_blob()], [sqlite3_value_text()], or +** Please pay particular attention to the fact that the pointer returned +** from [sqlite3_value_blob()], [sqlite3_value_text()], or ** [sqlite3_value_text16()] can be invalidated by a subsequent call to -** [sqlite3_value_bytes()], [sqlite3_value_bytes16()], [sqlite_value_text()], -** or [sqlite3_value_text16()]. +** [sqlite3_value_bytes()], [sqlite3_value_bytes16()], [sqlite3_value_text()], +** or [sqlite3_value_text16()]. +** +** These routines must be called from the same thread as +** the SQL function that supplied the [sqlite3_value*] parameters. +** +** INVARIANTS: +** +** {H15103} The [sqlite3_value_blob(V)] interface converts the +** [protected sqlite3_value] object V into a BLOB and then +** returns a pointer to the converted value. +** +** {H15106} The [sqlite3_value_bytes(V)] interface returns the +** number of bytes in the BLOB or string (exclusive of the +** zero terminator on the string) that was returned by the +** most recent call to [sqlite3_value_blob(V)] or +** [sqlite3_value_text(V)]. +** +** {H15109} The [sqlite3_value_bytes16(V)] interface returns the +** number of bytes in the string (exclusive of the +** zero terminator on the string) that was returned by the +** most recent call to [sqlite3_value_text16(V)], +** [sqlite3_value_text16be(V)], or [sqlite3_value_text16le(V)]. +** +** {H15112} The [sqlite3_value_double(V)] interface converts the +** [protected sqlite3_value] object V into a floating point value and +** returns a copy of that value. +** +** {H15115} The [sqlite3_value_int(V)] interface converts the +** [protected sqlite3_value] object V into a 64-bit signed integer and +** returns the lower 32 bits of that integer. +** +** {H15118} The [sqlite3_value_int64(V)] interface converts the +** [protected sqlite3_value] object V into a 64-bit signed integer and +** returns a copy of that integer. +** +** {H15121} The [sqlite3_value_text(V)] interface converts the +** [protected sqlite3_value] object V into a zero-terminated UTF-8 +** string and returns a pointer to that string. +** +** {H15124} The [sqlite3_value_text16(V)] interface converts the +** [protected sqlite3_value] object V into a zero-terminated 2-byte +** aligned UTF-16 native byte order +** string and returns a pointer to that string. +** +** {H15127} The [sqlite3_value_text16be(V)] interface converts the +** [protected sqlite3_value] object V into a zero-terminated 2-byte +** aligned UTF-16 big-endian +** string and returns a pointer to that string. +** +** {H15130} The [sqlite3_value_text16le(V)] interface converts the +** [protected sqlite3_value] object V into a zero-terminated 2-byte +** aligned UTF-16 little-endian +** string and returns a pointer to that string. +** +** {H15133} The [sqlite3_value_type(V)] interface returns +** one of [SQLITE_NULL], [SQLITE_INTEGER], [SQLITE_FLOAT], +** [SQLITE_TEXT], or [SQLITE_BLOB] as appropriate for +** the [sqlite3_value] object V. +** +** {H15136} The [sqlite3_value_numeric_type(V)] interface converts +** the [protected sqlite3_value] object V into either an integer or +** a floating point value if it can do so without loss of +** information, and returns one of [SQLITE_NULL], +** [SQLITE_INTEGER], [SQLITE_FLOAT], [SQLITE_TEXT], or +** [SQLITE_BLOB] as appropriate for the +** [protected sqlite3_value] object V after the conversion attempt. */ const void *sqlite3_value_blob(sqlite3_value*); int sqlite3_value_bytes(sqlite3_value*); int sqlite3_value_bytes16(sqlite3_value*); double sqlite3_value_double(sqlite3_value*); int sqlite3_value_int(sqlite3_value*); -sqlite_int64 sqlite3_value_int64(sqlite3_value*); +sqlite3_int64 sqlite3_value_int64(sqlite3_value*); const unsigned char *sqlite3_value_text(sqlite3_value*); const void *sqlite3_value_text16(sqlite3_value*); const void *sqlite3_value_text16le(sqlite3_value*); @@ -1753,76 +4307,168 @@ int sqlite3_value_type(sqlite3_value*); int sqlite3_value_numeric_type(sqlite3_value*); /* -** CAPI3REF: Obtain Aggregate Function Context +** CAPI3REF: Obtain Aggregate Function Context {H16210} ** ** The implementation of aggregate SQL functions use this routine to allocate -** a structure for storing their state. The first time this routine -** is called for a particular aggregate, a new structure of size nBytes -** is allocated, zeroed, and returned. On subsequent calls (for the -** same aggregate instance) the same buffer is returned. The implementation -** of the aggregate can use the returned buffer to accumulate data. +** a structure for storing their state. ** -** The buffer allocated is freed automatically by SQLite whan the aggregate +** The first time the sqlite3_aggregate_context() routine is called for a +** particular aggregate, SQLite allocates nBytes of memory, zeroes out that +** memory, and returns a pointer to it. On second and subsequent calls to +** sqlite3_aggregate_context() for the same aggregate function index, +** the same buffer is returned. The implementation of the aggregate can use +** the returned buffer to accumulate data. +** +** SQLite automatically frees the allocated buffer when the aggregate ** query concludes. ** -** The first parameter should be a copy of the -** [sqlite3_context | SQL function context] that is the first -** parameter to the callback routine that implements the aggregate -** function. +** The first parameter should be a copy of the +** [sqlite3_context | SQL function context] that is the first parameter +** to the callback routine that implements the aggregate function. +** +** This routine must be called from the same thread in which +** the aggregate SQL function is running. +** +** INVARIANTS: +** +** {H16211} The first invocation of [sqlite3_aggregate_context(C,N)] for +** a particular instance of an aggregate function (for a particular +** context C) causes SQLite to allocate N bytes of memory, +** zero that memory, and return a pointer to the allocated memory. +** +** {H16213} If a memory allocation error occurs during +** [sqlite3_aggregate_context(C,N)] then the function returns 0. +** +** {H16215} Second and subsequent invocations of +** [sqlite3_aggregate_context(C,N)] for the same context pointer C +** ignore the N parameter and return a pointer to the same +** block of memory returned by the first invocation. +** +** {H16217} The memory allocated by [sqlite3_aggregate_context(C,N)] is +** automatically freed on the next call to [sqlite3_reset()] +** or [sqlite3_finalize()] for the [prepared statement] containing +** the aggregate function associated with context C. */ void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); /* -** CAPI3REF: User Data For Functions +** CAPI3REF: User Data For Functions {H16240} ** -** The pUserData parameter to the [sqlite3_create_function()] -** and [sqlite3_create_function16()] routines -** used to register user functions is available to -** the implementation of the function using this call. +** The sqlite3_user_data() interface returns a copy of +** the pointer that was the pUserData parameter (the 5th parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. {END} +** +** This routine must be called from the same thread in which +** the application-defined function is running. +** +** INVARIANTS: +** +** {H16243} The [sqlite3_user_data(C)] interface returns a copy of the +** P pointer from the [sqlite3_create_function(D,X,N,E,P,F,S,L)] +** or [sqlite3_create_function16(D,X,N,E,P,F,S,L)] call that +** registered the SQL function associated with [sqlite3_context] C. */ void *sqlite3_user_data(sqlite3_context*); /* -** CAPI3REF: Function Auxiliary Data +** CAPI3REF: Database Connection For Functions {H16250} +** +** The sqlite3_context_db_handle() interface returns a copy of +** the pointer to the [database connection] (the 1st parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. +** +** INVARIANTS: +** +** {H16253} The [sqlite3_context_db_handle(C)] interface returns a copy of the +** D pointer from the [sqlite3_create_function(D,X,N,E,P,F,S,L)] +** or [sqlite3_create_function16(D,X,N,E,P,F,S,L)] call that +** registered the SQL function associated with [sqlite3_context] C. +*/ +sqlite3 *sqlite3_context_db_handle(sqlite3_context*); + +/* +** CAPI3REF: Function Auxiliary Data {H16270} ** ** The following two functions may be used by scalar SQL functions to -** associate meta-data with argument values. If the same value is passed to +** associate metadata with argument values. If the same value is passed to ** multiple invocations of the same SQL function during query execution, under -** some circumstances the associated meta-data may be preserved. This may +** some circumstances the associated metadata may be preserved. This may ** be used, for example, to add a regular-expression matching scalar ** function. The compiled version of the regular expression is stored as -** meta-data associated with the SQL value passed as the regular expression +** metadata associated with the SQL value passed as the regular expression ** pattern. The compiled regular expression can be reused on multiple ** invocations of the same function so that the original pattern string ** does not need to be recompiled on each invocation. ** -** The sqlite3_get_auxdata() interface returns a pointer to the meta-data -** associated with the Nth argument value to the current SQL function -** call, where N is the second parameter. If no meta-data has been set for -** that value, then a NULL pointer is returned. +** The sqlite3_get_auxdata() interface returns a pointer to the metadata +** associated by the sqlite3_set_auxdata() function with the Nth argument +** value to the application-defined function. If no metadata has been ever +** been set for the Nth argument of the function, or if the corresponding +** function parameter has changed since the meta-data was set, +** then sqlite3_get_auxdata() returns a NULL pointer. ** -** The sqlite3_set_auxdata() is used to associate meta-data with an SQL -** function argument. The third parameter is a pointer to the meta-data -** to be associated with the Nth user function argument value. The fourth -** parameter specifies a destructor that will be called on the meta- -** data pointer to release it when it is no longer required. If the -** destructor is NULL, it is not invoked. +** The sqlite3_set_auxdata() interface saves the metadata +** pointed to by its 3rd parameter as the metadata for the N-th +** argument of the application-defined function. Subsequent +** calls to sqlite3_get_auxdata() might return this data, if it has +** not been destroyed. +** If it is not NULL, SQLite will invoke the destructor +** function given by the 4th parameter to sqlite3_set_auxdata() on +** the metadata when the corresponding function parameter changes +** or when the SQL statement completes, whichever comes first. ** -** In practice, meta-data is preserved between function calls for +** SQLite is free to call the destructor and drop metadata on any +** parameter of any function at any time. The only guarantee is that +** the destructor will be called before the metadata is dropped. +** +** In practice, metadata is preserved between function calls for ** expressions that are constant at compile time. This includes literal ** values and SQL variables. +** +** These routines must be called from the same thread in which +** the SQL function is running. +** +** INVARIANTS: +** +** {H16272} The [sqlite3_get_auxdata(C,N)] interface returns a pointer +** to metadata associated with the Nth parameter of the SQL function +** whose context is C, or NULL if there is no metadata associated +** with that parameter. +** +** {H16274} The [sqlite3_set_auxdata(C,N,P,D)] interface assigns a metadata +** pointer P to the Nth parameter of the SQL function with context C. +** +** {H16276} SQLite will invoke the destructor D with a single argument +** which is the metadata pointer P following a call to +** [sqlite3_set_auxdata(C,N,P,D)] when SQLite ceases to hold +** the metadata. +** +** {H16277} SQLite ceases to hold metadata for an SQL function parameter +** when the value of that parameter changes. +** +** {H16278} When [sqlite3_set_auxdata(C,N,P,D)] is invoked, the destructor +** is called for any prior metadata associated with the same function +** context C and parameter N. +** +** {H16279} SQLite will call destructors for any metadata it is holding +** in a particular [prepared statement] S when either +** [sqlite3_reset(S)] or [sqlite3_finalize(S)] is called. */ -void *sqlite3_get_auxdata(sqlite3_context*, int); -void sqlite3_set_auxdata(sqlite3_context*, int, void*, void (*)(void*)); +void *sqlite3_get_auxdata(sqlite3_context*, int N); +void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); /* -** CAPI3REF: Constants Defining Special Destructor Behavior +** CAPI3REF: Constants Defining Special Destructor Behavior {H10280} ** -** These are special value for the destructor that is passed in as the +** These are special values for the destructor that is passed in as the ** final argument to routines like [sqlite3_result_blob()]. If the destructor ** argument is SQLITE_STATIC, it means that the content pointer is constant -** and will never change. It does not need to be destroyed. The +** and will never change. It does not need to be destroyed. The ** SQLITE_TRANSIENT value means that the content will likely change in ** the near future and that SQLite should make its own private copy of ** the content before returning. @@ -1835,36 +4481,214 @@ typedef void (*sqlite3_destructor_type)(void*); #define SQLITE_TRANSIENT ((sqlite3_destructor_type)-1) /* -** CAPI3REF: Setting The Result Of An SQL Function +** CAPI3REF: Setting The Result Of An SQL Function {H16400} ** ** These routines are used by the xFunc or xFinal callbacks that ** implement SQL functions and aggregates. See ** [sqlite3_create_function()] and [sqlite3_create_function16()] ** for additional information. ** -** These functions work very much like the -** [sqlite3_bind_blob | sqlite3_bind_*] family of functions used -** to bind values to host parameters in prepared statements. -** Refer to the -** [sqlite3_bind_blob | sqlite3_bind_* documentation] for -** additional information. +** These functions work very much like the [parameter binding] family of +** functions used to bind values to host parameters in prepared statements. +** Refer to the [SQL parameter] documentation for additional information. +** +** The sqlite3_result_blob() interface sets the result from +** an application-defined function to be the BLOB whose content is pointed +** to by the second parameter and which is N bytes long where N is the +** third parameter. +** +** The sqlite3_result_zeroblob() interfaces set the result of +** the application-defined function to be a BLOB containing all zero +** bytes and N bytes in size, where N is the value of the 2nd parameter. +** +** The sqlite3_result_double() interface sets the result from +** an application-defined function to be a floating point value specified +** by its 2nd argument. ** ** The sqlite3_result_error() and sqlite3_result_error16() functions -** cause the implemented SQL function to throw an exception. The -** parameter to sqlite3_result_error() or sqlite3_result_error16() -** is the text of an error message. +** cause the implemented SQL function to throw an exception. +** SQLite uses the string pointed to by the +** 2nd parameter of sqlite3_result_error() or sqlite3_result_error16() +** as the text of an error message. SQLite interprets the error +** message string from sqlite3_result_error() as UTF-8. SQLite +** interprets the string from sqlite3_result_error16() as UTF-16 in native +** byte order. If the third parameter to sqlite3_result_error() +** or sqlite3_result_error16() is negative then SQLite takes as the error +** message all text up through the first zero character. +** If the third parameter to sqlite3_result_error() or +** sqlite3_result_error16() is non-negative then SQLite takes that many +** bytes (not characters) from the 2nd parameter as the error message. +** The sqlite3_result_error() and sqlite3_result_error16() +** routines make a private copy of the error message text before +** they return. Hence, the calling function can deallocate or +** modify the text after they return without harm. +** The sqlite3_result_error_code() function changes the error code +** returned by SQLite as a result of an error in a function. By default, +** the error code is SQLITE_ERROR. A subsequent call to sqlite3_result_error() +** or sqlite3_result_error16() resets the error code to SQLITE_ERROR. ** -** The sqlite3_result_toobig() cause the function implementation -** to throw and error indicating that a string or BLOB is to long -** to represent. +** The sqlite3_result_toobig() interface causes SQLite to throw an error +** indicating that a string or BLOB is to long to represent. +** +** The sqlite3_result_nomem() interface causes SQLite to throw an error +** indicating that a memory allocation failed. +** +** The sqlite3_result_int() interface sets the return value +** of the application-defined function to be the 32-bit signed integer +** value given in the 2nd argument. +** The sqlite3_result_int64() interface sets the return value +** of the application-defined function to be the 64-bit signed integer +** value given in the 2nd argument. +** +** The sqlite3_result_null() interface sets the return value +** of the application-defined function to be NULL. +** +** The sqlite3_result_text(), sqlite3_result_text16(), +** sqlite3_result_text16le(), and sqlite3_result_text16be() interfaces +** set the return value of the application-defined function to be +** a text string which is represented as UTF-8, UTF-16 native byte order, +** UTF-16 little endian, or UTF-16 big endian, respectively. +** SQLite takes the text result from the application from +** the 2nd parameter of the sqlite3_result_text* interfaces. +** If the 3rd parameter to the sqlite3_result_text* interfaces +** is negative, then SQLite takes result text from the 2nd parameter +** through the first zero character. +** If the 3rd parameter to the sqlite3_result_text* interfaces +** is non-negative, then as many bytes (not characters) of the text +** pointed to by the 2nd parameter are taken as the application-defined +** function result. +** If the 4th parameter to the sqlite3_result_text* interfaces +** or sqlite3_result_blob is a non-NULL pointer, then SQLite calls that +** function as the destructor on the text or BLOB result when it has +** finished using that result. +** If the 4th parameter to the sqlite3_result_text* interfaces or +** sqlite3_result_blob is the special constant SQLITE_STATIC, then SQLite +** assumes that the text or BLOB result is in constant space and does not +** copy the it or call a destructor when it has finished using that result. +** If the 4th parameter to the sqlite3_result_text* interfaces +** or sqlite3_result_blob is the special constant SQLITE_TRANSIENT +** then SQLite makes a copy of the result into space obtained from +** from [sqlite3_malloc()] before it returns. +** +** The sqlite3_result_value() interface sets the result of +** the application-defined function to be a copy the +** [unprotected sqlite3_value] object specified by the 2nd parameter. The +** sqlite3_result_value() interface makes a copy of the [sqlite3_value] +** so that the [sqlite3_value] specified in the parameter may change or +** be deallocated after sqlite3_result_value() returns without harm. +** A [protected sqlite3_value] object may always be used where an +** [unprotected sqlite3_value] object is required, so either +** kind of [sqlite3_value] object can be used with this interface. +** +** If these routines are called from within the different thread +** than the one containing the application-defined function that received +** the [sqlite3_context] pointer, the results are undefined. +** +** INVARIANTS: +** +** {H16403} The default return value from any SQL function is NULL. +** +** {H16406} The [sqlite3_result_blob(C,V,N,D)] interface changes the +** return value of function C to be a BLOB that is N bytes +** in length and with content pointed to by V. +** +** {H16409} The [sqlite3_result_double(C,V)] interface changes the +** return value of function C to be the floating point value V. +** +** {H16412} The [sqlite3_result_error(C,V,N)] interface changes the return +** value of function C to be an exception with error code +** [SQLITE_ERROR] and a UTF-8 error message copied from V up to the +** first zero byte or until N bytes are read if N is positive. +** +** {H16415} The [sqlite3_result_error16(C,V,N)] interface changes the return +** value of function C to be an exception with error code +** [SQLITE_ERROR] and a UTF-16 native byte order error message +** copied from V up to the first zero terminator or until N bytes +** are read if N is positive. +** +** {H16418} The [sqlite3_result_error_toobig(C)] interface changes the return +** value of the function C to be an exception with error code +** [SQLITE_TOOBIG] and an appropriate error message. +** +** {H16421} The [sqlite3_result_error_nomem(C)] interface changes the return +** value of the function C to be an exception with error code +** [SQLITE_NOMEM] and an appropriate error message. +** +** {H16424} The [sqlite3_result_error_code(C,E)] interface changes the return +** value of the function C to be an exception with error code E. +** The error message text is unchanged. +** +** {H16427} The [sqlite3_result_int(C,V)] interface changes the +** return value of function C to be the 32-bit integer value V. +** +** {H16430} The [sqlite3_result_int64(C,V)] interface changes the +** return value of function C to be the 64-bit integer value V. +** +** {H16433} The [sqlite3_result_null(C)] interface changes the +** return value of function C to be NULL. +** +** {H16436} The [sqlite3_result_text(C,V,N,D)] interface changes the +** return value of function C to be the UTF-8 string +** V up to the first zero if N is negative +** or the first N bytes of V if N is non-negative. +** +** {H16439} The [sqlite3_result_text16(C,V,N,D)] interface changes the +** return value of function C to be the UTF-16 native byte order +** string V up to the first zero if N is negative +** or the first N bytes of V if N is non-negative. +** +** {H16442} The [sqlite3_result_text16be(C,V,N,D)] interface changes the +** return value of function C to be the UTF-16 big-endian +** string V up to the first zero if N is negative +** or the first N bytes or V if N is non-negative. +** +** {H16445} The [sqlite3_result_text16le(C,V,N,D)] interface changes the +** return value of function C to be the UTF-16 little-endian +** string V up to the first zero if N is negative +** or the first N bytes of V if N is non-negative. +** +** {H16448} The [sqlite3_result_value(C,V)] interface changes the +** return value of function C to be the [unprotected sqlite3_value] +** object V. +** +** {H16451} The [sqlite3_result_zeroblob(C,N)] interface changes the +** return value of function C to be an N-byte BLOB of all zeros. +** +** {H16454} The [sqlite3_result_error()] and [sqlite3_result_error16()] +** interfaces make a copy of their error message strings before +** returning. +** +** {H16457} If the D destructor parameter to [sqlite3_result_blob(C,V,N,D)], +** [sqlite3_result_text(C,V,N,D)], [sqlite3_result_text16(C,V,N,D)], +** [sqlite3_result_text16be(C,V,N,D)], or +** [sqlite3_result_text16le(C,V,N,D)] is the constant [SQLITE_STATIC] +** then no destructor is ever called on the pointer V and SQLite +** assumes that V is immutable. +** +** {H16460} If the D destructor parameter to [sqlite3_result_blob(C,V,N,D)], +** [sqlite3_result_text(C,V,N,D)], [sqlite3_result_text16(C,V,N,D)], +** [sqlite3_result_text16be(C,V,N,D)], or +** [sqlite3_result_text16le(C,V,N,D)] is the constant +** [SQLITE_TRANSIENT] then the interfaces makes a copy of the +** content of V and retains the copy. +** +** {H16463} If the D destructor parameter to [sqlite3_result_blob(C,V,N,D)], +** [sqlite3_result_text(C,V,N,D)], [sqlite3_result_text16(C,V,N,D)], +** [sqlite3_result_text16be(C,V,N,D)], or +** [sqlite3_result_text16le(C,V,N,D)] is some value other than +** the constants [SQLITE_STATIC] and [SQLITE_TRANSIENT] then +** SQLite will invoke the destructor D with V as its only argument +** when it has finished with the V value. */ void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); void sqlite3_result_double(sqlite3_context*, double); void sqlite3_result_error(sqlite3_context*, const char*, int); void sqlite3_result_error16(sqlite3_context*, const void*, int); void sqlite3_result_error_toobig(sqlite3_context*); +void sqlite3_result_error_nomem(sqlite3_context*); +void sqlite3_result_error_code(sqlite3_context*, int); void sqlite3_result_int(sqlite3_context*, int); -void sqlite3_result_int64(sqlite3_context*, sqlite_int64); +void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); void sqlite3_result_null(sqlite3_context*); void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); @@ -1874,46 +4698,95 @@ void sqlite3_result_value(sqlite3_context*, sqlite3_value*); void sqlite3_result_zeroblob(sqlite3_context*, int n); /* -** CAPI3REF: Define New Collating Sequences +** CAPI3REF: Define New Collating Sequences {H16600} ** ** These functions are used to add new collation sequences to the -** [sqlite3*] handle specified as the first argument. +** [database connection] specified as the first argument. ** ** The name of the new collation sequence is specified as a UTF-8 string ** for sqlite3_create_collation() and sqlite3_create_collation_v2() -** and a UTF-16 string for sqlite3_create_collation16(). In all cases +** and a UTF-16 string for sqlite3_create_collation16(). In all cases ** the name is passed as the second function argument. ** -** The third argument must be one of the constants [SQLITE_UTF8], +** The third argument may be one of the constants [SQLITE_UTF8], ** [SQLITE_UTF16LE] or [SQLITE_UTF16BE], indicating that the user-supplied ** routine expects to be passed pointers to strings encoded using UTF-8, -** UTF-16 little-endian or UTF-16 big-endian respectively. +** UTF-16 little-endian, or UTF-16 big-endian, respectively. The +** third argument might also be [SQLITE_UTF16_ALIGNED] to indicate that +** the routine expects pointers to 16-bit word aligned strings +** of UTF-16 in the native byte order of the host computer. ** ** A pointer to the user supplied routine must be passed as the fifth -** argument. If it is NULL, this is the same as deleting the collation -** sequence (so that SQLite cannot call it anymore). Each time the user -** supplied function is invoked, it is passed a copy of the void* passed as -** the fourth argument to sqlite3_create_collation() or -** sqlite3_create_collation16() as its first parameter. +** argument. If it is NULL, this is the same as deleting the collation +** sequence (so that SQLite cannot call it anymore). +** Each time the application supplied function is invoked, it is passed +** as its first parameter a copy of the void* passed as the fourth argument +** to sqlite3_create_collation() or sqlite3_create_collation16(). ** -** The remaining arguments to the user-supplied routine are two strings, -** each represented by a [length, data] pair and encoded in the encoding +** The remaining arguments to the application-supplied routine are two strings, +** each represented by a (length, data) pair and encoded in the encoding ** that was passed as the third argument when the collation sequence was -** registered. The user routine should return negative, zero or positive if -** the first string is less than, equal to, or greater than the second -** string. i.e. (STRING1 - STRING2). +** registered. {END} The application defined collation routine should +** return negative, zero or positive if the first string is less than, +** equal to, or greater than the second string. i.e. (STRING1 - STRING2). ** ** The sqlite3_create_collation_v2() works like sqlite3_create_collation() -** excapt that it takes an extra argument which is a destructor for +** except that it takes an extra argument which is a destructor for ** the collation. The destructor is called when the collation is ** destroyed and is passed a copy of the fourth parameter void* pointer -** of the sqlite3_create_collation_v2(). Collations are destroyed when -** they are overridden by later calls to the collation creation functions -** or when the [sqlite3*] database handle is closed using [sqlite3_close()]. +** of the sqlite3_create_collation_v2(). +** Collations are destroyed when they are overridden by later calls to the +** collation creation functions or when the [database connection] is closed +** using [sqlite3_close()]. ** -** The sqlite3_create_collation_v2() interface is experimental and -** subject to change in future releases. The other collation creation -** functions are stable. +** INVARIANTS: +** +** {H16603} A successful call to the +** [sqlite3_create_collation_v2(B,X,E,P,F,D)] interface +** registers function F as the comparison function used to +** implement collation X on the [database connection] B for +** databases having encoding E. +** +** {H16604} SQLite understands the X parameter to +** [sqlite3_create_collation_v2(B,X,E,P,F,D)] as a zero-terminated +** UTF-8 string in which case is ignored for ASCII characters and +** is significant for non-ASCII characters. +** +** {H16606} Successive calls to [sqlite3_create_collation_v2(B,X,E,P,F,D)] +** with the same values for B, X, and E, override prior values +** of P, F, and D. +** +** {H16609} If the destructor D in [sqlite3_create_collation_v2(B,X,E,P,F,D)] +** is not NULL then it is called with argument P when the +** collating function is dropped by SQLite. +** +** {H16612} A collating function is dropped when it is overloaded. +** +** {H16615} A collating function is dropped when the database connection +** is closed using [sqlite3_close()]. +** +** {H16618} The pointer P in [sqlite3_create_collation_v2(B,X,E,P,F,D)] +** is passed through as the first parameter to the comparison +** function F for all subsequent invocations of F. +** +** {H16621} A call to [sqlite3_create_collation(B,X,E,P,F)] is exactly +** the same as a call to [sqlite3_create_collation_v2()] with +** the same parameters and a NULL destructor. +** +** {H16624} Following a [sqlite3_create_collation_v2(B,X,E,P,F,D)], +** SQLite uses the comparison function F for all text comparison +** operations on the [database connection] B on text values that +** use the collating sequence named X. +** +** {H16627} The [sqlite3_create_collation16(B,X,E,P,F)] works the same +** as [sqlite3_create_collation(B,X,E,P,F)] except that the +** collation name X is understood as UTF-16 in native byte order +** instead of UTF-8. +** +** {H16630} When multiple comparison functions are available for the same +** collating sequence, SQLite chooses the one whose text encoding +** requires the least amount of conversion from the default +** text encoding of the database. */ int sqlite3_create_collation( sqlite3*, @@ -1932,37 +4805,56 @@ int sqlite3_create_collation_v2( ); int sqlite3_create_collation16( sqlite3*, - const char *zName, + const void *zName, int eTextRep, void*, int(*xCompare)(void*,int,const void*,int,const void*) ); /* -** CAPI3REF: Collation Needed Callbacks +** CAPI3REF: Collation Needed Callbacks {H16700} ** ** To avoid having to register all collation sequences before a database ** can be used, a single callback function may be registered with the -** database handle to be called whenever an undefined collation sequence is -** required. +** [database connection] to be called whenever an undefined collation +** sequence is required. ** ** If the function is registered using the sqlite3_collation_needed() API, ** then it is passed the names of undefined collation sequences as strings -** encoded in UTF-8. If sqlite3_collation_needed16() is used, the names -** are passed as UTF-16 in machine native byte order. A call to either -** function replaces any existing callback. +** encoded in UTF-8. {H16703} If sqlite3_collation_needed16() is used, +** the names are passed as UTF-16 in machine native byte order. +** A call to either function replaces any existing callback. ** ** When the callback is invoked, the first argument passed is a copy ** of the second argument to sqlite3_collation_needed() or -** sqlite3_collation_needed16(). The second argument is the database -** handle. The third argument is one of [SQLITE_UTF8], [SQLITE_UTF16BE], or -** [SQLITE_UTF16LE], indicating the most desirable form of the collation -** sequence function required. The fourth parameter is the name of the +** sqlite3_collation_needed16(). The second argument is the database +** connection. The third argument is one of [SQLITE_UTF8], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE], indicating the most desirable form of the collation +** sequence function required. The fourth parameter is the name of the ** required collation sequence. ** ** The callback function should register the desired collation using ** [sqlite3_create_collation()], [sqlite3_create_collation16()], or ** [sqlite3_create_collation_v2()]. +** +** INVARIANTS: +** +** {H16702} A successful call to [sqlite3_collation_needed(D,P,F)] +** or [sqlite3_collation_needed16(D,P,F)] causes +** the [database connection] D to invoke callback F with first +** parameter P whenever it needs a comparison function for a +** collating sequence that it does not know about. +** +** {H16704} Each successful call to [sqlite3_collation_needed()] or +** [sqlite3_collation_needed16()] overrides the callback registered +** on the same [database connection] by prior calls to either +** interface. +** +** {H16706} The name of the requested collating function passed in the +** 4th parameter to the callback is in UTF-8 if the callback +** was registered using [sqlite3_collation_needed()] and +** is in UTF-16 native byte order if the callback was +** registered using [sqlite3_collation_needed16()]. */ int sqlite3_collation_needed( sqlite3*, @@ -2001,259 +4893,459 @@ int sqlite3_rekey( ); /* -** CAPI3REF: Suspend Execution For A Short Time +** CAPI3REF: Suspend Execution For A Short Time {H10530} ** -** This function causes the current thread to suspend execution -** a number of milliseconds specified in its parameter. +** The sqlite3_sleep() function causes the current thread to suspend execution +** for at least a number of milliseconds specified in its parameter. ** -** If the operating system does not support sleep requests with -** millisecond time resolution, then the time will be rounded up to -** the nearest second. The number of milliseconds of sleep actually +** If the operating system does not support sleep requests with +** millisecond time resolution, then the time will be rounded up to +** the nearest second. The number of milliseconds of sleep actually ** requested from the operating system is returned. +** +** SQLite implements this interface by calling the xSleep() +** method of the default [sqlite3_vfs] object. +** +** INVARIANTS: +** +** {H10533} The [sqlite3_sleep(M)] interface invokes the xSleep +** method of the default [sqlite3_vfs|VFS] in order to +** suspend execution of the current thread for at least +** M milliseconds. +** +** {H10536} The [sqlite3_sleep(M)] interface returns the number of +** milliseconds of sleep actually requested of the operating +** system, which might be larger than the parameter M. */ int sqlite3_sleep(int); /* -** CAPI3REF: Name Of The Folder Holding Temporary Files +** CAPI3REF: Name Of The Folder Holding Temporary Files {H10310} ** ** If this global variable is made to point to a string which is -** the name of a folder (a.ka. directory), then all temporary files +** the name of a folder (a.k.a. directory), then all temporary files ** created by SQLite will be placed in that directory. If this variable -** is NULL pointer, then SQLite does a search for an appropriate temporary -** file directory. +** is a NULL pointer, then SQLite performs a search for an appropriate +** temporary file directory. ** -** Once [sqlite3_open()] has been called, changing this variable will -** invalidate the current temporary database, if any. Generally speaking, -** it is not safe to invoke this routine after [sqlite3_open()] has -** been called. +** It is not safe to modify this variable once a [database connection] +** has been opened. It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been call and remain unchanged thereafter. */ SQLITE_EXTERN char *sqlite3_temp_directory; /* -** CAPI3REF: Test To See If The Databse Is In Auto-Commit Mode +** CAPI3REF: Test For Auto-Commit Mode {H12930} +** KEYWORDS: {autocommit mode} ** -** Test to see whether or not the database connection is in autocommit -** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on -** by default. Autocommit is disabled by a BEGIN statement and reenabled -** by the next COMMIT or ROLLBACK. +** The sqlite3_get_autocommit() interface returns non-zero or +** zero if the given database connection is or is not in autocommit mode, +** respectively. Autocommit mode is on by default. +** Autocommit mode is disabled by a [BEGIN] statement. +** Autocommit mode is re-enabled by a [COMMIT] or [ROLLBACK]. +** +** If certain kinds of errors occur on a statement within a multi-statement +** transaction (errors including [SQLITE_FULL], [SQLITE_IOERR], +** [SQLITE_NOMEM], [SQLITE_BUSY], and [SQLITE_INTERRUPT]) then the +** transaction might be rolled back automatically. The only way to +** find out whether SQLite automatically rolled back the transaction after +** an error is to use this function. +** +** INVARIANTS: +** +** {H12931} The [sqlite3_get_autocommit(D)] interface returns non-zero or +** zero if the [database connection] D is or is not in autocommit +** mode, respectively. +** +** {H12932} Autocommit mode is on by default. +** +** {H12933} Autocommit mode is disabled by a successful [BEGIN] statement. +** +** {H12934} Autocommit mode is enabled by a successful [COMMIT] or [ROLLBACK] +** statement. +** +** ASSUMPTIONS: +** +** {A12936} If another thread changes the autocommit status of the database +** connection while this routine is running, then the return value +** is undefined. */ int sqlite3_get_autocommit(sqlite3*); /* -** CAPI3REF: Find The Database Handle Associated With A Prepared Statement +** CAPI3REF: Find The Database Handle Of A Prepared Statement {H13120} ** -** Return the [sqlite3*] database handle to which a -** [sqlite3_stmt | prepared statement] belongs. -** This is the same database handle that was -** the first argument to the [sqlite3_prepare_v2()] or its variants -** that was used to create the statement in the first place. +** The sqlite3_db_handle interface returns the [database connection] handle +** to which a [prepared statement] belongs. The database handle returned by +** sqlite3_db_handle is the same database handle that was the first argument +** to the [sqlite3_prepare_v2()] call (or its variants) that was used to +** create the statement in the first place. +** +** INVARIANTS: +** +** {H13123} The [sqlite3_db_handle(S)] interface returns a pointer +** to the [database connection] associated with the +** [prepared statement] S. */ sqlite3 *sqlite3_db_handle(sqlite3_stmt*); +/* +** CAPI3REF: Find the next prepared statement {H13140} +** +** This interface returns a pointer to the next [prepared statement] after +** pStmt associated with the [database connection] pDb. If pStmt is NULL +** then this interface returns a pointer to the first prepared statement +** associated with the database connection pDb. If no prepared statement +** satisfies the conditions of this routine, it returns NULL. +** +** INVARIANTS: +** +** {H13143} If D is a [database connection] that holds one or more +** unfinalized [prepared statements] and S is a NULL pointer, +** then [sqlite3_next_stmt(D, S)] routine shall return a pointer +** to one of the prepared statements associated with D. +** +** {H13146} If D is a [database connection] that holds no unfinalized +** [prepared statements] and S is a NULL pointer, then +** [sqlite3_next_stmt(D, S)] routine shall return a NULL pointer. +** +** {H13149} If S is a [prepared statement] in the [database connection] D +** and S is not the last prepared statement in D, then +** [sqlite3_next_stmt(D, S)] routine shall return a pointer +** to the next prepared statement in D after S. +** +** {H13152} If S is the last [prepared statement] in the +** [database connection] D then the [sqlite3_next_stmt(D, S)] +** routine shall return a NULL pointer. +** +** ASSUMPTIONS: +** +** {A13154} The [database connection] pointer D in a call to +** [sqlite3_next_stmt(D,S)] must refer to an open database +** connection and in particular must not be a NULL pointer. +*/ +sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); /* -** CAPI3REF: Commit And Rollback Notification Callbacks +** CAPI3REF: Commit And Rollback Notification Callbacks {H12950} ** -** These routines -** register callback functions to be invoked whenever a transaction -** is committed or rolled back. The pArg argument is passed through -** to the callback. If the callback on a commit hook function -** returns non-zero, then the commit is converted into a rollback. +** The sqlite3_commit_hook() interface registers a callback +** function to be invoked whenever a transaction is committed. +** Any callback set by a previous call to sqlite3_commit_hook() +** for the same database connection is overridden. +** The sqlite3_rollback_hook() interface registers a callback +** function to be invoked whenever a transaction is committed. +** Any callback set by a previous call to sqlite3_commit_hook() +** for the same database connection is overridden. +** The pArg argument is passed through to the callback. +** If the callback on a commit hook function returns non-zero, +** then the commit is converted into a rollback. ** -** If another function was previously registered, its pArg value is returned. -** Otherwise NULL is returned. +** If another function was previously registered, its +** pArg value is returned. Otherwise NULL is returned. +** +** The callback implementation must not do anything that will modify +** the database connection that invoked the callback. Any actions +** to modify the database connection must be deferred until after the +** completion of the [sqlite3_step()] call that triggered the commit +** or rollback hook in the first place. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. ** ** Registering a NULL function disables the callback. ** -** For the purposes of this API, a transaction is said to have been +** For the purposes of this API, a transaction is said to have been ** rolled back if an explicit "ROLLBACK" statement is executed, or -** an error or constraint causes an implicit rollback to occur. The -** callback is not invoked if a transaction is automatically rolled -** back because the database connection is closed. +** an error or constraint causes an implicit rollback to occur. +** The rollback callback is not invoked if a transaction is +** automatically rolled back because the database connection is closed. +** The rollback callback is not invoked if a transaction is +** rolled back because a commit callback returned non-zero. +** Check on this ** -** These are experimental interfaces and are subject to change. +** INVARIANTS: +** +** {H12951} The [sqlite3_commit_hook(D,F,P)] interface registers the +** callback function F to be invoked with argument P whenever +** a transaction commits on the [database connection] D. +** +** {H12952} The [sqlite3_commit_hook(D,F,P)] interface returns the P argument +** from the previous call with the same [database connection] D, +** or NULL on the first call for a particular database connection D. +** +** {H12953} Each call to [sqlite3_commit_hook()] overwrites the callback +** registered by prior calls. +** +** {H12954} If the F argument to [sqlite3_commit_hook(D,F,P)] is NULL +** then the commit hook callback is canceled and no callback +** is invoked when a transaction commits. +** +** {H12955} If the commit callback returns non-zero then the commit is +** converted into a rollback. +** +** {H12961} The [sqlite3_rollback_hook(D,F,P)] interface registers the +** callback function F to be invoked with argument P whenever +** a transaction rolls back on the [database connection] D. +** +** {H12962} The [sqlite3_rollback_hook(D,F,P)] interface returns the P +** argument from the previous call with the same +** [database connection] D, or NULL on the first call +** for a particular database connection D. +** +** {H12963} Each call to [sqlite3_rollback_hook()] overwrites the callback +** registered by prior calls. +** +** {H12964} If the F argument to [sqlite3_rollback_hook(D,F,P)] is NULL +** then the rollback hook callback is canceled and no callback +** is invoked when a transaction rolls back. */ void *sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); /* -** CAPI3REF: Data Change Notification Callbacks +** CAPI3REF: Data Change Notification Callbacks {H12970} ** -** Register a callback function with the database connection identified by the -** first argument to be invoked whenever a row is updated, inserted or deleted. -** Any callback set by a previous call to this function for the same -** database connection is overridden. +** The sqlite3_update_hook() interface registers a callback function +** with the [database connection] identified by the first argument +** to be invoked whenever a row is updated, inserted or deleted. +** Any callback set by a previous call to this function +** for the same database connection is overridden. ** -** The second argument is a pointer to the function to invoke when a -** row is updated, inserted or deleted. The first argument to the callback is -** a copy of the third argument to sqlite3_update_hook(). The second callback -** argument is one of SQLITE_INSERT, SQLITE_DELETE or SQLITE_UPDATE, depending -** on the operation that caused the callback to be invoked. The third and -** fourth arguments to the callback contain pointers to the database and -** table name containing the affected row. The final callback parameter is -** the rowid of the row. In the case of an update, this is the rowid after -** the update takes place. +** The second argument is a pointer to the function to invoke when a +** row is updated, inserted or deleted. +** The first argument to the callback is a copy of the third argument +** to sqlite3_update_hook(). +** The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], +** or [SQLITE_UPDATE], depending on the operation that caused the callback +** to be invoked. +** The third and fourth arguments to the callback contain pointers to the +** database and table name containing the affected row. +** The final callback parameter is the [rowid] of the row. +** In the case of an update, this is the [rowid] after the update takes place. ** ** The update hook is not invoked when internal system tables are ** modified (i.e. sqlite_master and sqlite_sequence). ** -** If another function was previously registered, its pArg value is returned. -** Otherwise NULL is returned. +** The update hook implementation must not do anything that will modify +** the database connection that invoked the update hook. Any actions +** to modify the database connection must be deferred until after the +** completion of the [sqlite3_step()] call that triggered the update hook. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +** If another function was previously registered, its pArg value +** is returned. Otherwise NULL is returned. +** +** INVARIANTS: +** +** {H12971} The [sqlite3_update_hook(D,F,P)] interface causes the callback +** function F to be invoked with first parameter P whenever +** a table row is modified, inserted, or deleted on +** the [database connection] D. +** +** {H12973} The [sqlite3_update_hook(D,F,P)] interface returns the value +** of P for the previous call on the same [database connection] D, +** or NULL for the first call. +** +** {H12975} If the update hook callback F in [sqlite3_update_hook(D,F,P)] +** is NULL then the no update callbacks are made. +** +** {H12977} Each call to [sqlite3_update_hook(D,F,P)] overrides prior calls +** to the same interface on the same [database connection] D. +** +** {H12979} The update hook callback is not invoked when internal system +** tables such as sqlite_master and sqlite_sequence are modified. +** +** {H12981} The second parameter to the update callback +** is one of [SQLITE_INSERT], [SQLITE_DELETE] or [SQLITE_UPDATE], +** depending on the operation that caused the callback to be invoked. +** +** {H12983} The third and fourth arguments to the callback contain pointers +** to zero-terminated UTF-8 strings which are the names of the +** database and table that is being updated. + +** {H12985} The final callback parameter is the [rowid] of the row after +** the change occurs. */ void *sqlite3_update_hook( sqlite3*, - void(*)(void *,int ,char const *,char const *,sqlite_int64), + void(*)(void *,int ,char const *,char const *,sqlite3_int64), void* ); /* -** CAPI3REF: Enable Or Disable Shared Pager Cache +** CAPI3REF: Enable Or Disable Shared Pager Cache {H10330} +** KEYWORDS: {shared cache} {shared cache mode} ** ** This routine enables or disables the sharing of the database cache -** and schema data structures between connections to the same database. -** Sharing is enabled if the argument is true and disabled if the argument -** is false. +** and schema data structures between [database connection | connections] +** to the same database. Sharing is enabled if the argument is true +** and disabled if the argument is false. ** -** Cache sharing is enabled and disabled on a thread-by-thread basis. -** Each call to this routine enables or disables cache sharing only for -** connections created in the same thread in which this routine is called. -** There is no mechanism for sharing cache between database connections -** running in different threads. +** Cache sharing is enabled and disabled for an entire process. {END} +** This is a change as of SQLite version 3.5.0. In prior versions of SQLite, +** sharing was enabled or disabled for each thread separately. ** -** Sharing must be disabled prior to shutting down a thread or else -** the thread will leak memory. Call this routine with an argument of -** 0 to turn off sharing. Or use the sqlite3_thread_cleanup() API. -** -** This routine must not be called when any database connections -** are active in the current thread. Enabling or disabling shared -** cache while there are active database connections will result -** in memory corruption. -** -** When the shared cache is enabled, the -** following routines must always be called from the same thread: -** [sqlite3_open()], [sqlite3_prepare_v2()], [sqlite3_step()], -** [sqlite3_reset()], [sqlite3_finalize()], and [sqlite3_close()]. -** This is due to the fact that the shared cache makes use of -** thread-specific storage so that it will be available for sharing -** with other connections. +** The cache sharing mode set by this interface effects all subsequent +** calls to [sqlite3_open()], [sqlite3_open_v2()], and [sqlite3_open16()]. +** Existing database connections continue use the sharing mode +** that was in effect at the time they were opened. ** ** Virtual tables cannot be used with a shared cache. When shared -** cache is enabled, the sqlite3_create_module() API used to register +** cache is enabled, the [sqlite3_create_module()] API used to register ** virtual tables will always return an error. ** -** This routine returns [SQLITE_OK] if shared cache was -** enabled or disabled successfully. An [SQLITE_ERROR | error code] -** is returned otherwise. +** This routine returns [SQLITE_OK] if shared cache was enabled or disabled +** successfully. An [error code] is returned otherwise. ** -** Shared cache is disabled by default for backward compatibility. +** Shared cache is disabled by default. But this might change in +** future releases of SQLite. Applications that care about shared +** cache setting should set it explicitly. +** +** INVARIANTS: +** +** {H10331} A successful invocation of [sqlite3_enable_shared_cache(B)] +** will enable or disable shared cache mode for any subsequently +** created [database connection] in the same process. +** +** {H10336} When shared cache is enabled, the [sqlite3_create_module()] +** interface will always return an error. +** +** {H10337} The [sqlite3_enable_shared_cache(B)] interface returns +** [SQLITE_OK] if shared cache was enabled or disabled successfully. +** +** {H10339} Shared cache is disabled by default. */ int sqlite3_enable_shared_cache(int); /* -** CAPI3REF: Attempt To Free Heap Memory +** CAPI3REF: Attempt To Free Heap Memory {H17340} ** -** Attempt to free N bytes of heap memory by deallocating non-essential -** memory allocations held by the database library (example: memory -** used to cache database pages to improve performance). +** The sqlite3_release_memory() interface attempts to free N bytes +** of heap memory by deallocating non-essential memory allocations +** held by the database library. {END} Memory used to cache database +** pages to improve performance is an example of non-essential memory. +** sqlite3_release_memory() returns the number of bytes actually freed, +** which might be more or less than the amount requested. ** -** This function is not a part of standard builds. It is only created -** if SQLite is compiled with the SQLITE_ENABLE_MEMORY_MANAGEMENT macro. +** INVARIANTS: +** +** {H17341} The [sqlite3_release_memory(N)] interface attempts to +** free N bytes of heap memory by deallocating non-essential +** memory allocations held by the database library. +** +** {H16342} The [sqlite3_release_memory(N)] returns the number +** of bytes actually freed, which might be more or less +** than the amount requested. */ int sqlite3_release_memory(int); /* -** CAPI3REF: Impose A Limit On Heap Size +** CAPI3REF: Impose A Limit On Heap Size {H17350} ** -** Place a "soft" limit on the amount of heap memory that may be allocated by -** SQLite within the current thread. If an internal allocation is requested -** that would exceed the specified limit, [sqlite3_release_memory()] is invoked -** one or more times to free up some space before the allocation is made. +** The sqlite3_soft_heap_limit() interface places a "soft" limit +** on the amount of heap memory that may be allocated by SQLite. +** If an internal allocation is requested that would exceed the +** soft heap limit, [sqlite3_release_memory()] is invoked one or +** more times to free up some space before the allocation is performed. ** -** The limit is called "soft", because if [sqlite3_release_memory()] cannot free -** sufficient memory to prevent the limit from being exceeded, the memory is -** allocated anyway and the current operation proceeds. -** -** Prior to shutting down a thread sqlite3_soft_heap_limit() must be set to -** zero (the default) or else the thread will leak memory. Alternatively, use -** the [sqlite3_thread_cleanup()] API. +** The limit is called "soft", because if [sqlite3_release_memory()] +** cannot free sufficient memory to prevent the limit from being exceeded, +** the memory is allocated anyway and the current operation proceeds. ** ** A negative or zero value for N means that there is no soft heap limit and -** [sqlite3_release_memory()] will only be called when memory is exhaused. +** [sqlite3_release_memory()] will only be called when memory is exhausted. ** The default value for the soft heap limit is zero. ** -** SQLite makes a best effort to honor the soft heap limit. But if it -** is unable to reduce memory usage below the soft limit, execution will -** continue without error or notification. This is why the limit is +** SQLite makes a best effort to honor the soft heap limit. +** But if the soft heap limit cannot be honored, execution will +** continue without error or notification. This is why the limit is ** called a "soft" limit. It is advisory only. ** -** This function is only available if the library was compiled with the -** SQLITE_ENABLE_MEMORY_MANAGEMENT option set. -** memory-management has been enabled. +** Prior to SQLite version 3.5.0, this routine only constrained the memory +** allocated by a single thread - the same thread in which this routine +** runs. Beginning with SQLite version 3.5.0, the soft heap limit is +** applied to all threads. The value specified for the soft heap limit +** is an upper bound on the total memory allocation for all threads. In +** version 3.5.0 there is no mechanism for limiting the heap usage for +** individual threads. +** +** INVARIANTS: +** +** {H16351} The [sqlite3_soft_heap_limit(N)] interface places a soft limit +** of N bytes on the amount of heap memory that may be allocated +** using [sqlite3_malloc()] or [sqlite3_realloc()] at any point +** in time. +** +** {H16352} If a call to [sqlite3_malloc()] or [sqlite3_realloc()] would +** cause the total amount of allocated memory to exceed the +** soft heap limit, then [sqlite3_release_memory()] is invoked +** in an attempt to reduce the memory usage prior to proceeding +** with the memory allocation attempt. +** +** {H16353} Calls to [sqlite3_malloc()] or [sqlite3_realloc()] that trigger +** attempts to reduce memory usage through the soft heap limit +** mechanism continue even if the attempt to reduce memory +** usage is unsuccessful. +** +** {H16354} A negative or zero value for N in a call to +** [sqlite3_soft_heap_limit(N)] means that there is no soft +** heap limit and [sqlite3_release_memory()] will only be +** called when memory is completely exhausted. +** +** {H16355} The default value for the soft heap limit is zero. +** +** {H16358} Each call to [sqlite3_soft_heap_limit(N)] overrides the +** values set by all prior calls. */ void sqlite3_soft_heap_limit(int); /* -** CAPI3REF: Clean Up Thread Local Storage +** CAPI3REF: Extract Metadata About A Column Of A Table {H12850} ** -** This routine makes sure that all thread-local storage has been -** deallocated for the current thread. +** This routine returns metadata about a specific column of a specific +** database table accessible using the [database connection] handle +** passed as the first function argument. ** -** This routine is not technically necessary. All thread-local storage -** will be automatically deallocated once memory-management and -** shared-cache are disabled and the soft heap limit has been set -** to zero. This routine is provided as a convenience for users who -** want to make absolutely sure they have not forgotten something -** prior to killing off a thread. -*/ -void sqlite3_thread_cleanup(void); - -/* -** CAPI3REF: Extract Metadata About A Column Of A Table -** -** This routine -** returns meta-data about a specific column of a specific database -** table accessible using the connection handle passed as the first function -** argument. -** -** The column is identified by the second, third and fourth parameters to +** The column is identified by the second, third and fourth parameters to ** this function. The second parameter is either the name of the database ** (i.e. "main", "temp" or an attached database) containing the specified ** table or NULL. If it is NULL, then all attached databases are searched -** for the table using the same algorithm as the database engine uses to +** for the table using the same algorithm used by the database engine to ** resolve unqualified table references. ** -** The third and fourth parameters to this function are the table and column -** name of the desired column, respectively. Neither of these parameters +** The third and fourth parameters to this function are the table and column +** name of the desired column, respectively. Neither of these parameters ** may be NULL. ** -** Meta information is returned by writing to the memory locations passed as -** the 5th and subsequent parameters to this function. Any of these -** arguments may be NULL, in which case the corresponding element of meta -** information is ommitted. +** Metadata is returned by writing to the memory locations passed as the 5th +** and subsequent parameters to this function. Any of these arguments may be +** NULL, in which case the corresponding element of metadata is omitted. ** -** 
-** Parameter     Output Type      Description
-** -----------------------------------
+** 

+** 
+** 
 Parameter  Output
Type 		  Description
 **
-**   5th         const char*      Data type
-**   6th         const char*      Name of the default collation sequence 
-**   7th         int              True if the column has a NOT NULL constraint
-**   8th         int              True if the column is part of the PRIMARY KEY
-**   9th         int              True if the column is AUTOINCREMENT
-** 
+** 
 5th  const char*  Data type
+** 
 6th  const char*  Name of default collation sequence
+** 
 7th  int          True if column has a NOT NULL constraint
+** 
 8th  int          True if column is part of the PRIMARY KEY
+** 
 9th  int          True if column is [AUTOINCREMENT]
+** 

+** 

 **
+** The memory pointed to by the character pointers returned for the
+** declaration type and collation sequence is valid only until the next
+** call to any SQLite API function.
 **
-** The memory pointed to by the character pointers returned for the 
-** declaration type and collation sequence is valid only until the next 
-** call to any sqlite API function.
+** If the specified table is actually a view, an [error code] is returned.
 **
-** If the specified table is actually a view, then an error is returned.
-**
-** If the specified column is "rowid", "oid" or "_rowid_" and an 
-** INTEGER PRIMARY KEY column has been explicitly declared, then the output 
+** If the specified column is "rowid", "oid" or "_rowid_" and an
+** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output
 ** parameters are set for the explicitly declared column. If there is no
-** explicitly declared IPK column, then the output parameters are set as 
-** follows:
+** explicitly declared [INTEGER PRIMARY KEY] column, then the output
+** parameters are set as follows:
 **
 ** 
 **     data type: "INTEGER"
@@ -2265,11 +5357,11 @@ void sqlite3_thread_cleanup(void);
 **
 ** This function may load one or more schemas from database files. If an
 ** error occurs during this process, or if the requested table or column
-** cannot be found, an SQLITE error code is returned and an error message
-** left in the database handle (to be retrieved using sqlite3_errmsg()).
+** cannot be found, an [error code] is returned and an error message left
+** in the [database connection] (to be retrieved using sqlite3_errmsg()).
 **
 ** This API is only available if the library was compiled with the
-** SQLITE_ENABLE_COLUMN_METADATA preprocessor symbol defined.
+** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol defined.
 */
 int sqlite3_table_column_metadata(
   sqlite3 *db,                /* Connection handle */
@@ -2280,24 +5372,34 @@ int sqlite3_table_column_metadata(
   char const **pzCollSeq,     /* OUTPUT: Collation sequence name */
   int *pNotNull,              /* OUTPUT: True if NOT NULL constraint exists */
   int *pPrimaryKey,           /* OUTPUT: True if column part of PK */
-  int *pAutoinc               /* OUTPUT: True if colums is auto-increment */
+  int *pAutoinc               /* OUTPUT: True if column is auto-increment */
 );
 
 /*
-** CAPI3REF: Load An Extension
+** CAPI3REF: Load An Extension {H12600} 
 **
-** Attempt to load an SQLite extension library contained in the file
-** zFile.  The entry point is zProc.  zProc may be 0 in which case the
-** name of the entry point defaults to "sqlite3_extension_init".
+** This interface loads an SQLite extension library from the named file.
 **
-** Return [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong.
+** {H12601} The sqlite3_load_extension() interface attempts to load an
+**          SQLite extension library contained in the file zFile.
 **
-** If an error occurs and pzErrMsg is not 0, then fill *pzErrMsg with 
-** error message text.  The calling function should free this memory
-** by calling [sqlite3_free()].
+** {H12602} The entry point is zProc.
 **
-** Extension loading must be enabled using [sqlite3_enable_load_extension()]
-** prior to calling this API or an error will be returned.
+** {H12603} zProc may be 0, in which case the name of the entry point
+**          defaults to "sqlite3_extension_init".
+**
+** {H12604} The sqlite3_load_extension() interface shall return
+**          [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong.
+**
+** {H12605} If an error occurs and pzErrMsg is not 0, then the
+**          [sqlite3_load_extension()] interface shall attempt to
+**          fill *pzErrMsg with error message text stored in memory
+**          obtained from [sqlite3_malloc()]. {END}  The calling function
+**          should free this memory by calling [sqlite3_free()].
+**
+** {H12606} Extension loading must be enabled using
+**          [sqlite3_enable_load_extension()] prior to calling this API,
+**          otherwise an error will be returned.
 */
 int sqlite3_load_extension(
   sqlite3 *db,          /* Load the extension into this database connection */
@@ -2307,62 +5409,64 @@ int sqlite3_load_extension(
 );
 
 /*
-** CAPI3REF:  Enable Or Disable Extension Loading
+** CAPI3REF: Enable Or Disable Extension Loading {H12620} 
 **
 ** So as not to open security holes in older applications that are
 ** unprepared to deal with extension loading, and as a means of disabling
-** extension loading while evaluating user-entered SQL, the following
-** API is provided to turn the [sqlite3_load_extension()] mechanism on and
-** off.  It is off by default.  See ticket #1863.
+** extension loading while evaluating user-entered SQL, the following API
+** is provided to turn the [sqlite3_load_extension()] mechanism on and off.
 **
-** Call this routine with onoff==1 to turn extension loading on
-** and call it with onoff==0 to turn it back off again.
+** Extension loading is off by default. See ticket #1863.
+**
+** {H12621} Call the sqlite3_enable_load_extension() routine with onoff==1
+**          to turn extension loading on and call it with onoff==0 to turn
+**          it back off again.
+**
+** {H12622} Extension loading is off by default.
 */
 int sqlite3_enable_load_extension(sqlite3 *db, int onoff);
 
 /*
-** CAPI3REF: Make Arrangements To Automatically Load An Extension
-**
-** Register an extension entry point that is automatically invoked
-** whenever a new database connection is opened using
-** [sqlite3_open()] or [sqlite3_open16()].
+** CAPI3REF: Automatically Load An Extensions {H12640} 
 **
 ** This API can be invoked at program startup in order to register
 ** one or more statically linked extensions that will be available
-** to all new database connections.
+** to all new [database connections]. {END}
 **
-** Duplicate extensions are detected so calling this routine multiple
-** times with the same extension is harmless.
+** This routine stores a pointer to the extension in an array that is
+** obtained from [sqlite3_malloc()].  If you run a memory leak checker
+** on your program and it reports a leak because of this array, invoke
+** [sqlite3_reset_auto_extension()] prior to shutdown to free the memory.
 **
-** This routine stores a pointer to the extension in an array
-** that is obtained from malloc().  If you run a memory leak
-** checker on your program and it reports a leak because of this
-** array, then invoke [sqlite3_automatic_extension_reset()] prior
-** to shutdown to free the memory.
+** {H12641} This function registers an extension entry point that is
+**          automatically invoked whenever a new [database connection]
+**          is opened using [sqlite3_open()], [sqlite3_open16()],
+**          or [sqlite3_open_v2()].
 **
-** Automatic extensions apply across all threads.
+** {H12642} Duplicate extensions are detected so calling this routine
+**          multiple times with the same extension is harmless.
 **
-** This interface is experimental and is subject to change or
-** removal in future releases of SQLite.
+** {H12643} This routine stores a pointer to the extension in an array
+**          that is obtained from [sqlite3_malloc()].
+**
+** {H12644} Automatic extensions apply across all threads.
 */
-int sqlite3_auto_extension(void *xEntryPoint);
-
+int sqlite3_auto_extension(void (*xEntryPoint)(void));
 
 /*
-** CAPI3REF: Reset Automatic Extension Loading
+** CAPI3REF: Reset Automatic Extension Loading {H12660} 
 **
-** Disable all previously registered automatic extensions.  This
-** routine undoes the effect of all prior [sqlite3_automatic_extension()]
-** calls.
+** This function disables all previously registered automatic
+** extensions. {END}  It undoes the effect of all prior
+** [sqlite3_auto_extension()] calls.
 **
-** This call disabled automatic extensions in all threads.
+** {H12661} This function disables all previously registered
+**          automatic extensions.
 **
-** This interface is experimental and is subject to change or
-** removal in future releases of SQLite.
+** {H12662} This function disables automatic extensions in all threads.
 */
 void sqlite3_reset_auto_extension(void);
 
-
 /*
 ****** EXPERIMENTAL - subject to change without notice **************
 **
@@ -2370,7 +5474,7 @@ void sqlite3_reset_auto_extension(void);
 ** to be experimental.  The interface might change in incompatible ways.
 ** If this is a problem for you, do not use the interface at this time.
 **
-** When the virtual-table mechanism stablizes, we will declare the
+** When the virtual-table mechanism stabilizes, we will declare the
 ** interface fixed, support it indefinitely, and remove this comment.
 */
 
@@ -2383,9 +5487,16 @@ typedef struct sqlite3_vtab_cursor sqlite3_vtab_cursor;
 typedef struct sqlite3_module sqlite3_module;
 
 /*
+** CAPI3REF: Virtual Table Object {H18000} 
+** KEYWORDS: sqlite3_module
+** EXPERIMENTAL
+**
 ** A module is a class of virtual tables.  Each module is defined
 ** by an instance of the following structure.  This structure consists
 ** mostly of methods for the module.
+**
+** This interface is experimental and is subject to change or
+** removal in future releases of SQLite.
 */
 struct sqlite3_module {
   int iVersion;
@@ -2405,8 +5516,8 @@ struct sqlite3_module {
   int (*xNext)(sqlite3_vtab_cursor*);
   int (*xEof)(sqlite3_vtab_cursor*);
   int (*xColumn)(sqlite3_vtab_cursor*, sqlite3_context*, int);
-  int (*xRowid)(sqlite3_vtab_cursor*, sqlite_int64 *pRowid);
-  int (*xUpdate)(sqlite3_vtab *, int, sqlite3_value **, sqlite_int64 *);
+  int (*xRowid)(sqlite3_vtab_cursor*, sqlite3_int64 *pRowid);
+  int (*xUpdate)(sqlite3_vtab *, int, sqlite3_value **, sqlite3_int64 *);
   int (*xBegin)(sqlite3_vtab *pVTab);
   int (*xSync)(sqlite3_vtab *pVTab);
   int (*xCommit)(sqlite3_vtab *pVTab);
@@ -2414,30 +5525,32 @@ struct sqlite3_module {
   int (*xFindFunction)(sqlite3_vtab *pVtab, int nArg, const char *zName,
                        void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
                        void **ppArg);
-
   int (*xRename)(sqlite3_vtab *pVtab, const char *zNew);
 };
 
 /*
+** CAPI3REF: Virtual Table Indexing Information {H18100} 
+** KEYWORDS: sqlite3_index_info
+** EXPERIMENTAL
+**
 ** The sqlite3_index_info structure and its substructures is used to
 ** pass information into and receive the reply from the xBestIndex
 ** method of an sqlite3_module.  The fields under **Inputs** are the
 ** inputs to xBestIndex and are read-only.  xBestIndex inserts its
 ** results into the **Outputs** fields.
 **
-** The aConstraint[] array records WHERE clause constraints of the
-** form:
+** The aConstraint[] array records WHERE clause constraints of the form:
 **
-**         column OP expr
+** 
column OP expr

 **
-** Where OP is =, <, <=, >, or >=.  The particular operator is stored
-** in aConstraint[].op.  The index of the column is stored in 
+** where OP is =, <, <=, >, or >=.  The particular operator is
+** stored in aConstraint[].op.  The index of the column is stored in
 ** aConstraint[].iColumn.  aConstraint[].usable is TRUE if the
 ** expr on the right-hand side can be evaluated (and thus the constraint
 ** is usable) and false if it cannot.
 **
 ** The optimizer automatically inverts terms of the form "expr OP column"
-** and makes other simplificatinos to the WHERE clause in an attempt to
+** and makes other simplifications to the WHERE clause in an attempt to
 ** get as many WHERE clause terms into the form shown above as possible.
 ** The aConstraint[] array only reports WHERE clause terms in the correct
 ** form that refer to the particular virtual table being queried.
@@ -2463,27 +5576,29 @@ struct sqlite3_module {
 ** particular lookup.  A full scan of a table with N entries should have
 ** a cost of N.  A binary search of a table of N entries should have a
 ** cost of approximately log(N).
+**
+** This interface is experimental and is subject to change or
+** removal in future releases of SQLite.
 */
 struct sqlite3_index_info {
   /* Inputs */
-  const int nConstraint;     /* Number of entries in aConstraint */
-  const struct sqlite3_index_constraint {
+  int nConstraint;           /* Number of entries in aConstraint */
+  struct sqlite3_index_constraint {
      int iColumn;              /* Column on left-hand side of constraint */
      unsigned char op;         /* Constraint operator */
      unsigned char usable;     /* True if this constraint is usable */
      int iTermOffset;          /* Used internally - xBestIndex should ignore */
-  } *const aConstraint;      /* Table of WHERE clause constraints */
-  const int nOrderBy;        /* Number of terms in the ORDER BY clause */
-  const struct sqlite3_index_orderby {
+  } *aConstraint;            /* Table of WHERE clause constraints */
+  int nOrderBy;              /* Number of terms in the ORDER BY clause */
+  struct sqlite3_index_orderby {
      int iColumn;              /* Column number */
      unsigned char desc;       /* True for DESC.  False for ASC. */
-  } *const aOrderBy;         /* The ORDER BY clause */
-
+  } *aOrderBy;               /* The ORDER BY clause */
   /* Outputs */
   struct sqlite3_index_constraint_usage {
     int argvIndex;           /* if >0, constraint is part of argv to xFilter */
     unsigned char omit;      /* Do not code a test for this constraint */
-  } *const aConstraintUsage;
+  } *aConstraintUsage;
   int idxNum;                /* Number used to identify the index */
   char *idxStr;              /* String, possibly obtained from sqlite3_malloc */
   int needToFreeIdxStr;      /* Free idxStr using sqlite3_free() if true */
@@ -2498,12 +5613,18 @@ struct sqlite3_index_info {
 #define SQLITE_INDEX_CONSTRAINT_MATCH 64
 
 /*
-** This routine is used to register a new module name with an SQLite
-** connection.  Module names must be registered before creating new
-** virtual tables on the module, or before using preexisting virtual
-** tables of the module.
+** CAPI3REF: Register A Virtual Table Implementation {H18200} 
+** EXPERIMENTAL
+**
+** This routine is used to register a new module name with a
+** [database connection].  Module names must be registered before
+** creating new virtual tables on the module, or before using
+** preexisting virtual tables of the module.
+**
+** This interface is experimental and is subject to change or
+** removal in future releases of SQLite.
 */
-int sqlite3_create_module(
+SQLITE_EXPERIMENTAL int sqlite3_create_module(
   sqlite3 *db,               /* SQLite connection to register module with */
   const char *zName,         /* Name of the module */
   const sqlite3_module *,    /* Methods for the module */
@@ -2511,11 +5632,14 @@ int sqlite3_create_module(
 );
 
 /*
-** This routine is identical to the sqlite3_create_module() method above,
+** CAPI3REF: Register A Virtual Table Implementation {H18210} 
+** EXPERIMENTAL
+**
+** This routine is identical to the [sqlite3_create_module()] method above,
 ** except that it allows a destructor function to be specified. It is
 ** even more experimental than the rest of the virtual tables API.
 */
-int sqlite3_create_module_v2(
+SQLITE_EXPERIMENTAL int sqlite3_create_module_v2(
   sqlite3 *db,               /* SQLite connection to register module with */
   const char *zName,         /* Name of the module */
   const sqlite3_module *,    /* Methods for the module */
@@ -2524,21 +5648,28 @@ int sqlite3_create_module_v2(
 );
 
 /*
+** CAPI3REF: Virtual Table Instance Object {H18010} 
+** KEYWORDS: sqlite3_vtab
+** EXPERIMENTAL
+**
 ** Every module implementation uses a subclass of the following structure
 ** to describe a particular instance of the module.  Each subclass will
-** be taylored to the specific needs of the module implementation.   The
-** purpose of this superclass is to define certain fields that are common
-** to all module implementations.
+** be tailored to the specific needs of the module implementation.
+** The purpose of this superclass is to define certain fields that are
+** common to all module implementations.
 **
 ** Virtual tables methods can set an error message by assigning a
-** string obtained from sqlite3_mprintf() to zErrMsg.  The method should
-** take care that any prior string is freed by a call to sqlite3_free()
+** string obtained from [sqlite3_mprintf()] to zErrMsg.  The method should
+** take care that any prior string is freed by a call to [sqlite3_free()]
 ** prior to assigning a new string to zErrMsg.  After the error message
 ** is delivered up to the client application, the string will be automatically
 ** freed by sqlite3_free() and the zErrMsg field will be zeroed.  Note
 ** that sqlite3_mprintf() and sqlite3_free() are used on the zErrMsg field
 ** since virtual tables are commonly implemented in loadable extensions which
 ** do not have access to sqlite3MPrintf() or sqlite3Free().
+**
+** This interface is experimental and is subject to change or
+** removal in future releases of SQLite.
 */
 struct sqlite3_vtab {
   const sqlite3_module *pModule;  /* The module for this virtual table */
@@ -2547,7 +5678,12 @@ struct sqlite3_vtab {
   /* Virtual table implementations will typically add additional fields */
 };
 
-/* Every module implementation uses a subclass of the following structure
+/*
+** CAPI3REF: Virtual Table Cursor Object  {H18020} 
+** KEYWORDS: sqlite3_vtab_cursor
+** EXPERIMENTAL
+**
+** Every module implementation uses a subclass of the following structure
 ** to describe cursors that point into the virtual table and are used
 ** to loop through the virtual table.  Cursors are created using the
 ** xOpen method of the module.  Each module implementation will define
@@ -2555,6 +5691,9 @@ struct sqlite3_vtab {
 **
 ** This superclass exists in order to define fields of the cursor that
 ** are common to all implementations.
+**
+** This interface is experimental and is subject to change or
+** removal in future releases of SQLite.
 */
 struct sqlite3_vtab_cursor {
   sqlite3_vtab *pVtab;      /* Virtual table of this cursor */
@@ -2562,13 +5701,22 @@ struct sqlite3_vtab_cursor {
 };
 
 /*
+** CAPI3REF: Declare The Schema Of A Virtual Table {H18280} 
+** EXPERIMENTAL
+**
 ** The xCreate and xConnect methods of a module use the following API
 ** to declare the format (the names and datatypes of the columns) of
 ** the virtual tables they implement.
+**
+** This interface is experimental and is subject to change or
+** removal in future releases of SQLite.
 */
-int sqlite3_declare_vtab(sqlite3*, const char *zCreateTable);
+SQLITE_EXPERIMENTAL int sqlite3_declare_vtab(sqlite3*, const char *zCreateTable);
 
 /*
+** CAPI3REF: Overload A Function For A Virtual Table {H18300} 
+** EXPERIMENTAL
+**
 ** Virtual tables can provide alternative implementations of functions
 ** using the xFindFunction method.  But global versions of those functions
 ** must exist in order to be overloaded.
@@ -2578,13 +5726,13 @@ int sqlite3_declare_vtab(sqlite3*, const char *zCreateTable);
 ** before this API is called, a new function is created.  The implementation
 ** of the new function always causes an exception to be thrown.  So
 ** the new function is not good for anything by itself.  Its only
-** purpose is to be a place-holder function that can be overloaded
+** purpose is to be a placeholder function that can be overloaded
 ** by virtual tables.
 **
 ** This API should be considered part of the virtual table interface,
 ** which is experimental and subject to change.
 */
-int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg);
+SQLITE_EXPERIMENTAL int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg);
 
 /*
 ** The interface to the virtual-table mechanism defined above (back up
@@ -2592,109 +5740,988 @@ int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg);
 ** to be experimental.  The interface might change in incompatible ways.
 ** If this is a problem for you, do not use the interface at this time.
 **
-** When the virtual-table mechanism stablizes, we will declare the
+** When the virtual-table mechanism stabilizes, we will declare the
 ** interface fixed, support it indefinitely, and remove this comment.
 **
 ****** EXPERIMENTAL - subject to change without notice **************
 */
 
 /*
-** CAPI3REF: A Handle To An Open BLOB
+** CAPI3REF: A Handle To An Open BLOB {H17800} 
+** KEYWORDS: {BLOB handle} {BLOB handles}
 **
-** An instance of the following opaque structure is used to 
-** represent an blob-handle.  A blob-handle is created by
-** [sqlite3_blob_open()] and destroyed by [sqlite3_blob_close()].
+** An instance of this object represents an open BLOB on which
+** [sqlite3_blob_open | incremental BLOB I/O] can be performed.
+** Objects of this type are created by [sqlite3_blob_open()]
+** and destroyed by [sqlite3_blob_close()].
 ** The [sqlite3_blob_read()] and [sqlite3_blob_write()] interfaces
-** can be used to read or write small subsections of the blob.
-** The [sqltie3_blob_size()] interface returns the size of the
-** blob in bytes.
+** can be used to read or write small subsections of the BLOB.
+** The [sqlite3_blob_bytes()] interface returns the size of the BLOB in bytes.
 */
 typedef struct sqlite3_blob sqlite3_blob;
 
 /*
-** CAPI3REF: Open A BLOB For Incremental I/O
+** CAPI3REF: Open A BLOB For Incremental I/O {H17810} 
 **
-** Open a handle to the blob located in row iRow,, column zColumn, 
-** table zTable in database zDb. i.e. the same blob that would
-** be selected by:
+** This interfaces opens a [BLOB handle | handle] to the BLOB located
+** in row iRow, column zColumn, table zTable in database zDb;
+** in other words, the same BLOB that would be selected by:
 **
 ** 
-**     SELECT zColumn FROM zDb.zTable WHERE rowid = iRow;
-** 

+**     SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow;
+** 
 {END}
 **
-** If the flags parameter is non-zero, the blob is opened for 
-** read and write access. If it is zero, the blob is opened for read 
-** access.
+** If the flags parameter is non-zero, the the BLOB is opened for read
+** and write access. If it is zero, the BLOB is opened for read access.
 **
-** On success, [SQLITE_OK] is returned and the new 
-** [sqlite3_blob | blob handle] is written to *ppBlob.
-** Otherwise an error code is returned and 
-** any value written to *ppBlob should not be used by the caller.
-** This function sets the database-handle error code and message
+** Note that the database name is not the filename that contains
+** the database but rather the symbolic name of the database that
+** is assigned when the database is connected using [ATTACH].
+** For the main database file, the database name is "main".
+** For TEMP tables, the database name is "temp".
+**
+** On success, [SQLITE_OK] is returned and the new [BLOB handle] is written
+** to *ppBlob. Otherwise an [error code] is returned and any value written
+** to *ppBlob should not be used by the caller.
+** This function sets the [database connection] error code and message
 ** accessible via [sqlite3_errcode()] and [sqlite3_errmsg()].
+**
+** If the row that a BLOB handle points to is modified by an
+** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects
+** then the BLOB handle is marked as "expired".
+** This is true if any column of the row is changed, even a column
+** other than the one the BLOB handle is open on.
+** Calls to [sqlite3_blob_read()] and [sqlite3_blob_write()] for
+** a expired BLOB handle fail with an return code of [SQLITE_ABORT].
+** Changes written into a BLOB prior to the BLOB expiring are not
+** rollback by the expiration of the BLOB.  Such changes will eventually
+** commit if the transaction continues to completion.
+**
+** INVARIANTS:
+**
+** {H17813} A successful invocation of the [sqlite3_blob_open(D,B,T,C,R,F,P)]
+**          interface shall open an [sqlite3_blob] object P on the BLOB
+**          in column C of the table T in the database B on
+**          the [database connection] D.
+**
+** {H17814} A successful invocation of [sqlite3_blob_open(D,...)] shall start
+**          a new transaction on the [database connection] D if that
+**          connection is not already in a transaction.
+**
+** {H17816} The [sqlite3_blob_open(D,B,T,C,R,F,P)] interface shall open
+**          the BLOB for read and write access if and only if the F
+**          parameter is non-zero.
+**
+** {H17819} The [sqlite3_blob_open()] interface shall return [SQLITE_OK] on
+**          success and an appropriate [error code] on failure.
+**
+** {H17821} If an error occurs during evaluation of [sqlite3_blob_open(D,...)]
+**          then subsequent calls to [sqlite3_errcode(D)],
+**          [sqlite3_extended_errcode()], 
+**          [sqlite3_errmsg(D)], and [sqlite3_errmsg16(D)] shall return
+**          information appropriate for that error.
+**
+** {H17824} If any column in the row that a [sqlite3_blob] has open is
+**          changed by a separate [UPDATE] or [DELETE] statement or by
+**          an [ON CONFLICT] side effect, then the [sqlite3_blob] shall
+**          be marked as invalid.
 */
 int sqlite3_blob_open(
   sqlite3*,
   const char *zDb,
   const char *zTable,
   const char *zColumn,
-  sqlite_int64 iRow,
+  sqlite3_int64 iRow,
   int flags,
   sqlite3_blob **ppBlob
 );
 
 /*
-** CAPI3REF:  Close A BLOB Handle
+** CAPI3REF: Close A BLOB Handle {H17830} 
 **
-** Close an open [sqlite3_blob | blob handle].
+** Closes an open [BLOB handle].
+**
+** Closing a BLOB shall cause the current transaction to commit
+** if there are no other BLOBs, no pending prepared statements, and the
+** database connection is in [autocommit mode].
+** If any writes were made to the BLOB, they might be held in cache
+** until the close operation if they will fit. {END}
+**
+** Closing the BLOB often forces the changes
+** out to disk and so if any I/O errors occur, they will likely occur
+** at the time when the BLOB is closed.  {H17833} Any errors that occur during
+** closing are reported as a non-zero return value.
+**
+** The BLOB is closed unconditionally.  Even if this routine returns
+** an error code, the BLOB is still closed.
+**
+** INVARIANTS:
+**
+** {H17833} The [sqlite3_blob_close(P)] interface closes an [sqlite3_blob]
+**          object P previously opened using [sqlite3_blob_open()].
+**
+** {H17836} Closing an [sqlite3_blob] object using
+**          [sqlite3_blob_close()] shall cause the current transaction to
+**          commit if there are no other open [sqlite3_blob] objects
+**          or [prepared statements] on the same [database connection] and
+**          the database connection is in [autocommit mode].
+**
+** {H17839} The [sqlite3_blob_close(P)] interfaces shall close the
+**          [sqlite3_blob] object P unconditionally, even if
+**          [sqlite3_blob_close(P)] returns something other than [SQLITE_OK].
 */
 int sqlite3_blob_close(sqlite3_blob *);
 
 /*
-** CAPI3REF:  Return The Size Of An Open BLOB
+** CAPI3REF: Return The Size Of An Open BLOB {H17840} 
 **
-** Return the size in bytes of the blob accessible via the open 
-** [sqlite3_blob | blob-handle] passed as an argument.
+** Returns the size in bytes of the BLOB accessible via the open
+** []BLOB handle] in its only argument.
+**
+** INVARIANTS:
+**
+** {H17843} The [sqlite3_blob_bytes(P)] interface returns the size
+**          in bytes of the BLOB that the [sqlite3_blob] object P
+**          refers to.
 */
 int sqlite3_blob_bytes(sqlite3_blob *);
 
 /*
-** CAPI3REF:  Read Data From A BLOB Incrementally
+** CAPI3REF: Read Data From A BLOB Incrementally {H17850} 
 **
-** This function is used to read data from an open 
-** [sqlite3_blob | blob-handle] into a caller supplied buffer.
-** n bytes of data are copied into buffer
-** z from the open blob, starting at offset iOffset.
+** This function is used to read data from an open [BLOB handle] into a
+** caller-supplied buffer. N bytes of data are copied into buffer Z
+** from the open BLOB, starting at offset iOffset.
 **
-** On success, SQLITE_OK is returned. Otherwise, an 
-** [SQLITE_ERROR | SQLite error code] or an
-** [SQLITE_IOERR_READ | extended error code] is returned.
+** If offset iOffset is less than N bytes from the end of the BLOB,
+** [SQLITE_ERROR] is returned and no data is read.  If N or iOffset is
+** less than zero, [SQLITE_ERROR] is returned and no data is read.
+**
+** An attempt to read from an expired [BLOB handle] fails with an
+** error code of [SQLITE_ABORT].
+**
+** On success, SQLITE_OK is returned.
+** Otherwise, an [error code] or an [extended error code] is returned.
+**
+** INVARIANTS:
+**
+** {H17853} A successful invocation of [sqlite3_blob_read(P,Z,N,X)] 
+**          shall reads N bytes of data out of the BLOB referenced by
+**          [BLOB handle] P beginning at offset X and store those bytes
+**          into buffer Z.
+**
+** {H17856} In [sqlite3_blob_read(P,Z,N,X)] if the size of the BLOB
+**          is less than N+X bytes, then the function shall leave the
+**          Z buffer unchanged and return [SQLITE_ERROR].
+**
+** {H17859} In [sqlite3_blob_read(P,Z,N,X)] if X or N is less than zero
+**          then the function shall leave the Z buffer unchanged
+**          and return [SQLITE_ERROR].
+**
+** {H17862} The [sqlite3_blob_read(P,Z,N,X)] interface shall return [SQLITE_OK]
+**          if N bytes are successfully read into buffer Z.
+**
+** {H17863} If the [BLOB handle] P is expired and X and N are within bounds
+**          then [sqlite3_blob_read(P,Z,N,X)] shall leave the Z buffer
+**          unchanged and return [SQLITE_ABORT].
+**
+** {H17865} If the requested read could not be completed,
+**          the [sqlite3_blob_read(P,Z,N,X)] interface shall return an
+**          appropriate [error code] or [extended error code].
+**
+** {H17868} If an error occurs during evaluation of [sqlite3_blob_read(P,...)]
+**          then subsequent calls to [sqlite3_errcode(D)],
+**          [sqlite3_extended_errcode()],
+**          [sqlite3_errmsg(D)], and [sqlite3_errmsg16(D)] shall return
+**          information appropriate for that error, where D is the
+**          [database connection] that was used to open the [BLOB handle] P.
 */
-int sqlite3_blob_read(sqlite3_blob *, void *z, int n, int iOffset);
+int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset);
 
 /*
-** CAPI3REF:  Write Data Into A BLOB Incrementally
+** CAPI3REF: Write Data Into A BLOB Incrementally {H17870} 
 **
-** This function is used to write data into an open 
-** [sqlite3_blob | blob-handle] from a user supplied buffer.
-** n bytes of data are copied from the buffer
-** pointed to by z into the open blob, starting at offset iOffset.
+** This function is used to write data into an open [BLOB handle] from a
+** caller-supplied buffer. N bytes of data are copied from the buffer Z
+** into the open BLOB, starting at offset iOffset.
 **
-** If the [sqlite3_blob | blob-handle] passed as the first argument
-** was not opened for writing (the flags parameter to [sqlite3_blob_open()]
-*** was zero), this function returns [SQLITE_READONLY].
+** If the [BLOB handle] passed as the first argument was not opened for
+** writing (the flags parameter to [sqlite3_blob_open()] was zero),
+** this function returns [SQLITE_READONLY].
 **
-** This function may only modify the contents of the blob, it is
-** not possible to increase the size of a blob using this API. If
-** offset iOffset is less than n bytes from the end of the blob, 
-** [SQLITE_ERROR] is returned and no data is written.
+** This function may only modify the contents of the BLOB; it is
+** not possible to increase the size of a BLOB using this API.
+** If offset iOffset is less than N bytes from the end of the BLOB,
+** [SQLITE_ERROR] is returned and no data is written.  If N is
+** less than zero [SQLITE_ERROR] is returned and no data is written.
 **
-** On success, SQLITE_OK is returned. Otherwise, an 
-** [SQLITE_ERROR | SQLite error code] or an
-** [SQLITE_IOERR_READ | extended error code] is returned.
+** An attempt to write to an expired [BLOB handle] fails with an
+** error code of [SQLITE_ABORT].  Writes to the BLOB that occurred
+** before the [BLOB handle] expired are not rolled back by the
+** expiration of the handle, though of course those changes might
+** have been overwritten by the statement that expired the BLOB handle
+** or by other independent statements.
+**
+** On success, SQLITE_OK is returned.
+** Otherwise, an  [error code] or an [extended error code] is returned.
+**
+** INVARIANTS:
+**
+** {H17873} A successful invocation of [sqlite3_blob_write(P,Z,N,X)]
+**          shall write N bytes of data from buffer Z into the BLOB 
+**          referenced by [BLOB handle] P beginning at offset X into
+**          the BLOB.
+**
+** {H17874} In the absence of other overridding changes, the changes
+**          written to a BLOB by [sqlite3_blob_write()] shall
+**          remain in effect after the associated [BLOB handle] expires.
+**
+** {H17875} If the [BLOB handle] P was opened for reading only then
+**          an invocation of [sqlite3_blob_write(P,Z,N,X)] shall leave
+**          the referenced BLOB unchanged and return [SQLITE_READONLY].
+**
+** {H17876} If the size of the BLOB referenced by [BLOB handle] P is
+**          less than N+X bytes then [sqlite3_blob_write(P,Z,N,X)] shall
+**          leave the BLOB unchanged and return [SQLITE_ERROR].
+**
+** {H17877} If the [BLOB handle] P is expired and X and N are within bounds
+**          then [sqlite3_blob_read(P,Z,N,X)] shall leave the BLOB
+**          unchanged and return [SQLITE_ABORT].
+**
+** {H17879} If X or N are less than zero then [sqlite3_blob_write(P,Z,N,X)]
+**          shall leave the BLOB referenced by [BLOB handle] P unchanged
+**          and return [SQLITE_ERROR].
+**
+** {H17882} The [sqlite3_blob_write(P,Z,N,X)] interface shall return
+**          [SQLITE_OK] if N bytes where successfully written into the BLOB.
+**
+** {H17885} If the requested write could not be completed,
+**          the [sqlite3_blob_write(P,Z,N,X)] interface shall return an
+**          appropriate [error code] or [extended error code].
+**
+** {H17888} If an error occurs during evaluation of [sqlite3_blob_write(D,...)]
+**          then subsequent calls to [sqlite3_errcode(D)],
+**          [sqlite3_extended_errcode()],
+**          [sqlite3_errmsg(D)], and [sqlite3_errmsg16(D)] shall return
+**          information appropriate for that error.
 */
 int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset);
 
+/*
+** CAPI3REF: Virtual File System Objects {H11200} 
+**
+** A virtual filesystem (VFS) is an [sqlite3_vfs] object
+** that SQLite uses to interact
+** with the underlying operating system.  Most SQLite builds come with a
+** single default VFS that is appropriate for the host computer.
+** New VFSes can be registered and existing VFSes can be unregistered.
+** The following interfaces are provided.
+**
+** The sqlite3_vfs_find() interface returns a pointer to a VFS given its name.
+** Names are case sensitive.
+** Names are zero-terminated UTF-8 strings.
+** If there is no match, a NULL pointer is returned.
+** If zVfsName is NULL then the default VFS is returned.
+**
+** New VFSes are registered with sqlite3_vfs_register().
+** Each new VFS becomes the default VFS if the makeDflt flag is set.
+** The same VFS can be registered multiple times without injury.
+** To make an existing VFS into the default VFS, register it again
+** with the makeDflt flag set.  If two different VFSes with the
+** same name are registered, the behavior is undefined.  If a
+** VFS is registered with a name that is NULL or an empty string,
+** then the behavior is undefined.
+**
+** Unregister a VFS with the sqlite3_vfs_unregister() interface.
+** If the default VFS is unregistered, another VFS is chosen as
+** the default.  The choice for the new VFS is arbitrary.
+**
+** INVARIANTS:
+**
+** {H11203} The [sqlite3_vfs_find(N)] interface returns a pointer to the
+**          registered [sqlite3_vfs] object whose name exactly matches
+**          the zero-terminated UTF-8 string N, or it returns NULL if
+**          there is no match.
+**
+** {H11206} If the N parameter to [sqlite3_vfs_find(N)] is NULL then
+**          the function returns a pointer to the default [sqlite3_vfs]
+**          object if there is one, or NULL if there is no default
+**          [sqlite3_vfs] object.
+**
+** {H11209} The [sqlite3_vfs_register(P,F)] interface registers the
+**          well-formed [sqlite3_vfs] object P using the name given
+**          by the zName field of the object.
+**
+** {H11212} Using the [sqlite3_vfs_register(P,F)] interface to register
+**          the same [sqlite3_vfs] object multiple times is a harmless no-op.
+**
+** {H11215} The [sqlite3_vfs_register(P,F)] interface makes the [sqlite3_vfs]
+**          object P the default [sqlite3_vfs] object if F is non-zero.
+**
+** {H11218} The [sqlite3_vfs_unregister(P)] interface unregisters the
+**          [sqlite3_vfs] object P so that it is no longer returned by
+**          subsequent calls to [sqlite3_vfs_find()].
+*/
+sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName);
+int sqlite3_vfs_register(sqlite3_vfs*, int makeDflt);
+int sqlite3_vfs_unregister(sqlite3_vfs*);
+
+/*
+** CAPI3REF: Mutexes {H17000} 
+**
+** The SQLite core uses these routines for thread
+** synchronization. Though they are intended for internal
+** use by SQLite, code that links against SQLite is
+** permitted to use any of these routines.
+**
+** The SQLite source code contains multiple implementations
+** of these mutex routines.  An appropriate implementation
+** is selected automatically at compile-time.  The following
+** implementations are available in the SQLite core:
+**
+** 
    
+** 
  •    SQLITE_MUTEX_OS2
+** 
  •    SQLITE_MUTEX_PTHREAD
+** 
  •    SQLITE_MUTEX_W32
+** 
  •    SQLITE_MUTEX_NOOP
+** 

+**
+** The SQLITE_MUTEX_NOOP implementation is a set of routines
+** that does no real locking and is appropriate for use in
+** a single-threaded application.  The SQLITE_MUTEX_OS2,
+** SQLITE_MUTEX_PTHREAD, and SQLITE_MUTEX_W32 implementations
+** are appropriate for use on OS/2, Unix, and Windows.
+**
+** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor
+** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex
+** implementation is included with the library. In this case the
+** application must supply a custom mutex implementation using the
+** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function
+** before calling sqlite3_initialize() or any other public sqlite3_
+** function that calls sqlite3_initialize().
+**
+** {H17011} The sqlite3_mutex_alloc() routine allocates a new
+** mutex and returns a pointer to it. {H17012} If it returns NULL
+** that means that a mutex could not be allocated. {H17013} SQLite
+** will unwind its stack and return an error. {H17014} The argument
+** to sqlite3_mutex_alloc() is one of these integer constants:
+**
+** 
    
+** 
  •   SQLITE_MUTEX_FAST
+** 
  •   SQLITE_MUTEX_RECURSIVE
+** 
  •   SQLITE_MUTEX_STATIC_MASTER
+** 
  •   SQLITE_MUTEX_STATIC_MEM
+** 
  •   SQLITE_MUTEX_STATIC_MEM2
+** 
  •   SQLITE_MUTEX_STATIC_PRNG
+** 
  •   SQLITE_MUTEX_STATIC_LRU
+** 
  •   SQLITE_MUTEX_STATIC_LRU2
+** 

+**
+** {H17015} The first two constants cause sqlite3_mutex_alloc() to create
+** a new mutex.  The new mutex is recursive when SQLITE_MUTEX_RECURSIVE
+** is used but not necessarily so when SQLITE_MUTEX_FAST is used. {END}
+** The mutex implementation does not need to make a distinction
+** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does
+** not want to.  {H17016} But SQLite will only request a recursive mutex in
+** cases where it really needs one.  {END} If a faster non-recursive mutex
+** implementation is available on the host platform, the mutex subsystem
+** might return such a mutex in response to SQLITE_MUTEX_FAST.
+**
+** {H17017} The other allowed parameters to sqlite3_mutex_alloc() each return
+** a pointer to a static preexisting mutex. {END}  Four static mutexes are
+** used by the current version of SQLite.  Future versions of SQLite
+** may add additional static mutexes.  Static mutexes are for internal
+** use by SQLite only.  Applications that use SQLite mutexes should
+** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or
+** SQLITE_MUTEX_RECURSIVE.
+**
+** {H17018} Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST
+** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc()
+** returns a different mutex on every call.  {H17034} But for the static
+** mutex types, the same mutex is returned on every call that has
+** the same type number.
+**
+** {H17019} The sqlite3_mutex_free() routine deallocates a previously
+** allocated dynamic mutex. {H17020} SQLite is careful to deallocate every
+** dynamic mutex that it allocates. {A17021} The dynamic mutexes must not be in
+** use when they are deallocated. {A17022} Attempting to deallocate a static
+** mutex results in undefined behavior. {H17023} SQLite never deallocates
+** a static mutex. {END}
+**
+** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt
+** to enter a mutex. {H17024} If another thread is already within the mutex,
+** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return
+** SQLITE_BUSY. {H17025}  The sqlite3_mutex_try() interface returns [SQLITE_OK]
+** upon successful entry.  {H17026} Mutexes created using
+** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread.
+** {H17027} In such cases the,
+** mutex must be exited an equal number of times before another thread
+** can enter.  {A17028} If the same thread tries to enter any other
+** kind of mutex more than once, the behavior is undefined.
+** {H17029} SQLite will never exhibit
+** such behavior in its own use of mutexes.
+**
+** Some systems (for example, Windows 95) do not support the operation
+** implemented by sqlite3_mutex_try().  On those systems, sqlite3_mutex_try()
+** will always return SQLITE_BUSY.  {H17030} The SQLite core only ever uses
+** sqlite3_mutex_try() as an optimization so this is acceptable behavior.
+**
+** {H17031} The sqlite3_mutex_leave() routine exits a mutex that was
+** previously entered by the same thread.  {A17032} The behavior
+** is undefined if the mutex is not currently entered by the
+** calling thread or is not currently allocated.  {H17033} SQLite will
+** never do either. {END}
+**
+** If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or
+** sqlite3_mutex_leave() is a NULL pointer, then all three routines
+** behave as no-ops.
+**
+** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()].
+*/
+sqlite3_mutex *sqlite3_mutex_alloc(int);
+void sqlite3_mutex_free(sqlite3_mutex*);
+void sqlite3_mutex_enter(sqlite3_mutex*);
+int sqlite3_mutex_try(sqlite3_mutex*);
+void sqlite3_mutex_leave(sqlite3_mutex*);
+
+/*
+** CAPI3REF: Mutex Methods Object {H17120} 
+** EXPERIMENTAL
+**
+** An instance of this structure defines the low-level routines
+** used to allocate and use mutexes.
+**
+** Usually, the default mutex implementations provided by SQLite are
+** sufficient, however the user has the option of substituting a custom
+** implementation for specialized deployments or systems for which SQLite
+** does not provide a suitable implementation. In this case, the user
+** creates and populates an instance of this structure to pass
+** to sqlite3_config() along with the [SQLITE_CONFIG_MUTEX] option.
+** Additionally, an instance of this structure can be used as an
+** output variable when querying the system for the current mutex
+** implementation, using the [SQLITE_CONFIG_GETMUTEX] option.
+**
+** The xMutexInit method defined by this structure is invoked as
+** part of system initialization by the sqlite3_initialize() function.
+** {H17001} The xMutexInit routine shall be called by SQLite once for each
+** effective call to [sqlite3_initialize()].
+**
+** The xMutexEnd method defined by this structure is invoked as
+** part of system shutdown by the sqlite3_shutdown() function. The
+** implementation of this method is expected to release all outstanding
+** resources obtained by the mutex methods implementation, especially
+** those obtained by the xMutexInit method. {H17003} The xMutexEnd()
+** interface shall be invoked once for each call to [sqlite3_shutdown()].
+**
+** The remaining seven methods defined by this structure (xMutexAlloc,
+** xMutexFree, xMutexEnter, xMutexTry, xMutexLeave, xMutexHeld and
+** xMutexNotheld) implement the following interfaces (respectively):
+**
+** 
    
+**   
  •   [sqlite3_mutex_alloc()] 
    • 
+**   
  •   [sqlite3_mutex_free()] 
    • 
+**   
  •   [sqlite3_mutex_enter()] 
    • 
+**   
  •   [sqlite3_mutex_try()] 
    • 
+**   
  •   [sqlite3_mutex_leave()] 
    • 
+**   
  •   [sqlite3_mutex_held()] 
    • 
+**   
  •   [sqlite3_mutex_notheld()] 
    • 
+** 

+**
+** The only difference is that the public sqlite3_XXX functions enumerated
+** above silently ignore any invocations that pass a NULL pointer instead
+** of a valid mutex handle. The implementations of the methods defined
+** by this structure are not required to handle this case, the results
+** of passing a NULL pointer instead of a valid mutex handle are undefined
+** (i.e. it is acceptable to provide an implementation that segfaults if
+** it is passed a NULL pointer).
+*/
+typedef struct sqlite3_mutex_methods sqlite3_mutex_methods;
+struct sqlite3_mutex_methods {
+  int (*xMutexInit)(void);
+  int (*xMutexEnd)(void);
+  sqlite3_mutex *(*xMutexAlloc)(int);
+  void (*xMutexFree)(sqlite3_mutex *);
+  void (*xMutexEnter)(sqlite3_mutex *);
+  int (*xMutexTry)(sqlite3_mutex *);
+  void (*xMutexLeave)(sqlite3_mutex *);
+  int (*xMutexHeld)(sqlite3_mutex *);
+  int (*xMutexNotheld)(sqlite3_mutex *);
+};
+
+/*
+** CAPI3REF: Mutex Verification Routines {H17080}  
+**
+** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines
+** are intended for use inside assert() statements. {H17081} The SQLite core
+** never uses these routines except inside an assert() and applications
+** are advised to follow the lead of the core.  {H17082} The core only
+** provides implementations for these routines when it is compiled
+** with the SQLITE_DEBUG flag.  {A17087} External mutex implementations
+** are only required to provide these routines if SQLITE_DEBUG is
+** defined and if NDEBUG is not defined.
+**
+** {H17083} These routines should return true if the mutex in their argument
+** is held or not held, respectively, by the calling thread.
+**
+** {X17084} The implementation is not required to provided versions of these
+** routines that actually work. If the implementation does not provide working
+** versions of these routines, it should at least provide stubs that always
+** return true so that one does not get spurious assertion failures.
+**
+** {H17085} If the argument to sqlite3_mutex_held() is a NULL pointer then
+** the routine should return 1.  {END} This seems counter-intuitive since
+** clearly the mutex cannot be held if it does not exist.  But the
+** the reason the mutex does not exist is because the build is not
+** using mutexes.  And we do not want the assert() containing the
+** call to sqlite3_mutex_held() to fail, so a non-zero return is
+** the appropriate thing to do.  {H17086} The sqlite3_mutex_notheld()
+** interface should also return 1 when given a NULL pointer.
+*/
+int sqlite3_mutex_held(sqlite3_mutex*);
+int sqlite3_mutex_notheld(sqlite3_mutex*);
+
+/*
+** CAPI3REF: Mutex Types {H17001} 
+**
+** The [sqlite3_mutex_alloc()] interface takes a single argument
+** which is one of these integer constants.
+**
+** The set of static mutexes may change from one SQLite release to the
+** next.  Applications that override the built-in mutex logic must be
+** prepared to accommodate additional static mutexes.
+*/
+#define SQLITE_MUTEX_FAST             0
+#define SQLITE_MUTEX_RECURSIVE        1
+#define SQLITE_MUTEX_STATIC_MASTER    2
+#define SQLITE_MUTEX_STATIC_MEM       3  /* sqlite3_malloc() */
+#define SQLITE_MUTEX_STATIC_MEM2      4  /* sqlite3_release_memory() */
+#define SQLITE_MUTEX_STATIC_PRNG      5  /* sqlite3_random() */
+#define SQLITE_MUTEX_STATIC_LRU       6  /* lru page list */
+#define SQLITE_MUTEX_STATIC_LRU2      7  /* lru page list */
+
+/*
+** CAPI3REF: Retrieve the mutex for a database connection {H17002} 
+**
+** This interface returns a pointer the [sqlite3_mutex] object that 
+** serializes access to the [database connection] given in the argument
+** when the [threading mode] is Serialized.
+** If the [threading mode] is Single-thread or Multi-thread then this
+** routine returns a NULL pointer.
+*/
+sqlite3_mutex *sqlite3_db_mutex(sqlite3*);
+
+/*
+** CAPI3REF: Low-Level Control Of Database Files {H11300} 
+**
+** {H11301} The [sqlite3_file_control()] interface makes a direct call to the
+** xFileControl method for the [sqlite3_io_methods] object associated
+** with a particular database identified by the second argument. {H11302} The
+** name of the database is the name assigned to the database by the
+** ATTACH SQL command that opened the
+** database. {H11303} To control the main database file, use the name "main"
+** or a NULL pointer. {H11304} The third and fourth parameters to this routine
+** are passed directly through to the second and third parameters of
+** the xFileControl method.  {H11305} The return value of the xFileControl
+** method becomes the return value of this routine.
+**
+** {H11306} If the second parameter (zDbName) does not match the name of any
+** open database file, then SQLITE_ERROR is returned. {H11307} This error
+** code is not remembered and will not be recalled by [sqlite3_errcode()]
+** or [sqlite3_errmsg()]. {A11308} The underlying xFileControl method might
+** also return SQLITE_ERROR.  {A11309} There is no way to distinguish between
+** an incorrect zDbName and an SQLITE_ERROR return from the underlying
+** xFileControl method. {END}
+**
+** See also: [SQLITE_FCNTL_LOCKSTATE]
+*/
+int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*);
+
+/*
+** CAPI3REF: Testing Interface {H11400} 
+**
+** The sqlite3_test_control() interface is used to read out internal
+** state of SQLite and to inject faults into SQLite for testing
+** purposes.  The first parameter is an operation code that determines
+** the number, meaning, and operation of all subsequent parameters.
+**
+** This interface is not for use by applications.  It exists solely
+** for verifying the correct operation of the SQLite library.  Depending
+** on how the SQLite library is compiled, this interface might not exist.
+**
+** The details of the operation codes, their meanings, the parameters
+** they take, and what they do are all subject to change without notice.
+** Unlike most of the SQLite API, this function is not guaranteed to
+** operate consistently from one release to the next.
+*/
+int sqlite3_test_control(int op, ...);
+
+/*
+** CAPI3REF: Testing Interface Operation Codes {H11410} 
+**
+** These constants are the valid operation code parameters used
+** as the first argument to [sqlite3_test_control()].
+**
+** These parameters and their meanings are subject to change
+** without notice.  These values are for testing purposes only.
+** Applications should not use any of these parameters or the
+** [sqlite3_test_control()] interface.
+*/
+#define SQLITE_TESTCTRL_PRNG_SAVE                5
+#define SQLITE_TESTCTRL_PRNG_RESTORE             6
+#define SQLITE_TESTCTRL_PRNG_RESET               7
+#define SQLITE_TESTCTRL_BITVEC_TEST              8
+#define SQLITE_TESTCTRL_FAULT_INSTALL            9
+#define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS     10
+
+/*
+** CAPI3REF: SQLite Runtime Status {H17200} 
+** EXPERIMENTAL
+**
+** This interface is used to retrieve runtime status information
+** about the preformance of SQLite, and optionally to reset various
+** highwater marks.  The first argument is an integer code for
+** the specific parameter to measure.  Recognized integer codes
+** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...].
+** The current value of the parameter is returned into *pCurrent.
+** The highest recorded value is returned in *pHighwater.  If the
+** resetFlag is true, then the highest record value is reset after
+** *pHighwater is written. Some parameters do not record the highest
+** value.  For those parameters
+** nothing is written into *pHighwater and the resetFlag is ignored.
+** Other parameters record only the highwater mark and not the current
+** value.  For these latter parameters nothing is written into *pCurrent.
+**
+** This routine returns SQLITE_OK on success and a non-zero
+** [error code] on failure.
+**
+** This routine is threadsafe but is not atomic.  This routine can
+** called while other threads are running the same or different SQLite
+** interfaces.  However the values returned in *pCurrent and
+** *pHighwater reflect the status of SQLite at different points in time
+** and it is possible that another thread might change the parameter
+** in between the times when *pCurrent and *pHighwater are written.
+**
+** See also: [sqlite3_db_status()]
+*/
+SQLITE_EXPERIMENTAL int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag);
+
+
+/*
+** CAPI3REF: Status Parameters {H17250} 
+** EXPERIMENTAL
+**
+** These integer constants designate various run-time status parameters
+** that can be returned by [sqlite3_status()].
+**
+** 

+** 
SQLITE_STATUS_MEMORY_USED

+** 
This parameter is the current amount of memory checked out
+** using [sqlite3_malloc()], either directly or indirectly.  The
+** figure includes calls made to [sqlite3_malloc()] by the application
+** and internal memory usage by the SQLite library.  Scratch memory
+** controlled by [SQLITE_CONFIG_SCRATCH] and auxiliary page-cache
+** memory controlled by [SQLITE_CONFIG_PAGECACHE] is not included in
+** this parameter.  The amount returned is the sum of the allocation
+** sizes as reported by the xSize method in [sqlite3_mem_methods].

+**
+** 
SQLITE_STATUS_MALLOC_SIZE

+** 
This parameter records the largest memory allocation request
+** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their
+** internal equivalents).  Only the value returned in the
+** *pHighwater parameter to [sqlite3_status()] is of interest.  
+** The value written into the *pCurrent parameter is undefined.

+**
+** 
SQLITE_STATUS_PAGECACHE_USED

+** 
This parameter returns the number of pages used out of the
+** [pagecache memory allocator] that was configured using 
+** [SQLITE_CONFIG_PAGECACHE].  The
+** value returned is in pages, not in bytes.

+**
+** 
SQLITE_STATUS_PAGECACHE_OVERFLOW

+** 
This parameter returns the number of bytes of page cache
+** allocation which could not be statisfied by the [SQLITE_CONFIG_PAGECACHE]
+** buffer and where forced to overflow to [sqlite3_malloc()].  The
+** returned value includes allocations that overflowed because they
+** where too large (they were larger than the "sz" parameter to
+** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because
+** no space was left in the page cache.

+**
+** 
SQLITE_STATUS_PAGECACHE_SIZE

+** 
This parameter records the largest memory allocation request
+** handed to [pagecache memory allocator].  Only the value returned in the
+** *pHighwater parameter to [sqlite3_status()] is of interest.  
+** The value written into the *pCurrent parameter is undefined.

+**
+** 
SQLITE_STATUS_SCRATCH_USED

+** 
This parameter returns the number of allocations used out of the
+** [scratch memory allocator] configured using
+** [SQLITE_CONFIG_SCRATCH].  The value returned is in allocations, not
+** in bytes.  Since a single thread may only have one scratch allocation
+** outstanding at time, this parameter also reports the number of threads
+** using scratch memory at the same time.

+**
+** 
SQLITE_STATUS_SCRATCH_OVERFLOW

+** 
This parameter returns the number of bytes of scratch memory
+** allocation which could not be statisfied by the [SQLITE_CONFIG_SCRATCH]
+** buffer and where forced to overflow to [sqlite3_malloc()].  The values
+** returned include overflows because the requested allocation was too
+** larger (that is, because the requested allocation was larger than the
+** "sz" parameter to [SQLITE_CONFIG_SCRATCH]) and because no scratch buffer
+** slots were available.
+** 

+**
+** 
SQLITE_STATUS_SCRATCH_SIZE

+** 
This parameter records the largest memory allocation request
+** handed to [scratch memory allocator].  Only the value returned in the
+** *pHighwater parameter to [sqlite3_status()] is of interest.  
+** The value written into the *pCurrent parameter is undefined.

+**
+** 
SQLITE_STATUS_PARSER_STACK

+** 
This parameter records the deepest parser stack.  It is only
+** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].

+** 

+**
+** New status parameters may be added from time to time.
+*/
+#define SQLITE_STATUS_MEMORY_USED          0
+#define SQLITE_STATUS_PAGECACHE_USED       1
+#define SQLITE_STATUS_PAGECACHE_OVERFLOW   2
+#define SQLITE_STATUS_SCRATCH_USED         3
+#define SQLITE_STATUS_SCRATCH_OVERFLOW     4
+#define SQLITE_STATUS_MALLOC_SIZE          5
+#define SQLITE_STATUS_PARSER_STACK         6
+#define SQLITE_STATUS_PAGECACHE_SIZE       7
+#define SQLITE_STATUS_SCRATCH_SIZE         8
+
+/*
+** CAPI3REF: Database Connection Status {H17500} 
+** EXPERIMENTAL
+**
+** This interface is used to retrieve runtime status information 
+** about a single [database connection].  The first argument is the
+** database connection object to be interrogated.  The second argument
+** is the parameter to interrogate.  Currently, the only allowed value
+** for the second parameter is [SQLITE_DBSTATUS_LOOKASIDE_USED].
+** Additional options will likely appear in future releases of SQLite.
+**
+** The current value of the requested parameter is written into *pCur
+** and the highest instantaneous value is written into *pHiwtr.  If
+** the resetFlg is true, then the highest instantaneous value is
+** reset back down to the current value.
+**
+** See also: [sqlite3_status()] and [sqlite3_stmt_status()].
+*/
+SQLITE_EXPERIMENTAL int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
+
+/*
+** CAPI3REF: Status Parameters for database connections {H17520} 
+** EXPERIMENTAL
+**
+** Status verbs for [sqlite3_db_status()].
+**
+** 

+** 
SQLITE_DBSTATUS_LOOKASIDE_USED

+** 
This parameter returns the number of lookaside memory slots currently
+** checked out.

+** 

+*/
+#define SQLITE_DBSTATUS_LOOKASIDE_USED     0
+
+
+/*
+** CAPI3REF: Prepared Statement Status {H17550} 
+** EXPERIMENTAL
+**
+** Each prepared statement maintains various
+** [SQLITE_STMTSTATUS_SORT | counters] that measure the number
+** of times it has performed specific operations.  These counters can
+** be used to monitor the performance characteristics of the prepared
+** statements.  For example, if the number of table steps greatly exceeds
+** the number of table searches or result rows, that would tend to indicate
+** that the prepared statement is using a full table scan rather than
+** an index.  
+**
+** This interface is used to retrieve and reset counter values from
+** a [prepared statement].  The first argument is the prepared statement
+** object to be interrogated.  The second argument
+** is an integer code for a specific [SQLITE_STMTSTATUS_SORT | counter]
+** to be interrogated. 
+** The current value of the requested counter is returned.
+** If the resetFlg is true, then the counter is reset to zero after this
+** interface call returns.
+**
+** See also: [sqlite3_status()] and [sqlite3_db_status()].
+*/
+SQLITE_EXPERIMENTAL int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg);
+
+/*
+** CAPI3REF: Status Parameters for prepared statements {H17570} 
+** EXPERIMENTAL
+**
+** These preprocessor macros define integer codes that name counter
+** values associated with the [sqlite3_stmt_status()] interface.
+** The meanings of the various counters are as follows:
+**
+** 

+** 
SQLITE_STMTSTATUS_FULLSCAN_STEP

+** 
This is the number of times that SQLite has stepped forward in
+** a table as part of a full table scan.  Large numbers for this counter
+** may indicate opportunities for performance improvement through 
+** careful use of indices.

+**
+** 
SQLITE_STMTSTATUS_SORT

+** 
This is the number of sort operations that have occurred.
+** A non-zero value in this counter may indicate an opportunity to
+** improvement performance through careful use of indices.

+**
+** 

+*/
+#define SQLITE_STMTSTATUS_FULLSCAN_STEP     1
+#define SQLITE_STMTSTATUS_SORT              2
+
+/*
+** CAPI3REF: Custom Page Cache Object
+** EXPERIMENTAL
+**
+** The sqlite3_pcache type is opaque.  It is implemented by
+** the pluggable module.  The SQLite core has no knowledge of
+** its size or internal structure and never deals with the
+** sqlite3_pcache object except by holding and passing pointers
+** to the object.
+**
+** See [sqlite3_pcache_methods] for additional information.
+*/
+typedef struct sqlite3_pcache sqlite3_pcache;
+
+/*
+** CAPI3REF: Application Defined Page Cache.
+** EXPERIMENTAL
+**
+** The [sqlite3_config]([SQLITE_CONFIG_PCACHE], ...) interface can
+** register an alternative page cache implementation by passing in an 
+** instance of the sqlite3_pcache_methods structure. The majority of the 
+** heap memory used by sqlite is used by the page cache to cache data read 
+** from, or ready to be written to, the database file. By implementing a 
+** custom page cache using this API, an application can control more 
+** precisely the amount of memory consumed by sqlite, the way in which 
+** said memory is allocated and released, and the policies used to 
+** determine exactly which parts of a database file are cached and for 
+** how long.
+**
+** The contents of the structure are copied to an internal buffer by sqlite
+** within the call to [sqlite3_config].
+**
+** The xInit() method is called once for each call to [sqlite3_initialize()]
+** (usually only once during the lifetime of the process). It is passed
+** a copy of the sqlite3_pcache_methods.pArg value. It can be used to set
+** up global structures and mutexes required by the custom page cache 
+** implementation. The xShutdown() method is called from within 
+** [sqlite3_shutdown()], if the application invokes this API. It can be used
+** to clean up any outstanding resources before process shutdown, if required.
+**
+** The xCreate() method is used to construct a new cache instance. The
+** first parameter, szPage, is the size in bytes of the pages that must
+** be allocated by the cache. szPage will not be a power of two. The
+** second argument, bPurgeable, is true if the cache being created will
+** be used to cache database pages read from a file stored on disk, or
+** false if it is used for an in-memory database. The cache implementation
+** does not have to do anything special based on the value of bPurgeable,
+** it is purely advisory. 
+**
+** The xCachesize() method may be called at any time by SQLite to set the
+** suggested maximum cache-size (number of pages stored by) the cache
+** instance passed as the first argument. This is the value configured using
+** the SQLite "[PRAGMA cache_size]" command. As with the bPurgeable parameter,
+** the implementation is not required to do anything special with this
+** value, it is advisory only.
+**
+** The xPagecount() method should return the number of pages currently
+** stored in the cache supplied as an argument.
+** 
+** The xFetch() method is used to fetch a page and return a pointer to it. 
+** A 'page', in this context, is a buffer of szPage bytes aligned at an
+** 8-byte boundary. The page to be fetched is determined by the key. The
+** mimimum key value is 1. After it has been retrieved using xFetch, the page 
+** is considered to be pinned.
+**
+** If the requested page is already in the page cache, then a pointer to
+** the cached buffer should be returned with its contents intact. If the
+** page is not already in the cache, then the expected behaviour of the
+** cache is determined by the value of the createFlag parameter passed
+** to xFetch, according to the following table:
+**
+** 
+**   
createFlagExpected Behaviour
+**   
0NULL should be returned. No new cache entry is created.
+**   
1If createFlag is set to 1, this indicates that 
+**                SQLite is holding pinned pages that can be unpinned
+**                by writing their contents to the database file (a
+**                relatively expensive operation). In this situation the
+**                cache implementation has two choices: it can return NULL,
+**                in which case SQLite will attempt to unpin one or more 
+**                pages before re-requesting the same page, or it can
+**                allocate a new page and return a pointer to it. If a new
+**                page is allocated, then it must be completely zeroed before 
+**                it is returned.
+**   
2If createFlag is set to 2, then SQLite is not holding any
+**                pinned pages associated with the specific cache passed
+**                as the first argument to xFetch() that can be unpinned. The
+**                cache implementation should attempt to allocate a new
+**                cache entry and return a pointer to it. Again, the new
+**                page should be zeroed before it is returned. If the xFetch()
+**                method returns NULL when createFlag==2, SQLite assumes that
+**                a memory allocation failed and returns SQLITE_NOMEM to the
+**                user.
+** 

+**
+** xUnpin() is called by SQLite with a pointer to a currently pinned page
+** as its second argument. If the third parameter, discard, is non-zero,
+** then the page should be evicted from the cache. In this case SQLite 
+** assumes that the next time the page is retrieved from the cache using
+** the xFetch() method, it will be zeroed. If the discard parameter is
+** zero, then the page is considered to be unpinned. The cache implementation
+** may choose to reclaim (free or recycle) unpinned pages at any time.
+** SQLite assumes that next time the page is retrieved from the cache
+** it will either be zeroed, or contain the same data that it did when it
+** was unpinned.
+**
+** The cache is not required to perform any reference counting. A single 
+** call to xUnpin() unpins the page regardless of the number of prior calls 
+** to xFetch().
+**
+** The xRekey() method is used to change the key value associated with the
+** page passed as the second argument from oldKey to newKey. If the cache
+** previously contains an entry associated with newKey, it should be
+** discarded. Any prior cache entry associated with newKey is guaranteed not
+** to be pinned.
+**
+** When SQLite calls the xTruncate() method, the cache must discard all
+** existing cache entries with page numbers (keys) greater than or equal
+** to the value of the iLimit parameter passed to xTruncate(). If any
+** of these pages are pinned, they are implicitly unpinned, meaning that
+** they can be safely discarded.
+**
+** The xDestroy() method is used to delete a cache allocated by xCreate().
+** All resources associated with the specified cache should be freed. After
+** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*]
+** handle invalid, and will not use it with any other sqlite3_pcache_methods
+** functions.
+*/
+typedef struct sqlite3_pcache_methods sqlite3_pcache_methods;
+struct sqlite3_pcache_methods {
+  void *pArg;
+  int (*xInit)(void*);
+  void (*xShutdown)(void*);
+  sqlite3_pcache *(*xCreate)(int szPage, int bPurgeable);
+  void (*xCachesize)(sqlite3_pcache*, int nCachesize);
+  int (*xPagecount)(sqlite3_pcache*);
+  void *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag);
+  void (*xUnpin)(sqlite3_pcache*, void*, int discard);
+  void (*xRekey)(sqlite3_pcache*, void*, unsigned oldKey, unsigned newKey);
+  void (*xTruncate)(sqlite3_pcache*, unsigned iLimit);
+  void (*xDestroy)(sqlite3_pcache*);
+};
+
 /*
 ** Undo the hack that converts floating point types to integer for
 ** builds on processors without floating point support.
diff --git a/sqlite3ext.h b/sqlite3ext.h
index ba6d378..5526646 100644
--- a/sqlite3ext.h
+++ b/sqlite3ext.h
@@ -15,7 +15,7 @@
 ** as extensions by SQLite should #include this file instead of 
 ** sqlite3.h.
 **
-** @(#) $Id: sqlite3ext.h,v 1.12 2007/07/20 10:48:36 drh Exp $
+** @(#) $Id: sqlite3ext.h,v 1.25 2008/10/12 00:27:54 shane Exp $
 */
 #ifndef _SQLITE3EXT_H_
 #define _SQLITE3EXT_H_
@@ -24,13 +24,13 @@
 typedef struct sqlite3_api_routines sqlite3_api_routines;
 
 /*
-** The following structure hold pointers to all of the SQLite API
+** The following structure holds pointers to all of the SQLite API
 ** routines.
 **
 ** WARNING:  In order to maintain backwards compatibility, add new
 ** interfaces to the end of this structure only.  If you insert new
 ** interfaces in the middle of this structure, then older different
-** versions of SQLite will not be able to load each others shared
+** versions of SQLite will not be able to load each others' shared
 ** libraries!
 */
 struct sqlite3_api_routines {
@@ -78,7 +78,7 @@ struct sqlite3_api_routines {
   int  (*complete)(const char*sql);
   int  (*complete16)(const void*sql);
   int  (*create_collation)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*));
-  int  (*create_collation16)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*));
+  int  (*create_collation16)(sqlite3*,const void*,int,void*,int(*)(void*,int,const void*,int,const void*));
   int  (*create_function)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*));
   int  (*create_function16)(sqlite3*,const void*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*));
   int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*);
@@ -149,11 +149,50 @@ struct sqlite3_api_routines {
   const void * (*value_text16le)(sqlite3_value*);
   int  (*value_type)(sqlite3_value*);
   char *(*vmprintf)(const char*,va_list);
+  /* Added ??? */
   int (*overload_function)(sqlite3*, const char *zFuncName, int nArg);
+  /* Added by 3.3.13 */
   int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**);
   int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**);
   int (*clear_bindings)(sqlite3_stmt*);
+  /* Added by 3.4.1 */
   int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,void (*xDestroy)(void *));
+  /* Added by 3.5.0 */
+  int (*bind_zeroblob)(sqlite3_stmt*,int,int);
+  int (*blob_bytes)(sqlite3_blob*);
+  int (*blob_close)(sqlite3_blob*);
+  int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,int,sqlite3_blob**);
+  int (*blob_read)(sqlite3_blob*,void*,int,int);
+  int (*blob_write)(sqlite3_blob*,const void*,int,int);
+  int (*create_collation_v2)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*),void(*)(void*));
+  int (*file_control)(sqlite3*,const char*,int,void*);
+  sqlite3_int64 (*memory_highwater)(int);
+  sqlite3_int64 (*memory_used)(void);
+  sqlite3_mutex *(*mutex_alloc)(int);
+  void (*mutex_enter)(sqlite3_mutex*);
+  void (*mutex_free)(sqlite3_mutex*);
+  void (*mutex_leave)(sqlite3_mutex*);
+  int (*mutex_try)(sqlite3_mutex*);
+  int (*open_v2)(const char*,sqlite3**,int,const char*);
+  int (*release_memory)(int);
+  void (*result_error_nomem)(sqlite3_context*);
+  void (*result_error_toobig)(sqlite3_context*);
+  int (*sleep)(int);
+  void (*soft_heap_limit)(int);
+  sqlite3_vfs *(*vfs_find)(const char*);
+  int (*vfs_register)(sqlite3_vfs*,int);
+  int (*vfs_unregister)(sqlite3_vfs*);
+  int (*xthreadsafe)(void);
+  void (*result_zeroblob)(sqlite3_context*,int);
+  void (*result_error_code)(sqlite3_context*,int);
+  int (*test_control)(int, ...);
+  void (*randomness)(int,void*);
+  sqlite3 *(*context_db_handle)(sqlite3_context*);
+  int (*extended_result_codes)(sqlite3*,int);
+  int (*limit)(sqlite3*,int,int);
+  sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*);
+  const char *(*sql)(sqlite3_stmt*);
+  int (*status)(int,int*,int*,int);
 };
 
 /*
@@ -169,7 +208,9 @@ struct sqlite3_api_routines {
 */
 #ifndef SQLITE_CORE
 #define sqlite3_aggregate_context      sqlite3_api->aggregate_context
+#ifndef SQLITE_OMIT_DEPRECATED
 #define sqlite3_aggregate_count        sqlite3_api->aggregate_count
+#endif
 #define sqlite3_bind_blob              sqlite3_api->bind_blob
 #define sqlite3_bind_double            sqlite3_api->bind_double
 #define sqlite3_bind_int               sqlite3_api->bind_int
@@ -225,14 +266,18 @@ struct sqlite3_api_routines {
 #define sqlite3_errmsg                 sqlite3_api->errmsg
 #define sqlite3_errmsg16               sqlite3_api->errmsg16
 #define sqlite3_exec                   sqlite3_api->exec
+#ifndef SQLITE_OMIT_DEPRECATED
 #define sqlite3_expired                sqlite3_api->expired
+#endif
 #define sqlite3_finalize               sqlite3_api->finalize
 #define sqlite3_free                   sqlite3_api->free
 #define sqlite3_free_table             sqlite3_api->free_table
 #define sqlite3_get_autocommit         sqlite3_api->get_autocommit
 #define sqlite3_get_auxdata            sqlite3_api->get_auxdata
 #define sqlite3_get_table              sqlite3_api->get_table
+#ifndef SQLITE_OMIT_DEPRECATED
 #define sqlite3_global_recover         sqlite3_api->global_recover
+#endif
 #define sqlite3_interrupt              sqlite3_api->interruptx
 #define sqlite3_last_insert_rowid      sqlite3_api->last_insert_rowid
 #define sqlite3_libversion             sqlite3_api->libversion
@@ -270,7 +315,9 @@ struct sqlite3_api_routines {
 #define sqlite3_thread_cleanup         sqlite3_api->thread_cleanup
 #define sqlite3_total_changes          sqlite3_api->total_changes
 #define sqlite3_trace                  sqlite3_api->trace
+#ifndef SQLITE_OMIT_DEPRECATED
 #define sqlite3_transfer_bindings      sqlite3_api->transfer_bindings
+#endif
 #define sqlite3_update_hook            sqlite3_api->update_hook
 #define sqlite3_user_data              sqlite3_api->user_data
 #define sqlite3_value_blob             sqlite3_api->value_blob
@@ -290,9 +337,44 @@ struct sqlite3_api_routines {
 #define sqlite3_prepare_v2             sqlite3_api->prepare_v2
 #define sqlite3_prepare16_v2           sqlite3_api->prepare16_v2
 #define sqlite3_clear_bindings         sqlite3_api->clear_bindings
+#define sqlite3_bind_zeroblob          sqlite3_api->bind_zeroblob
+#define sqlite3_blob_bytes             sqlite3_api->blob_bytes
+#define sqlite3_blob_close             sqlite3_api->blob_close
+#define sqlite3_blob_open              sqlite3_api->blob_open
+#define sqlite3_blob_read              sqlite3_api->blob_read
+#define sqlite3_blob_write             sqlite3_api->blob_write
+#define sqlite3_create_collation_v2    sqlite3_api->create_collation_v2
+#define sqlite3_file_control           sqlite3_api->file_control
+#define sqlite3_memory_highwater       sqlite3_api->memory_highwater
+#define sqlite3_memory_used            sqlite3_api->memory_used
+#define sqlite3_mutex_alloc            sqlite3_api->mutex_alloc
+#define sqlite3_mutex_enter            sqlite3_api->mutex_enter
+#define sqlite3_mutex_free             sqlite3_api->mutex_free
+#define sqlite3_mutex_leave            sqlite3_api->mutex_leave
+#define sqlite3_mutex_try              sqlite3_api->mutex_try
+#define sqlite3_open_v2                sqlite3_api->open_v2
+#define sqlite3_release_memory         sqlite3_api->release_memory
+#define sqlite3_result_error_nomem     sqlite3_api->result_error_nomem
+#define sqlite3_result_error_toobig    sqlite3_api->result_error_toobig
+#define sqlite3_sleep                  sqlite3_api->sleep
+#define sqlite3_soft_heap_limit        sqlite3_api->soft_heap_limit
+#define sqlite3_vfs_find               sqlite3_api->vfs_find
+#define sqlite3_vfs_register           sqlite3_api->vfs_register
+#define sqlite3_vfs_unregister         sqlite3_api->vfs_unregister
+#define sqlite3_threadsafe             sqlite3_api->xthreadsafe
+#define sqlite3_result_zeroblob        sqlite3_api->result_zeroblob
+#define sqlite3_result_error_code      sqlite3_api->result_error_code
+#define sqlite3_test_control           sqlite3_api->test_control
+#define sqlite3_randomness             sqlite3_api->randomness
+#define sqlite3_context_db_handle      sqlite3_api->context_db_handle
+#define sqlite3_extended_result_codes  sqlite3_api->extended_result_codes
+#define sqlite3_limit                  sqlite3_api->limit
+#define sqlite3_next_stmt              sqlite3_api->next_stmt
+#define sqlite3_sql                    sqlite3_api->sql
+#define sqlite3_status                 sqlite3_api->status
 #endif /* SQLITE_CORE */
 
-#define SQLITE_EXTENSION_INIT1     const sqlite3_api_routines *sqlite3_api;
+#define SQLITE_EXTENSION_INIT1     const sqlite3_api_routines *sqlite3_api = 0;
 #define SQLITE_EXTENSION_INIT2(v)  sqlite3_api = v;
 
 #endif /* _SQLITE3EXT_H_ */
diff --git a/sqliteInt.h b/sqliteInt.h
index 21e5dbd..fbfa728 100644
--- a/sqliteInt.h
+++ b/sqliteInt.h
@@ -11,13 +11,167 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.585 2007/08/08 12:11:21 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.824 2009/01/14 23:03:41 drh Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
+
+/*
+** Include the configuration header output by 'configure' if we're using the
+** autoconf-based build
+*/
+#ifdef _HAVE_SQLITE_CONFIG_H
+#include "config.h"
+#endif
+
 #include "sqliteLimit.h"
 
+/* Disable nuisance warnings on Borland compilers */
+#if defined(__BORLANDC__)
+#pragma warn -rch /* unreachable code */
+#pragma warn -ccc /* Condition is always true or false */
+#pragma warn -aus /* Assigned value is never used */
+#pragma warn -csu /* Comparing signed and unsigned */
+#pragma warn -spa /* Suspicious pointer arithmetic */
+#endif
 
+/* Needed for various definitions... */
+#ifndef _GNU_SOURCE
+# define _GNU_SOURCE
+#endif
+
+/*
+** Include standard header files as necessary
+*/
+#ifdef HAVE_STDINT_H
+#include 
+#endif
+#ifdef HAVE_INTTYPES_H
+#include 
+#endif
+
+/*
+ * This macro is used to "hide" some ugliness in casting an int
+ * value to a ptr value under the MSVC 64-bit compiler.   Casting
+ * non 64-bit values to ptr types results in a "hard" error with 
+ * the MSVC 64-bit compiler which this attempts to avoid.  
+ *
+ * A simple compiler pragma or casting sequence could not be found
+ * to correct this in all situations, so this macro was introduced.
+ *
+ * It could be argued that the intptr_t type could be used in this
+ * case, but that type is not available on all compilers, or 
+ * requires the #include of specific headers which differs between
+ * platforms.
+ */
+#define SQLITE_INT_TO_PTR(X)   ((void*)&((char*)0)[X])
+#define SQLITE_PTR_TO_INT(X)   ((int)(((char*)X)-(char*)0))
+
+/*
+** These #defines should enable >2GB file support on POSIX if the
+** underlying operating system supports it.  If the OS lacks
+** large file support, or if the OS is windows, these should be no-ops.
+**
+** Ticket #2739:  The _LARGEFILE_SOURCE macro must appear before any
+** system #includes.  Hence, this block of code must be the very first
+** code in all source files.
+**
+** Large file support can be disabled using the -DSQLITE_DISABLE_LFS switch
+** on the compiler command line.  This is necessary if you are compiling
+** on a recent machine (ex: Red Hat 7.2) but you want your code to work
+** on an older machine (ex: Red Hat 6.0).  If you compile on Red Hat 7.2
+** without this option, LFS is enable.  But LFS does not exist in the kernel
+** in Red Hat 6.0, so the code won't work.  Hence, for maximum binary
+** portability you should omit LFS.
+**
+** Similar is true for Mac OS X.  LFS is only supported on Mac OS X 9 and later.
+*/
+#ifndef SQLITE_DISABLE_LFS
+# define _LARGE_FILE       1
+# ifndef _FILE_OFFSET_BITS
+#   define _FILE_OFFSET_BITS 64
+# endif
+# define _LARGEFILE_SOURCE 1
+#endif
+
+
+/*
+** The SQLITE_THREADSAFE macro must be defined as either 0 or 1.
+** Older versions of SQLite used an optional THREADSAFE macro.
+** We support that for legacy
+*/
+#if !defined(SQLITE_THREADSAFE)
+#if defined(THREADSAFE)
+# define SQLITE_THREADSAFE THREADSAFE
+#else
+# define SQLITE_THREADSAFE 1
+#endif
+#endif
+
+/*
+** The SQLITE_DEFAULT_MEMSTATUS macro must be defined as either 0 or 1.
+** It determines whether or not the features related to 
+** SQLITE_CONFIG_MEMSTATUS are available by default or not. This value can
+** be overridden at runtime using the sqlite3_config() API.
+*/
+#if !defined(SQLITE_DEFAULT_MEMSTATUS)
+# define SQLITE_DEFAULT_MEMSTATUS 1
+#endif
+
+/*
+** Exactly one of the following macros must be defined in order to
+** specify which memory allocation subsystem to use.
+**
+**     SQLITE_SYSTEM_MALLOC          // Use normal system malloc()
+**     SQLITE_MEMDEBUG               // Debugging version of system malloc()
+**     SQLITE_MEMORY_SIZE            // internal allocator #1
+**     SQLITE_MMAP_HEAP_SIZE         // internal mmap() allocator
+**     SQLITE_POW2_MEMORY_SIZE       // internal power-of-two allocator
+**
+** If none of the above are defined, then set SQLITE_SYSTEM_MALLOC as
+** the default.
+*/
+#if defined(SQLITE_SYSTEM_MALLOC)+defined(SQLITE_MEMDEBUG)+\
+    defined(SQLITE_MEMORY_SIZE)+defined(SQLITE_MMAP_HEAP_SIZE)+\
+    defined(SQLITE_POW2_MEMORY_SIZE)>1
+# error "At most one of the following compile-time configuration options\
+ is allows: SQLITE_SYSTEM_MALLOC, SQLITE_MEMDEBUG, SQLITE_MEMORY_SIZE,\
+ SQLITE_MMAP_HEAP_SIZE, SQLITE_POW2_MEMORY_SIZE"
+#endif
+#if defined(SQLITE_SYSTEM_MALLOC)+defined(SQLITE_MEMDEBUG)+\
+    defined(SQLITE_MEMORY_SIZE)+defined(SQLITE_MMAP_HEAP_SIZE)+\
+    defined(SQLITE_POW2_MEMORY_SIZE)==0
+# define SQLITE_SYSTEM_MALLOC 1
+#endif
+
+/*
+** If SQLITE_MALLOC_SOFT_LIMIT is defined, then try to keep the
+** sizes of memory allocations below this value where possible.
+*/
+#if defined(SQLITE_POW2_MEMORY_SIZE) && !defined(SQLITE_MALLOC_SOFT_LIMIT)
+# define SQLITE_MALLOC_SOFT_LIMIT 1024
+#endif
+
+/*
+** We need to define _XOPEN_SOURCE as follows in order to enable
+** recursive mutexes on most Unix systems.  But Mac OS X is different.
+** The _XOPEN_SOURCE define causes problems for Mac OS X we are told,
+** so it is omitted there.  See ticket #2673.
+**
+** Later we learn that _XOPEN_SOURCE is poorly or incorrectly
+** implemented on some systems.  So we avoid defining it at all
+** if it is already defined or if it is unneeded because we are
+** not doing a threadsafe build.  Ticket #2681.
+**
+** See also ticket #2741.
+*/
+#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) && !defined(__APPLE__) && SQLITE_THREADSAFE
+#  define _XOPEN_SOURCE 500  /* Needed to enable pthread recursive mutexes */
+#endif
+
+/*
+** The TCL headers are only needed when compiling the TCL bindings.
+*/
 #if defined(SQLITE_TCL) || defined(TCLSH)
 # include 
 #endif
@@ -34,26 +188,90 @@
 #endif
 
 /*
-** These #defines should enable >2GB file support on Posix if the
-** underlying operating system supports it.  If the OS lacks
-** large file support, or if the OS is windows, these should be no-ops.
+** The testcase() macro is used to aid in coverage testing.  When 
+** doing coverage testing, the condition inside the argument to
+** testcase() must be evaluated both true and false in order to
+** get full branch coverage.  The testcase() macro is inserted
+** to help ensure adequate test coverage in places where simple
+** condition/decision coverage is inadequate.  For example, testcase()
+** can be used to make sure boundary values are tested.  For
+** bitmask tests, testcase() can be used to make sure each bit
+** is significant and used at least once.  On switch statements
+** where multiple cases go to the same block of code, testcase()
+** can insure that all cases are evaluated.
 **
-** Large file support can be disabled using the -DSQLITE_DISABLE_LFS switch
-** on the compiler command line.  This is necessary if you are compiling
-** on a recent machine (ex: RedHat 7.2) but you want your code to work
-** on an older machine (ex: RedHat 6.0).  If you compile on RedHat 7.2
-** without this option, LFS is enable.  But LFS does not exist in the kernel
-** in RedHat 6.0, so the code won't work.  Hence, for maximum binary
-** portability you should omit LFS.
-**
-** Similar is true for MacOS.  LFS is only supported on MacOS 9 and later.
 */
-#ifndef SQLITE_DISABLE_LFS
-# define _LARGE_FILE       1
-# ifndef _FILE_OFFSET_BITS
-#   define _FILE_OFFSET_BITS 64
-# endif
-# define _LARGEFILE_SOURCE 1
+#ifdef SQLITE_COVERAGE_TEST
+  void sqlite3Coverage(int);
+# define testcase(X)  if( X ){ sqlite3Coverage(__LINE__); }
+#else
+# define testcase(X)
+#endif
+
+/*
+** The TESTONLY macro is used to enclose variable declarations or
+** other bits of code that are needed to support the arguments
+** within testcase() and assert() macros.
+*/
+#if !defined(NDEBUG) || defined(SQLITE_COVERAGE_TEST)
+# define TESTONLY(X)  X
+#else
+# define TESTONLY(X)
+#endif
+
+/*
+** The ALWAYS and NEVER macros surround boolean expressions which 
+** are intended to always be true or false, respectively.  Such
+** expressions could be omitted from the code completely.  But they
+** are included in a few cases in order to enhance the resilience
+** of SQLite to unexpected behavior - to make the code "self-healing"
+** or "ductile" rather than being "brittle" and crashing at the first
+** hint of unplanned behavior.
+**
+** In other words, ALWAYS and NEVER are added for defensive code.
+**
+** When doing coverage testing ALWAYS and NEVER are hard-coded to
+** be true and false so that the unreachable code then specify will
+** not be counted as untested code.
+*/
+#if defined(SQLITE_COVERAGE_TEST)
+# define ALWAYS(X)      (1)
+# define NEVER(X)       (0)
+#elif !defined(NDEBUG)
+  int sqlite3Assert(void);
+# define ALWAYS(X)      ((X)?1:sqlite3Assert())
+# define NEVER(X)       ((X)?sqlite3Assert():0)
+#else
+# define ALWAYS(X)      (X)
+# define NEVER(X)       (X)
+#endif
+
+/*
+** The macro unlikely() is a hint that surrounds a boolean
+** expression that is usually false.  Macro likely() surrounds
+** a boolean expression that is usually true.  GCC is able to
+** use these hints to generate better code, sometimes.
+*/
+#if defined(__GNUC__) && 0
+# define likely(X)    __builtin_expect((X),1)
+# define unlikely(X)  __builtin_expect((X),0)
+#else
+# define likely(X)    !!(X)
+# define unlikely(X)  !!(X)
+#endif
+
+/*
+** Sometimes we need a small amount of code such as a variable initialization
+** to setup for a later assert() statement.  We do not want this code to
+** appear when assert() is disabled.  The following macro is therefore
+** used to contain that setup code.  The "VVA" acronym stands for
+** "Verification, Validation, and Accreditation".  In other words, the
+** code within VVA_ONLY() will only run during verification processes.
+*/
+#ifndef NDEBUG
+# define VVA_ONLY(X)  X
+#else
+# define VVA_ONLY(X)
 #endif
 
 #include "sqlite3.h"
@@ -65,8 +283,6 @@
 #include 
 #include 
 
-#define sqlite3_isnan(X)  ((X)!=(X))
-
 /*
 ** If compiling for a processor that lacks floating point support,
 ** substitute integer for floating-point
@@ -122,11 +338,11 @@
 #endif
 
 /*
-** Provide a default value for TEMP_STORE in case it is not specified
+** Provide a default value for SQLITE_TEMP_STORE in case it is not specified
 ** on the command-line
 */
-#ifndef TEMP_STORE
-# define TEMP_STORE 1
+#ifndef SQLITE_TEMP_STORE
+# define SQLITE_TEMP_STORE 1
 #endif
 
 /*
@@ -155,19 +371,39 @@
 **         cc '-DUINTPTR_TYPE=long long int' ...
 */
 #ifndef UINT32_TYPE
-# define UINT32_TYPE unsigned int
+# ifdef HAVE_UINT32_T
+#  define UINT32_TYPE uint32_t
+# else
+#  define UINT32_TYPE unsigned int
+# endif
 #endif
 #ifndef UINT16_TYPE
-# define UINT16_TYPE unsigned short int
+# ifdef HAVE_UINT16_T
+#  define UINT16_TYPE uint16_t
+# else
+#  define UINT16_TYPE unsigned short int
+# endif
 #endif
 #ifndef INT16_TYPE
-# define INT16_TYPE short int
+# ifdef HAVE_INT16_T
+#  define INT16_TYPE int16_t
+# else
+#  define INT16_TYPE short int
+# endif
 #endif
 #ifndef UINT8_TYPE
-# define UINT8_TYPE unsigned char
+# ifdef HAVE_UINT8_T
+#  define UINT8_TYPE uint8_t
+# else
+#  define UINT8_TYPE unsigned char
+# endif
 #endif
 #ifndef INT8_TYPE
-# define INT8_TYPE signed char
+# ifdef HAVE_INT8_T
+#  define INT8_TYPE int8_t
+# else
+#  define INT8_TYPE signed char
+# endif
 #endif
 #ifndef LONGDOUBLE_TYPE
 # define LONGDOUBLE_TYPE long double
@@ -178,14 +414,19 @@ typedef UINT32_TYPE u32;           /* 4-byte unsigned integer */
 typedef UINT16_TYPE u16;           /* 2-byte unsigned integer */
 typedef INT16_TYPE i16;            /* 2-byte signed integer */
 typedef UINT8_TYPE u8;             /* 1-byte unsigned integer */
-typedef UINT8_TYPE i8;             /* 1-byte signed integer */
+typedef INT8_TYPE i8;              /* 1-byte signed integer */
 
 /*
 ** Macros to determine whether the machine is big or little endian,
 ** evaluated at runtime.
 */
+#ifdef SQLITE_AMALGAMATION
+const int sqlite3one = 1;
+#else
 extern const int sqlite3one;
-#if defined(i386) || defined(__i386__) || defined(_M_IX86)
+#endif
+#if defined(i386) || defined(__i386__) || defined(_M_IX86)\
+                             || defined(__x86_64) || defined(__x86_64__)
 # define SQLITE_BIGENDIAN    0
 # define SQLITE_LITTLEENDIAN 1
 # define SQLITE_UTF16NATIVE  SQLITE_UTF16LE
@@ -195,6 +436,14 @@ extern const int sqlite3one;
 # define SQLITE_UTF16NATIVE (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE)
 #endif
 
+/*
+** Constants for the largest and smallest possible 64-bit signed integers.
+** These macros are designed to work correctly on both 32-bit and 64-bit
+** compilers.
+*/
+#define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
+#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
+
 /*
 ** An instance of the following structure is used to store the busy-handler
 ** callback for a given sqlite handle. 
@@ -211,83 +460,6 @@ struct BusyHandler {
   int nBusy;                 /* Incremented with each busy call */
 };
 
-/*
-** Defer sourcing vdbe.h and btree.h until after the "u8" and 
-** "BusyHandler typedefs.
-*/
-#include "vdbe.h"
-#include "btree.h"
-#include "pager.h"
-
-#ifdef SQLITE_MEMDEBUG
-/*
-** The following global variables are used for testing and debugging
-** only.  They only work if SQLITE_MEMDEBUG is defined.
-*/
-extern int sqlite3_nMalloc;      /* Number of sqliteMalloc() calls */
-extern int sqlite3_nFree;        /* Number of sqliteFree() calls */
-extern int sqlite3_iMallocFail;  /* Fail sqliteMalloc() after this many calls */
-extern int sqlite3_iMallocReset; /* Set iMallocFail to this when it reaches 0 */
-
-extern void *sqlite3_pFirst;         /* Pointer to linked list of allocations */
-extern int sqlite3_nMaxAlloc;        /* High water mark of ThreadData.nAlloc */
-extern int sqlite3_mallocDisallowed; /* assert() in sqlite3Malloc() if set */
-extern int sqlite3_isFail;           /* True if all malloc calls should fail */
-extern const char *sqlite3_zFile;    /* Filename to associate debug info with */
-extern int sqlite3_iLine;            /* Line number for debug info */
-
-#define ENTER_MALLOC (sqlite3_zFile = __FILE__, sqlite3_iLine = __LINE__)
-#define sqliteMalloc(x)          (ENTER_MALLOC, sqlite3Malloc(x,1))
-#define sqliteMallocRaw(x)       (ENTER_MALLOC, sqlite3MallocRaw(x,1))
-#define sqliteRealloc(x,y)       (ENTER_MALLOC, sqlite3Realloc(x,y))
-#define sqliteStrDup(x)          (ENTER_MALLOC, sqlite3StrDup(x))
-#define sqliteStrNDup(x,y)       (ENTER_MALLOC, sqlite3StrNDup(x,y))
-#define sqliteReallocOrFree(x,y) (ENTER_MALLOC, sqlite3ReallocOrFree(x,y))
-
-#else
-
-#define ENTER_MALLOC 0
-#define sqliteMalloc(x)          sqlite3Malloc(x,1)
-#define sqliteMallocRaw(x)       sqlite3MallocRaw(x,1)
-#define sqliteRealloc(x,y)       sqlite3Realloc(x,y)
-#define sqliteStrDup(x)          sqlite3StrDup(x)
-#define sqliteStrNDup(x,y)       sqlite3StrNDup(x,y)
-#define sqliteReallocOrFree(x,y) sqlite3ReallocOrFree(x,y)
-
-#endif
-
-/* Variable sqlite3MallocHasFailed is set to true after a malloc() 
-** failure occurs. 
-**
-** The sqlite3MallocFailed() macro returns true if a malloc has failed
-** in this thread since the last call to sqlite3ApiExit(), or false 
-** otherwise.
-*/
-extern int sqlite3MallocHasFailed;
-#define sqlite3MallocFailed() (sqlite3MallocHasFailed && sqlite3OsInMutex(1))
-
-#define sqliteFree(x)          sqlite3FreeX(x)
-#define sqliteAllocSize(x)     sqlite3AllocSize(x)
-
-/*
-** An instance of this structure might be allocated to store information
-** specific to a single thread.
-*/
-struct ThreadData {
-  int dummy;               /* So that this structure is never empty */
-
-#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
-  int nSoftHeapLimit;      /* Suggested max mem allocation.  No limit if <0 */
-  int nAlloc;              /* Number of bytes currently allocated */
-  Pager *pPager;           /* Linked list of all pagers in this thread */
-#endif
-
-#ifndef SQLITE_OMIT_SHARED_CACHE
-  u8 useSharedData;        /* True if shared pagers and schemas are enabled */
-  BtShared *pBtree;        /* Linked list of all currently open BTrees */
-#endif
-};
-
 /*
 ** Name of the master database table.  The master database table
 ** is a special table that holds the names and attributes of all
@@ -310,13 +482,66 @@ struct ThreadData {
 ** A convenience macro that returns the number of elements in
 ** an array.
 */
-#define ArraySize(X)    (sizeof(X)/sizeof(X[0]))
+#define ArraySize(X)    ((int)(sizeof(X)/sizeof(X[0])))
+
+/*
+** The following value as a destructor means to use sqlite3DbFree().
+** This is an internal extension to SQLITE_STATIC and SQLITE_TRANSIENT.
+*/
+#define SQLITE_DYNAMIC   ((sqlite3_destructor_type)sqlite3DbFree)
+
+/*
+** When SQLITE_OMIT_WSD is defined, it means that the target platform does
+** not support Writable Static Data (WSD) such as global and static variables.
+** All variables must either be on the stack or dynamically allocated from
+** the heap.  When WSD is unsupported, the variable declarations scattered
+** throughout the SQLite code must become constants instead.  The SQLITE_WSD
+** macro is used for this purpose.  And instead of referencing the variable
+** directly, we use its constant as a key to lookup the run-time allocated
+** buffer that holds real variable.  The constant is also the initializer
+** for the run-time allocated buffer.
+**
+** In the usual case where WSD is supported, the SQLITE_WSD and GLOBAL
+** macros become no-ops and have zero performance impact.
+*/
+#ifdef SQLITE_OMIT_WSD
+  #define SQLITE_WSD const
+  #define GLOBAL(t,v) (*(t*)sqlite3_wsd_find((void*)&(v), sizeof(v)))
+  #define sqlite3GlobalConfig GLOBAL(struct Sqlite3Config, sqlite3Config)
+  int sqlite3_wsd_init(int N, int J);
+  void *sqlite3_wsd_find(void *K, int L);
+#else
+  #define SQLITE_WSD 
+  #define GLOBAL(t,v) v
+  #define sqlite3GlobalConfig sqlite3Config
+#endif
+
+/*
+** The following macros are used to suppress compiler warnings and to
+** make it clear to human readers when a function parameter is deliberately 
+** left unused within the body of a function. This usually happens when
+** a function is called via a function pointer. For example the 
+** implementation of an SQL aggregate step callback may not use the
+** parameter indicating the number of arguments passed to the aggregate,
+** if it knows that this is enforced elsewhere.
+**
+** When a function parameter is not used at all within the body of a function,
+** it is generally named "NotUsed" or "NotUsed2" to make things even clearer.
+** However, these macros may also be used to suppress warnings related to
+** parameters that may or may not be used depending on compilation options.
+** For example those parameters only used in assert() statements. In these
+** cases the parameters are named as per the usual conventions.
+*/
+#define UNUSED_PARAMETER(x) (void)(x)
+#define UNUSED_PARAMETER2(x,y) UNUSED_PARAMETER(x),UNUSED_PARAMETER(y)
 
 /*
 ** Forward references to structures
 */
 typedef struct AggInfo AggInfo;
 typedef struct AuthContext AuthContext;
+typedef struct Bitvec Bitvec;
+typedef struct RowSet RowSet;
 typedef struct CollSeq CollSeq;
 typedef struct Column Column;
 typedef struct Db Db;
@@ -325,26 +550,45 @@ typedef struct Expr Expr;
 typedef struct ExprList ExprList;
 typedef struct FKey FKey;
 typedef struct FuncDef FuncDef;
+typedef struct FuncDefHash FuncDefHash;
 typedef struct IdList IdList;
 typedef struct Index Index;
 typedef struct KeyClass KeyClass;
 typedef struct KeyInfo KeyInfo;
+typedef struct Lookaside Lookaside;
+typedef struct LookasideSlot LookasideSlot;
 typedef struct Module Module;
 typedef struct NameContext NameContext;
 typedef struct Parse Parse;
+typedef struct Savepoint Savepoint;
 typedef struct Select Select;
 typedef struct SrcList SrcList;
-typedef struct ThreadData ThreadData;
+typedef struct StrAccum StrAccum;
 typedef struct Table Table;
 typedef struct TableLock TableLock;
 typedef struct Token Token;
 typedef struct TriggerStack TriggerStack;
 typedef struct TriggerStep TriggerStep;
 typedef struct Trigger Trigger;
+typedef struct UnpackedRecord UnpackedRecord;
+typedef struct Walker Walker;
+typedef struct WherePlan WherePlan;
 typedef struct WhereInfo WhereInfo;
 typedef struct WhereLevel WhereLevel;
 
+/*
+** Defer sourcing vdbe.h and btree.h until after the "u8" and 
+** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque
+** pointer types (i.e. FuncDef) defined above.
+*/
+#include "btree.h"
+#include "vdbe.h"
+#include "pager.h"
+#include "pcache.h"
+
 #include "os.h"
+#include "mutex.h"
+
 
 /*
 ** Each database file to be accessed by the system is an instance
@@ -357,7 +601,7 @@ struct Db {
   char *zName;         /* Name of this database */
   Btree *pBt;          /* The B*Tree structure for this database file */
   u8 inTrans;          /* 0: not writable.  1: Transaction.  2: Checkpoint */
-  u8 safety_level;     /* How aggressive at synching data to disk */
+  u8 safety_level;     /* How aggressive at syncing data to disk */
   void *pAux;               /* Auxiliary data.  Usually NULL */
   void (*xFreeAux)(void*);  /* Routine to free pAux */
   Schema *pSchema;     /* Pointer to database schema (possibly shared) */
@@ -413,6 +657,48 @@ struct Schema {
 #define DB_UnresetViews    0x0002  /* Some views have defined column names */
 #define DB_Empty           0x0004  /* The file is empty (length 0 bytes) */
 
+/*
+** The number of different kinds of things that can be limited
+** using the sqlite3_limit() interface.
+*/
+#define SQLITE_N_LIMIT (SQLITE_LIMIT_VARIABLE_NUMBER+1)
+
+/*
+** Lookaside malloc is a set of fixed-size buffers that can be used
+** to satisfy small transient memory allocation requests for objects
+** associated with a particular database connection.  The use of
+** lookaside malloc provides a significant performance enhancement
+** (approx 10%) by avoiding numerous malloc/free requests while parsing
+** SQL statements.
+**
+** The Lookaside structure holds configuration information about the
+** lookaside malloc subsystem.  Each available memory allocation in
+** the lookaside subsystem is stored on a linked list of LookasideSlot
+** objects.
+*/
+struct Lookaside {
+  u16 sz;                 /* Size of each buffer in bytes */
+  u8 bEnabled;            /* True if use lookaside.  False to ignore it */
+  u8 bMalloced;           /* True if pStart obtained from sqlite3_malloc() */
+  int nOut;               /* Number of buffers currently checked out */
+  int mxOut;              /* Highwater mark for nOut */
+  LookasideSlot *pFree;   /* List of available buffers */
+  void *pStart;           /* First byte of available memory space */
+  void *pEnd;             /* First byte past end of available space */
+};
+struct LookasideSlot {
+  LookasideSlot *pNext;    /* Next buffer in the list of free buffers */
+};
+
+/*
+** A hash table for function definitions.
+**
+** Hash each FuncDef structure into one of the FuncDefHash.a[] slots.
+** Collisions are on the FuncDef.pHash chain.
+*/
+struct FuncDefHash {
+  FuncDef *a[23];       /* Hash table for functions */
+};
 
 /*
 ** Each database is an instance of the following structure.
@@ -441,29 +727,39 @@ struct Schema {
 ** consistently.
 */
 struct sqlite3 {
+  sqlite3_vfs *pVfs;            /* OS Interface */
   int nDb;                      /* Number of backends currently in use */
   Db *aDb;                      /* All backends */
-  int flags;                    /* Miscellanous flags. See below */
+  int flags;                    /* Miscellaneous flags. See below */
+  int openFlags;                /* Flags passed to sqlite3_vfs.xOpen() */
   int errCode;                  /* Most recent error code (SQLITE_*) */
   int errMask;                  /* & result codes with this before returning */
   u8 autoCommit;                /* The auto-commit flag. */
   u8 temp_store;                /* 1: file 2: memory 0: default */
+  u8 mallocFailed;              /* True if we have seen a malloc failure */
+  u8 dfltLockMode;              /* Default locking-mode for attached dbs */
+  u8 dfltJournalMode;           /* Default journal mode for attached dbs */
+  signed char nextAutovac;      /* Autovac setting after VACUUM if >=0 */
+  int nextPagesize;             /* Pagesize after VACUUM if >0 */
   int nTable;                   /* Number of tables in the database */
   CollSeq *pDfltColl;           /* The default collating sequence (BINARY) */
   i64 lastRowid;                /* ROWID of most recent insert (see above) */
   i64 priorNewRowid;            /* Last randomly generated ROWID */
-  int magic;                    /* Magic number for detect library misuse */
+  u32 magic;                    /* Magic number for detect library misuse */
   int nChange;                  /* Value returned by sqlite3_changes() */
   int nTotalChange;             /* Value returned by sqlite3_total_changes() */
+  sqlite3_mutex *mutex;         /* Connection mutex */
+  int aLimit[SQLITE_N_LIMIT];   /* Limits */
   struct sqlite3InitInfo {      /* Information used during initialization */
     int iDb;                    /* When back is being initialized */
     int newTnum;                /* Rootpage of table being initialized */
     u8 busy;                    /* TRUE if currently initializing */
   } init;
   int nExtension;               /* Number of loaded extensions */
-  void **aExtension;            /* Array of shared libraray handles */
+  void **aExtension;            /* Array of shared library handles */
   struct Vdbe *pVdbe;           /* List of active virtual machines */
-  int activeVdbeCnt;            /* Number of vdbes currently executing */
+  int activeVdbeCnt;            /* Number of VDBEs currently executing */
+  int writeVdbeCnt;             /* Number of active VDBEs that are writing */
   void (*xTrace)(void*,const char*);        /* Trace function */
   void *pTraceArg;                          /* Argument to the trace function */
   void (*xProfile)(void*,const char*,u64);  /* Profiling function */
@@ -481,9 +777,10 @@ struct sqlite3 {
   char *zErrMsg;                /* Most recent error message (UTF-8 encoded) */
   char *zErrMsg16;              /* Most recent error message (UTF-16 encoded) */
   union {
-    int isInterrupted;          /* True if sqlite3_interrupt has been called */
+    volatile int isInterrupted; /* True if sqlite3_interrupt has been called */
     double notUsed1;            /* Spacer */
   } u1;
+  Lookaside lookaside;          /* Lookaside malloc configuration */
 #ifndef SQLITE_OMIT_AUTHORIZATION
   int (*xAuth)(void*,int,const char*,const char*,const char*,const char*);
                                 /* Access authorization function */
@@ -500,7 +797,7 @@ struct sqlite3 {
   sqlite3_vtab **aVTrans;       /* Virtual tables with open transactions */
   int nVTrans;                  /* Allocated size of aVTrans */
 #endif
-  Hash aFunc;                   /* All functions that can be in SQL exprs */
+  FuncDefHash aFunc;            /* Hash table of connection functions */
   Hash aCollSeq;                /* All collating sequences */
   BusyHandler busyHandler;      /* Busy callback */
   int busyTimeout;              /* Busy handler timeout, in msec */
@@ -508,7 +805,9 @@ struct sqlite3 {
 #ifdef SQLITE_SSE
   sqlite3_stmt *pFetch;         /* Used by SSE to fetch stored statements */
 #endif
-  u8 dfltLockMode;              /* Default locking-mode for attached dbs */
+  Savepoint *pSavepoint;        /* List of active savepoints */
+  int nSavepoint;               /* Number of non-transaction savepoints */
+  u8 isTransactionSavepoint;    /* True if the outermost savepoint is a TS */
 };
 
 /*
@@ -545,6 +844,9 @@ struct sqlite3 {
 #define SQLITE_LoadExtension  0x00020000  /* Enable load_extension */
 
 #define SQLITE_RecoveryMode   0x00040000  /* Ignore schema errors */
+#define SQLITE_SharedCache    0x00080000  /* Cache sharing is enabled */
+#define SQLITE_Vtab           0x00100000  /* There exists a virtual table */
+#define SQLITE_CommitBusy     0x00200000  /* In the process of committing */
 
 /*
 ** Possible values for the sqlite.magic field.
@@ -553,6 +855,7 @@ struct sqlite3 {
 */
 #define SQLITE_MAGIC_OPEN     0xa029a697  /* Database is open */
 #define SQLITE_MAGIC_CLOSED   0x9f3c2d33  /* Database is closed */
+#define SQLITE_MAGIC_SICK     0x4b771290  /* Error and awaiting close */
 #define SQLITE_MAGIC_BUSY     0xf03b7906  /* Database currently in use */
 #define SQLITE_MAGIC_ERROR    0xb5357930  /* An SQLITE_MISUSE error occurred */
 
@@ -565,16 +868,79 @@ struct sqlite3 {
 struct FuncDef {
   i16 nArg;            /* Number of arguments.  -1 means unlimited */
   u8 iPrefEnc;         /* Preferred text encoding (SQLITE_UTF8, 16LE, 16BE) */
-  u8 needCollSeq;      /* True if sqlite3GetFuncCollSeq() might be called */
   u8 flags;            /* Some combination of SQLITE_FUNC_* */
   void *pUserData;     /* User data parameter */
   FuncDef *pNext;      /* Next function with same name */
   void (*xFunc)(sqlite3_context*,int,sqlite3_value**); /* Regular function */
   void (*xStep)(sqlite3_context*,int,sqlite3_value**); /* Aggregate step */
-  void (*xFinalize)(sqlite3_context*);                /* Aggregate finializer */
-  char zName[1];       /* SQL name of the function.  MUST BE LAST */
+  void (*xFinalize)(sqlite3_context*);                /* Aggregate finalizer */
+  char *zName;         /* SQL name of the function. */
+  FuncDef *pHash;      /* Next with a different name but the same hash */
 };
 
+/*
+** Possible values for FuncDef.flags
+*/
+#define SQLITE_FUNC_LIKE     0x01 /* Candidate for the LIKE optimization */
+#define SQLITE_FUNC_CASE     0x02 /* Case-sensitive LIKE-type function */
+#define SQLITE_FUNC_EPHEM    0x04 /* Ephemeral.  Delete with VDBE */
+#define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */
+#define SQLITE_FUNC_PRIVATE  0x10 /* Allowed for internal use only */
+
+/*
+** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
+** used to create the initializers for the FuncDef structures.
+**
+**   FUNCTION(zName, nArg, iArg, bNC, xFunc)
+**     Used to create a scalar function definition of a function zName 
+**     implemented by C function xFunc that accepts nArg arguments. The
+**     value passed as iArg is cast to a (void*) and made available
+**     as the user-data (sqlite3_user_data()) for the function. If 
+**     argument bNC is true, then the FuncDef.needCollate flag is set.
+**
+**   AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal)
+**     Used to create an aggregate function definition implemented by
+**     the C functions xStep and xFinal. The first four parameters
+**     are interpreted in the same way as the first 4 parameters to
+**     FUNCTION().
+**
+**   LIKEFUNC(zName, nArg, pArg, flags)
+**     Used to create a scalar function definition of a function zName 
+**     that accepts nArg arguments and is implemented by a call to C 
+**     function likeFunc. Argument pArg is cast to a (void *) and made
+**     available as the function user-data (sqlite3_user_data()). The
+**     FuncDef.flags variable is set to the value passed as the flags
+**     parameter.
+*/
+#define FUNCTION(zName, nArg, iArg, bNC, xFunc) \
+  {nArg, SQLITE_UTF8, bNC*8, SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0}
+#define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \
+  {nArg, SQLITE_UTF8, bNC*8, pArg, 0, xFunc, 0, 0, #zName, 0}
+#define LIKEFUNC(zName, nArg, arg, flags) \
+  {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0}
+#define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \
+  {nArg, SQLITE_UTF8, nc*8, SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0}
+
+/*
+** All current savepoints are stored in a linked list starting at
+** sqlite3.pSavepoint. The first element in the list is the most recently
+** opened savepoint. Savepoints are added to the list by the vdbe
+** OP_Savepoint instruction.
+*/
+struct Savepoint {
+  char *zName;                        /* Savepoint name (nul-terminated) */
+  Savepoint *pNext;                   /* Parent savepoint (if any) */
+};
+
+/*
+** The following are used as the second parameter to sqlite3Savepoint(),
+** and as the P1 argument to the OP_Savepoint instruction.
+*/
+#define SAVEPOINT_BEGIN      0
+#define SAVEPOINT_RELEASE    1
+#define SAVEPOINT_ROLLBACK   2
+
+
 /*
 ** Each SQLite module (virtual table definition) is defined by an
 ** instance of the following structure, stored in the sqlite3.aModule
@@ -587,13 +953,6 @@ struct Module {
   void (*xDestroy)(void *);            /* Module destructor function */
 };
 
-/*
-** Possible values for FuncDef.flags
-*/
-#define SQLITE_FUNC_LIKE   0x01  /* Candidate for the LIKE optimization */
-#define SQLITE_FUNC_CASE   0x02  /* Case-sensitive LIKE-type function */
-#define SQLITE_FUNC_EPHEM  0x04  /* Ephermeral.  Delete with VDBE */
-
 /*
 ** information about each column of an SQL table is held in an instance
 ** of this structure.
@@ -616,7 +975,7 @@ struct Column {
 ** structure. Conceptually, a collating sequence consists of a name and
 ** a comparison routine that defines the order of that sequence.
 **
-** There may two seperate implementations of the collation function, one
+** There may two separate implementations of the collation function, one
 ** that processes text in UTF-8 encoding (CollSeq.xCmp) and another that
 ** processes text encoded in UTF-16 (CollSeq.xCmp16), using the machine
 ** native byte order. When a collation sequence is invoked, SQLite selects
@@ -642,7 +1001,7 @@ struct CollSeq {
 };
 
 /*
-** Allowed values of CollSeq flags:
+** Allowed values of CollSeq.type:
 */
 #define SQLITE_COLL_BINARY  1  /* The default memcmp() collating sequence */
 #define SQLITE_COLL_NOCASE  2  /* The built-in NOCASE collating sequence */
@@ -660,11 +1019,11 @@ struct CollSeq {
 **
 ** These used to have mnemonic name like 'i' for SQLITE_AFF_INTEGER and
 ** 't' for SQLITE_AFF_TEXT.  But we can save a little space and improve
-** the speed a little by number the values consecutively.  
+** the speed a little by numbering the values consecutively.  
 **
 ** But rather than start with 0 or 1, we begin with 'a'.  That way,
 ** when multiple affinity types are concatenated into a string and
-** used as the P3 operand, they will be more readable.
+** used as the P4 operand, they will be more readable.
 **
 ** Note also that the numeric types are grouped together so that testing
 ** for a numeric type is a single comparison.
@@ -677,6 +1036,19 @@ struct CollSeq {
 
 #define sqlite3IsNumericAffinity(X)  ((X)>=SQLITE_AFF_NUMERIC)
 
+/*
+** The SQLITE_AFF_MASK values masks off the significant bits of an
+** affinity value. 
+*/
+#define SQLITE_AFF_MASK     0x67
+
+/*
+** Additional bit values that can be ORed with an affinity without
+** changing the affinity.
+*/
+#define SQLITE_JUMPIFNULL   0x08  /* jumps if either operand is NULL */
+#define SQLITE_STOREP2      0x10  /* Store result in reg[P2] rather than jump */
+
 /*
 ** Each SQL table is represented in memory by an instance of the
 ** following structure.
@@ -693,14 +1065,14 @@ struct CollSeq {
 ** that the datatype of the PRIMARY KEY must be INTEGER for this field to
 ** be set.  An INTEGER PRIMARY KEY is used as the rowid for each row of
 ** the table.  If a table has no INTEGER PRIMARY KEY, then a random rowid
-** is generated for each row of the table.  Table.hasPrimKey is true if
+** is generated for each row of the table.  TF_HasPrimaryKey is set if
 ** the table has any PRIMARY KEY, INTEGER or otherwise.
 **
 ** Table.tnum is the page number for the root BTree page of the table in the
 ** database file.  If Table.iDb is the index of the database table backend
 ** in sqlite.aDb[].  0 is for the main database and 1 is for the file that
-** holds temporary tables and indices.  If Table.isEphem
-** is true, then the table is stored in a file that is automatically deleted
+** holds temporary tables and indices.  If TF_Ephemeral is set
+** then the table is stored in a file that is automatically deleted
 ** when the VDBE cursor to the table is closed.  In this case Table.tnum 
 ** refers VDBE cursor number that holds the table open, not to the root
 ** page number.  Transient tables are used to hold the results of a
@@ -708,46 +1080,55 @@ struct CollSeq {
 ** of a SELECT statement.
 */
 struct Table {
-  char *zName;     /* Name of the table */
-  int nCol;        /* Number of columns in this table */
-  Column *aCol;    /* Information about each column */
-  int iPKey;       /* If not less then 0, use aCol[iPKey] as the primary key */
-  Index *pIndex;   /* List of SQL indexes on this table. */
-  int tnum;        /* Root BTree node for this table (see note above) */
-  Select *pSelect; /* NULL for tables.  Points to definition if a view. */
-  int nRef;          /* Number of pointers to this Table */
-  Trigger *pTrigger; /* List of SQL triggers on this table */
-  FKey *pFKey;       /* Linked list of all foreign keys in this table */
-  char *zColAff;     /* String defining the affinity of each column */
+  sqlite3 *db;         /* Associated database connection.  Might be NULL. */
+  char *zName;         /* Name of the table or view */
+  int iPKey;           /* If not negative, use aCol[iPKey] as the primary key */
+  int nCol;            /* Number of columns in this table */
+  Column *aCol;        /* Information about each column */
+  Index *pIndex;       /* List of SQL indexes on this table. */
+  int tnum;            /* Root BTree node for this table (see note above) */
+  Select *pSelect;     /* NULL for tables.  Points to definition if a view. */
+  u16 nRef;            /* Number of pointers to this Table */
+  u8 tabFlags;         /* Mask of TF_* values */
+  u8 keyConf;          /* What to do in case of uniqueness conflict on iPKey */
+  Trigger *pTrigger;   /* List of SQL triggers on this table */
+  FKey *pFKey;         /* Linked list of all foreign keys in this table */
+  char *zColAff;       /* String defining the affinity of each column */
 #ifndef SQLITE_OMIT_CHECK
-  Expr *pCheck;      /* The AND of all CHECK constraints */
+  Expr *pCheck;        /* The AND of all CHECK constraints */
 #endif
 #ifndef SQLITE_OMIT_ALTERTABLE
-  int addColOffset;  /* Offset in CREATE TABLE statement to add a new column */
+  int addColOffset;    /* Offset in CREATE TABLE stmt to add a new column */
 #endif
-  u8 readOnly;     /* True if this table should not be written by the user */
-  u8 isEphem;      /* True if created using OP_OpenEphermeral */
-  u8 hasPrimKey;   /* True if there exists a primary key */
-  u8 keyConf;      /* What to do in case of uniqueness conflict on iPKey */
-  u8 autoInc;      /* True if the integer primary key is autoincrement */
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-  u8 isVirtual;             /* True if this is a virtual table */
-  u8 isCommit;              /* True once the CREATE TABLE has been committed */
-  Module *pMod;             /* Pointer to the implementation of the module */
-  sqlite3_vtab *pVtab;      /* Pointer to the module instance */
-  int nModuleArg;           /* Number of arguments to the module */
-  char **azModuleArg;       /* Text of all module args. [0] is module name */
+  Module *pMod;        /* Pointer to the implementation of the module */
+  sqlite3_vtab *pVtab; /* Pointer to the module instance */
+  int nModuleArg;      /* Number of arguments to the module */
+  char **azModuleArg;  /* Text of all module args. [0] is module name */
 #endif
-  Schema *pSchema;
+  Schema *pSchema;     /* Schema that contains this table */
+  Table *pNextZombie;  /* Next on the Parse.pZombieTab list */
 };
 
+/*
+** Allowed values for Tabe.tabFlags.
+*/
+#define TF_Readonly        0x01    /* Read-only system table */
+#define TF_Ephemeral       0x02    /* An ephemeral table */
+#define TF_HasPrimaryKey   0x04    /* Table has a primary key */
+#define TF_Autoincrement   0x08    /* Integer primary key is autoincrement */
+#define TF_Virtual         0x10    /* Is a virtual table */
+#define TF_NeedMetadata    0x20    /* aCol[].zType and aCol[].pColl missing */
+
+
+
 /*
 ** Test to see whether or not a table is a virtual table.  This is
 ** done as a macro so that it will be optimized out when virtual
 ** table support is omitted from the build.
 */
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-#  define IsVirtual(X)      ((X)->isVirtual)
+#  define IsVirtual(X)      (((X)->tabFlags & TF_Virtual)!=0)
 #  define IsHiddenColumn(X) ((X)->isHidden)
 #else
 #  define IsVirtual(X)      0
@@ -771,7 +1152,7 @@ struct Table {
 **
 ** Each REFERENCES clause generates an instance of the following structure
 ** which is attached to the from-table.  The to-table need not exist when
-** the from-table is created.  The existance of the to-table is not checked
+** the from-table is created.  The existence of the to-table is not checked
 ** until an attempt is made to insert data into the from-table.
 **
 ** The sqlite.aFKey hash table stores pointers to this structure
@@ -780,7 +1161,7 @@ struct Table {
 ** field.
 */
 struct FKey {
-  Table *pFrom;     /* The table that constains the REFERENCES clause */
+  Table *pFrom;     /* The table that contains the REFERENCES clause */
   FKey *pNextFrom;  /* Next foreign key in pFrom */
   char *zTo;        /* Name of table that the key points to */
   FKey *pNextTo;    /* Next foreign key that points to zTo */
@@ -796,7 +1177,7 @@ struct FKey {
 };
 
 /*
-** SQLite supports many different ways to resolve a contraint
+** SQLite supports many different ways to resolve a constraint
 ** error.  ROLLBACK processing means that a constraint violation
 ** causes the operation in process to fail and for the current transaction
 ** to be rolled back.  ABORT processing means the operation in process
@@ -839,19 +1220,45 @@ struct FKey {
 ** An instance of the following structure is passed as the first
 ** argument to sqlite3VdbeKeyCompare and is used to control the 
 ** comparison of the two index keys.
-**
-** If the KeyInfo.incrKey value is true and the comparison would
-** otherwise be equal, then return a result as if the second key
-** were larger.
 */
 struct KeyInfo {
+  sqlite3 *db;        /* The database connection */
   u8 enc;             /* Text encoding - one of the TEXT_Utf* values */
-  u8 incrKey;         /* Increase 2nd key by epsilon before comparison */
-  int nField;         /* Number of entries in aColl[] */
+  u16 nField;         /* Number of entries in aColl[] */
   u8 *aSortOrder;     /* If defined an aSortOrder[i] is true, sort DESC */
   CollSeq *aColl[1];  /* Collating sequence for each term of the key */
 };
 
+/*
+** An instance of the following structure holds information about a
+** single index record that has already been parsed out into individual
+** values.
+**
+** A record is an object that contains one or more fields of data.
+** Records are used to store the content of a table row and to store
+** the key of an index.  A blob encoding of a record is created by
+** the OP_MakeRecord opcode of the VDBE and is disassembled by the
+** OP_Column opcode.
+**
+** This structure holds a record that has already been disassembled
+** into its constituent fields.
+*/
+struct UnpackedRecord {
+  KeyInfo *pKeyInfo;  /* Collation and sort-order information */
+  u16 nField;         /* Number of entries in apMem[] */
+  u16 flags;          /* Boolean settings.  UNPACKED_... below */
+  Mem *aMem;          /* Values */
+};
+
+/*
+** Allowed values of UnpackedRecord.flags
+*/
+#define UNPACKED_NEED_FREE     0x0001  /* Memory is from sqlite3Malloc() */
+#define UNPACKED_NEED_DESTROY  0x0002  /* apMem[]s should all be destroyed */
+#define UNPACKED_IGNORE_ROWID  0x0004  /* Ignore trailing rowid on key1 */
+#define UNPACKED_INCRKEY       0x0008  /* Make this key an epsilon larger */
+#define UNPACKED_PREFIX_MATCH  0x0010  /* A prefix match is considered OK */
+
 /*
 ** Each SQL index is represented in memory by an
 ** instance of the following structure.
@@ -899,7 +1306,7 @@ struct Index {
 ** this structure.  Tokens are also used as part of an expression.
 **
 ** Note if Token.z==0 then Token.dyn and Token.n are undefined and
-** may contain random values.  Do not make any assuptions about Token.dyn
+** may contain random values.  Do not make any assumptions about Token.dyn
 ** and Token.n when Token.z==0.
 */
 struct Token {
@@ -946,7 +1353,7 @@ struct AggInfo {
     Expr *pExpr;             /* Expression encoding the function */
     FuncDef *pFunc;          /* The aggregate function implementation */
     int iMem;                /* Memory location that acts as accumulator */
-    int iDistinct;           /* Ephermeral table used to enforce DISTINCT */
+    int iDistinct;           /* Ephemeral table used to enforce DISTINCT */
   } *aFunc;
   int nFunc;              /* Number of entries in aFunc[] */
   int nFuncAlloc;         /* Number of slots allocated for aFunc[] */
@@ -1015,8 +1422,7 @@ struct Expr {
   int iRightJoinTable;   /* If EP_FromJoin, the right table of the join */
   Select *pSelect;       /* When the expression is a sub-select.  Also the
                          ** right side of " IN (