From f04f766d711e54f774f79305302364738735e054 Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Wed, 4 Sep 2013 12:05:29 +0900 Subject: [PATCH 1/8] TYPE statement attribute should be integer (RT#46873) --- dbdimp.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dbdimp.c b/dbdimp.c index 219a597..38e974a 100644 --- a/dbdimp.c +++ b/dbdimp.c @@ -1358,15 +1358,10 @@ sqlite_st_FETCH_attrib(SV *sth, imp_sth_t *imp_sth, SV *keysv) av_extend(av, i); retsv = sv_2mortal(newRV_noinc((SV*)av)); for (n = 0; n < i; n++) { - const char *fieldtype = sqlite3_column_decltype(imp_sth->stmt, n); int type = sqlite3_column_type(imp_sth->stmt, n); /* warn("got type: %d = %s\n", type, fieldtype); */ type = sqlite_type_to_odbc_type(type); - /* av_store(av, n, newSViv(type)); */ - if (fieldtype) - av_store(av, n, newSVpv(fieldtype, 0)); - else - av_store(av, n, newSVpv("VARCHAR", 0)); + av_store(av, n, newSViv(type)); } } else if (strEQ(key, "NULLABLE")) { From 90cf41a773cc3ec951718730ce391e2f5893e1b6 Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Wed, 4 Sep 2013 12:09:09 +0900 Subject: [PATCH 2/8] removed a type test in rt_40594_nullable.t --- t/rt_40594_nullable.t | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/t/rt_40594_nullable.t b/t/rt_40594_nullable.t index f7fadcc..d92296e 100644 --- a/t/rt_40594_nullable.t +++ b/t/rt_40594_nullable.t @@ -17,7 +17,7 @@ BEGIN { } } -plan tests => 7; +plan tests => 6; my $dbh = connect_ok(); @@ -28,7 +28,6 @@ ok $sth->execute; my $expected = { NUM_OF_FIELDS => 4, NAME_lc => [qw/id col1 col2 col3/], - TYPE => [qw/INTEGER varchar(2) varchar(2) char(2)/], NULLABLE => [qw/0 0 1 0/], }; From 09288710c58f7c515b4d509c72b25286e535c2ff Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Wed, 4 Sep 2013 12:10:18 +0900 Subject: [PATCH 3/8] these are not TODO any more --- t/27_metadata.t | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/t/27_metadata.t b/t/27_metadata.t index 1d1517b..13833c1 100644 --- a/t/27_metadata.t +++ b/t/27_metadata.t @@ -49,12 +49,9 @@ my $types = $sth->{TYPE}; my $names = $sth->{NAME}; # diag "Types: @$types\nNames: @$names"; is scalar @$types, scalar @$names, '$sth->{TYPE} array is same length as $sth->{NAME} array'; -# FIXME: This is wrong! $sth->{TYPE} should return an array of integers see: rt #46873 -TODO: { - local $TODO = '$sth->{TYPE} should return an array of integers.'; - isnt $types->[0], 'VARCHAR(2)', '$sth->{TYPE}[0] doesn\'t return a string'; - isnt $types->[1], 'CHAR(1)', '$sth->{TYPE}[1] doesn\'t return a string'; - like $types->[0], qr/^-?\d+$/, '$sth->{TYPE}[0] returns an integer'; - like $types->[1], qr/^-?\d+$/, '$sth->{TYPE}[1] returns an integer'; -} +# $sth->{TYPE} should return an array of integers see: rt #46873 +isnt $types->[0], 'VARCHAR(2)', '$sth->{TYPE}[0] doesn\'t return a string'; +isnt $types->[1], 'CHAR(1)', '$sth->{TYPE}[1] doesn\'t return a string'; +like $types->[0], qr/^-?\d+$/, '$sth->{TYPE}[0] returns an integer'; +like $types->[1], qr/^-?\d+$/, '$sth->{TYPE}[1] returns an integer'; From 6a86e54992eaaa9ebbff1b390fd9248816405812 Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Wed, 4 Sep 2013 16:05:32 +0900 Subject: [PATCH 4/8] introduced sqlite_prefer_numeric_type handle attribute --- dbdimp.c | 24 ++++++++++++++++++++---- dbdimp.h | 1 + t/27_metadata.t | 3 +++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/dbdimp.c b/dbdimp.c index 38e974a..37dc294 100644 --- a/dbdimp.c +++ b/dbdimp.c @@ -455,6 +455,7 @@ sqlite_db_login6(SV *dbh, imp_dbh_t *imp_dbh, char *dbname, char *user, char *pa imp_dbh->extended_result_codes = extended; imp_dbh->stmt_list = NULL; imp_dbh->began_transaction = FALSE; + imp_dbh->prefer_numeric_type = FALSE; sqlite3_busy_timeout(imp_dbh->db, SQL_TIMEOUT); @@ -737,6 +738,10 @@ sqlite_db_STORE_attrib(SV *dbh, imp_dbh_t *imp_dbh, SV *keysv, SV *valuesv) sqlite3_extended_result_codes(imp_dbh->db, imp_dbh->extended_result_codes); return TRUE; } + if (strEQ(key, "sqlite_prefer_numeric_type")) { + imp_dbh->prefer_numeric_type = !(! SvTRUE(valuesv)); + return TRUE; + } if (strEQ(key, "sqlite_unicode")) { #if PERL_UNICODE_DOES_NOT_WORK_WELL sqlite_trace(dbh, imp_dbh, 3, form("Unicode support is disabled for this version of perl.")); @@ -781,6 +786,9 @@ sqlite_db_FETCH_attrib(SV *dbh, imp_dbh_t *imp_dbh, SV *keysv) if (strEQ(key, "sqlite_extended_result_codes")) { return sv_2mortal(newSViv(imp_dbh->extended_result_codes ? 1 : 0)); } + if (strEQ(key, "sqlite_prefer_numeric_type")) { + return sv_2mortal(newSViv(imp_dbh->prefer_numeric_type ? 1 : 0)); + } if (strEQ(key, "sqlite_unicode")) { #if PERL_UNICODE_DOES_NOT_WORK_WELL sqlite_trace(dbh, imp_dbh, 3, "Unicode support is disabled for this version of perl."); @@ -1358,10 +1366,18 @@ sqlite_st_FETCH_attrib(SV *sth, imp_sth_t *imp_sth, SV *keysv) av_extend(av, i); retsv = sv_2mortal(newRV_noinc((SV*)av)); for (n = 0; n < i; n++) { - int type = sqlite3_column_type(imp_sth->stmt, n); - /* warn("got type: %d = %s\n", type, fieldtype); */ - type = sqlite_type_to_odbc_type(type); - av_store(av, n, newSViv(type)); + if (imp_dbh->prefer_numeric_type) { + int type = sqlite3_column_type(imp_sth->stmt, n); + /* warn("got type: %d = %s\n", type, fieldtype); */ + type = sqlite_type_to_odbc_type(type); + av_store(av, n, newSViv(type)); + } else { + const char *fieldtype = sqlite3_column_decltype(imp_sth->stmt, n); + if (fieldtype) + av_store(av, n, newSVpv(fieldtype, 0)); + else + av_store(av, n, newSVpv("VARCHAR", 0)); + } } } else if (strEQ(key, "NULLABLE")) { diff --git a/dbdimp.h b/dbdimp.h index b86f1c0..efe6828 100644 --- a/dbdimp.h +++ b/dbdimp.h @@ -53,6 +53,7 @@ struct imp_dbh_st { int extended_result_codes; stmt_list_s * stmt_list; bool began_transaction; + bool prefer_numeric_type; }; /* Statement Handle */ diff --git a/t/27_metadata.t b/t/27_metadata.t index 13833c1..3cd4a20 100644 --- a/t/27_metadata.t +++ b/t/27_metadata.t @@ -45,6 +45,9 @@ $dbh->do("INSERT INTO meta4 VALUES ('xyz', 'b')"); $sth = $dbh->prepare('SELECT * FROM meta4'); $sth->execute; $sth->fetch; + +$dbh->{sqlite_prefer_numeric_type} = 1; + my $types = $sth->{TYPE}; my $names = $sth->{NAME}; # diag "Types: @$types\nNames: @$names"; From 00dfbbaad688f82fd5dc373f9df9a35c52d18a5f Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Wed, 4 Sep 2013 16:22:41 +0900 Subject: [PATCH 5/8] implemented type_info_all --- lib/DBD/SQLite.pm | 101 +++++++++++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 38 deletions(-) diff --git a/lib/DBD/SQLite.pm b/lib/DBD/SQLite.pm index f676b2f..bd90819 100644 --- a/lib/DBD/SQLite.pm +++ b/lib/DBD/SQLite.pm @@ -191,6 +191,8 @@ sub regexp { package # hide from PAUSE DBD::SQLite::db; +use DBI qw/:sql_types/; + sub prepare { my $dbh = shift; my $sql = shift; @@ -773,45 +775,68 @@ sub statistics_info { return $sponge_sth; } +my @TypeInfoKeys = qw/ + TYPE_NAME + DATA_TYPE + COLUMN_SIZE + LITERAL_PREFIX + LITERAL_SUFFIX + CREATE_PARAMS + NULLABLE + CASE_SENSITIVE + SEARCHABLE + UNSIGNED_ATTRIBUTE + FIXED_PREC_SCALE + AUTO_UNIQUE_VALUE + LOCAL_TYPE_NAME + MINIMUM_SCALE + MAXIMUM_SCALE + SQL_DATA_TYPE + SQL_DATETIME_SUB + NUM_PREC_RADIX + INTERVAL_PRECISION +/; + +my %TypeInfo = ( + SQL_INTEGER ,=> { + TYPE_NAME => 'INTEGER', + DATA_TYPE => SQL_INTEGER, + NULLABLE => 2, # no for integer primary key, otherwise yes + SEARCHABLE => 3, + }, + SQL_DOUBLE ,=> { + TYPE_NAME => 'REAL', + DATA_TYPE => SQL_DOUBLE, + NULLABLE => 1, + SEARCHABLE => 3, + }, + SQL_VARCHAR ,=> { + TYPE_NAME => 'TEXT', + DATA_TYPE => SQL_VARCHAR, + LITERAL_PREFIX => "'", + LITERAL_SUFFIX => "'", + NULLABLE => 1, + SEARCHABLE => 3, + }, + SQL_BLOB ,=> { + TYPE_NAME => 'BLOB', + DATA_TYPE => SQL_BLOB, + NULLABLE => 1, + SEARCHABLE => 3, + }, + SQL_UNKNOWN_TYPE ,=> { + DATA_TYPE => SQL_UNKNOWN_TYPE, + }, +); + sub type_info_all { - return; # XXX code just copied from DBD::Oracle, not yet thought about -# return [ -# { -# TYPE_NAME => 0, -# DATA_TYPE => 1, -# COLUMN_SIZE => 2, -# LITERAL_PREFIX => 3, -# LITERAL_SUFFIX => 4, -# CREATE_PARAMS => 5, -# NULLABLE => 6, -# CASE_SENSITIVE => 7, -# SEARCHABLE => 8, -# UNSIGNED_ATTRIBUTE => 9, -# FIXED_PREC_SCALE => 10, -# AUTO_UNIQUE_VALUE => 11, -# LOCAL_TYPE_NAME => 12, -# MINIMUM_SCALE => 13, -# MAXIMUM_SCALE => 14, -# SQL_DATA_TYPE => 15, -# SQL_DATETIME_SUB => 16, -# NUM_PREC_RADIX => 17, -# }, -# [ 'CHAR', 1, 255, '\'', '\'', 'max length', 1, 1, 3, -# undef, '0', '0', undef, undef, undef, 1, undef, undef -# ], -# [ 'NUMBER', 3, 38, undef, undef, 'precision,scale', 1, '0', 3, -# '0', '0', '0', undef, '0', 38, 3, undef, 10 -# ], -# [ 'DOUBLE', 8, 15, undef, undef, undef, 1, '0', 3, -# '0', '0', '0', undef, undef, undef, 8, undef, 10 -# ], -# [ 'DATE', 9, 19, '\'', '\'', undef, 1, '0', 3, -# undef, '0', '0', undef, '0', '0', 11, undef, undef -# ], -# [ 'VARCHAR', 12, 1024*1024, '\'', '\'', 'max length', 1, 1, 3, -# undef, '0', '0', undef, undef, undef, 12, undef, undef -# ] -# ]; + my $idx = 0; + + my @info = ({map {$_ => $idx++} @TypeInfoKeys}); + for my $id (sort {$a <=> $b} keys %TypeInfo) { + push @info, [map {$TypeInfo{$id}{$_}} @TypeInfoKeys]; + } + return \@info; } my @COLUMN_INFO = qw( From 1aafac23823142dbe49a887bddd3cbb95c4ad47e Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Sat, 1 Dec 2018 17:42:57 +0900 Subject: [PATCH 6/8] added a note on sqlite_prefer_numeric_type --- lib/DBD/SQLite.pm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/DBD/SQLite.pm b/lib/DBD/SQLite.pm index bd90819..be28915 100644 --- a/lib/DBD/SQLite.pm +++ b/lib/DBD/SQLite.pm @@ -1549,6 +1549,10 @@ users because SQLite uses dynamic type system (that means, the datatype of a value is associated with the value itself, not with its container). +As of version 1.61_02, if you set C +database handle attribute to true, C statement handle +attribute returns an array of integer, as an experiment. + =head2 Performance SQLite is fast, very fast. Matt processed his 72MB log file with it, From 6faa8a3c906965c6c8dfb112257d81c34747e13c Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Sat, 20 Jan 2018 16:03:55 +0900 Subject: [PATCH 7/8] Implemented backup_to_dbh/backup_from_dbh (#30) --- SQLite.xs | 22 ++++++++++++ dbdimp.c | 86 ++++++++++++++++++++++++++++++++++++++++++++ lib/DBD/SQLite.pm | 15 ++++++++ t/34_online_backup.t | 44 ++++++++++++++++++++++- 4 files changed, 166 insertions(+), 1 deletion(-) diff --git a/SQLite.xs b/SQLite.xs index 2712ac6..e18e1cc 100644 --- a/SQLite.xs +++ b/SQLite.xs @@ -259,6 +259,28 @@ backup_to_file(dbh, filename) OUTPUT: RETVAL +static int +backup_from_dbh(dbh, from) + SV *dbh + SV *from + ALIAS: + DBD::SQLite::db::sqlite_backup_from_dbh = 1 + CODE: + RETVAL = sqlite_db_backup_from_dbh(aTHX_ dbh, from); + OUTPUT: + RETVAL + +static int +backup_to_dbh(dbh, to) + SV *dbh + SV *to + ALIAS: + DBD::SQLite::db::sqlite_backup_to_dbh = 1 + CODE: + RETVAL = sqlite_db_backup_to_dbh(aTHX_ dbh, to); + OUTPUT: + RETVAL + HV* table_column_metadata(dbh, dbname, tablename, columnname) SV* dbh diff --git a/dbdimp.c b/dbdimp.c index 37dc294..b6669ac 100644 --- a/dbdimp.c +++ b/dbdimp.c @@ -2615,6 +2615,49 @@ sqlite_db_backup_from_file(pTHX_ SV *dbh, char *filename) #endif } +int +sqlite_db_backup_from_dbh(pTHX_ SV *dbh, SV *from) +{ + D_imp_dbh(dbh); + +#if SQLITE_VERSION_NUMBER >= 3006011 + int rc; + sqlite3_backup *pBackup; + + imp_dbh_t *imp_dbh_from = (imp_dbh_t *)DBIh_COM(from); + + if (!DBIc_ACTIVE(imp_dbh)) { + sqlite_error(dbh, -2, "attempt to backup from file on inactive database handle"); + return FALSE; + } + + if (!DBIc_ACTIVE(imp_dbh_from)) { + sqlite_error(dbh, -2, "attempt to backup from inactive database handle"); + return FALSE; + } + + croak_if_db_is_null(); + + /* COMPAT: sqlite3_backup_* are only available for 3006011 or newer */ + pBackup = sqlite3_backup_init(imp_dbh->db, "main", imp_dbh_from->db, "main"); + if (pBackup) { + (void)sqlite3_backup_step(pBackup, -1); + (void)sqlite3_backup_finish(pBackup); + } + rc = sqlite3_errcode(imp_dbh->db); + + if ( rc != SQLITE_OK ) { + sqlite_error(dbh, rc, form("sqlite_backup_from_file failed with error %s", sqlite3_errmsg(imp_dbh->db))); + return FALSE; + } + + return TRUE; +#else + sqlite_error(dbh, SQLITE_ERROR, form("backup feature requires SQLite 3.6.11 and newer")); + return FALSE; +#endif +} + /* Accesses the SQLite Online Backup API, and copies the currently loaded * database into the passed filename. * Usual usage of this would be when you're operating on the :memory: @@ -2663,6 +2706,49 @@ sqlite_db_backup_to_file(pTHX_ SV *dbh, char *filename) #endif } +int +sqlite_db_backup_to_dbh(pTHX_ SV *dbh, SV *to) +{ + D_imp_dbh(dbh); + +#if SQLITE_VERSION_NUMBER >= 3006011 + int rc; + sqlite3_backup *pBackup; + + imp_dbh_t *imp_dbh_to = (imp_dbh_t *)DBIh_COM(to); + + if (!DBIc_ACTIVE(imp_dbh)) { + sqlite_error(dbh, -2, "attempt to backup to file on inactive database handle"); + return FALSE; + } + + if (!DBIc_ACTIVE(imp_dbh_to)) { + sqlite_error(dbh, -2, "attempt to backup to inactive database handle"); + return FALSE; + } + + croak_if_db_is_null(); + + /* COMPAT: sqlite3_backup_* are only available for 3006011 or newer */ + pBackup = sqlite3_backup_init(imp_dbh_to->db, "main", imp_dbh->db, "main"); + if (pBackup) { + (void)sqlite3_backup_step(pBackup, -1); + (void)sqlite3_backup_finish(pBackup); + } + rc = sqlite3_errcode(imp_dbh_to->db); + + if ( rc != SQLITE_OK ) { + sqlite_error(dbh, rc, form("sqlite_backup_to_file failed with error %s", sqlite3_errmsg(imp_dbh->db))); + return FALSE; + } + + return TRUE; +#else + sqlite_error(dbh, SQLITE_ERROR, form("backup feature requires SQLite 3.6.11 and newer")); + return FALSE; +#endif +} + int sqlite_db_limit(pTHX_ SV *dbh, int id, int new_value) { diff --git a/lib/DBD/SQLite.pm b/lib/DBD/SQLite.pm index be28915..2e4f0d7 100644 --- a/lib/DBD/SQLite.pm +++ b/lib/DBD/SQLite.pm @@ -47,6 +47,8 @@ sub driver { DBD::SQLite::db->install_method('sqlite_set_authorizer'); DBD::SQLite::db->install_method('sqlite_backup_from_file'); DBD::SQLite::db->install_method('sqlite_backup_to_file'); + DBD::SQLite::db->install_method('sqlite_backup_from_dbh'); + DBD::SQLite::db->install_method('sqlite_backup_to_dbh'); DBD::SQLite::db->install_method('sqlite_enable_load_extension'); DBD::SQLite::db->install_method('sqlite_load_extension'); DBD::SQLite::db->install_method('sqlite_register_fts3_perl_tokenizer'); @@ -2206,6 +2208,19 @@ special :memory: database, and you wish to populate it from an existing DB. This method accesses the SQLite Online Backup API, and will take a backup of the currently connected database, and write it out to the named file. +=head2 $dbh->sqlite_backup_from_dbh( $another_dbh ) + +This method accesses the SQLite Online Backup API, and will take a backup of +the database for the passed handle, copying it to, and overwriting, your current database +connection. This can be particularly handy if your current connection is to the +special :memory: database, and you wish to populate it from an existing DB. +You can use this to backup from an in-memory database to another in-memory database. + +=head2 $dbh->sqlite_backup_to_dbh( $another_dbh ) + +This method accesses the SQLite Online Backup API, and will take a backup of +the currently connected database, and write it out to the passed database handle. + =head2 $dbh->sqlite_enable_load_extension( $bool ) Calling this method with a true value enables loading (external) diff --git a/t/34_online_backup.t b/t/34_online_backup.t index 1ee67d6..5b817ba 100644 --- a/t/34_online_backup.t +++ b/t/34_online_backup.t @@ -12,7 +12,7 @@ BEGIN { requires_sqlite('3.6.11') } use Test::NoWarnings; use DBI; -plan tests => 6 * @CALL_FUNCS + 1; +plan tests => 11 * @CALL_FUNCS + 1; foreach my $call_func (@CALL_FUNCS) { # Connect to the test db and add some stuff: @@ -69,3 +69,45 @@ foreach my $call_func (@CALL_FUNCS) { unlink $dbfile; } + +foreach my $call_func (@CALL_FUNCS) { + # Connect to the test db and add some stuff: + my $foo = connect_ok( dbfile => ':memory:', RaiseError => 1 ); + $foo->do( + 'CREATE TABLE online_backup_test( id INTEGER PRIMARY KEY, foo INTEGER )' + ); + $foo->do("INSERT INTO online_backup_test (foo) VALUES ($$)"); + + my $dbh = DBI->connect( + 'dbi:SQLite:dbname=:memory:', + undef, undef, + { RaiseError => 1 } + ); + + ok($dbh->$call_func($foo, 'backup_from_dbh')); + + { + my ($count) = $dbh->selectrow_array( + "SELECT count(foo) FROM online_backup_test WHERE foo=$$" + ); + is($count, 1, "Found our process ID in backed-up table"); + } + + # Add more data then attempt to copy it back to file: + $dbh->do( + 'CREATE TABLE online_backup_test2 ( id INTEGER PRIMARY KEY, foo INTEGER )' + ); + $dbh->do("INSERT INTO online_backup_test2 (foo) VALUES ($$)"); + + # backup to dbh (foo): + ok($dbh->$call_func($foo, 'backup_to_dbh')); + + $dbh->disconnect; + + my ($count) = $foo->selectrow_array( + "SELECT count(foo) FROM online_backup_test2 WHERE foo=$$" + ); + is($count, 1, "Found our process ID in table back on disk"); + + $foo->disconnect; +} From 67af7629f932adf33999c51d8df52f63cc6dd519 Mon Sep 17 00:00:00 2001 From: Kenichi Ishigaki Date: Sat, 1 Dec 2018 17:52:34 +0900 Subject: [PATCH 8/8] added missing declarations --- dbdimp.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbdimp.h b/dbdimp.h index efe6828..09b8883 100644 --- a/dbdimp.h +++ b/dbdimp.h @@ -117,6 +117,8 @@ int sqlite_bind_col( SV *sth, imp_sth_t *imp_sth, SV *col, SV *ref, IV sql_type, int sqlite_db_busy_timeout (pTHX_ SV *dbh, SV *timeout ); int sqlite_db_backup_from_file(pTHX_ SV *dbh, char *filename); int sqlite_db_backup_to_file(pTHX_ SV *dbh, char *filename); +int sqlite_db_backup_from_dbh(pTHX_ SV *dbh, SV *from); +int sqlite_db_backup_to_dbh(pTHX_ SV *dbh, SV *to); void sqlite_db_collation_needed(pTHX_ SV *dbh, SV *callback ); SV* sqlite_db_commit_hook( pTHX_ SV *dbh, SV *hook ); SV* sqlite_db_rollback_hook( pTHX_ SV *dbh, SV *hook );