From c23579d93abf96672aab0328d33354a39c7f509d Mon Sep 17 00:00:00 2001 From: Max Maischein Date: Sat, 23 Apr 2022 11:17:57 +0200 Subject: [PATCH] Properly export the FTS5_ constants on demand # Conflicts: # constants.inc # lib/DBD/SQLite/Constants.pm # util/constants.pl --- constants.inc | 15 +++++++++++++ lib/DBD/SQLite/Constants.pm | 42 ++++++++++++++++++++++++++++++++++++- t/67_fts5.t | 7 ++++--- util/SQLiteUtil.pm | 18 +++++++++++++++- util/constants.pl | 27 ++++++++++++++++++++---- 5 files changed, 100 insertions(+), 9 deletions(-) diff --git a/constants.inc b/constants.inc index ed2e753..c19a284 100644 --- a/constants.inc +++ b/constants.inc @@ -1293,6 +1293,7 @@ _const_flags_for_file_open_operations() SQLITE_OPEN_READONLY = SQLITE_OPEN_READONLY SQLITE_OPEN_READWRITE = SQLITE_OPEN_READWRITE SQLITE_OPEN_CREATE = SQLITE_OPEN_CREATE + SQLITE_OPEN_SUPER_JOURNAL = SQLITE_OPEN_SUPER_JOURNAL SQLITE_OPEN_NOMUTEX = SQLITE_OPEN_NOMUTEX CODE: RETVAL = ix; @@ -1471,6 +1472,19 @@ _const_flags_for_file_open_operations_3037000_zero() #if SQLITE_VERSION_NUMBER >= 3008003 +IV +_const_fts5_tokenizer() + ALIAS: + FTS5_TOKENIZE_QUERY = FTS5_TOKENIZE_QUERY + FTS5_TOKENIZE_PREFIX = FTS5_TOKENIZE_PREFIX + FTS5_TOKENIZE_DOCUMENT = FTS5_TOKENIZE_DOCUMENT + FTS5_TOKENIZE_AUX = FTS5_TOKENIZE_AUX + FTS5_TOKEN_COLOCATED = FTS5_TOKEN_COLOCATED + CODE: + RETVAL = ix; + OUTPUT: + RETVAL + IV _const_function_flags_3008003() ALIAS: @@ -1820,6 +1834,7 @@ _const__flags_for_file_open_operations() OPEN_READONLY = SQLITE_OPEN_READONLY OPEN_READWRITE = SQLITE_OPEN_READWRITE OPEN_CREATE = SQLITE_OPEN_CREATE + OPEN_SUPER_JOURNAL = SQLITE_OPEN_SUPER_JOURNAL OPEN_NOMUTEX = SQLITE_OPEN_NOMUTEX CODE: RETVAL = ix; diff --git a/lib/DBD/SQLite/Constants.pm b/lib/DBD/SQLite/Constants.pm index 5be8f0a..16c26e0 100644 --- a/lib/DBD/SQLite/Constants.pm +++ b/lib/DBD/SQLite/Constants.pm @@ -190,6 +190,15 @@ our @EXPORT_OK = ( SQLITE_OPEN_URI /, + # fts5_tokenizer + qw/ + FTS5_TOKENIZE_AUX + FTS5_TOKENIZE_DOCUMENT + FTS5_TOKENIZE_PREFIX + FTS5_TOKENIZE_QUERY + FTS5_TOKEN_COLOCATED + /, + # function_flags qw/ SQLITE_DETERMINISTIC @@ -357,6 +366,11 @@ our %EXPORT_TAGS = ( SQLITE_ERROR_SNAPSHOT SQLITE_FLOAT SQLITE_FORMAT + FTS5_TOKENIZE_AUX + FTS5_TOKENIZE_DOCUMENT + FTS5_TOKENIZE_PREFIX + FTS5_TOKENIZE_QUERY + FTS5_TOKEN_COLOCATED SQLITE_FULL SQLITE_FUNCTION SQLITE_IGNORE @@ -650,6 +664,14 @@ our %EXPORT_TAGS = ( SQLITE_OPEN_URI /], + fts5_tokenizer => [qw/ + FTS5_TOKENIZE_AUX + FTS5_TOKENIZE_DOCUMENT + FTS5_TOKENIZE_PREFIX + FTS5_TOKENIZE_QUERY + FTS5_TOKEN_COLOCATED + /], + function_flags => [qw/ SQLITE_DETERMINISTIC SQLITE_DIRECTONLY @@ -736,7 +758,7 @@ DBD::SQLite::Constants - common SQLite constants =head1 DESCRIPTION -You can import necessary SQLite constants from this module. Available tags are C, C, C, C, C (C), C, C, C, C (C), C, C (C), C, C. See L for the complete list of constants. +You can import necessary SQLite constants from this module. Available tags are C, C, C, C, C (C), C, C, C, C (C), C, C, C (C), C, C. See L for the complete list of constants. This module does not export anything by default. @@ -1078,6 +1100,8 @@ This module does not export anything by default. =item SQLITE_OPEN_CREATE +=item SQLITE_OPEN_SUPER_JOURNAL + =item SQLITE_OPEN_NOMUTEX =item SQLITE_OPEN_FULLMUTEX @@ -1098,6 +1122,22 @@ This module does not export anything by default. =back +=head2 fts5_tokenizer + +=over 4 + +=item SQLITE_FTS5_TOKENIZE_QUERY + +=item SQLITE_FTS5_TOKENIZE_PREFIX + +=item SQLITE_FTS5_TOKENIZE_DOCUMENT + +=item SQLITE_FTS5_TOKENIZE_AUX + +=item SQLITE_FTS5_TOKEN_COLOCATED + +=back + =head2 function_flags =over 4 diff --git a/t/67_fts5.t b/t/67_fts5.t index bd5e37c..2919c82 100644 --- a/t/67_fts5.t +++ b/t/67_fts5.t @@ -48,6 +48,8 @@ BEGIN { } } +use DBD::SQLite::Constants ':fts5_tokenizer'; + use locale; sub locale_tokenizer { # see also: Search::Tokenizer @@ -59,14 +61,13 @@ sub locale_tokenizer { # see also: Search::Tokenizer while( $string =~ /$regex/g) { my ($start, $end) = ($-[0], $+[0]); my $term = substr($string, $start, my $len = $end-$start); - my $flags = 0; # SQLITE_FTS5_TOKEN; + my $flags = 0; + #my $flags = FTS5_TOKEN_COLOCATED; DBD::SQLite::db::fts5_xToken($ctx,$flags,$term,$start,$end); }; }; } -use DBD::SQLite; - for my $use_unicode (0, 1) { # connect diff --git a/util/SQLiteUtil.pm b/util/SQLiteUtil.pm index cfe8375..b88373b 100644 --- a/util/SQLiteUtil.pm +++ b/util/SQLiteUtil.pm @@ -143,6 +143,11 @@ my %since = ( STMTSTATUS_RUN => '3020000', STMTSTATUS_MEMUSED => '3020000', DBCONFIG_ENABLE_QPSG => '3020000', + SQLITE_FTS5_TOKEN => '3020000', + FTS5_TOKENIZE_QUERY => '3020000', + FTS5_TOKENIZE_PREFIX => '3020000', + FTS5_TOKENIZE_DOCUMENT => '3020000', + FTS5_TOKENIZE_AUX => '3020000', IOERR_BEGIN_ATOMIC => '3021000', IOERR_COMMIT_ATOMIC => '3021000', IOERR_ROLLBACK_ATOMIC => '3021000', @@ -313,6 +318,17 @@ sub extract_constants { } unshift @{$constants{_authorizer_return_codes}}, 'OK'; + # Fudge in the FTS5 constants, as these don't follow the common pattern + $constants{fts5_tokenizer} ||= []; + push @{$constants{fts5_tokenizer}}, + 'FTS5_TOKENIZE_QUERY', + 'FTS5_TOKENIZE_PREFIX', + 'FTS5_TOKENIZE_DOCUMENT', + 'FTS5_TOKENIZE_AUX', + 'FTS5_TOKEN_COLOCATED' + ; + + %constants; } @@ -335,7 +351,7 @@ sub srcdir { sub download_url { my $version = shift; my $year = $version->year; - join '', + join '', "http://www.sqlite.org/", ($version->year ? $version->year."/" : ""), "sqlite-".($version->archive_type)."-$version".$version->extension; diff --git a/util/constants.pl b/util/constants.pl index ea2d2a5..daf5437 100644 --- a/util/constants.pl +++ b/util/constants.pl @@ -26,6 +26,7 @@ my @dbd_sqlite_constants = ( ); my %constants = extract_constants(); + write_inc(%constants); write_pm(%constants); @@ -86,9 +87,18 @@ _const_$tag() END for my $name (@$list) { - my $prefix = $tag =~ /^_/ ? "" : "SQLITE_"; + my $prefix; + my $prefix2 = "SQLITE_"; + if( $tag =~ /^_/ ) { + $prefix = ""; + } elsif( $tag =~ /^fts5_/ ) { + $prefix = ""; + $prefix2 = ""; + } else { + $prefix = "SQLITE_"; + }; print $fh <<"END"; - $prefix$name = SQLITE_$name + $prefix$name = $prefix2$name END } @@ -111,7 +121,16 @@ END my $ix = 1; for my $name (@{$constants{$tag}}) { - my $prefix = $tag =~ /^_/ ? "" : "SQLITE_"; + my $prefix; + my $prefix2 = "SQLITE_"; + if( $tag =~ /^_/ ) { + $prefix = ""; + } elsif( $tag =~ /^fts5_/ ) { + $prefix = ""; + $prefix2 = ""; + } else { + $prefix = "SQLITE_"; + }; print $fh <<"END"; $prefix$name = $ix END @@ -166,7 +185,7 @@ END print $fh <<"END"; # $tag qw/ -@{[join "\n", map {" SQLITE_$_"} sort @{$constants{$tag}}]} +@{[join "\n", map {/^FTS5_/ ? " $_" : " SQLITE_$_"} sort @{$constants{$tag}}]} /, END