From 27394e17b1673d8bf47abb13f0f87e07a1c5f34e Mon Sep 17 00:00:00 2001
From: Kenichi Ishigaki <ishigaki@cpan.org>
Date: Mon, 12 Dec 2011 02:39:09 +0000
Subject: [PATCH] resolved #73159

---
 Changes                             |  1 +
 dbdimp.c                            |  4 +++
 t/rt_73159_fts_tokenizer_segfault.t | 38 +++++++++++++++++++++++++++++
 3 files changed, 43 insertions(+)
 create mode 100644 t/rt_73159_fts_tokenizer_segfault.t

diff --git a/Changes b/Changes
index 83ac9a4..1d439f3 100644
--- a/Changes
+++ b/Changes
@@ -3,6 +3,7 @@ Changes for Perl extension DBD-SQLite
 1.36_01 Tue 29 Nov 2011
     - Enabled SQLITE_ENABLE_FTS4
     - Enabled SQLITE_ENABLE_STAT3
+    - Resolved #73159: FTS tokenizer segfault (ISHIGAKI)
 
 1.35 Tue 29 Nov 2011
     - Updated to SQLite 3.7.9 (ISHIGAKI)
diff --git a/dbdimp.c b/dbdimp.c
index 18a307a..56ea851 100644
--- a/dbdimp.c
+++ b/dbdimp.c
@@ -2124,6 +2124,10 @@ static int perl_tokenizer_Create(
     int n_retval;
     SV *retval;
 
+    if (!argc) {
+        return SQLITE_ERROR;
+    }
+
     perl_tokenizer *t;
     t = (perl_tokenizer *) sqlite3_malloc(sizeof(*t));
     if( t==NULL ) return SQLITE_NOMEM;
diff --git a/t/rt_73159_fts_tokenizer_segfault.t b/t/rt_73159_fts_tokenizer_segfault.t
new file mode 100644
index 0000000..5fcaac8
--- /dev/null
+++ b/t/rt_73159_fts_tokenizer_segfault.t
@@ -0,0 +1,38 @@
+#!/usr/bin/perl
+
+use strict;
+BEGIN {
+	$|  = 1;
+	$^W = 1;
+}
+
+use t::lib::Test;
+use Test::More tests => 2;
+use DBI;
+
+my $dbh = connect_ok(RaiseError => 1, PrintError => 0);
+
+sub locale_tokenizer {
+  return sub {
+    my $string = shift;
+
+    use locale;
+    my $regex      = qr/\w+/;
+    my $term_index = 0;
+
+    return sub { # closure
+      $string =~ /$regex/g or return; # either match, or no more token
+      my ($start, $end) = ($-[0], $+[0]);
+      my $len           = $end-$start;
+      my $term          = substr($string, $start, $len);
+      return ($term, $len, $start, $end, $term_index++);
+    }
+  };
+}
+
+# "main::locale_tokenizer" is considered as another column name
+# because of the comma after "tokenize=perl"
+eval {
+  $dbh->do('CREATE VIRTUAL TABLE FIXMESSAGE USING FTS3(MESSAGE, tokenize=perl, "main::locale_tokenizer");');
+};
+ok $@, "cause an error but not segfault";