Created category for FTS3 support.

Added protocol to implement custom tokenizer logic.

src/fts3/FMDatabase+FTS3.h

@@ -0,0 +1,41 @@
+//
+//  FMDatabase+FTS3.h
+//  fmdb
+//
+//  Created by Andrew on 3/27/14.
+//  Copyright (c) 2014 Andrew Goodale. All rights reserved.
+//
+
+#import "FMDatabase.h"
+
+@protocol FMTokenizerDelegate;
+
+@interface FMDatabase (FTS3)
+
+- (BOOL)registerTokenizer:(id<FMTokenizerDelegate>)tokenizer withName:(NSString *)name;
+
+@end
+
+#pragma mark
+
+/* Extend this structure with your own custom cursor data */
+typedef struct FMTokenizerCursor
+{
+    void       *tokenizer;      /* Internal SQLite reference */
+    void       *tempBuffer;     /* Internal temporary memory */
+    CFStringRef inputString;    /* The input text being tokenized */
+    CFRange     currentRange;   /* The current offset within `inputString` */
+    CFStringRef tokenString;    /* The contents of the current token */
+    CFTypeRef   userObject;     /* Additional state for the cursor */
+    int         tokenIndex;     /* Index of next token to be returned */
+} FMTokenizerCursor;
+
+@protocol FMTokenizerDelegate
+
+- (void)openTokenizerCursor:(FMTokenizerCursor *)cursor;
+
+- (BOOL)nextTokenForCursor:(FMTokenizerCursor *)cursor;
+
+- (void)closeTokenizerCursor:(FMTokenizerCursor *)cursor;
+
+@end

src/fts3/FMDatabase+FTS3.m

@@ -0,0 +1,186 @@
+//
+//  FMDatabase+FTS3.m
+//  fmdb
+//
+//  Created by Andrew on 3/27/14.
+//  Copyright (c) 2014 Andrew Goodale. All rights reserved.
+//
+
+#import "FMDatabase+FTS3.h"
+#import "fts3_tokenizer.h"
+
+/* I know this is an evil global, but I'm not sure an app needs multiple tokenizers anyway...*/
+static id<FMTokenizerDelegate> __unsafe_unretained g_delegate = nil;
+
+/*
+ ** Class derived from sqlite3_tokenizer
+ */
+struct FMDBTokenizer
+{
+    sqlite3_tokenizer base;
+};
+
+/*
+ ** Create a new tokenizer instance.
+ */
+static int FMDBTokenizerCreate(int argc, const char * const *argv, sqlite3_tokenizer **ppTokenizer)
+{
+    struct FMDBTokenizer *t;
+    
+    t = (struct FMDBTokenizer *) sqlite3_malloc(sizeof(*t));
+    
+    if ( t==NULL ) {
+        return SQLITE_NOMEM;
+    }
+    
+    memset(t, 0, sizeof(*t));
+    *ppTokenizer = &t->base;
+    
+    return SQLITE_OK;
+}
+
+/*
+ ** Destroy a tokenizer
+ */
+static int FMDBTokenizerDestroy(sqlite3_tokenizer *pTokenizer)
+{
+    sqlite3_free(pTokenizer);
+    return SQLITE_OK;
+}
+
+/*
+ ** Prepare to begin tokenizing a particular string.  The input
+ ** string to be tokenized is zInput[0..nInput-1].  A cursor
+ ** used to incrementally tokenize this string is returned in
+ ** *ppCursor.
+ */
+static int FMDBTokenizerOpen(sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+                             const char *pInput, int nBytes,        /* String to be tokenized */
+                             sqlite3_tokenizer_cursor **ppCursor)   /* OUT: Tokenization cursor */
+{
+    FMTokenizerCursor *cursor = (FMTokenizerCursor *)sqlite3_malloc(sizeof(FMTokenizerCursor));
+    
+    if (cursor == NULL) {
+        return SQLITE_NOMEM;
+    }
+    
+    nBytes = (nBytes < 0) ? (int) strlen(pInput) : nBytes;
+    cursor->inputString = CFStringCreateWithBytesNoCopy(NULL, (const UInt8 *)pInput, nBytes,
+                                                        kCFStringEncodingUTF8, false, kCFAllocatorNull);
+    cursor->currentRange = CFRangeMake(0, 0);
+    cursor->tokenIndex = 0;
+    cursor->tokenString = NULL;
+    cursor->userObject = NULL;
+    cursor->tempBuffer = NULL;
+        
+    [g_delegate openTokenizerCursor:cursor];
+
+    *ppCursor = (sqlite3_tokenizer_cursor *)cursor;
+    return SQLITE_OK;
+}
+
+/*
+ ** Close a tokenization cursor previously opened by a call to
+ ** FMDBTokenizerOpen() above.
+ */
+static int FMDBTokenizerClose(sqlite3_tokenizer_cursor *pCursor)
+{
+    FMTokenizerCursor *cursor = (FMTokenizerCursor *)pCursor;
+    
+    [g_delegate closeTokenizerCursor:cursor];
+    
+    if (cursor->userObject) {
+        CFRelease(cursor->userObject);
+    }
+    if (cursor->tokenString) {
+        CFRelease(cursor->tokenString);
+    }
+    
+    CFRelease(cursor->inputString);
+    sqlite3_free(cursor->tempBuffer);
+    sqlite3_free(cursor);
+    
+    return SQLITE_OK;
+}
+
+
+/*
+ ** Extract the next token from a tokenization cursor.  The cursor must
+ ** have been opened by a prior call to FMDBTokenizerOpen().
+ */
+static int FMDBTokenizerNext(sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by Open */
+                             const char **pzToken,               /* OUT: *pzToken is the token text */
+                             int *pnBytes,                       /* OUT: Number of bytes in token */
+                             int *piStartOffset,                 /* OUT: Starting offset of token */
+                             int *piEndOffset,                   /* OUT: Ending offset of token */
+                             int *piPosition)                    /* OUT: Position integer of token */
+{
+    FMTokenizerCursor *cursor = (FMTokenizerCursor *)pCursor;
+    
+    if ([g_delegate nextTokenForCursor:cursor]) {
+        return SQLITE_DONE;
+    }
+    
+    const char *strToken = CFStringGetCStringPtr(cursor->tokenString, kCFStringEncodingUTF8);
+    
+    if (strToken == NULL) {
+        // Need to allocate some of our own memory for this.
+        if (cursor->tempBuffer == NULL) {
+            cursor->tempBuffer = sqlite3_malloc(128);   //TODO: fix hard-coded constant
+        }
+
+        CFStringGetCString(cursor->tokenString, cursor->tempBuffer, 128, kCFStringEncodingUTF8);
+        strToken = cursor->tempBuffer;
+    }
+    
+    *pzToken = strToken;
+    *pnBytes = (int) strlen(strToken);
+    *piStartOffset = (int) cursor->currentRange.location;
+    *piEndOffset = (int) (cursor->currentRange.location + cursor->currentRange.length);
+    *piPosition = cursor->tokenIndex++;
+    
+    return SQLITE_OK;
+}
+
+
+/*
+ ** The set of routines that bridge to the tokenizer delegate.
+ */
+static const sqlite3_tokenizer_module FMDBTokenizerModule =
+{
+    0,
+    FMDBTokenizerCreate,
+    FMDBTokenizerDestroy,
+    FMDBTokenizerOpen,
+    FMDBTokenizerClose,
+    FMDBTokenizerNext
+};
+
+#pragma mark
+
+@implementation FMDatabase (FTS3)
+
+- (BOOL)registerTokenizer:(id<FMTokenizerDelegate>)tokenizer withName:(NSString *)name
+{
+    NSParameterAssert(tokenizer);
+
+    if (g_delegate == tokenizer) {
+        return YES;
+    }
+    
+    const sqlite3_tokenizer_module *module = &FMDBTokenizerModule;
+    NSData *tokenizerData = [NSData dataWithBytes:&module  length:sizeof(module)];
+    
+    FMResultSet *results = [self executeQuery:@"SELECT fts3_tokenizer(?, ?)", name, tokenizerData];
+    
+    if ([results next]) {
+        [results close];
+        g_delegate = tokenizer;
+        
+        return YES;
+    }
+    
+    return NO;
+}
+
+@end

src/fts3/fts3_tokenizer.h

@@ -0,0 +1,161 @@
+/*
+** 2006 July 10
+**
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Defines the interface to tokenizers used by fulltext-search.  There
+** are three basic components:
+**
+** sqlite3_tokenizer_module is a singleton defining the tokenizer
+** interface functions.  This is essentially the class structure for
+** tokenizers.
+**
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
+** including customization information defined at creation time.
+**
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
+** tokens from a particular input.
+*/
+#ifndef _FTS3_TOKENIZER_H_
+#define _FTS3_TOKENIZER_H_
+
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
+** If tokenizers are to be allowed to call sqlite3_*() functions, then
+** we will need a way to register the API consistently.
+*/
+#include "sqlite3.h"
+
+/*
+** Structures used by the tokenizer interface. When a new tokenizer
+** implementation is registered, the caller provides a pointer to
+** an sqlite3_tokenizer_module containing pointers to the callback
+** functions that make up an implementation.
+**
+** When an fts3 table is created, it passes any arguments passed to
+** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
+** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
+** implementation. The xCreate() function in turn returns an 
+** sqlite3_tokenizer structure representing the specific tokenizer to
+** be used for the fts3 table (customized by the tokenizer clause arguments).
+**
+** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
+** method is called. It returns an sqlite3_tokenizer_cursor object
+** that may be used to tokenize a specific input buffer based on
+** the tokenization rules supplied by a specific sqlite3_tokenizer
+** object.
+*/
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
+
+struct sqlite3_tokenizer_module {
+
+  /*
+  ** Structure version. Should always be set to 0 or 1.
+  */
+  int iVersion;
+
+  /*
+  ** Create a new tokenizer. The values in the argv[] array are the
+  ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
+  ** TABLE statement that created the fts3 table. For example, if
+  ** the following SQL is executed:
+  **
+  **   CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
+  **
+  ** then argc is set to 2, and the argv[] array contains pointers
+  ** to the strings "arg1" and "arg2".
+  **
+  ** This method should return either SQLITE_OK (0), or an SQLite error 
+  ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
+  ** to point at the newly created tokenizer structure. The generic
+  ** sqlite3_tokenizer.pModule variable should not be initialised by
+  ** this callback. The caller will do so.
+  */
+  int (*xCreate)(
+    int argc,                           /* Size of argv array */
+    const char *const*argv,             /* Tokenizer argument strings */
+    sqlite3_tokenizer **ppTokenizer     /* OUT: Created tokenizer */
+  );
+
+  /*
+  ** Destroy an existing tokenizer. The fts3 module calls this method
+  ** exactly once for each successful call to xCreate().
+  */
+  int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
+
+  /*
+  ** Create a tokenizer cursor to tokenize an input buffer. The caller
+  ** is responsible for ensuring that the input buffer remains valid
+  ** until the cursor is closed (using the xClose() method). 
+  */
+  int (*xOpen)(
+    sqlite3_tokenizer *pTokenizer,       /* Tokenizer object */
+    const char *pInput, int nBytes,      /* Input buffer */
+    sqlite3_tokenizer_cursor **ppCursor  /* OUT: Created tokenizer cursor */
+  );
+
+  /*
+  ** Destroy an existing tokenizer cursor. The fts3 module calls this 
+  ** method exactly once for each successful call to xOpen().
+  */
+  int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
+
+  /*
+  ** Retrieve the next token from the tokenizer cursor pCursor. This
+  ** method should either return SQLITE_OK and set the values of the
+  ** "OUT" variables identified below, or SQLITE_DONE to indicate that
+  ** the end of the buffer has been reached, or an SQLite error code.
+  **
+  ** *ppToken should be set to point at a buffer containing the 
+  ** normalized version of the token (i.e. after any case-folding and/or
+  ** stemming has been performed). *pnBytes should be set to the length
+  ** of this buffer in bytes. The input text that generated the token is
+  ** identified by the byte offsets returned in *piStartOffset and
+  ** *piEndOffset. *piStartOffset should be set to the index of the first
+  ** byte of the token in the input buffer. *piEndOffset should be set
+  ** to the index of the first byte just past the end of the token in
+  ** the input buffer.
+  **
+  ** The buffer *ppToken is set to point at is managed by the tokenizer
+  ** implementation. It is only required to be valid until the next call
+  ** to xNext() or xClose(). 
+  */
+  /* TODO(shess) current implementation requires pInput to be
+  ** nul-terminated.  This should either be fixed, or pInput/nBytes
+  ** should be converted to zInput.
+  */
+  int (*xNext)(
+    sqlite3_tokenizer_cursor *pCursor,   /* Tokenizer cursor */
+    const char **ppToken, int *pnBytes,  /* OUT: Normalized text for token */
+    int *piStartOffset,  /* OUT: Byte offset of token in input buffer */
+    int *piEndOffset,    /* OUT: Byte offset of end of token in input buffer */
+    int *piPosition      /* OUT: Number of tokens returned before this one */
+  );
+
+  /***********************************************************************
+  ** Methods below this point are only available if iVersion>=1.
+  */
+
+  /* 
+  ** Configure the language id of a tokenizer cursor.
+  */
+  int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid);
+};
+
+struct sqlite3_tokenizer {
+  const sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+struct sqlite3_tokenizer_cursor {
+  sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+int fts3_global_term_cnt(int iTerm, int iCol);
+int fts3_term_cnt(int iTerm, int iCol);
+
+
+#endif /* _FTS3_TOKENIZER_H_ */