Jelajahi Sumber

Stop word tokenizer needs to ignore prefix queries.
Fixed a warning under 64-bit builds.

Andrew Goodale 11 tahun lalu
induk
melakukan
ff97584fff
2 mengubah file dengan 9 tambahan dan 2 penghapusan
  1. 1 1
      src/fts3/FMDatabase+FTS3.m
  2. 8 1
      src/fts3/FMTokenizers.m

+ 1 - 1
src/fts3/FMDatabase+FTS3.m

@@ -143,7 +143,7 @@ static int FMDBTokenizerNext(sqlite3_tokenizer_cursor *pCursor,  /* Cursor retur
                      cursor->outputBuf, sizeof(cursor->outputBuf), &usedBytes);
     
     *pzToken = (char *) cursor->outputBuf;
-    *pnBytes = usedBytes;
+    *pnBytes = (int) usedBytes;
     *piStartOffset = (int) cursor->currentRange.location;
     *piEndOffset = (int) (cursor->currentRange.location + cursor->currentRange.length);
     *piPosition = cursor->tokenIndex++;

+ 8 - 1
src/fts3/FMTokenizers.m

@@ -78,6 +78,8 @@ @implementation FMStopWordTokenizer
     id<FMTokenizerDelegate> m_baseTokenizer;
 }
 
+@synthesize words = m_words;
+
 + (instancetype)tokenizerWithFileURL:(NSURL *)wordFileURL
                        baseTokenizer:(id<FMTokenizerDelegate>)tokenizer
                                error:(NSError *__autoreleasing *)error
@@ -98,7 +100,7 @@ - (instancetype)initWithWords:(NSSet *)words baseTokenizer:(id<FMTokenizerDelega
     NSParameterAssert(tokenizer);
     
     if ((self = [super init])) {
-        _words = [words copy];
+        m_words = [words copy];
         m_baseTokenizer = tokenizer;
     }
     return self;
@@ -113,6 +115,11 @@ - (BOOL)nextTokenForCursor:(FMTokenizerCursor *)cursor
 {
     BOOL done = [m_baseTokenizer nextTokenForCursor:cursor];
     
+    // Don't use stop words for prefix queries since it's fine for the prefix to be in the stop list
+    if (CFStringHasSuffix(cursor->inputString, CFSTR("*"))) {
+        return done;
+    }
+    
     while (!done && [self.words containsObject:(__bridge id)(cursor->tokenString)]) {
         done = [m_baseTokenizer nextTokenForCursor:cursor];
     }