|
@@ -78,6 +78,8 @@ @implementation FMStopWordTokenizer
|
|
|
id<FMTokenizerDelegate> m_baseTokenizer;
|
|
id<FMTokenizerDelegate> m_baseTokenizer;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+@synthesize words = m_words;
|
|
|
|
|
+
|
|
|
+ (instancetype)tokenizerWithFileURL:(NSURL *)wordFileURL
|
|
+ (instancetype)tokenizerWithFileURL:(NSURL *)wordFileURL
|
|
|
baseTokenizer:(id<FMTokenizerDelegate>)tokenizer
|
|
baseTokenizer:(id<FMTokenizerDelegate>)tokenizer
|
|
|
error:(NSError *__autoreleasing *)error
|
|
error:(NSError *__autoreleasing *)error
|
|
@@ -98,7 +100,7 @@ - (instancetype)initWithWords:(NSSet *)words baseTokenizer:(id<FMTokenizerDelega
|
|
|
NSParameterAssert(tokenizer);
|
|
NSParameterAssert(tokenizer);
|
|
|
|
|
|
|
|
if ((self = [super init])) {
|
|
if ((self = [super init])) {
|
|
|
- _words = [words copy];
|
|
|
|
|
|
|
+ m_words = [words copy];
|
|
|
m_baseTokenizer = tokenizer;
|
|
m_baseTokenizer = tokenizer;
|
|
|
}
|
|
}
|
|
|
return self;
|
|
return self;
|
|
@@ -113,6 +115,11 @@ - (BOOL)nextTokenForCursor:(FMTokenizerCursor *)cursor
|
|
|
{
|
|
{
|
|
|
BOOL done = [m_baseTokenizer nextTokenForCursor:cursor];
|
|
BOOL done = [m_baseTokenizer nextTokenForCursor:cursor];
|
|
|
|
|
|
|
|
|
|
+ // Don't use stop words for prefix queries since it's fine for the prefix to be in the stop list
|
|
|
|
|
+ if (CFStringHasSuffix(cursor->inputString, CFSTR("*"))) {
|
|
|
|
|
+ return done;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
while (!done && [self.words containsObject:(__bridge id)(cursor->tokenString)]) {
|
|
while (!done && [self.words containsObject:(__bridge id)(cursor->tokenString)]) {
|
|
|
done = [m_baseTokenizer nextTokenForCursor:cursor];
|
|
done = [m_baseTokenizer nextTokenForCursor:cursor];
|
|
|
}
|
|
}
|