public abstract class JFlexTokenizer
extends org.apache.lucene.analysis.Tokenizer
Modifier and Type | Field and Description |
---|---|
protected org.apache.lucene.analysis.tokenattributes.OffsetAttribute |
offsetAtt |
protected org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute |
posIncrAtt |
protected org.apache.lucene.analysis.tokenattributes.TermAttribute |
termAtt |
Constructor and Description |
---|
JFlexTokenizer() |
Modifier and Type | Method and Description |
---|---|
void |
close() |
boolean |
incrementToken()
This will reinitalize internal AttributeImpls, or it returns false if end of input Reader ...
|
void |
reInit(char[] contents,
int length)
Reinitialize the tokenizer with new contents.
|
protected void |
setAttribs(java.lang.String str,
int start,
int end) |
abstract void |
yyclose() |
abstract boolean |
yylex() |
abstract void |
yyreset(java.io.Reader reader) |
addAttribute, addAttributeImpl, captureState, clearAttributes, cloneAttributes, copyTo, equals, getAttribute, getAttributeClassesIterator, getAttributeFactory, getAttributeImplsIterator, hasAttribute, hasAttributes, hashCode, reflectAsString, reflectWith, restoreState, toString
protected org.apache.lucene.analysis.tokenattributes.TermAttribute termAtt
protected org.apache.lucene.analysis.tokenattributes.OffsetAttribute offsetAtt
protected org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute posIncrAtt
public abstract boolean yylex() throws java.io.IOException
java.io.IOException
public abstract void yyreset(java.io.Reader reader)
public abstract void yyclose() throws java.io.IOException
java.io.IOException
public final void reInit(char[] contents, int length)
contents
- a char buffer with text to tokenizelength
- the number of characters to use from the char bufferpublic final void close() throws java.io.IOException
close
in interface java.io.Closeable
close
in interface java.lang.AutoCloseable
close
in class org.apache.lucene.analysis.Tokenizer
java.io.IOException
public boolean incrementToken() throws java.io.IOException
incrementToken
in class org.apache.lucene.analysis.TokenStream
java.io.IOException
protected void setAttribs(java.lang.String str, int start, int end)