public abstract class JFlexTokenizer
extends org.apache.lucene.analysis.Tokenizer
Modifier and Type | Field and Description |
---|---|
protected org.apache.lucene.analysis.tokenattributes.OffsetAttribute |
offsetAtt |
protected org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute |
posIncrAtt |
protected org.apache.lucene.analysis.tokenattributes.TermAttribute |
termAtt |
Constructor and Description |
---|
JFlexTokenizer() |
Modifier and Type | Method and Description |
---|---|
boolean |
incrementToken()
This will reinitalize internal AttributeImpls, or it returns false if end of input Reader ...
|
protected void |
setAttribs(java.lang.String str,
int start,
int end) |
abstract boolean |
yylex() |
addAttribute, addAttributeImpl, captureState, clearAttributes, cloneAttributes, equals, getAttribute, getAttributeClassesIterator, getAttributeFactory, getAttributeImplsIterator, hasAttribute, hasAttributes, hashCode, restoreState, toString
protected org.apache.lucene.analysis.tokenattributes.TermAttribute termAtt
protected org.apache.lucene.analysis.tokenattributes.OffsetAttribute offsetAtt
protected org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute posIncrAtt
public abstract boolean yylex() throws java.io.IOException
java.io.IOException
public boolean incrementToken() throws java.io.IOException
incrementToken
in class org.apache.lucene.analysis.TokenStream
java.io.IOException
protected void setAttribs(java.lang.String str, int start, int end)