An ANTLR3 lexer implements a base recongizer, a token source and a lexer interface. It constructs a base recognizer with default functions, then overrides any of these that are parser specific (usual default implementation of base recognizer.
#include <antlr3lexer.h>
Functions | |
ANTLR3_API pANTLR3_LEXER | antlr3LexerNew (ANTLR3_UINT32 sizeHint, pANTLR3_RECOGNIZER_SHARED_STATE state) |
ANTLR3_API pANTLR3_LEXER | antlr3LexerNewStream (ANTLR3_UINT32 sizeHint, pANTLR3_INPUT_STREAM input, pANTLR3_RECOGNIZER_SHARED_STATE state) |
static void | displayRecognitionError (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 *tokenNames) |
Default lexer error handler (works for 8 bit streams only!!!). | |
static pANTLR3_COMMON_TOKEN | emit (pANTLR3_LEXER lexer) |
static void | emitNew (pANTLR3_LEXER lexer, pANTLR3_COMMON_TOKEN token) |
static void | freeLexer (pANTLR3_LEXER lexer) |
Free the resources allocated by a lexer. | |
static ANTLR3_MARKER | getCharIndex (pANTLR3_LEXER lexer) |
static ANTLR3_UINT32 | getCharPositionInLine (pANTLR3_LEXER lexer) |
static void * | getCurrentInputSymbol (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream) |
static ANTLR3_UINT32 | getLine (pANTLR3_LEXER lexer) |
static void * | getMissingSymbol (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream, pANTLR3_EXCEPTION e, ANTLR3_UINT32 expectedTokenType, pANTLR3_BITSET_LIST follow) |
static pANTLR3_STRING | getText (pANTLR3_LEXER lexer) |
static void | matchAny (pANTLR3_LEXER lexer) |
static ANTLR3_BOOLEAN | matchc (pANTLR3_LEXER lexer, ANTLR3_UCHAR c) |
Implementation of matchc for the lexer, overrides any base implementation in the base recognizer. | |
static ANTLR3_BOOLEAN | matchRange (pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high) |
Implementation of match range for the lexer, overrides any base implementation in the base recognizer. | |
static ANTLR3_BOOLEAN | matchs (pANTLR3_LEXER lexer, ANTLR3_UCHAR *string) |
Implementation of matchs for the lexer, overrides any base implementation in the base recognizer. | |
static void | mTokens (pANTLR3_LEXER lexer) |
static pANTLR3_COMMON_TOKEN | nextToken (pANTLR3_TOKEN_SOURCE toksource) |
Default implementation of the nextToken() call for a lexer. | |
static ANTLR3_INLINE pANTLR3_COMMON_TOKEN | nextTokenStr (pANTLR3_TOKEN_SOURCE toksource) |
Returns the next available token from the current input stream. | |
static void | popCharStream (pANTLR3_LEXER lexer) |
Stops using the current input stream and reverts to any prior input stream on the stack. | |
static void | pushCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input) |
Change to a new input stream, remembering the old one. | |
static void | recover (pANTLR3_LEXER lexer) |
static void | reportError (pANTLR3_BASE_RECOGNIZER rec) |
static void | reset (pANTLR3_BASE_RECOGNIZER rec) |
static void | setCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input) |
ANTLR3_API pANTLR3_LEXER antlr3LexerNew | ( | ANTLR3_UINT32 | sizeHint, | |
pANTLR3_RECOGNIZER_SHARED_STATE | state | |||
) |
References ANTLR3_MALLOC, ANTLR3_TOKEN_EOF, ANTLR3_TOKEN_INVALID, ANTLR3_TRUE, ANTLR3_TYPE_LEXER, antlr3BaseRecognizerNew(), antlr3SetTokenAPI(), displayRecognitionError(), ANTLR3_BASE_RECOGNIZER_struct::displayRecognitionError, emit(), ANTLR3_LEXER_struct::emit, emitNew(), ANTLR3_LEXER_struct::emitNew, ANTLR3_TOKEN_SOURCE_struct::eofToken, ANTLR3_COMMON_TOKEN_struct::factoryMade, ANTLR3_BASE_RECOGNIZER_struct::free, ANTLR3_LEXER_struct::free, freeLexer(), getCharIndex(), ANTLR3_LEXER_struct::getCharIndex, getCharPositionInLine(), ANTLR3_LEXER_struct::getCharPositionInLine, getCurrentInputSymbol(), ANTLR3_BASE_RECOGNIZER_struct::getCurrentInputSymbol, getLine(), ANTLR3_LEXER_struct::getLine, getMissingSymbol(), ANTLR3_BASE_RECOGNIZER_struct::getMissingSymbol, getText(), ANTLR3_LEXER_struct::getText, matchAny(), ANTLR3_LEXER_struct::matchAny, matchc(), ANTLR3_LEXER_struct::matchc, matchRange(), ANTLR3_LEXER_struct::matchRange, matchs(), ANTLR3_LEXER_struct::matchs, mTokens(), ANTLR3_LEXER_struct::mTokens, nextToken(), ANTLR3_TOKEN_SOURCE_struct::nextToken, popCharStream(), ANTLR3_LEXER_struct::popCharStream, pushCharStream(), ANTLR3_LEXER_struct::pushCharStream, ANTLR3_LEXER_struct::rec, recover(), ANTLR3_LEXER_struct::recover, reportError(), ANTLR3_BASE_RECOGNIZER_struct::reportError, reset(), ANTLR3_BASE_RECOGNIZER_struct::reset, setCharStream(), ANTLR3_LEXER_struct::setCharStream, ANTLR3_COMMON_TOKEN_struct::setType, ANTLR3_TOKEN_SOURCE_struct::skipToken, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_COMMON_TOKEN_struct::strFactory, ANTLR3_TOKEN_SOURCE_struct::strFactory, ANTLR3_TOKEN_SOURCE_struct::super, ANTLR3_BASE_RECOGNIZER_struct::super, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokFactory, and ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokSource.
Referenced by antlr3LexerNewStream().
ANTLR3_API pANTLR3_LEXER antlr3LexerNewStream | ( | ANTLR3_UINT32 | sizeHint, | |
pANTLR3_INPUT_STREAM | input, | |||
pANTLR3_RECOGNIZER_SHARED_STATE | state | |||
) |
static void displayRecognitionError | ( | pANTLR3_BASE_RECOGNIZER | rec, | |
pANTLR3_UINT8 * | tokenNames | |||
) | [static] |
Default lexer error handler (works for 8 bit streams only!!!).
References ANTLR3_FPRINTF, ANTLR3_UINT32_CAST, ANTLR3_EXCEPTION_struct::c, ANTLR3_EXCEPTION_struct::charPositionInLine, ANTLR3_STRING_struct::chars, ANTLR3_INPUT_STREAM_struct::data, ANTLR3_RECOGNIZER_SHARED_STATE_struct::exception, ANTLR3_EXCEPTION_struct::index, ANTLR3_LEXER_struct::input, ANTLR3_EXCEPTION_struct::line, ANTLR3_EXCEPTION_struct::message, ANTLR3_EXCEPTION_struct::name, ANTLR3_LEXER_struct::rec, ANTLR3_INPUT_STREAM_struct::size, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_EXCEPTION_struct::streamName, ANTLR3_BASE_RECOGNIZER_struct::super, ANTLR3_STRING_struct::to8, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartCharIndex, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartCharPositionInLine, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartLine, and ANTLR3_EXCEPTION_struct::type.
static pANTLR3_COMMON_TOKEN emit | ( | pANTLR3_LEXER | lexer | ) | [static] |
References ANTLR3_TEXT_NONE, ANTLR3_TEXT_STRING, ANTLR3_RECOGNIZER_SHARED_STATE_struct::channel, ANTLR3_COMMON_TOKEN_struct::channel, ANTLR3_COMMON_TOKEN_struct::charPosition, ANTLR3_INPUT_STREAM_struct::currentLine, ANTLR3_RECOGNIZER_SHARED_STATE_struct::custom, ANTLR3_COMMON_TOKEN_struct::custom, ANTLR3_LEXER_struct::getCharIndex, ANTLR3_LEXER_struct::input, ANTLR3_COMMON_TOKEN_struct::line, ANTLR3_COMMON_TOKEN_struct::lineStart, ANTLR3_TOKEN_FACTORY_struct::newToken, ANTLR3_LEXER_struct::rec, ANTLR3_COMMON_TOKEN_struct::start, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_COMMON_TOKEN_struct::stop, ANTLR3_COMMON_TOKEN_struct::text, ANTLR3_RECOGNIZER_SHARED_STATE_struct::text, ANTLR3_COMMON_TOKEN_struct::textState, ANTLR3_RECOGNIZER_SHARED_STATE_struct::token, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartCharIndex, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartCharPositionInLine, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartLine, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokFactory, ANTLR3_COMMON_TOKEN_struct::tokText, ANTLR3_RECOGNIZER_SHARED_STATE_struct::type, ANTLR3_COMMON_TOKEN_struct::type, ANTLR3_RECOGNIZER_SHARED_STATE_struct::user1, ANTLR3_COMMON_TOKEN_struct::user1, ANTLR3_RECOGNIZER_SHARED_STATE_struct::user2, ANTLR3_COMMON_TOKEN_struct::user2, ANTLR3_RECOGNIZER_SHARED_STATE_struct::user3, and ANTLR3_COMMON_TOKEN_struct::user3.
Referenced by antlr3LexerNew(), and nextTokenStr().
static void emitNew | ( | pANTLR3_LEXER | lexer, | |
pANTLR3_COMMON_TOKEN | token | |||
) | [static] |
References ANTLR3_LEXER_struct::rec, ANTLR3_BASE_RECOGNIZER_struct::state, and ANTLR3_RECOGNIZER_SHARED_STATE_struct::token.
Referenced by antlr3LexerNew().
static void freeLexer | ( | pANTLR3_LEXER | lexer | ) | [static] |
Free the resources allocated by a lexer.
References ANTLR3_FREE, ANTLR3_TOKEN_FACTORY_struct::close, ANTLR3_BASE_RECOGNIZER_struct::free, ANTLR3_STACK_struct::free, ANTLR3_LEXER_struct::rec, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_RECOGNIZER_SHARED_STATE_struct::streams, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokFactory, and ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokSource.
Referenced by antlr3LexerNew().
static ANTLR3_MARKER getCharIndex | ( | pANTLR3_LEXER | lexer | ) | [static] |
References ANTLR3_INT_STREAM_struct::index, ANTLR3_LEXER_struct::input, and ANTLR3_INPUT_STREAM_struct::istream.
Referenced by antlr3LexerNew().
static ANTLR3_UINT32 getCharPositionInLine | ( | pANTLR3_LEXER | lexer | ) | [static] |
References ANTLR3_INPUT_STREAM_struct::charPositionInLine, and ANTLR3_LEXER_struct::input.
static void * getCurrentInputSymbol | ( | pANTLR3_BASE_RECOGNIZER | recognizer, | |
pANTLR3_INT_STREAM | istream | |||
) | [static] |
static ANTLR3_UINT32 getLine | ( | pANTLR3_LEXER | lexer | ) | [static] |
References ANTLR3_INPUT_STREAM_struct::getLine, and ANTLR3_LEXER_struct::input.
static void * getMissingSymbol | ( | pANTLR3_BASE_RECOGNIZER | recognizer, | |
pANTLR3_INT_STREAM | istream, | |||
pANTLR3_EXCEPTION | e, | |||
ANTLR3_UINT32 | expectedTokenType, | |||
pANTLR3_BITSET_LIST | follow | |||
) | [static] |
static pANTLR3_STRING getText | ( | pANTLR3_LEXER | lexer | ) | [static] |
References ANTLR3_INPUT_STREAM_struct::charByteSize, ANTLR3_LEXER_struct::getCharIndex, ANTLR3_LEXER_struct::input, ANTLR3_LEXER_struct::rec, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_INPUT_STREAM_struct::substr, ANTLR3_RECOGNIZER_SHARED_STATE_struct::text, and ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartCharIndex.
static void matchAny | ( | pANTLR3_LEXER | lexer | ) | [static] |
static ANTLR3_BOOLEAN matchc | ( | pANTLR3_LEXER | lexer, | |
ANTLR3_UCHAR | c | |||
) | [static] |
Implementation of matchc for the lexer, overrides any base implementation in the base recognizer.
References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_FALSE, ANTLR3_TRUE, ANTLR3_RECOGNIZER_SHARED_STATE_struct::backtracking, ANTLR3_INT_STREAM_struct::consume, ANTLR3_BASE_RECOGNIZER_struct::exConstruct, ANTLR3_RECOGNIZER_SHARED_STATE_struct::failed, ANTLR3_LEXER_struct::input, ANTLR3_INPUT_STREAM_struct::istream, ANTLR3_LEXER_struct::rec, ANTLR3_LEXER_struct::recover, and ANTLR3_BASE_RECOGNIZER_struct::state.
Referenced by antlr3LexerNew().
static ANTLR3_BOOLEAN matchRange | ( | pANTLR3_LEXER | lexer, | |
ANTLR3_UCHAR | low, | |||
ANTLR3_UCHAR | high | |||
) | [static] |
Implementation of match range for the lexer, overrides any base implementation in the base recognizer.
References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_FALSE, ANTLR3_TRUE, ANTLR3_RECOGNIZER_SHARED_STATE_struct::backtracking, ANTLR3_INT_STREAM_struct::consume, ANTLR3_BASE_RECOGNIZER_struct::exConstruct, ANTLR3_RECOGNIZER_SHARED_STATE_struct::failed, ANTLR3_LEXER_struct::input, ANTLR3_INPUT_STREAM_struct::istream, ANTLR3_LEXER_struct::rec, ANTLR3_LEXER_struct::recover, and ANTLR3_BASE_RECOGNIZER_struct::state.
Referenced by antlr3LexerNew().
static ANTLR3_BOOLEAN matchs | ( | pANTLR3_LEXER | lexer, | |
ANTLR3_UCHAR * | string | |||
) | [static] |
Implementation of matchs for the lexer, overrides any base implementation in the base recognizer.
References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_FALSE, ANTLR3_STRING_TERMINATOR, ANTLR3_TRUE, ANTLR3_RECOGNIZER_SHARED_STATE_struct::backtracking, ANTLR3_INT_STREAM_struct::consume, ANTLR3_BASE_RECOGNIZER_struct::exConstruct, ANTLR3_RECOGNIZER_SHARED_STATE_struct::failed, ANTLR3_LEXER_struct::input, ANTLR3_INPUT_STREAM_struct::istream, ANTLR3_LEXER_struct::rec, ANTLR3_LEXER_struct::recover, and ANTLR3_BASE_RECOGNIZER_struct::state.
Referenced by antlr3LexerNew().
static void mTokens | ( | pANTLR3_LEXER | lexer | ) | [static] |
References ANTLR3_FPRINTF.
Referenced by antlr3LexerNew().
static pANTLR3_COMMON_TOKEN nextToken | ( | pANTLR3_TOKEN_SOURCE | toksource | ) | [static] |
Default implementation of the nextToken() call for a lexer.
toksource | Points to the implementation of a token source. The lexer is addressed by the super structure pointer. |
References ANTLR3_TOKEN_EOF, nextTokenStr(), ANTLR3_LEXER_struct::popCharStream, ANTLR3_LEXER_struct::rec, ANTLR3_STACK_struct::size, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_RECOGNIZER_SHARED_STATE_struct::streams, ANTLR3_TOKEN_SOURCE_struct::super, and ANTLR3_COMMON_TOKEN_struct::type.
Referenced by antlr3LexerNew(), and antlr3RewriteRuleElementStreamNewAE().
static ANTLR3_INLINE pANTLR3_COMMON_TOKEN nextTokenStr | ( | pANTLR3_TOKEN_SOURCE | toksource | ) | [static] |
Returns the next available token from the current input stream.
toksource | Points to the implementation of a token source. The lexer is addressed by the super structure pointer. |
Loop until we get a non skipped token or EOF
References ANTLR3_INT_STREAM_struct::_LA, ANTLR3_CHARSTREAM_EOF, ANTLR3_FALSE, ANTLR3_TOKEN_DEFAULT_CHANNEL, ANTLR3_TRUE, ANTLR3_RECOGNIZER_SHARED_STATE_struct::channel, ANTLR3_INPUT_STREAM_struct::charPositionInLine, ANTLR3_LEXER_struct::ctx, ANTLR3_RECOGNIZER_SHARED_STATE_struct::custom, emit(), ANTLR3_TOKEN_SOURCE_struct::eofToken, ANTLR3_RECOGNIZER_SHARED_STATE_struct::error, ANTLR3_COMMON_TOKEN_struct::factoryMade, ANTLR3_RECOGNIZER_SHARED_STATE_struct::failed, ANTLR3_LEXER_struct::getCharIndex, ANTLR3_LEXER_struct::getLine, ANTLR3_LEXER_struct::input, ANTLR3_INPUT_STREAM_struct::istream, ANTLR3_INPUT_STREAM_struct::line, ANTLR3_LEXER_struct::mTokens, ANTLR3_INPUT_STREAM_struct::nextChar, ANTLR3_LEXER_struct::rec, ANTLR3_LEXER_struct::recover, ANTLR3_BASE_RECOGNIZER_struct::reportError, ANTLR3_COMMON_TOKEN_struct::setLine, ANTLR3_COMMON_TOKEN_struct::setStartIndex, ANTLR3_COMMON_TOKEN_struct::setStopIndex, ANTLR3_TOKEN_SOURCE_struct::skipToken, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_TOKEN_SOURCE_struct::super, ANTLR3_RECOGNIZER_SHARED_STATE_struct::text, ANTLR3_RECOGNIZER_SHARED_STATE_struct::token, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartCharIndex, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartCharPositionInLine, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartLine, ANTLR3_RECOGNIZER_SHARED_STATE_struct::user1, ANTLR3_RECOGNIZER_SHARED_STATE_struct::user2, and ANTLR3_RECOGNIZER_SHARED_STATE_struct::user3.
Referenced by nextToken().
static void popCharStream | ( | pANTLR3_LEXER | lexer | ) | [static] |
Stops using the current input stream and reverts to any prior input stream on the stack.
lexer | Description of parameter lexer. |
References ANTLR3_LEXER_struct::input, ANTLR3_INPUT_STREAM_struct::istream, ANTLR3_STACK_struct::pop, ANTLR3_LEXER_struct::rec, ANTLR3_INT_STREAM_struct::rewindLast, ANTLR3_LEXER_struct::setCharStream, ANTLR3_STACK_struct::size, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_RECOGNIZER_SHARED_STATE_struct::streams, and ANTLR3_STACK_struct::top.
Referenced by antlr3LexerNew().
static void pushCharStream | ( | pANTLR3_LEXER | lexer, | |
pANTLR3_INPUT_STREAM | input | |||
) | [static] |
Change to a new input stream, remembering the old one.
lexer | Pointer to the lexer instance to switch input streams for. | |
input | New input stream to install as the current one. |
References antlr3StackNew(), ANTLR3_LEXER_struct::input, ANTLR3_INPUT_STREAM_struct::istream, ANTLR3_INT_STREAM_struct::mark, ANTLR3_STACK_struct::push, ANTLR3_LEXER_struct::rec, ANTLR3_LEXER_struct::setCharStream, ANTLR3_BASE_RECOGNIZER_struct::state, and ANTLR3_RECOGNIZER_SHARED_STATE_struct::streams.
Referenced by antlr3LexerNew().
static void recover | ( | pANTLR3_LEXER | lexer | ) | [static] |
static void reportError | ( | pANTLR3_BASE_RECOGNIZER | rec | ) | [static] |
static void reset | ( | pANTLR3_BASE_RECOGNIZER | rec | ) | [static] |
References ANTLR3_TOKEN_DEFAULT_CHANNEL, ANTLR3_TOKEN_INVALID, ANTLR3_RECOGNIZER_SHARED_STATE_struct::channel, ANTLR3_LEXER_struct::rec, ANTLR3_TOKEN_FACTORY_struct::reset, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_BASE_RECOGNIZER_struct::super, ANTLR3_RECOGNIZER_SHARED_STATE_struct::text, ANTLR3_RECOGNIZER_SHARED_STATE_struct::token, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartCharIndex, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartCharPositionInLine, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartLine, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokFactory, and ANTLR3_RECOGNIZER_SHARED_STATE_struct::type.
static void setCharStream | ( | pANTLR3_LEXER | lexer, | |
pANTLR3_INPUT_STREAM | input | |||
) | [static] |
References antlr3RecognitionExceptionNew(), antlr3TokenFactoryNew(), ANTLR3_TOKEN_SOURCE_struct::eofToken, ANTLR3_BASE_RECOGNIZER_struct::exConstruct, ANTLR3_INPUT_STREAM_struct::fileName, ANTLR3_TOKEN_SOURCE_struct::fileName, ANTLR3_LEXER_struct::input, ANTLR3_LEXER_struct::rec, ANTLR3_TOKEN_FACTORY_struct::setInputStream, ANTLR3_BASE_RECOGNIZER_struct::state, ANTLR3_COMMON_TOKEN_struct::strFactory, ANTLR3_INPUT_STREAM_struct::strFactory, ANTLR3_TOKEN_SOURCE_struct::strFactory, ANTLR3_RECOGNIZER_SHARED_STATE_struct::text, ANTLR3_RECOGNIZER_SHARED_STATE_struct::token, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokenStartCharIndex, ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokFactory, and ANTLR3_RECOGNIZER_SHARED_STATE_struct::tokSource.
Referenced by antlr3LexerNew(), and antlr3LexerNewStream().