Replace lexertl with a custom lexeter implementation

QueryLexeter does not parse "atWord" because I couldn't find what it is used for.
This commit is contained in:
Luis Ángel San Martín
2021-01-12 18:56:59 +01:00
parent 5037f3ac92
commit a777aa3fe8
41 changed files with 187 additions and 12768 deletions

View File

@ -0,0 +1,94 @@
#include "query_lexer.h"
QueryLexer::QueryLexer(const std::string &input)
: input(input)
{
}
Token QueryLexer::next()
{
switch (peek()) {
case '\0':
return Token(Token::Type::eof);
case '(':
case ')':
return single(Token::Type::opcode);
case ' ':
case '\t':
case '\r':
case '\n':
return space();
case '"':
return quotedWord();
default:
return word();
}
}
char QueryLexer::peek()
{
return input[index];
}
char QueryLexer::get()
{
return input[index++];
}
Token QueryLexer::single(Token::Type type)
{
return Token(type, input.substr(index++, 1));
}
Token QueryLexer::space()
{
auto start = index;
get();
while (isSpace(peek()))
get();
return Token(Token::Type::space, input.substr(start, index - start));
}
Token QueryLexer::word()
{
auto start = index;
get();
auto current = peek();
while (current != '\0' && !isSpace(current) && current != '"' && current != '(' && current != ')') {
get();
current = peek();
}
return Token(Token::Type::word, input.substr(start, index - start));
}
Token QueryLexer::quotedWord()
{
auto start = index;
get();
auto current = peek();
while (current != '\0' && current != '"') {
get();
current = peek();
}
if (current == '"') {
get();
return Token(Token::Type::quotedWord, input.substr(start, index - start));
}
//This should be a lexical error, but the grammar doesn't support it
return Token(Token::Type::eof);
}
bool QueryLexer::isSpace(char c)
{
switch (c) {
case ' ':
case '\t':
case '\r':
case '\n':
return true;
default:
return false;
}
}