mirror/yacreader
1
0
Fork 0
mirror of https://github.com/YACReader/yacreader synced 2025-07-18 04:54:29 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove space and atWord tokens

Browse Source 
`atWord` wasn't used at all and spaces should be eaten by the lexer

And added `unspecified` token
This commit is contained in:
Luis Ángel San Martín
2021-01-14 08:39:16 +01:00
parent ddb140d430
commit f09c5955d8
4 changed files with 8 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
YACReaderLibrary/db/query_lexer.cpp
View File

@ -7,17 +7,16 @@ QueryLexer::QueryLexer(const std::string &input)
Token QueryLexer::next()
{
while (isSpace(peek())) {
get();
}
switch (peek()) {
case '\0':
return Token(Token::Type::eof);
case '(':
case ')':
return single(Token::Type::opcode);
case ' ':
case '\t':
case '\r':
case '\n':
return space();
case '"':
return quotedWord();
default:
@ -40,15 +39,6 @@ Token QueryLexer::single(Token::Type type)
return Token(type, input.substr(index++, 1));
}
Token QueryLexer::space()
{
auto start = index;
get();
while (isSpace(peek()))
get();
return Token(Token::Type::space, input.substr(start, index - start));
}
Token QueryLexer::word()
{
auto start = index;