Remove space and atWord tokens

`atWord` wasn't used at all and spaces should be eaten by the lexer

And added `unspecified` token

YACReaderLibrary/db/query_lexer.cpp

@@ -7,17 +7,16 @@ QueryLexer::QueryLexer(const std::string &input)
 
 Token QueryLexer::next()
 {
+    while (isSpace(peek())) {
+        get();
+    }
+
     switch (peek()) {
     case '\0':
         return Token(Token::Type::eof);
     case '(':
     case ')':
         return single(Token::Type::opcode);
-    case ' ':
-    case '\t':
-    case '\r':
-    case '\n':
-        return space();
     case '"':
         return quotedWord();
     default:
@@ -40,15 +39,6 @@ Token QueryLexer::single(Token::Type type)
     return Token(type, input.substr(index++, 1));
 }
 
-Token QueryLexer::space()
-{
-    auto start = index;
-    get();
-    while (isSpace(peek()))
-        get();
-    return Token(Token::Type::space, input.substr(start, index - start));
-}
-
 Token QueryLexer::word()
 {
     auto start = index;

YACReaderLibrary/db/query_lexer.h

@@ -9,10 +9,9 @@ public:
     enum class Type {
         eof,
         opcode,
-        atWord,
         word,
         quotedWord,
-        space
+        undefined
     };
 
     Token(Type type, std::string lexeme = "")
@@ -49,7 +48,6 @@ private:
     char get();
 
     Token single(Token::Type type);
-    Token space();
     Token word();
     Token quotedWord();
 

YACReaderLibrary/db/query_parser.cpp

@@ -136,9 +136,6 @@ bool QueryParser::isEof() const
 void QueryParser::advance()
 {
     currentToken = lexer.next();
-
-    if (tokenType() == Token::Type::space)
-        advance();
 }
 
 QueryParser::FieldType QueryParser::fieldType(const std::string &str)
@@ -184,7 +181,7 @@ QueryParser::TreeNode QueryParser::andExpression()
         return { "and", { lhs, andExpression() } };
     }
 
-    if ((isIn(tokenType(), { Token::Type::atWord, Token::Type::word, Token::Type::quotedWord }) || token() == "(") && lcaseToken() != "or") {
+    if ((isIn(tokenType(), { Token::Type::word, Token::Type::quotedWord }) || token() == "(") && lcaseToken() != "or") {
         return { "and", { lhs, andExpression() } };
     }
 
@@ -210,7 +207,7 @@ QueryParser::TreeNode QueryParser::locationExpression()
         }
         return res;
     }
-    if (!isIn(tokenType(), { Token::Type::atWord, Token::Type::word, Token::Type::quotedWord })) {
+    if (!isIn(tokenType(), { Token::Type::word, Token::Type::quotedWord })) {
         throw std::invalid_argument("Invalid syntax. Expected a lookup name or a word");
     }
     return baseToken();

YACReaderLibrary/db/query_parser.h

@@ -62,7 +62,7 @@ private:
     void advance();
 
     QueryLexer lexer = QueryLexer("");
-    Token currentToken = Token(Token::Type::eof);
+    Token currentToken = Token(Token::Type::undefined);
 
     template<typename T>
     static bool isIn(const T &e, const std::list<T> &v)