From f09c5955d8967078b2caa13feaf9607ed66a3e45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luis=20=C3=81ngel=20San=20Mart=C3=ADn?=
 <luisangelsm@gmail.com>
Date: Thu, 14 Jan 2021 08:39:16 +0100
Subject: [PATCH] Remove space and atWord tokens

`atWord` wasn't used at all and spaces should be eaten by the lexer

And added `unspecified` token
---
 YACReaderLibrary/db/query_lexer.cpp  | 18 ++++--------------
 YACReaderLibrary/db/query_lexer.h    |  4 +---
 YACReaderLibrary/db/query_parser.cpp |  7 ++-----
 YACReaderLibrary/db/query_parser.h   |  2 +-
 4 files changed, 8 insertions(+), 23 deletions(-)

diff --git a/YACReaderLibrary/db/query_lexer.cpp b/YACReaderLibrary/db/query_lexer.cpp
index 27944ff4..832b4978 100644
--- a/YACReaderLibrary/db/query_lexer.cpp
+++ b/YACReaderLibrary/db/query_lexer.cpp
@@ -7,17 +7,16 @@ QueryLexer::QueryLexer(const std::string &input)
 
 Token QueryLexer::next()
 {
+    while (isSpace(peek())) {
+        get();
+    }
+
     switch (peek()) {
     case '\0':
         return Token(Token::Type::eof);
     case '(':
     case ')':
         return single(Token::Type::opcode);
-    case ' ':
-    case '\t':
-    case '\r':
-    case '\n':
-        return space();
     case '"':
         return quotedWord();
     default:
@@ -40,15 +39,6 @@ Token QueryLexer::single(Token::Type type)
     return Token(type, input.substr(index++, 1));
 }
 
-Token QueryLexer::space()
-{
-    auto start = index;
-    get();
-    while (isSpace(peek()))
-        get();
-    return Token(Token::Type::space, input.substr(start, index - start));
-}
-
 Token QueryLexer::word()
 {
     auto start = index;
diff --git a/YACReaderLibrary/db/query_lexer.h b/YACReaderLibrary/db/query_lexer.h
index b2c892f6..4cc2b61f 100644
--- a/YACReaderLibrary/db/query_lexer.h
+++ b/YACReaderLibrary/db/query_lexer.h
@@ -9,10 +9,9 @@ public:
     enum class Type {
         eof,
         opcode,
-        atWord,
         word,
         quotedWord,
-        space
+        undefined
     };
 
     Token(Type type, std::string lexeme = "")
@@ -49,7 +48,6 @@ private:
     char get();
 
     Token single(Token::Type type);
-    Token space();
     Token word();
     Token quotedWord();
 
diff --git a/YACReaderLibrary/db/query_parser.cpp b/YACReaderLibrary/db/query_parser.cpp
index de9dc39e..5a10d238 100644
--- a/YACReaderLibrary/db/query_parser.cpp
+++ b/YACReaderLibrary/db/query_parser.cpp
@@ -136,9 +136,6 @@ bool QueryParser::isEof() const
 void QueryParser::advance()
 {
     currentToken = lexer.next();
-
-    if (tokenType() == Token::Type::space)
-        advance();
 }
 
 QueryParser::FieldType QueryParser::fieldType(const std::string &str)
@@ -184,7 +181,7 @@ QueryParser::TreeNode QueryParser::andExpression()
         return { "and", { lhs, andExpression() } };
     }
 
-    if ((isIn(tokenType(), { Token::Type::atWord, Token::Type::word, Token::Type::quotedWord }) || token() == "(") && lcaseToken() != "or") {
+    if ((isIn(tokenType(), { Token::Type::word, Token::Type::quotedWord }) || token() == "(") && lcaseToken() != "or") {
         return { "and", { lhs, andExpression() } };
     }
 
@@ -210,7 +207,7 @@ QueryParser::TreeNode QueryParser::locationExpression()
         }
         return res;
     }
-    if (!isIn(tokenType(), { Token::Type::atWord, Token::Type::word, Token::Type::quotedWord })) {
+    if (!isIn(tokenType(), { Token::Type::word, Token::Type::quotedWord })) {
         throw std::invalid_argument("Invalid syntax. Expected a lookup name or a word");
     }
     return baseToken();
diff --git a/YACReaderLibrary/db/query_parser.h b/YACReaderLibrary/db/query_parser.h
index 95fd48c8..aa688fc2 100644
--- a/YACReaderLibrary/db/query_parser.h
+++ b/YACReaderLibrary/db/query_parser.h
@@ -62,7 +62,7 @@ private:
     void advance();
 
     QueryLexer lexer = QueryLexer("");
-    Token currentToken = Token(Token::Type::eof);
+    Token currentToken = Token(Token::Type::undefined);
 
     template<typename T>
     static bool isIn(const T &e, const std::list<T> &v)