Add commit 43aab01 of BenHanson/lexertl14 from github

YACReaderLibrary/lexertl/generator.hpp

@@ -0,0 +1,738 @@
+// generator.hpp
+// Copyright (c) 2005-2018 Ben Hanson (http://www.benhanson.net/)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+#ifndef LEXERTL_GENERATOR_HPP
+#define LEXERTL_GENERATOR_HPP
+
+#include <algorithm>
+#include "partition/charset.hpp"
+#include "char_traits.hpp"
+#include "partition/equivset.hpp"
+#include <list>
+#include <memory>
+#include "parser/parser.hpp"
+#include "rules.hpp"
+#include "state_machine.hpp"
+#include <type_traits>
+
+namespace lexertl
+{
+template<typename rules, typename sm, typename char_traits = basic_char_traits
+    <typename sm::traits::input_char_type> >
+class basic_generator
+{
+public:
+    using id_type = typename rules::id_type;
+    using rules_char_type = typename rules::rules_char_type;
+    using sm_traits = typename sm::traits;
+    using parser = detail::basic_parser<rules_char_type, sm_traits>;
+    using charset_map = typename parser::charset_map;
+    using node = typename parser::node;
+    using node_ptr_vector = typename parser::node_ptr_vector;
+
+    static void build(const rules &rules_, sm &sm_)
+    {
+        const std::size_t size_ = rules_.statemap().size();
+        // Strong exception guarantee
+        // http://www.boost.org/community/exception_safety.html
+        internals internals_;
+        sm temp_sm_;
+        node_ptr_vector node_ptr_vector_;
+
+        internals_._eoi = rules_.eoi();
+        internals_.add_states(size_);
+
+        for (id_type index_ = 0; index_ < size_; ++index_)
+        {
+            if (rules_.regexes()[index_].empty())
+            {
+                std::ostringstream ss_;
+
+                ss_ << "Lexer states with no rules are not allowed "
+                    "(lexer state " << index_ << ".)";
+                throw runtime_error(ss_.str());
+            }
+            else
+            {
+                // Note that the following variables are per DFA.
+                // Map of regex charset tokens (strings) to index
+                charset_map charset_map_;
+                // Used to fix up $ and \n clashes.
+                id_type nl_id_ = sm_traits::npos();
+                // Regex syntax tree
+                observer_ptr<node> root_ = build_tree(rules_, index_,
+                    node_ptr_vector_, charset_map_, nl_id_);
+
+                build_dfa(charset_map_, root_, internals_, temp_sm_, index_,
+                    nl_id_);
+
+                if (internals_._dfa[index_].size() /
+                    internals_._dfa_alphabet[index_] >= sm_traits::npos())
+                {
+                    // Overflow
+                    throw runtime_error("The data type you have chosen "
+                        "cannot hold this many DFA rows.");
+                }
+            }
+        }
+
+        // If you get a compile error here the id_type from rules and
+        // state machine do no match.
+        create(internals_, temp_sm_, rules_.features(), lookup());
+        sm_.swap(temp_sm_);
+    }
+
+    static observer_ptr<node> build_tree(const rules &rules_,
+        const std::size_t dfa_, node_ptr_vector &node_ptr_vector_,
+        charset_map &charset_map_, id_type &nl_id_)
+    {
+        parser parser_(rules_.locale(), node_ptr_vector_, charset_map_,
+            rules_.eoi());
+        const auto &regexes_ = rules_.regexes();
+        auto regex_iter_ = regexes_[dfa_].cbegin();
+        auto regex_iter_end_ = regexes_[dfa_].cend();
+        const auto &ids_ = rules_.ids();
+        const auto &user_ids_ = rules_.user_ids();
+        auto id_iter_ = ids_[dfa_].cbegin();
+        auto user_id_iter_ = user_ids_[dfa_].cbegin();
+        const auto &next_dfas_ = rules_.next_dfas();
+        const auto &pushes_ = rules_.pushes();
+        const auto &pops_ = rules_.pops();
+        auto next_dfa_iter_ = next_dfas_[dfa_].cbegin();
+        auto push_dfa_iter_ = pushes_[dfa_].cbegin();
+        auto pop_dfa_iter_ = pops_[dfa_].cbegin();
+        const bool seen_bol_ = (rules_.features()[dfa_] & bol_bit) != 0;
+        observer_ptr<node> root_ = nullptr;
+
+        root_ = parser_.parse(*regex_iter_, *id_iter_, *user_id_iter_,
+            *next_dfa_iter_, *push_dfa_iter_, *pop_dfa_iter_,
+            rules_.flags(), nl_id_, seen_bol_);
+        ++regex_iter_;
+        ++id_iter_;
+        ++user_id_iter_;
+        ++next_dfa_iter_;
+        ++push_dfa_iter_;
+        ++pop_dfa_iter_;
+
+        // Build syntax trees
+        while (regex_iter_ != regex_iter_end_)
+        {
+            observer_ptr<node> rhs_ = parser_.parse(*regex_iter_, *id_iter_,
+                *user_id_iter_, *next_dfa_iter_, *push_dfa_iter_,
+                *pop_dfa_iter_, rules_.flags(), nl_id_,
+                (rules_.features()[dfa_] & bol_bit) != 0);
+
+            node_ptr_vector_.emplace_back
+                (std::make_unique<selection_node>(root_, rhs_));
+            root_ = node_ptr_vector_.back().get();
+
+            ++regex_iter_;
+            ++id_iter_;
+            ++user_id_iter_;
+            ++next_dfa_iter_;
+            ++push_dfa_iter_;
+            ++pop_dfa_iter_;
+        }
+
+        return root_;
+    }
+
+protected:
+    using compressed = std::integral_constant<bool, sm_traits::compressed>;
+    using equivset = detail::basic_equivset<id_type>;
+    using equivset_list = std::list<std::unique_ptr<equivset>>;
+    using equivset_ptr = std::unique_ptr<equivset>;
+    using sm_char_type = typename sm_traits::char_type;
+    using charset = detail::basic_charset<sm_char_type, id_type>;
+    using charset_ptr = std::unique_ptr<charset>;
+    using charset_list = std::list<std::unique_ptr<charset>>;
+    using internals = detail::basic_internals<id_type>;
+    using id_type_set = typename std::set<id_type>;
+    using id_type_vector = typename internals::id_type_vector;
+    using index_set = typename charset::index_set;
+    using index_set_vector = std::vector<index_set>;
+    using is_dfa = std::integral_constant<bool, sm_traits::is_dfa>;
+    using lookup = std::integral_constant<bool, sm_traits::lookup>;
+    using node_set = std::set<observer_ptr<const node>>;
+    using node_set_vector = std::vector<std::unique_ptr<node_set>>;
+    using node_vector = typename node::node_vector;
+    using node_vector_vector = std::vector<std::unique_ptr<node_vector>>;
+    using selection_node = typename parser::selection_node;
+    using size_t_vector = typename std::vector<std::size_t>;
+    using string_token = typename parser::string_token;
+
+    static void build_dfa(const charset_map &charset_map_,
+        const observer_ptr<node> root_, internals &internals_, sm &sm_,
+        const id_type dfa_index_, id_type &nl_id_)
+    {
+        // partitioned charset list
+        charset_list charset_list_;
+        // vector mapping token indexes to partitioned token index sets
+        index_set_vector set_mapping_;
+        auto &dfa_ = internals_._dfa[dfa_index_];
+        std::size_t dfa_alphabet_ = 0;
+        const node_vector &followpos_ = root_->firstpos();
+        node_set_vector seen_sets_;
+        node_vector_vector seen_vectors_;
+        size_t_vector hash_vector_;
+        id_type zero_id_ = sm_traits::npos();
+        id_type_set eol_set_;
+
+        set_mapping_.resize(charset_map_.size());
+        partition_charsets(charset_map_, charset_list_, is_dfa());
+        build_set_mapping(charset_list_, internals_, dfa_index_,
+            set_mapping_);
+
+        if (nl_id_ != sm_traits::npos())
+        {
+            nl_id_ = *set_mapping_[nl_id_].begin();
+            zero_id_ = sm_traits::compressed ?
+                *set_mapping_[charset_map_.find(string_token(0, 0))->
+                second].begin() : sm_traits::npos();
+        }
+
+        dfa_alphabet_ = charset_list_.size() + transitions_index +
+            (nl_id_ == sm_traits::npos() ? 0 : 1);
+
+        if (dfa_alphabet_ > sm_traits::npos())
+        {
+            // Overflow
+            throw runtime_error("The data type you have chosen cannot hold "
+                "the dfa alphabet.");
+        }
+
+        internals_._dfa_alphabet[dfa_index_] =
+            static_cast<id_type>(dfa_alphabet_);
+        // 'jam' state
+        dfa_.resize(dfa_alphabet_, 0);
+        closure(followpos_, seen_sets_, seen_vectors_, hash_vector_,
+            static_cast<id_type>(dfa_alphabet_), dfa_);
+
+        // Loop over states
+        for (id_type index_ = 0; index_ < static_cast<id_type>
+            (seen_vectors_.size()); ++index_)
+        {
+            equivset_list equiv_list_;
+
+            // Intersect charsets
+            build_equiv_list(*seen_vectors_[index_].get(), set_mapping_,
+                equiv_list_, is_dfa());
+
+            for (auto &equivset_ : equiv_list_)
+            {
+                const id_type transition_ = closure
+                    (equivset_->_followpos, seen_sets_, seen_vectors_,
+                    hash_vector_, static_cast<id_type>(dfa_alphabet_), dfa_);
+
+                if (transition_ != sm_traits::npos())
+                {
+                    observer_ptr<id_type> ptr_ = &dfa_.front() +
+                        ((index_ + 1) * dfa_alphabet_);
+
+                    // Prune abstemious transitions from end states.
+                    if (*ptr_ && !equivset_->_greedy) continue;
+
+                    set_transitions(transition_, equivset_.get(), dfa_, ptr_,
+                        index_, eol_set_);
+                }
+            }
+        }
+
+        fix_clashes(eol_set_, nl_id_, zero_id_, dfa_, dfa_alphabet_,
+            compressed());
+        append_dfa(charset_list_, internals_, sm_, dfa_index_, lookup());
+    }
+
+    static void set_transitions(const id_type transition_, equivset *equivset_,
+        typename internals::id_type_vector &dfa_, id_type *ptr_,
+        const id_type index_, id_type_set &eol_set_)
+    {
+        for (typename equivset::index_vector::const_iterator
+            equiv_iter_ = equivset_->_index_vector.begin(),
+            equiv_end_ = equivset_->_index_vector.end();
+            equiv_iter_ != equiv_end_; ++equiv_iter_)
+        {
+            const id_type i_ = *equiv_iter_;
+
+            if (i_ == parser::bol_token())
+            {
+                dfa_.front() = transition_;
+            }
+            else if (i_ == parser::eol_token())
+            {
+                ptr_[eol_index] = transition_;
+                eol_set_.insert(index_ + 1);
+            }
+            else
+            {
+                ptr_[i_ + transitions_index] = transition_;
+            }
+        }
+    }
+
+    // Uncompressed
+    static void fix_clashes(const id_type_set &eol_set_,
+        const id_type nl_id_, const id_type /*zero_id_*/,
+        typename internals::id_type_vector &dfa_,
+        const std::size_t dfa_alphabet_, const std::false_type &)
+    {
+        for (const auto &eol_ : eol_set_)
+        {
+            observer_ptr<id_type> ptr_ = &dfa_.front() + eol_ * dfa_alphabet_;
+            const id_type eol_state_ = ptr_[eol_index];
+            const id_type nl_state_ = ptr_[nl_id_ + transitions_index];
+
+            if (nl_state_)
+            {
+                ptr_[transitions_index + nl_id_] = 0;
+                ptr_ = &dfa_.front() + eol_state_ * dfa_alphabet_;
+
+                if (ptr_[transitions_index + nl_id_] == 0)
+                {
+                    ptr_[transitions_index + nl_id_] = nl_state_;
+                }
+            }
+        }
+    }
+
+    // Compressed
+    static void fix_clashes(const id_type_set &eol_set_,
+        const id_type nl_id_, const id_type zero_id_,
+        typename internals::id_type_vector &dfa_,
+        const std::size_t dfa_alphabet_, const std::true_type &)
+    {
+        std::size_t i_ = 0;
+
+        for (const auto &eol_ : eol_set_)
+        {
+            observer_ptr<id_type> ptr_ = &dfa_.front() + eol_ * dfa_alphabet_;
+            const id_type eol_state_ = ptr_[eol_index];
+            id_type nl_state_ = 0;
+
+            for (; i_ < (sm_traits::char_24_bit ? 2 : 1); ++i_)
+            {
+                ptr_ = &dfa_.front() + ptr_[transitions_index + zero_id_] *
+                    dfa_alphabet_;
+            }
+
+            nl_state_ = ptr_[transitions_index + nl_id_];
+
+            if (nl_state_)
+            {
+                ptr_ = &dfa_.front() + eol_state_ * dfa_alphabet_;
+
+                if (ptr_[transitions_index + zero_id_] != 0) continue;
+
+                ptr_[transitions_index + zero_id_] =
+                    static_cast<id_type>(dfa_.size() / dfa_alphabet_);
+                dfa_.resize(dfa_.size() + dfa_alphabet_, 0);
+
+                for (i_ = 0; i_ < (sm_traits::char_24_bit ? 1 : 0); ++i_)
+                {
+                    ptr_ = &dfa_.front() + dfa_.size() - dfa_alphabet_;
+                    ptr_[transitions_index + zero_id_] =
+                        static_cast<id_type>(dfa_.size() / dfa_alphabet_);
+                    dfa_.resize(dfa_.size() + dfa_alphabet_, 0);
+                }
+
+                ptr_ = &dfa_.front() + dfa_.size() - dfa_alphabet_;
+                ptr_[transitions_index + nl_id_] = nl_state_;
+            }
+        }
+    }
+
+    // char_state_machine version
+    static void append_dfa(const charset_list &charset_list_,
+        const internals &internals_, sm &sm_, const id_type dfa_index_,
+        const std::false_type &)
+    {
+        std::size_t size_ = charset_list_.size();
+        typename sm::string_token_vector token_vector_;
+
+        token_vector_.reserve(size_);
+
+        for (const auto &charset_ : charset_list_)
+        {
+            token_vector_.push_back(charset_->_token);
+        }
+
+        sm_.append(token_vector_, internals_, dfa_index_);
+    }
+
+    // state_machine version
+    static void append_dfa(const charset_list &, const internals &, sm &,
+        const id_type, const std::true_type &)
+    {
+        // Nothing to do - will use create() instead
+    }
+
+    // char_state_machine version
+    static void create(internals &, sm &, const id_type_vector &,
+        const std::false_type &)
+    {
+        // Nothing to do - will use append_dfa() instead
+    }
+
+    // state_machine version
+    static void create(internals &internals_, sm &sm_,
+        const id_type_vector &features_, const std::true_type &)
+    {
+        for (std::size_t i_ = 0, size_ = internals_._dfa.size();
+            i_ < size_; ++i_)
+        {
+            internals_._features |= features_[i_];
+        }
+
+        if (internals_._dfa.size() > 1)
+        {
+            internals_._features |= multi_state_bit;
+        }
+
+        sm_.data().swap(internals_);
+    }
+
+    // NFA version
+    static void partition_charsets(const charset_map &map_,
+        charset_list &lhs_, const std::false_type &)
+    {
+        fill_rhs_list(map_, lhs_);
+    }
+
+    // DFA version
+    static void partition_charsets(const charset_map &map_,
+        charset_list &lhs_, const std::true_type &)
+    {
+        charset_list rhs_;
+
+        fill_rhs_list(map_, rhs_);
+
+        if (!rhs_.empty())
+        {
+            typename charset_list::iterator iter_;
+            typename charset_list::iterator end_;
+            charset_ptr overlap_ = std::make_unique<charset>();
+
+            lhs_.emplace_back(std::move(rhs_.front()));
+            rhs_.pop_front();
+
+            while (!rhs_.empty())
+            {
+                charset_ptr r_(rhs_.front().release());
+
+                rhs_.pop_front();
+                iter_ = lhs_.begin();
+                end_ = lhs_.end();
+
+                while (!r_->empty() && iter_ != end_)
+                {
+                    auto l_iter_ = iter_;
+
+                    (*l_iter_)->intersect(*r_.get(), *overlap_.get());
+
+                    if (overlap_->empty())
+                    {
+                        ++iter_;
+                    }
+                    else if ((*l_iter_)->empty())
+                    {
+                        l_iter_->reset(overlap_.release());
+                        overlap_ = std::make_unique<charset>();
+                        ++iter_;
+                    }
+                    else if (r_->empty())
+                    {
+                        r_.reset(overlap_.release());
+                        overlap_ = std::make_unique<charset>();
+                        break;
+                    }
+                    else
+                    {
+                        iter_ = lhs_.insert(++iter_, charset_ptr());
+                        iter_->reset(overlap_.release());
+                        overlap_ = std::make_unique<charset>();
+                        ++iter_;
+                        end_ = lhs_.end();
+                    }
+                }
+
+                if (!r_->empty())
+                {
+                    lhs_.emplace_back(std::move(r_));
+                }
+            }
+        }
+    }
+
+    static void fill_rhs_list(const charset_map &map_, charset_list &list_)
+    {
+        for (const auto &pair_ : map_)
+        {
+            list_.emplace_back(std::make_unique<charset>
+                (pair_.first, pair_.second));
+        }
+    }
+
+    static void build_set_mapping(const charset_list &charset_list_,
+        internals &internals_, const id_type dfa_index_,
+        index_set_vector &set_mapping_)
+    {
+        auto iter_ = charset_list_.cbegin();
+        auto end_ = charset_list_.cend();
+
+        for (id_type index_ = 0; iter_ != end_; ++iter_, ++index_)
+        {
+            observer_ptr<const charset> cs_ = iter_->get();
+
+            fill_lookup(cs_->_token, &internals_._lookup[dfa_index_],
+                index_, lookup());
+
+            for (const id_type i_ : cs_->_index_set)
+            {
+                set_mapping_[i_].insert(index_);
+            }
+        }
+    }
+
+    // char_state_machine version
+    static void fill_lookup(const string_token &, observer_ptr<id_type_vector> ,
+        const id_type, const std::false_type &)
+    {
+        // Do nothing (lookup not used)
+    }
+
+    // state_machine version
+    static void fill_lookup(const string_token &charset_,
+        observer_ptr<id_type_vector> lookup_, const id_type index_,
+        const std::true_type &)
+    {
+        observer_ptr<id_type> ptr_ = &lookup_->front();
+
+        for (const auto &range_ : charset_._ranges)
+        {
+            for (typename char_traits::index_type char_ = range_.first;
+                char_ < range_.second; ++char_)
+            {
+                // Note char_ must be unsigned
+                ptr_[char_] = index_ + transitions_index;
+            }
+
+            // Note range_.second must be unsigned
+            ptr_[range_.second] = index_ + transitions_index;
+        }
+    }
+
+    static id_type closure(const node_vector &followpos_,
+        node_set_vector &seen_sets_, node_vector_vector &seen_vectors_,
+        size_t_vector &hash_vector_, const id_type size_, id_type_vector &dfa_)
+    {
+        bool end_state_ = false;
+        id_type id_ = 0;
+        id_type user_id_ = sm_traits::npos();
+        id_type next_dfa_ = 0;
+        id_type push_dfa_ = sm_traits::npos();
+        bool pop_dfa_ = false;
+        std::size_t hash_ = 0;
+
+        if (followpos_.empty()) return sm_traits::npos();
+
+        id_type index_ = 0;
+        std::unique_ptr<node_set> set_ptr_ = std::make_unique<node_set>();
+        std::unique_ptr<node_vector> vector_ptr_ =
+            std::make_unique<node_vector>();
+
+        for (observer_ptr<node> node_ : followpos_)
+        {
+            closure_ex(node_, end_state_, id_, user_id_, next_dfa_,
+                push_dfa_, pop_dfa_, *set_ptr_.get(),
+                *vector_ptr_.get(), hash_);
+        }
+
+        bool found_ = false;
+        auto hash_iter_ = hash_vector_.cbegin();
+        auto hash_end_ = hash_vector_.cend();
+        auto set_iter_ = seen_sets_.cbegin();
+
+        for (; hash_iter_ != hash_end_; ++hash_iter_, ++set_iter_)
+        {
+            found_ = *hash_iter_ == hash_ && *(*set_iter_) == *set_ptr_;
+            ++index_;
+
+            if (found_) break;
+        }
+
+        if (!found_)
+        {
+            seen_sets_.emplace_back(std::move(set_ptr_));
+            seen_vectors_.emplace_back(std::move(vector_ptr_));
+            hash_vector_.push_back(hash_);
+            // State 0 is the jam state...
+            index_ = static_cast<id_type>(seen_sets_.size());
+
+            const std::size_t old_size_ = dfa_.size();
+
+            dfa_.resize(old_size_ + size_, 0);
+
+            if (end_state_)
+            {
+                dfa_[old_size_] |= end_state_bit;
+
+                if (pop_dfa_)
+                {
+                    dfa_[old_size_] |= pop_dfa_bit;
+                }
+
+                dfa_[old_size_ + id_index] = id_;
+                dfa_[old_size_ + user_id_index] = user_id_;
+                dfa_[old_size_ + push_dfa_index] = push_dfa_;
+                dfa_[old_size_ + next_dfa_index] = next_dfa_;
+            }
+        }
+
+        return index_;
+    }
+
+    static void closure_ex(observer_ptr<node> node_, bool &end_state_,
+        id_type &id_, id_type &user_id_, id_type &next_dfa_,
+        id_type &push_dfa_, bool &pop_dfa_, node_set &set_ptr_,
+        node_vector &vector_ptr_, std::size_t &hash_)
+    {
+        const bool temp_end_state_ = node_->end_state();
+
+        if (temp_end_state_)
+        {
+            if (!end_state_)
+            {
+                end_state_ = true;
+                id_ = node_->id();
+                user_id_ = node_->user_id();
+                next_dfa_ = node_->next_dfa();
+                push_dfa_ = node_->push_dfa();
+                pop_dfa_ = node_->pop_dfa();
+            }
+        }
+
+        if (set_ptr_.insert(node_).second)
+        {
+            vector_ptr_.push_back(node_);
+            hash_ += reinterpret_cast<std::size_t>(node_);
+        }
+    }
+
+    // NFA version
+    static void build_equiv_list(const node_vector &vector_,
+        const index_set_vector &set_mapping_, equivset_list &lhs_,
+        const std::false_type &)
+    {
+        fill_rhs_list(vector_, set_mapping_, lhs_);
+    }
+
+    // DFA version
+    static void build_equiv_list(const node_vector &vector_,
+        const index_set_vector &set_mapping_, equivset_list &lhs_,
+        const std::true_type &)
+    {
+        equivset_list rhs_;
+
+        fill_rhs_list(vector_, set_mapping_, rhs_);
+
+        if (!rhs_.empty())
+        {
+            typename equivset_list::iterator iter_;
+            typename equivset_list::iterator end_;
+            equivset_ptr overlap_ = std::make_unique<equivset>();
+
+            lhs_.emplace_back(std::move(rhs_.front()));
+            rhs_.pop_front();
+
+            while (!rhs_.empty())
+            {
+                equivset_ptr r_(rhs_.front().release());
+
+                rhs_.pop_front();
+                iter_ = lhs_.begin();
+                end_ = lhs_.end();
+
+                while (!r_->empty() && iter_ != end_)
+                {
+                    auto l_iter_ = iter_;
+
+                    (*l_iter_)->intersect(*r_.get(), *overlap_.get());
+
+                    if (overlap_->empty())
+                    {
+                        ++iter_;
+                    }
+                    else if ((*l_iter_)->empty())
+                    {
+                        l_iter_->reset(overlap_.release());
+                        overlap_ = std::make_unique<equivset>();
+                        ++iter_;
+                    }
+                    else if (r_->empty())
+                    {
+                        r_.reset(overlap_.release());
+                        overlap_ = std::make_unique<equivset>();
+                        break;
+                    }
+                    else
+                    {
+                        iter_ = lhs_.insert(++iter_, equivset_ptr());
+                        iter_->reset(overlap_.release());
+                        overlap_ = std::make_unique<equivset>();
+                        ++iter_;
+                        end_ = lhs_.end();
+                    }
+                }
+
+                if (!r_->empty())
+                {
+                    lhs_.emplace_back(std::move(r_));
+                }
+            }
+        }
+    }
+
+    static void fill_rhs_list(const node_vector &vector_,
+        const index_set_vector &set_mapping_, equivset_list &list_)
+    {
+        for (observer_ptr<const node> node_ : vector_)
+        {
+            if (!node_->end_state())
+            {
+                const id_type token_ = node_->token();
+
+                if (token_ != node::null_token())
+                {
+                    if (token_ == parser::bol_token() ||
+                        token_ == parser::eol_token())
+                    {
+                        std::set<id_type> index_set_;
+
+                        index_set_.insert(token_);
+                        list_.emplace_back
+                            (std::make_unique<equivset>(index_set_,
+                                token_, node_->greedy(), node_->followpos()));
+                    }
+                    else
+                    {
+                        list_.emplace_back(std::make_unique<equivset>
+                            (set_mapping_[token_], token_, node_->greedy(),
+                                node_->followpos()));
+                    }
+                }
+            }
+        }
+    }
+};
+
+using generator = basic_generator<rules, state_machine>;
+using wgenerator = basic_generator<wrules, wstate_machine>;
+using u32generator = basic_generator<u32rules, u32state_machine>;
+using char_generator = basic_generator<rules, char_state_machine>;
+using wchar_generator = basic_generator<wrules, wchar_state_machine>;
+using u32char_generator = basic_generator<u32rules, u32char_state_machine>;
+}
+
+#endif