added Token::m_type/setType()/getType(); changed Token::process() to accept a const reference to a Matches; updated build test

TokenDefinition.cc

@@ -103,7 +103,7 @@ string TokenDefinition::getClassDefinition() const
     ret += "public:\n";
     if (m_process)
     {
-        ret += "    virtual void process(Matches matches);\n";
+        ret += "    virtual void process(const Matches & matches);\n";
     }
     ret += "protected:\n";
     ret += m_data + "\n";
@@ -116,7 +116,7 @@ string TokenDefinition::getProcessMethod() const
     string ret;
     if (m_code != "")
     {
-        ret += "void " + getClassName() + "::process(Matches matches) {\n";
+        ret += "void " + getClassName() + "::process(const Matches & matches) {\n";
         ret += m_code + "\n";
         ret += "}\n";
     }

tests/build/itest.I

@@ -6,11 +6,11 @@ OR          or
 NOT         not
 LPAREN      \(
 RPAREN      \)
-WS          \s+ %{
+WS          \s+
-    cout << "Hi there WS!!!!!" << endl;
+EQUALS      = %{ cout << "Saw '='" << endl; %}
+IDENTIFIER  [a-zA-Z_][a-zA-Z_0-9]* %{
+    cout << "Identify: '" << matches[0] << "'" << endl;
 %}
-EQUALS      =
-IDENTIFIER  [a-zA-Z_][a-zA-Z_0-9]*
 
 DEC_INT     [1-9]\d*\b
 ${
@@ -18,13 +18,16 @@ ${
 $}
 %{
     sscanf("%lld", matches[1].c_str(), &value);
+    cout << "value: " << value << endl;
 %}
 
 HEX_INT     0x([0-9a-fA-F]+)\b ${ uint64_t value; $} %{
     sscanf("%llx", matches[1].c_str(), &value);
+    cout << "value: " << value << endl;
 %}
 
 OCT_INT     0([0-7]*)\b
+BIN_INT     0b([01]+)\b
 
 [rules]
 

tests/build/main.cc

@@ -10,7 +10,8 @@ int main(int argc, char * argv[])
 {
     Parser p;
     stringstream t(string(
-                "hi there (one and two and three and four) or (two = nine)"
+                "hi there (one and two and three and four) or (two = nine)\n"
+                "0x42 12345 0 011 0b0011\n"
                 ));
     p.parse(t);
 }

tmpl/parser.cc

@@ -25,6 +25,10 @@ static TokenRef buildToken(int typeindex)
     {
         {%buildToken%}
     }
+    if (!token.isNull())
+    {
+        token->setType(typeindex);
+    }
     return token;
 }
 
@@ -119,19 +123,22 @@ bool I_CLASSNAME::parse(istream & i)
                 }
             }
         }
-        if (longest_match_index >= 0)
+        if (longest_match_index < 0)
-        {
-            Matches matches(tokens[longest_match_index].re,
-                    &buff[0], ovector, ovector_size);
-            TokenRef token = buildToken(longest_match_index);
-            buff_pos += longest_match_length;
-        }
-        else
         {
             /* no pattern matched the input at the current position */
             cerr << "Parse error" << endl;
             return false;
         }
+        Matches matches(tokens[longest_match_index].re,
+                &buff[0], ovector, ovector_size);
+        TokenRef token = buildToken(longest_match_index);
+        if (token.isNull())
+        {
+            cerr << "Internal Error: null token" << endl;
+            return false;
+        }
+        token->process(matches);
+        buff_pos += longest_match_length;
     }
 }
 
@@ -149,7 +156,7 @@ refptr<Node> Node::operator[](const std::string & index)
         : NULL;
 }
 
-void Token::process(Matches matches)
+void Token::process(const Matches & matches)
 {
     {%token_code%}
 }
@@ -159,7 +166,7 @@ Matches::Matches(pcre * re, const char * data, int * ovector, int ovec_size)
 {
 }
 
-std::string Matches::operator[](int index)
+std::string Matches::operator[](int index) const
 {
     if (0 <= index && index < (m_ovec_size / 3))
     {
@@ -173,7 +180,7 @@ std::string Matches::operator[](int index)
     return "";
 }
 
-std::string Matches::operator[](const std::string & index)
+std::string Matches::operator[](const std::string & index) const
 {
     int idx = pcre_get_stringnumber(m_re, index.c_str());
     if (idx > 0 && idx < (m_ovec_size / 3))

tmpl/parser.h

@@ -134,8 +134,8 @@ class Matches
 {
     public:
         Matches(pcre * re, const char * data, int * ovector, int ovec_size);
-        std::string operator[](int index);
+        std::string operator[](int index) const;
-        std::string operator[](const std::string & index);
+        std::string operator[](const std::string & index) const;
 
     protected:
         pcre * m_re;
@@ -159,9 +159,12 @@ typedef refptr<Node> NodeRef;
 class Token : public Node
 {
     public:
-        virtual void process(Matches matches);
+        virtual void process(const Matches & matches);
+        void setType(int type) { m_type = type; }
+        int getType() const { return m_type; }
 
     protected:
+        int m_type;
         {%token_data%}
 };
 typedef refptr<Token> TokenRef;