Track token position in AST Token node

spec/propane_spec.rb

@@ -1051,6 +1051,26 @@ EOF
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
+
+      it "stores the token position in the AST Token node" do
+        write_grammar <<EOF
+ast;
+
+token a;
+token b;
+token c;
+drop /\\s+/;
+Start -> T T T;
+T -> a;
+T -> b;
+T -> c;
+EOF
+        run_propane(language: language)
+        compile("spec/test_ast_token_positions.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/test_ast_token_positions.c

@@ -0,0 +1,33 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "abc";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert_eq(0, start->pT1->pToken->position.row);
+    assert_eq(0, start->pT1->pToken->position.col);
+    assert_eq(0, start->pT2->pToken->position.row);
+    assert_eq(1, start->pT2->pToken->position.col);
+    assert_eq(0, start->pT3->pToken->position.row);
+    assert_eq(2, start->pT3->pToken->position.col);
+
+    input = "\n\n  a\nc\n\n     a";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert_eq(2, start->pT1->pToken->position.row);
+    assert_eq(2, start->pT1->pToken->position.col);
+    assert_eq(3, start->pT2->pToken->position.row);
+    assert_eq(0, start->pT2->pToken->position.col);
+    assert_eq(5, start->pT3->pToken->position.row);
+    assert_eq(5, start->pT3->pToken->position.col);
+
+    return 0;
+}
+

spec/test_ast_token_positions.d

@@ -0,0 +1,34 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abc";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert_eq(0, start.pT1.pToken.position.row);
+    assert_eq(0, start.pT1.pToken.position.col);
+    assert_eq(0, start.pT2.pToken.position.row);
+    assert_eq(1, start.pT2.pToken.position.col);
+    assert_eq(0, start.pT3.pToken.position.row);
+    assert_eq(2, start.pT3.pToken.position.col);
+
+    input = "\n\n  a\nc\n\n     a";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert_eq(2, start.pT1.pToken.position.row);
+    assert_eq(2, start.pT1.pToken.position.col);
+    assert_eq(3, start.pT2.pToken.position.row);
+    assert_eq(0, start.pT2.pToken.position.col);
+    assert_eq(5, start.pT3.pToken.position.row);
+    assert_eq(5, start.pT3.pToken.position.col);
+}