Implement TokenFilter and refactor Pl_QPDFTokenizer

Implement a TokenFilter class and refactor Pl_QPDFTokenizer to use a
TokenFilter class called ContentNormalizer. Pl_QPDFTokenizer is now a
general filter that passes data through a TokenFilter.

1 files changed, 18 insertions, 0 deletions

diff --git a/libqpdf/QPDFTokenizer.cc b/libqpdf/QPDFTokenizer.cc
index 078b1af0..c3a017d0 100644
--- a/libqpdf/QPDFTokenizer.cc
+++ b/libqpdf/QPDFTokenizer.cc
@@ -7,6 +7,7 @@
 #include <qpdf/QTC.hh>
 #include <qpdf/QPDFExc.hh>
 #include <qpdf/QUtil.hh>
+#include <qpdf/QPDFObjectHandle.hh>
 
 #include <stdexcept>
 #include <string.h>
@@ -39,6 +40,23 @@ QPDFTokenizer::Members::~Members()
 {
 }
 
+QPDFTokenizer::Token::Token(token_type_e type, std::string const& value) :
+    type(type),
+    value(value),
+    raw_value(value)
+{
+    if (type == tt_string)
+    {
+        raw_value = QPDFObjectHandle::newString(value).unparse();
+    }
+    else if (type == tt_string)
+    {
+        raw_value = QPDFObjectHandle::newName(value).unparse();
+    }
+}
+
+
+
 QPDFTokenizer::QPDFTokenizer() :
     m(new Members())
 {