Implement TokenFilter and refactor Pl_QPDFTokenizer

Implement a TokenFilter class and refactor Pl_QPDFTokenizer to use a
TokenFilter class called ContentNormalizer. Pl_QPDFTokenizer is now a
general filter that passes data through a TokenFilter.

3 files changed, 43 insertions, 9 deletions

diff --git a/libqpdf/qpdf/ContentNormalizer.hh b/libqpdf/qpdf/ContentNormalizer.hh
new file mode 100644
index 00000000..504f15e8
--- /dev/null
+++ b/libqpdf/qpdf/ContentNormalizer.hh
@@ -0,0 +1,15 @@
+#ifndef __CONTENTNORMALIZER_HH__
+#define __CONTENTNORMALIZER_HH__
+
+#include <qpdf/QPDFObjectHandle.hh>
+
+class ContentNormalizer: public QPDFObjectHandle::TokenFilter
+{
+  public:
+    ContentNormalizer();
+    virtual ~ContentNormalizer();
+    virtual void handleToken(QPDFTokenizer::Token const&);
+    virtual void handleEOF();
+};
+
+#endif // __CONTENTNORMALIZER_HH__
diff --git a/libqpdf/qpdf/Pl_QPDFTokenizer.hh b/libqpdf/qpdf/Pl_QPDFTokenizer.hh
index 54507f68..9f4ac133 100644
--- a/libqpdf/qpdf/Pl_QPDFTokenizer.hh
+++ b/libqpdf/qpdf/Pl_QPDFTokenizer.hh
@@ -4,6 +4,8 @@
 #include <qpdf/Pipeline.hh>
 
 #include <qpdf/QPDFTokenizer.hh>
+#include <qpdf/PointerHolder.hh>
+#include <qpdf/QPDFObjectHandle.hh>
 
 //
 // Treat incoming text as a stream consisting of valid PDF tokens, but
@@ -16,7 +18,8 @@
 class Pl_QPDFTokenizer: public Pipeline
 {
   public:
-    Pl_QPDFTokenizer(char const* identifier, Pipeline* next);
+    Pl_QPDFTokenizer(char const* identifier,
+                     QPDFObjectHandle::TokenFilter* filter);
     virtual ~Pl_QPDFTokenizer();
     virtual void write(unsigned char* buf, size_t len);
     virtual void finish();
@@ -24,14 +27,25 @@ class Pl_QPDFTokenizer: public Pipeline
   private:
     void processChar(char ch);
     void checkUnread();
-    void writeNext(char const*, size_t len);
-    void writeToken(QPDFTokenizer::Token&);
-
-    QPDFTokenizer tokenizer;
-    bool just_wrote_nl;
-    bool last_char_was_cr;
-    bool unread_char;
-    char char_to_unread;
+
+    class Members
+    {
+        friend class Pl_QPDFTokenizer;
+
+      public:
+        ~Members();
+
+      private:
+        Members();
+        Members(Members const&);
+
+        QPDFObjectHandle::TokenFilter* filter;
+        QPDFTokenizer tokenizer;
+        bool last_char_was_cr;
+        bool unread_char;
+        char char_to_unread;
+    };
+    PointerHolder<Members> m;
 };
 
 #endif // __PL_QPDFTOKENIZER_HH__
diff --git a/libqpdf/qpdf/QPDF_Stream.hh b/libqpdf/qpdf/QPDF_Stream.hh
index 5350fc0d..86b796cf 100644
--- a/libqpdf/qpdf/QPDF_Stream.hh
+++ b/libqpdf/qpdf/QPDF_Stream.hh
@@ -20,6 +20,7 @@ class QPDF_Stream: public QPDFObject
     virtual QPDFObject::object_type_e getTypeCode() const;
     virtual char const* getTypeName() const;
     QPDFObjectHandle getDict() const;
+    bool isDataModified() const;
 
     // See comments in QPDFObjectHandle.hh for these methods.
     bool pipeStreamData(Pipeline*,
@@ -35,6 +36,8 @@ class QPDF_Stream: public QPDFObject
 	PointerHolder<QPDFObjectHandle::StreamDataProvider> provider,
 	QPDFObjectHandle const& filter,
 	QPDFObjectHandle const& decode_parms);
+    void addTokenFilter(
+        PointerHolder<QPDFObjectHandle::TokenFilter> token_filter);
 
     void replaceDict(QPDFObjectHandle new_dict);
 
@@ -72,6 +75,8 @@ class QPDF_Stream: public QPDFObject
     size_t length;
     PointerHolder<Buffer> stream_data;
     PointerHolder<QPDFObjectHandle::StreamDataProvider> stream_provider;
+    std::vector<
+        PointerHolder<QPDFObjectHandle::TokenFilter> > token_filters;
 };
 
 #endif // __QPDF_STREAM_HH__