Implement TokenFilter and refactor Pl_QPDFTokenizer

Implement a TokenFilter class and refactor Pl_QPDFTokenizer to use a
TokenFilter class called ContentNormalizer. Pl_QPDFTokenizer is now a
general filter that passes data through a TokenFilter.

1 files changed, 31 insertions, 3 deletions

diff --git a/libqpdf/QPDF_Stream.cc b/libqpdf/QPDF_Stream.cc
index 89b6b5a9..a026f9a4 100644
--- a/libqpdf/QPDF_Stream.cc
+++ b/libqpdf/QPDF_Stream.cc
@@ -13,7 +13,7 @@
 #include <qpdf/Pl_RunLength.hh>
 #include <qpdf/Pl_DCT.hh>
 #include <qpdf/Pl_Count.hh>
-
+#include <qpdf/ContentNormalizer.hh>
 #include <qpdf/QTC.hh>
 #include <qpdf/QPDF.hh>
 #include <qpdf/QPDFExc.hh>
@@ -91,6 +91,12 @@ QPDF_Stream::getDict() const
     return this->stream_dict;
 }
 
+bool
+QPDF_Stream::isDataModified() const
+{
+    return (! this->token_filters.empty());
+}
+
 PointerHolder<Buffer>
 QPDF_Stream::getStreamData(qpdf_stream_decode_level_e decode_level)
 {
@@ -440,21 +446,36 @@ QPDF_Stream::pipeStreamData(Pipeline* pipeline,
     // create to be deleted when this function finishes.
     std::vector<PointerHolder<Pipeline> > to_delete;
 
+    PointerHolder<ContentNormalizer> normalizer;
     if (filter)
     {
 	if (encode_flags & qpdf_ef_compress)
 	{
-	    pipeline = new Pl_Flate("compress object stream", pipeline,
+	    pipeline = new Pl_Flate("compress stream", pipeline,
 				    Pl_Flate::a_deflate);
 	    to_delete.push_back(pipeline);
 	}
 
 	if (encode_flags & qpdf_ef_normalize)
 	{
-	    pipeline = new Pl_QPDFTokenizer("normalizer", pipeline);
+            normalizer = new ContentNormalizer();
+            normalizer->setPipeline(pipeline);
+	    pipeline = new Pl_QPDFTokenizer(
+                "normalizer", normalizer.getPointer());
 	    to_delete.push_back(pipeline);
 	}
 
+        for (std::vector<PointerHolder<
+                 QPDFObjectHandle::TokenFilter> >::reverse_iterator iter =
+                 this->token_filters.rbegin();
+             iter != this->token_filters.rend(); ++iter)
+        {
+            (*iter)->setPipeline(pipeline);
+            pipeline = new Pl_QPDFTokenizer(
+                "token filter", (*iter).getPointer());
+            to_delete.push_back(pipeline);
+        }
+
 	for (std::vector<std::string>::reverse_iterator iter = filters.rbegin();
 	     iter != filters.rend(); ++iter)
 	{
@@ -613,6 +634,13 @@ QPDF_Stream::replaceStreamData(
 }
 
 void
+QPDF_Stream::addTokenFilter(
+    PointerHolder<QPDFObjectHandle::TokenFilter> token_filter)
+{
+    this->token_filters.push_back(token_filter);
+}
+
+void
 QPDF_Stream::replaceFilterData(QPDFObjectHandle const& filter,
 			       QPDFObjectHandle const& decode_parms,
 			       size_t length)