Enable finer grained control of stream decoding

This commit adds several API methods that enable control over which
types of filters QPDF will attempt to decode. It also adds support for
/RunLengthDecode and /DCTDecode filters for both encoding and
decoding.

6 files changed, 166 insertions, 47 deletions

diff --git a/libqpdf/QPDF.cc b/libqpdf/QPDF.cc
index d1360b14..9c79fc3a 100644
--- a/libqpdf/QPDF.cc
+++ b/libqpdf/QPDF.cc
@@ -45,7 +45,7 @@ QPDF::CopiedStreamDataProvider::provideStreamData(
 {
     QPDFObjectHandle foreign_stream =
         this->foreign_streams[QPDFObjGen(objid, generation)];
-    foreign_stream.pipeStreamData(pipeline, false, false, false);
+    foreign_stream.pipeStreamData(pipeline, 0, qpdf_dl_none);
 }
 
 void
@@ -2377,6 +2377,7 @@ QPDF::pipeStreamData(int objid, int generation,
 	    length -= len;
 	    pipeline->write(QUtil::unsigned_char_pointer(buf), len);
 	}
+        pipeline->finish();
         success = true;
     }
     catch (QPDFExc& e)
@@ -2398,13 +2399,16 @@ QPDF::pipeStreamData(int objid, int generation,
                          QUtil::int_to_string(generation) + ": " + e.what()));
         }
     }
-    try
-    {
-        pipeline->finish();
-    }
-    catch (std::exception&)
+    if (! success)
     {
-        // ignore
+        try
+        {
+            pipeline->finish();
+        }
+        catch (std::exception&)
+        {
+            // ignore
+        }
     }
     return success;
 }
diff --git a/libqpdf/QPDFObjectHandle.cc b/libqpdf/QPDFObjectHandle.cc
index a8a7e5a7..105ecad9 100644
--- a/libqpdf/QPDFObjectHandle.cc
+++ b/libqpdf/QPDFObjectHandle.cc
@@ -482,10 +482,10 @@ QPDFObjectHandle::replaceDict(QPDFObjectHandle new_dict)
 }
 
 PointerHolder<Buffer>
-QPDFObjectHandle::getStreamData()
+QPDFObjectHandle::getStreamData(qpdf_stream_decode_level_e level)
 {
     assertStream();
-    return dynamic_cast<QPDF_Stream*>(obj.getPointer())->getStreamData();
+    return dynamic_cast<QPDF_Stream*>(obj.getPointer())->getStreamData(level);
 }
 
 PointerHolder<Buffer>
@@ -496,13 +496,35 @@ QPDFObjectHandle::getRawStreamData()
 }
 
 bool
-QPDFObjectHandle::pipeStreamData(Pipeline* p, bool filter,
-				 bool normalize, bool compress,
+QPDFObjectHandle::pipeStreamData(Pipeline* p,
+                                 unsigned long encode_flags,
+                                 qpdf_stream_decode_level_e decode_level,
                                  bool suppress_warnings)
 {
     assertStream();
     return dynamic_cast<QPDF_Stream*>(obj.getPointer())->pipeStreamData(
-	p, filter, normalize, compress, suppress_warnings);
+	p, encode_flags, decode_level, suppress_warnings);
+}
+
+bool
+QPDFObjectHandle::pipeStreamData(Pipeline* p, bool filter,
+				 bool normalize, bool compress)
+{
+    unsigned long encode_flags = 0;
+    qpdf_stream_decode_level_e decode_level = qpdf_dl_none;
+    if (filter)
+    {
+        decode_level = qpdf_dl_generalized;
+        if (normalize)
+        {
+            encode_flags |= qpdf_ef_normalize;
+        }
+        if (compress)
+        {
+            encode_flags |= qpdf_ef_compress;
+        }
+    }
+    return pipeStreamData(p, encode_flags, decode_level, false);
 }
 
 void
@@ -825,7 +847,7 @@ QPDFObjectHandle::parseContentStream(QPDFObjectHandle stream_or_array,
                 all_description += ",";
             }
             all_description += " " + og;
-            if (! stream.pipeStreamData(&buf, true, false, false, false))
+            if (! stream.pipeStreamData(&buf, 0, qpdf_dl_specialized))
             {
                 QTC::TC("qpdf", "QPDFObjectHandle errors in parsecontent");
                 warn(stream.getOwningQPDF(),
diff --git a/libqpdf/QPDFWriter.cc b/libqpdf/QPDFWriter.cc
index 345613ad..fe25853a 100644
--- a/libqpdf/QPDFWriter.cc
+++ b/libqpdf/QPDFWriter.cc
@@ -54,8 +54,10 @@ QPDFWriter::init()
     output_buffer = 0;
     normalize_content_set = false;
     normalize_content = false;
-    stream_data_mode_set = false;
-    stream_data_mode = qpdf_s_compress;
+    compress_streams = true;
+    compress_streams_set = false;
+    stream_decode_level = qpdf_dl_none;
+    stream_decode_level_set = false;
     qdf_mode = false;
     precheck_streams = false;
     preserve_unreferenced_objects = false;
@@ -162,8 +164,42 @@ QPDFWriter::setObjectStreamMode(qpdf_object_stream_e mode)
 void
 QPDFWriter::setStreamDataMode(qpdf_stream_data_e mode)
 {
-    this->stream_data_mode_set = true;
-    this->stream_data_mode = mode;
+    switch (mode)
+    {
+      case qpdf_s_uncompress:
+        this->stream_decode_level =
+            std::max(qpdf_dl_generalized, this->stream_decode_level);
+        this->compress_streams = false;
+        break;
+
+      case qpdf_s_preserve:
+        this->stream_decode_level = qpdf_dl_none;
+        this->compress_streams = false;
+        break;
+
+      case qpdf_s_compress:
+        this->stream_decode_level =
+            std::max(qpdf_dl_generalized, this->stream_decode_level);
+        this->compress_streams = true;
+        break;
+    }
+    this->stream_decode_level_set = true;
+    this->compress_streams_set = true;
+}
+
+
+void
+QPDFWriter::setCompressStreams(bool val)
+{
+    this->compress_streams = val;
+    this->compress_streams_set = true;
+}
+
+void
+QPDFWriter::setDecodeLevel(qpdf_stream_decode_level_e val)
+{
+    this->stream_decode_level = val;
+    this->stream_decode_level_set = true;
 }
 
 void
@@ -1512,8 +1548,8 @@ QPDFWriter::unparseObject(QPDFObjectHandle object, int level,
 	{
 	    is_metadata = true;
 	}
-	bool filter = (this->stream_data_mode != qpdf_s_preserve);
-	if (this->stream_data_mode == qpdf_s_compress)
+	bool filter = (this->compress_streams || this->stream_decode_level);
+	if (this->compress_streams)
 	{
 	    // Don't filter if the stream is already compressed with
 	    // FlateDecode.  We don't want to make it worse by getting
@@ -1532,19 +1568,21 @@ QPDFWriter::unparseObject(QPDFObjectHandle object, int level,
 	}
 	bool normalize = false;
 	bool compress = false;
+        bool uncompress = false;
 	if (is_metadata &&
 	    ((! this->encrypted) || (this->encrypt_metadata == false)))
 	{
 	    QTC::TC("qpdf", "QPDFWriter not compressing metadata");
 	    filter = true;
 	    compress = false;
+           uncompress = true;
 	}
 	else if (this->normalize_content && normalized_streams.count(old_og))
 	{
 	    normalize = true;
 	    filter = true;
 	}
-	else if (filter && (this->stream_data_mode == qpdf_s_compress))
+	else if (filter && this->compress_streams)
 	{
 	    compress = true;
 	    QTC::TC("qpdf", "QPDFWriter compressing uncompressed stream");
@@ -1559,7 +1597,7 @@ QPDFWriter::unparseObject(QPDFObjectHandle object, int level,
                 QTC::TC("qpdf", "QPDFWriter precheck stream");
                 Pl_Discard discard;
                 filter = object.pipeStreamData(
-                    &discard, true, false, false, true);
+                    &discard, 0, qpdf_dl_all, true);
             }
             catch (std::exception&)
             {
@@ -1569,8 +1607,15 @@ QPDFWriter::unparseObject(QPDFObjectHandle object, int level,
 
 	pushPipeline(new Pl_Buffer("stream data"));
 	activatePipelineStack();
+
 	bool filtered =
-	    object.pipeStreamData(this->pipeline, filter, normalize, compress);
+	    object.pipeStreamData(
+                this->pipeline,
+                (((filter && normalize) ? qpdf_ef_normalize : 0) |
+                 ((filter && compress) ? qpdf_ef_compress : 0)),
+                (filter
+                 ? (uncompress ? qpdf_dl_all : this->stream_decode_level)
+                 : qpdf_dl_none));
 	PointerHolder<Buffer> stream_data;
 	popPipelineStack(&stream_data);
 	if (filtered)
@@ -1717,8 +1762,7 @@ QPDFWriter::writeObjectStream(QPDFObjectHandle object)
 
 	    // Set up a stream to write the stream data into a buffer.
 	    Pipeline* next = pushPipeline(new Pl_Buffer("object stream"));
-	    if (! ((this->stream_data_mode == qpdf_s_uncompress) ||
-		   this->qdf_mode))
+            if (! (this->stream_decode_level || this->qdf_mode))
 	    {
 		compressed = true;
 		next = pushPipeline(
@@ -2180,7 +2224,8 @@ QPDFWriter::prepareFileForWrite()
                 is_stream = true;
 		dict = node.getDict();
                 // See whether we are able to filter this stream.
-                filterable = node.pipeStreamData(0, true, false, false);
+                filterable = node.pipeStreamData(
+                    0, 0, this->stream_decode_level, true);
 	    }
             else if (pdf.getRoot().getObjectID() == node.getObjectID())
             {
@@ -2260,10 +2305,14 @@ QPDFWriter::write()
 	{
 	    this->normalize_content = true;
 	}
-	if (! this->stream_data_mode_set)
+	if (! this->compress_streams_set)
 	{
-	    this->stream_data_mode = qpdf_s_uncompress;
+	    this->compress_streams = false;
 	}
+        if (! this->stream_decode_level_set)
+        {
+            this->stream_decode_level = qpdf_dl_generalized;
+        }
     }
 
     if (this->encrypted)
@@ -2272,7 +2321,7 @@ QPDFWriter::write()
 	this->preserve_encryption = false;
     }
     else if (this->normalize_content ||
-	     (this->stream_data_mode == qpdf_s_uncompress) ||
+	     this->stream_decode_level ||
 	     this->qdf_mode)
     {
 	// Encryption makes looking at contents pretty useless.  If
@@ -2300,7 +2349,7 @@ QPDFWriter::write()
     }
 
     if (this->qdf_mode || this->normalize_content ||
-	(this->stream_data_mode == qpdf_s_uncompress))
+        this->stream_decode_level)
     {
 	initializeSpecialStreams();
     }
@@ -2586,7 +2635,7 @@ QPDFWriter::writeXRefStream(int xref_id, int max_id, qpdf_offset_t max_offset,
 
     Pipeline* p = pushPipeline(new Pl_Buffer("xref stream"));
     bool compressed = false;
-    if (! ((this->stream_data_mode == qpdf_s_uncompress) || this->qdf_mode))
+    if (! (this->stream_decode_level || this->qdf_mode))
     {
 	compressed = true;
 	if (! skip_compression)
diff --git a/libqpdf/QPDF_Stream.cc b/libqpdf/QPDF_Stream.cc
index 31d583b8..bcf9be92 100644
--- a/libqpdf/QPDF_Stream.cc
+++ b/libqpdf/QPDF_Stream.cc
@@ -9,6 +9,8 @@
 #include <qpdf/Pl_ASCII85Decoder.hh>
 #include <qpdf/Pl_ASCIIHexDecoder.hh>
 #include <qpdf/Pl_LZWDecoder.hh>
+#include <qpdf/Pl_RunLength.hh>
+#include <qpdf/Pl_DCT.hh>
 #include <qpdf/Pl_Count.hh>
 
 #include <qpdf/QTC.hh>
@@ -82,10 +84,10 @@ QPDF_Stream::getDict() const
 }
 
 PointerHolder<Buffer>
-QPDF_Stream::getStreamData()
+QPDF_Stream::getStreamData(qpdf_stream_decode_level_e decode_level)
 {
     Pl_Buffer buf("stream data buffer");
-    if (! pipeStreamData(&buf, true, false, false, false))
+    if (! pipeStreamData(&buf, 0, decode_level, false))
     {
 	throw std::logic_error("getStreamData called on unfilterable stream");
     }
@@ -97,7 +99,7 @@ PointerHolder<Buffer>
 QPDF_Stream::getRawStreamData()
 {
     Pl_Buffer buf("stream data buffer");
-    pipeStreamData(&buf, false, false, false, false);
+    pipeStreamData(&buf, 0, qpdf_dl_none, false);
     QTC::TC("qpdf", "QPDF_Stream getRawStreamData");
     return buf.getBuffer();
 }
@@ -178,6 +180,8 @@ QPDF_Stream::understandDecodeParams(
 
 bool
 QPDF_Stream::filterable(std::vector<std::string>& filters,
+                        bool& specialized_compression,
+                        bool& lossy_compression,
 			int& predictor, int& columns,
 			bool& early_code_change)
 {
@@ -254,11 +258,20 @@ QPDF_Stream::filterable(std::vector<std::string>& filters,
 	    filter = filter_abbreviations[filter];
 	}
 
-	if (! ((filter == "/Crypt") ||
-	       (filter == "/FlateDecode") ||
-	       (filter == "/LZWDecode") ||
-	       (filter == "/ASCII85Decode") ||
-	       (filter == "/ASCIIHexDecode")))
+        if (filter == "/RunLengthDecode")
+        {
+            specialized_compression = true;
+        }
+        else if (filter == "/DCTDecode")
+        {
+            specialized_compression = true;
+            lossy_compression = true;
+        }
+	else if (! ((filter == "/Crypt") ||
+                    (filter == "/FlateDecode") ||
+                    (filter == "/LZWDecode") ||
+                    (filter == "/ASCII85Decode") ||
+                    (filter == "/ASCIIHexDecode")))
 	{
 	    filterable = false;
 	}
@@ -350,17 +363,35 @@ QPDF_Stream::filterable(std::vector<std::string>& filters,
 }
 
 bool
-QPDF_Stream::pipeStreamData(Pipeline* pipeline, bool filter,
-			    bool normalize, bool compress,
+QPDF_Stream::pipeStreamData(Pipeline* pipeline,
+                            unsigned long encode_flags,
+                            qpdf_stream_decode_level_e decode_level,
                             bool suppress_warnings)
 {
     std::vector<std::string> filters;
     int predictor = 1;
     int columns = 0;
     bool early_code_change = true;
+    bool specialized_compression = false;
+    bool lossy_compression = false;
+    bool filter = (! ((encode_flags == 0) && (decode_level == qpdf_dl_none)));
     if (filter)
     {
-	filter = filterable(filters, predictor, columns, early_code_change);
+	filter = filterable(filters, specialized_compression, lossy_compression,
+                            predictor, columns, early_code_change);
+        if ((decode_level < qpdf_dl_all) && lossy_compression)
+        {
+            filter = false;
+        }
+        if ((decode_level < qpdf_dl_specialized) && specialized_compression)
+        {
+            filter = false;
+        }
+        QTC::TC("qpdf", "QPDF_Stream special filters",
+                (! filter) ? 0 :
+                lossy_compression ? 1 :
+                specialized_compression ? 2 :
+                3);
     }
 
     if (pipeline == 0)
@@ -375,14 +406,14 @@ QPDF_Stream::pipeStreamData(Pipeline* pipeline, bool filter,
 
     if (filter)
     {
-	if (compress)
+	if (encode_flags & qpdf_ef_compress)
 	{
 	    pipeline = new Pl_Flate("compress object stream", pipeline,
 				    Pl_Flate::a_deflate);
 	    to_delete.push_back(pipeline);
 	}
 
-	if (normalize)
+	if (encode_flags & qpdf_ef_normalize)
 	{
 	    pipeline = new Pl_QPDFTokenizer("normalizer", pipeline);
 	    to_delete.push_back(pipeline);
@@ -427,6 +458,17 @@ QPDF_Stream::pipeStreamData(Pipeline* pipeline, bool filter,
 					     early_code_change);
 		to_delete.push_back(pipeline);
 	    }
+	    else if (filter == "/RunLengthDecode")
+	    {
+		pipeline = new Pl_RunLength("runlength decode", pipeline,
+                                            Pl_RunLength::a_decode);
+		to_delete.push_back(pipeline);
+	    }
+	    else if (filter == "/DCTDecode")
+	    {
+		pipeline = new Pl_DCT("DCT decode", pipeline);
+		to_delete.push_back(pipeline);
+	    }
 	    else
 	    {
 		throw std::logic_error(
diff --git a/libqpdf/QPDF_linearization.cc b/libqpdf/QPDF_linearization.cc
index 424d6d6f..b05b1d4c 100644
--- a/libqpdf/QPDF_linearization.cc
+++ b/libqpdf/QPDF_linearization.cc
@@ -393,7 +393,7 @@ QPDF::readHintStream(Pipeline& pl, qpdf_offset_t offset, size_t length)
 		      this->file->getLastOffset(),
 		      "hint table length mismatch");
     }
-    H.pipeStreamData(&pl, true, false, false);
+    H.pipeStreamData(&pl, 0, qpdf_dl_specialized);
     return Hdict;
 }
 
diff --git a/libqpdf/qpdf/QPDF_Stream.hh b/libqpdf/qpdf/QPDF_Stream.hh
index d053fd0f..8b960f00 100644
--- a/libqpdf/qpdf/QPDF_Stream.hh
+++ b/libqpdf/qpdf/QPDF_Stream.hh
@@ -22,10 +22,11 @@ class QPDF_Stream: public QPDFObject
     QPDFObjectHandle getDict() const;
 
     // See comments in QPDFObjectHandle.hh for these methods.
-    bool pipeStreamData(Pipeline*, bool filter,
-			bool normalize, bool compress,
+    bool pipeStreamData(Pipeline*,
+                        unsigned long encode_flags,
+                        qpdf_stream_decode_level_e decode_level,
                         bool suppress_warnings);
-    PointerHolder<Buffer> getStreamData();
+    PointerHolder<Buffer> getStreamData(qpdf_stream_decode_level_e);
     PointerHolder<Buffer> getRawStreamData();
     void replaceStreamData(PointerHolder<Buffer> data,
 			   QPDFObjectHandle const& filter,
@@ -52,6 +53,7 @@ class QPDF_Stream: public QPDFObject
         std::string const& filter, QPDFObjectHandle decode_params,
         int& predictor, int& columns, bool& early_code_change);
     bool filterable(std::vector<std::string>& filters,
+                    bool& specialized_compression, bool& lossy_compression,
 		    int& predictor, int& columns, bool& early_code_change);
     void warn(QPDFExc const& e);