Bug fix: honor encryption key length with R=3 (fixes #212)

4 files changed, 82 insertions, 5 deletions

diff --git a/libqpdf/QPDF_encryption.cc b/libqpdf/QPDF_encryption.cc
index fd717c35..54d7ad03 100644
--- a/libqpdf/QPDF_encryption.cc
+++ b/libqpdf/QPDF_encryption.cc
@@ -183,7 +183,7 @@ truncate_password_V5(std::string const& password)
 }
 
 static void
-iterate_md5_digest(MD5& md5, MD5::Digest& digest, int iterations)
+iterate_md5_digest(MD5& md5, MD5::Digest& digest, int iterations, int key_len)
 {
     md5.digest(digest);
 
@@ -191,7 +191,7 @@ iterate_md5_digest(MD5& md5, MD5::Digest& digest, int iterations)
     {
 	MD5 m;
 	m.encodeDataIncrementally(reinterpret_cast<char*>(digest),
-                                  sizeof(digest));
+                                  key_len);
 	m.digest(digest);
     }
 }
@@ -437,7 +437,8 @@ QPDF::compute_encryption_key_from_password(
 	md5.encodeDataIncrementally(bytes, 4);
     }
     MD5::Digest digest;
-    iterate_md5_digest(md5, digest, ((data.getR() >= 3) ? 50 : 0));
+    iterate_md5_digest(md5, digest, ((data.getR() >= 3) ? 50 : 0),
+                       data.getLengthBytes());
     return std::string(reinterpret_cast<char*>(digest),
                        std::min(static_cast<int>(sizeof(digest)),
                                 data.getLengthBytes()));
@@ -463,7 +464,8 @@ compute_O_rc4_key(std::string const& user_password,
     md5.encodeDataIncrementally(
 	pad_or_truncate_password_V4(password).c_str(), key_bytes);
     MD5::Digest digest;
-    iterate_md5_digest(md5, digest, ((data.getR() >= 3) ? 50 : 0));
+    iterate_md5_digest(md5, digest, ((data.getR() >= 3) ? 50 : 0),
+                       data.getLengthBytes());
     memcpy(key, digest, OU_key_bytes_V4);
 }
 
@@ -933,6 +935,11 @@ QPDF::initializeEncryption()
     if (encryption_dict.getKey("/Length").isInteger())
     {
 	Length = encryption_dict.getKey("/Length").getIntValue();
+        if (R < 3)
+        {
+            // Force Length to 40 regardless of what the file says.
+            Length = 40;
+        }
 	if ((Length % 8) || (Length < 40) || (Length > 256))
 	{
 	    throw QPDFExc(qpdf_e_damaged_pdf, this->m->file->getName(),
diff --git a/qpdf/qtest/qpdf.test b/qpdf/qtest/qpdf.test
index 6a71fab8..603bbe01 100644
--- a/qpdf/qtest/qpdf.test
+++ b/qpdf/qtest/qpdf.test
@@ -453,7 +453,7 @@ $td->runtest("check output",
 show_ntests();
 # ----------
 $td->notify("--- ID and Encryption Parameter Issues ---");
-$n_tests += 12;
+$n_tests += 13;
 
 # Encrypt files whose /ID strings are other than 32 bytes long (bug
 # 2991412). Also linearize these files, which was reported in a
@@ -509,6 +509,19 @@ $td->runtest("short /O or /U",
               $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 
+# A file was sent to me privately as part of issue 212. This file was
+# encrypted and had /R=3 and /V=1 and was using a 40-bit key. qpdf was
+# failing to work properly on files with /R=3 and 40-bit keys. The
+# test file is not this private file, but the encryption parameters
+# were copied from it. Like the bug file, qpdf < 8.1 can't decrypt it.
+$td->runtest("/R 3 with 40-bit key",
+             {$td->COMMAND =>
+                  "qpdf --password=623 --check --show-encryption-key" .
+                  " encrypted-40-bit-R3.pdf"},
+             {$td->FILE => "encrypted-40-bit-R3.out",
+              $td->EXIT_STATUS => 0},
+             $td->NORMALIZE_NEWLINES);
+
 show_ntests();
 # ----------
 $td->notify("--- Min/force version ---");
diff --git a/qpdf/qtest/qpdf/encrypted-40-bit-R3.out b/qpdf/qtest/qpdf/encrypted-40-bit-R3.out
new file mode 100644
index 00000000..c79f942c
--- /dev/null
+++ b/qpdf/qtest/qpdf/encrypted-40-bit-R3.out
@@ -0,0 +1,18 @@
+checking encrypted-40-bit-R3.pdf
+PDF Version: 1.4
+R = 3
+P = -12
+User password = 623
+Encryption key = e390e220da
+extract for accessibility: allowed
+extract for any purpose: allowed
+print low resolution: allowed
+print high resolution: allowed
+modify document assembly: allowed
+modify forms: allowed
+modify annotations: allowed
+modify other: not allowed
+modify anything: not allowed
+File is not linearized
+No syntax or stream encoding errors found; the file may still contain
+errors that qpdf cannot detect
diff --git a/qpdf/qtest/qpdf/encrypted-40-bit-R3.pdf b/qpdf/qtest/qpdf/encrypted-40-bit-R3.pdf
new file mode 100644
index 00000000..11cec537
--- /dev/null
+++ b/qpdf/qtest/qpdf/encrypted-40-bit-R3.pdf
@@ -0,0 +1,39 @@
+%PDF-1.4
+%¿÷¢þ
+1 0 obj
+<< /Pages 2 0 R /Type /Catalog >>
+endobj
+2 0 obj
+<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
+endobj
+3 0 obj
+<< /Contents 4 0 R /MediaBox [ 0 0 612 792 ] /Parent 2 0 R /Resources << /Font << /F1 5 0 R >> /ProcSet 6 0 R >> /Type /Page >>
+endobj
+4 0 obj
+<< /Length 48 /Filter /FlateDecode >>
+stream
+þ'½ÆpM¸€Ý§%'15Í@ªÿº•àøùji{WCèS0ÿÇŠZK¹¢üŠìkéendstream
+endobj
+5 0 obj
+<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
+endobj
+6 0 obj
+[ /PDF /Text ]
+endobj
+7 0 obj
+<< /Filter /Standard /Length 40 /O <d803dba08ac8e191d78988a674bd14b81801c4a2ac82de11525b69ac42f04589> /P -12 /R 3 /U <16a5f5d4a63a2cd3265524fa2248466028bf4e5e4e758a4164004e56fffa0108> /V 1 >>
+endobj
+xref
+0 8
+0000000000 65535 f 
+0000000015 00000 n 
+0000000064 00000 n 
+0000000123 00000 n 
+0000000266 00000 n 
+0000000384 00000 n 
+0000000491 00000 n 
+0000000521 00000 n 
+trailer << /Root 1 0 R /Size 8 /ID [<dd22115171e32b2c977f0d40d45c9972><f0f0abef5d2698f46ff6776dea099b27>] /Encrypt 7 0 R >>
+startxref
+728
+%%EOF