Fix memory access error

A previous fix introduced a potentially memory overrun under certain
rare conditions. The test suite now once again passes with address
sanitizer.

1 files changed, 7 insertions, 7 deletions

diff --git a/libqpdf/QPDF_encryption.cc b/libqpdf/QPDF_encryption.cc
index 612a6204..b05e070b 100644
--- a/libqpdf/QPDF_encryption.cc
+++ b/libqpdf/QPDF_encryption.cc
@@ -437,11 +437,10 @@ QPDF::compute_encryption_key_from_password(
 	md5.encodeDataIncrementally(bytes, 4);
     }
     MD5::Digest digest;
-    iterate_md5_digest(md5, digest, ((data.getR() >= 3) ? 50 : 0),
-                       data.getLengthBytes());
-    return std::string(reinterpret_cast<char*>(digest),
-                       std::min(static_cast<int>(sizeof(digest)),
-                                data.getLengthBytes()));
+    int key_len = std::min(static_cast<int>(sizeof(digest)),
+                           data.getLengthBytes());
+    iterate_md5_digest(md5, digest, ((data.getR() >= 3) ? 50 : 0), key_len);
+    return std::string(reinterpret_cast<char*>(digest), key_len);
 }
 
 static void
@@ -464,8 +463,9 @@ compute_O_rc4_key(std::string const& user_password,
     md5.encodeDataIncrementally(
 	pad_or_truncate_password_V4(password).c_str(), key_bytes);
     MD5::Digest digest;
-    iterate_md5_digest(md5, digest, ((data.getR() >= 3) ? 50 : 0),
-                       data.getLengthBytes());
+    int key_len = std::min(static_cast<int>(sizeof(digest)),
+                           data.getLengthBytes());
+    iterate_md5_digest(md5, digest, ((data.getR() >= 3) ? 50 : 0), key_len);
     memcpy(key, digest, OU_key_bytes_V4);
 }