Add better tests for generation > 0 with object streams

This includes an expected failure for a file with a dangling
reference to an old generation.

6 files changed, 308 insertions, 44 deletions

diff --git a/qpdf/qtest/incremental.test b/qpdf/qtest/incremental.test
index db2a546d..b1edb918 100644
--- a/qpdf/qtest/incremental.test
+++ b/qpdf/qtest/incremental.test
@@ -18,7 +18,8 @@ my $n_tests = 6;
 
 # Since the beginning but discovered at the time of releasing 11.8.0:
 # qpdf doesn't delete earlier generations of an object when they are
-# reused.
+# reused. See also EXPECT_FAILURE in object-stream.test and
+# linearization.test.
 $td->runtest("handle delete and reuse",
              {$td->COMMAND => "qpdf --qdf --static-id incremental-1.pdf a.pdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0},
diff --git a/qpdf/qtest/linearization.test b/qpdf/qtest/linearization.test
index 553dfc3a..02cd778a 100644
--- a/qpdf/qtest/linearization.test
+++ b/qpdf/qtest/linearization.test
@@ -39,7 +39,8 @@ my @to_linearize =
      'lin-delete-and-reuse',    # linearized, then delete and reuse
      'object-stream',           # contains object streams
      'hybrid-xref',             # contains both xref tables and streams
-     'gen1',                    # has objects with generation > 0
+     'gen1',                    # has objects with generation > 0 and dangling references
+     'gen1-no-dangling',        # has objects with generation > 0
      'direct-outlines',         # /Outlines is a direct object
      @linearized_files,         # we should be able to relinearize
      );
@@ -83,23 +84,29 @@ foreach my $base (@to_linearize)
 {
     foreach my $omode (qw(disable preserve generate))
     {
+        my $xflags = 0;
+        if ($base eq 'gen1')
+        {
+            $xflags = $td->EXPECT_FAILURE;
+        }
         my $oarg = "-object-streams=$omode";
         my $sdarg = "";
         if (($base eq 'lin-special') || ($base eq 'object-stream'))
         {
             $sdarg = "--stream-data=uncompress";
         }
+        unlink "a.pdf", "b.pdf", "c.pdf";
         $td->runtest("linearize $base ($omode)",
                      {$td->COMMAND =>
                           "qpdf -linearize $oarg $sdarg" .
                           " --static-id $base.pdf a.pdf"},
-                     {$td->STRING => "",
-                      $td->EXIT_STATUS => 0});
+                     {$td->STRING => "", $td->EXIT_STATUS => 0},
+                     $xflags);
         $td->runtest("check linearization",
                      {$td->COMMAND => "qpdf --check-linearization a.pdf"},
                      {$td->STRING => "a.pdf: no linearization errors\n",
                       $td->EXIT_STATUS => 0},
-                     $td->NORMALIZE_NEWLINES);
+                     $td->NORMALIZE_NEWLINES | $xflags);
         # Relinearizing twice should produce identical results.  We
         # have to do it twice because, if objects changed ordering
         # during the original linearization, the hint tables won't
@@ -110,16 +117,17 @@ foreach my $base (@to_linearize)
         $td->runtest("relinearize $base 1",
                      {$td->COMMAND =>
                           "qpdf -linearize $sdarg --static-id a.pdf b.pdf"},
-                     {$td->STRING => "",
-                      $td->EXIT_STATUS => 0});
+                     {$td->STRING => "", $td->EXIT_STATUS => 0},
+                     $xflags);
         $td->runtest("relinearize $base 2",
                      {$td->COMMAND =>
                           "qpdf -linearize $sdarg --static-id b.pdf c.pdf"},
-                     {$td->STRING => "",
-                      $td->EXIT_STATUS => 0});
+                     {$td->STRING => "", $td->EXIT_STATUS => 0},
+                     $xflags);
         $td->runtest("compare files ($omode)",
                      {$td->FILE => "b.pdf"},
-                     {$td->FILE => "c.pdf"});
+                     {$td->FILE => "c.pdf"},
+                     $xflags);
         if (($base eq 'lin-special') || ($base eq 'object-stream'))
         {
             $td->runtest("check $base ($omode)",
diff --git a/qpdf/qtest/object-stream.test b/qpdf/qtest/object-stream.test
index 22b35af4..c9fa0664 100644
--- a/qpdf/qtest/object-stream.test
+++ b/qpdf/qtest/object-stream.test
@@ -16,7 +16,7 @@ cleanup();
 
 my $td = new TestDriver('object-stream');
 
-my $n_tests = 5 + (36 * 4) + (12 * 2);
+my $n_tests = 7 + (36 * 4) + (12 * 2);
 my $n_compare_pdfs = 36;
 
 for (my $n = 16; $n <= 19; ++$n)
@@ -82,8 +82,22 @@ $td->runtest("generate object streams for gen > 0",
              {$td->COMMAND => "qpdf --qdf --static-id" .
                   " --object-streams=generate gen1.pdf a.pdf"},
              {$td->STRING => "", $td->EXIT_STATUS => 0});
+# qpdf 11.8.0 -- it was discovered that qpdf was incorrectly handling
+# references to older generations of reused objects in incrementally
+# updated files.
 $td->runtest("check file",
              {$td->FILE => "a.pdf"},
+             {$td->FILE => "gen1.qdf"},
+             $td->EXPECT_FAILURE);
+
+$td->runtest("generate object streams for gen > 0",
+             {$td->COMMAND => "qpdf --qdf --static-id" .
+                  " --object-streams=generate gen1-no-dangling.pdf a.pdf"},
+             {$td->STRING => "", $td->EXIT_STATUS => 0});
+$td->runtest("check file",
+             {$td->FILE => "a.pdf"},
+             # Intentionally compare against gen1.pdf -- should have
+             # the same results as above.
              {$td->FILE => "gen1.qdf"});
 
 
diff --git a/qpdf/qtest/qpdf/gen1-no-dangling.pdf b/qpdf/qtest/qpdf/gen1-no-dangling.pdf
new file mode 100644
index 00000000..d03ee5f3
--- /dev/null
+++ b/qpdf/qtest/qpdf/gen1-no-dangling.pdf
@@ -0,0 +1,160 @@
+%PDF-1.3
+%¿÷¢þ
+%QDF-1.0
+
+1 0 obj
+<<
+  /Pages 2 0 R
+  /Type /Catalog
+>>
+endobj
+
+2 0 obj
+<<
+  /Count 1
+  /Kids [
+    3 0 R
+  ]
+  /Type /Pages
+>>
+endobj
+
+%% Page 1
+3 0 obj
+<<
+  /Contents 4 0 R
+  /MediaBox [
+    0
+    0
+    612
+    792
+  ]
+  /Parent 2 0 R
+  /Resources 6 0 R
+  /Type /Page
+>>
+endobj
+
+%% Contents for page 1
+4 0 obj
+<<
+  /Length 5 0 R
+>>
+stream
+BT
+  /F1 24 Tf
+  72 720 Td
+  (Potato) Tj
+ET
+endstream
+endobj
+
+5 0 obj
+44
+endobj
+
+6 0 obj
+<<
+  /Font 7 0 R
+  %ProcSet 8 0 R
+>>
+endobj
+
+7 0 obj
+<<
+  /F1 9 0 R
+>>
+endobj
+
+8 0 obj
+[
+  /PDF
+  /Text
+]
+endobj
+
+9 0 obj
+<<
+  /BaseFont /Helvetica
+  /Encoding /WinAnsiEncoding
+  /Name /F1
+  /Subtype /Type1
+  /Type /Font
+>>
+endobj
+
+xref
+0 10
+0000000000 65535 f 
+0000000025 00000 n 
+0000000079 00000 n 
+0000000161 00000 n 
+0000000319 00000 n 
+0000000418 00000 n 
+0000000437 00000 n 
+0000000490 00000 n 
+0000000524 00000 n 
+0000000559 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 10
+  /ID [<6b7a9692034323b8032e358b75d57a4c><44f7cee2c1913f0970a8a391cd03f327>]
+>>
+startxref
+677
+%%EOF
+
+% Delete object 8.
+xref
+0 1
+0000000008 65535 f 
+8 1
+0000000000 00001 f 
+trailer <<
+  /Root 1 0 R
+  /Size 10
+  /Prev 677
+  /ID [<6b7a9692034323b8032e358b75d57a4c><44f7cee2c1913f0970a8a391cd03f327>]
+>>
+startxref
+1043
+%%EOF
+
+% Reuse object 8 such that we have to traverse through it to get to a
+% compressible object.
+7 0 obj
+<<
+  /F1 8 1 R
+>>
+endobj
+
+8 1 obj
+<<
+  /BaseFont 9 0 R
+  /Encoding /WinAnsiEncoding
+  /Name /F1
+  /Subtype /Type1
+  /Type /Font
+>>
+endobj
+
+9 0 obj
+/Times-Roman
+endobj
+
+xref
+0 1
+0000000000 65535 f 
+7 3
+0000001339 00000 n 
+0000001373 00001 n 
+0000001486 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 10
+  /Prev 1043
+  /ID [<6b7a9692034323b8032e358b75d57a4c><44f7cee2c1913f0970a8a391cd03f327>]
+>>
+startxref
+1515
+%%EOF
diff --git a/qpdf/qtest/qpdf/gen1.pdf b/qpdf/qtest/qpdf/gen1.pdf
index 6bfbbefe..9800e5eb 100644
--- a/qpdf/qtest/qpdf/gen1.pdf
+++ b/qpdf/qtest/qpdf/gen1.pdf
@@ -1,39 +1,44 @@
 %PDF-1.3
-1 1 obj
+%¿÷¢þ
+%QDF-1.0
+
+1 0 obj
 <<
+  /Pages 2 0 R
   /Type /Catalog
-  /Pages 2 1 R
 >>
 endobj
 
-2 1 obj
+2 0 obj
 <<
-  /Type /Pages
+  /Count 1
   /Kids [
-    3 1 R
+    3 0 R
   ]
-  /Count 1
+  /Type /Pages
 >>
 endobj
 
-3 1 obj
+%% Page 1
+3 0 obj
 <<
+  /Contents 4 0 R
+  /MediaBox [
+    0
+    0
+    612
+    792
+  ]
+  /Parent 2 0 R
+  /Resources 6 0 R
   /Type /Page
-  /Parent 2 1 R
-  /MediaBox [0 0 612 792]
-  /Contents 4 1 R
-  /Resources <<
-    /ProcSet 5 1 R
-    /Font <<
-      /F1 6 1 R
-    >>
-  >>
 >>
 endobj
 
-4 1 obj
+%% Contents for page 1
+4 0 obj
 <<
-  /Length 44
+  /Length 5 0 R
 >>
 stream
 BT
@@ -44,36 +49,112 @@ ET
 endstream
 endobj
 
-5 1 obj
+5 0 obj
+44
+endobj
+
+6 0 obj
+<<
+  /Font 7 0 R
+  /ProcSet 8 0 R
+>>
+endobj
+
+7 0 obj
+<<
+  /F1 9 0 R
+>>
+endobj
+
+8 0 obj
 [
   /PDF
   /Text
 ]
 endobj
 
-6 1 obj
+9 0 obj
 <<
-  /Type /Font
-  /Subtype /Type1
-  /Name /F1
   /BaseFont /Helvetica
   /Encoding /WinAnsiEncoding
+  /Name /F1
+  /Subtype /Type1
+  /Type /Font
 >>
 endobj
 
 xref
-0 7
+0 10
+0000000000 65535 f 
+0000000025 00000 n 
+0000000079 00000 n 
+0000000161 00000 n 
+0000000319 00000 n 
+0000000418 00000 n 
+0000000437 00000 n 
+0000000490 00000 n 
+0000000524 00000 n 
+0000000559 00000 n 
+trailer <<
+  /Root 1 0 R
+  /Size 10
+  /ID [<6b7a9692034323b8032e358b75d57a4c><44f7cee2c1913f0970a8a391cd03f327>]
+>>
+startxref
+677
+%%EOF
+
+% Delete object 8.
+xref
+0 1
+0000000008 65535 f 
+8 1
+0000000000 00001 f 
+trailer <<
+  /Root 1 0 R
+  /Size 10
+  /Prev 677
+  /ID [<6b7a9692034323b8032e358b75d57a4c><44f7cee2c1913f0970a8a391cd03f327>]
+>>
+startxref
+1043
+%%EOF
+
+% Reuse object 8 such that we have to traverse through it to get to a
+% compressible object.
+7 0 obj
+<<
+  /F1 8 1 R
+>>
+endobj
+
+8 1 obj
+<<
+  /BaseFont 9 0 R
+  /Encoding /WinAnsiEncoding
+  /Name /F1
+  /Subtype /Type1
+  /Type /Font
+>>
+endobj
+
+9 0 obj
+/Times-Roman
+endobj
+
+xref
+0 1
 0000000000 65535 f 
-0000000009 00001 n 
-0000000063 00001 n 
-0000000135 00001 n 
-0000000307 00001 n 
-0000000403 00001 n 
-0000000438 00001 n 
+7 3
+0000001339 00000 n 
+0000001373 00001 n 
+0000001486 00000 n 
 trailer <<
-  /Size 7
-  /Root 1 1 R
+  /Root 1 0 R
+  /Size 10
+  /Prev 1043
+  /ID [<6b7a9692034323b8032e358b75d57a4c><44f7cee2c1913f0970a8a391cd03f327>]
 >>
 startxref
-556
+1515
 %%EOF
diff --git a/qpdf/qtest/qpdf/gen1.qdf b/qpdf/qtest/qpdf/gen1.qdf
index 802bf2bc..6409ec67 100644
--- a/qpdf/qtest/qpdf/gen1.qdf
+++ b/qpdf/qtest/qpdf/gen1.qdf