Convert object parsing errors to warnings

QPDFObjectHandle::parseInternal now issues warnings instead of
throwing exceptions for all error conditions that it finds (except
internal logic errors) and has stronger recovery for things like
invalid tokens and malformed dictionaries. This should improve qpdf's
ability to recover from a wide range of broken files that currently
cause it to fail.

20 files changed, 221 insertions, 72 deletions

diff --git a/qpdf/qpdf.testcov b/qpdf/qpdf.testcov
index 8144d2c3..20859984 100644
--- a/qpdf/qpdf.testcov
+++ b/qpdf/qpdf.testcov
@@ -61,7 +61,6 @@ QPDF trailer size not integer 0
 QPDF trailer prev not integer 0
 QPDFObjectHandle bad brace 0
 QPDFObjectHandle bad array close 0
-QPDFObjectHandle dictionary odd number of elements 0
 QPDF stream without length 0
 QPDF stream length not integer 0
 QPDF missing endstream 0
@@ -124,7 +123,7 @@ qpdf-c qpdf_next_warning returned warning 0
 qpdf-c called qpdf_set_suppress_warnings 0
 qpdf-c called qpdf_set_ignore_xref_streams 0
 qpdf-c called qpdf_set_attempt_recovery 0
-qpdf-c called qpdf_read 3
+qpdf-c called qpdf_read 2
 qpdf-c called qpdf_get_pdf_version 0
 qpdf-c called qpdf_get_user_password 0
 qpdf-c called qpdf_is_linearized 0
@@ -275,5 +274,7 @@ QPDFWriter deterministic with no data 0
 qpdf-c called qpdf_set_deterministic_ID 0
 QPDFObjectHandle indirect with 0 objid 0
 QPDF object id 0 0
-QPDF caught recursive xref reconstruction 0
 QPDF recursion loop in resolve 0
+QPDFObjectHandle treat word as string 0
+QPDFObjectHandle found fake 1
+QPDFObjectHandle no val for last key 0
diff --git a/qpdf/qtest/qpdf.test b/qpdf/qtest/qpdf.test
index 242ee149..d69ccdc2 100644
--- a/qpdf/qtest/qpdf.test
+++ b/qpdf/qtest/qpdf.test
@@ -220,22 +220,22 @@ $td->runtest("C API: qpdf version",
 
 # Files to reproduce various bugs
 foreach my $d (
-    ["51", "resolve loop"],
-    ["99", "object 0"],
-    ["99b", "object 0"],
-    ["100","xref reconstruction loop"],
-    ["101", "resolve for exception text"],
-    ["117", "other infinite loop"],
-    ["118", "other infinite loop"],
-    ["119", "other infinite loop"],
-    ["120", "other infinite loop"],
+    ["51", "resolve loop", 2],
+    ["99", "object 0", 2],
+    ["99b", "object 0", 2],
+    ["100", "xref reconstruction loop", 2],
+    ["101", "resolve for exception text", 2],
+    ["117", "other infinite loop", 2],
+    ["118", "other infinite loop", 2],
+    ["119", "other infinite loop", 3],
+    ["120", "other infinite loop", 2],
     )
 {
-    my ($n, $description) = @$d;
+    my ($n, $description, $exit_status) = @$d;
     $td->runtest($description,
                  {$td->COMMAND => "qpdf issue-$n.pdf a.pdf"},
                  {$td->FILE => "issue-$n.out",
-                  $td->EXIT_STATUS => 2},
+                  $td->EXIT_STATUS => $exit_status},
                  $td->NORMALIZE_NEWLINES);
 }
 
@@ -593,7 +593,7 @@ $td->runtest("no type key for page nodes",
              $td->NORMALIZE_NEWLINES);
 $td->runtest("ensure arguments to R are direct",
              {$td->COMMAND => "qpdf --check indirect-r-arg.pdf"},
-             {$td->FILE => "indirect-r-arg.out", $td->EXIT_STATUS => 2},
+             {$td->FILE => "indirect-r-arg.out", $td->EXIT_STATUS => 3},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("detect loops in pages structure",
              {$td->COMMAND => "qpdf --check pages-loop.pdf"},
@@ -784,16 +784,19 @@ my @badfiles = ("not a PDF file", 			# 1
 		"invalid stream /Filter and xref",	# 33
 		"obj/gen in wrong place",		# 34
 		"object stream of wrong type",		# 35
+                "bad dictionary key",                   # 36
 		);
 
-$n_tests += @badfiles + 5;
+$n_tests += @badfiles + 4;
 
 # Test 6 contains errors in the free table consistency, but we no
 # longer have any consistency check for this since it is not important
 # neither Acrobat nor other PDF viewers really care.  Tests 12 and 28
 # have error conditions that used to be fatal but are now considered
 # non-fatal.
-my %badtest_overrides = (6 => 0, 12 => 0, 28 => 0, 31 => 0);
+my %badtest_overrides = (6 => 0, 12 => 0, 13 => 0,
+                         14 => 0, 15 => 0, 17 => 0,
+                         28 => 0, 31 => 0, 36 => 0);
 for (my $i = 1; $i <= scalar(@badfiles); ++$i)
 {
     my $status = $badtest_overrides{$i};
@@ -810,11 +813,6 @@ $td->runtest("C API: errors",
 	     {$td->FILE => "c-read-errors.out",
 	      $td->EXIT_STATUS => 0},
 	     $td->NORMALIZE_NEWLINES);
-$td->runtest("C API: warnings and errors",
-	     {$td->COMMAND => "qpdf-ctest 2 bad17.pdf '' a.pdf"},
-	     {$td->FILE => "c-read-warnings-and-errors.out",
-	      $td->EXIT_STATUS => 0},
-	     $td->NORMALIZE_NEWLINES);
 $td->runtest("C API: errors writing",
 	     {$td->COMMAND => "qpdf-ctest 2 bad30.pdf '' a.pdf"},
 	     {$td->FILE => "c-write-errors.out",
@@ -842,7 +840,7 @@ $n_tests += @badfiles + 8;
 # though in some cases it may.  Acrobat Reader would not be able to
 # recover any of these files any better.
 my %recover_failures = ();
-for (1, 7, 13..21, 24, 29..30, 33, 35)
+for (1, 7, 16, 18..21, 24, 29..30, 33, 35)
 {
     $recover_failures{$_} = 1;
 }
diff --git a/qpdf/qtest/qpdf/bad13-recover.out b/qpdf/qtest/qpdf/bad13-recover.out
index c20d98a7..862fac7b 100644
--- a/qpdf/qtest/qpdf/bad13-recover.out
+++ b/qpdf/qtest/qpdf/bad13-recover.out
@@ -1,4 +1,7 @@
-WARNING: bad13.pdf: file is damaged
-WARNING: bad13.pdf (trailer, file position 753): unexpected brace token
-WARNING: bad13.pdf: Attempting to reconstruct cross-reference table
-bad13.pdf (trailer, file position 753): unexpected brace token
+WARNING: bad13.pdf (trailer, file position 753): treating unexpected brace token as null
+/QTest is implicit
+/QTest is direct and has type null (2)
+/QTest is null
+unparse: null
+unparseResolved: null
+test 1 done
diff --git a/qpdf/qtest/qpdf/bad13.out b/qpdf/qtest/qpdf/bad13.out
index 04c34e36..da69c50c 100644
--- a/qpdf/qtest/qpdf/bad13.out
+++ b/qpdf/qtest/qpdf/bad13.out
@@ -1 +1,7 @@
-bad13.pdf (trailer, file position 753): unexpected brace token
+WARNING: bad13.pdf (trailer, file position 753): treating unexpected brace token as null
+/QTest is implicit
+/QTest is direct and has type null (2)
+/QTest is null
+unparse: null
+unparseResolved: null
+test 0 done
diff --git a/qpdf/qtest/qpdf/bad14-recover.out b/qpdf/qtest/qpdf/bad14-recover.out
index e66441a8..7cc59a67 100644
--- a/qpdf/qtest/qpdf/bad14-recover.out
+++ b/qpdf/qtest/qpdf/bad14-recover.out
@@ -1,4 +1,7 @@
-WARNING: bad14.pdf: file is damaged
-WARNING: bad14.pdf (trailer, file position 753): unexpected brace token
-WARNING: bad14.pdf: Attempting to reconstruct cross-reference table
-bad14.pdf (trailer, file position 753): unexpected brace token
+WARNING: bad14.pdf (trailer, file position 753): treating unexpected brace token as null
+/QTest is implicit
+/QTest is direct and has type null (2)
+/QTest is null
+unparse: null
+unparseResolved: null
+test 1 done
diff --git a/qpdf/qtest/qpdf/bad14.out b/qpdf/qtest/qpdf/bad14.out
index 6d32b70c..cbb403ba 100644
--- a/qpdf/qtest/qpdf/bad14.out
+++ b/qpdf/qtest/qpdf/bad14.out
@@ -1 +1,7 @@
-bad14.pdf (trailer, file position 753): unexpected brace token
+WARNING: bad14.pdf (trailer, file position 753): treating unexpected brace token as null
+/QTest is implicit
+/QTest is direct and has type null (2)
+/QTest is null
+unparse: null
+unparseResolved: null
+test 0 done
diff --git a/qpdf/qtest/qpdf/bad15-recover.out b/qpdf/qtest/qpdf/bad15-recover.out
index 0575a338..d1ce2b90 100644
--- a/qpdf/qtest/qpdf/bad15-recover.out
+++ b/qpdf/qtest/qpdf/bad15-recover.out
@@ -1,4 +1,7 @@
-WARNING: bad15.pdf: file is damaged
-WARNING: bad15.pdf (trailer, file position 753): unexpected array close token
-WARNING: bad15.pdf: Attempting to reconstruct cross-reference table
-bad15.pdf (trailer, file position 753): unexpected array close token
+WARNING: bad15.pdf (trailer, file position 753): treating unexpected array close token as null
+/QTest is implicit
+/QTest is direct and has type null (2)
+/QTest is null
+unparse: null
+unparseResolved: null
+test 1 done
diff --git a/qpdf/qtest/qpdf/bad15.out b/qpdf/qtest/qpdf/bad15.out
index 54e799df..49de89c7 100644
--- a/qpdf/qtest/qpdf/bad15.out
+++ b/qpdf/qtest/qpdf/bad15.out
@@ -1 +1,7 @@
-bad15.pdf (trailer, file position 753): unexpected array close token
+WARNING: bad15.pdf (trailer, file position 753): treating unexpected array close token as null
+/QTest is implicit
+/QTest is direct and has type null (2)
+/QTest is null
+unparse: null
+unparseResolved: null
+test 0 done
diff --git a/qpdf/qtest/qpdf/bad16-recover.out b/qpdf/qtest/qpdf/bad16-recover.out
index 1256ceab..4cc66e23 100644
--- a/qpdf/qtest/qpdf/bad16-recover.out
+++ b/qpdf/qtest/qpdf/bad16-recover.out
@@ -1,4 +1,10 @@
-WARNING: bad16.pdf: file is damaged
 WARNING: bad16.pdf (trailer, file position 753): unexpected dictionary close token
+WARNING: bad16.pdf (trailer, file position 756): unexpected dictionary close token
+WARNING: bad16.pdf (trailer, file position 759): unknown token while reading object; treating as string
+WARNING: bad16.pdf: file is damaged
+WARNING: bad16.pdf (trailer, file position 773): EOF while reading token
 WARNING: bad16.pdf: Attempting to reconstruct cross-reference table
-bad16.pdf (trailer, file position 753): unexpected dictionary close token
+WARNING: bad16.pdf (trailer, file position 753): unexpected dictionary close token
+WARNING: bad16.pdf (trailer, file position 756): unexpected dictionary close token
+WARNING: bad16.pdf (trailer, file position 759): unknown token while reading object; treating as string
+bad16.pdf (trailer, file position 773): EOF while reading token
diff --git a/qpdf/qtest/qpdf/bad16.out b/qpdf/qtest/qpdf/bad16.out
index d3a72218..e3947f9d 100644
--- a/qpdf/qtest/qpdf/bad16.out
+++ b/qpdf/qtest/qpdf/bad16.out
@@ -1 +1,4 @@
-bad16.pdf (trailer, file position 753): unexpected dictionary close token
+WARNING: bad16.pdf (trailer, file position 753): unexpected dictionary close token
+WARNING: bad16.pdf (trailer, file position 756): unexpected dictionary close token
+WARNING: bad16.pdf (trailer, file position 759): unknown token while reading object; treating as string
+bad16.pdf (trailer, file position 773): EOF while reading token
diff --git a/qpdf/qtest/qpdf/bad17-recover.out b/qpdf/qtest/qpdf/bad17-recover.out
index d09805ae..f6f4dfa0 100644
--- a/qpdf/qtest/qpdf/bad17-recover.out
+++ b/qpdf/qtest/qpdf/bad17-recover.out
@@ -1,4 +1,7 @@
-WARNING: bad17.pdf: file is damaged
-WARNING: bad17.pdf (trailer, file position 753): dictionary ending here has an odd number of elements
-WARNING: bad17.pdf: Attempting to reconstruct cross-reference table
-bad17.pdf (trailer, file position 753): dictionary ending here has an odd number of elements
+WARNING: bad17.pdf (trailer, file position 715): dictionary ended prematurely; using null as value for last key
+/QTest is implicit
+/QTest is direct and has type null (2)
+/QTest is null
+unparse: null
+unparseResolved: null
+test 1 done
diff --git a/qpdf/qtest/qpdf/bad17.out b/qpdf/qtest/qpdf/bad17.out
index 7285b0ae..5744fc29 100644
--- a/qpdf/qtest/qpdf/bad17.out
+++ b/qpdf/qtest/qpdf/bad17.out
@@ -1 +1,7 @@
-bad17.pdf (trailer, file position 753): dictionary ending here has an odd number of elements
+WARNING: bad17.pdf (trailer, file position 715): dictionary ended prematurely; using null as value for last key
+/QTest is implicit
+/QTest is direct and has type null (2)
+/QTest is null
+unparse: null
+unparseResolved: null
+test 0 done
diff --git a/qpdf/qtest/qpdf/bad36-recover.out b/qpdf/qtest/qpdf/bad36-recover.out
new file mode 100644
index 00000000..59669a8a
--- /dev/null
+++ b/qpdf/qtest/qpdf/bad36-recover.out
@@ -0,0 +1,9 @@
+WARNING: bad36.pdf (trailer, file position 764): unknown token while reading object; treating as string
+WARNING: bad36.pdf (trailer, file position 715): expected dictionary key but found non-name object; inserting key /QPDFFake2
+WARNING: bad36.pdf (trailer, file position 715): dictionary ended prematurely; using null as value for last key
+/QTest is implicit
+/QTest is direct and has type null (2)
+/QTest is null
+unparse: null
+unparseResolved: null
+test 1 done
diff --git a/qpdf/qtest/qpdf/bad36.out b/qpdf/qtest/qpdf/bad36.out
new file mode 100644
index 00000000..f137afd9
--- /dev/null
+++ b/qpdf/qtest/qpdf/bad36.out
@@ -0,0 +1,9 @@
+WARNING: bad36.pdf (trailer, file position 764): unknown token while reading object; treating as string
+WARNING: bad36.pdf (trailer, file position 715): expected dictionary key but found non-name object; inserting key /QPDFFake2
+WARNING: bad36.pdf (trailer, file position 715): dictionary ended prematurely; using null as value for last key
+/QTest is implicit
+/QTest is direct and has type null (2)
+/QTest is null
+unparse: null
+unparseResolved: null
+test 0 done
diff --git a/qpdf/qtest/qpdf/bad36.pdf b/qpdf/qtest/qpdf/bad36.pdf
new file mode 100644
index 00000000..ea4a3c66
--- /dev/null
+++ b/qpdf/qtest/qpdf/bad36.pdf
@@ -0,0 +1,81 @@
+%PDF-1.3
+1 0 obj
+<<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+
+2 0 obj
+<<
+  /Type /Pages
+  /Kids [
+    3 0 R
+  ]
+  /Count 1
+>>
+endobj
+
+3 0 obj
+<<
+  /Type /Page
+  /Parent 2 0 R
+  /MediaBox [0 0 612 792]
+  /Contents 4 0 R
+  /Resources <<
+    /ProcSet 5 0 R
+    /Font <<
+      /F1 6 0 R
+    >>
+  >>
+>>
+endobj
+
+4 0 obj
+<<
+  /Length 44
+>>
+stream
+BT
+  /F1 24 Tf
+  72 720 Td
+  (Potato) Tj
+ET
+endstream
+endobj
+
+5 0 obj
+[
+  /PDF
+  /Text
+]
+endobj
+
+6 0 obj
+<<
+  /Type /Font
+  /Subtype /Type1
+  /Name /F1
+  /BaseFont /Helvetica
+  /Encoding /WinAnsiEncoding
+>>
+endobj
+
+xref
+0 7
+0000000000 65535 f 
+0000000009 00000 n 
+0000000063 00000 n 
+0000000135 00000 n 
+0000000307 00000 n 
+0000000403 00000 n 
+0000000438 00000 n 
+trailer <<
+  /Size 7
+  /Root 1 0 R
+  /QPDFFake1 (potato)
+  x /Something
+>>
+startxref
+556
+%%EOF
diff --git a/qpdf/qtest/qpdf/c-read-warnings-and-errors.out b/qpdf/qtest/qpdf/c-read-warnings-and-errors.out
deleted file mode 100644
index 117663e9..00000000
--- a/qpdf/qtest/qpdf/c-read-warnings-and-errors.out
+++ /dev/null
@@ -1,20 +0,0 @@
-warning: bad17.pdf: file is damaged
-  code: 5
-  file: bad17.pdf
-  pos : 0
-  text: file is damaged
-warning: bad17.pdf (trailer, file position 753): dictionary ending here has an odd number of elements
-  code: 5
-  file: bad17.pdf
-  pos : 753
-  text: dictionary ending here has an odd number of elements
-warning: bad17.pdf: Attempting to reconstruct cross-reference table
-  code: 5
-  file: bad17.pdf
-  pos : 0
-  text: Attempting to reconstruct cross-reference table
-error: bad17.pdf (trailer, file position 753): dictionary ending here has an odd number of elements
-  code: 5
-  file: bad17.pdf
-  pos : 753
-  text: dictionary ending here has an odd number of elements
diff --git a/qpdf/qtest/qpdf/indirect-r-arg.out b/qpdf/qtest/qpdf/indirect-r-arg.out
index e065ec16..e39d8f55 100644
--- a/qpdf/qtest/qpdf/indirect-r-arg.out
+++ b/qpdf/qtest/qpdf/indirect-r-arg.out
@@ -1 +1,7 @@
-indirect-r-arg.pdf (file position 76): unknown token while reading object (R)
+WARNING: indirect-r-arg.pdf (file position 76): unknown token while reading object; treating as string
+WARNING: indirect-r-arg.pdf (file position 62): expected dictionary key but found non-name object; inserting key /QPDFFake1
+WARNING: indirect-r-arg.pdf (file position 62): expected dictionary key but found non-name object; inserting key /QPDFFake2
+checking indirect-r-arg.pdf
+PDF Version: 1.3
+File is not encrypted
+File is not linearized
diff --git a/qpdf/qtest/qpdf/issue-100.out b/qpdf/qtest/qpdf/issue-100.out
index 37bd3207..691e2282 100644
--- a/qpdf/qtest/qpdf/issue-100.out
+++ b/qpdf/qtest/qpdf/issue-100.out
@@ -1,5 +1,13 @@
 WARNING: issue-100.pdf: file is damaged
 WARNING: issue-100.pdf (file position 736): xref not found
 WARNING: issue-100.pdf: Attempting to reconstruct cross-reference table
+WARNING: issue-100.pdf (file position 268): unknown token while reading object; treating as string
+WARNING: issue-100.pdf (file position 286): unknown token while reading object; treating as string
+WARNING: issue-100.pdf (file position 289): unknown token while reading object; treating as string
+WARNING: issue-100.pdf (file position 294): unknown token while reading object; treating as string
+WARNING: issue-100.pdf (file position 297): unknown token while reading object; treating as string
+WARNING: issue-100.pdf (file position 304): unknown token while reading object; treating as string
 WARNING: issue-100.pdf (object 5 0, file position 489): attempting to recover stream length
-issue-100.pdf (object 6 0, file position 59): expected n n obj
+WARNING: issue-100.pdf (trailer, file position 953): expected dictionary key but found non-name object; inserting key /QPDFFake1
+WARNING: issue-100.pdf (trailer, file position 953): dictionary ended prematurely; using null as value for last key
+operation for Dictionary object attempted on object of wrong type
diff --git a/qpdf/qtest/qpdf/issue-101.out b/qpdf/qtest/qpdf/issue-101.out
index 59bd8103..f2dc4715 100644
--- a/qpdf/qtest/qpdf/issue-101.out
+++ b/qpdf/qtest/qpdf/issue-101.out
@@ -1,6 +1,17 @@
 WARNING: issue-101.pdf: file is damaged
 WARNING: issue-101.pdf (file position 3526): xref not found
 WARNING: issue-101.pdf: Attempting to reconstruct cross-reference table
+WARNING: issue-101.pdf (file position 1242): expected dictionary key but found non-name object; inserting key /QPDFFake1
+WARNING: issue-101.pdf (file position 1242): dictionary ended prematurely; using null as value for last key
 WARNING: issue-101.pdf (object 5 0, file position 1509): attempting to recover stream length
-WARNING: issue-101.pdf (object 5 0, file position 2097): attempting to recover stream length
-issue-101.pdf (trailer, file position 2928): unknown token while reading object (ÿ)
+WARNING: issue-101.pdf (trailer, file position 2097): attempting to recover stream length
+WARNING: issue-101.pdf (trailer, file position 2928): unknown token while reading object; treating as string
+WARNING: issue-101.pdf (trailer, file position 2930): unknown token while reading object; treating as string
+WARNING: issue-101.pdf (trailer, file position 2928): expected dictionary key but found non-name object; inserting key /QPDFFake1
+WARNING: issue-101.pdf (trailer, file position 2928): expected dictionary key but found non-name object; inserting key /QPDFFake2
+WARNING: issue-101.pdf (trailer, file position 2928): expected dictionary key but found non-name object; inserting key /QPDFFake3
+WARNING: issue-101.pdf (trailer, file position 2996): attempting to recover stream length
+WARNING: issue-101.pdf (trailer, file position 3410): attempting to recover stream length
+WARNING: issue-101.pdf (trailer, file position 3631): attempting to recover stream length
+WARNING: issue-101.pdf (trailer, file position 4184): attempting to recover stream length
+issue-101.pdf (trailer, file position 4184): unable to recover stream data
diff --git a/qpdf/qtest/qpdf/issue-119.out b/qpdf/qtest/qpdf/issue-119.out
index bc6ffb3e..b83cfe16 100644
--- a/qpdf/qtest/qpdf/issue-119.out
+++ b/qpdf/qtest/qpdf/issue-119.out
@@ -1,2 +1,3 @@
-WARNING: issue-119.pdf (file position 336): loop detected resolving object 4 0
-issue-119.pdf (file position 298): dictionary key is not not a name token
+WARNING: issue-119.pdf (file position 298): expected dictionary key but found non-name object; inserting key /QPDFFake1
+WARNING: issue-119.pdf (file position 298): expected dictionary key but found non-name object; inserting key /QPDFFake2
+qpdf: operation succeeded with warnings; resulting file may have some problems