More robust handling of type errors

Give objects descriptions and context so it is possible to issue
warnings instead of fatal errors for attempts to access objects of the
wrong type.

12 files changed, 545 insertions, 118 deletions

diff --git a/libqpdf/QPDF.cc b/libqpdf/QPDF.cc
index 31c8d8e2..31f13118 100644
--- a/libqpdf/QPDF.cc
+++ b/libqpdf/QPDF.cc
@@ -106,6 +106,7 @@ QPDF::Members::~Members()
 QPDF::QPDF() :
     m(new Members())
 {
+    m->tokenizer.allowEOF();
 }
 
 QPDF::~QPDF()
@@ -272,10 +273,10 @@ QPDF::findHeader()
 bool
 QPDF::findStartxref()
 {
-    QPDFTokenizer::Token t = readToken(this->m->file, true);
+    QPDFTokenizer::Token t = readToken(this->m->file);
     if (t == QPDFTokenizer::Token(QPDFTokenizer::tt_word, "startxref"))
     {
-        t = readToken(this->m->file, true);
+        t = readToken(this->m->file);
         if (t.getType() == QPDFTokenizer::tt_integer)
         {
             // Position in front of offset token
@@ -421,7 +422,7 @@ QPDF::reconstruct_xref(QPDFExc& e)
         this->m->file->findAndSkipNextEOL();
         qpdf_offset_t next_line_start = this->m->file->tell();
         this->m->file->seek(line_start, SEEK_SET);
-        QPDFTokenizer::Token t1 = readToken(this->m->file, true, MAX_LEN);
+        QPDFTokenizer::Token t1 = readToken(this->m->file, MAX_LEN);
         qpdf_offset_t token_start =
             this->m->file->tell() - t1.getValue().length();
         if (token_start >= next_line_start)
@@ -440,9 +441,9 @@ QPDF::reconstruct_xref(QPDFExc& e)
             if (t1.getType() == QPDFTokenizer::tt_integer)
             {
                 QPDFTokenizer::Token t2 =
-                    readToken(this->m->file, true, MAX_LEN);
+                    readToken(this->m->file, MAX_LEN);
                 QPDFTokenizer::Token t3 =
-                    readToken(this->m->file, true, MAX_LEN);
+                    readToken(this->m->file, MAX_LEN);
                 if ((t2.getType() == QPDFTokenizer::tt_integer) &&
                     (t3 == QPDFTokenizer::Token(QPDFTokenizer::tt_word, "obj")))
                 {
@@ -1429,7 +1430,7 @@ bool
 QPDF::findEndstream()
 {
     // Find endstream or endobj. Position the input at that token.
-    QPDFTokenizer::Token t = readToken(this->m->file, true, 20);
+    QPDFTokenizer::Token t = readToken(this->m->file, 20);
     if ((t.getType() == QPDFTokenizer::tt_word) &&
         ((t.getValue() == "endobj") ||
          (t.getValue() == "endstream")))
@@ -1522,11 +1523,10 @@ QPDF::recoverStreamLength(PointerHolder<InputSource> input,
 }
 
 QPDFTokenizer::Token
-QPDF::readToken(PointerHolder<InputSource> input,
-                bool allow_bad, size_t max_len)
+QPDF::readToken(PointerHolder<InputSource> input, size_t max_len)
 {
     return this->m->tokenizer.readToken(
-        input, this->m->last_object_description, allow_bad, max_len);
+        input, this->m->last_object_description, true, max_len);
 }
 
 QPDFObjectHandle
@@ -1730,16 +1730,10 @@ QPDF::resolve(int objid, int generation)
     }
     ResolveRecorder rr(this, og);
 
-    if (! this->m->obj_cache.count(og))
+    // PDF spec says unknown objects resolve to the null object.
+    if ((! this->m->obj_cache.count(og)) && this->m->xref_table.count(og))
     {
-	if (! this->m->xref_table.count(og))
-	{
-	    // PDF spec says unknown objects resolve to the null object.
-	    return new QPDF_Null;
-	}
-
 	QPDFXRefEntry const& entry = this->m->xref_table[og];
-        bool success = false;
         try
         {
             switch (entry.getType())
@@ -1768,7 +1762,6 @@ QPDF::resolve(int objid, int generation)
                               QUtil::int_to_string(generation) +
                               " has unexpected xref entry type");
             }
-            success = true;
         }
         catch (QPDFExc& e)
         {
@@ -1782,16 +1775,24 @@ QPDF::resolve(int objid, int generation)
                          QUtil::int_to_string(generation) +
                          ": error reading object: " + e.what()));
         }
-        if (! success)
-        {
-            QTC::TC("qpdf", "QPDF resolve failure to null");
-            QPDFObjectHandle oh = QPDFObjectHandle::newNull();
-            this->m->obj_cache[og] =
-                ObjCache(QPDFObjectHandle::ObjAccessor::getObject(oh), -1, -1);
-        }
+    }
+    if (this->m->obj_cache.count(og) == 0)
+    {
+        QTC::TC("qpdf", "QPDF resolve failure to null");
+        QPDFObjectHandle oh = QPDFObjectHandle::newNull();
+        this->m->obj_cache[og] =
+            ObjCache(QPDFObjectHandle::ObjAccessor::getObject(oh), -1, -1);
     }
 
-    return this->m->obj_cache[og].object;
+    PointerHolder<QPDFObject> result(this->m->obj_cache[og].object);
+    if (! result->hasDescription())
+    {
+        result->setDescription(
+            this,
+            "object " + QUtil::int_to_string(objid) + " " +
+            QUtil::int_to_string(generation));
+    }
+    return result;
 }
 
 void
diff --git a/libqpdf/QPDFExc.cc b/libqpdf/QPDFExc.cc
index 728d4ce8..b816e913 100644
--- a/libqpdf/QPDFExc.cc
+++ b/libqpdf/QPDFExc.cc
@@ -32,7 +32,10 @@ QPDFExc::createWhat(std::string const& filename,
     }
     if (! (object.empty() && offset == 0))
     {
-	result += " (";
+        if (! filename.empty())
+        {
+            result += " (";
+        }
 	if (! object.empty())
 	{
 	    result += object;
@@ -45,7 +48,10 @@ QPDFExc::createWhat(std::string const& filename,
 	{
 	    result += "offset " + QUtil::int_to_string(offset);
 	}
-	result += ")";
+        if (! filename.empty())
+        {
+            result += ")";
+        }
     }
     if (! result.empty())
     {
diff --git a/libqpdf/QPDFObject.cc b/libqpdf/QPDFObject.cc
index 8df2b480..cffb8a56 100644
--- a/libqpdf/QPDFObject.cc
+++ b/libqpdf/QPDFObject.cc
@@ -1 +1,36 @@
 #include <qpdf/QPDFObject.hh>
+
+QPDFObject::Members::Members() :
+    owning_qpdf(0)
+{
+}
+
+QPDFObject::Members::~Members()
+{
+}
+
+QPDFObject::QPDFObject() :
+    m(new Members)
+{
+}
+
+void
+QPDFObject::setDescription(QPDF* qpdf, std::string const& description)
+{
+    this->m->owning_qpdf = qpdf;
+    this->m->object_description = description;
+}
+
+bool
+QPDFObject::getDescription(QPDF*& qpdf, std::string& description)
+{
+    qpdf = this->m->owning_qpdf;
+    description = this->m->object_description;
+    return this->m->owning_qpdf;
+}
+
+bool
+QPDFObject::hasDescription()
+{
+    return this->m->owning_qpdf;
+}
diff --git a/libqpdf/QPDFObjectHandle.cc b/libqpdf/QPDFObjectHandle.cc
index d48461bf..2e9cc996 100644
--- a/libqpdf/QPDFObjectHandle.cc
+++ b/libqpdf/QPDFObjectHandle.cc
@@ -190,6 +190,18 @@ QPDFObjectHandle::releaseResolved()
     }
 }
 
+void
+QPDFObjectHandle::setObjectDescriptionFromInput(
+    QPDFObjectHandle object, QPDF* context,
+    std::string const& description, PointerHolder<InputSource> input,
+    qpdf_offset_t offset)
+{
+    object.setObjectDescription(
+        context,
+        input->getName() + ", " + description +
+        " at offset " + QUtil::int_to_string(offset));
+}
+
 bool
 QPDFObjectHandle::isInitialized() const
 {
@@ -282,7 +294,8 @@ QPDFObjectHandle::getNumericValue()
     }
     else
     {
-	throw std::logic_error("getNumericValue called for non-numeric object");
+        typeWarning("number", "returning 0");
+        QTC::TC("qpdf", "QPDFObjectHandle numeric non-numeric");
     }
     return result;
 }
@@ -363,8 +376,16 @@ QPDFObjectHandle::isScalar()
 bool
 QPDFObjectHandle::getBoolValue()
 {
-    assertBool();
-    return dynamic_cast<QPDF_Bool*>(m->obj.getPointer())->getVal();
+    if (isBool())
+    {
+        return dynamic_cast<QPDF_Bool*>(m->obj.getPointer())->getVal();
+    }
+    else
+    {
+        typeWarning("boolean", "returning false");
+        QTC::TC("qpdf", "QPDFObjectHandle boolean returning false");
+        return false;
+    }
 }
 
 // Integer accessors
@@ -372,8 +393,16 @@ QPDFObjectHandle::getBoolValue()
 long long
 QPDFObjectHandle::getIntValue()
 {
-    assertInteger();
-    return dynamic_cast<QPDF_Integer*>(m->obj.getPointer())->getVal();
+    if (isInteger())
+    {
+        return dynamic_cast<QPDF_Integer*>(m->obj.getPointer())->getVal();
+    }
+    else
+    {
+        typeWarning("integer", "returning 0");
+        QTC::TC("qpdf", "QPDFObjectHandle integer returning 0");
+        return 0;
+    }
 }
 
 // Real accessors
@@ -381,8 +410,16 @@ QPDFObjectHandle::getIntValue()
 std::string
 QPDFObjectHandle::getRealValue()
 {
-    assertReal();
-    return dynamic_cast<QPDF_Real*>(m->obj.getPointer())->getVal();
+    if (isReal())
+    {
+        return dynamic_cast<QPDF_Real*>(m->obj.getPointer())->getVal();
+    }
+    else
+    {
+        typeWarning("real", "returning 0.0");
+        QTC::TC("qpdf", "QPDFObjectHandle real returning 0.0");
+        return "0.0";
+    }
 }
 
 // Name accessors
@@ -390,8 +427,16 @@ QPDFObjectHandle::getRealValue()
 std::string
 QPDFObjectHandle::getName()
 {
-    assertName();
-    return dynamic_cast<QPDF_Name*>(m->obj.getPointer())->getName();
+    if (isName())
+    {
+        return dynamic_cast<QPDF_Name*>(m->obj.getPointer())->getName();
+    }
+    else
+    {
+        typeWarning("name", "returning dummy name");
+        QTC::TC("qpdf", "QPDFObjectHandle name returning dummy name");
+        return "/QPDFFakeName";
+    }
 }
 
 // String accessors
@@ -399,15 +444,31 @@ QPDFObjectHandle::getName()
 std::string
 QPDFObjectHandle::getStringValue()
 {
-    assertString();
-    return dynamic_cast<QPDF_String*>(m->obj.getPointer())->getVal();
+    if (isString())
+    {
+        return dynamic_cast<QPDF_String*>(m->obj.getPointer())->getVal();
+    }
+    else
+    {
+        typeWarning("string", "returning empty string");
+        QTC::TC("qpdf", "QPDFObjectHandle string returning empty string");
+        return "";
+    }
 }
 
 std::string
 QPDFObjectHandle::getUTF8Value()
 {
-    assertString();
-    return dynamic_cast<QPDF_String*>(m->obj.getPointer())->getUTF8Val();
+    if (isString())
+    {
+        return dynamic_cast<QPDF_String*>(m->obj.getPointer())->getUTF8Val();
+    }
+    else
+    {
+        typeWarning("string", "returning empty string");
+        QTC::TC("qpdf", "QPDFObjectHandle string returning empty utf8");
+        return "";
+    }
 }
 
 // Operator and Inline Image accessors
@@ -415,15 +476,31 @@ QPDFObjectHandle::getUTF8Value()
 std::string
 QPDFObjectHandle::getOperatorValue()
 {
-    assertOperator();
-    return dynamic_cast<QPDF_Operator*>(m->obj.getPointer())->getVal();
+    if (isOperator())
+    {
+        return dynamic_cast<QPDF_Operator*>(m->obj.getPointer())->getVal();
+    }
+    else
+    {
+        typeWarning("operator", "returning fake value");
+        QTC::TC("qpdf", "QPDFObjectHandle operator returning fake value");
+        return "QPDFFAKE";
+    }
 }
 
 std::string
 QPDFObjectHandle::getInlineImageValue()
 {
-    assertInlineImage();
-    return dynamic_cast<QPDF_InlineImage*>(m->obj.getPointer())->getVal();
+    if (isInlineImage())
+    {
+        return dynamic_cast<QPDF_InlineImage*>(m->obj.getPointer())->getVal();
+    }
+    else
+    {
+        typeWarning("inlineimage", "returning empty data");
+        QTC::TC("qpdf", "QPDFObjectHandle inlineimage returning empty data");
+        return "";
+    }
 }
 
 // Array accessors
@@ -431,22 +508,66 @@ QPDFObjectHandle::getInlineImageValue()
 int
 QPDFObjectHandle::getArrayNItems()
 {
-    assertArray();
-    return dynamic_cast<QPDF_Array*>(m->obj.getPointer())->getNItems();
+    if (isArray())
+    {
+        return dynamic_cast<QPDF_Array*>(m->obj.getPointer())->getNItems();
+    }
+    else
+    {
+        typeWarning("array", "treating as empty");
+        QTC::TC("qpdf", "QPDFObjectHandle array treating as empty");
+        return 0;
+    }
 }
 
 QPDFObjectHandle
 QPDFObjectHandle::getArrayItem(int n)
 {
-    assertArray();
-    return dynamic_cast<QPDF_Array*>(m->obj.getPointer())->getItem(n);
+    QPDFObjectHandle result;
+    if (isArray() && (n < getArrayNItems()) && (n >= 0))
+    {
+        result = dynamic_cast<QPDF_Array*>(m->obj.getPointer())->getItem(n);
+    }
+    else
+    {
+        result = newNull();
+        if (isArray())
+        {
+            objectWarning("returning null for out of bounds array access");
+            QTC::TC("qpdf", "QPDFObjectHandle array bounds");
+        }
+        else
+        {
+            typeWarning("array", "returning null");
+            QTC::TC("qpdf", "QPDFObjectHandle array null for non-array");
+        }
+        QPDF* context = 0;
+        std::string description;
+        if (this->m->obj->getDescription(context, description))
+        {
+            result.setObjectDescription(
+                context,
+                description +
+                " -> null returned from invalid array access");
+        }
+    }
+    return result;
 }
 
 std::vector<QPDFObjectHandle>
 QPDFObjectHandle::getArrayAsVector()
 {
-    assertArray();
-    return dynamic_cast<QPDF_Array*>(m->obj.getPointer())->getAsVector();
+    std::vector<QPDFObjectHandle> result;
+    if (isArray())
+    {
+        result = dynamic_cast<QPDF_Array*>(m->obj.getPointer())->getAsVector();
+    }
+    else
+    {
+        typeWarning("array", "treating as empty");
+        QTC::TC("qpdf", "QPDFObjectHandle array treating as empty vector");
+    }
+    return result;
 }
 
 // Array mutators
@@ -454,36 +575,79 @@ QPDFObjectHandle::getArrayAsVector()
 void
 QPDFObjectHandle::setArrayItem(int n, QPDFObjectHandle const& item)
 {
-    assertArray();
-    return dynamic_cast<QPDF_Array*>(m->obj.getPointer())->setItem(n, item);
+    if (isArray())
+    {
+        dynamic_cast<QPDF_Array*>(m->obj.getPointer())->setItem(n, item);
+    }
+    else
+    {
+        typeWarning("array", "ignoring attempt to set item");
+        QTC::TC("qpdf", "QPDFObjectHandle array ignoring set item");
+    }
 }
 
 void
 QPDFObjectHandle::setArrayFromVector(std::vector<QPDFObjectHandle> const& items)
 {
-    assertArray();
-    return dynamic_cast<QPDF_Array*>(m->obj.getPointer())->setFromVector(items);
+    if (isArray())
+    {
+        dynamic_cast<QPDF_Array*>(m->obj.getPointer())->setFromVector(items);
+    }
+    else
+    {
+        typeWarning("array", "ignoring attempt to replace items");
+        QTC::TC("qpdf", "QPDFObjectHandle array ignoring replace items");
+    }
 }
 
 void
 QPDFObjectHandle::insertItem(int at, QPDFObjectHandle const& item)
 {
-    assertArray();
-    return dynamic_cast<QPDF_Array*>(m->obj.getPointer())->insertItem(at, item);
+    if (isArray())
+    {
+        dynamic_cast<QPDF_Array*>(m->obj.getPointer())->insertItem(at, item);
+    }
+    else
+    {
+        typeWarning("array", "ignoring attempt to insert item");
+        QTC::TC("qpdf", "QPDFObjectHandle array ignoring insert item");
+    }
 }
 
 void
 QPDFObjectHandle::appendItem(QPDFObjectHandle const& item)
 {
-    assertArray();
-    return dynamic_cast<QPDF_Array*>(m->obj.getPointer())->appendItem(item);
+    if (isArray())
+    {
+        dynamic_cast<QPDF_Array*>(m->obj.getPointer())->appendItem(item);
+    }
+    else
+    {
+        typeWarning("array", "ignoring attempt to append item");
+        QTC::TC("qpdf", "QPDFObjectHandle array ignoring append item");
+    }
 }
 
 void
 QPDFObjectHandle::eraseItem(int at)
 {
-    assertArray();
-    return dynamic_cast<QPDF_Array*>(m->obj.getPointer())->eraseItem(at);
+    if (isArray() && (at < getArrayNItems()) && (at >= 0))
+    {
+        dynamic_cast<QPDF_Array*>(m->obj.getPointer())->eraseItem(at);
+    }
+    else
+    {
+        if (isArray())
+        {
+            objectWarning("ignoring attempt to erase out of bounds array item");
+            QTC::TC("qpdf", "QPDFObjectHandle erase array bounds");
+        }
+        else
+        {
+            typeWarning("array", "ignoring attempt to erase item");
+            QTC::TC("qpdf", "QPDFObjectHandle array ignoring erase item");
+        }
+    }
 }
 
 // Dictionary accessors
@@ -491,29 +655,79 @@ QPDFObjectHandle::eraseItem(int at)
 bool
 QPDFObjectHandle::hasKey(std::string const& key)
 {
-    assertDictionary();
-    return dynamic_cast<QPDF_Dictionary*>(m->obj.getPointer())->hasKey(key);
+    if (isDictionary())
+    {
+        return dynamic_cast<QPDF_Dictionary*>(m->obj.getPointer())->hasKey(key);
+    }
+    else
+    {
+        typeWarning("dictionary",
+                    "returning false for a key containment request");
+        QTC::TC("qpdf", "QPDFObjectHandle dictionary false for hasKey");
+        return false;
+    }
 }
 
 QPDFObjectHandle
 QPDFObjectHandle::getKey(std::string const& key)
 {
-    assertDictionary();
-    return dynamic_cast<QPDF_Dictionary*>(m->obj.getPointer())->getKey(key);
+    QPDFObjectHandle result;
+    if (isDictionary())
+    {
+        result = dynamic_cast<QPDF_Dictionary*>(
+            m->obj.getPointer())->getKey(key);
+    }
+    else
+    {
+        typeWarning(
+            "dictionary", "returning null for attempted key retrieval");
+        QTC::TC("qpdf", "QPDFObjectHandle dictionary null for getKey");
+        result = newNull();
+        QPDF* qpdf = 0;
+        std::string description;
+        if (this->m->obj->getDescription(qpdf, description))
+        {
+            result.setObjectDescription(
+                qpdf,
+                description +
+                " -> null returned from getting key " +
+                key + " from non-Dictionary");
+        }
+    }
+    return result;
 }
 
 std::set<std::string>
 QPDFObjectHandle::getKeys()
 {
-    assertDictionary();
-    return dynamic_cast<QPDF_Dictionary*>(m->obj.getPointer())->getKeys();
+    std::set<std::string> result;
+    if (isDictionary())
+    {
+        result = dynamic_cast<QPDF_Dictionary*>(m->obj.getPointer())->getKeys();
+    }
+    else
+    {
+        typeWarning("dictionary", "treating as empty");
+        QTC::TC("qpdf", "QPDFObjectHandle dictionary empty set for getKeys");
+    }
+    return result;
 }
 
 std::map<std::string, QPDFObjectHandle>
 QPDFObjectHandle::getDictAsMap()
 {
-    assertDictionary();
-    return dynamic_cast<QPDF_Dictionary*>(m->obj.getPointer())->getAsMap();
+    std::map<std::string, QPDFObjectHandle> result;
+    if (isDictionary())
+    {
+        result = dynamic_cast<QPDF_Dictionary*>(
+            m->obj.getPointer())->getAsMap();
+    }
+    else
+    {
+        typeWarning("dictionary", "treating as empty");
+        QTC::TC("qpdf", "QPDFObjectHandle dictionary empty map for asMap");
+    }
+    return result;
 }
 
 // Array and Name accessors
@@ -551,27 +765,48 @@ QPDFObjectHandle::getOwningQPDF()
 
 void
 QPDFObjectHandle::replaceKey(std::string const& key,
-			    QPDFObjectHandle const& value)
+			    QPDFObjectHandle value)
 {
-    assertDictionary();
-    return dynamic_cast<QPDF_Dictionary*>(
-	m->obj.getPointer())->replaceKey(key, value);
+    if (isDictionary())
+    {
+        dynamic_cast<QPDF_Dictionary*>(
+            m->obj.getPointer())->replaceKey(key, value);
+    }
+    else
+    {
+        typeWarning("dictionary", "ignoring key replacement request");
+        QTC::TC("qpdf", "QPDFObjectHandle dictionary ignoring replaceKey");
+    }
 }
 
 void
 QPDFObjectHandle::removeKey(std::string const& key)
 {
-    assertDictionary();
-    return dynamic_cast<QPDF_Dictionary*>(m->obj.getPointer())->removeKey(key);
+    if (isDictionary())
+    {
+        dynamic_cast<QPDF_Dictionary*>(m->obj.getPointer())->removeKey(key);
+    }
+    else
+    {
+        typeWarning("dictionary", "ignoring key removal request");
+        QTC::TC("qpdf", "QPDFObjectHandle dictionary ignoring removeKey");
+    }
 }
 
 void
 QPDFObjectHandle::replaceOrRemoveKey(std::string const& key,
 				     QPDFObjectHandle value)
 {
-    assertDictionary();
-    return dynamic_cast<QPDF_Dictionary*>(
-	m->obj.getPointer())->replaceOrRemoveKey(key, value);
+    if (isDictionary())
+    {
+        dynamic_cast<QPDF_Dictionary*>(
+            m->obj.getPointer())->replaceOrRemoveKey(key, value);
+    }
+    else
+    {
+        typeWarning("dictionary", "ignoring key removal/replacement request");
+        QTC::TC("qpdf", "QPDFObjectHandle dictionary ignoring removereplace");
+    }
 }
 
 // Stream accessors
@@ -1173,35 +1408,45 @@ QPDFObjectHandle::parseInternal(PointerHolder<InputSource> input,
     std::vector<parser_state_e> state_stack;
     state_stack.push_back(st_top);
     std::vector<qpdf_offset_t> offset_stack;
-    offset_stack.push_back(input->tell());
+    qpdf_offset_t offset = input->tell();
+    offset_stack.push_back(offset);
     bool done = false;
     while (! done)
     {
         std::vector<QPDFObjectHandle>& olist = olist_stack.back();
         parser_state_e state = state_stack.back();
-        qpdf_offset_t offset = offset_stack.back();
+        offset = offset_stack.back();
 
 	object = QPDFObjectHandle();
 
 	QPDFTokenizer::Token token =
-            tokenizer.readToken(input, object_description);
+            tokenizer.readToken(input, object_description, true);
 
 	switch (token.getType())
 	{
           case QPDFTokenizer::tt_eof:
-            if (content_stream)
+            if (! content_stream)
             {
-                state = st_eof;
-            }
-            else
-            {
-                // When not in content stream mode, EOF is tt_bad and
-                // throws an exception before we get here.
-                throw std::logic_error(
-                    "EOF received while not in content stream mode");
+                QTC::TC("qpdf", "QPDFObjectHandle eof in parseInternal");
+                warn(context,
+                     QPDFExc(qpdf_e_damaged_pdf, input->getName(),
+                             object_description,
+                             input->getLastOffset(),
+                             "unexpected EOF"));
             }
+            state = st_eof;
             break;
 
+          case QPDFTokenizer::tt_bad:
+	    QTC::TC("qpdf", "QPDFObjectHandle bad token in parse");
+            warn(context,
+                 QPDFExc(qpdf_e_damaged_pdf, input->getName(),
+                         object_description,
+                         input->getLastOffset(),
+                         token.getErrorMessage()));
+            object = newNull();
+	    break;
+
 	  case QPDFTokenizer::tt_brace_open:
 	  case QPDFTokenizer::tt_brace_close:
 	    QTC::TC("qpdf", "QPDFObjectHandle bad brace");
@@ -1375,11 +1620,19 @@ QPDFObjectHandle::parseInternal(PointerHolder<InputSource> input,
                              "parse error while reading object"));
             }
             done = true;
-            // Leave object uninitialized to indicate EOF
+            // In content stream mode, leave object uninitialized to
+            // indicate EOF
+            if (! content_stream)
+            {
+                object = newNull();
+            }
             break;
 
           case st_dictionary:
           case st_array:
+            setObjectDescriptionFromInput(
+                object, context, object_description, input,
+                input->getLastOffset());
             olist.push_back(object);
             break;
 
@@ -1402,6 +1655,8 @@ QPDFObjectHandle::parseInternal(PointerHolder<InputSource> input,
             if (old_state == st_array)
             {
                 object = newArray(olist);
+                setObjectDescriptionFromInput(
+                    object, context, object_description, input, offset);
             }
             else if (old_state == st_dictionary)
             {
@@ -1458,6 +1713,8 @@ QPDFObjectHandle::parseInternal(PointerHolder<InputSource> input,
                                  "dictionary ended prematurely; "
                                  "using null as value for last key"));
                         val = newNull();
+                        setObjectDescriptionFromInput(
+                            val, context, object_description, input, offset);
                     }
                     else
                     {
@@ -1466,6 +1723,8 @@ QPDFObjectHandle::parseInternal(PointerHolder<InputSource> input,
                     dict[key_obj.getName()] = val;
                 }
                 object = newDictionary(dict);
+                setObjectDescriptionFromInput(
+                    object, context, object_description, input, offset);
             }
             olist_stack.pop_back();
             offset_stack.pop_back();
@@ -1480,6 +1739,8 @@ QPDFObjectHandle::parseInternal(PointerHolder<InputSource> input,
         }
     }
 
+    setObjectDescriptionFromInput(
+        object, context, object_description, input, offset);
     return object;
 }
 
@@ -1635,6 +1896,26 @@ QPDFObjectHandle::newReserved(QPDF* qpdf)
     return result;
 }
 
+void
+QPDFObjectHandle::setObjectDescription(QPDF* owning_qpdf,
+                                       std::string const& object_description)
+{
+    if (isInitialized() && this->m->obj.getPointer())
+    {
+        this->m->obj->setDescription(owning_qpdf, object_description);
+    }
+}
+
+bool
+QPDFObjectHandle::hasObjectDescription()
+{
+    if (isInitialized() && this->m->obj.getPointer())
+    {
+        return this->m->obj->hasDescription();
+    }
+    return false;
+}
+
 QPDFObjectHandle
 QPDFObjectHandle::shallowCopy()
 {
@@ -1793,85 +2074,127 @@ QPDFObjectHandle::assertInitialized() const
 }
 
 void
-QPDFObjectHandle::assertType(char const* type_name, bool istype) const
+QPDFObjectHandle::typeWarning(char const* expected_type,
+                              std::string const& warning)
+{
+    QPDF* context = 0;
+    std::string description;
+    if (this->m->obj->getDescription(context, description))
+    {
+        warn(context,
+             QPDFExc(
+                 qpdf_e_damaged_pdf,
+                 "", description, 0,
+                 std::string("operation for ") + expected_type +
+                 " attempted on object of type " +
+                 getTypeName() + ": " + warning));
+    }
+    else
+    {
+        assertType(expected_type, false);
+    }
+}
+
+void
+QPDFObjectHandle::objectWarning(std::string const& warning)
+{
+    QPDF* context = 0;
+    std::string description;
+    if (this->m->obj->getDescription(context, description))
+    {
+        warn(context,
+             QPDFExc(
+                 qpdf_e_damaged_pdf,
+                 "", description, 0,
+                 warning));
+    }
+    else
+    {
+        throw std::logic_error(warning);
+    }
+}
+
+void
+QPDFObjectHandle::assertType(char const* type_name, bool istype)
 {
     if (! istype)
     {
 	throw std::logic_error(std::string("operation for ") + type_name +
-			       " object attempted on object of wrong type");
+			       " attempted on object of type " +
+                               getTypeName());
     }
 }
 
 void
 QPDFObjectHandle::assertNull()
 {
-    assertType("Null", isNull());
+    assertType("null", isNull());
 }
 
 void
 QPDFObjectHandle::assertBool()
 {
-    assertType("Boolean", isBool());
+    assertType("boolean", isBool());
 }
 
 void
 QPDFObjectHandle::assertInteger()
 {
-    assertType("Integer", isInteger());
+    assertType("integer", isInteger());
 }
 
 void
 QPDFObjectHandle::assertReal()
 {
-    assertType("Real", isReal());
+    assertType("real", isReal());
 }
 
 void
 QPDFObjectHandle::assertName()
 {
-    assertType("Name", isName());
+    assertType("name", isName());
 }
 
 void
 QPDFObjectHandle::assertString()
 {
-    assertType("String", isString());
+    assertType("string", isString());
 }
 
 void
 QPDFObjectHandle::assertOperator()
 {
-    assertType("Operator", isOperator());
+    assertType("operator", isOperator());
 }
 
 void
 QPDFObjectHandle::assertInlineImage()
 {
-    assertType("InlineImage", isInlineImage());
+    assertType("inlineimage", isInlineImage());
 }
 
 void
 QPDFObjectHandle::assertArray()
 {
-    assertType("Array", isArray());
+    assertType("array", isArray());
 }
 
 void
 QPDFObjectHandle::assertDictionary()
 {
-    assertType("Dictionary", isDictionary());
+    assertType("dictionary", isDictionary());
 }
 
 void
 QPDFObjectHandle::assertStream()
 {
-    assertType("Stream", isStream());
+    assertType("stream", isStream());
 }
 
 void
 QPDFObjectHandle::assertReserved()
 {
-    assertType("Reserved", isReserved());
+    assertType("reserved", isReserved());
 }
 
 void
@@ -1887,13 +2210,13 @@ QPDFObjectHandle::assertIndirect()
 void
 QPDFObjectHandle::assertScalar()
 {
-    assertType("Scalar", isScalar());
+    assertType("scalar", isScalar());
 }
 
 void
 QPDFObjectHandle::assertNumber()
 {
-    assertType("Number", isNumber());
+    assertType("number", isNumber());
 }
 
 bool
@@ -1928,7 +2251,8 @@ QPDFObjectHandle::dereference()
 	    this->m->qpdf, this->m->objid, this->m->generation);
 	if (obj.getPointer() == 0)
 	{
-	    QTC::TC("qpdf", "QPDFObjectHandle indirect to unknown");
+            // QPDF::resolve never returns an uninitialized object, but
+            // check just in case.
 	    this->m->obj = new QPDF_Null();
 	}
         else if (dynamic_cast<QPDF_Reserved*>(obj.getPointer()))
diff --git a/libqpdf/QPDFTokenizer.cc b/libqpdf/QPDFTokenizer.cc
index c3a017d0..95551e7c 100644
--- a/libqpdf/QPDFTokenizer.cc
+++ b/libqpdf/QPDFTokenizer.cc
@@ -640,7 +640,9 @@ QPDFTokenizer::readToken(PointerHolder<InputSource> input,
                 presented_eof = true;
                 if ((this->m->type == tt_eof) && (! this->m->allow_eof))
                 {
-                    QTC::TC("qpdf", "QPDFTokenizer EOF when not allowed");
+                    // Nothing in the qpdf library calls readToken
+                    // without allowEOF anymore, so this case is not
+                    // exercised.
                     this->m->type = tt_bad;
                     this->m->error_message = "unexpected EOF";
                     offset = input->getLastOffset();
@@ -677,7 +679,10 @@ QPDFTokenizer::readToken(PointerHolder<InputSource> input,
 	input->unreadCh(char_to_unread);
     }
 
-    input->setLastOffset(offset);
+    if (token.getType() != tt_eof)
+    {
+        input->setLastOffset(offset);
+    }
 
     if (token.getType() == tt_bad)
     {
diff --git a/libqpdf/QPDF_Array.cc b/libqpdf/QPDF_Array.cc
index c526174f..1a4ba61d 100644
--- a/libqpdf/QPDF_Array.cc
+++ b/libqpdf/QPDF_Array.cc
@@ -1,4 +1,5 @@
 #include <qpdf/QPDF_Array.hh>
+#include <qpdf/QUtil.hh>
 #include <stdexcept>
 
 QPDF_Array::QPDF_Array(std::vector<QPDFObjectHandle> const& items) :
@@ -46,6 +47,12 @@ QPDF_Array::getTypeName() const
     return "array";
 }
 
+void
+QPDF_Array::setDescription(QPDF* qpdf, std::string const& description)
+{
+    this->QPDFObject::setDescription(qpdf, description);
+}
+
 int
 QPDF_Array::getNItems() const
 {
diff --git a/libqpdf/QPDF_Dictionary.cc b/libqpdf/QPDF_Dictionary.cc
index 0af2f4bf..df640354 100644
--- a/libqpdf/QPDF_Dictionary.cc
+++ b/libqpdf/QPDF_Dictionary.cc
@@ -51,6 +51,12 @@ QPDF_Dictionary::getTypeName() const
     return "dictionary";
 }
 
+void
+QPDF_Dictionary::setDescription(QPDF* qpdf, std::string const& description)
+{
+    this->QPDFObject::setDescription(qpdf, description);
+}
+
 bool
 QPDF_Dictionary::hasKey(std::string const& key)
 {
@@ -70,7 +76,15 @@ QPDF_Dictionary::getKey(std::string const& key)
     }
     else
     {
-	return QPDFObjectHandle::newNull();
+        QPDFObjectHandle null = QPDFObjectHandle::newNull();
+        QPDF* qpdf = 0;
+        std::string description;
+        if (getDescription(qpdf, description))
+        {
+            null.setObjectDescription(
+                qpdf, description + " -> dictionary key " + key);
+        }
+	return null;
     }
 }
 
@@ -93,13 +107,12 @@ QPDF_Dictionary::getKeys()
 std::map<std::string, QPDFObjectHandle> const&
 QPDF_Dictionary::getAsMap() const
 {
-
     return this->items;
 }
 
 void
 QPDF_Dictionary::replaceKey(std::string const& key,
-			    QPDFObjectHandle const& value)
+			    QPDFObjectHandle value)
 {
     // add or replace value
     this->items[key] = value;
diff --git a/libqpdf/QPDF_Stream.cc b/libqpdf/QPDF_Stream.cc
index 7b84d10c..384652e2 100644
--- a/libqpdf/QPDF_Stream.cc
+++ b/libqpdf/QPDF_Stream.cc
@@ -39,6 +39,7 @@ QPDF_Stream::QPDF_Stream(QPDF* qpdf, int objid, int generation,
 	    "stream object instantiated with non-dictionary "
 	    "object for dictionary");
     }
+    setStreamDescription();
 }
 
 QPDF_Stream::~QPDF_Stream()
@@ -85,6 +86,35 @@ QPDF_Stream::getTypeName() const
     return "stream";
 }
 
+void
+QPDF_Stream::setDescription(QPDF* qpdf, std::string const& description)
+{
+    this->QPDFObject::setDescription(qpdf, description);
+    setDictDescription();
+}
+
+void
+QPDF_Stream::setStreamDescription()
+{
+    setDescription(
+        this->qpdf,
+        "stream object " + QUtil::int_to_string(this->objid) + " " +
+        QUtil::int_to_string(this->generation));
+}
+
+void
+QPDF_Stream::setDictDescription()
+{
+    QPDF* qpdf = 0;
+    std::string description;
+    if ((! this->stream_dict.hasObjectDescription()) &&
+        getDescription(qpdf, description))
+    {
+        this->stream_dict.setObjectDescription(
+            qpdf, description + " -> stream dictionary");
+    }
+}
+
 QPDFObjectHandle
 QPDF_Stream::getDict() const
 {
@@ -688,6 +718,7 @@ void
 QPDF_Stream::replaceDict(QPDFObjectHandle new_dict)
 {
     this->stream_dict = new_dict;
+    setDictDescription();
     QPDFObjectHandle length_obj = new_dict.getKey("/Length");
     if (length_obj.isInteger())
     {
diff --git a/libqpdf/QPDF_linearization.cc b/libqpdf/QPDF_linearization.cc
index 3d04ab90..ecf81bee 100644
--- a/libqpdf/QPDF_linearization.cc
+++ b/libqpdf/QPDF_linearization.cc
@@ -121,10 +121,10 @@ QPDF::isLinearized()
             ++p;
         }
 
-        QPDFTokenizer::Token t1 = readToken(this->m->file, true);
-        QPDFTokenizer::Token t2 = readToken(this->m->file, true);
-        QPDFTokenizer::Token t3 = readToken(this->m->file, true);
-        QPDFTokenizer::Token t4 = readToken(this->m->file, true);
+        QPDFTokenizer::Token t1 = readToken(this->m->file);
+        QPDFTokenizer::Token t2 = readToken(this->m->file);
+        QPDFTokenizer::Token t3 = readToken(this->m->file);
+        QPDFTokenizer::Token t4 = readToken(this->m->file);
         if ((t1.getType() == QPDFTokenizer::tt_integer) &&
             (t2.getType() == QPDFTokenizer::tt_integer) &&
             (t3 == QPDFTokenizer::Token(QPDFTokenizer::tt_word, "obj")) &&
diff --git a/libqpdf/qpdf/QPDF_Array.hh b/libqpdf/qpdf/QPDF_Array.hh
index e81f8664..8a23da35 100644
--- a/libqpdf/qpdf/QPDF_Array.hh
+++ b/libqpdf/qpdf/QPDF_Array.hh
@@ -14,6 +14,7 @@ class QPDF_Array: public QPDFObject
     virtual std::string unparse();
     virtual QPDFObject::object_type_e getTypeCode() const;
     virtual char const* getTypeName() const;
+    virtual void setDescription(QPDF*, std::string const&);
 
     int getNItems() const;
     QPDFObjectHandle getItem(int n) const;
diff --git a/libqpdf/qpdf/QPDF_Dictionary.hh b/libqpdf/qpdf/QPDF_Dictionary.hh
index 5b5630cf..cea63835 100644
--- a/libqpdf/qpdf/QPDF_Dictionary.hh
+++ b/libqpdf/qpdf/QPDF_Dictionary.hh
@@ -16,6 +16,7 @@ class QPDF_Dictionary: public QPDFObject
     virtual std::string unparse();
     virtual QPDFObject::object_type_e getTypeCode() const;
     virtual char const* getTypeName() const;
+    virtual void setDescription(QPDF*, std::string const&);
 
     // hasKey() and getKeys() treat keys with null values as if they
     // aren't there.  getKey() returns null for the value of a
@@ -26,7 +27,7 @@ class QPDF_Dictionary: public QPDFObject
     std::map<std::string, QPDFObjectHandle> const& getAsMap() const;
 
     // Replace value of key, adding it if it does not exist
-    void replaceKey(std::string const& key, QPDFObjectHandle const&);
+    void replaceKey(std::string const& key, QPDFObjectHandle);
     // Remove key, doing nothing if key does not exist
     void removeKey(std::string const& key);
     // If object is null, replace key; otherwise, remove key
diff --git a/libqpdf/qpdf/QPDF_Stream.hh b/libqpdf/qpdf/QPDF_Stream.hh
index 86b796cf..98b8c11f 100644
--- a/libqpdf/qpdf/QPDF_Stream.hh
+++ b/libqpdf/qpdf/QPDF_Stream.hh
@@ -19,6 +19,7 @@ class QPDF_Stream: public QPDFObject
     virtual std::string unparse();
     virtual QPDFObject::object_type_e getTypeCode() const;
     virtual char const* getTypeName() const;
+    virtual void setDescription(QPDF*, std::string const&);
     QPDFObjectHandle getDict() const;
     bool isDataModified() const;
 
@@ -66,6 +67,8 @@ class QPDF_Stream: public QPDFObject
                     int& colors, int& bits_per_component,
                     bool& early_code_change);
     void warn(QPDFExc const& e);
+    void setDictDescription();
+    void setStreamDescription();
 
     QPDF* qpdf;
     int objid;