1 files changed, 24 insertions, 49 deletions

diff --git a/libqpdf/Pl_QPDFTokenizer.cc b/libqpdf/Pl_QPDFTokenizer.cc
index 577c5cc7..bd5d88ab 100644
--- a/libqpdf/Pl_QPDFTokenizer.cc
+++ b/libqpdf/Pl_QPDFTokenizer.cc
@@ -1,13 +1,13 @@
 #include <qpdf/Pl_QPDFTokenizer.hh>
 #include <qpdf/QTC.hh>
+#include <qpdf/QUtil.hh>
+#include <qpdf/BufferInputSource.hh>
 #include <stdexcept>
 #include <string.h>
 
 Pl_QPDFTokenizer::Members::Members() :
     filter(0),
-    last_char_was_cr(false),
-    unread_char(false),
-    char_to_unread('\0')
+    buf("tokenizer buffer")
 {
 }
 
@@ -33,61 +33,36 @@ Pl_QPDFTokenizer::~Pl_QPDFTokenizer()
 }
 
 void
-Pl_QPDFTokenizer::processChar(char ch)
+Pl_QPDFTokenizer::write(unsigned char* data, size_t len)
 {
-    this->m->tokenizer.presentCharacter(ch);
-    QPDFTokenizer::Token token;
-    if (this->m->tokenizer.getToken(
-            token, this->m->unread_char, this->m->char_to_unread))
-    {
-	this->m->filter->handleToken(token);
-        if ((token.getType() == QPDFTokenizer::tt_word) &&
-            (token.getValue() == "ID"))
-        {
-            QTC::TC("qpdf", "Pl_QPDFTokenizer found ID");
-            this->m->tokenizer.expectInlineImage();
-        }
-    }
-}
-
-
-void
-Pl_QPDFTokenizer::checkUnread()
-{
-    if (this->m->unread_char)
-    {
-	processChar(this->m->char_to_unread);
-	if (this->m->unread_char)
-	{
-	    throw std::logic_error(
-		"INTERNAL ERROR: unread_char still true after processing "
-		"unread character");
-	}
-    }
-}
-
-void
-Pl_QPDFTokenizer::write(unsigned char* buf, size_t len)
-{
-    checkUnread();
-    for (size_t i = 0; i < len; ++i)
-    {
-	processChar(buf[i]);
-	checkUnread();
-    }
+    this->m->buf.write(data, len);
 }
 
 void
 Pl_QPDFTokenizer::finish()
 {
-    this->m->tokenizer.presentEOF();
-    QPDFTokenizer::Token token;
-    if (this->m->tokenizer.getToken(
-            token, this->m->unread_char, this->m->char_to_unread))
+    this->m->buf.finish();
+    PointerHolder<InputSource> input =
+        new BufferInputSource("tokenizer data",
+                              this->m->buf.getBuffer(), true);
+
+    while (true)
     {
+        QPDFTokenizer::Token token = this->m->tokenizer.readToken(
+            input, "offset " + QUtil::int_to_string(input->tell()),
+            true);
 	this->m->filter->handleToken(token);
+        if (token.getType() == QPDFTokenizer::tt_eof)
+        {
+            break;
+        }
+        else if ((token.getType() == QPDFTokenizer::tt_word) &&
+                 (token.getValue() == "ID"))
+        {
+            QTC::TC("qpdf", "Pl_QPDFTokenizer found ID");
+            this->m->tokenizer.expectInlineImage(input);
+        }
     }
-
     this->m->filter->handleEOF();
     QPDFObjectHandle::TokenFilter::PipelineAccessor::setPipeline(
         m->filter, 0);