1 files changed, 42 insertions, 0 deletions
diff --git a/qpdf/qtest/tokenizer.test b/qpdf/qtest/tokenizer.test
new file mode 100644
index 00000000..b6e440b8
--- /dev/null
+++ b/qpdf/qtest/tokenizer.test
@@ -0,0 +1,42 @@
+#!/usr/bin/env perl
+require 5.008;
+use warnings;
+use strict;
+
+unshift(@INC, '.');
+require qpdf_test_helpers;
+
+chdir("qpdf") or die "chdir testdir failed: $!\n";
+
+require TestDriver;
+
+cleanup();
+
+my $td = new TestDriver('tokenizer');
+
+my $n_tests = 4;
+
+$td->runtest("tokenizer with no ignorable",
+             {$td->COMMAND => "test_tokenizer -no-ignorable tokens.pdf"},
+             {$td->FILE => "tokens-no-ignorable.out", $td->EXIT_STATUS => 0},
+             $td->NORMALIZE_NEWLINES);
+
+$td->runtest("tokenizer",
+             {$td->COMMAND => "test_tokenizer tokens.pdf"},
+             {$td->FILE => "tokens.out", $td->EXIT_STATUS => 0},
+             $td->NORMALIZE_NEWLINES);
+
+$td->runtest("tokenizer with max_len",
+             {$td->COMMAND => "test_tokenizer -maxlen 50 tokens.pdf"},
+             {$td->FILE => "tokens-maxlen.out", $td->EXIT_STATUS => 0},
+             $td->NORMALIZE_NEWLINES);
+
+$td->runtest("ignore bad token",
+             {$td->COMMAND =>
+                  "qpdf --show-xref bad-token-startxref.pdf"},
+             {$td->FILE => "bad-token-startxref.out",
+              $td->EXIT_STATUS => 0},
+             $td->NORMALIZE_NEWLINES);
+
+cleanup();
+$td->report($n_tests);