Don't lose character after \d or \dd parsing string (fixes #1050)

8 files changed, 1043 insertions, 1003 deletions

diff --git a/ChangeLog b/ChangeLog
index 4ea991c9..2674cc06 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2023-10-14  Jay Berkenbilt  <ejb@ql.org>
+
+        * Fix serious bug: qpdf could discard a the character after an
+        escaped octal string. For content, this would only happen with QDF
+        or when normalizing content, but it could have happened in a
+        binary string. This bug was introduced between 10.6.3 and 11.0.0.
+        Fixes #1050.
+
 2023-10-07  Jay Berkenbilt  <ejb@ql.org>
 
         * 11.6.2: release
diff --git a/libqpdf/QPDFTokenizer.cc b/libqpdf/QPDFTokenizer.cc
index d98af8a9..ca09708a 100644
--- a/libqpdf/QPDFTokenizer.cc
+++ b/libqpdf/QPDFTokenizer.cc
@@ -145,9 +145,8 @@ QPDFTokenizer::presentCharacter(char ch)
 void
 QPDFTokenizer::handleCharacter(char ch)
 {
-    // State machine is implemented such that the final character may not be handled.  This happens
-    // whenever you have to use a character from the next token to detect the end of the current
-    // token.
+    // In some cases, functions called below may call a second handler. This happens whenever you
+    // have to use a character from the next token to detect the end of the current token.
 
     switch (this->state) {
     case st_top:
@@ -692,16 +691,21 @@ QPDFTokenizer::inHexstring2nd(char ch)
 void
 QPDFTokenizer::inCharCode(char ch)
 {
+    bool handled = false;
     if (('0' <= ch) && (ch <= '7')) {
         this->char_code = 8 * this->char_code + (int(ch) - int('0'));
         if (++(this->digit_count) < 3) {
             return;
         }
-        // We've accumulated \ddd.  PDF Spec says to ignore high-order overflow.
+        handled = true;
     }
+    // We've accumulated \ddd or we have \d or \dd followed by other than an octal digit. The PDF
+    // Spec says to ignore high-order overflow.
     this->val += char(this->char_code % 256);
     this->state = st_in_string;
-    return;
+    if (!handled) {
+        inString(ch);
+    }
 }
 
 void
diff --git a/qpdf/qtest/qpdf/tokens-maxlen.out b/qpdf/qtest/qpdf/tokens-maxlen.out
index 5e0046f1..80964b30 100644
--- a/qpdf/qtest/qpdf/tokens-maxlen.out
+++ b/qpdf/qtest/qpdf/tokens-maxlen.out
@@ -181,352 +181,352 @@ skipping to endstream
 7121: space: \x0a
 7122: word: stream
 skipping to endstream
-7469: word: endstream
-7478: space: \x0a
-7479: word: endobj
-7485: space: \x0a\x0a
-7487: integer: 44
-7489: space:  
-7490: integer: 0
-7491: space:  
-7492: word: obj
-7495: space: \x0a
-7496: integer: 340
-7499: space: \x0a
-7500: word: endobj
-7506: space: \x0a\x0a
-7508: comment: %% Contents for page 5
-7530: space: \x0a
-7531: comment: %% Original object ID: 41 0
-7558: space: \x0a
-7559: integer: 45
-7561: space:  
-7562: integer: 0
-7563: space:  
-7564: word: obj
-7567: space: \x0a
-7568: dict_open: <<
-7570: space: \x0a  
-7573: name: /Length
-7580: space:  
-7581: integer: 46
-7583: space:  
-7584: integer: 0
-7585: space:  
-7586: word: R
-7587: space: \x0a
-7588: dict_close: >>
-7590: space: \x0a
-7591: word: stream
+7524: word: endstream
+7533: space: \x0a
+7534: word: endobj
+7540: space: \x0a\x0a
+7542: integer: 44
+7544: space:  
+7545: integer: 0
+7546: space:  
+7547: word: obj
+7550: space: \x0a
+7551: integer: 395
+7554: space: \x0a
+7555: word: endobj
+7561: space: \x0a\x0a
+7563: comment: %% Contents for page 5
+7585: space: \x0a
+7586: comment: %% Original object ID: 41 0
+7613: space: \x0a
+7614: integer: 45
+7616: space:  
+7617: integer: 0
+7618: space:  
+7619: word: obj
+7622: space: \x0a
+7623: dict_open: <<
+7625: space: \x0a  
+7628: name: /Length
+7635: space:  
+7636: integer: 46
+7638: space:  
+7639: integer: 0
+7640: space:  
+7641: word: R
+7642: space: \x0a
+7643: dict_close: >>
+7645: space: \x0a
+7646: word: stream
 skipping to endstream
-7666: word: endstream
-7675: space: \x0a
-7676: word: endobj
-7682: space: \x0a
-7683: comment: %QDF: ignore_newline
-7703: space: \x0a\x0a
-7705: integer: 46
-7707: space:  
-7708: integer: 0
-7709: space:  
-7710: word: obj
-7713: space: \x0a
-7714: integer: 67
-7716: space: \x0a
-7717: word: endobj
-7723: space: \x0a\x0a
-7725: comment: %% Contents for page 6
-7747: space: \x0a
-7748: comment: %% Original object ID: 42 0
-7775: space: \x0a
-7776: integer: 47
-7778: space:  
-7779: integer: 0
-7780: space:  
-7781: word: obj
-7784: space: \x0a
-7785: dict_open: <<
-7787: space: \x0a  
-7790: name: /Length
-7797: space:  
-7798: integer: 48
-7800: space:  
-7801: integer: 0
-7802: space:  
-7803: word: R
-7804: space: \x0a
-7805: dict_close: >>
-7807: space: \x0a
-7808: word: stream
+7721: word: endstream
+7730: space: \x0a
+7731: word: endobj
+7737: space: \x0a
+7738: comment: %QDF: ignore_newline
+7758: space: \x0a\x0a
+7760: integer: 46
+7762: space:  
+7763: integer: 0
+7764: space:  
+7765: word: obj
+7768: space: \x0a
+7769: integer: 67
+7771: space: \x0a
+7772: word: endobj
+7778: space: \x0a\x0a
+7780: comment: %% Contents for page 6
+7802: space: \x0a
+7803: comment: %% Original object ID: 42 0
+7830: space: \x0a
+7831: integer: 47
+7833: space:  
+7834: integer: 0
+7835: space:  
+7836: word: obj
+7839: space: \x0a
+7840: dict_open: <<
+7842: space: \x0a  
+7845: name: /Length
+7852: space:  
+7853: integer: 48
+7855: space:  
+7856: integer: 0
+7857: space:  
+7858: word: R
+7859: space: \x0a
+7860: dict_close: >>
+7862: space: \x0a
+7863: word: stream
 skipping to endstream
-7859: word: endstream
-7868: space: \x0a
-7869: word: endobj
-7875: space: \x0a\x0a
-7877: integer: 48
-7879: space:  
-7880: integer: 0
-7881: space:  
-7882: word: obj
-7885: space: \x0a
-7886: integer: 44
-7888: space: \x0a
-7889: word: endobj
-7895: space: \x0a\x0a
-7897: comment: %% Contents for page 7
-7919: space: \x0a
-7920: comment: %% Original object ID: 43 0
-7947: space: \x0a
-7948: integer: 49
-7950: space:  
-7951: integer: 0
-7952: space:  
-7953: word: obj
-7956: space: \x0a
-7957: dict_open: <<
-7959: space: \x0a  
-7962: name: /Length
-7969: space:  
-7970: integer: 50
-7972: space:  
-7973: integer: 0
-7974: space:  
-7975: word: R
-7976: space: \x0a
-7977: dict_close: >>
-7979: space: \x0a
-7980: word: stream
+7914: word: endstream
+7923: space: \x0a
+7924: word: endobj
+7930: space: \x0a\x0a
+7932: integer: 48
+7934: space:  
+7935: integer: 0
+7936: space:  
+7937: word: obj
+7940: space: \x0a
+7941: integer: 44
+7943: space: \x0a
+7944: word: endobj
+7950: space: \x0a\x0a
+7952: comment: %% Contents for page 7
+7974: space: \x0a
+7975: comment: %% Original object ID: 43 0
+8002: space: \x0a
+8003: integer: 49
+8005: space:  
+8006: integer: 0
+8007: space:  
+8008: word: obj
+8011: space: \x0a
+8012: dict_open: <<
+8014: space: \x0a  
+8017: name: /Length
+8024: space:  
+8025: integer: 50
+8027: space:  
+8028: integer: 0
+8029: space:  
+8030: word: R
+8031: space: \x0a
+8032: dict_close: >>
+8034: space: \x0a
+8035: word: stream
 skipping to endstream
-8306: word: endstream
-8315: space: \x0a
-8316: word: endobj
-8322: space: \x0a
-8323: comment: %QDF: ignore_newline
-8343: space: \x0a\x0a
-8345: integer: 50
-8347: space:  
-8348: integer: 0
-8349: space:  
-8350: word: obj
-8353: space: \x0a
-8354: integer: 318
-8357: space: \x0a
-8358: word: endobj
-8364: space: \x0a\x0a
-8366: comment: %% Contents for page 8
-8388: space: \x0a
-8389: comment: %% Original object ID: 44 0
-8416: space: \x0a
-8417: integer: 51
-8419: space:  
-8420: integer: 0
-8421: space:  
-8422: word: obj
-8425: space: \x0a
-8426: dict_open: <<
-8428: space: \x0a  
-8431: name: /Length
-8438: space:  
-8439: integer: 52
-8441: space:  
-8442: integer: 0
-8443: space:  
-8444: word: R
-8445: space: \x0a
-8446: dict_close: >>
-8448: space: \x0a
-8449: word: stream
+8361: word: endstream
+8370: space: \x0a
+8371: word: endobj
+8377: space: \x0a
+8378: comment: %QDF: ignore_newline
+8398: space: \x0a\x0a
+8400: integer: 50
+8402: space:  
+8403: integer: 0
+8404: space:  
+8405: word: obj
+8408: space: \x0a
+8409: integer: 318
+8412: space: \x0a
+8413: word: endobj
+8419: space: \x0a\x0a
+8421: comment: %% Contents for page 8
+8443: space: \x0a
+8444: comment: %% Original object ID: 44 0
+8471: space: \x0a
+8472: integer: 51
+8474: space:  
+8475: integer: 0
+8476: space:  
+8477: word: obj
+8480: space: \x0a
+8481: dict_open: <<
+8483: space: \x0a  
+8486: name: /Length
+8493: space:  
+8494: integer: 52
+8496: space:  
+8497: integer: 0
+8498: space:  
+8499: word: R
+8500: space: \x0a
+8501: dict_close: >>
+8503: space: \x0a
+8504: word: stream
 skipping to endstream
-8500: word: endstream
-8509: space: \x0a
-8510: word: endobj
-8516: space: \x0a\x0a
-8518: integer: 52
-8520: space:  
-8521: integer: 0
-8522: space:  
-8523: word: obj
-8526: space: \x0a
-8527: integer: 44
-8529: space: \x0a
-8530: word: endobj
-8536: space: \x0a\x0a
-8538: comment: %% Contents for page 9
-8560: space: \x0a
-8561: comment: %% Original object ID: 45 0
-8588: space: \x0a
-8589: integer: 53
-8591: space:  
-8592: integer: 0
-8593: space:  
-8594: word: obj
-8597: space: \x0a
-8598: dict_open: <<
-8600: space: \x0a  
-8603: name: /Length
-8610: space:  
-8611: integer: 54
-8613: space:  
-8614: integer: 0
-8615: space:  
-8616: word: R
-8617: space: \x0a
-8618: dict_close: >>
-8620: space: \x0a
-8621: word: stream
+8555: word: endstream
+8564: space: \x0a
+8565: word: endobj
+8571: space: \x0a\x0a
+8573: integer: 52
+8575: space:  
+8576: integer: 0
+8577: space:  
+8578: word: obj
+8581: space: \x0a
+8582: integer: 44
+8584: space: \x0a
+8585: word: endobj
+8591: space: \x0a\x0a
+8593: comment: %% Contents for page 9
+8615: space: \x0a
+8616: comment: %% Original object ID: 45 0
+8643: space: \x0a
+8644: integer: 53
+8646: space:  
+8647: integer: 0
+8648: space:  
+8649: word: obj
+8652: space: \x0a
+8653: dict_open: <<
+8655: space: \x0a  
+8658: name: /Length
+8665: space:  
+8666: integer: 54
+8668: space:  
+8669: integer: 0
+8670: space:  
+8671: word: R
+8672: space: \x0a
+8673: dict_close: >>
+8675: space: \x0a
+8676: word: stream
 skipping to endstream
-8672: word: endstream
-8681: space: \x0a
-8682: word: endobj
-8688: space: \x0a\x0a
-8690: integer: 54
-8692: space:  
-8693: integer: 0
-8694: space:  
-8695: word: obj
-8698: space: \x0a
-8699: integer: 44
-8701: space: \x0a
-8702: word: endobj
-8708: space: \x0a\x0a
-8710: comment: %% Contents for page 10
-8733: space: \x0a
-8734: comment: %% Original object ID: 46 0
-8761: space: \x0a
-8762: integer: 55
-8764: space:  
-8765: integer: 0
-8766: space:  
-8767: word: obj
-8770: space: \x0a
-8771: dict_open: <<
-8773: space: \x0a  
-8776: name: /Length
-8783: space:  
-8784: integer: 56
-8786: space:  
-8787: integer: 0
-8788: space:  
-8789: word: R
-8790: space: \x0a
-8791: dict_close: >>
-8793: space: \x0a
-8794: word: stream
+8727: word: endstream
+8736: space: \x0a
+8737: word: endobj
+8743: space: \x0a\x0a
+8745: integer: 54
+8747: space:  
+8748: integer: 0
+8749: space:  
+8750: word: obj
+8753: space: \x0a
+8754: integer: 44
+8756: space: \x0a
+8757: word: endobj
+8763: space: \x0a\x0a
+8765: comment: %% Contents for page 10
+8788: space: \x0a
+8789: comment: %% Original object ID: 46 0
+8816: space: \x0a
+8817: integer: 55
+8819: space:  
+8820: integer: 0
+8821: space:  
+8822: word: obj
+8825: space: \x0a
+8826: dict_open: <<
+8828: space: \x0a  
+8831: name: /Length
+8838: space:  
+8839: integer: 56
+8841: space:  
+8842: integer: 0
+8843: space:  
+8844: word: R
+8845: space: \x0a
+8846: dict_close: >>
+8848: space: \x0a
+8849: word: stream
 skipping to endstream
-8845: word: endstream
-8854: space: \x0a
-8855: word: endobj
-8861: space: \x0a\x0a
-8863: integer: 56
-8865: space:  
-8866: integer: 0
-8867: space:  
-8868: word: obj
-8871: space: \x0a
-8872: integer: 44
-8874: space: \x0a
-8875: word: endobj
-8881: space: \x0a\x0a
-8883: comment: %% Contents for page 11
-8906: space: \x0a
-8907: comment: %% Original object ID: 47 0
-8934: space: \x0a
-8935: integer: 57
-8937: space:  
-8938: integer: 0
-8939: space:  
-8940: word: obj
-8943: space: \x0a
-8944: dict_open: <<
-8946: space: \x0a  
-8949: name: /Length
-8956: space:  
-8957: integer: 58
-8959: space:  
-8960: integer: 0
-8961: space:  
-8962: word: R
-8963: space: \x0a
-8964: dict_close: >>
-8966: space: \x0a
-8967: word: stream
+8900: word: endstream
+8909: space: \x0a
+8910: word: endobj
+8916: space: \x0a\x0a
+8918: integer: 56
+8920: space:  
+8921: integer: 0
+8922: space:  
+8923: word: obj
+8926: space: \x0a
+8927: integer: 44
+8929: space: \x0a
+8930: word: endobj
+8936: space: \x0a\x0a
+8938: comment: %% Contents for page 11
+8961: space: \x0a
+8962: comment: %% Original object ID: 47 0
+8989: space: \x0a
+8990: integer: 57
+8992: space:  
+8993: integer: 0
+8994: space:  
+8995: word: obj
+8998: space: \x0a
+8999: dict_open: <<
+9001: space: \x0a  
+9004: name: /Length
+9011: space:  
+9012: integer: 58
+9014: space:  
+9015: integer: 0
+9016: space:  
+9017: word: R
+9018: space: \x0a
+9019: dict_close: >>
+9021: space: \x0a
+9022: word: stream
 skipping to endstream
-9018: word: endstream
-9027: space: \x0a
-9028: word: endobj
-9034: space: \x0a\x0a
-9036: integer: 58
-9038: space:  
-9039: integer: 0
-9040: space:  
-9041: word: obj
-9044: space: \x0a
-9045: integer: 44
-9047: space: \x0a
-9048: word: endobj
-9054: space: \x0a\x0a
-9056: integer: 59
-9058: space:  
-9059: integer: 0
-9060: space:  
-9061: word: obj
-9064: space: \x0a
-9065: dict_open: <<
-9067: space: \x0a  
-9070: name: /Type
-9075: space:  
-9076: name: /XRef
-9081: space: \x0a  
-9084: name: /Length
-9091: space:  
-9092: integer: 240
-9095: space: \x0a  
-9098: name: /W
-9100: space:  
-9101: array_open: [
-9102: space:  
-9103: integer: 1
-9104: space:  
-9105: integer: 2
-9106: space:  
-9107: integer: 1
-9108: space:  
-9109: array_close: ]
-9110: space: \x0a  
-9113: name: /Root
-9118: space:  
-9119: integer: 2
-9120: space:  
-9121: integer: 0
-9122: space:  
-9123: word: R
-9124: space: \x0a  
-9127: name: /Size
-9132: space:  
-9133: integer: 60
-9135: space: \x0a  
-9138: name: /ID
-9141: space:  
-9142: array_open: [
-9143: string: \x88\x04\x8e\x17\xc9a\xe0\x94\xff\xec\xe9\x8c\xb8\x8cF\xd0 (raw: <88048e17c961e094ffece98cb88c46d0>)
-9177: string: \xed\xd6\x0f\xe8\xee\x87\xf8\x871\xa8o\x81\x9f\xe6Q\x99 (raw: <edd60fe8ee87f88731a86f819fe65199>)
-9211: array_close: ]
-9212: space: \x0a
-9213: dict_close: >>
-9215: space: \x0a
-9216: word: stream
+9073: word: endstream
+9082: space: \x0a
+9083: word: endobj
+9089: space: \x0a\x0a
+9091: integer: 58
+9093: space:  
+9094: integer: 0
+9095: space:  
+9096: word: obj
+9099: space: \x0a
+9100: integer: 44
+9102: space: \x0a
+9103: word: endobj
+9109: space: \x0a\x0a
+9111: integer: 59
+9113: space:  
+9114: integer: 0
+9115: space:  
+9116: word: obj
+9119: space: \x0a
+9120: dict_open: <<
+9122: space: \x0a  
+9125: name: /Type
+9130: space:  
+9131: name: /XRef
+9136: space: \x0a  
+9139: name: /Length
+9146: space:  
+9147: integer: 240
+9150: space: \x0a  
+9153: name: /W
+9155: space:  
+9156: array_open: [
+9157: space:  
+9158: integer: 1
+9159: space:  
+9160: integer: 2
+9161: space:  
+9162: integer: 1
+9163: space:  
+9164: array_close: ]
+9165: space: \x0a  
+9168: name: /Root
+9173: space:  
+9174: integer: 2
+9175: space:  
+9176: integer: 0
+9177: space:  
+9178: word: R
+9179: space: \x0a  
+9182: name: /Size
+9187: space:  
+9188: integer: 60
+9190: space: \x0a  
+9193: name: /ID
+9196: space:  
+9197: array_open: [
+9198: string: \x88\x04\x8e\x17\xc9a\xe0\x94\xff\xec\xe9\x8c\xb8\x8cF\xd0 (raw: <88048e17c961e094ffece98cb88c46d0>)
+9232: string: \xed\xd6\x0f\xe8\xee\x87\xf8\x871\xa8o\x81\x9f\xe6Q\x99 (raw: <edd60fe8ee87f88731a86f819fe65199>)
+9266: array_close: ]
+9267: space: \x0a
+9268: dict_close: >>
+9270: space: \x0a
+9271: word: stream
 skipping to endstream
-9464: word: endstream
-9473: space: \x0a
-9474: word: endobj
-9480: space: \x0a\x0a
-9482: word: startxref
-9491: space: \x0a
-9492: integer: 9056
-9496: space: \x0a
-9497: comment: %%EOF
-9502: space: \x0a
-9503: eof
+9519: word: endstream
+9528: space: \x0a
+9529: word: endobj
+9535: space: \x0a\x0a
+9537: word: startxref
+9546: space: \x0a
+9547: integer: 9111
+9551: space: \x0a
+9552: comment: %%EOF
+9557: space: \x0a
+9558: eof
 --- END FILE ---
 --- BEGIN PAGE 1 ---
 0: word: BT
@@ -669,69 +669,73 @@ skipping to endstream
 117: space: \x0a  
 120: string: qu\x0a\x0a\x0a\x0a\x0a\x0aack (raw: (qu\x0a\x0d\x0a\x0a\x0d\x0d\x0a\x0aack))
 135: space: \x0a  
-138: integer: 72
-140: space:  
-141: integer: 720
-144: space:  
-145: word: Td
-147: space: \x0a  
-150: real: 3.14
-154: space: \x0a  
-157: real: 3.
-159: space: \x0a  
-162: real: .14
-165: space: \x0a  
-168: real: +3.14
-173: space: \x0a  
-176: real: +3.
-179: space: \x0a  
-182: real: +.14
-186: space: \x0a  
-189: real: -3.14
-194: space: \x0a  
-197: real: -3.
-200: space: \x0a  
-203: real: -.14
-207: space: \x0a  
-210: integer: +16059
-216: space: \x0a  
-219: integer: -16059
-225: space: \x0a  
-228: word: +.
-230: space: \x0a  
-233: bad: <fade\x0aET (invalid character (T) in hexstring)
-241: space: \x0a
-242: bad: ) (unexpected ))
-243: bad: > (unexpected >)
-244: word: quack
-249: space:  
-250: name: /name\x00oops (raw: /name#oops) (name with stray # will not work with PDF >= 1.2)
-260: space:  
-261: name: /name (raw: /n#61me)
-268: space:  
-269: word: one
-272: space:  
-273: bool: true
-277: space:  
-278: word: two
-281: space:  
-282: bool: false
-287: space:  
-288: word: three
-293: space:  
-294: null: null
-298: space:  
-299: word: four
-303: space: \x0a
-304: word: !@#$^&
-310: brace_open: {
-311: brace_close: }
-312: word: *-_+=
-317: space: \x0a
-318: word: abc123def3.14true
-335: space: \x0a
-336: bad: <ff\x0a (EOF while reading token)
-340: eof
+138: string: \x048!8Q\x04!Q\x04 (raw: (\48\418\121\4\41\121\4))
+162: space: \x0a  
+165: string: \x048!8Q\x04!Q! (raw: (\48\418\121\4\41\121\41))
+190: space: \x0a  
+193: integer: 72
+195: space:  
+196: integer: 720
+199: space:  
+200: word: Td
+202: space: \x0a  
+205: real: 3.14
+209: space: \x0a  
+212: real: 3.
+214: space: \x0a  
+217: real: .14
+220: space: \x0a  
+223: real: +3.14
+228: space: \x0a  
+231: real: +3.
+234: space: \x0a  
+237: real: +.14
+241: space: \x0a  
+244: real: -3.14
+249: space: \x0a  
+252: real: -3.
+255: space: \x0a  
+258: real: -.14
+262: space: \x0a  
+265: integer: +16059
+271: space: \x0a  
+274: integer: -16059
+280: space: \x0a  
+283: word: +.
+285: space: \x0a  
+288: bad: <fade\x0aET (invalid character (T) in hexstring)
+296: space: \x0a
+297: bad: ) (unexpected ))
+298: bad: > (unexpected >)
+299: word: quack
+304: space:  
+305: name: /name\x00oops (raw: /name#oops) (name with stray # will not work with PDF >= 1.2)
+315: space:  
+316: name: /name (raw: /n#61me)
+323: space:  
+324: word: one
+327: space:  
+328: bool: true
+332: space:  
+333: word: two
+336: space:  
+337: bool: false
+342: space:  
+343: word: three
+348: space:  
+349: null: null
+353: space:  
+354: word: four
+358: space: \x0a
+359: word: !@#$^&
+365: brace_open: {
+366: brace_close: }
+367: word: *-_+=
+372: space: \x0a
+373: word: abc123def3.14true
+390: space: \x0a
+391: bad: <ff\x0a (EOF while reading token)
+395: eof
 --- END PAGE 4 ---
 --- BEGIN PAGE 5 ---
 0: word: BT
diff --git a/qpdf/qtest/qpdf/tokens-no-ignorable.out b/qpdf/qtest/qpdf/tokens-no-ignorable.out
index a378c118..4523ea7f 100644
--- a/qpdf/qtest/qpdf/tokens-no-ignorable.out
+++ b/qpdf/qtest/qpdf/tokens-no-ignorable.out
@@ -81,172 +81,172 @@ skipping to endstream
 7119: dict_close: >>
 7122: word: stream
 skipping to endstream
-7469: word: endstream
-7479: word: endobj
-7487: integer: 44
-7490: integer: 0
-7492: word: obj
-7496: integer: 340
-7500: word: endobj
-7559: integer: 45
-7562: integer: 0
-7564: word: obj
-7568: dict_open: <<
-7573: name: /Length
-7581: integer: 46
-7584: integer: 0
-7586: word: R
-7588: dict_close: >>
-7591: word: stream
+7524: word: endstream
+7534: word: endobj
+7542: integer: 44
+7545: integer: 0
+7547: word: obj
+7551: integer: 395
+7555: word: endobj
+7614: integer: 45
+7617: integer: 0
+7619: word: obj
+7623: dict_open: <<
+7628: name: /Length
+7636: integer: 46
+7639: integer: 0
+7641: word: R
+7643: dict_close: >>
+7646: word: stream
 skipping to endstream
-7666: word: endstream
-7676: word: endobj
-7705: integer: 46
-7708: integer: 0
-7710: word: obj
-7714: integer: 67
-7717: word: endobj
-7776: integer: 47
-7779: integer: 0
-7781: word: obj
-7785: dict_open: <<
-7790: name: /Length
-7798: integer: 48
-7801: integer: 0
-7803: word: R
-7805: dict_close: >>
-7808: word: stream
+7721: word: endstream
+7731: word: endobj
+7760: integer: 46
+7763: integer: 0
+7765: word: obj
+7769: integer: 67
+7772: word: endobj
+7831: integer: 47
+7834: integer: 0
+7836: word: obj
+7840: dict_open: <<
+7845: name: /Length
+7853: integer: 48
+7856: integer: 0
+7858: word: R
+7860: dict_close: >>
+7863: word: stream
 skipping to endstream
-7859: word: endstream
-7869: word: endobj
-7877: integer: 48
-7880: integer: 0
-7882: word: obj
-7886: integer: 44
-7889: word: endobj
-7948: integer: 49
-7951: integer: 0
-7953: word: obj
-7957: dict_open: <<
-7962: name: /Length
-7970: integer: 50
-7973: integer: 0
-7975: word: R
-7977: dict_close: >>
-7980: word: stream
+7914: word: endstream
+7924: word: endobj
+7932: integer: 48
+7935: integer: 0
+7937: word: obj
+7941: integer: 44
+7944: word: endobj
+8003: integer: 49
+8006: integer: 0
+8008: word: obj
+8012: dict_open: <<
+8017: name: /Length
+8025: integer: 50
+8028: integer: 0
+8030: word: R
+8032: dict_close: >>
+8035: word: stream
 skipping to endstream
-8306: word: endstream
-8316: word: endobj
-8345: integer: 50
-8348: integer: 0
-8350: word: obj
-8354: integer: 318
-8358: word: endobj
-8417: integer: 51
-8420: integer: 0
-8422: word: obj
-8426: dict_open: <<
-8431: name: /Length
-8439: integer: 52
-8442: integer: 0
-8444: word: R
-8446: dict_close: >>
-8449: word: stream
+8361: word: endstream
+8371: word: endobj
+8400: integer: 50
+8403: integer: 0
+8405: word: obj
+8409: integer: 318
+8413: word: endobj
+8472: integer: 51
+8475: integer: 0
+8477: word: obj
+8481: dict_open: <<
+8486: name: /Length
+8494: integer: 52
+8497: integer: 0
+8499: word: R
+8501: dict_close: >>
+8504: word: stream
 skipping to endstream
-8500: word: endstream
-8510: word: endobj
-8518: integer: 52
-8521: integer: 0
-8523: word: obj
-8527: integer: 44
-8530: word: endobj
-8589: integer: 53
-8592: integer: 0
-8594: word: obj
-8598: dict_open: <<
-8603: name: /Length
-8611: integer: 54
-8614: integer: 0
-8616: word: R
-8618: dict_close: >>
-8621: word: stream
+8555: word: endstream
+8565: word: endobj
+8573: integer: 52
+8576: integer: 0
+8578: word: obj
+8582: integer: 44
+8585: word: endobj
+8644: integer: 53
+8647: integer: 0
+8649: word: obj
+8653: dict_open: <<
+8658: name: /Length
+8666: integer: 54
+8669: integer: 0
+8671: word: R
+8673: dict_close: >>
+8676: word: stream
 skipping to endstream
-8672: word: endstream
-8682: word: endobj
-8690: integer: 54
-8693: integer: 0
-8695: word: obj
-8699: integer: 44
-8702: word: endobj
-8762: integer: 55
-8765: integer: 0
-8767: word: obj
-8771: dict_open: <<
-8776: name: /Length
-8784: integer: 56
-8787: integer: 0
-8789: word: R
-8791: dict_close: >>
-8794: word: stream
+8727: word: endstream
+8737: word: endobj
+8745: integer: 54
+8748: integer: 0
+8750: word: obj
+8754: integer: 44
+8757: word: endobj
+8817: integer: 55
+8820: integer: 0
+8822: word: obj
+8826: dict_open: <<
+8831: name: /Length
+8839: integer: 56
+8842: integer: 0
+8844: word: R
+8846: dict_close: >>
+8849: word: stream
 skipping to endstream
-8845: word: endstream
-8855: word: endobj
-8863: integer: 56
-8866: integer: 0
-8868: word: obj
-8872: integer: 44
-8875: word: endobj
-8935: integer: 57
-8938: integer: 0
-8940: word: obj
-8944: dict_open: <<
-8949: name: /Length
-8957: integer: 58
-8960: integer: 0
-8962: word: R
-8964: dict_close: >>
-8967: word: stream
+8900: word: endstream
+8910: word: endobj
+8918: integer: 56
+8921: integer: 0
+8923: word: obj
+8927: integer: 44
+8930: word: endobj
+8990: integer: 57
+8993: integer: 0
+8995: word: obj
+8999: dict_open: <<
+9004: name: /Length
+9012: integer: 58
+9015: integer: 0
+9017: word: R
+9019: dict_close: >>
+9022: word: stream
 skipping to endstream
-9018: word: endstream
-9028: word: endobj
-9036: integer: 58
-9039: integer: 0
-9041: word: obj
-9045: integer: 44
-9048: word: endobj
-9056: integer: 59
-9059: integer: 0
-9061: word: obj
-9065: dict_open: <<
-9070: name: /Type
-9076: name: /XRef
-9084: name: /Length
-9092: integer: 240
-9098: name: /W
-9101: array_open: [
-9103: integer: 1
-9105: integer: 2
-9107: integer: 1
-9109: array_close: ]
-9113: name: /Root
-9119: integer: 2
-9121: integer: 0
-9123: word: R
-9127: name: /Size
-9133: integer: 60
-9138: name: /ID
-9142: array_open: [
-9143: string: \x88\x04\x8e\x17\xc9a\xe0\x94\xff\xec\xe9\x8c\xb8\x8cF\xd0 (raw: <88048e17c961e094ffece98cb88c46d0>)
-9177: string: \xed\xd6\x0f\xe8\xee\x87\xf8\x871\xa8o\x81\x9f\xe6Q\x99 (raw: <edd60fe8ee87f88731a86f819fe65199>)
-9211: array_close: ]
-9213: dict_close: >>
-9216: word: stream
+9073: word: endstream
+9083: word: endobj
+9091: integer: 58
+9094: integer: 0
+9096: word: obj
+9100: integer: 44
+9103: word: endobj
+9111: integer: 59
+9114: integer: 0
+9116: word: obj
+9120: dict_open: <<
+9125: name: /Type
+9131: name: /XRef
+9139: name: /Length
+9147: integer: 240
+9153: name: /W
+9156: array_open: [
+9158: integer: 1
+9160: integer: 2
+9162: integer: 1
+9164: array_close: ]
+9168: name: /Root
+9174: integer: 2
+9176: integer: 0
+9178: word: R
+9182: name: /Size
+9188: integer: 60
+9193: name: /ID
+9197: array_open: [
+9198: string: \x88\x04\x8e\x17\xc9a\xe0\x94\xff\xec\xe9\x8c\xb8\x8cF\xd0 (raw: <88048e17c961e094ffece98cb88c46d0>)
+9232: string: \xed\xd6\x0f\xe8\xee\x87\xf8\x871\xa8o\x81\x9f\xe6Q\x99 (raw: <edd60fe8ee87f88731a86f819fe65199>)
+9266: array_close: ]
+9268: dict_close: >>
+9271: word: stream
 skipping to endstream
-9464: word: endstream
-9474: word: endobj
-9482: word: startxref
-9492: integer: 9056
-9503: eof
+9519: word: endstream
+9529: word: endobj
+9537: word: startxref
+9547: integer: 9111
+9558: eof
 --- END FILE ---
 --- BEGIN PAGE 1 ---
 0: word: BT
@@ -330,41 +330,43 @@ skipping to endstream
 87: string: qu\x0aack (raw: (qu\\x0d\x0dack))
 100: string: qu\x0a\x0a\x0a\x0a\x0a\x0aack (raw: (qu\\x0a\x0a\x0d\x0a\x0a\x0d\x0d\x0a\x0aack))
 120: string: qu\x0a\x0a\x0a\x0a\x0a\x0aack (raw: (qu\x0a\x0d\x0a\x0a\x0d\x0d\x0a\x0aack))
-138: integer: 72
-141: integer: 720
-145: word: Td
-150: real: 3.14
-157: real: 3.
-162: real: .14
-168: real: +3.14
-176: real: +3.
-182: real: +.14
-189: real: -3.14
-197: real: -3.
-203: real: -.14
-210: integer: +16059
-219: integer: -16059
-228: word: +.
-233: bad: <fade\x0aET (invalid character (T) in hexstring)
-242: bad: ) (unexpected ))
-243: bad: > (unexpected >)
-244: word: quack
-250: name: /name\x00oops (raw: /name#oops) (name with stray # will not work with PDF >= 1.2)
-261: name: /name (raw: /n#61me)
-269: word: one
-273: bool: true
-278: word: two
-282: bool: false
-288: word: three
-294: null: null
-299: word: four
-304: word: !@#$^&
-310: brace_open: {
-311: brace_close: }
-312: word: *-_+=
-318: word: abc123def3.14true
-336: bad: <ff\x0a (EOF while reading token)
-340: eof
+138: string: \x048!8Q\x04!Q\x04 (raw: (\48\418\121\4\41\121\4))
+165: string: \x048!8Q\x04!Q! (raw: (\48\418\121\4\41\121\41))
+193: integer: 72
+196: integer: 720
+200: word: Td
+205: real: 3.14
+212: real: 3.
+217: real: .14
+223: real: +3.14
+231: real: +3.
+237: real: +.14
+244: real: -3.14
+252: real: -3.
+258: real: -.14
+265: integer: +16059
+274: integer: -16059
+283: word: +.
+288: bad: <fade\x0aET (invalid character (T) in hexstring)
+297: bad: ) (unexpected ))
+298: bad: > (unexpected >)
+299: word: quack
+305: name: /name\x00oops (raw: /name#oops) (name with stray # will not work with PDF >= 1.2)
+316: name: /name (raw: /n#61me)
+324: word: one
+328: bool: true
+333: word: two
+337: bool: false
+343: word: three
+349: null: null
+354: word: four
+359: word: !@#$^&
+365: brace_open: {
+366: brace_close: }
+367: word: *-_+=
+373: word: abc123def3.14true
+391: bad: <ff\x0a (EOF while reading token)
+395: eof
 --- END PAGE 4 ---
 --- BEGIN PAGE 5 ---
 0: word: BT
diff --git a/qpdf/qtest/qpdf/tokens.out b/qpdf/qtest/qpdf/tokens.out
index b145245b..6601ff97 100644
--- a/qpdf/qtest/qpdf/tokens.out
+++ b/qpdf/qtest/qpdf/tokens.out
@@ -181,352 +181,352 @@ skipping to endstream
 7121: space: \x0a
 7122: word: stream
 skipping to endstream
-7469: word: endstream
-7478: space: \x0a
-7479: word: endobj
-7485: space: \x0a\x0a
-7487: integer: 44
-7489: space:  
-7490: integer: 0
-7491: space:  
-7492: word: obj
-7495: space: \x0a
-7496: integer: 340
-7499: space: \x0a
-7500: word: endobj
-7506: space: \x0a\x0a
-7508: comment: %% Contents for page 5
-7530: space: \x0a
-7531: comment: %% Original object ID: 41 0
-7558: space: \x0a
-7559: integer: 45
-7561: space:  
-7562: integer: 0
-7563: space:  
-7564: word: obj
-7567: space: \x0a
-7568: dict_open: <<
-7570: space: \x0a  
-7573: name: /Length
-7580: space:  
-7581: integer: 46
-7583: space:  
-7584: integer: 0
-7585: space:  
-7586: word: R
-7587: space: \x0a
-7588: dict_close: >>
-7590: space: \x0a
-7591: word: stream
+7524: word: endstream
+7533: space: \x0a
+7534: word: endobj
+7540: space: \x0a\x0a
+7542: integer: 44
+7544: space:  
+7545: integer: 0
+7546: space:  
+7547: word: obj
+7550: space: \x0a
+7551: integer: 395
+7554: space: \x0a
+7555: word: endobj
+7561: space: \x0a\x0a
+7563: comment: %% Contents for page 5
+7585: space: \x0a
+7586: comment: %% Original object ID: 41 0
+7613: space: \x0a
+7614: integer: 45
+7616: space:  
+7617: integer: 0
+7618: space:  
+7619: word: obj
+7622: space: \x0a
+7623: dict_open: <<
+7625: space: \x0a  
+7628: name: /Length
+7635: space:  
+7636: integer: 46
+7638: space:  
+7639: integer: 0
+7640: space:  
+7641: word: R
+7642: space: \x0a
+7643: dict_close: >>
+7645: space: \x0a
+7646: word: stream
 skipping to endstream
-7666: word: endstream
-7675: space: \x0a
-7676: word: endobj
-7682: space: \x0a
-7683: comment: %QDF: ignore_newline
-7703: space: \x0a\x0a
-7705: integer: 46
-7707: space:  
-7708: integer: 0
-7709: space:  
-7710: word: obj
-7713: space: \x0a
-7714: integer: 67
-7716: space: \x0a
-7717: word: endobj
-7723: space: \x0a\x0a
-7725: comment: %% Contents for page 6
-7747: space: \x0a
-7748: comment: %% Original object ID: 42 0
-7775: space: \x0a
-7776: integer: 47
-7778: space:  
-7779: integer: 0
-7780: space:  
-7781: word: obj
-7784: space: \x0a
-7785: dict_open: <<
-7787: space: \x0a  
-7790: name: /Length
-7797: space:  
-7798: integer: 48
-7800: space:  
-7801: integer: 0
-7802: space:  
-7803: word: R
-7804: space: \x0a
-7805: dict_close: >>
-7807: space: \x0a
-7808: word: stream
+7721: word: endstream
+7730: space: \x0a
+7731: word: endobj
+7737: space: \x0a
+7738: comment: %QDF: ignore_newline
+7758: space: \x0a\x0a
+7760: integer: 46
+7762: space:  
+7763: integer: 0
+7764: space:  
+7765: word: obj
+7768: space: \x0a
+7769: integer: 67
+7771: space: \x0a
+7772: word: endobj
+7778: space: \x0a\x0a
+7780: comment: %% Contents for page 6
+7802: space: \x0a
+7803: comment: %% Original object ID: 42 0
+7830: space: \x0a
+7831: integer: 47
+7833: space:  
+7834: integer: 0
+7835: space:  
+7836: word: obj
+7839: space: \x0a
+7840: dict_open: <<
+7842: space: \x0a  
+7845: name: /Length
+7852: space:  
+7853: integer: 48
+7855: space:  
+7856: integer: 0
+7857: space:  
+7858: word: R
+7859: space: \x0a
+7860: dict_close: >>
+7862: space: \x0a
+7863: word: stream
 skipping to endstream
-7859: word: endstream
-7868: space: \x0a
-7869: word: endobj
-7875: space: \x0a\x0a
-7877: integer: 48
-7879: space:  
-7880: integer: 0
-7881: space:  
-7882: word: obj
-7885: space: \x0a
-7886: integer: 44
-7888: space: \x0a
-7889: word: endobj
-7895: space: \x0a\x0a
-7897: comment: %% Contents for page 7
-7919: space: \x0a
-7920: comment: %% Original object ID: 43 0
-7947: space: \x0a
-7948: integer: 49
-7950: space:  
-7951: integer: 0
-7952: space:  
-7953: word: obj
-7956: space: \x0a
-7957: dict_open: <<
-7959: space: \x0a  
-7962: name: /Length
-7969: space:  
-7970: integer: 50
-7972: space:  
-7973: integer: 0
-7974: space:  
-7975: word: R
-7976: space: \x0a
-7977: dict_close: >>
-7979: space: \x0a
-7980: word: stream
+7914: word: endstream
+7923: space: \x0a
+7924: word: endobj
+7930: space: \x0a\x0a
+7932: integer: 48
+7934: space:  
+7935: integer: 0
+7936: space:  
+7937: word: obj
+7940: space: \x0a
+7941: integer: 44
+7943: space: \x0a
+7944: word: endobj
+7950: space: \x0a\x0a
+7952: comment: %% Contents for page 7
+7974: space: \x0a
+7975: comment: %% Original object ID: 43 0
+8002: space: \x0a
+8003: integer: 49
+8005: space:  
+8006: integer: 0
+8007: space:  
+8008: word: obj
+8011: space: \x0a
+8012: dict_open: <<
+8014: space: \x0a  
+8017: name: /Length
+8024: space:  
+8025: integer: 50
+8027: space:  
+8028: integer: 0
+8029: space:  
+8030: word: R
+8031: space: \x0a
+8032: dict_close: >>
+8034: space: \x0a
+8035: word: stream
 skipping to endstream
-8306: word: endstream
-8315: space: \x0a
-8316: word: endobj
-8322: space: \x0a
-8323: comment: %QDF: ignore_newline
-8343: space: \x0a\x0a
-8345: integer: 50
-8347: space:  
-8348: integer: 0
-8349: space:  
-8350: word: obj
-8353: space: \x0a
-8354: integer: 318
-8357: space: \x0a
-8358: word: endobj
-8364: space: \x0a\x0a
-8366: comment: %% Contents for page 8
-8388: space: \x0a
-8389: comment: %% Original object ID: 44 0
-8416: space: \x0a
-8417: integer: 51
-8419: space:  
-8420: integer: 0
-8421: space:  
-8422: word: obj
-8425: space: \x0a
-8426: dict_open: <<
-8428: space: \x0a  
-8431: name: /Length
-8438: space:  
-8439: integer: 52
-8441: space:  
-8442: integer: 0
-8443: space:  
-8444: word: R
-8445: space: \x0a
-8446: dict_close: >>
-8448: space: \x0a
-8449: word: stream
+8361: word: endstream
+8370: space: \x0a
+8371: word: endobj
+8377: space: \x0a
+8378: comment: %QDF: ignore_newline
+8398: space: \x0a\x0a
+8400: integer: 50
+8402: space:  
+8403: integer: 0
+8404: space:  
+8405: word: obj
+8408: space: \x0a
+8409: integer: 318
+8412: space: \x0a
+8413: word: endobj
+8419: space: \x0a\x0a
+8421: comment: %% Contents for page 8
+8443: space: \x0a
+8444: comment: %% Original object ID: 44 0
+8471: space: \x0a
+8472: integer: 51
+8474: space:  
+8475: integer: 0
+8476: space:  
+8477: word: obj
+8480: space: \x0a
+8481: dict_open: <<
+8483: space: \x0a  
+8486: name: /Length
+8493: space:  
+8494: integer: 52
+8496: space:  
+8497: integer: 0
+8498: space:  
+8499: word: R
+8500: space: \x0a
+8501: dict_close: >>
+8503: space: \x0a
+8504: word: stream
 skipping to endstream
-8500: word: endstream
-8509: space: \x0a
-8510: word: endobj
-8516: space: \x0a\x0a
-8518: integer: 52
-8520: space:  
-8521: integer: 0
-8522: space:  
-8523: word: obj
-8526: space: \x0a
-8527: integer: 44
-8529: space: \x0a
-8530: word: endobj
-8536: space: \x0a\x0a
-8538: comment: %% Contents for page 9
-8560: space: \x0a
-8561: comment: %% Original object ID: 45 0
-8588: space: \x0a
-8589: integer: 53
-8591: space:  
-8592: integer: 0
-8593: space:  
-8594: word: obj
-8597: space: \x0a
-8598: dict_open: <<
-8600: space: \x0a  
-8603: name: /Length
-8610: space:  
-8611: integer: 54
-8613: space:  
-8614: integer: 0
-8615: space:  
-8616: word: R
-8617: space: \x0a
-8618: dict_close: >>
-8620: space: \x0a
-8621: word: stream
+8555: word: endstream
+8564: space: \x0a
+8565: word: endobj
+8571: space: \x0a\x0a
+8573: integer: 52
+8575: space:  
+8576: integer: 0
+8577: space:  
+8578: word: obj
+8581: space: \x0a
+8582: integer: 44
+8584: space: \x0a
+8585: word: endobj
+8591: space: \x0a\x0a
+8593: comment: %% Contents for page 9
+8615: space: \x0a
+8616: comment: %% Original object ID: 45 0
+8643: space: \x0a
+8644: integer: 53
+8646: space:  
+8647: integer: 0
+8648: space:  
+8649: word: obj
+8652: space: \x0a
+8653: dict_open: <<
+8655: space: \x0a  
+8658: name: /Length
+8665: space:  
+8666: integer: 54
+8668: space:  
+8669: integer: 0
+8670: space:  
+8671: word: R
+8672: space: \x0a
+8673: dict_close: >>
+8675: space: \x0a
+8676: word: stream
 skipping to endstream
-8672: word: endstream
-8681: space: \x0a
-8682: word: endobj
-8688: space: \x0a\x0a
-8690: integer: 54
-8692: space:  
-8693: integer: 0
-8694: space:  
-8695: word: obj
-8698: space: \x0a
-8699: integer: 44
-8701: space: \x0a
-8702: word: endobj
-8708: space: \x0a\x0a
-8710: comment: %% Contents for page 10
-8733: space: \x0a
-8734: comment: %% Original object ID: 46 0
-8761: space: \x0a
-8762: integer: 55
-8764: space:  
-8765: integer: 0
-8766: space:  
-8767: word: obj
-8770: space: \x0a
-8771: dict_open: <<
-8773: space: \x0a  
-8776: name: /Length
-8783: space:  
-8784: integer: 56
-8786: space:  
-8787: integer: 0
-8788: space:  
-8789: word: R
-8790: space: \x0a
-8791: dict_close: >>
-8793: space: \x0a
-8794: word: stream
+8727: word: endstream
+8736: space: \x0a
+8737: word: endobj
+8743: space: \x0a\x0a
+8745: integer: 54
+8747: space:  
+8748: integer: 0
+8749: space:  
+8750: word: obj
+8753: space: \x0a
+8754: integer: 44
+8756: space: \x0a
+8757: word: endobj
+8763: space: \x0a\x0a
+8765: comment: %% Contents for page 10
+8788: space: \x0a
+8789: comment: %% Original object ID: 46 0
+8816: space: \x0a
+8817: integer: 55
+8819: space:  
+8820: integer: 0
+8821: space:  
+8822: word: obj
+8825: space: \x0a
+8826: dict_open: <<
+8828: space: \x0a  
+8831: name: /Length
+8838: space:  
+8839: integer: 56
+8841: space:  
+8842: integer: 0
+8843: space:  
+8844: word: R
+8845: space: \x0a
+8846: dict_close: >>
+8848: space: \x0a
+8849: word: stream
 skipping to endstream
-8845: word: endstream
-8854: space: \x0a
-8855: word: endobj
-8861: space: \x0a\x0a
-8863: integer: 56
-8865: space:  
-8866: integer: 0
-8867: space:  
-8868: word: obj
-8871: space: \x0a
-8872: integer: 44
-8874: space: \x0a
-8875: word: endobj
-8881: space: \x0a\x0a
-8883: comment: %% Contents for page 11
-8906: space: \x0a
-8907: comment: %% Original object ID: 47 0
-8934: space: \x0a
-8935: integer: 57
-8937: space:  
-8938: integer: 0
-8939: space:  
-8940: word: obj
-8943: space: \x0a
-8944: dict_open: <<
-8946: space: \x0a  
-8949: name: /Length
-8956: space:  
-8957: integer: 58
-8959: space:  
-8960: integer: 0
-8961: space:  
-8962: word: R
-8963: space: \x0a
-8964: dict_close: >>
-8966: space: \x0a
-8967: word: stream
+8900: word: endstream
+8909: space: \x0a
+8910: word: endobj
+8916: space: \x0a\x0a
+8918: integer: 56
+8920: space:  
+8921: integer: 0
+8922: space:  
+8923: word: obj
+8926: space: \x0a
+8927: integer: 44
+8929: space: \x0a
+8930: word: endobj
+8936: space: \x0a\x0a
+8938: comment: %% Contents for page 11
+8961: space: \x0a
+8962: comment: %% Original object ID: 47 0
+8989: space: \x0a
+8990: integer: 57
+8992: space:  
+8993: integer: 0
+8994: space:  
+8995: word: obj
+8998: space: \x0a
+8999: dict_open: <<
+9001: space: \x0a  
+9004: name: /Length
+9011: space:  
+9012: integer: 58
+9014: space:  
+9015: integer: 0
+9016: space:  
+9017: word: R
+9018: space: \x0a
+9019: dict_close: >>
+9021: space: \x0a
+9022: word: stream
 skipping to endstream
-9018: word: endstream
-9027: space: \x0a
-9028: word: endobj
-9034: space: \x0a\x0a
-9036: integer: 58
-9038: space:  
-9039: integer: 0
-9040: space:  
-9041: word: obj
-9044: space: \x0a
-9045: integer: 44
-9047: space: \x0a
-9048: word: endobj
-9054: space: \x0a\x0a
-9056: integer: 59
-9058: space:  
-9059: integer: 0
-9060: space:  
-9061: word: obj
-9064: space: \x0a
-9065: dict_open: <<
-9067: space: \x0a  
-9070: name: /Type
-9075: space:  
-9076: name: /XRef
-9081: space: \x0a  
-9084: name: /Length
-9091: space:  
-9092: integer: 240
-9095: space: \x0a  
-9098: name: /W
-9100: space:  
-9101: array_open: [
-9102: space:  
-9103: integer: 1
-9104: space:  
-9105: integer: 2
-9106: space:  
-9107: integer: 1
-9108: space:  
-9109: array_close: ]
-9110: space: \x0a  
-9113: name: /Root
-9118: space:  
-9119: integer: 2
-9120: space:  
-9121: integer: 0
-9122: space:  
-9123: word: R
-9124: space: \x0a  
-9127: name: /Size
-9132: space:  
-9133: integer: 60
-9135: space: \x0a  
-9138: name: /ID
-9141: space:  
-9142: array_open: [
-9143: string: \x88\x04\x8e\x17\xc9a\xe0\x94\xff\xec\xe9\x8c\xb8\x8cF\xd0 (raw: <88048e17c961e094ffece98cb88c46d0>)
-9177: string: \xed\xd6\x0f\xe8\xee\x87\xf8\x871\xa8o\x81\x9f\xe6Q\x99 (raw: <edd60fe8ee87f88731a86f819fe65199>)
-9211: array_close: ]
-9212: space: \x0a
-9213: dict_close: >>
-9215: space: \x0a
-9216: word: stream
+9073: word: endstream
+9082: space: \x0a
+9083: word: endobj
+9089: space: \x0a\x0a
+9091: integer: 58
+9093: space:  
+9094: integer: 0
+9095: space:  
+9096: word: obj
+9099: space: \x0a
+9100: integer: 44
+9102: space: \x0a
+9103: word: endobj
+9109: space: \x0a\x0a
+9111: integer: 59
+9113: space:  
+9114: integer: 0
+9115: space:  
+9116: word: obj
+9119: space: \x0a
+9120: dict_open: <<
+9122: space: \x0a  
+9125: name: /Type
+9130: space:  
+9131: name: /XRef
+9136: space: \x0a  
+9139: name: /Length
+9146: space:  
+9147: integer: 240
+9150: space: \x0a  
+9153: name: /W
+9155: space:  
+9156: array_open: [
+9157: space:  
+9158: integer: 1
+9159: space:  
+9160: integer: 2
+9161: space:  
+9162: integer: 1
+9163: space:  
+9164: array_close: ]
+9165: space: \x0a  
+9168: name: /Root
+9173: space:  
+9174: integer: 2
+9175: space:  
+9176: integer: 0
+9177: space:  
+9178: word: R
+9179: space: \x0a  
+9182: name: /Size
+9187: space:  
+9188: integer: 60
+9190: space: \x0a  
+9193: name: /ID
+9196: space:  
+9197: array_open: [
+9198: string: \x88\x04\x8e\x17\xc9a\xe0\x94\xff\xec\xe9\x8c\xb8\x8cF\xd0 (raw: <88048e17c961e094ffece98cb88c46d0>)
+9232: string: \xed\xd6\x0f\xe8\xee\x87\xf8\x871\xa8o\x81\x9f\xe6Q\x99 (raw: <edd60fe8ee87f88731a86f819fe65199>)
+9266: array_close: ]
+9267: space: \x0a
+9268: dict_close: >>
+9270: space: \x0a
+9271: word: stream
 skipping to endstream
-9464: word: endstream
-9473: space: \x0a
-9474: word: endobj
-9480: space: \x0a\x0a
-9482: word: startxref
-9491: space: \x0a
-9492: integer: 9056
-9496: space: \x0a
-9497: comment: %%EOF
-9502: space: \x0a
-9503: eof
+9519: word: endstream
+9528: space: \x0a
+9529: word: endobj
+9535: space: \x0a\x0a
+9537: word: startxref
+9546: space: \x0a
+9547: integer: 9111
+9551: space: \x0a
+9552: comment: %%EOF
+9557: space: \x0a
+9558: eof
 --- END FILE ---
 --- BEGIN PAGE 1 ---
 0: word: BT
@@ -669,69 +669,73 @@ skipping to endstream
 117: space: \x0a  
 120: string: qu\x0a\x0a\x0a\x0a\x0a\x0aack (raw: (qu\x0a\x0d\x0a\x0a\x0d\x0d\x0a\x0aack))
 135: space: \x0a  
-138: integer: 72
-140: space:  
-141: integer: 720
-144: space:  
-145: word: Td
-147: space: \x0a  
-150: real: 3.14
-154: space: \x0a  
-157: real: 3.
-159: space: \x0a  
-162: real: .14
-165: space: \x0a  
-168: real: +3.14
-173: space: \x0a  
-176: real: +3.
-179: space: \x0a  
-182: real: +.14
-186: space: \x0a  
-189: real: -3.14
-194: space: \x0a  
-197: real: -3.
-200: space: \x0a  
-203: real: -.14
-207: space: \x0a  
-210: integer: +16059
-216: space: \x0a  
-219: integer: -16059
-225: space: \x0a  
-228: word: +.
-230: space: \x0a  
-233: bad: <fade\x0aET (invalid character (T) in hexstring)
-241: space: \x0a
-242: bad: ) (unexpected ))
-243: bad: > (unexpected >)
-244: word: quack
-249: space:  
-250: name: /name\x00oops (raw: /name#oops) (name with stray # will not work with PDF >= 1.2)
-260: space:  
-261: name: /name (raw: /n#61me)
-268: space:  
-269: word: one
-272: space:  
-273: bool: true
-277: space:  
-278: word: two
-281: space:  
-282: bool: false
-287: space:  
-288: word: three
-293: space:  
-294: null: null
-298: space:  
-299: word: four
-303: space: \x0a
-304: word: !@#$^&
-310: brace_open: {
-311: brace_close: }
-312: word: *-_+=
-317: space: \x0a
-318: word: abc123def3.14true
-335: space: \x0a
-336: bad: <ff\x0a (EOF while reading token)
-340: eof
+138: string: \x048!8Q\x04!Q\x04 (raw: (\48\418\121\4\41\121\4))
+162: space: \x0a  
+165: string: \x048!8Q\x04!Q! (raw: (\48\418\121\4\41\121\41))
+190: space: \x0a  
+193: integer: 72
+195: space:  
+196: integer: 720
+199: space:  
+200: word: Td
+202: space: \x0a  
+205: real: 3.14
+209: space: \x0a  
+212: real: 3.
+214: space: \x0a  
+217: real: .14
+220: space: \x0a  
+223: real: +3.14
+228: space: \x0a  
+231: real: +3.
+234: space: \x0a  
+237: real: +.14
+241: space: \x0a  
+244: real: -3.14
+249: space: \x0a  
+252: real: -3.
+255: space: \x0a  
+258: real: -.14
+262: space: \x0a  
+265: integer: +16059
+271: space: \x0a  
+274: integer: -16059
+280: space: \x0a  
+283: word: +.
+285: space: \x0a  
+288: bad: <fade\x0aET (invalid character (T) in hexstring)
+296: space: \x0a
+297: bad: ) (unexpected ))
+298: bad: > (unexpected >)
+299: word: quack
+304: space:  
+305: name: /name\x00oops (raw: /name#oops) (name with stray # will not work with PDF >= 1.2)
+315: space:  
+316: name: /name (raw: /n#61me)
+323: space:  
+324: word: one
+327: space:  
+328: bool: true
+332: space:  
+333: word: two
+336: space:  
+337: bool: false
+342: space:  
+343: word: three
+348: space:  
+349: null: null
+353: space:  
+354: word: four
+358: space: \x0a
+359: word: !@#$^&
+365: brace_open: {
+366: brace_close: }
+367: word: *-_+=
+372: space: \x0a
+373: word: abc123def3.14true
+390: space: \x0a
+391: bad: <ff\x0a (EOF while reading token)
+395: eof
 --- END PAGE 4 ---
 --- BEGIN PAGE 5 ---
 0: word: BT
diff --git a/qpdf/qtest/qpdf/tokens.pdf b/qpdf/qtest/qpdf/tokens.pdf
index e9278c4c..a96c1eb9 100644
--- a/qpdf/qtest/qpdf/tokens.pdf
+++ b/qpdf/qtest/qpdf/tokens.pdf
diff --git a/qpdf/qtest/tokenizer.test b/qpdf/qtest/tokenizer.test
index b6e440b8..aa2a6cd3 100644
--- a/qpdf/qtest/tokenizer.test
+++ b/qpdf/qtest/tokenizer.test
@@ -14,7 +14,7 @@ cleanup();
 
 my $td = new TestDriver('tokenizer');
 
-my $n_tests = 4;
+my $n_tests = 5;
 
 $td->runtest("tokenizer with no ignorable",
              {$td->COMMAND => "test_tokenizer -no-ignorable tokens.pdf"},
@@ -38,5 +38,11 @@ $td->runtest("ignore bad token",
               $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 
+$td->runtest("quoted char edge cases",
+             {$td->COMMAND => "test_driver 96 -"},
+             {$td->STRING => "test 96 done\n",
+              $td->EXIT_STATUS => 0},
+             $td->NORMALIZE_NEWLINES);
+
 cleanup();
 $td->report($n_tests);
diff --git a/qpdf/test_driver.cc b/qpdf/test_driver.cc
index 2e661fdb..798bd623 100644
--- a/qpdf/test_driver.cc
+++ b/qpdf/test_driver.cc
@@ -3341,6 +3341,17 @@ test_95(QPDF& pdf, char const* arg2)
     assert(!oh_d.isScalar());
 }
 
+static void
+test_96(QPDF& pdf, char const* arg2)
+{
+    // Test edge cases with quoted characters
+
+    auto s = R"((\48\418\121\4))"_qpdf;
+    assert(s.unparseBinary() == "<043821385104>");
+    s = R"((\48\418\121\41))"_qpdf;
+    assert(s.unparseBinary() == "<043821385121>");
+}
+
 void
 runtest(int n, char const* filename1, char const* arg2)
 {
@@ -3348,7 +3359,7 @@ runtest(int n, char const* filename1, char const* arg2)
     // the test suite to see how the test is invoked to find the file
     // that the test is supposed to operate on.
 
-    std::set<int> ignore_filename = {61, 81, 83, 84, 85, 86, 87, 92, 95};
+    std::set<int> ignore_filename = {61, 81, 83, 84, 85, 86, 87, 92, 95, 96};
 
     if (n == 0) {
         // Throw in some random test cases that don't fit anywhere
@@ -3441,7 +3452,8 @@ runtest(int n, char const* filename1, char const* arg2)
         {72, test_72}, {73, test_73}, {74, test_74}, {75, test_75}, {76, test_76}, {77, test_77},
         {78, test_78}, {79, test_79}, {80, test_80}, {81, test_81}, {82, test_82}, {83, test_83},
         {84, test_84}, {85, test_85}, {86, test_86}, {87, test_87}, {88, test_88}, {89, test_89},
-        {90, test_90}, {91, test_91}, {92, test_92}, {93, test_93}, {94, test_94}, {95, test_95}};
+        {90, test_90}, {91, test_91}, {92, test_92}, {93, test_93}, {94, test_94}, {95, test_95},
+        {96, test_96}};
 
     auto fn = test_functions.find(n);
     if (fn == test_functions.end()) {