QUtil::read_lines_from_file: optional EOL preservation

3 files changed, 43 insertions, 4 deletions

diff --git a/ChangeLog b/ChangeLog
index b4dd7c04..64518c5e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2020-01-13  Jay Berkenbilt  <ejb@ql.org>
+
+	* QUtil::read_lines_from_file: add new versions that use FILE*,
+	use FILE* instead if std::ifstream internally to support correct
+	handling of Unicode filenames in Windows, and add the option to
+	preserve line endings.
+
 2019-11-17  Jay Berkenbilt  <ejb@ql.org>
 
 	* 9.1.0: release
diff --git a/libqpdf/QUtil.cc b/libqpdf/QUtil.cc
index 42647504..70066b32 100644
--- a/libqpdf/QUtil.cc
+++ b/libqpdf/QUtil.cc
@@ -1116,11 +1116,18 @@ QUtil::read_lines_from_file(std::function<bool(char&)> next_char,
 	}
 	if (c == '\n')
 	{
-            // Remove any carriage return that preceded the
-            // newline and discard the newline
-            if ((! buf->empty()) && ((*(buf->rbegin())) == '\r'))
+            if (preserve_eol)
             {
-                buf->erase(buf->length() - 1);
+                buf->append(1, c);
+            }
+            else
+            {
+                // Remove any carriage return that preceded the
+                // newline and discard the newline
+                if ((! buf->empty()) && ((*(buf->rbegin())) == '\r'))
+                {
+                    buf->erase(buf->length() - 1);
+                }
             }
 	    buf = 0;
 	}
diff --git a/libtests/qutil.cc b/libtests/qutil.cc
index 0331d715..935cdfc2 100644
--- a/libtests/qutil.cc
+++ b/libtests/qutil.cc
@@ -418,6 +418,31 @@ void read_from_file_test()
         fclose(fp);
     }
 
+    // Test with EOL preservation
+    std::list<std::string> lines2 =
+        QUtil::read_lines_from_file("other-file", true);
+    auto line = lines2.begin();
+    assert(37 == (*line).length());
+    assert('.' == (*line).at(35));
+    assert('\n' == (*line).at(36));
+    ++line;
+    assert(24 == (*line).length());
+    assert('.' == (*line).at(21));
+    assert('\r' == (*line).at(22));
+    assert('\n' == (*line).at(23));
+    ++line;
+    assert(24591 == (*line).length());
+    assert('.' == (*line).at(24589));
+    assert('\n' == (*line).at(24590));
+    // Test the other versions and make sure we get the same results
+    {
+        std::ifstream infs("other-file", std::ios_base::binary);
+        assert(QUtil::read_lines_from_file(infs, true) == lines2);
+        FILE* fp = QUtil::safe_fopen("other-file", "rb");
+        assert(QUtil::read_lines_from_file(fp, true) == lines2);
+        fclose(fp);
+    }
+
     PointerHolder<char> buf;
     size_t size = 0;
     QUtil::read_file_into_memory("other-file", buf, size);