Merge pull request #752 from jberkenbilt/report-mem-usage

Report mem usage

9 files changed, 123 insertions, 1 deletions

diff --git a/libqpdf/CMakeLists.txt b/libqpdf/CMakeLists.txt
index 46d35959..51f7476d 100644
--- a/libqpdf/CMakeLists.txt
+++ b/libqpdf/CMakeLists.txt
@@ -376,6 +376,29 @@ int main(int argc, char* argv[]) {
   endif()
 endfunction()
 
+check_c_source_compiles(
+"#include <malloc.h>
+#include <stdio.h>
+int main(int argc, char* argv[]) {
+    malloc_info(0, stdout);
+    return 0;
+}"
+    HAVE_MALLOC_INFO)
+
+check_c_source_compiles(
+"#include <stdio.h>
+#include <stdlib.h>
+int main(int argc, char* argv[]) {
+    char* buf;
+    size_t size;
+    FILE* f;
+    f = open_memstream(&buf, &size);
+    fclose(f);
+    free(buf);
+    return 0;
+}"
+    HAVE_OPEN_MEMSTREAM)
+
 qpdf_check_ll_fmt("%lld" fmt_lld)
 qpdf_check_ll_fmt("%I64d" fmt_i64d)
 qpdf_check_ll_fmt("%I64lld" fmt_i64lld)
diff --git a/libqpdf/QPDFJob.cc b/libqpdf/QPDFJob.cc
index 15a81854..7bd563aa 100644
--- a/libqpdf/QPDFJob.cc
+++ b/libqpdf/QPDFJob.cc
@@ -417,7 +417,8 @@ QPDFJob::Members::Members() :
     check_is_encrypted(false),
     check_requires_password(false),
     json_input(false),
-    json_output(false)
+    json_output(false),
+    report_mem_usage(false)
 {
 }
 
@@ -625,6 +626,14 @@ QPDFJob::run()
                                      << ": operation succeeded with warnings\n";
         }
     }
+    if (m->report_mem_usage) {
+        // Call get_max_memory_usage before generating output. When
+        // debugging, it's easier if print statements from
+        // get_max_memory_usage are not interleaved with the output.
+        auto mem_usage = QUtil::get_max_memory_usage();
+        *this->m->log->getWarn()
+            << "qpdf-max-memory-usage " << mem_usage << "\n";
+    }
 }
 
 bool
diff --git a/libqpdf/QPDFJob_config.cc b/libqpdf/QPDFJob_config.cc
index 8a9c1470..3e148fca 100644
--- a/libqpdf/QPDFJob_config.cc
+++ b/libqpdf/QPDFJob_config.cc
@@ -503,6 +503,13 @@ QPDFJob::Config::removePageLabels()
 }
 
 QPDFJob::Config*
+QPDFJob::Config::reportMemUsage()
+{
+    o.m->report_mem_usage = true;
+    return this;
+}
+
+QPDFJob::Config*
 QPDFJob::Config::requiresPassword()
 {
     o.m->check_requires_password = true;
diff --git a/libqpdf/QUtil.cc b/libqpdf/QUtil.cc
index d565ece0..98a8f318 100644
--- a/libqpdf/QUtil.cc
+++ b/libqpdf/QUtil.cc
@@ -37,6 +37,9 @@
 # include <sys/stat.h>
 # include <unistd.h>
 #endif
+#ifdef HAVE_MALLOC_INFO
+# include <malloc.h>
+#endif
 
 // First element is 24
 static unsigned short pdf_doc_low_to_unicode[] = {
@@ -1968,3 +1971,73 @@ QUtil::call_main_from_wmain(
 }
 
 #endif // QPDF_NO_WCHAR_T
+
+size_t
+QUtil::get_max_memory_usage()
+{
+#if defined(HAVE_MALLOC_INFO) && defined(HAVE_OPEN_MEMSTREAM)
+    static std::regex tag_re("<(/?\\w+)([^>]*?)>");
+    static std::regex attr_re("(\\w+)=\"(.*?)\"");
+
+    char* buf;
+    size_t size;
+    FILE* f = open_memstream(&buf, &size);
+    if (f == nullptr) {
+        return 0;
+    }
+    malloc_info(0, f);
+    fclose(f);
+    if (QUtil::get_env("QPDF_DEBUG_MEM_USAGE")) {
+        fprintf(stderr, "%s", buf);
+    }
+
+    // Warning: this code uses regular expression to extract data from
+    // an XML string. This is generally a bad idea, but we're going to
+    // do it anyway because QUtil.hh warns against using this function
+    // for other than development/testing, and if this function fails
+    // to generate reasonable output during performance testing, it
+    // will be noticed.
+
+    // This is my best guess at how to interpret malloc_info. Anyway
+    // it seems to provide useful information for detecting code
+    // changes that drastically change memory usage.
+    size_t result = 0;
+    try {
+        std::cregex_iterator m_begin(buf, buf + size, tag_re);
+        std::cregex_iterator cr_end;
+        std::sregex_iterator sr_end;
+
+        int in_heap = 0;
+        for (auto m = m_begin; m != cr_end; ++m) {
+            std::string tag(m->str(1));
+            if (tag == "heap") {
+                ++in_heap;
+            } else if (tag == "/heap") {
+                --in_heap;
+            } else if (in_heap == 0) {
+                std::string rest = m->str(2);
+                std::map<std::string, std::string> attrs;
+                std::sregex_iterator a_begin(rest.begin(), rest.end(), attr_re);
+                for (auto m2 = a_begin; m2 != sr_end; ++m2) {
+                    attrs[m2->str(1)] = m2->str(2);
+                }
+                if (tag == "total") {
+                    if (attrs.count("size") > 0) {
+                        result += QIntC::to_size(
+                            QUtil::string_to_ull(attrs["size"].c_str()));
+                    }
+                } else if (tag == "system" && attrs["type"] == "max") {
+                    result += QIntC::to_size(
+                        QUtil::string_to_ull(attrs["size"].c_str()));
+                }
+            }
+        }
+    } catch (...) {
+        // ignore -- just return 0
+    }
+    free(buf);
+    return result;
+#else
+    return 0;
+#endif
+}
diff --git a/libqpdf/qpdf/auto_job_help.hh b/libqpdf/qpdf/auto_job_help.hh
index 7c3bb266..eb272a04 100644
--- a/libqpdf/qpdf/auto_job_help.hh
+++ b/libqpdf/qpdf/auto_job_help.hh
@@ -883,6 +883,9 @@ for debugging qpdf.
 ap.addOptionHelp("--test-json-schema", "testing", "test generated json against schema", R"(This is used by qpdf's test suite to check consistency between
 the output of qpdf --json and the output of qpdf --json-help.
 )");
+ap.addOptionHelp("--report-mem-usage", "testing", "best effort report of memory usage", R"(This is used by qpdf's performance test suite to report the
+maximum amount of memory used in supported environments.
+)");
 }
 static void add_help(QPDFArgParser& ap)
 {
diff --git a/libqpdf/qpdf/auto_job_init.hh b/libqpdf/qpdf/auto_job_init.hh
index b90592e0..ad110d16 100644
--- a/libqpdf/qpdf/auto_job_init.hh
+++ b/libqpdf/qpdf/auto_job_init.hh
@@ -69,6 +69,7 @@ this->ap.addBare("raw-stream-data", [this](){c_main->rawStreamData();});
 this->ap.addBare("recompress-flate", [this](){c_main->recompressFlate();});
 this->ap.addBare("remove-page-labels", [this](){c_main->removePageLabels();});
 this->ap.addBare("replace-input", b(&ArgParser::argReplaceInput));
+this->ap.addBare("report-mem-usage", [this](){c_main->reportMemUsage();});
 this->ap.addBare("requires-password", [this](){c_main->requiresPassword();});
 this->ap.addBare("show-encryption", [this](){c_main->showEncryption();});
 this->ap.addBare("show-encryption-key", [this](){c_main->showEncryptionKey();});
diff --git a/libqpdf/qpdf/auto_job_json_init.hh b/libqpdf/qpdf/auto_job_json_init.hh
index 8f8fb987..1cd69368 100644
--- a/libqpdf/qpdf/auto_job_json_init.hh
+++ b/libqpdf/qpdf/auto_job_json_init.hh
@@ -409,6 +409,9 @@ popHandler(); // key: pages
 pushKey("removePageLabels");
 addBare([this]() { c_main->removePageLabels(); });
 popHandler(); // key: removePageLabels
+pushKey("reportMemUsage");
+addBare([this]() { c_main->reportMemUsage(); });
+popHandler(); // key: reportMemUsage
 pushKey("rotate");
 addParameter([this](std::string const& p) { c_main->rotate(p); });
 popHandler(); // key: rotate
diff --git a/libqpdf/qpdf/auto_job_schema.hh b/libqpdf/qpdf/auto_job_schema.hh
index aa69c192..9272c596 100644
--- a/libqpdf/qpdf/auto_job_schema.hh
+++ b/libqpdf/qpdf/auto_job_schema.hh
@@ -144,6 +144,7 @@ static constexpr char const* JOB_SCHEMA_DATA = R"({
     }
   ],
   "removePageLabels": "remove explicit page numbers",
+  "reportMemUsage": "best effort report of memory usage",
   "rotate": "rotate pages",
   "overlay": {
     "file": "source file for overlay",
diff --git a/libqpdf/qpdf/qpdf-config.h.in b/libqpdf/qpdf/qpdf-config.h.in
index 8a22b875..500f55cc 100644
--- a/libqpdf/qpdf/qpdf-config.h.in
+++ b/libqpdf/qpdf/qpdf-config.h.in
@@ -21,6 +21,8 @@
 #cmakedefine HAVE_LOCALTIME_R 1
 #cmakedefine HAVE_RANDOM 1
 #cmakedefine HAVE_TM_GMTOFF 1
+#cmakedefine HAVE_MALLOC_INFO 1
+#cmakedefine HAVE_OPEN_MEMSTREAM 1
 
 /* printf format for long long */
 #cmakedefine LL_FMT "${LL_FMT}"