Stop adding pre-built documentation to the source distribution

10 files changed, 178 insertions, 91 deletions

diff --git a/manual/README b/manual/README
deleted file mode 100644
index 88118ac0..00000000
--- a/manual/README
+++ /dev/null
@@ -1,3 +0,0 @@
-This directory contains sources to the documentation.  If you are
-looking for pre-built documentation, please look in the "doc"
-directory.
diff --git a/manual/build.mk b/manual/build.mk
index b07bf830..bca1253d 100644
--- a/manual/build.mk
+++ b/manual/build.mk
@@ -6,7 +6,10 @@ HTML_TARGET := $(HTML_OUT)/index.html
 PDF_OUT := $(DOC_OUT)/latex
 PDF_TARGET := $(PDF_OUT)/qpdf.pdf
 
-TARGETS_manual := doc/qpdf.1 doc/fix-qdf.1 doc/zlib-flate.1
+TARGETS_manual := \
+    $(DOC_OUT)/qpdf.1 \
+    $(DOC_OUT)/fix-qdf.1 \
+    $(DOC_OUT)/zlib-flate.1
 ifeq ($(BUILD_HTML),1)
 TARGETS_manual += $(HTML_TARGET) $(S_HTML_TARGET)
 endif
@@ -22,23 +25,31 @@ MANUAL_DEPS = $(wildcard manual/*.rst) manual/conf.py
 # the error "_pickle.UnpicklingError: pickle data was truncated"
 $(HTML_TARGET): $(MANUAL_DEPS)
 	$(SPHINX) -M html manual $(DOC_OUT) -W
-	mkdir -p doc
-	rm -rf doc/html
-	cp -r $(DOC_OUT)/html doc
 
 $(S_HTML_TARGET): $(MANUAL_DEPS) | $(HTML_TARGET)
 	$(SPHINX) -M singlehtml manual $(DOC_OUT) -W
-	mkdir -p doc
-	rm -rf doc/singlehtml
-	cp -r $(DOC_OUT)/singlehtml doc
 
 $(PDF_TARGET): $(MANUAL_DEPS) | $(S_HTML_TARGET) $(HTML_TARGET)
 	$(SPHINX) -M latexpdf manual $(DOC_OUT) -W
-	mkdir -p doc
-	cp $(PDF_TARGET) doc/qpdf-manual.pdf
 
-doc/%.1: manual/%.1.in
-	mkdir -p doc
-	sed -e 's:@PACKAGE_VERSION@:$(PACKAGE_VERSION):g' \
-	    -e 's:@docdir@:$(docdir):g' \
-	    < $< > $@
+$(DOC_OUT)/%.1: manual/%.1.in
+	sed -e 's:@PACKAGE_VERSION@:$(PACKAGE_VERSION):g' < $< > $@
+
+# The doc-dist target must not removed $(DOC_DEST) so that it works to
+# do stuff like make doc-dist DOC_DEST=$(DESTDIR)/$(docdir). Make sure
+# what this does is consistent with ../README-doc.txt and the
+# information in the manual and ../README.md.
+.PHONY: doc-dist
+doc-dist: build_manual
+	@if test x"$(DOC_DEST)" = x; then \
+	    echo DOC_DEST must be set 1>& 2; \
+	    false; \
+	fi
+	if test -d $(DOC_DEST); then \
+	    $(RM) -rf $(DOC_DEST)/*html $(DOC_DEST)/*.pdf; \
+	else \
+	    mkdir -p $(DOC_DEST); \
+	fi
+	cp -r $(DOC_OUT)/html doc
+	cp -r $(DOC_OUT)/singlehtml doc
+	cp $(PDF_TARGET) $(DOC_DEST)/qpdf-manual.pdf
diff --git a/manual/conf.py b/manual/conf.py
index 3fb14c5a..92e4a3de 100644
--- a/manual/conf.py
+++ b/manual/conf.py
@@ -11,6 +11,7 @@ import sphinx_rtd_theme  # noQA F401
 project = 'QPDF'
 copyright = '2005-2021, Jay Berkenbilt'
 author = 'Jay Berkenbilt'
+# make_dist and the CI build lexically find the release version from this file.
 release = '10.5.0'
 version = release
 extensions = [
diff --git a/manual/fix-qdf.1.in b/manual/fix-qdf.1.in
index ef3b8022..69a588d7 100644
--- a/manual/fix-qdf.1.in
+++ b/manual/fix-qdf.1.in
@@ -14,8 +14,6 @@ the same file with stream lengths, cross-reference table entries, and
 object stream offset tables regenerated.
 .PP
 For details about fix-qdf and about PDF files in QDF mode, please see
-the qpdf manual, which can be found in @docdir@/html/index.html or
-@docdir@/qpdf-manual.pdf. The latest version of the documentation can
-be found at https://qpdf.readthedocs.io.
+the qpdf manual, which can be found at https://qpdf.readthedocs.io.
 .SH "SEE ALSO"
 qpdf(1)
diff --git a/manual/index.rst b/manual/index.rst
index 992850e7..56413cf6 100644
--- a/manual/index.rst
+++ b/manual/index.rst
@@ -21,6 +21,7 @@ documentation, please visit `https://qpdf.readthedocs.io
    license
    download
    installation
+   packaging
    cli
    qdf
    library
diff --git a/manual/installation.rst b/manual/installation.rst
index 197977e7..0bd13e31 100644
--- a/manual/installation.rst
+++ b/manual/installation.rst
@@ -60,14 +60,7 @@ suite. Note that in no case are these items required to use qpdf.
 If you do not enable this, then you do not need to have tiff and
 ghostscript.
 
-Pre-built documentation is distributed with qpdf, so you should
-generally not need to rebuild the documentation. In order to build the
-documentation from source, you need to install `Sphinx
-<https://sphinx-doc.org>`__ along with the ``sphinx_rtd_doc`` theme
-(``pip install sphinx sphinx_rtd_theme``). To build the PDF version of
-the documentation, you need ``pdflatex``, ``latexmk``, and a fairly
-complete LaTeX installation. Detailed requirements can be found in the
-Sphinx documentation.
+For information on building the documentation, see :ref:`build-doc`.
 
 .. _building:
 
@@ -113,6 +106,23 @@ hand-crafted non-recursive Makefile that requires gnu make. If you're
 really interested, please read the comments in the top-level
 :file:`Makefile`.
 
+.. _build-doc:
+
+Building Documentation
+----------------------
+
+The qpdf manual is written in reStructured Text and built with `Sphinx
+<https://www.sphinx-doc.org>`__ using the `Read the Docs Sphinx Theme
+<https://sphinx-rtd-theme.readthedocs.io>`__. In order to build the
+HTML documentation from source, you need to install sphinx and the
+theme, which you can typically do with ``pip install sphinx
+sphinx_rtd_theme``. To build the PDF version of the documentation, you
+need ``pdflatex``, ``latexmk``, and a fairly complete LaTeX
+installation. Detailed requirements can be found in the Sphinx
+documentation. To see how the documentation is built for the qpdf
+distribution, refer to the :file:`build-scripts/build-doc` file in the
+qpdf source distribution.
+
 .. _crypto:
 
 Crypto Providers
@@ -298,46 +308,3 @@ provide their own implementations for basic filters like
 Implementing the registration functions and internal storage of
 registered providers was also easier using C++-11's functional
 interfaces, which was another reason to require C++-11 at this time.
-
-.. _packaging:
-
-Notes for Packagers
--------------------
-
-If you are packaging qpdf for an operating system distribution, here are
-some things you may want to keep in mind:
-
-- Starting in qpdf version 9.1.1, qpdf no longer has a runtime
-  dependency on perl. This is because fix-qdf was rewritten in C++.
-  However, qpdf still has a build-time dependency on perl.
-
-- Make sure you are getting the intended behavior with regard to crypto
-  providers. Read :ref:`crypto.build` for details.
-
-- Passing :samp:`--enable-show-failed-test-output` to
-  :command:`./configure` will cause any failed test
-  output to be written to the console. This can be very useful for
-  seeing test failures generated by autobuilders where you can't access
-  qtest.log after the fact.
-
-- If qpdf's build environment detects the presence of autoconf and
-  related tools, it will check to ensure that automatically generated
-  files are up-to-date with recorded checksums and fail if it detects a
-  discrepancy. This feature is intended to prevent you from
-  accidentally forgetting to regenerate automatic files after modifying
-  their sources. If your packaging environment automatically refreshes
-  automatic files, it can cause this check to fail. Suppress qpdf's
-  checks by passing :samp:`--disable-check-autofiles`
-  to :command:`/.configure`. This is safe since qpdf's
-  :command:`autogen.sh` just runs autotools in the
-  normal way.
-
-- QPDF's :command:`make install` does not install
-  completion files by default, but as a packager, it's good if you
-  install them wherever your distribution expects such files to go. You
-  can find completion files to install in the
-  :file:`completions` directory.
-
-- Packagers are encouraged to install the source files from the
-  :file:`examples` directory along with qpdf
-  development packages.
diff --git a/manual/packaging.rst b/manual/packaging.rst
new file mode 100644
index 00000000..6ff77e51
--- /dev/null
+++ b/manual/packaging.rst
@@ -0,0 +1,120 @@
+.. _packaging:
+
+Notes for Packagers
+===================
+
+If you are packaging qpdf for an operating system distribution, this
+chapter is for you. Otherwise, feel free to skip.
+
+Build Options
+-------------
+
+- Perl must be present at build time. Prior to qpdf version 9.1.1,
+  there was a runtime dependency on perl, but this is no longer the
+  case.
+
+- Make sure you are getting the intended behavior with regard to crypto
+  providers. Read :ref:`crypto.build` for details.
+
+- Passing :samp:`--enable-show-failed-test-output` to
+  :command:`./configure` will cause any failed test
+  output to be written to the console. This can be very useful for
+  seeing test failures generated by autobuilders where you can't access
+  qtest.log after the fact.
+
+- If qpdf's build environment detects the presence of autoconf and
+  related tools, it will check to ensure that automatically generated
+  files are up-to-date with recorded checksums and fail if it detects a
+  discrepancy. This feature is intended to prevent you from
+  accidentally forgetting to regenerate automatic files after modifying
+  their sources. If your packaging environment automatically refreshes
+  automatic files, it can cause this check to fail. Suppress qpdf's
+  checks by passing :samp:`--disable-check-autofiles`
+  to :command:`/.configure`. This is safe since qpdf's
+  :command:`autogen.sh` just runs autotools in the
+  normal way.
+
+- QPDF's :command:`make install` does not install completion files by
+  default, but as a packager, it's good if you install them wherever
+  your distribution expects such files to go. You can find completion
+  files to install in the :file:`completions` directory.
+
+- Packagers are encouraged to install the source files from the
+  :file:`examples` directory along with qpdf development packages.
+
+.. _packaging-doc:
+
+Packaging Documentation
+-----------------------
+
+Starting in qpdf version 10.5, pre-built documentation is no longer
+distributed with the qpdf source distribution. Here are a few options
+you may want to consider for your packages:
+
+- **Do nothing**
+
+  When you run ``make install``, the file :file:`README-doc.txt` is
+  installed in ``$(docdir)``. That file tells the reader where to find
+  the documentation online and where to go to download offline copies
+  of the documentation. This is the option selected by the debian
+  packages.
+
+- **Embed pre-built documentation**
+
+  You can obtain pre-built documentation and extract its contents into
+  your distribution. This is what the Windows binary distributions
+  available from the qpdf release site do. You can find the pre-build
+  documentation in the release area in the file
+  :file:`qpdf-{version}-doc.zip`.
+
+- **Build the documentation yourself**
+
+  You can build the documentation as part of your build process. Be
+  sure to pass ``--enable-doc-maintenance`` to ``./configure``, and
+  install it with ``make doc-dist DOC_DEST=...``. This is what the
+  AppImage build does. The latest version of Sphinx at the time of the
+  initial conversion a sphinx-based documentation was 4.3.2. Older
+  versions are not guaranteed to work.
+
+Documentation Packaging Rationale
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This section describes the reason for things being the way they are.
+It's for information only; you don't have to know any of this to
+package qpdf.
+
+What is the reason for this change? Prior to qpdf 10.5, the qpdf
+manual was a docbook XML file. The generated documents were the
+product of running the file through build-time style sheets and
+contained no copyrighted material of their own. Starting with version
+10.5, the manual is written produced with `Sphinx
+<https://www.sphinx-doc.org>`__. This change was made to make it much
+easier to automatically generate portions of the documentation and to
+make the documentation easier to work with. The HTML output of Sphinx
+is also much more readable, usable, and suitable for online
+consumption than the output of the docbook style sheets. The downsides
+are that the generated HTML documentation now contains Javascript code
+and embedded fonts, and the PDF version of the documentation is no
+longer as suitable for printing (at least as of the 10.5 distribution)
+since external link targets are no longer shown and cross references
+no longer contain page number information. The presence of copyrighted
+material in the generated documentation, even though things are
+licensed with MIT and BSD licenses, complicates the job of the
+packager in various ways. For one thing, it means the
+:file:`NOTICE.md` file in the source repository would have to keep up
+with the copyright information for files that are not controlled in
+the repository. Additionally, some distributions (notably
+Debian/Ubuntu) discourage inclusion of sphinx-generated documentation
+in packages, preferring you instead to build the documentation as part
+of the package build process and to depend at runtime on a shared
+package that contains the code. At the time of the conversion of the
+qpdf manual from docbook to sphinx, newer versions of both sphinx and
+the html theme were required than were available in some of most of
+the Debian/Ubuntu versions for which qpdf was packaged.
+
+Since always-on Internet connectivity is much more common than it used
+to be, many users of qpdf would prefer to consume the documentation
+online anyway, and the lack of pre-built documentation in the
+distribution won't be as big of a deal. However there are still some
+people who can't or choose not to view documentation online. For them,
+pre-built documentation is still available.
diff --git a/manual/qpdf.1.in b/manual/qpdf.1.in
index 852313ee..fb568a13 100644
--- a/manual/qpdf.1.in
+++ b/manual/qpdf.1.in
@@ -15,6 +15,4 @@ options for inspecting or checking PDF files, some of which are
 useful primarily to PDF developers.
 .PP
 For a summary of qpdf's options, please run \fBqpdf \-\-help\fR. A
-complete manual can be found in @docdir@/html/index.html or
-@docdir@/qpdf-manual.pdf. The latest version of the documentation can
-be found at https://qpdf.readthedocs.io.
+complete manual can be found at https://qpdf.readthedocs.io.
diff --git a/manual/release-notes.rst b/manual/release-notes.rst
index 7db44c08..15089f40 100644
--- a/manual/release-notes.rst
+++ b/manual/release-notes.rst
@@ -6,29 +6,25 @@ Release Notes
 For a detailed list of changes, please see the file
 :file:`ChangeLog` in the source distribution.
 
-10.5.0: December 20, 2021
+10.5.0: December 21, 2021
   - Packaging changes
 
-    - The structure of the ``doc`` directory is different. The PDF
-      documentation is in the same place, but the files for the
-      previous HTML documentation are no longer there. Instead, there
-      are ``html`` and ``singlehtml`` directories, each of which
-      contain ``index.html`` and other files and directories. The
-      distribution files and ``make install`` target handle this, but
-      if you are building your own packages and including
-      documentation, please double check to make sure that you are
-      including the right documentation files.
+    - Pre-built documentation is no longer distributed with the source
+      distribution. The AppImage and Windows binary distributions
+      still contain embedded documentation, and a separate ``doc``
+      distribution file is available from the qpdf release site.
+      Documentation is now available at `https://qpdf.readthedocs.io
+      <https://qpdf.readthedocs.io>`__ for every major/minor version
+      starting with version 10.5. Please see :ref:`packaging-doc` for
+      details on how packagers should handle documentation.
 
     - The documentation sources have been switched from docbook to
       reStructuredText processed with `Sphinx
-      <https://sphinx-doc.org>`__. This will break previous
+      <https://www.sphinx-doc.org>`__. This will break previous
       documentation links. A redirect is in place on the main website.
       A top-to-bottom review of the documentation is planned for an
       upcoming release.
 
-    - Documentation is now available at `https://qpdf.readthedocs.io
-      <https://qpdf.readthedocs.io>`__.
-
   - Library Enhancements
 
     - Since qpdf version 8, using object accessor methods on an
diff --git a/manual/zlib-flate.1.in b/manual/zlib-flate.1.in
index 20b20bbc..99cc5cc9 100644
--- a/manual/zlib-flate.1.in
+++ b/manual/zlib-flate.1.in
@@ -21,8 +21,6 @@ This program should not be used as a general purpose compression
 tool.  Use something like gzip(1) instead.
 .PP
 For details about qpdf, please see the qpdf manual, which can be found
-in @docdir@/html/index.html or @docdir@/qpdf-manual.pdf. The latest
-version of the documentation can be found at
-https://qpdf.readthedocs.io.
+at https://qpdf.readthedocs.io.
 .SH "SEE ALSO"
 qpdf(1), gzip(1)