Update documentation build/installation

18 files changed, 76 insertions, 112 deletions

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 70128365..5543a7d8 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -19,21 +19,20 @@ on:
     # have reliable testing.
     - cron: '12 4 * * 5'
 jobs:
-  Distfiles:
-    # Generate distfiles.zip, which is a prerequisite for the mac and
-    # Windows builds. Only the mac and Windows builds actually "need"
-    # it, but declaring all the jobs to need it forces it to run
-    # first.
+  Prebuild:
+    # Run steps that are needed by the Windows build but are easier to
+    # build on Linux. Although only the Windows builds need this,
+    # other jobs depend on it to force it to run early.
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: 'Create distfiles.zip'
-        run: build-scripts/make-distfiles
+      - name: 'Run pre-build steps'
+        run: build-scripts/prebuild
       - name: 'Upload extra distribution files'
         uses: actions/upload-artifact@v1
         with:
-          name: distfiles
-          path: distfiles.zip
+          name: doc
+          path: doc.zip
       - name: 'Upload external libs'
         uses: actions/upload-artifact@v1
         with:
@@ -52,7 +51,7 @@ jobs:
           path: distribution
   Windows:
     runs-on: windows-latest
-    needs: Distfiles
+    needs: Prebuild
     strategy:
       fail-fast: false
       max-parallel: 4
@@ -64,10 +63,10 @@ jobs:
         shell: bash
         run: git config --global core.autocrlf input
       - uses: actions/checkout@v2
-      - name: 'Download distribution files'
+      - name: 'Download documentation'
         uses: actions/download-artifact@v2
         with:
-          name: distfiles
+          name: doc
           path: .
       - name: 'Download external libs'
         uses: actions/download-artifact@v2
@@ -84,13 +83,9 @@ jobs:
           path: distribution
   macOS:
     runs-on: macos-latest
-    needs: Distfiles
+    needs: Prebuild
     steps:
       - uses: actions/checkout@v2
-      - name: 'Download distribution files'
-        uses: actions/download-artifact@v2
-        with:
-          name: distfiles
       - name: 'Download external libs'
         uses: actions/download-artifact@v2
         with:
@@ -100,7 +95,7 @@ jobs:
         run: build-scripts/build-mac
   AppImage:
     runs-on: ubuntu-latest
-    needs: Distfiles
+    needs: Prebuild
     steps:
       - uses: actions/checkout@v2
       - name: 'Build AppImage'
@@ -112,21 +107,21 @@ jobs:
           path: distribution
   Linux32:
     runs-on: ubuntu-latest
-    needs: Distfiles
+    needs: Prebuild
     steps:
       - uses: actions/checkout@v2
       - name: 'Linux 32-bit'
         run: build-scripts/build-linux32
   Fuzzers:
     runs-on: ubuntu-latest
-    needs: Distfiles
+    needs: Prebuild
     steps:
       - uses: actions/checkout@v2
       - name: 'Build Fuzzer'
         run: build-scripts/build-fuzzer
   Sanitizers:
     runs-on: ubuntu-latest
-    needs: Distfiles
+    needs: Prebuild
     steps:
       - uses: actions/checkout@v2
       - name: 'Sanitizer Tests'
diff --git a/.gitignore b/.gitignore
index 3a086e8a..b42b920b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,8 +5,8 @@ autoconf.mk
 autom4te.cache/
 config.log
 config.status
-distfiles.zip
 doc
+doc.zip
 examples/build/
 external-libs
 fuzz/build/
diff --git a/Makefile b/Makefile
index 6a6412dc..64bb8bcd 100644
--- a/Makefile
+++ b/Makefile
@@ -97,23 +97,16 @@ $(foreach B,$(BUILD_ITEMS),$(eval \
   clean_$(B): ; \
 	$(RM) -r $(B)/$(OUTPUT_DIR)))
 
-DISTFILES = doc/qpdf-manual.html doc/qpdf-manual.pdf
-distfiles.zip: $(DISTFILES)
-	$(RM) distfiles.zip
-	zip -r distfiles.zip $(DISTFILES) doc/_static
-
 distclean: clean
 	$(RM) -r autoconf.mk autom4te.cache config.log config.status libtool
 	$(RM) libqpdf/qpdf/qpdf-config.h
 	$(RM) manual/html.xsl
 	$(RM) manual/print.xsl
-	$(RM) doc/*.1
 	$(RM) libqpdf.pc libqpdf.map
 
 maintainer-clean: distclean
-	$(RM) doc/qpdf-manual.*
 	$(RM) -r install-mingw* install-msvc* external-libs
-	$(RM) distfiles.zip
+	$(RM) -rf doc
 
 .PHONY: $(TEST_TARGETS)
 
diff --git a/README-maintainer b/README-maintainer
index b9198fbc..47d64b2d 100644
--- a/README-maintainer
+++ b/README-maintainer
@@ -119,7 +119,7 @@ RELEASE PREPARATION
   git --no-pager grep -i -n -P "copyright.*$(expr $(date +%Y) - 1).*berkenbilt"
 
   Also update the copyright in these places:
-  * manual
+  * manual/conf.py
   * debian package -- search for copyright.*berkenbilt in debian/copyright
   * qtest-driver, TestDriver.pm in qtest source
 
@@ -210,14 +210,12 @@ RELEASE PREPARATION
   * Make sure version numbers are consistent in the following locations:
     * configure.ac
     * libqpdf/QPDF.cc
-    * manual/qpdf-manual.xml
+    * manual/conf.py
     * qpdf/qpdf.cc
     `make_dist` verifies this consistency.
 
   * Update release notes in manual. Look at diffs and ChangeLog.
-    Update release date in `manual/qpdf-manual.xml`. Remember to
-    ensure that the entities at the top of the document are consistent
-    with the release notes for both version and release date.
+    Update release date in `manual/release-notes.rst`.
 
   * Add a release entry to ChangeLog: "x.y.z: release"
 
@@ -327,7 +325,7 @@ Template for release notes:
 ```
 This is qpdf version x.y.z. (Brief description)
 
-For a full list of changes from previous releases, please see the [release notes](http://qpdf.sourceforge.net/files/qpdf-manual.html#release-notes). See also [README-what-to-download](./README-what-to-download.md) for details about the available source and binary distributions.
+For a full list of changes from previous releases, please see the [release notes](https://qpdf.sourceforge.io/doc/html/release-notes.html). See also [README-what-to-download](./README-what-to-download.md) for details about the available source and binary distributions.
 ```
 
 # Publish release
@@ -348,7 +346,7 @@ rsync -vrlcO ./ jay_berkenbilt,qpdf@frs.sourceforge.net:/home/frs/project/q/qp/q
 
 (cd /tmp; mkdir -p z; cd z; \
  tar xf ~/Q/storage/releases/qpdf/qpdf/$version/qpdf-$version.tar.gz qpdf-$version/doc; \
- scp -p qpdf-$version/doc/qpdf-* jay_berkenbilt,qpdf@frs.sourceforge.net:htdocs/files/)
+ rsync -avx --delete --force --exclude '*.1' qpdf-$version/doc/ jay_berkenbilt,qpdf@frs.sourceforge.net:htdocs/doc/)
 
 * Upload the debian package and Ubuntu ppa backports.
 
diff --git a/README.md b/README.md
index 11777352..3e7e9eb5 100644
--- a/README.md
+++ b/README.md
@@ -68,7 +68,7 @@ The PDF file format used to rely on RC4 for encryption. Using 256-bit keys alway
 
 # Building from a pristine checkout
 
-When building qpdf from a pristine checkout from version control, generated documentation files are not present. You may either generate them (by passing `--enable-doc-maintenance` to `./configure` and satisfying the extra build-time dependencies) or obtain them from a released source package, which includes them. If you want to grab just the files that are in the source distribution but not in the repository, extract a source distribution in a temporary directory, and run `make CLEAN=1 distfiles.zip`. This will create a file called `distfiles.zip`, which can you can extract in a checkout of the source repository. This step is optional unless you are running make install and want the html and PDF versions of the documentation to be installed.
+When building qpdf from a pristine checkout from version control, generated HTML and PDF documentation files are not present. You don't need them unless you are going to run `make install` and want the documentation to be installed. If you want them, you can either extract the `doc` directory from a source distribution, or you can satisfy the additional requirements for building documentation and pass `--enable-doc-maintenance` to `./configure`.
 
 # Building from source distribution on UNIX/Linux
 
diff --git a/TODO b/TODO
index 18746cf6..4435aa84 100644
--- a/TODO
+++ b/TODO
@@ -1,8 +1,6 @@
 Next
 ====
 
-* Add spell check to CI
-
 * High-level API/doc overhaul: #593
 
 * Refactor test_driver.cc so that runtest is not one huge function.
@@ -19,24 +17,6 @@ Next
   thrown when an uninitialized trailer is accessed provides useful
   information. Test from the C API as well as the C++ API.
 
-Doc conversion
-==============
-
-Before release:
-
-* Figure out what to do about the fact that the release notes are now
-  at #release-notes instead of #ref.release-notes. This invalidates
-  the link in all previous release announcements, but there's not much
-  I can do about, and it doesn't seem worth fixing. Maybe mention it
-  somewhere?
-* README-maintainer: Fix installation of documentation to website
-
-Soon:
-
-* :ref: -- would be nice if it were suitable for printed documentation
-* Decide about readthedocs; if using, with multiple versions/latest
-* Generate stuff (options, code samples, etc.) as needed
-
 Documentation
 =============
 
@@ -52,6 +32,8 @@ Documentation
 * See #530 -- add an appendix explaining PDF encryption in general
   plus how it's handled by qpdf. Or maybe this should go on the wiki.
 
+* Decide about readthedocs including supporting multiple released
+  versions of the docs and docs from main.
 
 Document-level work
 ===================
diff --git a/build-scripts/build-mac b/build-scripts/build-mac
index eb41f817..497ea569 100755
--- a/build-scripts/build-mac
+++ b/build-scripts/build-mac
@@ -8,9 +8,6 @@ cd jpeg-*
 make -k
 sudo make install
 cd ..
-if [ -f distfiles/distfiles.zip ]; then
-    unzip distfiles/distfiles.zip
-fi
 ./configure --enable-werror --enable-show-failed-test-output
 make -j$(nproc) -k
 make -k check
diff --git a/build-scripts/build-windows b/build-scripts/build-windows
index 992c54ae..81543668 100755
--- a/build-scripts/build-windows
+++ b/build-scripts/build-windows
@@ -20,7 +20,7 @@ if [[ $tool == mingw ]]; then
 elif [[ $tool == msvc ]]; then
     cl
 fi
-unzip distfiles.zip
+unzip doc.zip
 unzip qpdf-external-libs-bin.zip
 cwd=`pwd`
 PATH=$cwd/libqpdf/build:$PATH
diff --git a/build-scripts/make-distfiles b/build-scripts/prebuild
index f30f4595..f8d1f61b 100755
--- a/build-scripts/make-distfiles
+++ b/build-scripts/prebuild
@@ -7,5 +7,6 @@ sudo apt-get -y install \
    python3-pip texlive-latex-extra latexmk inkscape imagemagick
 pip3 install sphinx
 ./configure --enable-doc-maintenance
-make -j$(nproc) distfiles.zip
+make -j$(nproc) build_manual
+zip -r doc.zip doc/*html doc/*.pdf
 build-scripts/download-external-libs
diff --git a/cSpell.json b/cSpell.json
index 7ecfe5c7..ba39675d 100644
--- a/cSpell.json
+++ b/cSpell.json
@@ -11,6 +11,7 @@
     "afdh",
     "afdhph",
     "ageneration",
+    "agogo",
     "aitems",
     "annots",
     "aobjid",
@@ -73,6 +74,7 @@
     "dctdecode",
     "decrypter",
     "deduplicating",
+    "deps",
     "destdir",
     "dests",
     "devel",
@@ -80,7 +82,6 @@
     "diffutils",
     "directpagerefcount",
     "distclean",
-    "distfiles",
     "ditems",
     "docbook",
     "docdir",
@@ -299,6 +300,7 @@
     "pluggable",
     "pngify",
     "poppler",
+    "prebuild",
     "precheck",
     "prepended",
     "prepending",
diff --git a/make/installwin.mk b/make/installwin.mk
index 4acf9e3f..af17ba81 100644
--- a/make/installwin.mk
+++ b/make/installwin.mk
@@ -17,11 +17,12 @@ installwin: all
 	cp qpdf/$(OUTPUT_DIR)/fix-qdf.exe $(DEST)/bin
 	cp include/qpdf/*.h $(DEST)/include/qpdf
 	cp include/qpdf/*.hh $(DEST)/include/qpdf
-	if [ -f doc/qpdf-manual.html ]; then \
-	    mkdir $(DEST)/doc/_static; \
-	    cp doc/qpdf-manual.html $(DEST)/doc; \
-	    cp doc/_static/* $(DEST)/doc/_static; \
-	fi
 	if [ -f doc/qpdf-manual.pdf ]; then \
 	    cp doc/qpdf-manual.pdf $(DEST)/doc; \
 	fi
+	if [ -d doc/html ]; then \
+	    cp -r doc/html $(DEST)/doc; \
+	fi
+	if [ -d doc/singlehtml ]; then \
+	    cp -r doc/singlehtml $(DEST)/doc; \
+	fi
diff --git a/make/libtool.mk b/make/libtool.mk
index eda53732..e023bc60 100644
--- a/make/libtool.mk
+++ b/make/libtool.mk
@@ -121,8 +121,6 @@ install-libs: build_libqpdf
 # NOTE: If installing any new executables, remember to update the
 # lambda layer code in build-scripts/build-appimage.
 
-# NOTE: See comments in manual/build.mk about html documentation.
-
 # Ensure that installwin in make/installwin.mk is consistent with
 # this.
 
@@ -139,12 +137,13 @@ install: all install-libs
 	$(LIBTOOL) --mode=install ./install-sh \
 		qpdf/$(OUTPUT_DIR)/fix-qdf \
 		$(DESTDIR)$(bindir)/fix-qdf
-	if [ -f doc/qpdf-manual.html ]; then \
-		./mkinstalldirs -m 0755 $(DESTDIR)$(docdir)/_static; \
-		./install-sh -m 0644 doc/qpdf-manual.html $(DESTDIR)$(docdir); \
-		./install-sh -m 0644 doc/_static/* $(DESTDIR)$(docdir)/_static; \
-	fi
 	if [ -f doc/qpdf-manual.pdf ]; then \
 		./install-sh -m 0644 doc/qpdf-manual.pdf $(DESTDIR)$(docdir); \
 	fi
+	if [ -d doc/html ]; then \
+		cp -r doc/html $(DESTDIR)/$(docdir); \
+	fi
+	if [ -d doc/singlehtml ]; then \
+		cp -r doc/singlehtml $(DESTDIR)/$(docdir); \
+	fi
 	./install-sh -m 0644 doc/*.1 $(DESTDIR)$(mandir)/man1
diff --git a/manual/build.mk b/manual/build.mk
index 6d38bc3a..b07bf830 100644
--- a/manual/build.mk
+++ b/manual/build.mk
@@ -8,10 +8,10 @@ PDF_TARGET := $(PDF_OUT)/qpdf.pdf
 
 TARGETS_manual := doc/qpdf.1 doc/fix-qdf.1 doc/zlib-flate.1
 ifeq ($(BUILD_HTML),1)
-TARGETS_manual += doc/qpdf-manual.html $(HTML_TARGET)
+TARGETS_manual += $(HTML_TARGET) $(S_HTML_TARGET)
 endif
 ifeq ($(BUILD_PDF),1)
-TARGETS_manual += doc/qpdf-manual.pdf
+TARGETS_manual += $(PDF_TARGET)
 endif
 
 MANUAL_DEPS = $(wildcard manual/*.rst) manual/conf.py
@@ -22,33 +22,20 @@ MANUAL_DEPS = $(wildcard manual/*.rst) manual/conf.py
 # the error "_pickle.UnpicklingError: pickle data was truncated"
 $(HTML_TARGET): $(MANUAL_DEPS)
 	$(SPHINX) -M html manual $(DOC_OUT) -W
+	mkdir -p doc
+	rm -rf doc/html
+	cp -r $(DOC_OUT)/html doc
 
 $(S_HTML_TARGET): $(MANUAL_DEPS) | $(HTML_TARGET)
 	$(SPHINX) -M singlehtml manual $(DOC_OUT) -W
+	mkdir -p doc
+	rm -rf doc/singlehtml
+	cp -r $(DOC_OUT)/singlehtml doc
 
 $(PDF_TARGET): $(MANUAL_DEPS) | $(S_HTML_TARGET) $(HTML_TARGET)
 	$(SPHINX) -M latexpdf manual $(DOC_OUT) -W
-
-# This depends on sphinx-build's singlehtml target creating index.html
-# and a _static directory. If that changes, this code has to be
-# adjusted. It will also be necessary to adjust the install target in
-# make/libtool.mk.
-doc/qpdf-manual.html: $(S_HTML_TARGET)
-	mkdir -p doc
-	@if [ "$(shell find $(S_HTML_OUT)/ -mindepth 1 -type d -print)" != \
-             "$(S_HTML_OUT)/_static" ]; then \
-	    echo "***"; \
-	    echo Expected only directory in $(S_HTML_OUT) to be _static; \
-	    echo "***"; \
-	    false; \
-	fi
-	cp $< $@
-	mkdir -p doc/_static
-	cp -p $(S_HTML_OUT)/_static/* doc/_static
-
-doc/qpdf-manual.pdf: $(PDF_TARGET)
 	mkdir -p doc
-	cp $< $@
+	cp $(PDF_TARGET) doc/qpdf-manual.pdf
 
 doc/%.1: manual/%.1.in
 	mkdir -p doc
diff --git a/manual/fix-qdf.1.in b/manual/fix-qdf.1.in
index b14f4304..dbc4c106 100644
--- a/manual/fix-qdf.1.in
+++ b/manual/fix-qdf.1.in
@@ -14,5 +14,5 @@ the same file with stream lengths, cross-reference table entries, and
 object stream offset tables regenerated.
 .PP
 For details about fix-qdf and about PDF files in QDF mode, please see
-the qpdf manual, which can be found in @docdir@/qpdf-manual.html or
+the qpdf manual, which can be found in @docdir@/html/index.html or
 @docdir@/qpdf-manual.pdf.
diff --git a/manual/installation.rst b/manual/installation.rst
index 8862034d..d7989e66 100644
--- a/manual/installation.rst
+++ b/manual/installation.rst
@@ -64,7 +64,7 @@ Pre-built documentation is distributed with qpdf, so you should
 generally not need to rebuild the documentation. In order to build the
 documentation from source, you need to install `Sphinx
 <https://sphinx-doc.org>`__. To build the PDF version of the
-documentation, you need `pdflatex`, `latexmk`, and a fairly complete
+documentation, you need ``pdflatex``, ``latexmk``, and a fairly complete
 LaTeX installation. Detailed requirements can be found in the Sphinx
 documentation.
 
diff --git a/manual/qpdf.1.in b/manual/qpdf.1.in
index 18c6f704..7d8112dd 100644
--- a/manual/qpdf.1.in
+++ b/manual/qpdf.1.in
@@ -16,4 +16,4 @@ useful primarily to PDF developers.
 .PP
 For a summary of qpdf's options, please run
 \fBqpdf \-\-help\fR.  A complete manual can be found in
-@docdir@/qpdf-manual.html or @docdir@/qpdf-manual.pdf.
+@docdir@/html/index.html or @docdir@/qpdf-manual.pdf.
diff --git a/manual/release-notes.rst b/manual/release-notes.rst
index 5a8fd307..adb2a48d 100644
--- a/manual/release-notes.rst
+++ b/manual/release-notes.rst
@@ -7,6 +7,25 @@ For a detailed list of changes, please see the file
 :file:`ChangeLog` in the source distribution.
 
 10.5.0: XXX Month dd, YYYY
+  - Packaging changes
+
+    - The structure of the ``doc`` directory is different. The PDF
+      documentation is in the same place, but the files for the
+      previous HTML documentation are no longer there. Instead, there
+      are ``html`` and ``singlehtml`` directories, each of which
+      contain ``index.html`` and other files and directories. The
+      distribution files and ``make install`` target handle this, but
+      if you are building your own packages and including
+      documentation, please double check to make sure that you are
+      including the right documentation files.
+
+    - The documentation sources have been switched from docbook to
+      reStructuredText processed with `Sphinx
+      <https://sphinx-doc.org>`__. This will break previous
+      documentation links. A redirect is in place on the main website.
+      A top-to-bottom review of the documentation is planned for an
+      upcoming release.
+
   - Library Enhancements
 
     - Since qpdf version 8, using object accessor methods on an
@@ -58,16 +77,6 @@ For a detailed list of changes, please see the file
 
     - Add ``qpdf_oh_get_type_code`` and ``qpdf_oh_get_type_name``.
 
-  - Documentation change
-
-    - The documentation sources have been switched from docbook to
-      reStructuredText processed with `Sphinx
-      <https://sphinx-doc.org>`__. This is mostly transparent (other
-      than format change) with the exception that all section links
-      have changed. What used to be `#ref.something` is now
-      `#something`. A top-to-bottom review of the documentation is
-      planned for an upcoming release.
-
 10.4.0: November 16, 2021
   - Handling of Weak Cryptography Algorithms
 
diff --git a/manual/zlib-flate.1.in b/manual/zlib-flate.1.in
index e74eb3f5..77349c36 100644
--- a/manual/zlib-flate.1.in
+++ b/manual/zlib-flate.1.in
@@ -21,6 +21,6 @@ This program should not be used as a general purpose compression
 tool.  Use something like gzip(1) instead.
 .PP
 For details about qpdf, please see the qpdf manual, which can be found
-in @docdir@/qpdf-manual.html or @docdir@/qpdf-manual.pdf.
+in @docdir@/html/index.html or @docdir@/qpdf-manual.pdf.
 .SH "SEE ALSO"
 qpdf(1), gzip(1)