Fix sign and conversion warnings (major)

This makes all integer type conversions that have potential data loss
explicit with calls that do range checks and raise an exception. After
this commit, qpdf builds with no warnings when -Wsign-conversion
-Wconversion is used with gcc or clang or when -W3 -Wd4800 is used
with MSVC. This significantly reduces the likelihood of potential
crashes from bogus integer values.

There are some parts of the code that take int when they should take
size_t or an offset. Such places would make qpdf not support files
with more than 2^31 of something that usually wouldn't be so large. In
the event that such a file shows up and is valid, at least qpdf would
raise an error in the right spot so the issue could be legitimately
addressed rather than failing in some weird way because of a silent
overflow condition.

9 files changed, 52 insertions, 37 deletions

diff --git a/libqpdf/qpdf/BitStream.hh b/libqpdf/qpdf/BitStream.hh
index bdbf2645..8be7c64d 100644
--- a/libqpdf/qpdf/BitStream.hh
+++ b/libqpdf/qpdf/BitStream.hh
@@ -4,28 +4,33 @@
 #define BITSTREAM_HH
 
 #include <qpdf/DLL.h>
+#include <stddef.h>
 
 class BitStream
 {
   public:
     QPDF_DLL
-    BitStream(unsigned char const* p, int nbytes);
+    BitStream(unsigned char const* p, size_t nbytes);
     QPDF_DLL
     void reset();
     QPDF_DLL
-    unsigned long long getBits(int nbits);
+    unsigned long long getBits(size_t nbits);
     QPDF_DLL
-    long long getBitsSigned(int nbits);
+    long long getBitsSigned(size_t nbits);
+    // Only call getBitsInt when requesting a number of bits that will
+    // definitely fit in an int.
+    QPDF_DLL
+    int getBitsInt(size_t nbits);
     QPDF_DLL
     void skipToNextByte();
 
   private:
     unsigned char const* start;
-    int nbytes;
+    size_t nbytes;
 
     unsigned char const* p;
-    unsigned int bit_offset;
-    unsigned int bits_available;
+    size_t bit_offset;
+    size_t bits_available;
 };
 
 #endif // BITSTREAM_HH
diff --git a/libqpdf/qpdf/BitWriter.hh b/libqpdf/qpdf/BitWriter.hh
index 76e28b8e..3ca91977 100644
--- a/libqpdf/qpdf/BitWriter.hh
+++ b/libqpdf/qpdf/BitWriter.hh
@@ -4,6 +4,7 @@
 #define BITWRITER_HH
 
 #include <qpdf/DLL.h>
+#include <stddef.h>
 
 class Pipeline;
 
@@ -15,9 +16,11 @@ class BitWriter
     QPDF_DLL
     BitWriter(Pipeline* pl);
     QPDF_DLL
-    void writeBits(unsigned long long val, unsigned int bits);
+    void writeBits(unsigned long long val, size_t bits);
     QPDF_DLL
-    void writeBitsSigned(long long val, unsigned int bits);
+    void writeBitsSigned(long long val, size_t bits);
+    QPDF_DLL
+    void writeBitsInt(int val, size_t bits);
     // Force any partial byte to be written to the pipeline.
     QPDF_DLL
     void flush();
@@ -25,7 +28,7 @@ class BitWriter
   private:
     Pipeline* pl;
     unsigned char ch;
-    unsigned int bit_offset;
+    size_t bit_offset;
 };
 
 #endif // BITWRITER_HH
diff --git a/libqpdf/qpdf/MD5.hh b/libqpdf/qpdf/MD5.hh
index 5cefe08c..02cdb3cf 100644
--- a/libqpdf/qpdf/MD5.hh
+++ b/libqpdf/qpdf/MD5.hh
@@ -3,6 +3,7 @@
 
 #include <qpdf/DLL.h>
 #include <qpdf/qpdf-config.h>
+#include <qpdf/Types.h>
 #ifdef HAVE_INTTYPES_H
 # include <inttypes.h>
 #endif
@@ -25,9 +26,9 @@ class MD5
     QPDF_DLL
     void encodeString(char const* input_string);
 
-    // encodes file and finalizes
+    // encodes file and finalizes; offset < 0 reads whole file
     QPDF_DLL
-    void encodeFile(char const* filename, int up_to_size = -1);
+    void encodeFile(char const* filename, qpdf_offset_t up_to_offset = -1);
 
     // appends string to current md5 object
     QPDF_DLL
@@ -35,7 +36,7 @@ class MD5
 
     // appends arbitrary data to current md5 object
     QPDF_DLL
-    void encodeDataIncrementally(char const* input_data, int len);
+    void encodeDataIncrementally(char const* input_data, size_t len);
 
     // computes a raw digest
     QPDF_DLL
@@ -52,16 +53,17 @@ class MD5
 
     // Convenience functions
     QPDF_DLL
-    static std::string getDataChecksum(char const* buf, int len);
+    static std::string getDataChecksum(char const* buf, size_t len);
     QPDF_DLL
     static std::string getFileChecksum(char const* filename,
-				       int up_to_size = -1);
+				       qpdf_offset_t up_to_offset = -1);
     QPDF_DLL
     static bool checkDataChecksum(char const* const checksum,
-				  char const* buf, int len);
+				  char const* buf, size_t len);
     QPDF_DLL
     static bool checkFileChecksum(char const* const checksum,
-				  char const* filename, int up_to_size = -1);
+				  char const* filename,
+                                  qpdf_offset_t up_to_offset = -1);
 
   private:
     // POINTER defines a generic pointer type
@@ -74,12 +76,12 @@ class MD5
     typedef uint32_t UINT4;
 
     void init();
-    void update(unsigned char *, unsigned int);
+    void update(unsigned char *, size_t);
     void final();
 
     static void transform(UINT4 [4], unsigned char [64]);
-    static void encode(unsigned char *, UINT4 *, unsigned int);
-    static void decode(UINT4 *, unsigned char *, unsigned int);
+    static void encode(unsigned char *, UINT4 *, size_t);
+    static void decode(UINT4 *, unsigned char *, size_t);
 
     UINT4 state[4];		// state (ABCD)
     UINT4 count[2];		// number of bits, modulo 2^64 (lsb first)
diff --git a/libqpdf/qpdf/Pl_AES_PDF.hh b/libqpdf/qpdf/Pl_AES_PDF.hh
index ac56e59a..4aaf68bb 100644
--- a/libqpdf/qpdf/Pl_AES_PDF.hh
+++ b/libqpdf/qpdf/Pl_AES_PDF.hh
@@ -16,7 +16,8 @@ class Pl_AES_PDF: public Pipeline
     QPDF_DLL
     // key should be a pointer to key_bytes bytes of data
     Pl_AES_PDF(char const* identifier, Pipeline* next,
-	       bool encrypt, unsigned char const* key, unsigned int key_bytes);
+	       bool encrypt, unsigned char const* key,
+               size_t key_bytes);
     QPDF_DLL
     virtual ~Pl_AES_PDF();
 
diff --git a/libqpdf/qpdf/Pl_ASCII85Decoder.hh b/libqpdf/qpdf/Pl_ASCII85Decoder.hh
index 4778f61a..cef09425 100644
--- a/libqpdf/qpdf/Pl_ASCII85Decoder.hh
+++ b/libqpdf/qpdf/Pl_ASCII85Decoder.hh
@@ -18,7 +18,7 @@ class Pl_ASCII85Decoder: public Pipeline
   private:
     void flush();
 
-    char inbuf[5];
+    unsigned char inbuf[5];
     size_t pos;
     size_t eod;
 };
diff --git a/libqpdf/qpdf/Pl_LZWDecoder.hh b/libqpdf/qpdf/Pl_LZWDecoder.hh
index b4f517ff..0456310f 100644
--- a/libqpdf/qpdf/Pl_LZWDecoder.hh
+++ b/libqpdf/qpdf/Pl_LZWDecoder.hh
@@ -21,23 +21,23 @@ class Pl_LZWDecoder: public Pipeline
 
   private:
     void sendNextCode();
-    void handleCode(int code);
-    unsigned char getFirstChar(int code);
+    void handleCode(unsigned int code);
+    unsigned char getFirstChar(unsigned int code);
     void addToTable(unsigned char next);
 
     // members used for converting bits to codes
     unsigned char buf[3];
-    int code_size;
-    int next;
-    int byte_pos;
-    int bit_pos;		// left to right: 01234567
-    int bits_available;
+    unsigned int code_size;
+    unsigned int next;
+    unsigned int byte_pos;
+    unsigned int bit_pos;       // left to right: 01234567
+    unsigned int bits_available;
 
     // members used for handle LZW decompression
     bool code_change_delta;
     bool eod;
     std::vector<Buffer> table;
-    int last_code;
+    unsigned int last_code;
 };
 
 #endif // PL_LZWDECODER_HH
diff --git a/libqpdf/qpdf/Pl_RC4.hh b/libqpdf/qpdf/Pl_RC4.hh
index 27eab5d0..9885f0ad 100644
--- a/libqpdf/qpdf/Pl_RC4.hh
+++ b/libqpdf/qpdf/Pl_RC4.hh
@@ -8,7 +8,7 @@
 class Pl_RC4: public Pipeline
 {
   public:
-    static int const def_bufsize = 65536;
+    static size_t const def_bufsize = 65536;
 
     // key_len of -1 means treat key_data as a null-terminated string
     QPDF_DLL
diff --git a/libqpdf/qpdf/RC4.hh b/libqpdf/qpdf/RC4.hh
index efd91069..a2aa5dce 100644
--- a/libqpdf/qpdf/RC4.hh
+++ b/libqpdf/qpdf/RC4.hh
@@ -1,6 +1,8 @@
 #ifndef RC4_HH
 #define RC4_HH
 
+#include <stddef.h>
+
 class RC4
 {
   public:
@@ -8,7 +10,8 @@ class RC4
     RC4(unsigned char const* key_data, int key_len = -1);
 
     // out_data = 0 means to encrypt/decrypt in place
-    void process(unsigned char* in_data, int len, unsigned char* out_data = 0);
+    void process(unsigned char* in_data, size_t len,
+                 unsigned char* out_data = 0);
 
   private:
     class RC4Key
diff --git a/libqpdf/qpdf/rijndael.h b/libqpdf/qpdf/rijndael.h
index 8b4c88f6..fa838fd9 100644
--- a/libqpdf/qpdf/rijndael.h
+++ b/libqpdf/qpdf/rijndael.h
@@ -8,14 +8,15 @@
 #ifdef HAVE_STDINT_H
 # include <stdint.h>
 #endif
+#include <stddef.h>
 
-int rijndaelSetupEncrypt(uint32_t *rk, const unsigned char *key,
-  int keybits);
-int rijndaelSetupDecrypt(uint32_t *rk, const unsigned char *key,
-  int keybits);
-void rijndaelEncrypt(const uint32_t *rk, int nrounds,
+unsigned int rijndaelSetupEncrypt(uint32_t *rk, const unsigned char *key,
+  size_t keybits);
+unsigned int rijndaelSetupDecrypt(uint32_t *rk, const unsigned char *key,
+  size_t keybits);
+void rijndaelEncrypt(const uint32_t *rk, unsigned int nrounds,
   const unsigned char plaintext[16], unsigned char ciphertext[16]);
-void rijndaelDecrypt(const uint32_t *rk, int nrounds,
+void rijndaelDecrypt(const uint32_t *rk, unsigned int nrounds,
   const unsigned char ciphertext[16], unsigned char plaintext[16]);
 
 #define KEYLENGTH(keybits) ((keybits)/8)