PDFBOX-6189: Safely validate dimension to avoid OOM, by subbudvk; closes #437

git-svn-id: https://svn.apache.org/repos/asf/pdfbox/branches/2.0@1932960 13f79535-47bb-0310-9956-ffa450edef68

Author: THausherr

pdfbox/src/main/java/org/apache/pdfbox/filter/CCITTFaxFilter.java

@@ -60,8 +60,43 @@ public DecodeResult decode(InputStream encoded, OutputStream decoded,
         // decompress data
         int k = decodeParms.getInt(COSName.K, 0);
         boolean encodedByteAlign = decodeParms.getBoolean(COSName.ENCODED_BYTE_ALIGN, false);
-        int arraySize = (cols + 7) / 8 * rows;
-        // TODO possible options??
+        if (cols <= 0 || rows <= 0)
+        {
+            throw new IOException("Invalid CCITT image dimensions: cols=" + cols + ", rows=" + rows);
+        }
+
+        long arraySizeLong = ((long) cols + 7) / 8 * rows;
+
+        long maxBytes = 256 * 1024 * 1024L;
+        String sysProp = System.getProperty(Filter.SYSPROP_CCITTFAX_MAXBYTES);
+
+        if (sysProp != null)
+        {
+            try
+            {
+                long parsed = Long.parseLong(sysProp);
+                if (parsed > 0)
+                {
+                    maxBytes = parsed;
+                }
+                // else ignore zero/negative values
+            }
+            catch (NumberFormatException e)
+            {
+                // ignore invalid value, keep default
+            }
+        }
+
+        if (arraySizeLong > maxBytes)
+        {
+            throw new IOException(
+                "CCITT decode buffer too large (" + arraySizeLong + " bytes) for cols=" + cols +
+                ", rows=" + rows + "; max allowed=" + maxBytes +
+                "; increase " + Filter.SYSPROP_CCITTFAX_MAXBYTES + " to override"
+            );
+        }
+
+        int arraySize = (int) arraySizeLong;
         byte[] decompressed = new byte[arraySize];
         CCITTFaxDecoderStream s;
         int type;

pdfbox/src/main/java/org/apache/pdfbox/filter/Filter.java

@@ -50,6 +50,16 @@ public abstract class Filter
      */
     public static final String SYSPROP_DEFLATELEVEL = "org.apache.pdfbox.filter.deflatelevel";
 
+    /**
+     * CCITTFax decode buffer size cap System Property. Sets the maximum number of bytes that
+     * CCITTFaxFilter is allowed to pre-allocate for a single image decode buffer. PDF-controlled
+     * /Columns and /Rows values are validated against this limit before allocation to prevent
+     * denial-of-service via crafted image dimensions. The default is 256 MB. To raise the cap for
+     * high-resolution legitimate documents, use
+     * {@code System.setProperty(Filter.SYSPROP_CCITTFAX_MAXBYTES, String.valueOf(512 * 1024 * 1024L));}
+     */
+    public static final String SYSPROP_CCITTFAX_MAXBYTES = "org.apache.pdfbox.filter.ccittmaxbytes";
+
     /**
      * Constructor.
      */