shithub: jbig2

Info  •  Files  •  Log

jbig2dec: tweak overflow check in jbig2_(re)alloc

If num * size overflows under x64, the value may be negative and still
fit into a 32-bit integer. The proper check unfortunately requires a
division.

Note: The maximum allowed allocation is (size_t)-0x100 instead of
SIZE_MAX so that debug CRTs which check for the allocation of e.g.
(size_t)-1 never assert.

--- a/jbig2.c

+++ b/jbig2.c

@@ -60,11 +60,9 @@

 jbig2_alloc (Jbig2Allocator *allocator, size_t size, size_t num)

 {

   /* check for integer multiplication overflow */

-  int64_t check = ((int64_t)num)*((int64_t)size);

-  if (check != (int)check)

+  if (num > 0 && size >= (size_t)-0x100 / num)

     return NULL;

-  else

-    return allocator->alloc (allocator, (int)check);

+  return allocator->alloc(allocator, size * num);

 }

 void

@@ -77,11 +75,9 @@

 jbig2_realloc (Jbig2Allocator *allocator, void *p, size_t size, size_t num)

 {

   /* check for integer multiplication overflow */

-  int64_t check = ((int64_t)num)*((int64_t)size);

-  if (check != (int)check)

+  if (num > 0 && size >= (size_t)-0x100 / num)

     return NULL;

-  else

-    return allocator->realloc (allocator, p, (int)check);

+  return allocator->realloc(allocator, p, size * num);

 }

 static int