Add a check for unsigned integer wrapping.

src/gc.c

@@ -167,14 +167,16 @@ mrb_malloc(mrb_state *mrb, size_t len)
 void*
 mrb_calloc(mrb_state *mrb, size_t nelem, size_t len)
 {
-  void *p;
+  void *p = NULL;
   size_t size;
 
-  size = nelem * len;
-  p = mrb_realloc(mrb, 0, size);
+  if (nelem <= SIZE_MAX / len) {
+    size = nelem * len;
+    p = mrb_realloc(mrb, 0, size);
 
-  if (p && size > 0)
-    memset(p, 0, size);
+    if (p && size > 0)
+      memset(p, 0, size);
+  }
 
   return p;
 }