ary_expand_capa(): size calculation by size_t; fix #3353

Also more size checks added.

src/array.c

@@ -165,9 +165,9 @@ ary_make_shared(mrb_state *mrb, struct RArray *a)
 }
 
 static void
-ary_expand_capa(mrb_state *mrb, struct RArray *a, mrb_int len)
+ary_expand_capa(mrb_state *mrb, struct RArray *a, size_t len)
 {
-  mrb_int capa = a->aux.capa;
+  size_t capa = a->aux.capa;
 
   if (len > ARY_MAX_SIZE) {
     mrb_raise(mrb, E_ARGUMENT_ERROR, "array size too big");
@@ -177,14 +177,16 @@ ary_expand_capa(mrb_state *mrb, struct RArray *a, mrb_int len)
     capa = ARY_DEFAULT_LEN;
   }
   while (capa < len) {
-    if (capa <= ARY_MAX_SIZE / 2) {
     capa *= 2;
-    } else {
+    if (capa > ARY_MAX_SIZE) {
       capa = ARY_MAX_SIZE;
     }
   }
+  if (capa < len || capa > MRB_INT_MAX) {
+    mrb_raise(mrb, E_ARGUMENT_ERROR, "array size too big");
+  }
 
-  if (capa > a->aux.capa) {
+  if (capa > (size_t)a->aux.capa) {
     mrb_value *expanded_ptr = (mrb_value *)mrb_realloc(mrb, a->ptr, sizeof(mrb_value)*capa);
 
     a->aux.capa = capa;