add function for checked mrb_int subtraction

include/mruby/numeric.h

@@ -45,6 +45,16 @@ mrb_int_add_overflow(mrb_int augend, mrb_int addend, mrb_int *sum)
   return !!(((x ^ z) & (y ^ z)) & MRB_INT_OVERFLOW_MASK);
 }
 
+static inline mrb_bool
+mrb_int_sub_overflow(mrb_int minuend, mrb_int subtrahend, mrb_int *difference)
+{
+  mrb_uint x = (mrb_uint)minuend;
+  mrb_uint y = (mrb_uint)subtrahend;
+  mrb_uint z = (mrb_uint)(x - y);
+  *difference = (mrb_int)z;
+  return !!(((x ^ z) & (~y ^ z)) & MRB_INT_OVERFLOW_MASK);
+}
+
 #undef MRB_INT_OVERFLOW_MASK
 #undef mrb_uint
 #undef MRB_UINT_MAKE

src/numeric.c

@@ -1145,9 +1145,7 @@ mrb_fixnum_minus(mrb_state *mrb, mrb_value x, mrb_value y)
     mrb_int b, c;
 
     b = mrb_fixnum(y);
-    c = a - b;
-    if (((a < 0) ^ (b < 0)) != 0 && (a < 0) != (c < 0)) {
-      /* integer overflow */
+    if (mrb_int_sub_overflow(a, b, &c)) {
       return mrb_float_value(mrb, (mrb_float)a - (mrb_float)b);
     }
     return mrb_fixnum_value(c);

src/vm.c

@@ -1669,12 +1669,7 @@ RETRY_TRY_BLOCK:
 
           x = mrb_fixnum(regs[a]);
           y = mrb_fixnum(regs[a+1]);
-          z = x - y;
-#ifdef MRB_WORD_BOXING
-          z = (z << MRB_FIXNUM_SHIFT) / (1 << MRB_FIXNUM_SHIFT);
-#endif
-          if (((x < 0) ^ (y < 0)) != 0 && (x < 0) != (z < 0)) {
-            /* integer overflow */
+          if (mrb_int_sub_overflow(x, y, &z)) {
             SET_FLT_VALUE(mrb, regs[a], (mrb_float)x - (mrb_float)y);
             break;
           }
@@ -1876,10 +1871,9 @@ RETRY_TRY_BLOCK:
         {
           mrb_int x = regs_a[0].attr_i;
           mrb_int y = GETARG_C(i);
-          mrb_int z = x - y;
+          mrb_int z;
 
-          if ((x < 0) != (z < 0) && ((x < 0) ^ (y < 0)) != 0) {
-            /* integer overflow */
+          if (mrb_int_sub_overflow(x, y, &z)) {
             SET_FLT_VALUE(mrb, regs_a[0], (mrb_float)x - (mrb_float)y);
           }
           else {