Squashed commit of the following:

commit 2d7d545c4c4bfce7fdcbcbe9baaeb437915742f0
Merge: 625a1249 b178914b
Author: Yukihiro "Matz" Matsumoto <matz@ruby.or.jp>
Date:   Fri Jun 5 14:35:13 2020 +0900

    Merge branch 'fix-mrb_open-with-nomem' of https://github.com/dearblue/mruby into dearblue-fix-mrb_open-with-nomem

commit b178914b111dda79a8f36ec4eb3e9d37b76f982e
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sat Jan 19 22:22:44 2019 +0900

    Fix invalid pointer free inside other heap's block

     1. `e = mrb_obj_alloc(...)`
     2. `e->stack = mrb->c->stack` (`mrb->c->stack` is anywhere in the range `stbase...stend`)
     3. And raised exception by `mrb_malloc()`!
     4. `mrb_free(e->stack)` by GC part (wrong free)

commit 52e3d5d8585daf86af3ed12db5ab0efefbc9b956
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sat Jan 19 21:55:36 2019 +0900

    Fix memory leak for temporary symbols when out of memory

commit 4c5499b88e47cc6012ad7d7379cb6bc74c6a0b60
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sun Jan 20 11:42:07 2019 +0900

    Fix uninitialized pointer dereference for debug section

commit 8e993167dec62a9709d6faacd517729ddcedf4f9
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sun Jan 20 11:41:09 2019 +0900

    Fix memory leak for temporary filenames when out of memory

commit 8b422577e6eae68a28121b88421d937e8707b487
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sun Jan 20 10:57:51 2019 +0900

    Fix memory leak for irep when out of memory

commit 6b35ebf49a0aa3edb6bbda770ed58681e9c2e6af
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sun Jan 20 10:55:50 2019 +0900

    Fix uninitialized pointer dereference when do not finished initializing irep

commit 2531f2631e67e0462749618e2344c733a29238f0
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sun Jan 20 10:48:15 2019 +0900

    Fix NULL pointer dereference when do not finished initializing irep

commit e2d6896ebad13694800af49c2625e106b8440ddf
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sat Jan 19 12:54:19 2019 +0900

    Fix memory leak for irep when out of memory by `mrb_proc_new()`

commit b6214ff8a0a1c73bc9554e39053878ac50bb683f
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sat Jan 19 12:53:07 2019 +0900

    Fix memory leak for `khash_t` in `kh_init_size()` when out of memory by `kh_alloc()`

commit 19162dd6c11f0093d0011e7cab83b8f9e84c2c07
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sun Jan 20 02:15:07 2019 +0900

    Fix memory leak for symbol string when out of memory in `kh_put()`

commit 15e67297ff54bc14ef359d6d1e745d760a4a255a
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sun Jan 20 02:12:24 2019 +0900

    Fix keep wrong symbol index when out of memory

commit 3f8e2b375244f5441e8d62efa13c6e6a9afecb14
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sun Jan 20 02:08:13 2019 +0900

    Fix keep wrong symbol capacity when out of memory

commit a3cfe755ab3e758046c3f4e30938ac8d567ed046
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sat Jan 19 10:11:37 2019 +0900

    Fix NULL pointer dereference `mrb->c` by `mark_context()`

commit d9c7b6be6eb54630b64eea5c35be241e551676e5
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sun Jan 20 15:25:09 2019 +0900

    Fix protect exception for print error message

commit 100642750e4d549f2e8050f8d6cabdf8825d4495
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sun Jan 20 11:59:02 2019 +0900

    Protect exception for mruby core initialization

commit 7a0418304ec70764fa215bef3599f5f735222075
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Fri Jan 18 20:38:27 2019 +0900

    Fix memory leak for string object when out of memory

    The `mrb_str_pool()` function has a path to call `malloc()` twice.

    If occurs `NoMemoryError` exception in second `malloc()`,
    first `malloc()` pointer is not freed.

commit fef1c152ce4e52b9e4a34dc23aca5b02907ac639
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sat Jan 19 13:05:09 2019 +0900

    Fix stack overflow when out of memory

    As a result of this change, no backtrace information is set
    for NoMemoryError (`mrb->nomem_err`).

    Detailes:

    When generating a backtrace, called `mrb_intern_lit()`,
    `mrb_str_new_cstr()` and `mrb_obj_iv_set()` function with
    `exc_debug_info()` function in `src/error.c`.

    If a `NoMemoryError` exception occurs at this time,
    the `exc_debug_info()` function will be called again,
    and in the same way `NoMemoryError` exception raised will result
    in an infinite loop to occurs stack overflow (and SIGSEGV).

commit da7d7f881bbbad9988a3a2b7bad8f2b72ff06bc6
Author: dearblue <dearblue@users.noreply.github.com>
Date:   Sun Jan 20 12:00:38 2019 +0900

    Fix NULL pointer dereference `mrb->nomem_err` when not initialized

    Add internal functions (not `static`):

      * `mrb_raise_nomemory()`
      * `mrb_core_init_abort()`

include/mruby/khash.h

@@ -73,6 +73,7 @@ static const uint8_t __m_either[] = {0x03, 0x0c, 0x30, 0xc0};
   void kh_clear_##name(mrb_state *mrb, kh_##name##_t *h);               \
   khint_t kh_get_##name(mrb_state *mrb, kh_##name##_t *h, khkey_t key);           \
   khint_t kh_put_##name(mrb_state *mrb, kh_##name##_t *h, khkey_t key, int *ret); \
+  void kh_put_prepare_##name(mrb_state *mrb, kh_##name##_t *h);                   \
   void kh_resize_##name(mrb_state *mrb, kh_##name##_t *h, khint_t new_n_buckets); \
   void kh_del_##name(mrb_state *mrb, kh_##name##_t *h, khint_t x);                \
   kh_##name##_t *kh_copy_##name(mrb_state *mrb, kh_##name##_t *h);
@@ -95,16 +96,25 @@ kh_fill_flags(uint8_t *p, uint8_t c, size_t len)
    __hash_equal: hash comparation function
 */
 #define KHASH_DEFINE(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
-  void kh_alloc_##name(mrb_state *mrb, kh_##name##_t *h)                \
+  mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);                 \
+  int kh_alloc_simple_##name(mrb_state *mrb, kh_##name##_t *h)          \
   {                                                                     \
     khint_t sz = h->n_buckets;                                          \
     size_t len = sizeof(khkey_t) + (kh_is_map ? sizeof(khval_t) : 0);   \
-    uint8_t *p = (uint8_t*)mrb_malloc(mrb, sizeof(uint8_t)*sz/4+len*sz); \
+    uint8_t *p = (uint8_t*)mrb_malloc_simple(mrb, sizeof(uint8_t)*sz/4+len*sz); \
+    if (!p) { return 1; }                                               \
     h->size = h->n_occupied = 0;                                        \
     h->keys = (khkey_t *)p;                                             \
     h->vals = kh_is_map ? (khval_t *)(p+sizeof(khkey_t)*sz) : NULL;     \
     h->ed_flags = p+len*sz;                                             \
     kh_fill_flags(h->ed_flags, 0xaa, sz/4);                             \
+    return 0;                                                           \
+  }                                                                     \
+  void kh_alloc_##name(mrb_state *mrb, kh_##name##_t *h)                \
+  {                                                                     \
+    if (kh_alloc_simple_##name(mrb, h)) {                               \
+      mrb_raise_nomemory(mrb);                                          \
+    }                                                                   \
   }                                                                     \
   kh_##name##_t *kh_init_##name##_size(mrb_state *mrb, khint_t size) {  \
     kh_##name##_t *h = (kh_##name##_t*)mrb_calloc(mrb, 1, sizeof(kh_##name##_t)); \
@@ -112,7 +122,10 @@ kh_fill_flags(uint8_t *p, uint8_t c, size_t len)
       size = KHASH_MIN_SIZE;                                            \
     khash_power2(size);                                                 \
     h->n_buckets = size;                                                \
-    kh_alloc_##name(mrb, h);                                            \
+    if (kh_alloc_simple_##name(mrb, h)) {                               \
+      mrb_free(mrb, h);                                                 \
+      mrb_raise_nomemory(mrb);                                          \
+    }                                                                   \
     return h;                                                           \
   }                                                                     \
   kh_##name##_t *kh_init_##name(mrb_state *mrb) {                       \
@@ -171,12 +184,16 @@ kh_fill_flags(uint8_t *p, uint8_t c, size_t len)
       mrb_free(mrb, old_keys);                                          \
     }                                                                   \
   }                                                                     \
-  khint_t kh_put_##name(mrb_state *mrb, kh_##name##_t *h, khkey_t key, int *ret) \
+  void kh_put_prepare_##name(mrb_state *mrb, kh_##name##_t *h)          \
   {                                                                     \
-    khint_t k, del_k, step = 0;                                         \
     if (h->n_occupied >= khash_upper_bound(h)) {                        \
       kh_resize_##name(mrb, h, h->n_buckets*2);                         \
     }                                                                   \
+  }                                                                     \
+  khint_t kh_put_##name(mrb_state *mrb, kh_##name##_t *h, khkey_t key, int *ret) \
+  {                                                                     \
+    khint_t k, del_k, step = 0;                                         \
+    kh_put_prepare_##name(mrb, h);                                      \
     k = __hash_func(mrb,key) & khash_mask(h);                           \
     del_k = kh_end(h);                                                  \
     while (!__ac_isempty(h->ed_flags, k)) {                             \
@@ -239,6 +256,7 @@ kh_fill_flags(uint8_t *p, uint8_t c, size_t len)
 #define kh_destroy(name, mrb, h) kh_destroy_##name(mrb, h)
 #define kh_clear(name, mrb, h) kh_clear_##name(mrb, h)
 #define kh_resize(name, mrb, h, s) kh_resize_##name(mrb, h, s)
+#define kh_put_prepare(name, mrb, h) kh_put_prepare_##name(mrb, h)
 #define kh_put(name, mrb, h, k) kh_put_##name(mrb, h, k, NULL)
 #define kh_put2(name, mrb, h, k, r) kh_put_##name(mrb, h, k, r)
 #define kh_get(name, mrb, h, k) kh_get_##name(mrb, h, k)

src/debug.c

@@ -204,11 +204,14 @@ mrb_debug_info_free(mrb_state *mrb, mrb_irep_debug_info *d)
 
   if (!d) { return; }
 
-  for (i = 0; i < d->flen; ++i) {
-    mrb_assert(d->files[i]);
-    mrb_free(mrb, d->files[i]->lines.ptr);
-    mrb_free(mrb, d->files[i]);
+  if (d->files) {
+    for (i = 0; i < d->flen; ++i) {
+      if (d->files[i]) {
+        mrb_free(mrb, d->files[i]->lines.ptr);
+        mrb_free(mrb, d->files[i]);
+      }
+    }
+    mrb_free(mrb, d->files);
   }
-  mrb_free(mrb, d->files);
   mrb_free(mrb, d);
 }

src/error.c

@@ -185,6 +185,16 @@ mrb_exc_set(mrb_state *mrb, mrb_value exc)
   }
 }
 
+static mrb_noreturn void
+exc_throw(mrb_state *mrb, mrb_value exc)
+{
+  if (!mrb->jmp) {
+    mrb_p(mrb, exc);
+    abort();
+  }
+  MRB_THROW(mrb->jmp);
+}
+
 MRB_API mrb_noreturn void
 mrb_exc_raise(mrb_state *mrb, mrb_value exc)
 {
@@ -197,11 +207,7 @@ mrb_exc_raise(mrb_state *mrb, mrb_value exc)
     }
     mrb_exc_set(mrb, exc);
   }
-  if (!mrb->jmp) {
-    mrb_p(mrb, exc);
-    abort();
-  }
-  MRB_THROW(mrb->jmp);
+  exc_throw(mrb, exc);
 }
 
 MRB_API mrb_noreturn void
@@ -550,6 +556,52 @@ mrb_argnum_error(mrb_state *mrb, mrb_int argc, int min, int max)
 #undef FMT
 }
 
+void mrb_core_init_printabort(void);
+
+int
+mrb_core_init_protect(mrb_state *mrb, void (*body)(mrb_state *, void *), void *opaque)
+{
+  struct mrb_jmpbuf *prev_jmp = mrb->jmp;
+  struct mrb_jmpbuf c_jmp;
+  int err = 1;
+
+  MRB_TRY(&c_jmp) {
+    mrb->jmp = &c_jmp;
+    body(mrb, opaque);
+    err = 0;
+  } MRB_CATCH(&c_jmp) {
+    if (mrb->exc) {
+      mrb_p(mrb, mrb_obj_value(mrb->exc));
+      mrb->exc = NULL;
+    }
+    else {
+      mrb_core_init_printabort();
+    }
+  } MRB_END_EXC(&c_jmp);
+
+  mrb->jmp = prev_jmp;
+
+  return err;
+}
+
+mrb_noreturn void
+mrb_core_init_abort(mrb_state *mrb)
+{
+  mrb->exc = NULL;
+  exc_throw(mrb, mrb_nil_value());
+}
+
+mrb_noreturn void
+mrb_raise_nomemory(mrb_state *mrb)
+{
+  if (mrb->nomem_err) {
+    mrb_exc_raise(mrb, mrb_obj_value(mrb->nomem_err));
+  }
+  else {
+    mrb_core_init_abort(mrb);
+  }
+}
+
 void
 mrb_init_exception(mrb_state *mrb)
 {

src/gc.c

@@ -201,6 +201,8 @@ gettimeofday_time(void)
 
 #define objects(p) ((RVALUE *)p->objects)
 
+mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);
+
 MRB_API void*
 mrb_realloc_simple(mrb_state *mrb, void *p,  size_t len)
 {
@@ -224,12 +226,12 @@ mrb_realloc(mrb_state *mrb, void *p, size_t len)
   if (len == 0) return p2;
   if (p2 == NULL) {
     if (mrb->gc.out_of_memory) {
-      mrb_exc_raise(mrb, mrb_obj_value(mrb->nomem_err));
+      mrb_raise_nomemory(mrb);
       /* mrb_panic(mrb); */
     }
     else {
       mrb->gc.out_of_memory = TRUE;
-      mrb_exc_raise(mrb, mrb_obj_value(mrb->nomem_err));
+      mrb_raise_nomemory(mrb);
     }
   }
   else {
@@ -1294,6 +1296,7 @@ mrb_full_gc(mrb_state *mrb)
 {
   mrb_gc *gc = &mrb->gc;
 
+  if (!mrb->c) return;
   if (gc->disabled || gc->iterating) return;
 
   GC_INVOKE_TIME_REPORT("mrb_full_gc()");

src/load.c

@@ -14,6 +14,7 @@
 #include <mruby/string.h>
 #include <mruby/debug.h>
 #include <mruby/error.h>
+#include <mruby/data.h>
 
 #if SIZE_MAX < UINT32_MAX
 # error size_t must be at least 32 bits wide
@@ -55,6 +56,14 @@ str_to_double(mrb_state *mrb, const char *p, size_t len)
 
 mrb_value mrb_str_len_to_inum(mrb_state *mrb, const char *str, mrb_int len, mrb_int base, int badcheck);
 
+static void
+tempirep_free(mrb_state *mrb, void *p)
+{
+  if (p) mrb_irep_decref(mrb, (mrb_irep *)p);
+}
+
+static const mrb_data_type tempirep_type = { "temporary irep", tempirep_free };
+
 static mrb_irep*
 read_irep_record_1(mrb_state *mrb, const uint8_t *bin, size_t *len, uint8_t flags)
 {
@@ -63,8 +72,11 @@ read_irep_record_1(mrb_state *mrb, const uint8_t *bin, size_t *len, uint8_t flag
   ptrdiff_t diff;
   uint16_t tt, pool_data_len, snl;
   int plen;
-  int ai = mrb_gc_arena_save(mrb);
+  struct RData *irep_obj = mrb_data_object_alloc(mrb, mrb->object_class, NULL, &tempirep_type);
   mrb_irep *irep = mrb_add_irep(mrb);
+  int ai = mrb_gc_arena_save(mrb);
+
+  irep_obj->data = irep;
 
   /* skip record size */
   src += sizeof(uint32_t);
@@ -184,36 +196,47 @@ read_irep_record_1(mrb_state *mrb, const uint8_t *bin, size_t *len, uint8_t flag
     }
   }
 
-  irep->reps = (mrb_irep**)mrb_malloc(mrb, sizeof(mrb_irep*)*irep->rlen);
+  irep->reps = (mrb_irep**)mrb_calloc(mrb, irep->rlen, sizeof(mrb_irep*));
 
   diff = src - bin;
   mrb_assert_int_fit(ptrdiff_t, diff, size_t, SIZE_MAX);
   *len = (size_t)diff;
 
+  irep_obj->data = NULL;
+
   return irep;
 }
 
 static mrb_irep*
 read_irep_record(mrb_state *mrb, const uint8_t *bin, size_t *len, uint8_t flags)
 {
+  struct RData *irep_obj = mrb_data_object_alloc(mrb, mrb->object_class, NULL, &tempirep_type);
+  mrb_int ai = mrb_gc_arena_save(mrb);
   mrb_irep *irep = read_irep_record_1(mrb, bin, len, flags);
   int i;
 
+  mrb_gc_arena_restore(mrb, ai);
   if (irep == NULL) {
     return NULL;
   }
 
+  irep_obj->data = irep;
+
   bin += *len;
   for (i=0; i<irep->rlen; i++) {
     size_t rlen;
 
     irep->reps[i] = read_irep_record(mrb, bin, &rlen, flags);
+    mrb_gc_arena_restore(mrb, ai);
     if (irep->reps[i] == NULL) {
       return NULL;
     }
     bin += rlen;
     *len += rlen;
   }
+
+  irep_obj->data = NULL;
+
   return irep;
 }
 
@@ -237,21 +260,21 @@ read_debug_record(mrb_state *mrb, const uint8_t *start, mrb_irep* irep, size_t *
 
   if (irep->debug_info) { return MRB_DUMP_INVALID_IREP; }
 
-  irep->debug_info = (mrb_irep_debug_info*)mrb_malloc(mrb, sizeof(mrb_irep_debug_info));
+  irep->debug_info = (mrb_irep_debug_info*)mrb_calloc(mrb, 1, sizeof(mrb_irep_debug_info));
   irep->debug_info->pc_count = (uint32_t)irep->ilen;
 
   record_size = (size_t)bin_to_uint32(bin);
   bin += sizeof(uint32_t);
 
   irep->debug_info->flen = bin_to_uint16(bin);
-  irep->debug_info->files = (mrb_irep_debug_info_file**)mrb_malloc(mrb, sizeof(mrb_irep_debug_info*) * irep->debug_info->flen);
+  irep->debug_info->files = (mrb_irep_debug_info_file**)mrb_calloc(mrb, irep->debug_info->flen, sizeof(mrb_irep_debug_info*));
   bin += sizeof(uint16_t);
 
   for (f_idx = 0; f_idx < irep->debug_info->flen; ++f_idx) {
     mrb_irep_debug_info_file *file;
     uint16_t filename_idx;
 
-    file = (mrb_irep_debug_info_file *)mrb_malloc(mrb, sizeof(*file));
+    file = (mrb_irep_debug_info_file *)mrb_calloc(mrb, 1, sizeof(*file));
     irep->debug_info->files[f_idx] = file;
 
     file->start_pos = bin_to_uint32(bin);
@@ -281,8 +304,8 @@ read_debug_record(mrb_state *mrb, const uint8_t *start, mrb_irep* irep, size_t *
       case mrb_debug_line_flat_map: {
         uint32_t l;
 
-        file->lines.flat_map = (mrb_irep_debug_info_line*)mrb_malloc(
-            mrb, sizeof(mrb_irep_debug_info_line) * (size_t)(file->line_entry_count));
+        file->lines.flat_map = (mrb_irep_debug_info_line*)mrb_calloc(
+            mrb, (size_t)(file->line_entry_count), sizeof(mrb_irep_debug_info_line));
         for (l = 0; l < file->line_entry_count; ++l) {
           file->lines.flat_map[l].start_pos = bin_to_uint32(bin);
           bin += sizeof(uint32_t);
@@ -329,6 +352,7 @@ read_section_debug(mrb_state *mrb, const uint8_t *start, mrb_irep *irep, uint8_t
   int result;
   uint16_t filenames_len;
   mrb_sym *filenames;
+  mrb_value filenames_obj;
 
   bin = start;
   header = (struct rite_section_debug_header *)bin;
@@ -336,7 +360,8 @@ read_section_debug(mrb_state *mrb, const uint8_t *start, mrb_irep *irep, uint8_t
 
   filenames_len = bin_to_uint16(bin);
   bin += sizeof(uint16_t);
-  filenames = (mrb_sym*)mrb_malloc(mrb, sizeof(mrb_sym) * (size_t)filenames_len);
+  filenames_obj = mrb_str_new(mrb, NULL, sizeof(mrb_sym) * (size_t)filenames_len);
+  filenames = (mrb_sym*)RSTRING_PTR(filenames_obj);
   for (i = 0; i < filenames_len; ++i) {
     uint16_t f_len = bin_to_uint16(bin);
     bin += sizeof(uint16_t);
@@ -360,7 +385,7 @@ read_section_debug(mrb_state *mrb, const uint8_t *start, mrb_irep *irep, uint8_t
   }
 
 debug_exit:
-  mrb_free(mrb, filenames);
+  mrb_str_resize(mrb, filenames_obj, 0);
   return result;
 }
 
@@ -418,6 +443,7 @@ read_section_lv(mrb_state *mrb, const uint8_t *start, mrb_irep *irep, uint8_t fl
   int result;
   uint32_t syms_len;
   mrb_sym *syms;
+  mrb_value syms_obj;
   mrb_sym (*intern_func)(mrb_state*, const char*, size_t) =
     (flags & FLAG_SRC_MALLOC)? mrb_intern : mrb_intern_static;
 
@@ -427,7 +453,8 @@ read_section_lv(mrb_state *mrb, const uint8_t *start, mrb_irep *irep, uint8_t fl
 
   syms_len = bin_to_uint32(bin);
   bin += sizeof(uint32_t);
-  syms = (mrb_sym*)mrb_malloc(mrb, sizeof(mrb_sym) * (size_t)syms_len);
+  syms_obj = mrb_str_new(mrb, NULL, sizeof(mrb_sym) * (size_t)syms_len);
+  syms = (mrb_sym*)RSTRING_PTR(syms_obj);
   for (i = 0; i < syms_len; ++i) {
     uint16_t const str_len = bin_to_uint16(bin);
     bin += sizeof(uint16_t);
@@ -447,7 +474,7 @@ read_section_lv(mrb_state *mrb, const uint8_t *start, mrb_irep *irep, uint8_t fl
   }
 
 lv_exit:
-  mrb_free(mrb, syms);
+  mrb_str_resize(mrb, syms_obj, 0);
   return result;
 }
 
@@ -484,6 +511,7 @@ static mrb_irep*
 read_irep(mrb_state *mrb, const uint8_t *bin, size_t bufsize, uint8_t flags)
 {
   int result;
+  struct RData *irep_obj = NULL;
   mrb_irep *irep = NULL;
   const struct rite_section_header *section_header;
   uint16_t crc;
@@ -504,12 +532,15 @@ read_irep(mrb_state *mrb, const uint8_t *bin, size_t bufsize, uint8_t flags)
     return NULL;
   }
 
+  irep_obj = mrb_data_object_alloc(mrb, mrb->object_class, NULL, &tempirep_type);
+
   bin += sizeof(struct rite_binary_header);
   do {
     section_header = (const struct rite_section_header *)bin;
     if (memcmp(section_header->section_ident, RITE_SECTION_IREP_IDENT, sizeof(section_header->section_ident)) == 0) {
       irep = read_section_irep(mrb, bin, flags);
       if (!irep) return NULL;
+      irep_obj->data = irep;
     }
     else if (memcmp(section_header->section_ident, RITE_SECTION_DEBUG_IDENT, sizeof(section_header->section_ident)) == 0) {
       if (!irep) return NULL;   /* corrupted data */
@@ -528,6 +559,8 @@ read_irep(mrb_state *mrb, const uint8_t *bin, size_t bufsize, uint8_t flags)
     bin += bin_to_uint32(section_header->section_size);
   } while (memcmp(section_header->section_ident, RITE_BINARY_EOF, sizeof(section_header->section_ident)) != 0);
 
+  irep_obj->data = NULL;
+
   return irep;
 }
 
@@ -579,7 +612,16 @@ load_irep(mrb_state *mrb, mrb_irep *irep, mrbc_context *c)
 MRB_API mrb_value
 mrb_load_irep_cxt(mrb_state *mrb, const uint8_t *bin, mrbc_context *c)
 {
-  return load_irep(mrb, mrb_read_irep(mrb, bin), c);
+  struct RData *irep_obj = mrb_data_object_alloc(mrb, mrb->object_class, NULL, &tempirep_type);
+  mrb_irep *irep = mrb_read_irep(mrb, bin);
+  mrb_value ret;
+
+  irep_obj->data = irep;
+  mrb_irep_incref(mrb, irep);
+  ret = load_irep(mrb, irep, c);
+  irep_obj->data = NULL;
+  mrb_irep_decref(mrb, irep);
+  return ret;
 }
 
 MRB_API mrb_value

src/print.c

@@ -7,24 +7,48 @@
 #include <mruby.h>
 #include <mruby/string.h>
 #include <mruby/variable.h>
+#include <mruby/error.h>
+#include <string.h>
 
 #ifndef MRB_DISABLE_STDIO
+static void
+printcstr(const char *str, size_t len, FILE *stream)
+{
+  if (str) {
+    fwrite(str, len, 1, stream);
+    putc('\n', stream);
+  }
+}
+
 static void
 printstr(mrb_value obj, FILE *stream)
 {
   if (mrb_string_p(obj)) {
-    fwrite(RSTRING_PTR(obj), RSTRING_LEN(obj), 1, stream);
-    putc('\n', stream);
+    printcstr(RSTRING_PTR(obj), RSTRING_LEN(obj), stream);
   }
 }
 #else
+# define printcstr(str, len, stream) (void)0
 # define printstr(obj, stream) (void)0
 #endif
 
+void
+mrb_core_init_printabort(void)
+{
+  static const char *str = "Failed mruby core initialization";
+  printcstr(str, strlen(str), stdout);
+}
+
 MRB_API void
 mrb_p(mrb_state *mrb, mrb_value obj)
 {
-  printstr(mrb_inspect(mrb, obj), stdout);
+  if (mrb_type(obj) == MRB_TT_EXCEPTION && mrb_obj_ptr(obj) == mrb->nomem_err) {
+    static const char *str = "Out of memory";
+    printcstr(str, strlen(str), stdout);
+  }
+  else {
+    printstr(mrb_inspect(mrb, obj), stdout);
+  }
 }
 
 MRB_API void

src/proc.c

@@ -8,6 +8,7 @@
 #include <mruby/class.h>
 #include <mruby/proc.h>
 #include <mruby/opcode.h>
+#include <mruby/data.h>
 
 static const mrb_code call_iseq[] = {
   OP_CALL,
@@ -122,7 +123,14 @@ mrb_proc_new_cfunc_with_env(mrb_state *mrb, mrb_func_t func, mrb_int argc, const
   p->flags |= MRB_PROC_ENVSET;
   mrb_field_write_barrier(mrb, (struct RBasic*)p, (struct RBasic*)e);
   MRB_ENV_UNSHARE_STACK(e);
+
+  /* NOTE: Prevents keeping invalid addresses when NoMemoryError is raised from `mrb_malloc()`. */
+  e->stack = NULL;
+  MRB_ENV_SET_STACK_LEN(e, 0);
+
   e->stack = (mrb_value*)mrb_malloc(mrb, sizeof(mrb_value) * argc);
+  MRB_ENV_SET_STACK_LEN(e, argc);
+
   if (argv) {
     for (i = 0; i < argc; ++i) {
       e->stack[i] = argv[i];
@@ -286,14 +294,25 @@ mrb_proc_arity(const struct RProc *p)
   return arity;
 }
 
+static void
+tempirep_free(mrb_state *mrb, void *p)
+{
+  if (p) mrb_irep_free(mrb, (mrb_irep *)p);
+}
+
+static const mrb_data_type tempirep_type = { "temporary irep", tempirep_free };
+
 void
 mrb_init_proc(mrb_state *mrb)
 {
   struct RProc *p;
   mrb_method_t m;
-  mrb_irep *call_irep = (mrb_irep *)mrb_malloc(mrb, sizeof(mrb_irep));
+  struct RData *irep_obj = mrb_data_object_alloc(mrb, mrb->object_class, NULL, &tempirep_type);
+  mrb_irep *call_irep;
   static const mrb_irep mrb_irep_zero = { 0 };
 
+  call_irep = (mrb_irep *)mrb_malloc(mrb, sizeof(mrb_irep));
+  irep_obj->data = call_irep;
   *call_irep = mrb_irep_zero;
   call_irep->flags = MRB_ISEQ_NO_FREE;
   call_irep->iseq = call_iseq;
@@ -305,6 +324,7 @@ mrb_init_proc(mrb_state *mrb)
   mrb_define_method(mrb, mrb->proc_class, "arity", proc_arity, MRB_ARGS_NONE());
 
   p = mrb_proc_new(mrb, call_irep);
+  irep_obj->data = NULL;
   MRB_METHOD_FROM_PROC(m, p);
   mrb_define_method_raw(mrb, mrb->proc_class, mrb_intern_lit(mrb, "call"), m);
   mrb_define_method_raw(mrb, mrb->proc_class, mrb_intern_lit(mrb, "[]"), m);

src/state.c

@@ -19,11 +19,25 @@ void mrb_init_mrbgems(mrb_state*);
 void mrb_gc_init(mrb_state*, mrb_gc *gc);
 void mrb_gc_destroy(mrb_state*, mrb_gc *gc);
 
+int mrb_core_init_protect(mrb_state *mrb, void (*body)(mrb_state *, void *), void *opaque);
+
+static void
+init_gc_and_core(mrb_state *mrb, void *opaque)
+{
+  static const struct mrb_context mrb_context_zero = { 0 };
+
+  mrb_gc_init(mrb, &mrb->gc);
+  mrb->c = (struct mrb_context*)mrb_malloc(mrb, sizeof(struct mrb_context));
+  *mrb->c = mrb_context_zero;
+  mrb->root_c = mrb->c;
+
+  mrb_init_core(mrb);
+}
+
 MRB_API mrb_state*
 mrb_open_core(mrb_allocf f, void *ud)
 {
   static const mrb_state mrb_state_zero = { 0 };
-  static const struct mrb_context mrb_context_zero = { 0 };
   mrb_state *mrb;
 
   if (f == NULL) f = mrb_default_allocf;
@@ -35,12 +49,10 @@ mrb_open_core(mrb_allocf f, void *ud)
   mrb->allocf = f;
   mrb->atexit_stack_len = 0;
 
-  mrb_gc_init(mrb, &mrb->gc);
-  mrb->c = (struct mrb_context*)mrb_malloc(mrb, sizeof(struct mrb_context));
-  *mrb->c = mrb_context_zero;
-  mrb->root_c = mrb->c;
-
-  mrb_init_core(mrb);
+  if (mrb_core_init_protect(mrb, init_gc_and_core, NULL)) {
+    mrb_close(mrb);
+    return NULL;
+  }
 
   return mrb;
 }
@@ -65,6 +77,12 @@ mrb_open(void)
   return mrb;
 }
 
+static void
+init_mrbgems(mrb_state *mrb, void *opaque)
+{
+  mrb_init_mrbgems(mrb);
+}
+
 MRB_API mrb_state*
 mrb_open_allocf(mrb_allocf f, void *ud)
 {
@@ -75,7 +93,10 @@ mrb_open_allocf(mrb_allocf f, void *ud)
   }
 
 #ifndef DISABLE_GEMS
-  mrb_init_mrbgems(mrb);
+  if (mrb_core_init_protect(mrb, init_mrbgems, NULL)) {
+    mrb_close(mrb);
+    return NULL;
+  }
   mrb_gc_arena_restore(mrb, 0);
 #endif
   return mrb;
@@ -131,9 +152,11 @@ mrb_irep_free(mrb_state *mrb, mrb_irep *irep)
   }
   mrb_free(mrb, irep->pool);
   mrb_free(mrb, irep->syms);
-  for (i=0; i<irep->rlen; i++) {
-    if (irep->reps[i])
-      mrb_irep_decref(mrb, irep->reps[i]);
+  if (irep->reps) {
+    for (i=0; i<irep->rlen; i++) {
+      if (irep->reps[i])
+        mrb_irep_decref(mrb, irep->reps[i]);
+    }
   }
   mrb_free(mrb, irep->reps);
   mrb_free(mrb, irep->lv);

src/symbol.c

@@ -172,11 +172,13 @@ sym_intern(mrb_state *mrb, const char *name, size_t len, mrb_bool lit)
   if (sym > 0) return sym;
 
   /* registering a new symbol */
-  sym = ++mrb->symidx;
+  sym = mrb->symidx + 1;
   if (mrb->symcapa < sym) {
-    if (mrb->symcapa == 0) mrb->symcapa = 100;
-    else mrb->symcapa = (size_t)(mrb->symcapa * 6 / 5);
-    mrb->symtbl = (symbol_name*)mrb_realloc(mrb, mrb->symtbl, sizeof(symbol_name)*(mrb->symcapa+1));
+    size_t symcapa = mrb->symcapa;
+    if (symcapa == 0) symcapa = 100;
+    else symcapa = (size_t)(symcapa * 6 / 5);
+    mrb->symtbl = (symbol_name*)mrb_realloc(mrb, mrb->symtbl, sizeof(symbol_name)*(symcapa+1));
+    mrb->symcapa = symcapa;
   }
   sname = &mrb->symtbl[sym];
   sname->len = (uint16_t)len;
@@ -201,7 +203,7 @@ sym_intern(mrb_state *mrb, const char *name, size_t len, mrb_bool lit)
   else {
     sname->prev = 0;
   }
-  mrb->symhash[hash] = sym;
+  mrb->symhash[hash] = mrb->symidx = sym;
 
   return sym<<SYMBOL_NORMAL_SHIFT;
 }