Merge branch 'improve-source-scanning-for-presym' of...

Merge branch 'improve-source-scanning-for-presym' of https://github.com/shuujii/mruby into shuujii-improve-source-scanning-for-presym

Rakefile

@@ -25,12 +25,11 @@ end
 # load custom rules
 load "#{MRUBY_ROOT}/tasks/core.rake"
 load "#{MRUBY_ROOT}/tasks/mrblib.rake"
-
 load "#{MRUBY_ROOT}/tasks/mrbgems.rake"
 load "#{MRUBY_ROOT}/tasks/libmruby.rake"
-
+load "#{MRUBY_ROOT}/tasks/bin.rake"
+load "#{MRUBY_ROOT}/tasks/presym.rake"
 load "#{MRUBY_ROOT}/tasks/benchmark.rake"
-
 load "#{MRUBY_ROOT}/tasks/gitlab.rake"
 load "#{MRUBY_ROOT}/tasks/doc.rake"
 
@@ -38,219 +37,19 @@ load "#{MRUBY_ROOT}/tasks/doc.rake"
 # generic build targets, rules
 task :default => :all
 
-bin_path = ENV['INSTALL_DIR'] || "#{MRUBY_ROOT}/bin"
-
-if MRuby.targets['host']
-  target = MRuby.targets['host']
-  depfiles = target.bins.map do |bin|
-    install_path = target.exefile("#{bin_path}/#{bin}")
-    source_path = target.exefile("#{target.build_dir}/bin/#{bin}")
-
-    file install_path => source_path do |t|
-      install_D t.prerequisites.first, t.name
-    end
-    install_path
-  end
-else
-  depfiles = []
-end
-
-cfiles  = Dir.glob("#{MRUBY_ROOT}/src/*.c")
-rbfiles = Dir.glob("#{MRUBY_ROOT}/mrblib/**/*.rb")
-psfiles = []
-MRuby.each_target do |target|
-  gems.each do |gem|
-    current_dir = gem.dir.relative_path_from(Dir.pwd)
-    relative_from_root = gem.dir.relative_path_from(MRUBY_ROOT)
-    current_build_dir = File.expand_path "#{build_dir}/#{relative_from_root}"
-
-    if current_build_dir !~ /^#{Regexp.escape(build_dir)}/
-      current_build_dir = "#{build_dir}/mrbgems/#{gem.name}"
-    end
-
-    gem.bins.each do |bin|
-      exec = exefile("#{build_dir}/bin/#{bin}")
-      objs = Dir.glob("#{current_dir}/tools/#{bin}/*.{c,cpp,cxx,cc}").map { |f| objfile(f.pathmap("#{current_build_dir}/tools/#{bin}/%n")) }
-
-      file exec => objs + target.libraries do |t|
-        gem_flags = gems.map { |g| g.linker.flags }
-        gem_flags_before_libraries = gems.map { |g| g.linker.flags_before_libraries }
-        gem_flags_after_libraries = gems.map { |g| g.linker.flags_after_libraries }
-        gem_libraries = gems.map { |g| g.linker.libraries }
-        gem_library_paths = gems.map { |g| g.linker.library_paths }
-        linker.run t.name, t.prerequisites, gem_libraries, gem_library_paths, gem_flags, gem_flags_before_libraries, gem_flags_after_libraries
-      end
-
-      if target == MRuby.targets['host']
-        install_path = MRuby.targets['host'].exefile("#{bin_path}/#{bin}")
-
-        file install_path => exec do |t|
-          install_D t.prerequisites.first, t.name
-        end
-        depfiles += [ install_path ]
-      else
-        depfiles += [ exec ]
-      end
-    end
-
-    cfiles += Dir.glob(gem.dir+"/{src,core,tools/*}/*.c")
-    if gem.cdump?
-      rbfiles += Dir.glob(gem.dir+"/mrblib/**/*.rb")
-      psfiles += Dir.glob(gem.dir+"/**/presym")
-    end
-  end
-end
-
-mkdir_p "#{MRUBY_ROOT}/build"
-symbols = []
-psfiles.each do |file|
-  symbols += File.readlines(file).grep_v(/^# /)
-end
-symbols.each{|x| x.chomp!}
-presym_file="#{MRUBY_ROOT}/build/presym"
-presym_inc="#{presym_file}.inc"
-op_table = {
-  "!" => "not",
-  "!=" => "neq",
-  "!~" => "nmatch",
-  "%" => "mod",
-  "&" => "and",
-  "&&" => "andand",
-  "*" => "mul",
-  "**" => "pow",
-  "+" => "add",
-  "+@" => "plus",
-  "-" => "sub",
-  "-@" => "minus",
-  "/" => "div",
-  "<" => "lt",
-  "<=" => "le",
-  "<<" => "lshift",
-  "<=>" => "cmp",
-  "==" => "eq",
-  "===" => "eqq",
-  "=~" => "match",
-  ">" => "gt",
-  ">=" => "ge",
-  ">>" => "rshift",
-  "[]" => "aref",
-  "[]=" => "aset",
-  "^" => "xor",
-  "`" => "tick",
-  "|" => "or",
-  "||" => "oror",
-  "~" => "neg",
-}
-macro_to_symbol = {
-#      Macro               Symbol
-# [prefix, suffix] => [prefix, suffix]
-  ["CV"  , ""    ] => ["@@"  , ""    ],
-  ["IV"  , ""    ] => ["@"   , ""    ],
-  [""    , "_B"  ] => [""    , "!"   ],
-  [""    , "_Q"  ] => [""    , "?"   ],
-  [""    , "_E"  ] => [""    , "="   ],
-  [""    , ""    ] => [""    , ""    ],
-}
-
-file presym_file => cfiles+rbfiles+psfiles+[__FILE__] do
-  prefix_re = Regexp.union(*macro_to_symbol.keys.map(&:first).uniq)
-  suffix_re = Regexp.union(*macro_to_symbol.keys.map(&:last).uniq)
-  macro_re = /MRB_(#{prefix_re})SYM(#{suffix_re})\((\w+)\)/o
-  csymbols = cfiles.map do |f|
-    src = File.read(f)
-    src.gsub!(/\/\/.+(\n|$)/, "\n")
-    [src.scan(/intern_lit\([^\n"]*"([^\n "]*)"/),
-     src.scan(/mrb_define_method\([^\n"]*"([^\n"]*)"/),
-     src.scan(/mrb_define_class_method\([^\n"]*"([^\n"]*)"/),
-     src.scan(/mrb_define_class\([^\n"]*"([^\n"]*)"/),
-     src.scan(/mrb_define_module\([^\n"]*"([^\n"]*)"/),
-     src.scan(/mrb_define_module_function\([^\n"]*"([^\n"]*)"/),
-     src.scan(/mrb_define_const\([^\n"]*"([^\n"]*)"/),
-     src.scan(/mrb_define_global_const\([^\n"]*"([^\n"]*)"/),
-     src.scan(macro_re).map{|prefix, suffix, name|
-       macro_to_symbol[[prefix, suffix]] * name
-     }]
-  end
-  csymbols += File.readlines("#{MRUBY_ROOT}/include/mruby.h").grep(/define E_/).join.scan(/MRB_SYM\((\w+)\)/)
-
-  rbsymbols = rbfiles.map do |f|
-    src = File.read(f)
-    src.force_encoding(Encoding::BINARY)
-    [src.scan(/\bclass +([A-Z]\w*)/),
-     src.scan(/\bmodule +([A-Z]\w*)/),
-     src.scan(/\bdef +(\w+[!?=]?)/),
-     src.scan(/\balias +(\w+[!?]?)/),
-     src.scan(/\b([A-Z]\w*) *=[^=]/),
-     src.scan(/(\$[a-zA-Z_]\w*)/),
-     src.scan(/(\$[$!?0-9]\w*)/),
-     src.scan(/(@@?[a-zA-Z_]\w*)/),
-     src.scan(/[^.]\.([a-zA-Z_]\w*[!?]?)/),
-     src.scan(/\.([a-zA-Z_]\w* *=)/).map{|x|x.map{|s|s.gsub(' ', '')}},
-     src.scan(/\b([a-zA-Z_]\w*):/),
-     src.scan(/:([a-zA-Z_]\w*[!?=]?)/),
-     src.scan(/[\(\[\{ ]:"([^"]+)"/).map{|x|x.map{|s|s.gsub('\#', '#')}},
-     src.scan(/[ \(\[\{]:'([^']+)'/)
-    ]
-  end
-  symbols = (symbols+csymbols+rbsymbols+op_table.keys).flatten.compact.uniq.grep_v(/#/).map{|x| x.gsub("\n", '\n')}.sort_by!{|x| [x.bytesize, x]}
-  presyms = File.readlines(presym_file) rescue []
-  presyms.each{|x| x.chomp!}
-  if presyms != symbols
-    _pp "GEN", presym_file.relative_path
-    File.write(presym_file, symbols.join("\n"))
-    Rake::Task[presym_inc].invoke
-  end
-end
-
-task presym_inc do
-  presyms = File.readlines(presym_file)
-  presyms.each{|x| x.chomp!}
-  symbol_to_macro = macro_to_symbol.invert
-  prefix_re = Regexp.union(*symbol_to_macro.keys.map(&:first).uniq)
-  suffix_re = Regexp.union(*symbol_to_macro.keys.map(&:last).uniq)
-  sym_re = /\A(#{prefix_re})?([\w&&\D]\w*)(#{suffix_re})?\z/o
-  _pp "GEN", presym_inc.relative_path
-  File.open(presym_inc, "w") do |f|
-    f.puts "/* MRB_PRESYM_NAMED(lit, num, type, name) */"
-    f.puts "/* MRB_PRESYM_UNNAMED(lit, num) */"
-    presyms.each.with_index(1) do |sym, num|
-      if sym_re =~ sym && (affixes = symbol_to_macro[[$1, $3]])
-        f.puts %|MRB_PRESYM_NAMED("#{sym}", #{num}, #{affixes * 'SYM'}, #{$2})|
-      elsif name = op_table[sym]
-        f.puts %|MRB_PRESYM_NAMED("#{sym}", #{num}, OPSYM, #{name})|
-      elsif
-        f.puts %|MRB_PRESYM_UNNAMED("#{sym}", #{num})|
-      end
-    end
-    f.print "#define MRB_PRESYM_MAX #{presyms.size}"
-  end
-end
-
-desc "preallocated symbols"
-task :gensym => presym_file
-
-depfiles += MRuby.targets.map { |n, t|
-  t.libmruby_enabled? ? t.libraries : t.libmruby_core_static
-}.flatten
-
-depfiles += MRuby.targets.reject { |n, t| n == 'host' }.map { |n, t|
-  t.bins.map { |bin| t.exefile("#{t.build_dir}/bin/#{bin}") }
-}.flatten
-
 desc "build all targets, install (locally) in-repo"
-task :all => :build do
+task :all => :gensym do
+  Rake::Task[:build].invoke
   puts
   puts "Build summary:"
   puts
-  MRuby.each_target do
-    print_build_summary
+  MRuby.each_target do |build|
+    build.print_build_summary
   end
   MRuby::Lockfile.write
 end
 
-task :build => :gensym do
-  depfiles.each {|dep| Rake::Task[dep].invoke}
-end
+task :build => MRuby.targets.flat_map{|_, build| build.products}
 
 desc "run all mruby tests"
 task :test
@@ -274,19 +73,17 @@ end
 
 desc "clean all built and in-repo installed artifacts"
 task :clean do
-  MRuby.each_target do |t|
-    rm_rf t.build_dir
+  MRuby.each_target do |build|
+    rm_rf build.products
+    rm_rf build.build_dir
   end
-  rm_f depfiles
-  rm_f presym_file
-  rm_f presym_inc
   puts "Cleaned up target build folder"
 end
 
 desc "clean everything!"
 task :deep_clean => ["clean", "clean_doc"] do
-  MRuby.each_target do |t|
-    rm_rf t.gem_clone_dir
+  MRuby.each_target do |build|
+    rm_rf build.gem_clone_dir
   end
   puts "Cleaned up mrbgems build folder"
 end

build_config/ci/msvc.rb

 def setup_option(conf)
-  conf.cc.flags[0].delete("/Zi") unless ENV['CFLAGS']
-  conf.cxx.flags[0].delete("/Zi") unless ENV['CFLAGS'] || ENV['CXXFLAGS']
+  conf.cc.compile_options.sub!(%r{/Zi }, "") unless ENV['CFLAGS']
+  conf.cxx.compile_options.sub!(%r{/Zi }, "") unless ENV['CFLAGS'] || ENV['CXXFLAGS']
   conf.linker.flags << "/DEBUG:NONE" unless ENV['LDFLAGS']
 end
 

build_config/mrbc.rb

+MRuby::Build.new do |conf|
+  if ENV['VisualStudioVersion'] || ENV['VSINSTALLDIR']
+    conf.toolchain :visualcpp
+  else
+    conf.toolchain :gcc
+  end
+
+  conf.build_mrbc_exec
+  conf.disable_libmruby
+  conf.disable_presym
+end

include/mruby.h

@@ -93,7 +93,6 @@
 #include <mruby/common.h>
 #include <mruby/value.h>
 #include <mruby/gc.h>
-#include <mruby/presym.h>
 #include <mruby/version.h>
 
 #ifndef MRB_NO_FLOAT
@@ -1410,6 +1409,8 @@ MRB_API void mrb_show_copyright(mrb_state *mrb);
 
 MRB_API mrb_value mrb_format(mrb_state *mrb, const char *format, ...);
 
+#include <mruby/presym.h>
+
 #if 0
 /* memcpy and memset does not work with gdb reverse-next on my box */
 /* use naive memcpy and memset instead */

include/mruby/presym.h

@@ -7,42 +7,36 @@
 #ifndef MRUBY_PRESYM_H
 #define MRUBY_PRESYM_H
 
-#undef MRB_PRESYM_MAX
-#ifdef MRB_USE_ALL_SYMBOLS
-#define MRB_PRESYM_NAMED(lit, num, type, name) MRB_##type##__##name = (num),
+#if defined(MRB_PRESYM_SCANNING)
+# include <mruby/presym/scanning.h>
+#elif defined(MRB_NO_PRESYM)
+# include <mruby/presym/disable.h>
 #else
-#define MRB_PRESYM_NAMED(lit, num, type, name) MRB_##type##__##name = (num<<1),
+# include <mruby/presym/enable.h>
 #endif
-#define MRB_PRESYM_UNNAMED(lit, num)
-
-enum mruby_presym {
-#include <../build/presym.inc>
-};
-
-#undef MRB_PRESYM_NAMED
-#undef MRB_PRESYM_UNNAMED
 
 /*
- * For `MRB_OPSYM`, specify the names corresponding to operators (refer to
- * `op_table` in `Rakefile` for the names that can be specified for it).
- * Other than that, describe only word characters excluding leading and
- * ending punctuations.
+ * Where `mrb_intern_lit` is allowed for symbol interning, it is directly
+ * replaced by the symbol ID if presym is enabled by using the following
+ * macros.
+ *
+ *   MRB_OPSYM(xor)  //=> ^      (Operator)
+ *   MRB_CVSYM(xor)  //=> @@xor  (Class Variable)
+ *   MRB_IVSYM(xor)  //=> @xor   (Instance Variable)
+ *   MRB_SYM_B(xor)  //=> xor!   (Method with Bang)
+ *   MRB_SYM_Q(xor)  //=> xor?   (Method with Question mark)
+ *   MRB_SYM_E(xor)  //=> xor=   (Method with Equal)
+ *   MRB_SYM(xor)    //=> xor    (Word characters)
+ *
+ * For `MRB_OPSYM`, specify the names corresponding to operators (see
+ * `MRuby::Presym::OPERATORS` in `lib/mruby/presym.rb for the names that can
+ * be specified for it). Other than that, describe only word characters
+ * excluding leading and ending punctuations.
  *
- * Example:
- *   MRB_OPSYM(and)  //=> &
- *   MRB_CVSYM(foo)  //=> @@foo
- *   MRB_IVSYM(foo)  //=> @foo
- *   MRB_SYM_B(foo)  //=> foo!
- *   MRB_SYM_Q(foo)  //=> foo?
- *   MRB_SYM_E(foo)  //=> foo=
- *   MRB_SYM(foo)    //=> foo
+ * These macros are expanded to `mrb_intern_lit` if presym is disabled,
+ * therefore the mruby state variable is required. The above macros can be
+ * used when the variable name is `mrb`. If you want to use other variable
+ * names, you need to use macros with `_2` suffix, such as `MRB_SYM_2`.
  */
-#define MRB_OPSYM(name) MRB_OPSYM__##name  /* Operator */
-#define MRB_CVSYM(name) MRB_CVSYM__##name  /* Class Variable */
-#define MRB_IVSYM(name) MRB_IVSYM__##name  /* Instance Variable */
-#define MRB_SYM_B(name) MRB_SYM_B__##name  /* Method with Bang */
-#define MRB_SYM_Q(name) MRB_SYM_Q__##name  /* Method with Question mark */
-#define MRB_SYM_E(name) MRB_SYM_E__##name  /* Method with Equal */
-#define MRB_SYM(name)   MRB_SYM__##name    /* Word characters */
 
 #endif  /* MRUBY_PRESYM_H */

include/mruby/presym/disable.h

+/**
+** @file mruby/presym/scanning.h - Disable Preallocated Symbols
+**
+** See Copyright Notice in mruby.h
+*/
+
+#ifndef MRUBY_PRESYM_DISABLE_H
+#define MRUBY_PRESYM_DISABLE_H
+
+#include <string.h>
+
+#define MRB_PRESYM_MAX 0
+
+#define MRB_OPSYM(name) MRB_OPSYM__##name(mrb)
+#define MRB_CVSYM(name) mrb_intern_lit(mrb, "@@" #name)
+#define MRB_IVSYM(name) mrb_intern_lit(mrb, "@" #name)
+#define MRB_SYM_B(name) mrb_intern_lit(mrb, #name "!")
+#define MRB_SYM_Q(name) mrb_intern_lit(mrb, #name "?")
+#define MRB_SYM_E(name) mrb_intern_lit(mrb, #name "=")
+#define MRB_SYM(name) mrb_intern_lit(mrb, #name)
+
+#define MRB_OPSYM_2(mrb, name) MRB_OPSYM__##name(mrb)
+#define MRB_CVSYM_2(mrb, name) mrb_intern_lit(mrb, "@@" #name)
+#define MRB_IVSYM_2(mrb, name) mrb_intern_lit(mrb, "@" #name)
+#define MRB_SYM_B_2(mrb, name) mrb_intern_lit(mrb, #name "!")
+#define MRB_SYM_Q_2(mrb, name) mrb_intern_lit(mrb, #name "?")
+#define MRB_SYM_E_2(mrb, name) mrb_intern_lit(mrb, #name "=")
+#define MRB_SYM_2(mrb, name) mrb_intern_lit(mrb, #name)
+
+#define MRB_OPSYM__not(mrb) mrb_intern_lit(mrb, "!")
+#define MRB_OPSYM__mod(mrb) mrb_intern_lit(mrb, "%")
+#define MRB_OPSYM__and(mrb) mrb_intern_lit(mrb, "&")
+#define MRB_OPSYM__mul(mrb) mrb_intern_lit(mrb, "*")
+#define MRB_OPSYM__add(mrb) mrb_intern_lit(mrb, "+")
+#define MRB_OPSYM__sub(mrb) mrb_intern_lit(mrb, "-")
+#define MRB_OPSYM__div(mrb) mrb_intern_lit(mrb, "/")
+#define MRB_OPSYM__lt(mrb) mrb_intern_lit(mrb, "<")
+#define MRB_OPSYM__gt(mrb) mrb_intern_lit(mrb, ">")
+#define MRB_OPSYM__xor(mrb) mrb_intern_lit(mrb, "^")
+#define MRB_OPSYM__tick(mrb) mrb_intern_lit(mrb, "`")
+#define MRB_OPSYM__or(mrb) mrb_intern_lit(mrb, "|")
+#define MRB_OPSYM__neg(mrb) mrb_intern_lit(mrb, "~")
+#define MRB_OPSYM__neq(mrb) mrb_intern_lit(mrb, "!=")
+#define MRB_OPSYM__nmatch(mrb) mrb_intern_lit(mrb, "!~")
+#define MRB_OPSYM__andand(mrb) mrb_intern_lit(mrb, "&&")
+#define MRB_OPSYM__pow(mrb) mrb_intern_lit(mrb, "**")
+#define MRB_OPSYM__plus(mrb) mrb_intern_lit(mrb, "+@")
+#define MRB_OPSYM__minus(mrb) mrb_intern_lit(mrb, "-@")
+#define MRB_OPSYM__lshift(mrb) mrb_intern_lit(mrb, "<<")
+#define MRB_OPSYM__le(mrb) mrb_intern_lit(mrb, "<=")
+#define MRB_OPSYM__eq(mrb) mrb_intern_lit(mrb, "==")
+#define MRB_OPSYM__match(mrb) mrb_intern_lit(mrb, "=~")
+#define MRB_OPSYM__ge(mrb) mrb_intern_lit(mrb, ">=")
+#define MRB_OPSYM__rshift(mrb) mrb_intern_lit(mrb, ">>")
+#define MRB_OPSYM__aref(mrb) mrb_intern_lit(mrb, "[]")
+#define MRB_OPSYM__oror(mrb) mrb_intern_lit(mrb, "||")
+#define MRB_OPSYM__cmp(mrb) mrb_intern_lit(mrb, "<=>")
+#define MRB_OPSYM__eqq(mrb) mrb_intern_lit(mrb, "===")
+#define MRB_OPSYM__aset(mrb) mrb_intern_lit(mrb, "[]=")
+
+#define MRB_PRESYM_DEFINE_VAR_AND_INITER(name, size, ...)                     \
+  static mrb_sym name[size];                                                  \
+  static void init_##name(mrb_state *mrb) {                                   \
+    mrb_sym name__[] = {__VA_ARGS__};                                         \
+    memcpy(name, name__, sizeof(name));                                       \
+  }
+
+#endif  /* MRUBY_PRESYM_DISABLE_H */

include/mruby/presym/enable.h

+/**
+** @file mruby/presym/scanning.h - Enable Preallocated Symbols
+**
+** See Copyright Notice in mruby.h
+*/
+
+#ifndef MRUBY_PRESYM_ENABLE_H
+#define MRUBY_PRESYM_ENABLE_H
+
+#undef MRB_PRESYM_MAX
+#ifdef MRB_USE_ALL_SYMBOLS
+# define MRB_PRESYM_NAMED(lit, num, type, name) MRB_##type##__##name = (num),
+#else
+# define MRB_PRESYM_NAMED(lit, num, type, name) MRB_##type##__##name = (num<<1),
+#endif
+#define MRB_PRESYM_UNNAMED(lit, num)
+
+enum mruby_presym {
+# include <mruby/presym.inc>
+};
+
+#undef MRB_PRESYM_NAMED
+#undef MRB_PRESYM_UNNAMED
+
+#define MRB_OPSYM(name) MRB_OPSYM__##name
+#define MRB_CVSYM(name) MRB_CVSYM__##name
+#define MRB_IVSYM(name) MRB_IVSYM__##name
+#define MRB_SYM_B(name) MRB_SYM_B__##name
+#define MRB_SYM_Q(name) MRB_SYM_Q__##name
+#define MRB_SYM_E(name) MRB_SYM_E__##name
+#define MRB_SYM(name) MRB_SYM__##name
+
+#define MRB_OPSYM_2(mrb, name) MRB_OPSYM__##name
+#define MRB_CVSYM_2(mrb, name) MRB_CVSYM__##name
+#define MRB_IVSYM_2(mrb, name) MRB_IVSYM__##name
+#define MRB_SYM_B_2(mrb, name) MRB_SYM_B__##name
+#define MRB_SYM_Q_2(mrb, name) MRB_SYM_Q__##name
+#define MRB_SYM_E_2(mrb, name) MRB_SYM_E__##name
+#define MRB_SYM_2(mrb, name) MRB_SYM__##name
+
+#define MRB_PRESYM_DEFINE_VAR_AND_INITER(name, size, ...)                     \
+  static const mrb_sym name[] = {__VA_ARGS__};                                \
+  static void init_##name(mrb_state *mrb) {}
+
+#endif  /* MRUBY_PRESYM_ENABLE_H */

include/mruby/presym/scanning.h

+/**
+** @file mruby/presym/scanning.h - Scanning Preallocated Symbols
+**
+** See Copyright Notice in mruby.h
+*/
+
+#ifndef MRUBY_PRESYM_SCANNING_H
+#define MRUBY_PRESYM_SCANNING_H
+
+#define MRB_PRESYM_SCANNING_TAGGED(arg) <@! arg !@>
+
+#undef mrb_intern_lit
+#define mrb_intern_lit(mrb, name) MRB_PRESYM_SCANNING_TAGGED(name)
+#define mrb_define_method(mrb, c, name, f, a) MRB_PRESYM_SCANNING_TAGGED(name)
+#define mrb_define_class_method(mrb, c, name, f, a) MRB_PRESYM_SCANNING_TAGGED(name)
+#define mrb_define_class(mrb, name, s) MRB_PRESYM_SCANNING_TAGGED(name)
+#define mrb_define_module(mrb, name) MRB_PRESYM_SCANNING_TAGGED(name)
+#define mrb_define_module_function(mrb, c, name, f, s) MRB_PRESYM_SCANNING_TAGGED(name)
+#define mrb_define_const(mrb, c, name, v) MRB_PRESYM_SCANNING_TAGGED(name)
+#define mrb_define_global_const(mrb, name, v) MRB_PRESYM_SCANNING_TAGGED(name)
+
+#define MRB_OPSYM(name) MRB_OPSYM__##name(mrb)
+#define MRB_CVSYM(name) MRB_PRESYM_SCANNING_TAGGED("@@" #name)
+#define MRB_IVSYM(name) MRB_PRESYM_SCANNING_TAGGED("@" #name)
+#define MRB_SYM_B(name) MRB_PRESYM_SCANNING_TAGGED(#name "!")
+#define MRB_SYM_Q(name) MRB_PRESYM_SCANNING_TAGGED(#name "?")
+#define MRB_SYM_E(name) MRB_PRESYM_SCANNING_TAGGED(#name "=")
+#define MRB_SYM(name) MRB_PRESYM_SCANNING_TAGGED(#name)
+
+#define MRB_OPSYM_2(mrb, name) MRB_OPSYM__##name(mrb)
+#define MRB_CVSYM_2(mrb, name) MRB_PRESYM_SCANNING_TAGGED("@@" #name)
+#define MRB_IVSYM_2(mrb, name) MRB_PRESYM_SCANNING_TAGGED("@" #name)
+#define MRB_SYM_B_2(mrb, name) MRB_PRESYM_SCANNING_TAGGED(#name "!")
+#define MRB_SYM_Q_2(mrb, name) MRB_PRESYM_SCANNING_TAGGED(#name "?")
+#define MRB_SYM_E_2(mrb, name) MRB_PRESYM_SCANNING_TAGGED(#name "=")
+#define MRB_SYM_2(mrb, name) MRB_PRESYM_SCANNING_TAGGED(#name)
+
+#define MRB_OPSYM__not(mrb) MRB_PRESYM_SCANNING_TAGGED("!")
+#define MRB_OPSYM__mod(mrb) MRB_PRESYM_SCANNING_TAGGED("%")
+#define MRB_OPSYM__and(mrb) MRB_PRESYM_SCANNING_TAGGED("&")
+#define MRB_OPSYM__mul(mrb) MRB_PRESYM_SCANNING_TAGGED("*")
+#define MRB_OPSYM__add(mrb) MRB_PRESYM_SCANNING_TAGGED("+")
+#define MRB_OPSYM__sub(mrb) MRB_PRESYM_SCANNING_TAGGED("-")
+#define MRB_OPSYM__div(mrb) MRB_PRESYM_SCANNING_TAGGED("/")
+#define MRB_OPSYM__lt(mrb) MRB_PRESYM_SCANNING_TAGGED("<")
+#define MRB_OPSYM__gt(mrb) MRB_PRESYM_SCANNING_TAGGED(">")
+#define MRB_OPSYM__xor(mrb) MRB_PRESYM_SCANNING_TAGGED("^")
+#define MRB_OPSYM__tick(mrb) MRB_PRESYM_SCANNING_TAGGED("`")
+#define MRB_OPSYM__or(mrb) MRB_PRESYM_SCANNING_TAGGED("|")
+#define MRB_OPSYM__neg(mrb) MRB_PRESYM_SCANNING_TAGGED("~")
+#define MRB_OPSYM__neq(mrb) MRB_PRESYM_SCANNING_TAGGED("!=")
+#define MRB_OPSYM__nmatch(mrb) MRB_PRESYM_SCANNING_TAGGED("!~")
+#define MRB_OPSYM__andand(mrb) MRB_PRESYM_SCANNING_TAGGED("&&")
+#define MRB_OPSYM__pow(mrb) MRB_PRESYM_SCANNING_TAGGED("**")
+#define MRB_OPSYM__plus(mrb) MRB_PRESYM_SCANNING_TAGGED("+@")
+#define MRB_OPSYM__minus(mrb) MRB_PRESYM_SCANNING_TAGGED("-@")
+#define MRB_OPSYM__lshift(mrb) MRB_PRESYM_SCANNING_TAGGED("<<")
+#define MRB_OPSYM__le(mrb) MRB_PRESYM_SCANNING_TAGGED("<=")
+#define MRB_OPSYM__eq(mrb) MRB_PRESYM_SCANNING_TAGGED("==")
+#define MRB_OPSYM__match(mrb) MRB_PRESYM_SCANNING_TAGGED("=~")
+#define MRB_OPSYM__ge(mrb) MRB_PRESYM_SCANNING_TAGGED(">=")
+#define MRB_OPSYM__rshift(mrb) MRB_PRESYM_SCANNING_TAGGED(">>")
+#define MRB_OPSYM__aref(mrb) MRB_PRESYM_SCANNING_TAGGED("[]")
+#define MRB_OPSYM__oror(mrb) MRB_PRESYM_SCANNING_TAGGED("||")
+#define MRB_OPSYM__cmp(mrb) MRB_PRESYM_SCANNING_TAGGED("<=>")
+#define MRB_OPSYM__eqq(mrb) MRB_PRESYM_SCANNING_TAGGED("===")
+#define MRB_OPSYM__aset(mrb) MRB_PRESYM_SCANNING_TAGGED("[]=")
+
+#endif  /* MRUBY_PRESYM_SCANNING_H */

lib/mruby/build.rb

@@ -5,6 +5,7 @@ require "mruby/build/command"
 module MRuby
   autoload :Gem, "mruby/gem"
   autoload :Lockfile, "mruby/lockfile"
+  autoload :Presym, "mruby/presym"
 
   class << self
     def targets
@@ -51,6 +52,7 @@ module MRuby
   class Build
     class << self
       attr_accessor :current
+
       def mruby_config_path
         path = ENV['MRUBY_CONFIG'] || ENV['CONFIG']
         if path.nil? || path.empty?
@@ -61,11 +63,16 @@ module MRuby
         end
         path
       end
+
+      def install_dir
+        @install_dir ||= ENV['INSTALL_DIR'] || "#{MRUBY_ROOT}/bin"
+      end
     end
+
     include Rake::DSL
     include LoadGems
     attr_accessor :name, :bins, :exts, :file_separator, :build_dir, :gem_clone_dir
-    attr_reader :libmruby_core_objs, :libmruby_objs, :gems, :toolchains, :gem_dir_to_repo_url
+    attr_reader :products, :libmruby_core_objs, :libmruby_objs, :gems, :toolchains, :presym, :mrbc_build, :gem_dir_to_repo_url
 
     alias libmruby libmruby_objs
 
@@ -73,16 +80,16 @@ module MRuby
     COMMANDS = COMPILERS + %w(linker archiver yacc gperf git exts mrbc)
     attr_block MRuby::Build::COMMANDS
 
-    Exts = Struct.new(:object, :executable, :library)
+    Exts = Struct.new(:object, :executable, :library, :preprocessed)
 
-    def initialize(name='host', build_dir=nil, &block)
+    def initialize(name='host', build_dir=nil, internal: false, &block)
       @name = name.to_s
 
       unless current = MRuby.targets[@name]
         if ENV['OS'] == 'Windows_NT'
-          @exts = Exts.new('.o', '.exe', '.a')
+          @exts = Exts.new('.o', '.exe', '.a', '.i')
         else
-          @exts = Exts.new('.o', '', '.a')
+          @exts = Exts.new('.o', '', '.a', '.i')
         end
 
         build_dir = build_dir || ENV['MRUBY_BUILD_DIR'] || "#{MRUBY_ROOT}/build"
@@ -101,6 +108,7 @@ module MRuby
         @git = Command::Git.new(self)
         @mrbc = Command::Mrbc.new(self)
 
+        @products = []
         @bins = []
         @gems = MRuby::Gem::List.new
         @libmruby_core_objs = []
@@ -113,6 +121,9 @@ module MRuby
         @enable_bintest = false
         @enable_test = false
         @enable_lock = true
+        @enable_presym = true
+        @mrbcfile_external = false
+        @internal = internal
         @toolchains = []
         @gem_dir_to_repo_url = {}
 
@@ -121,7 +132,14 @@ module MRuby
 
       MRuby::Build.current = current
       current.instance_eval(&block)
-      current.build_mrbc_exec if current.libmruby_enabled? && @name == "host"
+      if current.libmruby_enabled? && !current.mrbcfile_external?
+        if current.presym_enabled?
+          current.create_mrbc_build if current.host? || current.gems["mruby-bin-mrbc"]
+        elsif current.host?
+          current.build_mrbc_exec
+        end
+      end
+      current.presym = Presym.new(current) if current.presym_enabled?
       current.build_mrbtest if current.test_enabled?
     end
 
@@ -149,6 +167,17 @@ module MRuby
       @enable_debug = true
     end
 
+    def presym_enabled?
+      @enable_presym
+    end
+
+    def disable_presym
+      if @enable_presym
+        @enable_presym = false
+        compilers.each{|c| c.defines << "MRB_NO_PRESYM"}
+      end
+    end
+
     def disable_lock
       @enable_lock = false
     end
@@ -200,8 +229,29 @@ module MRuby
       @cxx_abi_enabled = true
     end
 
-    def compile_as_cxx src, cxx_src, obj = nil, includes = []
-      obj = objfile(cxx_src) if obj.nil?
+    def compile_as_cxx(src, cxx_src = nil, obj = nil, includes = [])
+      #
+      # If `cxx_src` is specified, this method behaves the same as before as
+      # compatibility mode, but `.d` file is not read.
+      #
+      # If `cxx_src` is omitted, `.d` file is read by using mruby standard
+      # Rake rule (C++ source name is also changed).
+      #
+      if cxx_src
+        obj ||= cxx_src + @exts.object
+        dsts = [obj]
+        dsts << (cxx_src + @exts.preprocessed) if presym_enabled?
+        defines = []
+        include_paths = ["#{MRUBY_ROOT}/src", *includes]
+        dsts.each do |dst|
+          file dst => cxx_src do |t|
+            cxx.run t.name, t.prerequisites.first, defines, include_paths
+          end
+        end
+      else
+        cxx_src = "#{build_dir}/#{src.relative_path})".ext << "-cxx.cxx"
+        obj = cxx_src.ext(@exts.object)
+      end
 
       file cxx_src => [src, __FILE__] do |t|
         mkdir_p File.dirname t.name
@@ -219,10 +269,6 @@ extern "C" {
 EOS
       end
 
-      file obj => cxx_src do |t|
-        cxx.run t.name, t.prerequisites.first, [], ["#{MRUBY_ROOT}/src"] + includes
-      end
-
       obj
     end
 
@@ -263,11 +309,11 @@ EOS
     end
 
     def build_mrbtest
-      gem :core => 'mruby-test'
+      gem :core => 'mruby-test' unless @gems['mruby-test']
     end
 
     def build_mrbc_exec
-      gem :core => 'mruby-bin-mrbc'
+      gem :core => 'mruby-bin-mrbc' unless @gems['mruby-bin-mrbc']
     end
 
     def locks
@@ -278,10 +324,23 @@ EOS
       return @mrbcfile if @mrbcfile
 
       gem_name = "mruby-bin-mrbc"
-      gem = gems[gem_name] || MRuby.targets["host"].gems[gem_name]
+      gem = @gems[gem_name]
+      gem ||= (host = MRuby.targets["host"]) && host.gems[gem_name]
+      unless gem
+        fail "external mrbc or mruby-bin-mrbc gem in current('#{@name}') or 'host' build is required"
+      end
       @mrbcfile = exefile("#{gem.build.build_dir}/bin/mrbc")
     end
 
+    def mrbcfile=(path)
+      @mrbcfile = path
+      @mrbcfile_external = true
+    end
+
+    def mrbcfile_external?
+      @mrbcfile_external
+    end
+
     def compilers
       COMPILERS.map do |c|
         instance_variable_get("@#{c}")
@@ -289,7 +348,7 @@ EOS
     end
 
     def define_rules
-      use_mrdb = @gems.find{|g| g.name == "mruby-bin-debugger"}
+      use_mrdb = @gems["mruby-bin-debugger"]
       compilers.each do |compiler|
         if respond_to?(:enable_gems?) && enable_gems?
           compiler.defines -= %w(MRB_NO_GEMS)
@@ -298,7 +357,10 @@ EOS
         end
         compiler.defines |= %w(MRB_USE_DEBUG_HOOK) if use_mrdb
       end
-      cc.define_rules(build_dir, MRUBY_ROOT)
+      [@cc, *(@cxx if cxx_exception_enabled?)].each do |compiler|
+        compiler.define_rules(@build_dir, MRUBY_ROOT, @exts.object)
+        compiler.define_rules(@build_dir, MRUBY_ROOT, @exts.preprocessed) if presym_enabled?
+      end
     end
 
     def filename(name)
@@ -359,7 +421,8 @@ EOS
       puts ">>> Bintest #{name} <<<"
       targets = @gems.select { |v| File.directory? "#{v.dir}/bintest" }.map { |v| filename v.dir }
       targets << filename(".") if File.directory? "./bintest"
-      env = {"BUILD_DIR" => @build_dir}
+      mrbc = @gems["mruby-bin-mrbc"] ? exefile("#{@build_dir}/bin/mrbc") : mrbcfile
+      env = {"BUILD_DIR" => @build_dir, "MRBCFILE" => mrbc}
       sh env, "ruby test/bintest.rb#{verbose_flag} #{targets.join ' '}"
     end
 
@@ -393,6 +456,41 @@ EOS
     def libraries
       [libmruby_static]
     end
+
+    def host?
+      @name == "host"
+    end
+
+    def internal?
+      @internal
+    end
+
+    protected
+
+    attr_writer :presym
+
+    def create_mrbc_build
+      exclusions = %i[@name @build_dir @gems @enable_test @enable_bintest @internal]
+      name = "#{@name}/mrbc"
+      MRuby.targets.delete(name)
+      build = self.class.new(name, internal: true){}
+      instance_variables.each do |n|
+        next if exclusions.include?(n)
+        v = instance_variable_get(n)
+        v = case v
+            when nil, true, false, Numeric; v
+            when String, Command; v.clone
+            else Marshal.load(Marshal.dump(v))  # deep clone
+            end
+        build.instance_variable_set(n, v)
+      end
+      build.build_mrbc_exec
+      build.disable_libmruby
+      build.disable_presym
+      @mrbc_build = build
+      self.mrbcfile = build.mrbcfile
+      build
+    end
   end # Build
 
   class CrossBuild < Build
@@ -405,7 +503,7 @@ EOS
     def initialize(name, build_dir=nil, &block)
       @test_runner = Command::CrossTestRunner.new(self)
       super
-      unless MRuby.targets['host']
+      unless mrbcfile_external? || MRuby.targets['host']
         # add minimal 'host'
         MRuby::Build.new('host') do |conf|
           if ENV['VisualStudioVersion'] || ENV['VSINSTALLDIR']
@@ -413,16 +511,13 @@ EOS
           else
             toolchain :gcc
           end
-          conf.gem :core => 'mruby-bin-mrbc'
+          conf.build_mrbc_exec
           conf.disable_libmruby
+          conf.disable_presym
         end
       end
     end
 
-    def mrbcfile
-      MRuby.targets['host'].mrbcfile
-    end
-
     def run_test
       @test_runner.runner_options << verbose_flag
       mrbtest = exefile("#{build_dir}/bin/mrbtest")
@@ -433,5 +528,9 @@ EOS
         @test_runner.run(mrbtest)
       end
     end
+
+    protected
+
+    def create_mrbc_build; end
   end # CrossBuild
 end # MRuby

lib/mruby/build/command.rb

@@ -42,6 +42,7 @@ module MRuby
     attr_accessor :label, :flags, :include_paths, :defines, :source_exts
     attr_accessor :compile_options, :option_define, :option_include_path, :out_ext
     attr_accessor :cxx_compile_flag, :cxx_exception_flag, :cxx_invalid_flags
+    attr_writer :preprocess_options
 
     def initialize(build, source_exts=[], label: "CC")
       super(build)
@@ -55,9 +56,15 @@ module MRuby
       @option_define = %q[-D"%s"]
       @compile_options = %q[%{flags} -o "%{outfile}" -c "%{infile}"]
       @cxx_invalid_flags = []
+      @out_ext = build.exts.object
     end
 
     alias header_search_paths include_paths
+
+    def preprocess_options
+      @preprocess_options ||= @compile_options.sub(/(?:\A|\s)\K-c(?=\s)/, "-E -P")
+    end
+
     def search_header_path(name)
       header_search_paths.find do |v|
         File.exist? build.filename("#{v}/#{name}").sub(/^"(.*)"$/, '\1')
@@ -79,13 +86,20 @@ module MRuby
 
     def run(outfile, infile, _defines=[], _include_paths=[], _flags=[])
       mkdir_p File.dirname(outfile)
-      _pp @label, infile.relative_path, outfile.relative_path
-      _run compile_options, { :flags => all_flags(_defines, _include_paths, _flags),
-                              :infile => filename(infile), :outfile => filename(outfile) }
+      flags = all_flags(_defines, _include_paths, _flags)
+      if File.extname(outfile) == build.exts.object
+        label = @label
+        opts = compile_options
+      else
+        label = "CPP"
+        opts = preprocess_options
+        flags << " -DMRB_PRESYM_SCANNING"
+      end
+      _pp label, infile.relative_path, outfile.relative_path
+      _run opts, flags: flags, infile: filename(infile), outfile: filename(outfile)
     end
 
-    def define_rules(build_dir, source_dir='')
-      @out_ext = build.exts.object
+    def define_rules(build_dir, source_dir='', out_ext=build.exts.object)
       gemrake = File.join(source_dir, "mrbgem.rake")
       rakedep = File.exist?(gemrake) ? [ gemrake ] : []
 
@@ -143,10 +157,10 @@ module MRuby
     #   /src/value_array.h:
     #
     def get_dependencies(file)
-      file = file.ext('d') unless File.extname(file) == '.d'
-      return [MRUBY_CONFIG] unless File.exist?(file)
+      dep_file = "#{file}.d"
+      return [MRUBY_CONFIG] unless File.exist?(dep_file)
 
-      deps = File.read(file).gsub("\\\n ", "").split("\n").map do |dep_line|
+      deps = File.read(dep_file).gsub("\\\n ", "").split("\n").map do |dep_line|
         # dep_line:
         # - "/build/host/src/array.o:   /src/array.c   /include/mruby/common.h ..."
         # - ""
@@ -187,6 +201,10 @@ module MRuby
       [libraries, _libraries].flatten.map{ |d| option_library % d }.join(' ')
     end
 
+    def run_attrs
+      [@libraries, @library_paths, @flags, @flags_before_libraries, @flags_after_libraries]
+    end
+
     def run(outfile, objfiles, _libraries=[], _library_paths=[], _flags=[], _flags_before_libraries=[], _flags_after_libraries=[])
       mkdir_p File.dirname(outfile)
       library_flags = [libraries, _libraries].flatten.map { |d| option_library % d }

lib/mruby/gem.rb

@@ -7,7 +7,6 @@ module MRuby
     class << self
       attr_accessor :current
     end
-    LinkerConfig = Struct.new(:libraries, :library_paths, :flags, :flags_before_libraries, :flags_after_libraries)
 
     class Specification
       include Rake::DSL
@@ -50,13 +49,13 @@ module MRuby
       end
 
       def setup
-        return if defined?(@linker)  # return if already set up
+        return if defined?(@bins)  # return if already set up
 
         MRuby::Gem.current = self
         MRuby::Build::COMMANDS.each do |command|
           instance_variable_set("@#{command}", @build.send(command).clone)
         end
-        @linker = LinkerConfig.new([], [], [], [], [])
+        @linker.run_attrs.each(&:clear)
 
         @rbfiles = Dir.glob("#{@dir}/mrblib/**/*.rb").sort
         @objs = Dir.glob("#{@dir}/src/*.{c,cpp,cxx,cc,m,asm,s,S}").map do |f|
@@ -72,6 +71,7 @@ module MRuby
         @test_args = {}
 
         @bins = []
+        @cdump = true
 
         @requirements = []
         @export_include_paths = []
@@ -81,7 +81,6 @@ module MRuby
 
         @generate_functions = !(@rbfiles.empty? && @objs.empty?)
         @objs << objfile("#{build_dir}/gem_init") if @generate_functions
-        @cdump = core?  # by default core gems use cdump and others use mrb dump
 
         if !name || !licenses || !authors
           fail "#{name || dir} required to set name, license(s) and author(s)"
@@ -96,10 +95,11 @@ module MRuby
       end
 
       def setup_compilers
-        (core? ? [cc] : compilers).each do |compiler|
-          compiler.define_rules build_dir, "#{dir}"
+        (core? ? [@cc, *(@cxx if build.cxx_exception_enabled?)] : compilers).each do |compiler|
+          compiler.define_rules build_dir, @dir, @build.exts.preprocessed if build.presym_enabled?
+          compiler.define_rules build_dir, @dir, @build.exts.object
           compiler.defines << %Q[MRBGEM_#{funcname.upcase}_VERSION=#{version}]
-          compiler.include_paths << "#{dir}/include" if File.directory? "#{dir}/include"
+          compiler.include_paths << "#{@dir}/include" if File.directory? "#{@dir}/include"
         end
 
         define_gem_init_builder if @generate_functions
@@ -114,12 +114,12 @@ module MRuby
         return false
       end
 
-      def enable_cdump
-        @cdump = true
+      def disable_cdump
+        @cdump = false
       end
 
       def cdump?
-        @cdump
+        build.presym_enabled? && @cdump
       end
 
       def core?
@@ -175,7 +175,6 @@ module MRuby
       end
 
       def define_gem_init_builder
-        file objfile("#{build_dir}/gem_init") => [ "#{build_dir}/gem_init.c", File.join(dir, "mrbgem.rake") ]
         file "#{build_dir}/gem_init.c" => [build.mrbcfile, __FILE__] + [rbfiles].flatten do |t|
           mkdir_p build_dir
           generate_gem_init("#{build_dir}/gem_init.c")
@@ -187,7 +186,7 @@ module MRuby
         open(fname, 'w') do |f|
           print_gem_init_header f
           unless rbfiles.empty?
-            if @cdump
+            if cdump?
               build.mrbc.run f, rbfiles, "gem_mrblib_#{funcname}_proc"
             else
               build.mrbc.run f, rbfiles, "gem_mrblib_irep_#{funcname}", false
@@ -198,10 +197,10 @@ module MRuby
           f.puts %Q[]
           f.puts %Q[void GENERATED_TMP_mrb_#{funcname}_gem_init(mrb_state *mrb) {]
           f.puts %Q[  int ai = mrb_gc_arena_save(mrb);]
-          f.puts %Q[  struct REnv *e;] unless rbfiles.empty?
+          f.puts %Q[  gem_mrblib_#{funcname}_proc_init_syms(mrb);] if !rbfiles.empty? && cdump?
           f.puts %Q[  mrb_#{funcname}_gem_init(mrb);] if objs != [objfile("#{build_dir}/gem_init")]
           unless rbfiles.empty?
-            if @cdump
+            if cdump?
               f.puts %Q[  mrb_load_proc(mrb, gem_mrblib_#{funcname}_proc);]
             else
               f.puts %Q[  mrb_load_irep(mrb, gem_mrblib_irep_#{funcname});]
@@ -211,7 +210,7 @@ module MRuby
             f.puts %Q[    mrb_close(mrb);]
             f.puts %Q[    exit(EXIT_FAILURE);]
             f.puts %Q[  }]
-            f.puts %Q[  e = mrb->c->cibase->env;]
+            f.puts %Q[  struct REnv *e = mrb->c->cibase->env;]
             f.puts %Q[  mrb->c->cibase->env = NULL;]
             f.puts %Q[  mrb_env_unshare(mrb, e);]
           end
@@ -241,8 +240,6 @@ module MRuby
           f.puts %Q[#include <mruby.h>]
         else
           f.puts %Q[#include <stdlib.h>]
-          f.puts %Q[#include <mruby.h>]
-          f.puts %Q[#include <mruby/proc.h>]
         end
       end
 
@@ -251,7 +248,7 @@ module MRuby
         f.puts %Q[#include <stdio.h>]
         f.puts %Q[#include <stdlib.h>]
         f.puts %Q[#include <mruby.h>]
-        f.puts %Q[#include <mruby/proc.h>]
+        f.puts %Q[#include <mruby/irep.h>]
         f.puts %Q[#include <mruby/variable.h>]
         f.puts %Q[#include <mruby/hash.h>] unless test_args.empty?
       end

lib/mruby/presym.rb

+module MRuby
+  class Presym
+    include Rake::DSL
+
+    OPERATORS = {
+      "!" => "not",
+      "%" => "mod",
+      "&" => "and",
+      "*" => "mul",
+      "+" => "add",
+      "-" => "sub",
+      "/" => "div",
+      "<" => "lt",
+      ">" => "gt",
+      "^" => "xor",
+      "`" => "tick",
+      "|" => "or",
+      "~" => "neg",
+      "!=" => "neq",
+      "!~" => "nmatch",
+      "&&" => "andand",
+      "**" => "pow",
+      "+@" => "plus",
+      "-@" => "minus",
+      "<<" => "lshift",
+      "<=" => "le",
+      "==" => "eq",
+      "=~" => "match",
+      ">=" => "ge",
+      ">>" => "rshift",
+      "[]" => "aref",
+      "||" => "oror",
+      "<=>" => "cmp",
+      "===" => "eqq",
+      "[]=" => "aset",
+    }.freeze
+
+    SYMBOL_TO_MACRO = {
+    #      Symbol      =>      Macro
+    # [prefix, suffix] => [prefix, suffix]
+      ["@@"  , ""    ] => ["CV"  , ""    ],
+      ["@"   , ""    ] => ["IV"  , ""    ],
+      [""    , "!"   ] => [""    , "_B"  ],
+      [""    , "?"   ] => [""    , "_Q"  ],
+      [""    , "="   ] => [""    , "_E"  ],
+      [""    , ""    ] => [""    , ""    ],
+    }.freeze
+
+    C_STR_LITERAL_RE = /"(?:[^\\\"]|\\.)*"/
+
+    def initialize(build)
+      @build = build
+    end
+
+    def scan(paths)
+      presym_hash = {}
+      paths.each {|path| read_preprocessed(presym_hash, path)}
+      presym_hash.keys.sort_by!{|sym| [c_literal_size(sym), sym]}
+    end
+
+    def read_list
+      File.readlines(list_path, mode: "r:binary").each(&:chomp!)
+    end
+
+    def write_list(presyms)
+      _pp "GEN", list_path.relative_path
+      File.binwrite(list_path, presyms.join("\n") << "\n")
+    end
+
+    def write_header(presyms)
+      prefix_re = Regexp.union(*SYMBOL_TO_MACRO.keys.map(&:first).uniq)
+      suffix_re = Regexp.union(*SYMBOL_TO_MACRO.keys.map(&:last).uniq)
+      sym_re = /\A(#{prefix_re})?([\w&&\D]\w*)(#{suffix_re})?\z/o
+      _pp "GEN", header_path.relative_path
+      mkdir_p(File.dirname(header_path))
+      File.open(header_path, "w:binary") do |f|
+        f.puts "/* MRB_PRESYM_NAMED(lit, num, type, name) */"
+        f.puts "/* MRB_PRESYM_UNNAMED(lit, num) */"
+        presyms.each.with_index(1) do |sym, num|
+          if sym_re =~ sym && (affixes = SYMBOL_TO_MACRO[[$1, $3]])
+            f.puts %|MRB_PRESYM_NAMED("#{sym}", #{num}, #{affixes * 'SYM'}, #{$2})|
+          elsif name = OPERATORS[sym]
+            f.puts %|MRB_PRESYM_NAMED("#{sym}", #{num}, OPSYM, #{name})|
+          elsif
+            f.puts %|MRB_PRESYM_UNNAMED("#{sym}", #{num})|
+          end
+        end
+        f.puts "#define MRB_PRESYM_MAX #{presyms.size}"
+      end
+    end
+
+    def list_path
+      @list_pat ||= "#{@build.build_dir}/presym".freeze
+    end
+
+    def header_path
+      @header_path ||= "#{@build.build_dir}/include/mruby/presym.inc".freeze
+    end
+
+    private
+
+    def read_preprocessed(presym_hash, path)
+      File.binread(path).scan(/<@! (.*?) !@>/) do |part,|
+        literals = part.scan(C_STR_LITERAL_RE)
+        presym_hash[literals.map{|l| l[1..-2]}.join] = true unless literals.empty?
+      end
+    end
+
+    def c_literal_size(literal_without_quote)
+      literal_without_quote.size  # TODO: consider escape sequence
+    end
+  end
+end

mrbgems/mruby-bin-mruby/mrbgem.rake

@@ -7,6 +7,6 @@ MRuby::Gem::Specification.new('mruby-bin-mruby') do |spec|
   spec.add_test_dependency('mruby-print', :core => 'mruby-print')
 
   if build.cxx_exception_enabled?
-    build.compile_as_cxx("#{spec.dir}/tools/mruby/mruby.c", "#{spec.build_dir}/tools/mruby/mruby.cxx")
+    build.compile_as_cxx("#{spec.dir}/tools/mruby/mruby.c")
   end
 end

mrbgems/mruby-compiler/core/codegen.c

@@ -819,7 +819,7 @@ for_body(codegen_scope *s, node *tree)
   genop_2(s, OP_BLOCK, cursp(), s->irep->rlen-1);
   push();pop(); /* space for a block */
   pop();
-  idx = new_sym(s, MRB_SYM(each));
+  idx = new_sym(s, MRB_SYM_2(s->mrb, each));
   genop_3(s, OP_SENDB, cursp(), idx, 0);
 }
 
@@ -1547,14 +1547,14 @@ codegen(codegen_scope *s, node *tree, int val)
               gen_move(s, cursp(), exc, 0);
               push_n(2); pop_n(2); /* space for one arg and a block */
               pop();
-              genop_3(s, OP_SEND, cursp(), new_sym(s, MRB_SYM(__case_eqq)), 1);
+              genop_3(s, OP_SEND, cursp(), new_sym(s, MRB_SYM_2(s->mrb, __case_eqq)), 1);
             }
             else {
               if (n4) {
                 codegen(s, n4->car, VAL);
               }
               else {
-                genop_2(s, OP_GETCONST, cursp(), new_sym(s, MRB_SYM(StandardError)));
+                genop_2(s, OP_GETCONST, cursp(), new_sym(s, MRB_SYM_2(s->mrb, StandardError)));
                 push();
               }
               pop();
@@ -1668,7 +1668,7 @@ codegen(codegen_scope *s, node *tree, int val)
         {
           node *n = tree->car->cdr;
           mrb_sym mid = nsym(n->cdr->car);
-          mrb_sym mnil = MRB_SYM_Q(nil);
+          mrb_sym mnil = MRB_SYM_Q_2(s->mrb, nil);
           if (mid == mnil && n->cdr->cdr->car == NULL) {
             nil_p = TRUE;
             codegen(s, n->car, VAL);
@@ -1804,10 +1804,10 @@ codegen(codegen_scope *s, node *tree, int val)
             gen_move(s, cursp(), head, 0);
             push(); push(); pop(); pop(); pop();
             if (nint(n->car->car) == NODE_SPLAT) {
-              genop_3(s, OP_SEND, cursp(), new_sym(s, MRB_SYM(__case_eqq)), 1);
+              genop_3(s, OP_SEND, cursp(), new_sym(s, MRB_SYM_2(s->mrb, __case_eqq)), 1);
             }
             else {
-              genop_3(s, OP_SEND, cursp(), new_sym(s, MRB_OPSYM(eqq)), 1);
+              genop_3(s, OP_SEND, cursp(), new_sym(s, MRB_OPSYM_2(s->mrb, eqq)), 1);
             }
           }
           else {
@@ -2333,7 +2333,7 @@ codegen(codegen_scope *s, node *tree, int val)
       pop_n(n+1);
       genop_2S(s, OP_BLKPUSH, cursp(), (ainfo<<4)|(lv & 0xf));
       if (sendv) n = CALL_MAXARGS;
-      genop_3(s, OP_SEND, cursp(), new_sym(s, MRB_SYM(call)), n);
+      genop_3(s, OP_SEND, cursp(), new_sym(s, MRB_SYM_2(s->mrb, call)), n);
       if (val) push();
     }
     break;
@@ -2582,7 +2582,7 @@ codegen(codegen_scope *s, node *tree, int val)
 
       default:
         if (val) {
-          int sym = new_sym(s, MRB_OPSYM(minus));
+          int sym = new_sym(s, MRB_OPSYM_2(s->mrb, minus));
           codegen(s, tree, VAL);
           pop();
           genop_3(s, OP_SEND, cursp(), sym, 0);
@@ -2655,7 +2655,7 @@ codegen(codegen_scope *s, node *tree, int val)
     {
       node *n;
       int ai = mrb_gc_arena_save(s->mrb);
-      int sym = new_sym(s, MRB_SYM(Kernel));
+      int sym = new_sym(s, MRB_SYM_2(s->mrb, Kernel));
 
       genop_1(s, OP_LOADSELF, cursp());
       push();
@@ -2674,7 +2674,7 @@ codegen(codegen_scope *s, node *tree, int val)
       }
       push();                   /* for block */
       pop_n(3);
-      sym = new_sym(s, MRB_OPSYM(tick)); /* ` */
+      sym = new_sym(s, MRB_OPSYM_2(s->mrb, tick)); /* ` */
       genop_3(s, OP_SEND, cursp(), sym, 1);
       if (val) push();
       mrb_gc_arena_restore(s->mrb, ai);
@@ -2694,7 +2694,7 @@ codegen(codegen_scope *s, node *tree, int val)
       genop_bs(s, OP_STRING, cursp(), off);
       push(); push();
       pop_n(3);
-      sym = new_sym(s, MRB_OPSYM(tick)); /* ` */
+      sym = new_sym(s, MRB_OPSYM_2(s->mrb, tick)); /* ` */
       genop_3(s, OP_SEND, cursp(), sym, 1);
       if (val) push();
       mrb_gc_arena_restore(s->mrb, ai);
@@ -2735,7 +2735,7 @@ codegen(codegen_scope *s, node *tree, int val)
       }
       push(); /* space for a block */
       pop_n(argc+2);
-      sym = new_sym(s, MRB_SYM(compile));
+      sym = new_sym(s, MRB_SYM_2(s->mrb, compile));
       genop_3(s, OP_SEND, cursp(), sym, argc);
       mrb_gc_arena_restore(s->mrb, ai);
       push();
@@ -2789,7 +2789,7 @@ codegen(codegen_scope *s, node *tree, int val)
       }
       push(); /* space for a block */
       pop_n(argc+2);
-      sym = new_sym(s, MRB_SYM(compile));
+      sym = new_sym(s, MRB_SYM_2(s->mrb, compile));
       genop_3(s, OP_SEND, cursp(), sym, argc);
       mrb_gc_arena_restore(s->mrb, ai);
       push();

mrbgems/mruby-compiler/mrbgem.rake

@@ -5,13 +5,13 @@ MRuby::Gem::Specification.new 'mruby-compiler' do |spec|
 
   as_cxx_srcs = %w[codegen y.tab].map{|name| "#{dir}/core/#{name}.c"}
   objs = Dir.glob("#{dir}/core/*.c").map do |src|
-    dst = src.pathmap("#{build_dir}/core/%n")
     if build.cxx_exception_enabled? && as_cxx_srcs.include?(src)
-      build.compile_as_cxx(src, "#{dst}.cxx")
+      build.compile_as_cxx(src)
     else
-      objfile(dst)
+      objfile(src.pathmap("#{build_dir}/core/%n"))
     end
   end
+  objs << objfile("#{build_dir}/core/y.tab")
   build.libmruby_core_objs << objs
 
   lex_def = "#{dir}/core/lex.def"

mrbgems/mruby-error/mrbgem.rake

@@ -4,7 +4,7 @@ MRuby::Gem::Specification.new('mruby-error') do |spec|
   spec.summary = 'extensional error handling'
 
   if build.cxx_exception_enabled?
-    @objs << build.compile_as_cxx("#{spec.dir}/src/exception.c", "#{spec.build_dir}/src/exception.cxx")
-    @objs.delete_if { |v| v == objfile("#{spec.build_dir}/src/exception") }
+    objs << build.compile_as_cxx("#{spec.dir}/src/exception.c")
+    objs.delete_if { |v| v == objfile("#{spec.build_dir}/src/exception") }
   end
 end

src/class.c

@@ -2792,7 +2792,8 @@ static const mrb_code new_iseq[] = {
   OP_RETURN, 0x0             /* OP_RETURN    R0 */
 };
 
-const mrb_sym new_syms[] = { MRB_SYM(allocate), MRB_SYM(initialize) };
+MRB_PRESYM_DEFINE_VAR_AND_INITER(new_syms, 2, MRB_SYM(allocate), MRB_SYM(initialize))
+
 static const mrb_irep new_irep = {
   3, 6, 0, MRB_IREP_STATIC,
   new_iseq, NULL, new_syms, NULL, NULL, NULL,
@@ -2805,6 +2806,7 @@ init_class_new(mrb_state *mrb, struct RClass *cls)
   struct RProc *p;
   mrb_method_t m;
 
+  init_new_syms(mrb);
   p = mrb_proc_new(mrb, &new_irep);
   MRB_METHOD_FROM_PROC(m, p);
   mrb_define_method_raw(mrb, cls, MRB_SYM(new), m);

src/debug.c

@@ -66,7 +66,7 @@ mrb_debug_get_filename(mrb_state *mrb, const mrb_irep *irep, uint32_t pc)
 MRB_API int32_t
 mrb_debug_get_line(mrb_state *mrb, const mrb_irep *irep, uint32_t pc)
 {
-  if (irep && pc < irep->ilen) {
+  if (irep && pc >= 0 && pc < irep->ilen) {
     mrb_irep_debug_info_file* f = NULL;
     if (!irep->debug_info) {
       return -1;

src/dump.c

@@ -10,7 +10,6 @@
 #include <mruby/dump.h>
 #include <mruby/string.h>
 #include <mruby/irep.h>
-#include <mruby/numeric.h>
 #include <mruby/debug.h>
 
 #ifndef MRB_NO_FLOAT
@@ -24,6 +23,45 @@ static size_t get_irep_record_size_1(mrb_state *mrb, const mrb_irep *irep);
 # error This code cannot be built on your environment.
 #endif
 
+#define OPERATOR_SYMBOL(sym_name, name) {name, sym_name, sizeof(sym_name)-1}
+struct operator_symbol {
+  const char *name;
+  const char *sym_name;
+  uint16_t sym_name_len;
+};
+static const struct operator_symbol operator_table[] = {
+  OPERATOR_SYMBOL("!", "not"),
+  OPERATOR_SYMBOL("%", "mod"),
+  OPERATOR_SYMBOL("&", "and"),
+  OPERATOR_SYMBOL("*", "mul"),
+  OPERATOR_SYMBOL("+", "add"),
+  OPERATOR_SYMBOL("-", "sub"),
+  OPERATOR_SYMBOL("/", "div"),
+  OPERATOR_SYMBOL("<", "lt"),
+  OPERATOR_SYMBOL(">", "gt"),
+  OPERATOR_SYMBOL("^", "xor"),
+  OPERATOR_SYMBOL("`", "tick"),
+  OPERATOR_SYMBOL("|", "or"),
+  OPERATOR_SYMBOL("~", "neg"),
+  OPERATOR_SYMBOL("!=", "neq"),
+  OPERATOR_SYMBOL("!~", "nmatch"),
+  OPERATOR_SYMBOL("&&", "andand"),
+  OPERATOR_SYMBOL("**", "pow"),
+  OPERATOR_SYMBOL("+@", "plus"),
+  OPERATOR_SYMBOL("-@", "minus"),
+  OPERATOR_SYMBOL("<<", "lshift"),
+  OPERATOR_SYMBOL("<=", "le"),
+  OPERATOR_SYMBOL("==", "eq"),
+  OPERATOR_SYMBOL("=~", "match"),
+  OPERATOR_SYMBOL(">=", "ge"),
+  OPERATOR_SYMBOL(">>", "rshift"),
+  OPERATOR_SYMBOL("[]", "aref"),
+  OPERATOR_SYMBOL("||", "oror"),
+  OPERATOR_SYMBOL("<=>", "cmp"),
+  OPERATOR_SYMBOL("===", "eqq"),
+  OPERATOR_SYMBOL("[]=", "aset"),
+};
+
 static size_t
 get_irep_header_size(mrb_state *mrb)
 {
@@ -888,6 +926,8 @@ mrb_dump_irep_cfunc(mrb_state *mrb, const mrb_irep *irep, uint8_t flags, FILE *f
       return MRB_DUMP_WRITE_FAULT;
     }
     if (fprintf(fp,
+          "#include <mruby.h>\n"
+          "#include <mruby/proc.h>\n"
           "#ifdef __cplusplus\n"
           "extern const uint8_t %s[];\n"
           "#endif\n"
@@ -960,26 +1000,132 @@ dump_pool(mrb_state *mrb, const mrb_pool_value *p, FILE *fp)
   return MRB_DUMP_OK;
 }
 
-mrb_bool mrb_sym_static_p(mrb_state *mrb, mrb_sym sym);
+static mrb_bool
+sym_name_word_p(const char *name, mrb_int len)
+{
+  if (len == 0) return FALSE;
+  if (name[0] != '_' && !ISALPHA(name[0])) return FALSE;
+  for (int i = 1; i < len; i++) {
+    if (name[i] != '_' && !ISALNUM(name[i])) return FALSE;
+  }
+  return TRUE;
+}
+
+static mrb_bool
+sym_name_with_equal_p(const char *name, mrb_int len)
+{
+  return len >= 2 && name[len-1] == '=' && sym_name_word_p(name, len-1);
+}
+
+static mrb_bool
+sym_name_with_question_mark_p(const char *name, mrb_int len)
+{
+  return len >= 2 && name[len-1] == '?' && sym_name_word_p(name, len-1);
+}
+
+static mrb_bool
+sym_name_with_bang_p(const char *name, mrb_int len)
+{
+  return len >= 2 && name[len-1] == '!' && sym_name_word_p(name, len-1);
+}
+
+static mrb_bool
+sym_name_ivar_p(const char *name, mrb_int len)
+{
+  return len >= 2 && name[0] == '@' && sym_name_word_p(name+1, len-1);
+}
+
+static mrb_bool
+sym_name_cvar_p(const char *name, mrb_int len)
+{
+  return len >= 3 && name[0] == '@' && sym_name_ivar_p(name+1, len-1);
+}
+
+const char *
+sym_operator_p(const char *name, mrb_int len)
+{
+  mrb_sym start, idx;
+  mrb_sym table_size = sizeof(operator_table)/sizeof(struct operator_symbol);
+  int cmp;
+  const struct operator_symbol *op_sym;
+  for (start = 0; table_size != 0; table_size/=2) {
+    idx = start+table_size/2;
+    op_sym = &operator_table[idx];
+    cmp = len-op_sym->sym_name_len;
+    if (cmp == 0) {
+      cmp = memcmp(name, op_sym->sym_name, len);
+      if (cmp == 0) return op_sym->name;
+    }
+    if (0 < cmp) {
+      start = ++idx;
+      --table_size;
+    }
+  }
+  return NULL;
+}
 
 static int
-dump_sym(mrb_state *mrb, mrb_sym sym, FILE *fp)
+dump_sym(mrb_state *mrb, mrb_sym sym, const char *var_name, int idx, mrb_value init_syms_code, FILE *fp, mrb_bool *presymp)
 {
-  const char *name;
   if (sym == 0) return MRB_DUMP_INVALID_ARGUMENT;
-  name = mrb_sym_name(mrb, sym);
-  if (!name) {
-    fprintf(stderr, "undefined symbol (%d) - define presym\n", sym);
+
+  mrb_int len;
+  const char *name = mrb_sym_name_len(mrb, sym, &len), *op_name;
+  if (!name) return MRB_DUMP_INVALID_ARGUMENT;
+  if (presymp) *presymp = TRUE;
+  if (sym_name_word_p(name, len)) {
+    fprintf(fp, "MRB_SYM(%s)", name);
+  }
+  else if (sym_name_with_equal_p(name, len)) {
+    fprintf(fp, "MRB_SYM_E(%.*s)", (int)(len-1), name);
+  }
+  else if (sym_name_with_question_mark_p(name, len)) {
+    fprintf(fp, "MRB_SYM_Q(%.*s)", (int)(len-1), name);
+  }
+  else if (sym_name_with_bang_p(name, len)) {
+    fprintf(fp, "MRB_SYM_B(%.*s)", (int)(len-1), name);
   }
-  if (!mrb_sym_static_p(mrb, sym)) {
-    fprintf(stderr, "no static symbol (%s) - define presym\n", name);
+  else if (sym_name_ivar_p(name, len)) {
+    fprintf(fp, "MRB_IVSYM(%s)", name+1);
   }
-  fprintf(fp, "%d /* %s */,", sym, name);
+  else if (sym_name_cvar_p(name, len)) {
+    fprintf(fp, "MRB_CVSYM(%s)", name+2);
+  }
+  else if ((op_name = sym_operator_p(name, len))) {
+    fprintf(fp, "MRB_OPSYM(%s)", op_name);
+  }
+  else {
+    mrb_assert(var_name);
+    char buf[32];
+    mrb_str_cat_lit(mrb, init_syms_code, "  ");
+    mrb_str_cat_cstr(mrb, init_syms_code, var_name);
+    snprintf(buf, sizeof(buf), "[%d] = ", idx);
+    mrb_str_cat_cstr(mrb, init_syms_code, buf);
+    mrb_str_cat_lit(mrb, init_syms_code, "mrb_intern_lit(mrb, \"");
+    mrb_str_cat_cstr(mrb, init_syms_code, mrb_sym_dump(mrb, sym));
+    mrb_str_cat_lit(mrb, init_syms_code, "\");\n");
+    *presymp = FALSE;
+    fputs("0", fp);
+  }
+  fputs(", ", fp);
   return MRB_DUMP_OK;
 }
 
+static const char*
+sym_var_name(mrb_state *mrb, const char *initname, const char *key, int n)
+{
+  char buf[32];
+  mrb_value s = mrb_str_new_cstr(mrb, initname);
+  mrb_str_cat_lit(mrb, s, "_");
+  mrb_str_cat_cstr(mrb, s, key);
+  mrb_str_cat_lit(mrb, s, "_");
+  snprintf(buf, sizeof(buf), "%d", n);
+  mrb_str_cat_cstr(mrb, s, buf);
+  return RSTRING_PTR(s);
+}
+
 static int
-dump_irep_struct(mrb_state *mrb, const mrb_irep *irep, uint8_t flags, FILE *fp, const char *name, int n, int *mp)
+dump_irep_struct(mrb_state *mrb, const mrb_irep *irep, uint8_t flags, FILE *fp, const char *name, int n, mrb_value init_syms_code, int *mp)
 {
   int i, len;
   int max = *mp;
@@ -988,7 +1134,7 @@ dump_irep_struct(mrb_state *mrb, const mrb_irep *irep, uint8_t flags, FILE *fp,
   if (irep->reps) {
     for (i=0,len=irep->rlen; i<len; i++) {
       *mp += len;
-      if (dump_irep_struct(mrb, irep->reps[i], flags, fp, name, max+i, mp) != MRB_DUMP_OK)
+      if (dump_irep_struct(mrb, irep->reps[i], flags, fp, name, max+i, init_syms_code, mp) != MRB_DUMP_OK)
         return MRB_DUMP_INVALID_ARGUMENT;
     }
     fprintf(fp,   "static const mrb_irep *%s_reps_%d[%d] = {\n", name, n, len);
@@ -1009,12 +1155,19 @@ dump_irep_struct(mrb_state *mrb, const mrb_irep *irep, uint8_t flags, FILE *fp,
   }
   /* dump syms */
   if (irep->syms) {
+    int ai = mrb_gc_arena_save(mrb);
+    const char *var_name = sym_var_name(mrb, name, "syms", n);
+    mrb_bool all_presym = TRUE, presym;
     len=irep->slen;
-    fprintf(fp,   "static const mrb_sym %s_syms_%d[%d] = {", name, n, len);
+    fprintf(fp,   "mrb_DEFINE_SYMS_VAR(%s, %d, (", var_name, len);
     for (i=0; i<len; i++) {
-      dump_sym(mrb, irep->syms[i], fp);
+      dump_sym(mrb, irep->syms[i], var_name, i, init_syms_code, fp, &presym);
+      all_presym &= presym;
     }
-    fputs("};\n", fp);
+    fputs("), ", fp);
+    if (all_presym) fputs("const", fp);
+    fputs(");\n", fp);
+    mrb_gc_arena_restore(mrb, ai);
   }
   /* dump iseq */
   len=irep->ilen+sizeof(struct mrb_irep_catch_handler)*irep->clen;
@@ -1029,7 +1182,7 @@ dump_irep_struct(mrb_state *mrb, const mrb_irep *irep, uint8_t flags, FILE *fp,
     len=irep->nlocals;
     fprintf(fp,   "static const mrb_sym %s_lv_%d[%d] = {", name, n, len-1);
     for (i=0; i+1<len; i++) {
-      fprintf(fp, "%uU, ", irep->lv[i]);
+      dump_sym(mrb, irep->lv[i], NULL, 0, mrb_nil_value(), fp, NULL);
     }
     fputs("};\n", fp);
   }
@@ -1070,20 +1223,28 @@ dump_irep_struct(mrb_state *mrb, const mrb_irep *irep, uint8_t flags, FILE *fp,
 int
 mrb_dump_irep_cstruct(mrb_state *mrb, const mrb_irep *irep, uint8_t flags, FILE *fp, const char *initname)
 {
-  int max = 1;
-  int n;
-
   if (fp == NULL || initname == NULL || initname[0] == '\0') {
     return MRB_DUMP_INVALID_ARGUMENT;
   }
   if (fprintf(fp, "#include <mruby.h>\n" "#include <mruby/proc.h>\n\n") < 0) {
     return MRB_DUMP_WRITE_FAULT;
   }
-  n = dump_irep_struct(mrb, irep, flags, fp, initname, 0, &max);
+  fputs("#define mrb_BRACED(...) {__VA_ARGS__}\n", fp);
+  fputs("#define mrb_DEFINE_SYMS_VAR(name, len, syms, qualifier) \\\n", fp);
+  fputs("  static qualifier mrb_sym name[len] = mrb_BRACED syms\n", fp);
+  fputs("\n", fp);
+  mrb_value init_syms_code = mrb_str_new_capa(mrb, 0);
+  int max = 1;
+  int n = dump_irep_struct(mrb, irep, flags, fp, initname, 0, init_syms_code, &max);
   if (n != MRB_DUMP_OK) return n;
   fprintf(fp, "#ifdef __cplusplus\nextern const struct RProc %s[];\n#endif\n", initname);
   fprintf(fp, "const struct RProc %s[] = {{\n", initname);
   fprintf(fp, "NULL,NULL,MRB_TT_PROC,7,0,{&%s_irep_0},NULL,{NULL},\n}};\n", initname);
+  fputs("static void\n", fp);
+  fprintf(fp, "%s_init_syms(mrb_state *mrb)\n", initname);
+  fputs("{\n", fp);
+  fputs(RSTRING_PTR(init_syms_code), fp);
+  fputs("}\n", fp);
   return MRB_DUMP_OK;
 }
 

src/symbol.c

@@ -12,15 +12,19 @@
 #include <mruby/dump.h>
 #include <mruby/class.h>
 
-#undef MRB_PRESYM_MAX
-#define MRB_PRESYM_NAMED(lit, num, type, name) {lit, sizeof(lit)-1},
-#define MRB_PRESYM_UNNAMED(lit, num) {lit, sizeof(lit)-1},
+#ifndef MRB_NO_PRESYM
+
+# undef MRB_PRESYM_MAX
+# define MRB_PRESYM_NAMED(lit, num, type, name) {lit, sizeof(lit)-1},
+# define MRB_PRESYM_UNNAMED(lit, num) {lit, sizeof(lit)-1},
 
 static const struct {
   const char *name;
   uint16_t len;
 } presym_table[] = {
-#include <../build/presym.inc>
+#ifndef MRB_PRESYM_SCANNING
+# include <mruby/presym.inc>
+#endif
 };
 
 static mrb_sym
@@ -51,6 +55,8 @@ presym_sym2name(mrb_sym sym, mrb_int *lenp)
   return presym_table[sym-1].name;
 }
 
+#endif  /* MRB_NO_PRESYM */
+
 /* ------------------------------------------------------ */
 typedef struct symbol_name {
   mrb_bool lit : 1;
@@ -147,9 +153,11 @@ find_symbol(mrb_state *mrb, const char *name, size_t len, uint8_t *hashp)
   symbol_name *sname;
   uint8_t hash;
 
+#ifndef MRB_NO_PRESYM
   /* presym */
   i = presym_find(name, len);
   if (i > 0) return i<<SYMBOL_SHIFT;
+#endif
 
   /* inline symbol */
   i = sym_inline_pack(name, len);
@@ -306,10 +314,12 @@ sym2name_len(mrb_state *mrb, mrb_sym sym, char *buf, mrb_int *lenp)
   if (SYMBOL_INLINE_P(sym)) return sym_inline_unpack(sym, buf, lenp);
 
   sym >>= SYMBOL_SHIFT;
+#ifndef MRB_NO_PRESYM
   {
     const char *name = presym_sym2name(sym, lenp);
     if (name) return name;
   }
+#endif
   sym -= MRB_PRESYM_MAX;
 
   if (sym == 0 || mrb->symidx < sym) {

tasks/benchmark.rake

@@ -50,7 +50,7 @@ end
 
 
 MRuby.each_target do |target|
-  next if target.name == 'host'
+  next if target.name == 'host' || target.internal?
   mruby_bin = "#{target.build_dir}/bin/mruby"
 
   bm_files.each do |bm_file|

tasks/bin.rake

+install_task = ->(src) do
+  dst = "#{MRuby::Build.install_dir}/#{File.basename(src)}"
+  file dst => src do
+    install_D src, dst
+  end
+  dst
+end
+
+MRuby.each_target do |build|
+  if build.host? && build.mrbc_build && !build.gems["mruby-bin-mrbc"]
+    exe = build.exefile("#{build.mrbc_build.build_dir}/bin/mrbc")
+    build.products << install_task.(exe)
+  end
+
+  build.bins.each do |bin|
+    exe = build.exefile("#{build.build_dir}/bin/#{bin}")
+    build.products << (build.host? ? install_task.(exe) : exe)
+  end
+
+  linker_attrs = build.gems.map{|gem| gem.linker.run_attrs}.transpose
+  build.gems.each do |gem|
+    gem.bins.each do |bin|
+      exe = build.exefile("#{build.build_dir}/bin/#{bin}")
+      objs = Dir["#{gem.dir}/tools/#{bin}/*.{c,cpp,cxx,cc}"].map do |f|
+        build.objfile(f.pathmap("#{gem.build_dir}/tools/#{bin}/%n"))
+      end
+
+      file exe => objs.concat(build.libraries) do |t|
+        build.linker.run t.name, t.prerequisites, *linker_attrs
+      end
+
+      build.products << (build.host? ? install_task.(exe) : exe)
+    end
+  end
+end

tasks/core.rake

@@ -2,11 +2,10 @@ as_cxx_srcs = %w[vm error gc].map{|name| "#{MRUBY_ROOT}/src/#{name}.c"}
 
 MRuby.each_target do
   objs = Dir.glob("#{MRUBY_ROOT}/src/*.c").map do |src|
-    dst = src.pathmap("#{build_dir}/src/%n")
     if cxx_exception_enabled? && as_cxx_srcs.include?(src)
-      compile_as_cxx(src, "#{dst}.cxx")
+      compile_as_cxx(src)
     else
-      objfile(dst)
+      objfile(src.pathmap("#{build_dir}/src/%n"))
     end
   end
   self.libmruby_core_objs << objs

tasks/libmruby.rake

@@ -3,6 +3,8 @@ MRuby.each_target do
     archiver.run t.name, t.prerequisites
   end
 
+  products << libmruby_core_static
+
   next unless libmruby_enabled?
 
   file libmruby_static => libmruby_objs.flatten do |t|
@@ -27,4 +29,6 @@ MRuby.each_target do
       f.puts "MRUBY_LIBMRUBY_PATH = #{libmruby_static}"
     end
   end
+
+  products << libmruby_static
 end

tasks/mrblib.rake

@@ -2,13 +2,18 @@ MRuby.each_target do
   next unless libmruby_enabled?
 
   src = "#{build_dir}/mrblib/mrblib.c"
-  obj = objfile(src.ext)
   rbfiles = Dir["#{MRUBY_ROOT}/mrblib/*.rb"].sort!
 
-  self.libmruby_objs << obj
+  self.libmruby_objs << objfile(src.ext)
 
-  file obj => src
   file src => [mrbcfile, __FILE__, *rbfiles] do |t|
+    if presym_enabled?
+      cdump = true
+      suffix = "proc"
+    else
+      cdump = false
+      suffix = "irep"
+    end
     mkdir_p File.dirname(t.name)
     File.open(t.name, 'w') do |f|
       _pp "GEN", "mrblib/*.rb", "#{t.name.relative_path}"
@@ -19,14 +24,17 @@ MRuby.each_target do
       f.puts %Q[ *   This file was generated!]
       f.puts %Q[ *   All manual changes will get lost.]
       f.puts %Q[ */]
-      mrbc.run f, rbfiles, 'mrblib_proc'
-      f.puts <<INIT_END
-void
-mrb_init_mrblib(mrb_state *mrb)
-{
-  mrb_load_proc(mrb, mrblib_proc);
-}
-INIT_END
+      unless presym_enabled?
+        f.puts %Q[#include <mruby.h>]
+        f.puts %Q[#include <mruby/irep.h>]
+      end
+      mrbc.run f, rbfiles, "mrblib_#{suffix}", cdump
+      f.puts %Q[void]
+      f.puts %Q[mrb_init_mrblib(mrb_state *mrb)]
+      f.puts %Q[{]
+      f.puts %Q[  mrblib_#{suffix}_init_syms(mrb);] if cdump
+      f.puts %Q[  mrb_load_#{suffix}(mrb, mrblib_#{suffix});]
+      f.puts %Q[}]
     end
   end
 end

tasks/presym.rake

+all_prerequisites = ->(task_name, prereqs) do
+  Rake::Task[task_name].prerequisites.each do |prereq_name|
+    next if prereqs[prereq_name]
+    prereqs[prereq_name] = true
+    all_prerequisites.(Rake::Task[prereq_name].name, prereqs)
+  end
+end
+
+MRuby.each_target do |build|
+  gensym_task = task(:gensym)
+  next unless build.presym_enabled?
+
+  presym = build.presym
+
+  include_dir = "#{build.build_dir}/include"
+  build.compilers.each{|c| c.include_paths << include_dir}
+  build.gems.each{|gem| gem.compilers.each{|c| c.include_paths << include_dir}}
+
+  prereqs = {}
+  pps = []
+  mrbtest = "#{build.class.install_dir}/mrbtest"
+  mrbc_build_dir = "#{build.mrbc_build.build_dir}/" if build.mrbc_build
+  build.products.each do |product|
+    all_prerequisites.(product, prereqs) unless product == mrbtest
+  end
+  prereqs.each_key do |prereq|
+    next unless File.extname(prereq) == build.exts.object
+    next if mrbc_build_dir && prereq.start_with?(mrbc_build_dir)
+    pps << prereq.ext(build.exts.preprocessed)
+  end
+
+  file presym.list_path => pps do
+    presyms = presym.scan(pps)
+    current_presyms = presym.read_list if File.exist?(presym.list_path)
+    update = presyms != current_presyms
+    presym.write_list(presyms) if update
+    presym.write_header(presyms) if update || !File.exist?(presym.header_path)
+  end
+
+  gensym_task.enhance([presym.list_path])
+end

tasks/toolchains/gcc.rake

@@ -3,6 +3,7 @@ MRuby::Toolchain.new(:gcc) do |conf, params|
   compiler_flags = %w(-g -O3 -Wall -Wundef)
   c_mandatory_flags = %w(-std=gnu99)
   cxx_invalid_flags = %w(-Werror-implicit-function-declaration)
+  compile_opt = '%{flags} -MMD -MF "%{outfile}.d" -o "%{outfile}" "%{infile}"'
 
   [conf.cc, conf.objc, conf.asm, conf.cxx].each do |compiler|
     if compiler == conf.cxx
@@ -14,7 +15,8 @@ MRuby::Toolchain.new(:gcc) do |conf, params|
     end
     compiler.option_include_path = %q[-I"%s"]
     compiler.option_define = '-D%s'
-    compiler.compile_options = %q[%{flags} -MMD -o "%{outfile}" -c "%{infile}"]
+    compiler.compile_options = "-c #{compile_opt}"
+    compiler.preprocess_options = "-E -P #{compile_opt}"
     compiler.cxx_compile_flag = '-x c++ -std=gnu++03'
     compiler.cxx_exception_flag = '-fexceptions'
     compiler.cxx_invalid_flags = c_mandatory_flags + cxx_invalid_flags

tasks/toolchains/openwrt.rake

 # usage of environmental variables to set the
 # cross compiling toolchain proper
 MRuby::Toolchain.new(:openwrt) do |conf|
-  [conf.cc, conf.objc, conf.asm].each do |cc|
-    cc.command = ENV['TARGET_CC']
-    cc.flags = ENV['TARGET_CFLAGS']
-    cc.include_paths = ["#{MRUBY_ROOT}/include"]
+  [conf.cc, conf.cxx, conf.objc, conf.asm].each do |cc|
+    if cc == conf.cxx
+      cc.command = ENV['TARGET_CXX']
+      cc.flags = ENV['TARGET_CXXFLAGS']
+    else
+      cc.command = ENV['TARGET_CC']
+      cc.flags = ENV['TARGET_CFLAGS']
+    end
     cc.option_include_path = %q[-I"%s"]
     cc.option_define = '-D%s'
-    cc.compile_options = %q[%{flags} -MMD -o "%{outfile}" -c "%{infile}"]
+    cc.compile_options = %q[%{flags} -MMD -MF "%{outfile}.d" -o "%{outfile}" -c "%{infile}"]
   end
 
-  [conf.cxx].each do |cxx|
-    cxx.command = ENV['TARGET_CXX']
-    cxx.flags = ENV['TARGET_CXXFLAGS']
-    cxx.include_paths = ["#{MRUBY_ROOT}/include"]
-    cxx.option_include_path = %q[-I"%s"]
-    cxx.option_define = '-D%s'
-    cxx.compile_options = %q[%{flags} -MMD -o "%{outfile}" -c "%{infile}"]
-   end
-
   conf.linker do |linker|
     linker.command = ENV['TARGET_CC']
     linker.flags = ENV['TARGET_LDFLAGS']

tasks/toolchains/visualcpp.rake

 MRuby::Toolchain.new(:visualcpp) do |conf, _params|
-  conf.cc do |cc|
-    cc.command = ENV['CC'] || 'cl.exe'
-    # C4013: implicit function declaration
-    cc.flags = [ENV['CFLAGS'] || %w(/c /nologo /W3 /we4013 /Zi /MD /O2 /D_CRT_SECURE_NO_WARNINGS)]
-    cc.defines = %w(MRB_STACK_EXTEND_DOUBLING)
-    cc.option_include_path = %q[/I"%s"]
-    cc.option_define = '/D%s'
-    cc.compile_options = %Q[%{flags} /Fo"%{outfile}" "%{infile}"]
-    cc.cxx_compile_flag = '/TP'
-    cc.cxx_exception_flag = '/EHs'
-  end
-
-  conf.cxx do |cxx|
-    cxx.command = ENV['CXX'] || 'cl.exe'
-    cxx.flags = [ENV['CXXFLAGS'] || ENV['CFLAGS'] || %w(/c /nologo /W3 /Zi /MD /O2 /EHs /D_CRT_SECURE_NO_WARNINGS)]
-    cxx.defines = %w(MRB_STACK_EXTEND_DOUBLING)
-    cxx.option_include_path = %q[/I"%s"]
-    cxx.option_define = '/D%s'
-    cxx.compile_options = %Q[%{flags} /Fo"%{outfile}" "%{infile}"]
-    cxx.cxx_compile_flag = '/TP'
-    cxx.cxx_exception_flag = '/EHs'
+  compiler_flags = %w(/nologo /W3 /MD /O2 /D_CRT_SECURE_NO_WARNINGS)
+  [conf.cc, conf.cxx].each do |compiler|
+    if compiler == conf.cc
+      compiler.command = ENV['CC'] || 'cl.exe'
+      # C4013: implicit function declaration
+      compiler.flags = [*(ENV['CFLAGS'] || compiler_flags + %w(/we4013))]
+    else
+      compiler.command = ENV['CXX'] || 'cl.exe'
+      compiler.flags = [*(ENV['CXXFLAGS'] || ENV['CFLAGS'] || compiler_flags + %w(/EHs))]
+    end
+    compiler.defines = %w(MRB_STACK_EXTEND_DOUBLING)
+    compiler.option_include_path = %q[/I"%s"]
+    compiler.option_define = '/D%s'
+    compiler.compile_options = %Q[/Zi /c /Fo"%{outfile}" %{flags} "%{infile}"]
+    compiler.preprocess_options = %Q[/EP %{flags} "%{infile}" > "%{outfile}"]
+    compiler.cxx_compile_flag = '/TP'
+    compiler.cxx_exception_flag = '/EHs'
   end
 
   conf.linker do |linker|

test/bintest.rb

@@ -4,7 +4,8 @@ require 'test/assert.rb'
 GEMNAME = ""
 
 def cmd(s)
-  path = "#{ENV['BUILD_DIR']}/bin/#{s}"
+  path = s == "mrbc" ? ENV['MRBCFILE'] : "#{ENV['BUILD_DIR']}/bin/#{s}"
+  path = path.sub(/\.exe\z/, "")
   if /mswin(?!ce)|mingw|bccwin/ =~ RbConfig::CONFIG['host_os']
     path = "#{path}.exe".tr("/", "\\")
   end