Merge pull request #5194 from shuujii/create-presym-files-for-each-build-target

Create presym files for each build target

Rakefile

@@ -25,9 +25,9 @@ end
 # load custom rules
 load "#{MRUBY_ROOT}/tasks/core.rake"
 load "#{MRUBY_ROOT}/tasks/mrblib.rake"
-
 load "#{MRUBY_ROOT}/tasks/mrbgems.rake"
 load "#{MRUBY_ROOT}/tasks/libmruby.rake"
+load "#{MRUBY_ROOT}/tasks/presym.rake"
 
 load "#{MRUBY_ROOT}/tasks/benchmark.rake"
 
@@ -55,11 +55,8 @@ else
   depfiles = []
 end
 
-cfiles  = Dir.glob("#{MRUBY_ROOT}/src/*.c")
-rbfiles = Dir.glob("#{MRUBY_ROOT}/mrblib/**/*.rb")
-psfiles = []
 MRuby.each_target do |target|
-  gems.map do |gem|
+  gems.each do |gem|
     current_dir = gem.dir.relative_path_from(Dir.pwd)
     relative_from_root = gem.dir.relative_path_from(MRUBY_ROOT)
     current_build_dir = File.expand_path "#{build_dir}/#{relative_from_root}"
@@ -92,142 +89,11 @@ MRuby.each_target do |target|
         depfiles += [ exec ]
       end
     end
-
-    cfiles += Dir.glob(gem.dir+"/{src,core}/*.c")
-    if gem.cdump?
-      rbfiles += Dir.glob(gem.dir+"/mrblib/**/*.rb")
-      psfiles += Dir.glob(gem.dir+"/**/presym")
-    end
-  end
-end
-
-mkdir_p "#{MRUBY_ROOT}/build"
-symbols = []
-psfiles.each do |file|
-  symbols += File.readlines(file).grep_v(/^# /)
-end
-symbols.each{|x| x.chomp!}
-presym_file="#{MRUBY_ROOT}/build/presym"
-presym_inc="#{presym_file}.inc"
-op_table = {
-  "!" => "not",
-  "!=" => "neq",
-  "!~" => "nmatch",
-  "%" => "mod",
-  "&" => "and",
-  "&&" => "andand",
-  "*" => "mul",
-  "**" => "pow",
-  "+" => "add",
-  "+@" => "plus",
-  "-" => "sub",
-  "-@" => "minus",
-  "/" => "div",
-  "<" => "lt",
-  "<=" => "le",
-  "<<" => "lshift",
-  "<=>" => "cmp",
-  "==" => "eq",
-  "===" => "eqq",
-  "=~" => "match",
-  ">" => "gt",
-  ">=" => "ge",
-  ">>" => "rshift",
-  "[]" => "aref",
-  "[]=" => "aset",
-  "^" => "xor",
-  "`" => "tick",
-  "|" => "or",
-  "||" => "oror",
-  "~" => "neg",
-}
-macro_to_symbol = {
-#      Macro               Symbol
-# [prefix, suffix] => [prefix, suffix]
-  ["CV"  , ""    ] => ["@@"  , ""    ],
-  ["IV"  , ""    ] => ["@"   , ""    ],
-  [""    , "_B"  ] => [""    , "!"   ],
-  [""    , "_Q"  ] => [""    , "?"   ],
-  [""    , "_E"  ] => [""    , "="   ],
-  [""    , ""    ] => [""    , ""    ],
-}
-
-file presym_file => cfiles+rbfiles+psfiles+[__FILE__] do
-  prefix_re = Regexp.union(*macro_to_symbol.keys.uniq.map(&:first))
-  suffix_re = Regexp.union(*macro_to_symbol.keys.uniq.map(&:last))
-  macro_re = /MRB_(#{prefix_re})SYM(#{suffix_re})\((\w+)\)/o
-  csymbols = cfiles.map do |f|
-    src = File.read(f)
-    src.gsub!(/\/\/.+(\n|$)/, "\n")
-    [src.scan(/intern_lit\([^\n"]*"([^\n "]*)"/),
-     src.scan(/mrb_define_method\([^\n"]*"([^\n"]*)"/),
-     src.scan(/mrb_define_class_method\([^\n"]*"([^\n"]*)"/),
-     src.scan(/mrb_define_class\([^\n"]*"([^\n"]*)"/),
-     src.scan(/mrb_define_module\([^\n"]*"([^\n"]*)"/),
-     src.scan(/mrb_define_module_function\([^\n"]*"([^\n"]*)"/),
-     src.scan(/mrb_define_const\([^\n"]*"([^\n"]*)"/),
-     src.scan(/mrb_define_global_const\([^\n"]*"([^\n"]*)"/),
-     src.scan(macro_re).map{|prefix, suffix, name|
-       macro_to_symbol[[prefix, suffix]] * name
-     }]
-  end
-  csymbols += File.readlines("#{MRUBY_ROOT}/include/mruby.h").grep(/define E_/).join.scan(/MRB_SYM\((\w+)\)/)
-
-  rbsymbols = rbfiles.map do |f|
-    src = File.read(f)
-    src.force_encoding(Encoding::BINARY)
-    [src.scan(/\bclass +([A-Z]\w*)/),
-     src.scan(/\bmodule +([A-Z]\w*)/),
-     src.scan(/\bdef +(\w+[!?=]?)/),
-     src.scan(/\balias +(\w+[!?]?)/),
-     src.scan(/\b([A-Z]\w*) *=[^=]/),
-     src.scan(/(\$[a-zA-Z_]\w*)/),
-     src.scan(/(\$[$!?0-9]\w*)/),
-     src.scan(/(@@?[a-zA-Z_]\w*)/),
-     src.scan(/[^.]\.([a-zA-Z_]\w*[!?]?)/),
-     src.scan(/\.([a-zA-Z_]\w* *=)/).map{|x|x.map{|s|s.gsub(' ', '')}},
-     src.scan(/\b([a-zA-Z_]\w*):/),
-     src.scan(/:([a-zA-Z_]\w*[!?=]?)/),
-     src.scan(/[\(\[\{ ]:"([^"]+)"/).map{|x|x.map{|s|s.gsub('\#', '#')}},
-     src.scan(/[ \(\[\{]:'([^']+)'/)
-    ]
-  end
-  symbols = (symbols+csymbols+rbsymbols+op_table.keys).flatten.compact.uniq.grep_v(/#/).map{|x| x.gsub("\n", '\n')}.sort_by!{|x| [x.bytesize, x]}
-  presyms = File.readlines(presym_file) rescue []
-  presyms.each{|x| x.chomp!}
-  if presyms != symbols
-    _pp "GEN", presym_file.relative_path
-    File.write(presym_file, symbols.join("\n"))
-    Rake::Task[presym_inc].invoke
-  end
-end
-
-task presym_inc do
-  presyms = File.readlines(presym_file)
-  presyms.each{|x| x.chomp!}
-  symbol_to_macro = macro_to_symbol.invert
-  prefix_re = Regexp.union(*symbol_to_macro.keys.uniq.map(&:first))
-  suffix_re = Regexp.union(*symbol_to_macro.keys.uniq.map(&:last))
-  macro_re = /\A(#{prefix_re})?([\w&&\D]\w*)(#{suffix_re})?\z/o
-  _pp "GEN", presym_inc.relative_path
-  File.open(presym_inc, "w") do |f|
-    f.puts "/* MRB_PRESYM_NAMED(lit, num, type, name) */"
-    f.puts "/* MRB_PRESYM_UNNAMED(lit, num) */"
-    presyms.each.with_index(1) do |sym, num|
-      if macro_re =~ sym && (affixes = symbol_to_macro[[$1, $3]])
-        f.puts %|MRB_PRESYM_NAMED("#{sym}", #{num}, #{affixes * 'SYM'}, #{$2})|
-      elsif name = op_table[sym]
-        f.puts %|MRB_PRESYM_NAMED("#{sym}", #{num}, OPSYM, #{name})|
-      elsif
-        f.puts %|MRB_PRESYM_UNNAMED("#{sym}", #{num})|
-      end
-    end
-    f.print "#define MRB_PRESYM_MAX #{presyms.size}"
   end
 end
 
 desc "preallocated symbols"
-task :gensym => presym_file
+task :gensym => MRuby.targets.values.map(&:presym_file)
 
 depfiles += MRuby.targets.map { |n, t|
   t.libraries
@@ -286,7 +152,5 @@ task :deep_clean => ["clean", "clean_doc"] do
   MRuby.each_target do |t|
     rm_rf t.gem_clone_dir
   end
-  rm_f presym_file
-  rm_f presym_inc
   puts "Cleaned up mrbgems build folder"
 end

doc/guides/compile.md

@@ -348,10 +348,6 @@ root directory. The structure of this directory will look like this:
 
 	+- build
 	   |
-	   +-  presym         <- List of preallocated symbolx
-       |
-	   +-  presym.inc     <- C source file for preallocated symbols
-       |
 	   +-  host
 	       |
 	       +- bin          <- Binaries (mirb, mrbc and mruby)
@@ -403,10 +399,6 @@ In case of a cross-compilation to *i386* the *build* directory structure looks
 like this:
 
 	+- build
-	   |
-	   +-  presym         <- List of preallocated symbolx
-       |
-	   +-  presym.inc     <- C source file for preallocated symbols
 	   |
 	   +-  host
 	   |   |

include/mruby/presym.h

@@ -16,7 +16,7 @@
 #define MRB_PRESYM_UNNAMED(lit, num)
 
 enum mruby_presym {
-#include <../build/presym.inc>
+#include <mruby/presym.inc>
 };
 
 #undef MRB_PRESYM_NAMED

lib/mruby/build.rb

@@ -373,6 +373,14 @@ EOS
     def libraries
       [libmruby_static]
     end
+
+    def presym_file
+      "#{build_dir}/presym"
+    end
+
+    def presym_inc
+      "#{build_dir}/include/mruby/presym.inc"
+    end
   end # Build
 
   class CrossBuild < Build

src/symbol.c

@@ -20,7 +20,7 @@ static const struct {
   const char *name;
   uint16_t len;
 } presym_table[] = {
-#include <../build/presym.inc>
+#include <mruby/presym.inc>
 };
 
 static mrb_sym

tasks/presym.rake

+op_table = {
+  "!" => "not",
+  "!=" => "neq",
+  "!~" => "nmatch",
+  "%" => "mod",
+  "&" => "and",
+  "&&" => "andand",
+  "*" => "mul",
+  "**" => "pow",
+  "+" => "add",
+  "+@" => "plus",
+  "-" => "sub",
+  "-@" => "minus",
+  "/" => "div",
+  "<" => "lt",
+  "<=" => "le",
+  "<<" => "lshift",
+  "<=>" => "cmp",
+  "==" => "eq",
+  "===" => "eqq",
+  "=~" => "match",
+  ">" => "gt",
+  ">=" => "ge",
+  ">>" => "rshift",
+  "[]" => "aref",
+  "[]=" => "aset",
+  "^" => "xor",
+  "`" => "tick",
+  "|" => "or",
+  "||" => "oror",
+  "~" => "neg",
+}
+macro_to_symbol = {
+#      Macro               Symbol
+# [prefix, suffix] => [prefix, suffix]
+  ["CV"  , ""    ] => ["@@"  , ""    ],
+  ["IV"  , ""    ] => ["@"   , ""    ],
+  [""    , "_B"  ] => [""    , "!"   ],
+  [""    , "_Q"  ] => [""    , "?"   ],
+  [""    , "_E"  ] => [""    , "="   ],
+  [""    , ""    ] => [""    , ""    ],
+}
+
+core_cfiles = nil
+core_rbfiles = nil
+MRuby.each_target do |build|
+  core_cfiles ||= Dir.glob("#{MRUBY_ROOT}/src/*.c")
+  core_rbfiles ||= Dir.glob("#{MRUBY_ROOT}/mrblib/*.rb")
+  cfiles = core_cfiles.dup
+  rbfiles = core_rbfiles.dup
+  psfiles = []
+  presym_file = build.presym_file
+  presym_inc = build.presym_inc
+
+  build.compilers.each {|c| c.include_paths << "#{build.build_dir}/include"}
+  build.gems.each do |gem|
+    cfiles.concat(Dir.glob(gem.dir+"/{src,core}/*.c"))
+    if gem.cdump?
+      rbfiles.concat(gem.rbfiles)
+      psfiles.concat(Dir.glob(gem.dir+"/**/presym"))
+      gem.compilers.each {|c| c.include_paths << "#{build.build_dir}/include"}
+    end
+  end
+
+  file presym_file => [*cfiles, *rbfiles, *psfiles, __FILE__] do
+    prefix_re = Regexp.union(*macro_to_symbol.keys.uniq.map(&:first))
+    suffix_re = Regexp.union(*macro_to_symbol.keys.uniq.map(&:last))
+    macro_re = /MRB_(#{prefix_re})SYM(#{suffix_re})\((\w+)\)/o
+    csymbols = cfiles.map do |f|
+      src = File.read(f)
+      src.gsub!(/\/\/.+(\n|$)/, "\n")
+      [src.scan(/intern_lit\([^\n"]*"([^\n "]*)"/),
+        src.scan(/mrb_define_method\([^\n"]*"([^\n"]*)"/),
+        src.scan(/mrb_define_class_method\([^\n"]*"([^\n"]*)"/),
+        src.scan(/mrb_define_class\([^\n"]*"([^\n"]*)"/),
+        src.scan(/mrb_define_module\([^\n"]*"([^\n"]*)"/),
+        src.scan(/mrb_define_module_function\([^\n"]*"([^\n"]*)"/),
+        src.scan(/mrb_define_const\([^\n"]*"([^\n"]*)"/),
+        src.scan(/mrb_define_global_const\([^\n"]*"([^\n"]*)"/),
+        src.scan(macro_re).map{|prefix, suffix, name|
+          macro_to_symbol[[prefix, suffix]] * name
+        }]
+    end
+    csymbols += File.readlines("#{MRUBY_ROOT}/include/mruby.h").grep(/define E_/).join.scan(/MRB_SYM\((\w+)\)/)
+
+    rbsymbols = rbfiles.map do |f|
+      src = File.read(f)
+      src.force_encoding(Encoding::BINARY)
+      [src.scan(/\bclass +([A-Z]\w*)/),
+        src.scan(/\bmodule +([A-Z]\w*)/),
+        src.scan(/\bdef +(\w+[!?=]?)/),
+        src.scan(/\balias +(\w+[!?]?)/),
+        src.scan(/\b([A-Z]\w*) *=[^=]/),
+        src.scan(/(\$[a-zA-Z_]\w*)/),
+        src.scan(/(\$[$!?0-9]\w*)/),
+        src.scan(/(@@?[a-zA-Z_]\w*)/),
+        src.scan(/[^.]\.([a-zA-Z_]\w*[!?]?)/),
+        src.scan(/\.([a-zA-Z_]\w* *=)/).map{|x|x.map{|s|s.gsub(' ', '')}},
+        src.scan(/\b([a-zA-Z_]\w*):/),
+        src.scan(/:([a-zA-Z_]\w*[!?=]?)/),
+        src.scan(/[\(\[\{ ]:"([^"]+)"/).map{|x|x.map{|s|s.gsub('\#', '#')}},
+        src.scan(/[ \(\[\{]:'([^']+)'/)
+      ]
+    end
+    symbols = [csymbols, rbsymbols, op_table.keys, psfiles.map{|file| symbols.concat(File.readlines(file).grep_v(/^# /))}]
+    symbols = symbols.flatten.compact.uniq.grep_v(/#/).map{|x| x.gsub("\n", '\n')}.sort_by!{|x| [x.bytesize, x]}
+    presyms = File.readlines(presym_file) rescue []
+    presyms.each{|x| x.chomp!}
+    if presyms != symbols
+      _pp "GEN", presym_file.relative_path
+      mkdir_p(File.dirname(presym_file))
+      File.write(presym_file, symbols.join("\n"))
+      Rake::Task["gensym:#{presym_inc}"].invoke
+    end
+  end
+
+  task "gensym:#{presym_inc}" do
+    presyms = File.readlines(presym_file)
+    presyms.each{|x| x.chomp!}
+    symbol_to_macro = macro_to_symbol.invert
+    prefix_re = Regexp.union(*symbol_to_macro.keys.uniq.map(&:first))
+    suffix_re = Regexp.union(*symbol_to_macro.keys.uniq.map(&:last))
+    macro_re = /\A(#{prefix_re})?([\w&&\D]\w*)(#{suffix_re})?\z/o
+    mkdir_p(File.dirname(presym_inc))
+    _pp "GEN", presym_inc.relative_path
+    File.open(presym_inc, "w") do |f|
+      f.puts "/* MRB_PRESYM_NAMED(lit, num, type, name) */"
+      f.puts "/* MRB_PRESYM_UNNAMED(lit, num) */"
+      presyms.each.with_index(1) do |sym, num|
+        if macro_re =~ sym && (affixes = symbol_to_macro[[$1, $3]])
+          f.puts %|MRB_PRESYM_NAMED("#{sym}", #{num}, #{affixes * 'SYM'}, #{$2})|
+        elsif name = op_table[sym]
+          f.puts %|MRB_PRESYM_NAMED("#{sym}", #{num}, OPSYM, #{name})|
+        elsif
+          f.puts %|MRB_PRESYM_UNNAMED("#{sym}", #{num})|
+        end
+      end
+      f.print "#define MRB_PRESYM_MAX #{presyms.size}"
+    end
+  end
+end