Merge branch 'parallelization' - close #23

.gitignore

@@ -1,6 +1,7 @@
+/.bundle/
 /.yardoc/
 /_yardoc/
+/build_test_run/
 /coverage/
 /doc/
 /pkg/
-/build_test_run/

Rakefile.rb

@@ -15,6 +15,7 @@ CLOBBER.include %w[pkg]
 
 RSpec::Core::RakeTask.new(:spec, :example_string) do |task, args|
   if args.example_string
+    ENV["partial_specs"] = "1"
     task.rspec_opts = %W[-e "#{args.example_string}" -f documentation]
   end
 end

build_tests/build_dir/Rsconsfile

@@ -0,0 +1,5 @@
+Rscons::Environment.new do |env|
+  env.append('CPPPATH' => Dir['src/**/*/'].sort)
+  env.build_dir(%r{^src/([^/]+)/}, 'build_\\1/')
+  env.Program('build_dir.exe', Dir['src/**/*.c'])
+end

build_tests/build_dir/build_dirs_and_root.rb

@@ -0,0 +1,6 @@
+env = Rscons::Environment.new do |env|
+  env.append('CPPPATH' => Dir['src/**/*/'].sort)
+  env.build_dir("src", "build")
+  env.build_root = "build_root"
+  env.Program('build_dir.exe', Dir['src/**/*.c'])
+end

build_tests/build_dir/build_hooks.rb

@@ -0,0 +1,12 @@
+Rscons::Environment.new(echo: :command) do |env|
+  env.append('CPPPATH' => Dir['src/**/*/'].sort)
+  env.build_dir(%r{^src/([^/]+)/}, 'build_\\1/')
+  env.add_build_hook do |build_op|
+    if build_op[:target] =~ %r{build_one/.*\.o}
+      build_op[:vars]["CFLAGS"] << "-O1"
+    elsif build_op[:target] =~ %r{build_two/.*\.o}
+      build_op[:vars]["CFLAGS"] << "-O2"
+    end
+  end
+  env.Program('build_hook.exe', Dir['src/**/*.c'].sort)
+end

build_tests/build_dir/carat.rb

@@ -0,0 +1,8 @@
+Rscons::Environment.new(echo: :command) do |env|
+  env.append('CPPPATH' => Dir['src/**/*/'].sort)
+  env.build_root = "build_root"
+  FileUtils.mkdir_p(env.build_root)
+  FileUtils.mv("src/one/one.c", "build_root")
+  env.Object("^/one.o", "^/one.c")
+  env.Program("build_dir.exe", Dir['src/**/*.c'] + ["^/one.o"])
+end

build_tests/build_dir/copy.rb

@@ -0,0 +1,3 @@
+Rscons::Environment.new do |env|
+  env.Copy("inst.exe", "copy.rb")
+end

build_tests/build_dir/csuffix.rb

@@ -0,0 +1,6 @@
+Rscons::Environment.new do |env|
+  env["CSUFFIX"] = %w[.yargh .c]
+  env["CFLAGS"] += %w[-x c]
+  env["CPPPATH"] += Dir["src/**/"]
+  env.Program("build_dir.exe", Dir["src/**/*.{c,yargh}"])
+end

build_tests/build_dir/install.rb

@@ -0,0 +1,3 @@
+Rscons::Environment.new do |env|
+  env.Install("inst.exe", "install.rb")
+end

build_tests/build_dir/install_directory.rb

@@ -0,0 +1,9 @@
+Rscons::Environment.new do |env|
+  env.Directory("inst")
+  env.Install("inst", "install_directory.rb")
+
+  env.Install("noexist/src", "src")
+
+  env.Directory("exist/src")
+  env.Install("exist/src", "src")
+end

build_tests/build_dir/multiple_targets_same_name.rb

@@ -0,0 +1,5 @@
+Rscons::Environment.new do |env|
+  env["CPPPATH"] << "src/two"
+  env.Object("one.o", "src/one/one.c")
+  env.Object("one.o", "src/two/two.c")
+end

build_tests/build_dir/no_match_build_dir.rb

@@ -0,0 +1,6 @@
+Rscons::Environment.new do |env|
+  env.append('CPPPATH' => Dir['src/**/*/'].sort)
+  env.build_dir("src2", "build")
+  env.build_root = "build_root"
+  env.Program('build_dir.exe', Dir['src/**/*.c'])
+end

build_tests/build_dir/post_build_hook_expansion.rb

@@ -0,0 +1,10 @@
+Rscons::Environment.new do |env|
+  env["CPPPATH"] << "src/two"
+  env.Object("one.o", "src/one/one.c")
+  env.add_post_build_hook do |build_op|
+    if build_op[:target] == "one.o"
+      env["MODULE"] = "two"
+      env.Object("${MODULE}.o", "src/${MODULE}/${MODULE}.c")
+    end
+  end
+end

build_tests/build_dir/slashes.rb

@@ -0,0 +1,6 @@
+Rscons::Environment.new do |env|
+  env.append("CPPPATH" => Dir["src/**/*/"].sort)
+  env.build_dir("src/one/", "build_one/")
+  env.build_dir("src/two", "build_two")
+  env.Program("build_dir.exe", Dir["src/**/*.c"])
+end

build_tests/cfile/Rsconsfile

@@ -0,0 +1,4 @@
+Rscons::Environment.new do |env|
+  env.CFile("lexer.c", "lexer.l")
+  env.CFile("parser.c", "parser.y")
+end

build_tests/cfile/error_unknown_extension.rb

@@ -0,0 +1,3 @@
+Rscons::Environment.new do |env|
+  env.CFile("file.c", "foo.bar")
+end

build_tests/cfile/lexer.l

@@ -0,0 +1,4 @@
+%{
+%}
+
+%%

build_tests/cfile/parser.y

@@ -0,0 +1,9 @@
+%{
+%}
+
+%token ONE
+
+%%
+
+one: ONE
+   ;

build_tests/clone_env/Rsconsfile

@@ -0,0 +1,12 @@
+debug = Rscons::Environment.new(echo: :command) do |env|
+  env.build_dir('src', 'debug')
+  env['CFLAGS'] = '-O2'
+  env['CPPFLAGS'] = '-DSTRING="Debug Version"'
+  env.Program('program-debug.exe', Dir['src/*.c'])
+end
+
+release = debug.clone do |env|
+  env["CPPFLAGS"] = '-DSTRING="Release Version"'
+  env.build_dir('src', 'release')
+  env.Program('program-release.exe', Dir['src/*.c'])
+end

build_tests/clone_env/clone_all.rb

@@ -0,0 +1,15 @@
+env1 = Rscons::Environment.new(echo: :command) do |env|
+  env.build_dir('src', 'build')
+  env['CFLAGS'] = '-O2'
+  env.add_build_hook do |build_op|
+    build_op[:vars]['CPPFLAGS'] = '-DSTRING="Hello"'
+  end
+  env.add_post_build_hook do |build_op|
+    $stdout.puts "post #{build_op[:target]}"
+  end
+  env.Program('program.exe', Dir['src/*.c'])
+end
+
+env2 = env1.clone(clone: :all) do |env|
+  env.Program('program2.exe', Dir['src/*.c'])
+end

build_tests/custom_builder/Rsconsfile

@@ -0,0 +1,16 @@
+class MySource < Rscons::Builder
+  def run(target, sources, cache, env, vars)
+    File.open(target, 'w') do |fh|
+      fh.puts <<EOF
+#define THE_VALUE 5678
+EOF
+    end
+    target
+  end
+end
+
+Rscons::Environment.new do |env|
+  env.add_builder(MySource.new)
+  env.MySource('inc.h', [])
+  env.Program('program.exe', Dir['*.c'])
+end

build_tests/custom_builder/build_after.rb

@@ -0,0 +1,8 @@
+Rscons::Environment.new do |env|
+  env.Command("inc.c",
+              [],
+              "CMD" => %w[ruby gen.rb ${_TARGET}],
+              "CMD_DESC" => "Generating")
+  env.build_after("program.o", "inc.c")
+  env.Program("program.exe", ["program.c", "inc.c"])
+end

build_tests/custom_builder/cvar_expansion.rb

@@ -0,0 +1,18 @@
+class MySource < Rscons::Builder
+  def run(target, sources, cache, env, vars)
+    File.open(target, 'w') do |fh|
+      fh.puts <<EOF
+#define THE_VALUE 678
+EOF
+    end
+    target
+  end
+end
+
+env = Rscons::Environment.new do |env|
+  env["hdr"] = "inc.h"
+  env["src"] = "program.c"
+  env.add_builder(MySource.new)
+  env.MySource('${hdr}')
+  env.Program('program.exe', "${src}")
+end

build_tests/custom_builder/cvar_lambda.rb

@@ -0,0 +1,28 @@
+class MySource < Rscons::Builder
+  def run(target, sources, cache, env, vars)
+    File.open(target, 'w') do |fh|
+      fh.puts <<EOF
+#define THE_VALUE #{env.expand_varref("${the_value}")}
+EOF
+    end
+    target
+  end
+end
+
+e1 = Rscons::Environment.new do |env|
+  env.add_builder(MySource.new)
+  env["one"] = "5"
+  env[:cfg] = {val: "9"}
+  env["two"] = lambda do |args|
+    args[:env][:cfg][:val]
+  end
+  env["the_value"] = lambda do |args|
+    "${one}${two}78"
+  end
+end
+
+e1.clone do |env|
+  env[:cfg][:val] = "6"
+  env.MySource('inc.h', [])
+  env.Program('program.exe', Dir['*.c'])
+end

build_tests/custom_builder/gen.rb

@@ -0,0 +1,9 @@
+sleep 1.0
+c_file = ARGV.first
+h_file = File.basename(c_file, ".c") + ".h"
+File.open(c_file, "w") do |fh|
+  fh.puts
+end
+File.open(h_file, "w") do |fh|
+  fh.puts("#define THE_VALUE 191")
+end

build_tests/custom_builder/multiple_targets.rb

@@ -0,0 +1,19 @@
+class CHGen < Rscons::Builder
+  def run(target, sources, cache, env, vars)
+    c_fname = target
+    h_fname = target.sub(/\.c$/, ".h")
+    unless cache.up_to_date?([c_fname, h_fname], "", sources, env)
+      puts "CHGen #{c_fname}"
+      File.open(c_fname, "w") {|fh| fh.puts "int THE_VALUE = 42;"}
+      File.open(h_fname, "w") {|fh| fh.puts "extern int THE_VALUE;"}
+      cache.register_build([c_fname, h_fname], "", sources, env)
+    end
+    target
+  end
+end
+
+Rscons::Environment.new do |env|
+  env.add_builder(CHGen.new)
+  env.CHGen("inc.c", ["program.c"])
+  env.Program("program.exe", %w[program.c inc.c])
+end

build_tests/d/Rsconsfile

@@ -0,0 +1,3 @@
+Rscons::Environment.new(echo: :command) do |env|
+  env.Program("hello-d.exe", Dir["*.d"])
+end

build_tests/header/Rsconsfile

@@ -0,0 +1,3 @@
+Rscons::Environment.new do |env|
+  env.Program('header.exe', Dir['*.c'])
+end

build_tests/json_to_yaml/Rsconsfile

@@ -0,0 +1,15 @@
+Rscons::Environment.new do |env|
+  require 'json'
+  require 'yaml'
+  env.add_builder(:JsonToYaml) do |target, sources, cache, env, vars|
+    unless cache.up_to_date?(target, :JsonToYaml, sources, env)
+      cache.mkdir_p(File.dirname(target))
+      File.open(target, 'w') do |f|
+        f.write(YAML.dump(JSON.load(IO.read(sources.first))))
+      end
+      cache.register_build(target, :JsonToYaml, sources, env)
+    end
+    target
+  end
+  env.JsonToYaml('foo.yml', 'foo.json')
+end

build_tests/library/Rsconsfile

@@ -0,0 +1,4 @@
+Rscons::Environment.new(echo: :command) do |env|
+  env.Program('library.exe', ['lib.a', 'three.c'])
+  env.Library("lib.a", ['one.c', 'two.c'], 'CPPFLAGS' => ['-Dmake_lib'])
+end

build_tests/library/override_arcmd.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new(echo: :command) do |env|
+  env["ARCMD"] = %w[ar rcf ${_TARGET} ${_SOURCES}]
+  env.Library("lib.a", Dir["*.c"].sort)
+end

build_tests/preprocess/Rsconsfile

@@ -0,0 +1,3 @@
+Rscons::Environment.new do |env|
+  env.Preprocess("pp", "foo.h")
+end

build_tests/simple/Rsconsfile

@@ -0,0 +1,3 @@
+Rscons::Environment.new do |env|
+  env.Program('simple.exe', Dir['*.c'])
+end

build_tests/simple/build_root_builder_no_sources.rb

@@ -0,0 +1,10 @@
+class TestBuilder < Rscons::Builder
+  def run(target, sources, cache, env, vars)
+    target
+  end
+end
+Rscons::Environment.new do |env|
+  env.build_root = "build"
+  env.add_builder(TestBuilder.new)
+  env.TestBuilder("file")
+end

build_tests/simple/build_sources.rb

@@ -0,0 +1,22 @@
+class MyProgram < Rscons::Builder
+  def run(options)
+    target, sources, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars)
+    objects = env.build_sources(sources, [".o"], cache, vars)
+    command = %W[gcc -o #{target}] + objects
+    return false unless env.execute("#{name} #{target}", command)
+    target
+  end
+end
+
+Rscons::Environment.new do |env|
+  env.add_builder(MyProgram.new)
+  env.Object("simple.o", "simple.c")
+  File.open("two.c", "wb") do |fh|
+    fh.puts <<-EOF
+      void two(void)
+      {
+      }
+    EOF
+  end
+  env.MyProgram("simple.exe", ["simple.o", "two.c"])
+end

build_tests/simple/cache_command_change.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new do |env|
+  env["LIBS"] += ["c"]
+  env.Program('simple.exe', Dir['*.c'])
+end

build_tests/simple/cache_dep_checksum_change.rb

@@ -0,0 +1,3 @@
+Rscons::Environment.new do |env|
+  env.Copy("simple.copy", "simple.c")
+end

build_tests/simple/cache_new_dep1.rb

@@ -0,0 +1,6 @@
+Rscons::Environment.new do |env|
+  env.Object("simple.o", "simple.c")
+  env.process
+  env["LDCMD"] = %w[gcc -o ${_TARGET} simple.o]
+  env.Program('simple.exe', [])
+end

build_tests/simple/cache_new_dep2.rb

@@ -0,0 +1,5 @@
+Rscons::Environment.new do |env|
+  env.Object("simple.o", "simple.c")
+  env["LDCMD"] = %w[gcc -o ${_TARGET} simple.o]
+  env.Program('simple.exe', ["simple.o"])
+end

build_tests/simple/cache_successful_builds_when_one_fails.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new do |env|
+  env.Object("simple.o", "simple.c")
+  env.Object("two.o", "two.c")
+end

build_tests/simple/cache_user_dep.rb

@@ -0,0 +1,5 @@
+Rscons::Environment.new do |env|
+  target = env.Program("simple.exe", "simple.c")
+  user_deps = File.read("user_deps", mode: "rb").split(" ")
+  target.depends(*user_deps)
+end

build_tests/simple/clear_targets.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new do |env|
+  env.Program("simple.exe", "simple.c")
+  env.clear_targets
+end

build_tests/simple/command.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new(echo: :command) do |env|
+  env["LD"] = "gcc"
+  env.Program('simple.exe', Dir['*.c'])
+end

build_tests/simple/command_builder.rb

@@ -0,0 +1,7 @@
+Rscons::Environment.new do |env|
+  command = %W[gcc -o ${_TARGET} ${_SOURCES}]
+  env.Command("simple.exe",
+              "simple.c",
+              "CMD" => command,
+              "CMD_DESC" => "BuildIt")
+end

build_tests/simple/cvar_array.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new do |env|
+  env["sources"] = Dir["*.c"].sort
+  env.Program("simple.exe", "${sources}")
+end

build_tests/simple/directory.rb

@@ -0,0 +1,3 @@
+Rscons::Environment.new do |env|
+  env.Directory("teh_dir")
+end

build_tests/simple/disassemble.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new do |env|
+  env.Object("simple.o", "simple.c")
+  env.Disassemble("simple.txt", "simple.o")
+end

build_tests/simple/dump.rb

@@ -0,0 +1,5 @@
+env = Rscons::Environment.new do |env|
+  env["CFLAGS"] += %w[-O2 -fomit-frame-pointer]
+  env[:foo] = :bar
+end
+env.dump

build_tests/simple/error_unknown_suffix.rb

@@ -0,0 +1,6 @@
+Rscons::Environment.new do |env|
+  File.open("foo.xyz", "wb") do |fh|
+    fh.puts("hi")
+  end
+  env.Object("foo.o", "foo.xyz")
+end

build_tests/simple/link_flag_change.rb

@@ -0,0 +1,5 @@
+Rscons::Environment.new(echo: :command) do |env|
+  env["LD"] = "gcc"
+  env["LIBPATH"] += ["libdir"]
+  env.Program('simple.exe', Dir['*.c'])
+end

build_tests/simple/override_cccmd.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new(echo: :command) do |env|
+  env.Object("simple.o", "simple.c",
+             "CCCMD" => %w[${CC} -c -o ${_TARGET} -Dfoobar ${_SOURCES}])
+end

build_tests/simple/override_depfilesuffix.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new(echo: :command) do |env|
+  env["DEPFILESUFFIX"] = ".deppy"
+  env.Object("simple.o", "simple.c")
+end

build_tests/simple/phony_target.rb

@@ -0,0 +1,12 @@
+Rscons::Environment.new do |env|
+  env.build_root = "build"
+  env.add_builder(:Checker) do |target, sources, cache, env, vars|
+    unless cache.up_to_date?(target, :Checker, sources, env)
+      puts "Checker #{sources.first}" if env.echo != :off
+      cache.register_build(target, :Checker, sources, env)
+    end
+    target
+  end
+  env.Program("simple.exe", "simple.c")
+  env.Checker(:checker, "simple.exe")
+end

build_tests/simple/preprocess.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new do |env|
+  env.Preprocess("simplepp.c", "simple.c")
+  env.Program("simple.exe", "simplepp.c")
+end

build_tests/simple/progsuffix.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new do |env|
+  env["PROGSUFFIX"] = ".out"
+  env.Program("simple", Dir["*.c"])
+end

build_tests/simple/register_target_in_build_hook.rb

@@ -0,0 +1,8 @@
+Rscons::Environment.new do |env|
+  env.Program("simple.exe", Dir["*.c"])
+  env.add_build_hook do |build_op|
+    if build_op[:target].end_with?(".o")
+      env.Disassemble("#{build_op[:target]}.txt", build_op[:target])
+    end
+  end
+end

build_tests/simple/run_builder.rb

@@ -0,0 +1,12 @@
+class MyObject < Rscons::Builder
+  def run(options)
+    target, sources, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars)
+    env.run_builder(env.builders["Object"], target, sources, cache, vars)
+  end
+end
+
+Rscons::Environment.new do |env|
+  env.add_builder(MyObject.new)
+  env.MyObject("simple.o", "simple.c")
+  env.Program("simple.exe", "simple.o")
+end

build_tests/simple/standard_build.rb

@@ -0,0 +1,17 @@
+class MyCommand < Rscons::Builder
+  def run(target, sources, cache, env, vars)
+    vars = vars.merge({
+      "_TARGET" => target,
+      "_SOURCES" => sources,
+    })
+    command = env.build_command("${CMD}", vars)
+    cmd_desc = vars["CMD_DESC"] || "MyCommand"
+    standard_build("#{cmd_desc} #{target}", target, command, sources, env, cache)
+  end
+end
+
+Rscons::Environment.new do |env|
+  env.add_builder(MyCommand.new)
+  command = %w[gcc -c -o ${_TARGET} ${_SOURCES}]
+  env.MyCommand("simple.o", "simple.c", "CMD" => command)
+end

build_tests/simple/threading.rb

@@ -0,0 +1,28 @@
+class ThreadedTestBuilder < Rscons::Builder
+  def run(options)
+    command = ["ruby", "-e", %[sleep 1]]
+    Rscons::ThreadedCommand.new(
+      command,
+      short_description: "ThreadedTestBuilder #{options[:target]}")
+  end
+  def finalize(options)
+    true
+  end
+end
+
+class NonThreadedTestBuilder < Rscons::Builder
+  def run(options)
+    puts "NonThreadedTestBuilder #{options[:target]}"
+    sleep 1
+    options[:target]
+  end
+end
+
+Rscons::Environment.new do |env|
+  env.add_builder(ThreadedTestBuilder.new)
+  env.add_builder(NonThreadedTestBuilder.new)
+  env.ThreadedTestBuilder("a")
+  env.ThreadedTestBuilder("b")
+  env.ThreadedTestBuilder("c")
+  env.NonThreadedTestBuilder("d")
+end

build_tests/simple/user_dependencies.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new do |env|
+  program = env.Program("simple.exe", Dir["*.c"])
+  env.depends(program, "program.ld")
+end

build_tests/simple_cc/Rsconsfile

@@ -0,0 +1,3 @@
+Rscons::Environment.new do |env|
+  env.Program('simple.exe', Dir['*.cc'])
+end

build_tests/simple_cc/cxxsuffix.rb

@@ -0,0 +1,5 @@
+Rscons::Environment.new do |env|
+  env["CXXSUFFIX"] = %w[.cccc .cc]
+  env["CXXFLAGS"] += %w[-x c++]
+  env.Program("simple.exe", Dir["*.cc"] + Dir["*.cccc"])
+end

build_tests/simple_cc/preprocess.rb

@@ -0,0 +1,4 @@
+Rscons::Environment.new do |env|
+  env.Preprocess("simplepp.cc", "simple.cc")
+  env.Program("simple.exe", "simplepp.cc")
+end

build_tests/two_sources/Rsconsfile

@@ -0,0 +1,4 @@
+Rscons::Environment.new(echo: :command) do |env|
+  env.Object("one.o", "one.c", 'CPPFLAGS' => ['-DONE'])
+  env.Program('two_sources.exe', ['one.o', 'two.c'])
+end

build_tests/two_sources/assuffix.rb

@@ -0,0 +1,7 @@
+Rscons::Environment.new do |env|
+  env["ASSUFFIX"] = %w[.ssss .sss]
+  env["CFLAGS"] += %w[-S]
+  env.Object("one.ssss", "one.c", "CPPFLAGS" => ["-DONE"])
+  env.Object("two.sss", "two.c")
+  env.Program("two_sources.exe", %w[one.ssss two.sss], "ASFLAGS" => env["ASFLAGS"] + %w[-x assembler])
+end

build_tests/two_sources/cache_strict_deps.rb

@@ -0,0 +1,19 @@
+class StrictBuilder < Rscons::Builder
+  def run(options)
+    target, sources, cache, env = options.values_at(:target, :sources, :cache, :env)
+    command = %W[gcc -o #{target}] + sources.sort
+    unless cache.up_to_date?(target, command, sources, env, strict_deps: true)
+      return false unless env.execute("StrictBuilder #{target}", command)
+      cache.register_build(target, command, sources, env)
+    end
+    target
+  end
+end
+
+Rscons::Environment.new(echo: :command) do |env|
+  env.add_builder(StrictBuilder.new)
+  env.Object("one.o", "one.c", "CCFLAGS" => %w[-DONE])
+  env.Object("two.o", "two.c")
+  sources = File.read("sources", mode: "rb").split(" ")
+  env.StrictBuilder("program.exe", sources)
+end

build_tests/two_sources/libsuffix.rb

@@ -0,0 +1,6 @@
+Rscons::Environment.new() do |env|
+  env["LIBSUFFIX"] = %w[.aaaa .aaa]
+  env.Library("one.aaaa", "one.c", "CPPFLAGS" => ["-DONE"])
+  env.Library("two.aaa", "two.c")
+  env.Program("two_sources.exe", %w[one.aaaa two.aaa])
+end

build_tests/two_sources/objsuffix.rb

@@ -0,0 +1,6 @@
+Rscons::Environment.new do |env|
+  env["OBJSUFFIX"] = %w[.oooo .ooo]
+  env.Object("one.oooo", "one.c", "CPPFLAGS" => ["-DONE"])
+  env.Object("two.ooo", "two.c")
+  env.Program("two_sources.exe", %w[one.oooo two.ooo])
+end

lib/rscons.rb

@@ -2,6 +2,8 @@ require_relative "rscons/build_target"
 require_relative "rscons/builder"
 require_relative "rscons/cache"
 require_relative "rscons/environment"
+require_relative "rscons/job_set"
+require_relative "rscons/threaded_command"
 require_relative "rscons/varset"
 require_relative "rscons/version"
 
@@ -40,6 +42,10 @@ module Rscons
 
   class << self
 
+    # @return [Integer]
+    #   The number of threads to use when scheduling subprocesses.
+    attr_accessor :n_threads
+
     # Remove all generated files.
     #
     # @return [void]
@@ -146,7 +152,45 @@ module Rscons
       @command_executer = val
     end
 
+    private
+
+    # Determine the number of threads to use by default.
+    #
+    # @return [Integer]
+    #   The number of threads to use by default.
+    def determine_n_threads
+      # If the user specifies the number of threads in the environment, then
+      # respect that.
+      if ENV["RSCONS_NTHREADS"] =~ /^(\d+)$/
+        return $1.to_i
+      end
+
+      # Otherwise try to figure out how many threads are available on the
+      # host hardware.
+      begin
+        case RbConfig::CONFIG["host_os"]
+        when /linux/
+          return File.read("/proc/cpuinfo").scan(/^processor\s*:/).size
+        when /mswin|mingw/
+          if `wmic cpu get NumberOfLogicalProcessors /value` =~ /NumberOfLogicalProcessors=(\d+)/
+            return $1.to_i
+          end
+        when /darwin/
+          if `sysctl -n hw.ncpu` =~ /(\d+)/
+            return $1.to_i
+          end
+        end
+      rescue
+      end
+
+      # If we can't figure it out, default to 1.
+      1
+    end
+
   end
+
+  @n_threads = determine_n_threads
+
 end
 
 # Unbuffer $stdout

lib/rscons/builder.rb

@@ -56,22 +56,128 @@ module Rscons
       false
     end
 
+    # Set up a build operation using this builder.
+    #
+    # This method is called when a build target is registered using this
+    # builder. This method should not do any building, but should perform any
+    # setup needed and register any prerequisite build targets that need to be
+    # built before the target being requested here.
+    #
+    # If the builder needs no special setup, it does not need to override this
+    # method. If there is any information produced in this method that will be
+    # needed later in the build, it can be stored in the return value from this
+    # method, which will be passed to the {#run} method.
+    #
+    # @since 1.10.0
+    #
+    # @param options [Hash]
+    #   Options.
+    # @option options [String] :target
+    #   Target file name.
+    # @option options [Array<String>] :sources
+    #   Source file name(s).
+    # @option options [Environment] :env
+    #   The Environment executing the builder.
+    # @option options [Hash,VarSet] :vars
+    #   Extra construction variables.
+    #
+    # @return [Object]
+    #   Any object that the builder author wishes to be saved and passed back
+    #   in to the {#run} method.
+    def setup(options)
+    end
+
     # Run the builder to produce a build target.
     #
-    # @param target [String] Target file name.
-    # @param sources [Array<String>] Source file name(s).
-    # @param cache [Cache] The Cache object.
-    # @param env [Environment] The Environment executing the builder.
-    # @param vars [Hash,VarSet] Extra construction variables.
+    # The run method supports two different signatures - an older signature
+    # with five separate arguments, and a newer one with one Hash argument. A
+    # builder author can use either signature, and Rscons will automatically
+    # determine which arguments to pass when invoking the run method based on
+    # the method's arity.
     #
-    # @return [String,false]
+    # @overload run(target, sources, cache, env, vars)
+    #
+    #   @param target [String]
+    #     Target file name.
+    #   @param sources [Array<String>]
+    #     Source file name(s).
+    #   @param cache [Cache]
+    #     The Cache object.
+    #   @param env [Environment]
+    #     The Environment executing the builder.
+    #   @param vars [Hash,VarSet]
+    #     Extra construction variables.
+    #
+    # @overload run(options)
+    #
+    #   @since 1.10.0
+    #
+    #   @param options [Hash]
+    #     Run options.
+    #   @option options [String] :target
+    #     Target file name.
+    #   @option options [Array<String>] :sources
+    #     Source file name(s).
+    #   @option options [Cache] :cache
+    #     The Cache object.
+    #   @option options [Environment] :env
+    #     The Environment executing the builder.
+    #   @option options [Hash,VarSet] :vars
+    #     Extra construction variables.
+    #   @option options [Object] :setup_info
+    #     Whatever value was returned from this builder's {#setup} method call.
+    #
+    # @return [ThreadedCommand,String,false]
     #   Name of the target file on success or false on failure.
-    def run(target, sources, cache, env, vars)
+    #   Since 1.10.0, this method may return an instance of {ThreadedCommand}.
+    #   In that case, the build operation has not actually been completed yet
+    #   but the command to do so will be executed by Rscons in a separate
+    #   thread. This allows for build parallelization. If a {ThreadedCommand}
+    #   object is returned, the {#finalize} method will be called after the
+    #   command has completed. The {#finalize} method should then be used to
+    #   record cache info, if needed, and to return the true result of the
+    #   build operation. The builder can store information to be passed in to
+    #   the {#finalize} method by populating the :builder_info field of the
+    #   {ThreadedCommand} object returned here.
+    def run(options)
       raise "This method must be overridden in a subclass"
     end
 
+    # Finalize a build operation.
+    #
+    # This method is called after the {#run} method if the {#run} method
+    # returns a {ThreadedCommand} object.
+    #
+    # @since 1.10.0
+    #
+    # @param options [Hash]
+    #   Options.
+    # @option options [String] :target
+    #   Target file name.
+    # @option options [Array<String>] :sources
+    #   Source file name(s).
+    # @option options [Cache] :cache
+    #   The Cache object.
+    # @option options [Environment] :env
+    #   The Environment executing the builder.
+    # @option options [Hash,VarSet] :vars
+    #   Extra construction variables.
+    # @option options [Object] :setup_info
+    #   Whatever value was returned from this builder's {#setup} method call.
+    # @option options [true,false,nil] :command_status
+    #   If the {#run} method returns a {ThreadedCommand}, this field will
+    #   contain the return value from executing the command with
+    #   Kernel.system().
+    # @option options [ThreadedCommand] :tc
+    #   The {ThreadedCommand} object that was returned by the #run method.
+    #
+    # @return [String,false]
+    #   Name of the target file on success or false on failure.
+    def finalize(options)
+    end
+
     # Check if the cache is up to date for the target and if not execute the
-    # build command.
+    # build command. This method does not support parallelization.
     #
     # @param short_cmd_string [String]
     #   Short description of build action to be printed when env.echo ==
@@ -96,5 +202,54 @@ module Rscons
       end
       target
     end
+
+    # Check if the cache is up to date for the target and if not create a
+    # {ThreadedCommand} object to execute the build command in a thread.
+    #
+    # @since 1.10.0
+    #
+    # @param short_cmd_string [String]
+    #   Short description of build action to be printed when env.echo ==
+    #   :short.
+    # @param target [String] Name of the target file.
+    # @param command [Array<String>]
+    #   The command to execute to build the target.
+    # @param sources [Array<String>] Source file name(s).
+    # @param env [Environment] The Environment executing the builder.
+    # @param cache [Cache] The Cache object.
+    #
+    # @return [String,ThreadedCommand]
+    #   The name of the target if it is already up to date or the
+    #   {ThreadedCommand} object created to update it.
+    def standard_threaded_build(short_cmd_string, target, command, sources, env, cache)
+      if cache.up_to_date?(target, command, sources, env)
+        target
+      else
+        unless Rscons.phony_target?(target)
+          cache.mkdir_p(File.dirname(target))
+          FileUtils.rm_f(target)
+        end
+        ThreadedCommand.new(
+          command,
+          short_description: short_cmd_string)
+      end
+    end
+
+    # Register build results from a {ThreadedCommand} with the cache.
+    #
+    # @since 1.10.0
+    #
+    # @param options [Hash]
+    #   Builder finalize options.
+    #
+    # @return [String, nil]
+    #   The target name on success or nil on failure.
+    def standard_finalize(options)
+      if options[:command_status]
+        target, sources, cache, env = options.values_at(:target, :sources, :cache, :env)
+        cache.register_build(target, options[:tc].command, sources, env)
+        target
+      end
+    end
   end
 end

lib/rscons/builders/cfile.rb

@@ -7,6 +7,7 @@ module Rscons
     #   env.CFile("parser.tab.cc", "parser.yy")
     #   env.CFile("lex.yy.cc", "parser.ll")
     class CFile < Builder
+
       # Return default construction variables for the builder.
       #
       # @param env [Environment] The Environment using the builder.
@@ -48,8 +49,20 @@ module Rscons
             raise "Unknown source file #{sources.first.inspect} for CFile builder"
           end
         command = env.build_command("${#{cmd}_CMD}", vars)
-        standard_build("#{cmd} #{target}", target, command, sources, env, cache)
+        standard_threaded_build("#{cmd} #{target}", target, command, sources, env, cache)
       end
+
+      # Finalize a build.
+      #
+      # @param options [Hash]
+      #   Finalize options.
+      #
+      # @return [String, nil]
+      #   The target name on success or nil on failure.
+      def finalize(options)
+        standard_finalize(options)
+      end
+
     end
   end
 end

lib/rscons/builders/command.rb

@@ -9,25 +9,36 @@ module Rscons
     #   env.Command("docs.html", "docs.md",
     #               CMD => %w[pandoc -fmarkdown -thtml -o${_TARGET} ${_SOURCES}])
     class Command < Builder
+
       # Run the builder to produce a build target.
       #
-      # @param target [String] Target file name.
-      # @param sources [Array<String>] Source file name(s).
-      # @param cache [Cache] The Cache object.
-      # @param env [Environment] The Environment executing the builder.
-      # @param vars [Hash,VarSet] Extra construction variables.
+      # @param options [Hash] Builder run options.
       #
-      # @return [String,false]
-      #   Name of the target file on success or false on failure.
-      def run(target, sources, cache, env, vars)
+      # @return [String, ThreadedCommand]
+      #   Target file name if target is up to date or a {ThreadedCommand}
+      #   to execute to build the target.
+      def run(options)
+        target, sources, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars)
         vars = vars.merge({
           "_TARGET" => target,
           "_SOURCES" => sources,
         })
         command = env.build_command("${CMD}", vars)
         cmd_desc = vars["CMD_DESC"] || "Command"
-        standard_build("#{cmd_desc} #{target}", target, command, sources, env, cache)
+        standard_threaded_build("#{cmd_desc} #{target}", target, command, sources, env, cache)
       end
+
+      # Finalize a build.
+      #
+      # @param options [Hash]
+      #   Finalize options.
+      #
+      # @return [String, nil]
+      #   The target name on success or nil on failure.
+      def finalize(options)
+        standard_finalize(options)
+      end
+
     end
   end
 end

lib/rscons/builders/disassemble.rb

@@ -2,6 +2,7 @@ module Rscons
   module Builders
     # The Disassemble builder produces a disassembly listing of a source file.
     class Disassemble < Builder
+
       # Return default construction variables for the builder.
       #
       # @param env [Environment] The Environment using the builder.
@@ -28,13 +29,28 @@ module Rscons
       def run(target, sources, cache, env, vars)
         vars = vars.merge("_SOURCES" => sources)
         command = env.build_command("${DISASM_CMD}", vars)
-        unless cache.up_to_date?(target, command, sources, env)
+        if cache.up_to_date?(target, command, sources, env)
+          target
+        else
           cache.mkdir_p(File.dirname(target))
-          return false unless env.execute("Disassemble #{target}", command, options: {out: target})
-          cache.register_build(target, command, sources, env)
+          ThreadedCommand.new(
+            command,
+            short_description: "Disassemble #{target}",
+            system_options: {out: target})
         end
-        target
       end
+
+      # Finalize a build.
+      #
+      # @param options [Hash]
+      #   Finalize options.
+      #
+      # @return [String, nil]
+      #   The target name on success or nil on failure.
+      def finalize(options)
+        standard_finalize(options)
+      end
+
     end
   end
 end

lib/rscons/builders/library.rb

@@ -2,6 +2,7 @@ module Rscons
   module Builders
     # A default Rscons builder that produces a static library archive.
     class Library < Builder
+
       # Return default construction variables for the builder.
       #
       # @param env [Environment] The Environment using the builder.
@@ -16,28 +17,48 @@ module Rscons
         }
       end
 
+      # Set up a build operation using this builder.
+      #
+      # @param options [Hash] Builder setup options.
+      #
+      # @return [Object]
+      #   Any object that the builder author wishes to be saved and passed back
+      #   in to the {#run} method.
+      def setup(options)
+        target, sources, env, vars = options.values_at(:target, :sources, :env, :vars)
+        suffixes = env.expand_varref(["${OBJSUFFIX}", "${LIBSUFFIX}"], vars)
+        # Register builders to build each source to an object file or library.
+        env.register_builds(target, sources, suffixes, vars)
+      end
+
       # Run the builder to produce a build target.
       #
-      # @param target [String] Target file name.
-      # @param sources [Array<String>] Source file name(s).
-      # @param cache [Cache] The Cache object.
-      # @param env [Environment] The Environment executing the builder.
-      # @param vars [Hash,VarSet] Extra construction variables.
+      # @param options [Hash] Builder run options.
       #
       # @return [String,false]
       #   Name of the target file on success or false on failure.
-      def run(target, sources, cache, env, vars)
-        # build sources to linkable objects
-        objects = env.build_sources(sources, env.expand_varref(["${OBJSUFFIX}", "${LIBSUFFIX}"], vars).flatten, cache, vars)
-        if objects
-          vars = vars.merge({
-            '_TARGET' => target,
-            '_SOURCES' => objects,
-          })
-          command = env.build_command("${ARCMD}", vars)
-          standard_build("AR #{target}", target, command, objects, env, cache)
-        end
+      def run(options)
+        target, sources, cache, env, vars, objects = options.values_at(:target, :sources, :cache, :env, :vars, :setup_info)
+        vars = vars.merge({
+          '_TARGET' => target,
+          '_SOURCES' => objects,
+        })
+        options[:sources] = objects
+        command = env.build_command("${ARCMD}", vars)
+        standard_threaded_build("AR #{target}", target, command, objects, env, cache)
       end
+
+      # Finalize a build.
+      #
+      # @param options [Hash]
+      #   Finalize options.
+      #
+      # @return [String, nil]
+      #   The target name on success or nil on failure.
+      def finalize(options)
+        standard_finalize(options)
+      end
+
     end
   end
 end

lib/rscons/builders/object.rb

@@ -3,6 +3,7 @@ module Rscons
     # A default Rscons builder which knows how to produce an object file from
     # various types of source files.
     class Object < Builder
+
       # Mapping of known sources from which to build object files.
       KNOWN_SUFFIXES = {
         "AS" => "ASSUFFIX",
@@ -76,15 +77,13 @@ module Rscons
 
       # Run the builder to produce a build target.
       #
-      # @param target [String] Target file name.
-      # @param sources [Array<String>] Source file name(s).
-      # @param cache [Cache] The Cache object.
-      # @param env [Environment] The Environment executing the builder.
-      # @param vars [Hash,VarSet] Extra construction variables.
+      # @param options [Hash] Builder run options.
       #
-      # @return [String,false]
-      #   Name of the target file on success or false on failure.
-      def run(target, sources, cache, env, vars)
+      # @return [String, ThreadedCommand]
+      #   Target file name if target is up to date or a {ThreadedCommand}
+      #   to execute to build the target.
+      def run(options)
+        target, sources, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars)
         vars = vars.merge({
           '_TARGET' => target,
           '_SOURCES' => sources,
@@ -96,19 +95,29 @@ module Rscons
           v.nil? and raise "Error: unknown input file type: #{sources.first.inspect}"
         end.first
         command = env.build_command("${#{com_prefix}CMD}", vars)
-        unless cache.up_to_date?(target, command, sources, env)
-          cache.mkdir_p(File.dirname(target))
-          FileUtils.rm_f(target)
-          return false unless env.execute("#{com_prefix} #{target}", command)
-          deps = sources
+        # Store vars back into options so new keys are accessible in #finalize.
+        options[:vars] = vars
+        standard_threaded_build("#{com_prefix} #{target}", target, command, sources, env, cache)
+      end
+
+      # Finalize the build operation.
+      #
+      # @param options [Hash] Builder finalize options.
+      #
+      # @return [String, nil]
+      #   Name of the target file on success or nil on failure.
+      def finalize(options)
+        if options[:command_status]
+          target, deps, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars)
           if File.exists?(vars['_DEPFILE'])
             deps += Environment.parse_makefile_deps(vars['_DEPFILE'], target)
             FileUtils.rm_f(vars['_DEPFILE'])
           end
-          cache.register_build(target, command, deps.uniq, env)
+          cache.register_build(target, options[:tc].command, deps.uniq, env)
+          target
         end
-        target
       end
+
     end
   end
 end

lib/rscons/builders/preprocess.rb

@@ -20,15 +20,13 @@ module Rscons
 
       # Run the builder to produce a build target.
       #
-      # @param target [String] Target file name.
-      # @param sources [Array<String>] Source file name(s).
-      # @param cache [Cache] The Cache object.
-      # @param env [Environment] The Environment executing the builder.
-      # @param vars [Hash,VarSet] Extra construction variables.
+      # @param options [Hash] Builder run options.
       #
-      # @return [String,false]
-      #   Name of the target file on success or false on failure.
-      def run(target, sources, cache, env, vars)
+      # @return [String, ThreadedCommand]
+      #   Target file name if target is up to date or a {ThreadedCommand}
+      #   to execute to build the target.
+      def run(options)
+        target, sources, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars)
         if sources.find {|s| s.end_with?(*env.expand_varref("${CXXSUFFIX}", vars))}
           pp_cc = "${CXX}"
           depgen = "${CXXDEPGEN}"
@@ -42,17 +40,27 @@ module Rscons
                           "_SOURCES" => sources,
                           "_DEPFILE" => Rscons.set_suffix(target, env.expand_varref("${DEPFILESUFFIX}", vars)))
         command = env.build_command("${CPP_CMD}", vars)
-        unless cache.up_to_date?(target, command, sources, env)
-          cache.mkdir_p(File.dirname(target))
-          return false unless env.execute("Preprocess #{target}", command)
-          deps = sources
-          if File.exists?(vars["_DEPFILE"])
-            deps += Environment.parse_makefile_deps(vars["_DEPFILE"], nil)
-            FileUtils.rm_f(vars["_DEPFILE"])
+        # Store vars back into options so new keys are accessible in #finalize.
+        options[:vars] = vars
+        standard_threaded_build("#{name} #{target}", target, command, sources, env, cache)
+      end
+
+      # Finalize the build operation.
+      #
+      # @param options [Hash] Builder finalize options.
+      #
+      # @return [String, nil]
+      #   Name of the target file on success or nil on failure.
+      def finalize(options)
+        if options[:command_status]
+          target, deps, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars)
+          if File.exists?(vars['_DEPFILE'])
+            deps += Environment.parse_makefile_deps(vars['_DEPFILE'], nil)
+            FileUtils.rm_f(vars['_DEPFILE'])
           end
-          cache.register_build(target, command, deps.uniq, env)
+          cache.register_build(target, options[:tc].command, deps.uniq, env)
+          target
         end
-        target
       end
 
     end

lib/rscons/builders/program.rb

@@ -3,6 +3,7 @@ module Rscons
     # A default Rscons builder that knows how to link object files into an
     # executable program.
     class Program < Builder
+
       # Return default construction variables for the builder.
       #
       # @param env [Environment] The Environment using the builder.
@@ -45,20 +46,28 @@ module Rscons
         super(my_options)
       end
 
+      # Set up a build operation using this builder.
+      #
+      # @param options [Hash] Builder setup options.
+      #
+      # @return [Object]
+      #   Any object that the builder author wishes to be saved and passed back
+      #   in to the {#run} method.
+      def setup(options)
+        target, sources, env, vars = options.values_at(:target, :sources, :env, :vars)
+        suffixes = env.expand_varref(["${OBJSUFFIX}", "${LIBSUFFIX}"], vars)
+        # Register builders to build each source to an object file or library.
+        env.register_builds(target, sources, suffixes, vars)
+      end
+
       # Run the builder to produce a build target.
       #
-      # @param target [String] Target file name.
-      # @param sources [Array<String>] Source file name(s).
-      # @param cache [Cache] The Cache object.
-      # @param env [Environment] The Environment executing the builder.
-      # @param vars [Hash,VarSet] Extra construction variables.
+      # @param options [Hash] Builder run options.
       #
       # @return [String,false]
       #   Name of the target file on success or false on failure.
-      def run(target, sources, cache, env, vars)
-        # build sources to linkable objects
-        objects = env.build_sources(sources, env.expand_varref(["${OBJSUFFIX}", "${LIBSUFFIX}"], vars).flatten, cache, vars)
-        return false unless objects
+      def run(options)
+        target, sources, cache, env, vars, objects = options.values_at(:target, :sources, :cache, :env, :vars, :setup_info)
         ld = env.expand_varref("${LD}", vars)
         ld = if ld != ""
                ld
@@ -74,9 +83,22 @@ module Rscons
           '_SOURCES' => objects,
           'LD' => ld,
         })
+        options[:sources] = objects
         command = env.build_command("${LDCMD}", vars)
-        standard_build("LD #{target}", target, command, objects, env, cache)
+        standard_threaded_build("LD #{target}", target, command, objects, env, cache)
       end
+
+      # Finalize a build.
+      #
+      # @param options [Hash]
+      #   Finalize options.
+      #
+      # @return [String, nil]
+      #   The target name on success or nil on failure.
+      def finalize(options)
+        standard_finalize(options)
+      end
+
     end
   end
 end

lib/rscons/cache.rb

@@ -2,7 +2,6 @@ require "digest/md5"
 require "fileutils"
 require "json"
 require "set"
-require "singleton"
 require "rscons/version"
 
 module Rscons
@@ -51,7 +50,6 @@ module Rscons
   #     },
   #   }
   class Cache
-    include Singleton
 
     # Name of the file to store cache information in
     CACHE_FILE = ".rsconscache"
@@ -59,6 +57,13 @@ module Rscons
     # Prefix for phony cache entries.
     PHONY_PREFIX = ":PHONY:"
 
+    class << self
+      # Access the singleton instance.
+      def instance
+        @instance ||= Cache.new
+      end
+    end
+
     # Create a Cache object and load in the previous contents from the cache
     # file.
     def initialize

lib/rscons/cli.rb

@@ -35,6 +35,10 @@ module Rscons
             rsconsfile = f
           end
 
+          opts.on("-j NTHREADS", "Use NTHREADS parallel jobs (local default #{Rscons.n_threads})") do |n_threads|
+            Rscons.n_threads = n_threads.to_i
+          end
+
           opts.on_tail("--version", "Show version") do
             puts "Rscons version #{Rscons::VERSION}"
             exit 0

lib/rscons/environment.rb

@@ -1,6 +1,7 @@
 require "fileutils"
 require "set"
 require "shellwords"
+require "thwait"
 
 module Rscons
   # The Environment class is the main programmatic interface to Rscons. It
@@ -37,8 +38,10 @@ module Rscons
     # If a block is given, the Environment object is yielded to the block and
     # when the block returns, the {#process} method is automatically called.
     def initialize(options = {})
+      @threaded_commands = Set.new
+      @registered_build_dependencies = {}
       @varset = VarSet.new
-      @targets = {}
+      @job_set = JobSet.new(@registered_build_dependencies)
       @user_deps = {}
       @builders = {}
       @build_dirs = []
@@ -280,40 +283,62 @@ module Rscons
     #
     # @return [void]
     def process
-      while @targets.size > 0
-        expand_paths!
-        targets = @targets
-        @targets = {}
-        cache = Cache.instance
-        cache.clear_checksum_cache!
-        targets_processed = Set.new
-        process_target = proc do |target|
-          unless targets_processed.include?(target)
-            targets_processed << target
-            targets[target].each do |target_params|
-              target_params[:sources].each do |src|
-                if targets.include?(src) and not targets_processed.include?(src)
-                  process_target.call(src)
-                end
-              end
-              result = run_builder(target_params[:builder],
-                                   target,
-                                   target_params[:sources],
-                                   cache,
-                                   target_params[:vars] || {})
-              unless result
-                raise BuildError.new("Failed to build #{target}")
-              end
+      cache = Cache.instance
+      begin
+        while @job_set.size > 0 or @threaded_commands.size > 0
+
+          targets_still_building = @threaded_commands.map do |tc|
+            tc.build_operation[:target]
+          end
+          job = @job_set.get_next_job_to_run(targets_still_building)
+
+          # TODO: have Cache determine when checksums may be invalid based on
+          # file size and/or timestamp.
+          cache.clear_checksum_cache!
+
+          if job
+            result = run_builder(job[:builder],
+                                 job[:target],
+                                 job[:sources],
+                                 cache,
+                                 job[:vars],
+                                 allow_delayed_execution: true,
+                                 setup_info: job[:setup_info])
+            unless result
+              raise BuildError.new("Failed to build #{job[:target]}")
             end
           end
-        end
-        begin
-          targets.each_key do |target|
-            process_target.call(target)
+
+          completed_tcs = Set.new
+          # First do a non-blocking wait to pick up any threads that have
+          # completed since last time.
+          while tc = wait_for_threaded_commands(nonblock: true)
+            completed_tcs << tc
           end
-        ensure
-          cache.write
+
+          # If needed, do a blocking wait.
+          if (completed_tcs.empty? and job.nil?) or @threaded_commands.size >= Rscons.n_threads
+            completed_tcs << wait_for_threaded_commands
+          end
+
+          # Process all completed {ThreadedCommand} objects.
+          completed_tcs.each do |tc|
+            result = finalize_builder(tc)
+            if result
+              @build_hooks[:post].each do |build_hook_block|
+                build_hook_block.call(tc.build_operation)
+              end
+            else
+              unless @echo == :command
+                $stdout.puts "Failed command was: #{command_to_s(tc.command)}"
+              end
+              raise BuildError.new("Failed to build #{tc.build_operation[:target]}")
+            end
+          end
+
         end
+      ensure
+        cache.write
       end
     end
 
@@ -321,7 +346,7 @@ module Rscons
     #
     # @return [void]
     def clear_targets
-      @targets = {}
+      @job_set.clear!
     end
 
     # Expand a construction variable reference.
@@ -354,11 +379,8 @@ module Rscons
     #
     # @return [true,false,nil] Return value from Kernel.system().
     def execute(short_desc, command, options = {})
-      print_command = proc do
-        puts command.map { |c| c =~ /\s/ ? "'#{c}'" : c }.join(' ')
-      end
       if @echo == :command
-        print_command.call
+        puts command_to_s(command)
       elsif @echo == :short
         puts short_desc
       end
@@ -366,8 +388,7 @@ module Rscons
       options_args = options[:options] ? [options[:options]] : []
       system(*env_args, *Rscons.command_executer, *command, *options_args).tap do |result|
         unless result or @echo == :command
-          $stdout.write "Failed command was: "
-          print_command.call
+          $stdout.puts "Failed command was: #{command_to_s(command)}"
         end
       end
     end
@@ -389,7 +410,7 @@ module Rscons
         sources = Array(sources)
         builder = @builders[method.to_s]
         build_target = builder.create_build_target(env: self, target: target, sources: sources)
-        add_target(build_target.to_s, builder, sources, vars, rest)
+        add_target(build_target.to_s, builder, sources, vars || {}, rest)
         build_target
       else
         super
@@ -402,17 +423,27 @@ module Rscons
     # @param builder [Builder] The {Builder} to use to build the target.
     # @param sources [Array<String>] Source file name(s).
     # @param vars [Hash] Construction variable overrides.
-    # @param args [Object] Any extra arguments passed to the {Builder}.
+    # @param args [Object] Deprecated; unused.
     #
     # @return [void]
     def add_target(target, builder, sources, vars, args)
-      @targets[target] ||= []
-      @targets[target] << {
+      target = expand_path(target) if @build_root
+      target = expand_varref(target)
+      sources = sources.map do |source|
+        source = expand_path(source) if @build_root
+        expand_varref(source)
+      end.flatten
+      setup_info = builder.setup(
+        target: target,
+        sources: sources,
+        env: self,
+        vars: vars)
+      @job_set.add_job(
         builder: builder,
+        target: target,
         sources: sources,
         vars: vars,
-        args: args,
-      }
+        setup_info: setup_info)
     end
 
     # Manually record a given target as depending on the specified files.
@@ -428,6 +459,42 @@ module Rscons
       @user_deps[target] = (@user_deps[target] + user_deps).uniq
     end
 
+    # Manually record the given target(s) as needing to be built after the
+    # given prerequisite(s).
+    #
+    # For example, consider a builder registered to generate gen.c which also
+    # generates gen.h as a side-effect. If program.c includes gen.h, then it
+    # should not be compiled before gen.h has been generated. When using
+    # multiple threads to build, Rscons may attempt to compile program.c before
+    # gen.h has been generated because it does not know that gen.h will be
+    # generated along with gen.c. One way to prevent that situation would be
+    # to first process the Environment with just the code-generation builders
+    # in place and then register the compilation builders. Another way is to
+    # use this method to record that a certain target should not be built until
+    # another has completed. For example, for the situation previously
+    # described:
+    #   env.build_after("program.o", "gen.c")
+    #
+    # @since 1.10.0
+    #
+    # @param targets [String, Array<String>]
+    #   Target files to wait to build until the prerequisites are finished
+    #   building.
+    # @param prerequisites [String, Array<String>]
+    #   Files that must be built before building the specified targets.
+    #
+    # @return [void]
+    def build_after(targets, prerequisites)
+      targets = Array(targets)
+      prerequisites = Array(prerequisites)
+      targets.each do |target|
+        @registered_build_dependencies[target] ||= Set.new
+        prerequisites.each do |prerequisite|
+          @registered_build_dependencies[target] << prerequisite
+        end
+      end
+    end
+
     # Return the list of user dependencies for a given target.
     #
     # @param target [String] Target file name.
@@ -444,6 +511,8 @@ module Rscons
     #
     # This method is used internally by Rscons builders.
     #
+    # @deprecated Use {#register_builds} instead.
+    #
     # @param sources [Array<String>] List of source files to build.
     # @param suffixes [Array<String>]
     #   List of suffixes to try to convert source files into.
@@ -471,6 +540,49 @@ module Rscons
       end
     end
 
+    # Find and register builders to build source files into files containing
+    # one of the suffixes given by suffixes.
+    #
+    # This method is used internally by Rscons builders. It should be called
+    # from the builder's #setup method.
+    #
+    # @since 1.10.0
+    #
+    # @param target [String]
+    #   The target that depends on these builds.
+    # @param sources [Array<String>]
+    #   List of source file(s) to build.
+    # @param suffixes [Array<String>]
+    #   List of suffixes to try to convert source files into.
+    # @param vars [Hash]
+    #   Extra variables to pass to the builders.
+    #
+    # @return [Array<String>]
+    #   List of the output file name(s).
+    def register_builds(target, sources, suffixes, vars)
+      @registered_build_dependencies[target] ||= Set.new
+      sources.map do |source|
+        if source.end_with?(*suffixes)
+          source
+        else
+          output_fname = nil
+          suffixes.each do |suffix|
+            attempt_output_fname = get_build_fname(source, suffix)
+            builder = @builders.values.find do |builder|
+              builder.produces?(attempt_output_fname, source, self)
+            end
+            if builder
+              output_fname = attempt_output_fname
+              self.__send__(builder.name, output_fname, source, vars)
+              @registered_build_dependencies[target] << output_fname
+              break
+            end
+          end
+          output_fname or raise "Could not find a builder for #{source.inspect}."
+        end
+      end
+    end
+
     # Invoke a builder to build the given target based on the given sources.
     #
     # @param builder [Builder] The Builder to use.
@@ -478,25 +590,66 @@ module Rscons
     # @param sources [Array<String>] List of source files.
     # @param cache [Cache] The Cache.
     # @param vars [Hash] Extra variables to pass to the builder.
+    # @param options [Hash]
+    #   @since 1.10.0
+    #   Options.
+    # @option options [Boolean] :allow_delayed_execution
+    #   @since 1.10.0
+    #   Allow a threaded command to be scheduled but not yet completed before
+    #   this method returns.
+    # @option options [Object] :setup_info
+    #   Arbitrary builder info returned by Builder#setup.
     #
     # @return [String,false] Return value from the {Builder}'s +run+ method.
-    def run_builder(builder, target, sources, cache, vars)
+    def run_builder(builder, target, sources, cache, vars, options = {})
       vars = @varset.merge(vars)
+      build_operation = {
+        builder: builder,
+        target: target,
+        sources: sources,
+        cache: cache,
+        env: self,
+        vars: vars,
+        setup_info: options[:setup_info]
+      }
       call_build_hooks = lambda do |sec|
         @build_hooks[sec].each do |build_hook_block|
-          build_operation = {
-            builder: builder,
-            target: target,
-            sources: sources,
-            vars: vars,
-            env: self,
-          }
           build_hook_block.call(build_operation)
         end
       end
+
+      # Invoke pre-build hooks.
       call_build_hooks[:pre]
-      rv = builder.run(target, sources, cache, self, vars)
-      call_build_hooks[:post] if rv
+
+      # Call the builder's #run method.
+      if builder.method(:run).arity == 5
+        rv = builder.run(target, sources, cache, self, vars)
+      else
+        rv = builder.run(build_operation)
+      end
+
+      if rv.is_a?(ThreadedCommand)
+        # Store the build operation so the post-build hooks can be called
+        # with it when the threaded command completes.
+        rv.build_operation = build_operation
+        start_threaded_command(rv)
+        unless options[:allow_delayed_execution]
+          # Delayed command execution is not allowed, so we need to execute
+          # the command and finalize the builder now.
+          tc = wait_for_threaded_commands(which: [rv])
+          rv = finalize_builder(tc)
+          if rv
+            call_build_hooks[:post]
+          else
+            unless @echo == :command
+              $stdout.puts "Failed command was: #{command_to_s(tc.command)}"
+            end
+          end
+        end
+      else
+        call_build_hooks[:post] if rv
+      end
+
       rv
     end
 
@@ -676,26 +829,102 @@ module Rscons
 
     private
 
-    # Expand target and source paths before invoking builders.
+    # Start a threaded command in a new thread.
     #
-    # This method expand construction variable references in the target and
-    # source file names before passing them to the builder. It also expands
-    # "^/" prefixes to the Environment's build root if a build root is defined.
+    # @param tc [ThreadedCommand]
+    #   The ThreadedCommand to start.
     #
     # @return [void]
-    def expand_paths!
-      @targets = @targets.reduce({}) do |result, (target, target_params_list)|
-        target = expand_path(target) if @build_root
-        target = expand_varref(target)
-        result[target] = target_params_list.map do |target_params|
-          sources = target_params[:sources].map do |source|
-            source = expand_path(source) if @build_root
-            expand_varref(source)
-          end.flatten
-          target_params.merge(sources: sources)
+    def start_threaded_command(tc)
+      if @echo == :command
+        puts command_to_s(tc.command)
+      elsif @echo == :short
+        if tc.short_description
+          puts tc.short_description
         end
-        result
       end
+
+      env_args = tc.system_env ? [tc.system_env] : []
+      options_args = tc.system_options ? [tc.system_options] : []
+      system_args = [*env_args, *Rscons.command_executer, *tc.command, *options_args]
+
+      tc.thread = Thread.new do
+        system(*system_args)
+      end
+      @threaded_commands << tc
+    end
+
+    # Wait for threaded commands to complete.
+    #
+    # @param options [Hash]
+    #   Options.
+    # @option options [Set<ThreadedCommand>, Array<ThreadedCommand>] :which
+    #   Which {ThreadedCommand} objects to wait for. If not specified, this
+    #   method will wait for any.
+    # @option options [Boolean] :nonblock
+    #   Set to true to not block.
+    #
+    # @return [ThreadedCommand, nil]
+    #   The {ThreadedCommand} object that is finished.
+    def wait_for_threaded_commands(options = {})
+      options[:which] ||= @threaded_commands
+      threads = options[:which].map(&:thread)
+      if finished_thread = find_finished_thread(threads, options[:nonblock])
+        threaded_command = @threaded_commands.find do |tc|
+          tc.thread == finished_thread
+        end
+        @threaded_commands.delete(threaded_command)
+        threaded_command
+      end
+    end
+
+    # Check if any of the requested threads are finished.
+    #
+    # @param threads [Array<Thread>]
+    #   The threads to check.
+    # @param nonblock [Boolean]
+    #   Whether to be non-blocking. If true, nil will be returned if no thread
+    #   is finished. If false, the method will wait until one of the threads
+    #   is finished.
+    #
+    # @return [Thread, nil]
+    #   The finished thread, if any.
+    def find_finished_thread(threads, nonblock)
+      if nonblock
+        threads.find do |thread|
+          !thread.alive?
+        end
+      else
+        if threads.empty?
+          raise "No threads to wait for"
+        end
+        ThreadsWait.new(*threads).next_wait
+      end
+    end
+
+    # Return a string representation of a command.
+    #
+    # @param command [Array<String>]
+    #   The command.
+    #
+    # @return [String]
+    #   The string representation of the command.
+    def command_to_s(command)
+      command.map { |c| c =~ /\s/ ? "'#{c}'" : c }.join(' ')
+    end
+
+    # Call a builder's #finalize method after a ThreadedCommand terminates.
+    #
+    # @param tc [ThreadedCommand]
+    #   The ThreadedCommand returned from the builder's #run method.
+    #
+    # @return [String, false]
+    #   Result of Builder#finalize.
+    def finalize_builder(tc)
+      tc.build_operation[:builder].finalize(
+        tc.build_operation.merge(
+          command_status: tc.thread.value,
+          tc: tc))
     end
 
     # Parse dependencies for a given target from a Makefile.

lib/rscons/job_set.rb

@@ -0,0 +1,93 @@
+require "set"
+
+module Rscons
+  # Class to keep track of a set of jobs that need to be performed.
+  class JobSet
+
+    # Create a JobSet
+    #
+    # @param build_dependencies [Hash]
+    #   Hash mapping targets to a set of build dependencies. A job will not be
+    #   returned as ready to run if any of its dependencies are still building.
+    def initialize(build_dependencies)
+      @jobs = {}
+      @build_dependencies = build_dependencies
+    end
+
+    # Add a job to the JobSet.
+    #
+    # @param options [Hash]
+    #   Options.
+    # @option options [Symbol, String] :target
+    #   Build target name.
+    # @option options [Builder] :builder
+    #   The {Builder} to use to build the target.
+    # @option options [Array<String>] :sources
+    #   Source file name(s).
+    # @option options [Hash] :vars
+    #   Construction variable overrides.
+    def add_job(options)
+      # We allow multiple jobs to be registered per target for cases like:
+      #   env.Directory("dest")
+      #   env.Install("dest", "bin")
+      #   env.Install("dest", "share")
+      @jobs[options[:target]] ||= []
+      @jobs[options[:target]] << options
+    end
+
+    # Get the next job that is ready to run from the JobSet.
+    #
+    # This method will remove the job from the JobSet.
+    #
+    # @param targets_still_building [Array<String>]
+    #   Targets that are not finished building. This is used to avoid returning
+    #   a job as available to run if it depends on one of the targets that are
+    #   still building as a source.
+    #
+    # @return [nil, Hash]
+    #   The next job to run.
+    def get_next_job_to_run(targets_still_building)
+      @jobs.keys.each do |target|
+        skip = false
+        (@jobs[target][0][:sources] + (@build_dependencies[target] || []).to_a).each do |src|
+          if @jobs.include?(src)
+            # Skip this target because it depends on another target later in
+            # the job set.
+            skip = true
+            break
+          end
+          if targets_still_building.include?(src)
+            # Skip this target because it depends on another target that is
+            # still being built.
+            skip = true
+            break
+          end
+        end
+        next if skip
+        job = @jobs[target][0]
+        if @jobs[target].size > 1
+          @jobs[target].slice!(0)
+        else
+          @jobs.delete(target)
+        end
+        return job
+      end
+
+      nil
+    end
+
+    # Remove all jobs from the JobSet.
+    def clear!
+      @jobs.clear
+    end
+
+    # Get the JobSet size.
+    #
+    # @return [Integer]
+    #   JobSet size.
+    def size
+      @jobs.size
+    end
+
+  end
+end

lib/rscons/threaded_command.rb

@@ -0,0 +1,63 @@
+module Rscons
+  # If a builder returns an instance of this class from its #run method, then
+  # Rscons will execute the command specified in a thread and allow other
+  # builders to continue executing in parallel.
+  class ThreadedCommand
+
+    # @return [Array<String>]
+    #   The command to execute.
+    attr_reader :command
+
+    # @return [Object]
+    #   Arbitrary object to store builder-specific info. This object value will
+    #   be passed back into the builder's #finalize method.
+    attr_reader :builder_info
+
+    # @return [String]
+    #   Short description of the command. This will be printed to standard
+    #   output if the Environment's echo mode is :short.
+    attr_reader :short_description
+
+    # @return [Hash]
+    #   Environment Hash to pass to Kernel#system.
+    attr_reader :system_env
+
+    # @return [Hash]
+    #   Options Hash to pass to Kernel#system.
+    attr_reader :system_options
+
+    # @return [Hash]
+    #   Field for Rscons to store the build operation while this threaded
+    #   command is executing.
+    attr_accessor :build_operation
+
+    # @return [Thread]
+    #   The thread waiting on this command to terminate.
+    attr_accessor :thread
+
+    # Create a ThreadedCommand object.
+    #
+    # @param command [Array<String>]
+    #   The command to execute.
+    # @param options [Hash]
+    #   Optional parameters.
+    # @option options [Object] :builder_info
+    #   Arbitrary object to store builder-specific info. This object value will
+    #   be passed back into the builder's #finalize method.
+    # @option options [String] :short_description
+    #   Short description of the command. This will be printed to standard
+    #   output if the Environment's echo mode is :short.
+    # @option options [Hash] :system_env
+    #   Environment Hash to pass to Kernel#system.
+    # @option options [Hash] :system_options
+    #   Options Hash to pass to Kernel#system.
+    def initialize(command, options = {})
+      @command = command
+      @builder_info = options[:builder_info]
+      @short_description = options[:short_description]
+      @system_env = options[:system_env]
+      @system_options = options[:system_options]
+    end
+
+  end
+end

spec/rscons/builder_spec.rb

@@ -2,7 +2,7 @@ module Rscons
   describe Builder do
     describe "#run" do
       it "raises an error if called directly and not through a subclass" do
-        expect{subject.run(:target, :sources, :cache, :env, :vars)}.to raise_error /This method must be overridden in a subclass/
+        expect{subject.run({})}.to raise_error /This method must be overridden in a subclass/
       end
     end
   end

spec/rscons/builders/cfile_spec.rb

@@ -1,28 +0,0 @@
-module Rscons
-  module Builders
-    describe CFile do
-      let(:env) {Environment.new}
-      subject {CFile.new}
-
-      it "invokes bison to create a .c file from a .y file" do
-        expect(subject).to receive(:standard_build).with("YACC parser.c", "parser.c", ["bison", "-d", "-o", "parser.c", "parser.y"], ["parser.y"], env, :cache)
-        subject.run("parser.c", ["parser.y"], :cache, env, {})
-      end
-
-      it "invokes a custom lexer to create a .cc file from a .ll file" do
-        env["LEX"] = "custom_lex"
-        expect(subject).to receive(:standard_build).with("LEX lexer.cc", "lexer.cc", ["custom_lex", "-o", "lexer.cc", "parser.ll"], ["parser.ll"], env, :cache)
-        subject.run("lexer.cc", ["parser.ll"], :cache, env, {})
-      end
-
-      it "supports overriding construction variables" do
-        expect(subject).to receive(:standard_build).with("LEX lexer.c", "lexer.c", ["hi", "parser.l"], ["parser.l"], env, :cache)
-        subject.run("lexer.c", ["parser.l"], :cache, env, "LEX_CMD" => ["hi", "${_SOURCES}"])
-      end
-
-      it "raises an error when an unknown source file is specified" do
-        expect {subject.run("file.c", ["foo.bar"], :cache, env, {})}.to raise_error /Unknown source file .foo.bar. for CFile builder/
-      end
-    end
-  end
-end

spec/rscons/builders/command_spec.rb

@@ -1,18 +0,0 @@
-module Rscons
-  module Builders
-    describe Command do
-
-      let(:command) { ['pandoc', '-fmarkdown', '-thtml', '-o${_TARGET}', '${_SOURCES}'] }
-      let(:env) {Environment.new}
-      subject {Command.new}
-
-      it "invokes the command to build the target" do
-        expected_cmd = ['pandoc', '-fmarkdown', '-thtml', '-ofoo.html', 'foo.md']
-        expect(subject).to receive(:standard_build).with("PANDOC foo.html", "foo.html", expected_cmd, ["foo.md"], env, :cache)
-        subject.run("foo.html", ["foo.md"], :cache, env,
-                    "CMD" => command, "CMD_DESC" => "PANDOC")
-      end
-
-    end
-  end
-end

spec/rscons/builders/disassemble_spec.rb

@@ -1,17 +0,0 @@
-module Rscons
-  module Builders
-    describe Disassemble do
-      let(:env) {Environment.new}
-      subject {Disassemble.new}
-
-      it "supports overriding DISASM_CMD construction variable" do
-        cache = "cache"
-        allow(cache).to receive(:up_to_date?) { false }
-        allow(cache).to receive(:mkdir_p) { }
-        allow(cache).to receive(:register_build) { }
-        expect(env).to receive(:execute).with("Disassemble a_file.txt", ["my_disasm", "a_file.exe"], anything).and_return(true)
-        subject.run("a_file.txt", ["a_file.exe"], cache, env, "DISASM_CMD" => ["my_disasm", "${_SOURCES}"])
-      end
-    end
-  end
-end

spec/rscons/builders/library_spec.rb

@@ -1,18 +0,0 @@
-module Rscons
-  module Builders
-    describe Library do
-      let(:env) {Environment.new}
-      subject {Library.new}
-
-      it "supports overriding AR construction variable" do
-        expect(subject).to receive(:standard_build).with("AR prog.a", "prog.a", ["sp-ar", "rcs", "prog.a", "prog.o"], ["prog.o"], env, :cache)
-        subject.run("prog.a", ["prog.o"], :cache, env, "AR" => "sp-ar")
-      end
-
-      it "supports overriding ARCMD construction variable" do
-        expect(subject).to receive(:standard_build).with("AR prog.a", "prog.a", ["special", "AR!", "prog.o"], ["prog.o"], env, :cache)
-        subject.run("prog.a", ["prog.o"], :cache, env, "ARCMD" => ["special", "AR!", "${_SOURCES}"])
-      end
-    end
-  end
-end

spec/rscons/builders/object_spec.rb

@@ -1,35 +0,0 @@
-module Rscons
-  module Builders
-    describe Object do
-      let(:env) {Environment.new}
-      let(:cache) {double(Cache)}
-      subject {Object.new}
-
-      it "supports overriding CCCMD construction variable" do
-        expect(cache).to receive(:up_to_date?).and_return(false)
-        expect(cache).to receive(:mkdir_p)
-        expect(FileUtils).to receive(:rm_f)
-        expect(env).to receive(:execute).with("CC mod.o", ["llc", "mod.c"]).and_return(true)
-        expect(File).to receive(:exists?).and_return(false)
-        expect(cache).to receive(:register_build)
-
-        subject.run("mod.o", ["mod.c"], cache, env, "CCCMD" => ["llc", "${_SOURCES}"])
-      end
-
-      it "supports overriding DEPFILESUFFIX construction variable" do
-        expect(cache).to receive(:up_to_date?).and_return(false)
-        expect(cache).to receive(:mkdir_p)
-        expect(FileUtils).to receive(:rm_f)
-        expect(env).to receive(:execute).with(anything, %w[gcc -c -o f.o -MMD -MF f.d in.c]).and_return(true)
-        expect(File).to receive(:exists?).with("f.d").and_return(false)
-        expect(cache).to receive(:register_build)
-
-        subject.run("f.o", ["in.c"], cache, env, "DEPFILESUFFIX" => ".d")
-      end
-
-      it "raises an error when given a source file with an unknown suffix" do
-        expect { subject.run("mod.o", ["mod.xyz"], :cache, env, {}) }.to raise_error /unknown input file type: "mod.xyz"/
-      end
-    end
-  end
-end

spec/rscons/builders/preprocess_spec.rb

@@ -1,48 +0,0 @@
-module Rscons
-  module Builders
-    describe Preprocess do
-
-      let(:env) {Environment.new}
-      subject {Preprocess.new}
-
-      it "supports overriding CC construction variable" do
-        cache = double(Cache)
-        command = %w[my_cpp -E -MMD -MF module.mf -o module.pp module.c]
-        expect(cache).to receive(:up_to_date?).with("module.pp", command, %w[module.c], env).and_return(false)
-        expect(cache).to receive(:mkdir_p).with(".")
-        expect(env).to receive(:execute).with("Preprocess module.pp", command).and_return(true)
-        expect(File).to receive(:exists?).with("module.mf").and_return(true)
-        expect(Environment).to receive(:parse_makefile_deps).with("module.mf", nil).and_return(%w[module.c one.h two.h])
-        expect(FileUtils).to receive(:rm_f).with("module.mf")
-        expect(cache).to receive(:register_build).with("module.pp", command, %w[module.c one.h two.h], env)
-
-        expect(subject.run("module.pp", ["module.c"], cache, env, "CC" => "my_cpp")).to eq("module.pp")
-      end
-
-      it "supports overriding CPP_CMD construction variable" do
-        cache = double(Cache)
-        command = %w[my_cpp module.c]
-        expect(cache).to receive(:up_to_date?).with("module.pp", command, %w[module.c], env).and_return(false)
-        expect(cache).to receive(:mkdir_p).with(".")
-        expect(env).to receive(:execute).with("Preprocess module.pp", command).and_return(true)
-        expect(File).to receive(:exists?).with("module.mf").and_return(true)
-        expect(Environment).to receive(:parse_makefile_deps).with("module.mf", nil).and_return(%w[module.c one.h two.h])
-        expect(FileUtils).to receive(:rm_f).with("module.mf")
-        expect(cache).to receive(:register_build).with("module.pp", command, %w[module.c one.h two.h], env)
-
-        expect(subject.run("module.pp", ["module.c"], cache, env, "CPP_CMD" => ["my_cpp", "${_SOURCES}"])).to eq("module.pp")
-      end
-
-      it "returns false if executing the preprocessor fails" do
-        cache = double(Cache)
-        command = %w[gcc -E -MMD -MF module.mf -o module.pp module.c]
-        expect(cache).to receive(:up_to_date?).with("module.pp", command, %w[module.c], env).and_return(false)
-        expect(cache).to receive(:mkdir_p).with(".")
-        expect(env).to receive(:execute).with("Preprocess module.pp", command).and_return(false)
-
-        expect(subject.run("module.pp", ["module.c"], cache, env, {})).to eq(false)
-      end
-
-    end
-  end
-end

spec/rscons/builders/program_spec.rb

@@ -1,18 +0,0 @@
-module Rscons
-  module Builders
-    describe Program do
-      let(:env) {Environment.new}
-      subject {Program.new}
-
-      it "supports overriding CC construction variable" do
-        expect(subject).to receive(:standard_build).with("LD prog", "prog", ["sp-c++", "-o", "prog", "prog.o"], ["prog.o"], env, :cache)
-        subject.run("prog", ["prog.o"], :cache, env, "CC" => "sp-c++")
-      end
-
-      it "supports overriding LDCMD construction variable" do
-        expect(subject).to receive(:standard_build).with("LD prog.exe", "prog.exe", ["special", "LD!", "prog.o"], ["prog.o"], env, :cache)
-        subject.run("prog.exe", ["prog.o"], :cache, env, "LDCMD" => ["special", "LD!", "${_SOURCES}"])
-      end
-    end
-  end
-end

spec/rscons/cache_spec.rb

@@ -13,194 +13,6 @@ module Rscons
       end
     end
 
-    describe "#initialize" do
-      context "when corrupt" do
-        it "prints a warning and defaults to an empty hash" do
-          expect(JSON).to receive(:load).and_return("string")
-          expect($stderr).to receive(:puts).with(/Warning:.*was.corrupt/)
-          c = Cache.instance
-          c.send(:initialize!)
-          expect(c.instance_variable_get(:@cache).is_a?(Hash)).to be_truthy
-        end
-      end
-    end
-
-    describe "#clear" do
-      it "removes the cache file" do
-        expect(FileUtils).to receive(:rm_f).with(Cache::CACHE_FILE)
-        allow(JSON).to receive(:load) {{}}
-        Cache.instance.clear
-      end
-    end
-
-    describe "#write" do
-      it "fills in 'version' and write to file" do
-        cache = {}
-        fh = $stdout
-        expect(fh).to receive(:puts)
-        expect(File).to receive(:open).and_yield(fh)
-        build_from(cache).write
-        expect(cache["version"]).to eq Rscons::VERSION
-      end
-    end
-
-    describe "#up_to_date?" do
-      empty_env = "env"
-      before do
-        allow(empty_env).to receive(:get_user_deps) { nil }
-      end
-
-      it "returns false when target file does not exist" do
-        expect(File).to receive(:exists?).with("target").and_return(false)
-        expect(build_from({}).up_to_date?("target", "command", [], empty_env)).to be_falsey
-      end
-
-      it "returns false when target is not registered in the cache" do
-        expect(File).to receive(:exists?).with("target").and_return(true)
-        expect(build_from({}).up_to_date?("target", "command", [], empty_env)).to be_falsey
-      end
-
-      it "returns false when the target's checksum does not match" do
-        _cache = {"targets" => {"target" => {"checksum" => "abc"}}}
-        cache = build_from(_cache)
-        expect(File).to receive(:exists?).with("target").and_return(true)
-        expect(cache).to receive(:calculate_checksum).with("target").and_return("def")
-        expect(cache.up_to_date?("target", "command", [], empty_env)).to be_falsey
-      end
-
-      it "returns false when the build command has changed" do
-        _cache = {"targets" => {"target" => {"checksum" => "abc", "command" => Digest::MD5.hexdigest("old command".inspect)}}}
-        cache = build_from(_cache)
-        expect(File).to receive(:exists?).with("target").and_return(true)
-        expect(cache).to receive(:calculate_checksum).with("target").and_return("abc")
-        expect(cache.up_to_date?("target", "command", [], empty_env)).to be_falsey
-      end
-
-      it "returns false when there is a new dependency" do
-        _cache = {"targets" => {"target" => {"checksum" => "abc",
-                                         "command" => Digest::MD5.hexdigest("command".inspect),
-                                         "deps" => [{"fname" => "dep.1"}]}}}
-        cache = build_from(_cache)
-        expect(File).to receive(:exists?).with("target").and_return(true)
-        expect(cache).to receive(:calculate_checksum).with("target").and_return("abc")
-        expect(cache.up_to_date?("target", "command", ["dep.1", "dep.2"], empty_env)).to be_falsey
-      end
-
-      it "returns false when a dependency's checksum has changed" do
-        _cache = {"targets" => {"target" => {"checksum" => "abc",
-                                         "command" => Digest::MD5.hexdigest("command".inspect),
-                                         "deps" => [{"fname" => "dep.1",
-                                                 "checksum" => "dep.1.chk"},
-                                                {"fname" => "dep.2",
-                                                 "checksum" => "dep.2.chk"},
-                                                {"fname" => "extra.dep",
-                                                 "checksum" => "extra.dep.chk"}],
-                                         "user_deps" => []}}}
-        cache = build_from(_cache)
-        expect(File).to receive(:exists?).with("target").and_return(true)
-        expect(cache).to receive(:calculate_checksum).with("target").and_return("abc")
-        expect(cache).to receive(:calculate_checksum).with("dep.1").and_return("dep.1.chk")
-        expect(cache).to receive(:calculate_checksum).with("dep.2").and_return("dep.2.changed")
-        expect(cache.up_to_date?("target", "command", ["dep.1", "dep.2"], empty_env)).to be_falsey
-      end
-
-      it "returns false with strict_deps=true when cache has an extra dependency" do
-        _cache = {"targets" => {"target" => {"checksum" => "abc",
-                                         "command" => Digest::MD5.hexdigest("command".inspect),
-                                         "deps" => [{"fname" => "dep.1",
-                                                 "checksum" => "dep.1.chk"},
-                                                {"fname" => "dep.2",
-                                                 "checksum" => "dep.2.chk"},
-                                                {"fname" => "extra.dep",
-                                                 "checksum" => "extra.dep.chk"}],
-                                         "user_deps" => []}}}
-        cache = build_from(_cache)
-        expect(File).to receive(:exists?).with("target").and_return(true)
-        expect(cache).to receive(:calculate_checksum).with("target").and_return("abc")
-        expect(cache.up_to_date?("target", "command", ["dep.1", "dep.2"], empty_env, strict_deps: true)).to be_falsey
-      end
-
-      it "returns false when there is a new user dependency" do
-        _cache = {"targets" => {"target" => {"checksum" => "abc",
-                                         "command" => Digest::MD5.hexdigest("command".inspect),
-                                         "deps" => [{"fname" => "dep.1"}],
-                                         "user_deps" => []}}}
-        cache = build_from(_cache)
-        env = "env"
-        expect(env).to receive(:get_user_deps).with("target").and_return(["file.ld"])
-        expect(File).to receive(:exists?).with("target").and_return(true)
-        expect(cache).to receive(:calculate_checksum).with("target").and_return("abc")
-        expect(cache.up_to_date?("target", "command", ["dep.1"], env)).to be_falsey
-      end
-
-      it "returns false when a user dependency checksum has changed" do
-        _cache = {"targets" => {"target" => {"checksum" => "abc",
-                                         "command" => Digest::MD5.hexdigest("command".inspect),
-                                         "deps" => [{"fname" => "dep.1",
-                                                 "checksum" => "dep.1.chk"},
-                                                {"fname" => "dep.2",
-                                                 "checksum" => "dep.2.chk"},
-                                                {"fname" => "extra.dep",
-                                                 "checksum" => "extra.dep.chk"}],
-                                         "user_deps" => [{"fname" => "user.dep",
-                                                      "checksum" => "user.dep.chk"}]}}}
-        cache = build_from(_cache)
-        env = "env"
-        expect(env).to receive(:get_user_deps).with("target").and_return(["user.dep"])
-        expect(File).to receive(:exists?).with("target").and_return(true)
-        expect(cache).to receive(:calculate_checksum).with("target").and_return("abc")
-        expect(cache).to receive(:calculate_checksum).with("dep.1").and_return("dep.1.chk")
-        expect(cache).to receive(:calculate_checksum).with("dep.2").and_return("dep.2.chk")
-        expect(cache).to receive(:calculate_checksum).with("extra.dep").and_return("extra.dep.chk")
-        expect(cache).to receive(:calculate_checksum).with("user.dep").and_return("INCORRECT")
-        expect(cache.up_to_date?("target", "command", ["dep.1", "dep.2"], env)).to be_falsey
-      end
-
-      it "returns true when no condition for false is met" do
-        _cache = {"targets" => {"target" => {"checksum" => "abc",
-                                         "command" => Digest::MD5.hexdigest("command".inspect),
-                                         "deps" => [{"fname" => "dep.1",
-                                                 "checksum" => "dep.1.chk"},
-                                                {"fname" => "dep.2",
-                                                 "checksum" => "dep.2.chk"},
-                                                {"fname" => "extra.dep",
-                                                 "checksum" => "extra.dep.chk"}],
-                                         "user_deps" => []}}}
-        cache = build_from(_cache)
-        expect(File).to receive(:exists?).with("target").and_return(true)
-        expect(cache).to receive(:calculate_checksum).with("target").and_return("abc")
-        expect(cache).to receive(:calculate_checksum).with("dep.1").and_return("dep.1.chk")
-        expect(cache).to receive(:calculate_checksum).with("dep.2").and_return("dep.2.chk")
-        expect(cache).to receive(:calculate_checksum).with("extra.dep").and_return("extra.dep.chk")
-        expect(cache.up_to_date?("target", "command", ["dep.1", "dep.2"], empty_env)).to be_truthy
-      end
-    end
-
-    describe "#register_build" do
-      it "stores the given information in the cache" do
-        _cache = {}
-        cache = build_from(_cache)
-        env = "env"
-        expect(env).to receive(:get_user_deps).with("the target").and_return(["user.dep"])
-        expect(cache).to receive(:calculate_checksum).with("the target").and_return("the checksum")
-        expect(cache).to receive(:calculate_checksum).with("dep 1").and_return("dep 1 checksum")
-        expect(cache).to receive(:calculate_checksum).with("dep 2").and_return("dep 2 checksum")
-        expect(cache).to receive(:calculate_checksum).with("user.dep").and_return("user.dep checksum")
-        cache.register_build("the target", "the command", ["dep 1", "dep 2"], env)
-        cached_target = cache.instance_variable_get(:@cache)["targets"]["the target"]
-        expect(cached_target).to_not be_nil
-        expect(cached_target["command"]).to eq Digest::MD5.hexdigest("the command".inspect)
-        expect(cached_target["checksum"]).to eq "the checksum"
-        expect(cached_target["deps"]).to eq [
-          {"fname" => "dep 1", "checksum" => "dep 1 checksum"},
-          {"fname" => "dep 2", "checksum" => "dep 2 checksum"},
-        ]
-        expect(cached_target["user_deps"]).to eq [
-          {"fname" => "user.dep", "checksum" => "user.dep checksum"},
-        ]
-      end
-    end
-
     describe "#targets" do
       it "returns a list of targets that are cached" do
         cache = {"targets" => {"t1" => {}, "t2" => {}, "t3" => {}}}

spec/rscons/environment_spec.rb

@@ -162,77 +162,6 @@ module Rscons
       end
     end
 
-    describe "#process" do
-      it "runs builders for all of the targets specified" do
-        env = Environment.new
-        env.Program("a.out", "main.c")
-
-        cache = "cache"
-        expect(Cache).to receive(:instance).and_return(cache)
-        expect(cache).to receive(:clear_checksum_cache!)
-        expect(env).to receive(:run_builder).with(anything, "a.out", ["main.c"], cache, {}).and_return(true)
-        expect(cache).to receive(:write)
-
-        env.process
-      end
-
-      it "builds dependent targets first" do
-        env = Environment.new
-        env.Program("a.out", "main.o")
-        env.Object("main.o", "other.cc")
-
-        cache = "cache"
-        expect(Cache).to receive(:instance).and_return(cache)
-        expect(cache).to receive(:clear_checksum_cache!)
-        expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}).and_return("main.o")
-        expect(env).to receive(:run_builder).with(anything, "a.out", ["main.o"], cache, {}).and_return("a.out")
-        expect(cache).to receive(:write)
-
-        env.process
-      end
-
-      it "raises a BuildError when building fails" do
-        env = Environment.new
-        env.Program("a.out", "main.o")
-        env.Object("main.o", "other.cc")
-
-        cache = "cache"
-        expect(Cache).to receive(:instance).and_return(cache)
-        expect(cache).to receive(:clear_checksum_cache!)
-        expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}).and_return(false)
-        expect(cache).to receive(:write)
-
-        expect { env.process }.to raise_error BuildError, /Failed.to.build.main.o/
-      end
-
-      it "writes the cache when the Builder raises an exception" do
-        env = Environment.new
-        env.Object("module.o", "module.c")
-
-        cache = "cache"
-        expect(Cache).to receive(:instance).and_return(cache)
-        expect(cache).to receive(:clear_checksum_cache!)
-        allow(env).to receive(:run_builder) do |builder, target, sources, cache, vars|
-          raise "Ruby exception thrown by builder"
-        end
-        expect(cache).to receive(:write)
-
-        expect { env.process }.to raise_error RuntimeError, /Ruby exception thrown by builder/
-      end
-    end
-
-    describe "#clear_targets" do
-      it "resets @targets to an empty hash" do
-        env = Environment.new
-        env.Program("a.out", "main.o")
-        expect(env.instance_variable_get(:@targets).keys).to eq(["a.out"])
-
-        env.clear_targets
-
-        expect(env.instance_variable_get(:@targets).keys).to eq([])
-      end
-    end
-
     describe "#build_command" do
       it "returns a command based on the variables in the Environment" do
         env = Environment.new
@@ -276,8 +205,7 @@ module Rscons
             env = Environment.new(echo: :short)
             expect(env).to receive(:puts).with("short desc")
             expect(env).to receive(:system).with(*Rscons.command_executer, "a", "command").and_return(false)
-            expect($stdout).to receive(:write).with("Failed command was: ")
-            expect(env).to receive(:puts).with("a command")
+            expect($stdout).to receive(:puts).with("Failed command was: a command")
             env.execute("short desc", ["a", "command"])
           end
         end
@@ -299,18 +227,6 @@ module Rscons
         expect {env.foobar}.to raise_error /undefined method .foobar./
       end
 
-      it "records the target when the target method is a known builder" do
-        env = Environment.new
-        expect(env.instance_variable_get(:@targets)).to eq({})
-        env.Object("target.o", ["src1.c", "src2.c"], var: "val")
-        target = env.instance_variable_get(:@targets)["target.o"]
-        expect(target).to_not be_nil
-        expect(target[0][:builder].is_a?(Builder)).to be_truthy
-        expect(target[0][:sources]).to eq ["src1.c", "src2.c"]
-        expect(target[0][:vars]).to eq({var: "val"})
-        expect(target[0][:args]).to eq []
-      end
-
       it "raises an error when vars is not a Hash" do
         env = Environment.new
         expect { env.Program("a.out", "main.c", "other") }.to raise_error /Unexpected construction variable set/
@@ -338,42 +254,6 @@ module Rscons
       end
     end
 
-    describe "#build_sources" do
-      class ABuilder < Builder
-        def produces?(target, source, env)
-          target =~ /\.ab_out$/ and source =~ /\.ab_in$/
-        end
-      end
-
-      it "finds and invokes a builder to produce output files with the requested suffixes" do
-        cache = "cache"
-        env = Environment.new
-        env.add_builder(ABuilder.new)
-        expect(env.builders["Object"]).to receive(:run).with("mod.o", ["mod.c"], cache, env, anything).and_return("mod.o")
-        expect(env.builders["ABuilder"]).to receive(:run).with("mod2.ab_out", ["mod2.ab_in"], cache, env, anything).and_return("mod2.ab_out")
-        expect(env.build_sources(["precompiled.o", "mod.c", "mod2.ab_in"], [".o", ".ab_out"], cache, {})).to eq ["precompiled.o", "mod.o", "mod2.ab_out"]
-      end
-    end
-
-    describe "#run_builder" do
-      it "modifies the construction variables using given build hooks and invokes the builder" do
-        env = Environment.new
-        env.add_build_hook do |build_op|
-          if build_op[:sources].first =~ %r{src/special}
-            build_op[:vars]["CFLAGS"] += ["-O3", "-DSPECIAL"]
-          end
-        end
-        allow(env.builders["Object"]).to receive(:run) do |target, sources, cache, env, vars|
-          expect(vars["CFLAGS"]).to eq []
-        end
-        env.run_builder(env.builders["Object"], "build/normal/module.o", ["src/normal/module.c"], "cache", {})
-        allow(env.builders["Object"]).to receive(:run) do |target, sources, cache, env, vars|
-          expect(vars["CFLAGS"]).to eq ["-O3", "-DSPECIAL"]
-        end
-        env.run_builder(env.builders["Object"], "build/special/module.o", ["src/special/module.c"], "cache", {})
-      end
-    end
-
     describe "#shell" do
       it "executes the given shell command and returns the results" do
         env = Environment.new
@@ -446,6 +326,13 @@ module Rscons
       end
     end
 
+    describe "#find_finished_thread" do
+      it "raises an error if called with nonblock=false and no threads to wait for" do
+        env = Environment.new
+        expect {env.__send__(:find_finished_thread, [], false)}.to raise_error /No threads to wait for/
+      end
+    end
+
     describe ".parse_makefile_deps" do
       it 'handles dependencies on one line' do
         expect(File).to receive(:read).with('makefile').and_return(<<EOS)

spec/rscons_spec.rb

@@ -122,4 +122,70 @@ describe Rscons do
       end
     end
   end
+
+  describe ".determine_n_threads" do
+    context "when specified by environment variable" do
+      before(:each) do
+        expect(ENV).to receive(:[]).with("RSCONS_NTHREADS").and_return("3")
+      end
+      it "returns the user-specified number of threads to use" do
+        expect(Rscons.__send__(:determine_n_threads)).to eq(3)
+      end
+    end
+
+    context "when not specified by environment variable" do
+      before(:each) do
+        expect(ENV).to receive(:[]).with("RSCONS_NTHREADS").and_return(nil)
+      end
+
+      context "on Linux" do
+        before(:each) do
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return("linux")
+        end
+        it "returns the number of processors from /proc/cpuinfo" do
+          expect(File).to receive(:read).with("/proc/cpuinfo").and_return(<<EOF)
+processor : 0
+processor   : 1
+EOF
+          expect(Rscons.__send__(:determine_n_threads)).to eq(2)
+        end
+      end
+
+      context "on Windows" do
+        before(:each) do
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return("mingw")
+        end
+        it "returns the number of logical processors that wmic reports" do
+          expect(Rscons).to receive(:`).with("wmic cpu get NumberOfLogicalProcessors /value").and_return("NumberOfLogicalProcessors=7")
+          expect(Rscons.__send__(:determine_n_threads)).to eq(7)
+        end
+      end
+
+      context "on Darwin" do
+        before(:each) do
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return("darwin")
+        end
+        it "returns the number of threads that sysctl reports" do
+          expect(Rscons).to receive(:`).with("sysctl -n hw.ncpu").and_return("6")
+          expect(Rscons.__send__(:determine_n_threads)).to eq(6)
+        end
+      end
+
+      context "on an unknown platform" do
+        before(:each) do
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return("other")
+        end
+        it "returns 1" do
+          expect(Rscons.__send__(:determine_n_threads)).to eq(1)
+        end
+      end
+
+      context "when an error occurs" do
+        it "returns 1" do
+          expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_raise("foo")
+          expect(Rscons.__send__(:determine_n_threads)).to eq(1)
+        end
+      end
+    end
+  end
 end

spec/spec_helper.rb

@@ -2,6 +2,14 @@ require "simplecov"
 
 SimpleCov.start do
   add_filter "/spec/"
+  add_filter "/.bundle/"
+  if ENV["partial_specs"]
+    command_name "RSpec-partial"
+  else
+    command_name "RSpec"
+  end
+  project_name "Rscons"
+  merge_timeout 3600
 end
 
 require "rscons"