From e1d8dfbab672d2f19ecf0c3af8f7e0f3f2385858 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Thu, 16 Feb 2017 18:09:02 -0500 Subject: [PATCH 01/69] add JobSet class --- lib/rscons.rb | 1 + lib/rscons/job_set.rb | 56 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 lib/rscons/job_set.rb diff --git a/lib/rscons.rb b/lib/rscons.rb index a7e8f08..c760982 100644 --- a/lib/rscons.rb +++ b/lib/rscons.rb @@ -2,6 +2,7 @@ require_relative "rscons/build_target" require_relative "rscons/builder" require_relative "rscons/cache" require_relative "rscons/environment" +require_relative "rscons/job_set" require_relative "rscons/varset" require_relative "rscons/version" diff --git a/lib/rscons/job_set.rb b/lib/rscons/job_set.rb new file mode 100644 index 0000000..76de53e --- /dev/null +++ b/lib/rscons/job_set.rb @@ -0,0 +1,56 @@ +require "set" + +module Rscons + # Class to keep track of a set of jobs that need to be performed. + class JobSet + + # Create a JobSet + def initialize + @jobs = {} + end + + # Add a job to the JobSet. + # + # @param target [Symbol, String] + # Build target name. + # @param builder [Builder] + # The {Builder} to use to build the target. + # @param sources [Array] + # Source file name(s). + # @param vars [Hash] + # Construction variable overrides. + def add_job(builder, target, sources, vars) + @jobs[target] = { + builder: builder, + target: target, + sources: sources, + vars: vars, + } + end + + # Get the next job that is ready to run from the JobSet. + # + # This method will remove the job from the JobSet. + # + # @return [nil, Hash] + # The next job to run. + def get_next_job_to_run + if @jobs.size > 0 + evaluated_targets = Set.new + attempt = lambda do |target| + evaluated_targets << target + @jobs[target][:sources].each do |src| + if @jobs.include?(src) and not evaluated_targets.include?(src) + return attempt[src] + end + end + job = @jobs[target].merge(target: target) + @jobs.delete(target) + return job + end + attempt[@jobs.first.first] + end + end + + end +end From 3601359c08325c459360094b7cfe1f864727cc74 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Thu, 16 Feb 2017 18:44:20 -0500 Subject: [PATCH 02/69] process jobs from the JobSet --- lib/rscons/environment.rb | 86 +++++++++++---------------------- lib/rscons/job_set.rb | 22 +++++++-- spec/build_tests_spec.rb | 11 +++++ spec/rscons/environment_spec.rb | 30 ++---------- 4 files changed, 61 insertions(+), 88 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 81cd63d..57712d6 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -38,7 +38,7 @@ module Rscons # when the block returns, the {#process} method is automatically called. def initialize(options = {}) @varset = VarSet.new - @targets = {} + @job_set = JobSet.new @user_deps = {} @builders = {} @build_dirs = [] @@ -280,40 +280,24 @@ module Rscons # # @return [void] def process - while @targets.size > 0 - expand_paths! - targets = @targets - @targets = {} - cache = Cache.instance - cache.clear_checksum_cache! - targets_processed = Set.new - process_target = proc do |target| - unless targets_processed.include?(target) - targets_processed << target - targets[target].each do |target_params| - target_params[:sources].each do |src| - if targets.include?(src) and not targets_processed.include?(src) - process_target.call(src) - end - end - result = run_builder(target_params[:builder], - target, - target_params[:sources], - cache, - target_params[:vars] || {}) - unless result - raise BuildError.new("Failed to build #{target}") - end - end + cache = Cache.instance + begin + while job = @job_set.get_next_job_to_run + expand_paths(job) + # TODO: have Cache determine when checksums may be invalid based on + # file size and/or timestamp. + cache.clear_checksum_cache! + result = run_builder(job[:builder], + job[:target], + job[:sources], + cache, + job[:vars]) + unless result + raise BuildError.new("Failed to build #{job[:target]}") end end - begin - targets.each_key do |target| - process_target.call(target) - end - ensure - cache.write - end + ensure + cache.write end end @@ -321,7 +305,7 @@ module Rscons # # @return [void] def clear_targets - @targets = {} + @job_set.clear! end # Expand a construction variable reference. @@ -389,7 +373,7 @@ module Rscons sources = Array(sources) builder = @builders[method.to_s] build_target = builder.create_build_target(env: self, target: target, sources: sources) - add_target(build_target.to_s, builder, sources, vars, rest) + add_target(build_target.to_s, builder, sources, vars || {}, rest) build_target else super @@ -402,17 +386,11 @@ module Rscons # @param builder [Builder] The {Builder} to use to build the target. # @param sources [Array] Source file name(s). # @param vars [Hash] Construction variable overrides. - # @param args [Object] Any extra arguments passed to the {Builder}. + # @param args [Object] Deprecated; unused. # # @return [void] def add_target(target, builder, sources, vars, args) - @targets[target] ||= [] - @targets[target] << { - builder: builder, - sources: sources, - vars: vars, - args: args, - } + @job_set.add_job(builder, target, sources, vars) end # Manually record a given target as depending on the specified files. @@ -676,26 +654,20 @@ module Rscons private - # Expand target and source paths before invoking builders. + # Expand target and source paths of a job before invoking the builder. # # This method expand construction variable references in the target and # source file names before passing them to the builder. It also expands # "^/" prefixes to the Environment's build root if a build root is defined. # # @return [void] - def expand_paths! - @targets = @targets.reduce({}) do |result, (target, target_params_list)| - target = expand_path(target) if @build_root - target = expand_varref(target) - result[target] = target_params_list.map do |target_params| - sources = target_params[:sources].map do |source| - source = expand_path(source) if @build_root - expand_varref(source) - end.flatten - target_params.merge(sources: sources) - end - result - end + def expand_paths(job) + job[:target] = expand_path(job[:target]) if @build_root + job[:target] = expand_varref(job[:target]) + job[:sources] = job[:sources].map do |source| + source = expand_path(source) if @build_root + expand_varref(source) + end.flatten end # Parse dependencies for a given target from a Makefile. diff --git a/lib/rscons/job_set.rb b/lib/rscons/job_set.rb index 76de53e..05c6726 100644 --- a/lib/rscons/job_set.rb +++ b/lib/rscons/job_set.rb @@ -20,7 +20,12 @@ module Rscons # @param vars [Hash] # Construction variable overrides. def add_job(builder, target, sources, vars) - @jobs[target] = { + # We allow multiple jobs to be registered per target for cases like: + # env.Directory("dest") + # env.Install("dest", "bin") + # env.Install("dest", "share") + @jobs[target] ||= [] + @jobs[target] << { builder: builder, target: target, sources: sources, @@ -39,18 +44,27 @@ module Rscons evaluated_targets = Set.new attempt = lambda do |target| evaluated_targets << target - @jobs[target][:sources].each do |src| + @jobs[target][0][:sources].each do |src| if @jobs.include?(src) and not evaluated_targets.include?(src) return attempt[src] end end - job = @jobs[target].merge(target: target) - @jobs.delete(target) + job = @jobs[target][0].merge(target: target) + if @jobs[target].size > 1 + @jobs[target].slice!(0) + else + @jobs.delete(target) + end return job end attempt[@jobs.first.first] end end + # Remove all jobs from the JobSet. + def clear! + @jobs.clear + end + end end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 98c9918..8baf883 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -966,4 +966,15 @@ EOF end end + context "Environment#clear_targets" do + it "clears registered targets" do + test_dir('header') + env = Rscons::Environment.new do |env| + env.Program('header', Dir['*.c']) + env.clear_targets + end + expect(lines).to eq [] + end + end + end diff --git a/spec/rscons/environment_spec.rb b/spec/rscons/environment_spec.rb index aabab74..802bc5b 100644 --- a/spec/rscons/environment_spec.rb +++ b/spec/rscons/environment_spec.rb @@ -169,7 +169,7 @@ module Rscons cache = "cache" expect(Cache).to receive(:instance).and_return(cache) - expect(cache).to receive(:clear_checksum_cache!) + allow(cache).to receive(:clear_checksum_cache!) expect(env).to receive(:run_builder).with(anything, "a.out", ["main.c"], cache, {}).and_return(true) expect(cache).to receive(:write) @@ -183,7 +183,7 @@ module Rscons cache = "cache" expect(Cache).to receive(:instance).and_return(cache) - expect(cache).to receive(:clear_checksum_cache!) + allow(cache).to receive(:clear_checksum_cache!) expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}).and_return("main.o") expect(env).to receive(:run_builder).with(anything, "a.out", ["main.o"], cache, {}).and_return("a.out") expect(cache).to receive(:write) @@ -198,7 +198,7 @@ module Rscons cache = "cache" expect(Cache).to receive(:instance).and_return(cache) - expect(cache).to receive(:clear_checksum_cache!) + allow(cache).to receive(:clear_checksum_cache!) expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}).and_return(false) expect(cache).to receive(:write) @@ -221,18 +221,6 @@ module Rscons end end - describe "#clear_targets" do - it "resets @targets to an empty hash" do - env = Environment.new - env.Program("a.out", "main.o") - expect(env.instance_variable_get(:@targets).keys).to eq(["a.out"]) - - env.clear_targets - - expect(env.instance_variable_get(:@targets).keys).to eq([]) - end - end - describe "#build_command" do it "returns a command based on the variables in the Environment" do env = Environment.new @@ -299,18 +287,6 @@ module Rscons expect {env.foobar}.to raise_error /undefined method .foobar./ end - it "records the target when the target method is a known builder" do - env = Environment.new - expect(env.instance_variable_get(:@targets)).to eq({}) - env.Object("target.o", ["src1.c", "src2.c"], var: "val") - target = env.instance_variable_get(:@targets)["target.o"] - expect(target).to_not be_nil - expect(target[0][:builder].is_a?(Builder)).to be_truthy - expect(target[0][:sources]).to eq ["src1.c", "src2.c"] - expect(target[0][:vars]).to eq({var: "val"}) - expect(target[0][:args]).to eq [] - end - it "raises an error when vars is not a Hash" do env = Environment.new expect { env.Program("a.out", "main.c", "other") }.to raise_error /Unexpected construction variable set/ From 551b8fa3656bf562ccd764c1bf84bcc66a696bfe Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Thu, 16 Feb 2017 19:08:03 -0500 Subject: [PATCH 03/69] add integration test to verify cache is written if a builder fails --- lib/rscons/cache.rb | 14 ++++++++++++-- spec/build_tests_spec.rb | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/lib/rscons/cache.rb b/lib/rscons/cache.rb index 50c5afb..9c71339 100644 --- a/lib/rscons/cache.rb +++ b/lib/rscons/cache.rb @@ -2,7 +2,6 @@ require "digest/md5" require "fileutils" require "json" require "set" -require "singleton" require "rscons/version" module Rscons @@ -51,7 +50,6 @@ module Rscons # }, # } class Cache - include Singleton # Name of the file to store cache information in CACHE_FILE = ".rsconscache" @@ -59,6 +57,18 @@ module Rscons # Prefix for phony cache entries. PHONY_PREFIX = ":PHONY:" + class << self + # Access the singleton instance. + def instance + @instance ||= Cache.new + end + + # Reset the cache (for unit/integration test purposes) + def reset! + @instance = nil + end + end + # Create a Cache object and load in the previous contents from the cache # file. def initialize diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 8baf883..ce2a91c 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -54,6 +54,7 @@ describe Rscons do def test_dir(build_test_directory) FileUtils.cp_r("build_tests/#{build_test_directory}", BUILD_TEST_RUN_DIR) Dir.chdir(BUILD_TEST_RUN_DIR) + @saved_stderr.reopen(".stderr") end def file_sub(fname) @@ -798,6 +799,38 @@ EOF ]) end + it "does not re-run previously successful builders if one fails" do + test_dir('simple') + File.open("two.c", "w") do |fh| + fh.puts("FOO") + end + expect do + Rscons::Environment.new do |env| + env.Program("simple", %w[simple.c two.c]) + end + end.to raise_error /Failed to build simple/ + result = lines + expect(result.size).to be > 2 + expect(result[0, 2]).to eq [ + "CC simple.o", + "CC two.o", + ] + expect(File.exists?("simple.o")).to be_truthy + expect(File.exists?("two.o")).to be_falsey + expect(File.exists?("two_sources#{Rscons::Environment.new["PROGSUFFIX"]}")).to be_falsey + + Rscons::Cache.reset! + + File.open("two.c", "w") {|fh|} + Rscons::Environment.new do |env| + env.Program("simple", %w[simple.c two.c]) + end + expect(lines).to eq [ + "CC two.o", + "LD simple", + ] + end + context "Directory builder" do it "creates the requested directory" do test_dir("simple") From 7cba8c84245b854324571810d50632cee10d5d80 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 10 May 2017 14:57:04 -0400 Subject: [PATCH 04/69] JobSet: target is already stored in job info --- lib/rscons/job_set.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rscons/job_set.rb b/lib/rscons/job_set.rb index 05c6726..64c65d6 100644 --- a/lib/rscons/job_set.rb +++ b/lib/rscons/job_set.rb @@ -49,7 +49,7 @@ module Rscons return attempt[src] end end - job = @jobs[target][0].merge(target: target) + job = @jobs[target][0] if @jobs[target].size > 1 @jobs[target].slice!(0) else From 695bf840924578768799aed99da6d94b8a4d6c08 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 10 May 2017 15:06:16 -0400 Subject: [PATCH 05/69] expand target and source paths when registering a builder --- lib/rscons/environment.rb | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 57712d6..05e5ad3 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -283,7 +283,6 @@ module Rscons cache = Cache.instance begin while job = @job_set.get_next_job_to_run - expand_paths(job) # TODO: have Cache determine when checksums may be invalid based on # file size and/or timestamp. cache.clear_checksum_cache! @@ -390,6 +389,12 @@ module Rscons # # @return [void] def add_target(target, builder, sources, vars, args) + target = expand_path(target) if @build_root + target = expand_varref(target) + sources = sources.map do |source| + source = expand_path(source) if @build_root + expand_varref(source) + end.flatten @job_set.add_job(builder, target, sources, vars) end @@ -654,22 +659,6 @@ module Rscons private - # Expand target and source paths of a job before invoking the builder. - # - # This method expand construction variable references in the target and - # source file names before passing them to the builder. It also expands - # "^/" prefixes to the Environment's build root if a build root is defined. - # - # @return [void] - def expand_paths(job) - job[:target] = expand_path(job[:target]) if @build_root - job[:target] = expand_varref(job[:target]) - job[:sources] = job[:sources].map do |source| - source = expand_path(source) if @build_root - expand_varref(source) - end.flatten - end - # Parse dependencies for a given target from a Makefile. # # This method is used internally by Rscons builders. From dab870854a1edcf0fd1492c81f581b44035d2be0 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 12 May 2017 13:50:45 -0400 Subject: [PATCH 06/69] add Rscons.n_threads --- lib/rscons.rb | 42 +++++++++++++++++++++++++++++ spec/rscons_spec.rb | 66 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/lib/rscons.rb b/lib/rscons.rb index c760982..2ef06d0 100644 --- a/lib/rscons.rb +++ b/lib/rscons.rb @@ -41,6 +41,10 @@ module Rscons class << self + # @return [Integer] + # The number of threads to use when scheduling subprocesses. + attr_accessor :n_threads + # Remove all generated files. # # @return [void] @@ -147,7 +151,45 @@ module Rscons @command_executer = val end + private + + # Determine the number of threads to use by default. + # + # @return [Integer] + # The number of threads to use by default. + def determine_n_threads + # If the user specifies the number of threads in the environment, then + # respect that. + if ENV["RSCONS_NTHREADS"] =~ /^(\d+)$/ + return $1.to_i + end + + # Otherwise try to figure out how many threads are available on the + # host hardware. + begin + case RbConfig::CONFIG["host_os"] + when /linux/ + return File.read("/proc/cpuinfo").scan(/^processor\s*:/).size + when /mswin|mingw/ + if `wmic cpu get NumberOfLogicalProcessors /value` =~ /NumberOfLogicalProcessors=(\d+)/ + return $1.to_i + end + when /darwin/ + if `sysctl -n hw.ncpu` =~ /(\d+)/ + return $1.to_i + end + end + rescue + end + + # If we can't figure it out, default to 1. + 1 + end + end + + @n_threads = determine_n_threads + end # Unbuffer $stdout diff --git a/spec/rscons_spec.rb b/spec/rscons_spec.rb index 6c6660b..6468f9e 100644 --- a/spec/rscons_spec.rb +++ b/spec/rscons_spec.rb @@ -122,4 +122,70 @@ describe Rscons do end end end + + describe ".determine_n_threads" do + context "when specified by environment variable" do + before(:each) do + expect(ENV).to receive(:[]).with("RSCONS_NTHREADS").and_return("3") + end + it "returns the user-specified number of threads to use" do + expect(Rscons.__send__(:determine_n_threads)).to eq(3) + end + end + + context "when not specified by environment variable" do + before(:each) do + expect(ENV).to receive(:[]).with("RSCONS_NTHREADS").and_return(nil) + end + + context "on Linux" do + before(:each) do + expect(RbConfig::CONFIG).to receive(:[]).with("host_os").and_return("linux") + end + it "returns the number of processors from /proc/cpuinfo" do + expect(File).to receive(:read).with("/proc/cpuinfo").and_return(< Date: Fri, 12 May 2017 16:16:07 -0400 Subject: [PATCH 07/69] add Builder#setup; add new Builder#run signature --- lib/rscons/builder.rb | 63 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/lib/rscons/builder.rb b/lib/rscons/builder.rb index 0cfeb69..210a550 100644 --- a/lib/rscons/builder.rb +++ b/lib/rscons/builder.rb @@ -56,13 +56,66 @@ module Rscons false end + # Set up a build operation using this builder. + # + # This method is called when a build target is registered using this + # builder. This method should not do any building, but should perform any + # setup needed and register any prerequisite build targets that need to be + # built before the target being requested here. + # + # If the builder needs no special setup, it does not need to override this + # method. If there is any information produced in this method that will be + # needed later in the build, it can be stored in the return value from this + # method, which will be passed to the {#run} method. + # + # @param target [String] + # Target file name. + # @param sources [Array] + # Source file name(s). + # @param env [Environment] + # The Environment executing the builder. + # @param vars [Hash,VarSet] + # Extra construction variables. + # + # @return [Object] + # Any object that the builder author wishes to be saved and passed back + # in to the {#run} method. + def setup(target, sources, env, vars) + end + # Run the builder to produce a build target. # - # @param target [String] Target file name. - # @param sources [Array] Source file name(s). - # @param cache [Cache] The Cache object. - # @param env [Environment] The Environment executing the builder. - # @param vars [Hash,VarSet] Extra construction variables. + # The run method supports two different signatures - an older signature + # with five separate arguments, and a newer one with one Hash argument. A + # builder author can use either signature, and Rscons will automatically + # determine which arguments to pass when invoking the run method based on + # the method's arity. + # + # @overload run(target, sources, cache, env, vars) + # @param target [String] + # Target file name. + # @param sources [Array] + # Source file name(s). + # @param cache [Cache] + # The Cache object. + # @param env [Environment] + # The Environment executing the builder. + # @param vars [Hash,VarSet] + # Extra construction variables. + # @overload run(options) + # @param options [Hash] Run options. + # @option options [String] :target + # Target file name. + # @option options [Array] :sources + # Source file name(s). + # @option options [Cache] :cache + # The Cache object. + # @option options [Environment] :env + # The Environment executing the builder. + # @option options [Hash,VarSet] :vars + # Extra construction variables. + # @option options [Object] :setup_info + # Whatever value was returned from this builder's {#setup} method call. # # @return [String,false] # Name of the target file on success or false on failure. From 9b6d2c5111433a5a7cddbc6368bce82a7b9df0f0 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 12 May 2017 16:16:25 -0400 Subject: [PATCH 08/69] add Environment#register_builds --- lib/rscons/environment.rb | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 05e5ad3..dfe798f 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -427,6 +427,8 @@ module Rscons # # This method is used internally by Rscons builders. # + # @deprecated Use {#register_builds} instead. + # # @param sources [Array] List of source files to build. # @param suffixes [Array] # List of suffixes to try to convert source files into. @@ -454,6 +456,43 @@ module Rscons end end + # Find and register builders to build source files into files containing + # one of the suffixes given by suffixes. + # + # This method is used internally by Rscons builders. It should be called + # from the builder's #setup method. + # + # @param sources [Array] + # List of source file(s) to build. + # @param suffixes [Array] + # List of suffixes to try to convert source files into. + # @param vars [Hash] + # Extra variables to pass to the builders. + # + # @return [Array] + # List of the output file name(s). + def register_builds(sources, suffixes, vars) + sources.map do |source| + if source.end_with?(*suffixes) + source + else + output_fname = nil + suffixes.each do |suffix| + attempt_output_fname = get_build_fname(source, suffix) + builder = @builders.values.find do |builder| + builder.produces?(attempt_output_fname, source, self) + end + if builder + output_fname = attempt_output_fname + self.__send__(builder.name, output_fname, source, vars) + break + end + end + output_fname or raise "Could not find a builder for #{source.inspect}." + end + end + end + # Invoke a builder to build the given target based on the given sources. # # @param builder [Builder] The Builder to use. From 19a00a7b84645fc4625020ef2a641d0dc0eef00c Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 12 May 2017 16:25:34 -0400 Subject: [PATCH 09/69] prepare to call new Builder#run interface --- lib/rscons/environment.rb | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index dfe798f..3cad39b 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -517,7 +517,22 @@ module Rscons end end call_build_hooks[:pre] - rv = builder.run(target, sources, cache, self, vars) + use_new_run_method_signature = + begin + builder.method(:run).arity == 1 + rescue NameError + false + end + if use_new_run_method_signature + rv = builder.run( + target: target, + sources: sources, + cache: cache, + env: self, + vars: vars) + else + rv = builder.run(target, sources, cache, self, vars) + end call_build_hooks[:post] if rv rv end From 9cfc0c20b7e75ae596070c021bcf3291a8f98ce1 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 12 May 2017 16:27:43 -0400 Subject: [PATCH 10/69] call Builder#setup --- lib/rscons/builder.rb | 12 +++++++----- lib/rscons/environment.rb | 5 +++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/rscons/builder.rb b/lib/rscons/builder.rb index 210a550..b13dd49 100644 --- a/lib/rscons/builder.rb +++ b/lib/rscons/builder.rb @@ -68,19 +68,21 @@ module Rscons # needed later in the build, it can be stored in the return value from this # method, which will be passed to the {#run} method. # - # @param target [String] + # @param options [Hash] + # Options. + # @option options [String] :target # Target file name. - # @param sources [Array] + # @option options [Array] :sources # Source file name(s). - # @param env [Environment] + # @option options [Environment] :env # The Environment executing the builder. - # @param vars [Hash,VarSet] + # @option options [Hash,VarSet] :vars # Extra construction variables. # # @return [Object] # Any object that the builder author wishes to be saved and passed back # in to the {#run} method. - def setup(target, sources, env, vars) + def setup(options) end # Run the builder to produce a build target. diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 3cad39b..82afdb4 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -395,6 +395,11 @@ module Rscons source = expand_path(source) if @build_root expand_varref(source) end.flatten + setup_info = builder.setup( + target: target, + sources: sources, + env: self, + vars: vars) @job_set.add_job(builder, target, sources, vars) end From d46dc2014c0f17f22173f2192e385c86f34c854d Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 12 May 2017 16:30:02 -0400 Subject: [PATCH 11/69] update JobSet interface to just take a Hash of job parameters --- lib/rscons/environment.rb | 6 +++++- lib/rscons/job_set.rb | 21 +++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 82afdb4..d85f525 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -400,7 +400,11 @@ module Rscons sources: sources, env: self, vars: vars) - @job_set.add_job(builder, target, sources, vars) + @job_set.add_job( + builder: builder, + target: target, + sources: sources, + vars: vars) end # Manually record a given target as depending on the specified files. diff --git a/lib/rscons/job_set.rb b/lib/rscons/job_set.rb index 64c65d6..333b17e 100644 --- a/lib/rscons/job_set.rb +++ b/lib/rscons/job_set.rb @@ -11,26 +11,23 @@ module Rscons # Add a job to the JobSet. # - # @param target [Symbol, String] + # @param options [Hash] + # Options. + # @option options [Symbol, String] :target # Build target name. - # @param builder [Builder] + # @option options [Builder] :builder # The {Builder} to use to build the target. - # @param sources [Array] + # @option options [Array] :sources # Source file name(s). - # @param vars [Hash] + # @option options [Hash] :vars # Construction variable overrides. - def add_job(builder, target, sources, vars) + def add_job(options) # We allow multiple jobs to be registered per target for cases like: # env.Directory("dest") # env.Install("dest", "bin") # env.Install("dest", "share") - @jobs[target] ||= [] - @jobs[target] << { - builder: builder, - target: target, - sources: sources, - vars: vars, - } + @jobs[options[:target]] ||= [] + @jobs[options[:target]] << options end # Get the next job that is ready to run from the JobSet. From 3e4897c04bd8b27f265f5b86d3924129c30e8076 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 12 May 2017 16:30:30 -0400 Subject: [PATCH 12/69] store setup_info in the job --- lib/rscons/environment.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index d85f525..0fbc213 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -404,7 +404,8 @@ module Rscons builder: builder, target: target, sources: sources, - vars: vars) + vars: vars, + setup_info: setup_info) end # Manually record a given target as depending on the specified files. From 7a31039e3583f69c4def0bb6f32795660879afd1 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 12 May 2017 21:07:25 -0400 Subject: [PATCH 13/69] add @since tags for new methods --- lib/rscons/builder.rb | 7 +++++++ lib/rscons/environment.rb | 2 ++ 2 files changed, 9 insertions(+) diff --git a/lib/rscons/builder.rb b/lib/rscons/builder.rb index b13dd49..9bf36ef 100644 --- a/lib/rscons/builder.rb +++ b/lib/rscons/builder.rb @@ -68,6 +68,8 @@ module Rscons # needed later in the build, it can be stored in the return value from this # method, which will be passed to the {#run} method. # + # @since 1.10.0 + # # @param options [Hash] # Options. # @option options [String] :target @@ -94,6 +96,7 @@ module Rscons # the method's arity. # # @overload run(target, sources, cache, env, vars) + # # @param target [String] # Target file name. # @param sources [Array] @@ -104,7 +107,11 @@ module Rscons # The Environment executing the builder. # @param vars [Hash,VarSet] # Extra construction variables. + # # @overload run(options) + # + # @since 1.10.0 + # # @param options [Hash] Run options. # @option options [String] :target # Target file name. diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 0fbc213..e5a57a9 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -472,6 +472,8 @@ module Rscons # This method is used internally by Rscons builders. It should be called # from the builder's #setup method. # + # @since 1.10.0 + # # @param sources [Array] # List of source file(s) to build. # @param suffixes [Array] From ef4f9882cd514a54dbe129d7f455a769b795baf2 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 12 May 2017 21:08:14 -0400 Subject: [PATCH 14/69] change Builder#run base class method to new signature --- lib/rscons/builder.rb | 2 +- spec/rscons/builder_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/rscons/builder.rb b/lib/rscons/builder.rb index 9bf36ef..ed8e4b6 100644 --- a/lib/rscons/builder.rb +++ b/lib/rscons/builder.rb @@ -128,7 +128,7 @@ module Rscons # # @return [String,false] # Name of the target file on success or false on failure. - def run(target, sources, cache, env, vars) + def run(options) raise "This method must be overridden in a subclass" end diff --git a/spec/rscons/builder_spec.rb b/spec/rscons/builder_spec.rb index 7515956..8e4ebc2 100644 --- a/spec/rscons/builder_spec.rb +++ b/spec/rscons/builder_spec.rb @@ -2,7 +2,7 @@ module Rscons describe Builder do describe "#run" do it "raises an error if called directly and not through a subclass" do - expect{subject.run(:target, :sources, :cache, :env, :vars)}.to raise_error /This method must be overridden in a subclass/ + expect{subject.run({})}.to raise_error /This method must be overridden in a subclass/ end end end From 6b8fda706d5f4bb9df97376f400dd9373cf6ebd6 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Mon, 15 May 2017 10:57:34 -0400 Subject: [PATCH 15/69] add ThreadedCommand class --- lib/rscons.rb | 1 + lib/rscons/threaded_command.rb | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 lib/rscons/threaded_command.rb diff --git a/lib/rscons.rb b/lib/rscons.rb index 2ef06d0..02f6e0e 100644 --- a/lib/rscons.rb +++ b/lib/rscons.rb @@ -3,6 +3,7 @@ require_relative "rscons/builder" require_relative "rscons/cache" require_relative "rscons/environment" require_relative "rscons/job_set" +require_relative "rscons/threaded_command" require_relative "rscons/varset" require_relative "rscons/version" diff --git a/lib/rscons/threaded_command.rb b/lib/rscons/threaded_command.rb new file mode 100644 index 0000000..e885d50 --- /dev/null +++ b/lib/rscons/threaded_command.rb @@ -0,0 +1,31 @@ +module Rscons + # If a builder returns an instance of this class from its #run method, then + # Rscons will execute the command specified in a thread and allow other + # builders to continue executing in parallel. + class ThreadedCommand + + # @return [Array] + # The command to execute. + attr_reader :command + + # @return [Object] + # Arbitrary object to store builder-specific info. This object value will + # be passed back into the builder's #finalize method. + attr_reader :builder_info + + # Create a ThreadedCommand object. + # + # @param command [Array] + # The command to execute. + # @param options [Hash] + # Optional parameters. + # @option options [Object] :builder_info + # Arbitrary object to store builder-specific info. This object value will + # be passed back into the builder's #finalize method. + def initialize(command, options = {}) + @command = command + @builder_info = options[:builder_info] + end + + end +end From e4adaab003be2bbfd941926a7aaf95bf53a61626 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 16 May 2017 09:18:08 -0400 Subject: [PATCH 16/69] add Builder#finalize --- lib/rscons/builder.rb | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/lib/rscons/builder.rb b/lib/rscons/builder.rb index ed8e4b6..9bf8898 100644 --- a/lib/rscons/builder.rb +++ b/lib/rscons/builder.rb @@ -112,7 +112,8 @@ module Rscons # # @since 1.10.0 # - # @param options [Hash] Run options. + # @param options [Hash] + # Run options. # @option options [String] :target # Target file name. # @option options [Array] :sources @@ -126,12 +127,43 @@ module Rscons # @option options [Object] :setup_info # Whatever value was returned from this builder's {#setup} method call. # - # @return [String,false] + # @return [ThreadedCommand,String,false] # Name of the target file on success or false on failure. + # Since 1.10.0, this method may return an instance of {ThreadedCommand}. + # In that case, the build operation has not actually been completed yet + # but the command to do so will be executed by Rscons in a separate + # thread. This allows for build parallelization. If a {ThreadedCommand} + # object is returned, the {#finalize} method will be called after the + # command has completed. The {#finalize} method should then be used to + # record cache info, if needed, and to return the true result of the + # build operation. The builder can store information to be passed in to + # the {#finalize} method by populating the :builder_info field of the + # {ThreadedCommand} object returned here. def run(options) raise "This method must be overridden in a subclass" end + # Finalize a build operation. + # + # This method is called after the {#run} method if the {#run} method does + # not return an error. + # + # @param options [Hash] + # Options. + # @option options [true,false,nil] :command_status + # If the {#run} method returns a {ThreadedCommand}, this field will + # contain the return value from executing the command with + # Kernel.system(). + # @option options [Object] :builder_info + # If the {#run} method returns a {ThreadedCommand}, this field will + # contain the value passed in to the :builder_info field of the + # {ThreadedCommand} object. + # + # @return [String,false] + # Name of the target file on success or false on failure. + def finalize(options) + end + # Check if the cache is up to date for the target and if not execute the # build command. # From 9c13634eaf9fc491dbfd5673b3f34dac2b71a8b1 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 17 May 2017 08:47:17 -0400 Subject: [PATCH 17/69] start adding support for threaded commands in Environment#run_builder --- lib/rscons/environment.rb | 44 +++++++++++++++++++++++++++------- lib/rscons/threaded_command.rb | 5 ++++ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index e5a57a9..74f3c11 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -512,29 +512,39 @@ module Rscons # @param sources [Array] List of source files. # @param cache [Cache] The Cache. # @param vars [Hash] Extra variables to pass to the builder. + # @param options [Hash] Options. + # @option options [Boolean] :allow_delayed_execution + # @since 1.10.0 + # Allow a threaded command to be scheduled but not yet completed before + # this method returns. # # @return [String,false] Return value from the {Builder}'s +run+ method. - def run_builder(builder, target, sources, cache, vars) + def run_builder(builder, target, sources, cache, vars, options = {}) vars = @varset.merge(vars) + build_operation = { + builder: builder, + target: target, + sources: sources, + vars: vars, + env: self, + } call_build_hooks = lambda do |sec| @build_hooks[sec].each do |build_hook_block| - build_operation = { - builder: builder, - target: target, - sources: sources, - vars: vars, - env: self, - } build_hook_block.call(build_operation) end end + + # Invoke pre-build hooks. call_build_hooks[:pre] + use_new_run_method_signature = begin builder.method(:run).arity == 1 rescue NameError false end + + # Call the builder's #run method. if use_new_run_method_signature rv = builder.run( target: target, @@ -545,7 +555,23 @@ module Rscons else rv = builder.run(target, sources, cache, self, vars) end - call_build_hooks[:post] if rv + + if rv.is_a?(ThreadedCommand) + if options[:allow_delayed_execution] + # Store the build operation so the post-build hooks can be called + # with it when the threaded command completes. + rv.build_operation = build_operation + else + # Delayed command execution is not allowed, so we need to execute + # the command and finalize the builder now. + # TODO + #rv = builder.finalize() + call_build_hooks[:post] if rv + end + else + call_build_hooks[:post] if rv + end + rv end diff --git a/lib/rscons/threaded_command.rb b/lib/rscons/threaded_command.rb index e885d50..4ee4bff 100644 --- a/lib/rscons/threaded_command.rb +++ b/lib/rscons/threaded_command.rb @@ -13,6 +13,11 @@ module Rscons # be passed back into the builder's #finalize method. attr_reader :builder_info + # @return [Hash] + # Field for Rscons to store the build operation while this threaded + # command is executing. + attr_accessor :build_operation + # Create a ThreadedCommand object. # # @param command [Array] From 5fe55a584ea17b15037dbcfd39f4dfe35425295a Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 17 May 2017 08:49:53 -0400 Subject: [PATCH 18/69] add short description field to ThreadedCommand --- lib/rscons/threaded_command.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/rscons/threaded_command.rb b/lib/rscons/threaded_command.rb index 4ee4bff..b24e29c 100644 --- a/lib/rscons/threaded_command.rb +++ b/lib/rscons/threaded_command.rb @@ -13,6 +13,11 @@ module Rscons # be passed back into the builder's #finalize method. attr_reader :builder_info + # @return [String] + # Short description of the command. This will be printed to standard + # output if the Environment's echo mode is :short. + attr_reader :short_description + # @return [Hash] # Field for Rscons to store the build operation while this threaded # command is executing. @@ -27,9 +32,13 @@ module Rscons # @option options [Object] :builder_info # Arbitrary object to store builder-specific info. This object value will # be passed back into the builder's #finalize method. + # @option options [String] :short_description + # Short description of the command. This will be printed to standard + # output if the Environment's echo mode is :short. def initialize(command, options = {}) @command = command @builder_info = options[:builder_info] + @short_description = options[:short_description] end end From 01851c28720ce6d5ebdd53613b8596b6e5db4bd2 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 17 May 2017 09:04:05 -0400 Subject: [PATCH 19/69] refactor into new Environment#command_to_s --- lib/rscons/environment.rb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 74f3c11..9894539 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -337,11 +337,8 @@ module Rscons # # @return [true,false,nil] Return value from Kernel.system(). def execute(short_desc, command, options = {}) - print_command = proc do - puts command.map { |c| c =~ /\s/ ? "'#{c}'" : c }.join(' ') - end if @echo == :command - print_command.call + puts command_to_s(command) elsif @echo == :short puts short_desc end @@ -350,7 +347,7 @@ module Rscons system(*env_args, *Rscons.command_executer, *command, *options_args).tap do |result| unless result or @echo == :command $stdout.write "Failed command was: " - print_command.call + puts command_to_s(command) end end end @@ -751,6 +748,17 @@ module Rscons private + # Return a string representation of a command. + # + # @param command [Array] + # The command. + # + # @return [String] + # The string representation of the command. + def command_to_s(command) + command.map { |c| c =~ /\s/ ? "'#{c}'" : c }.join(' ') + end + # Parse dependencies for a given target from a Makefile. # # This method is used internally by Rscons builders. From 5d36aa74a00bed3cfa4398e417cb81e4603cd070 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 17 May 2017 09:07:51 -0400 Subject: [PATCH 20/69] add more fields to ThreadedCommand --- lib/rscons/threaded_command.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/rscons/threaded_command.rb b/lib/rscons/threaded_command.rb index b24e29c..323212b 100644 --- a/lib/rscons/threaded_command.rb +++ b/lib/rscons/threaded_command.rb @@ -18,11 +18,23 @@ module Rscons # output if the Environment's echo mode is :short. attr_reader :short_description + # @return [Hash] + # Environment Hash to pass to Kernel#system. + attr_reader :system_env + + # @return [Hash] + # Options Hash to pass to Kernel#system. + attr_reader :system_options + # @return [Hash] # Field for Rscons to store the build operation while this threaded # command is executing. attr_accessor :build_operation + # @return [Thread] + # The thread waiting on this command to terminate. + attr_accessor :thread + # Create a ThreadedCommand object. # # @param command [Array] @@ -35,10 +47,16 @@ module Rscons # @option options [String] :short_description # Short description of the command. This will be printed to standard # output if the Environment's echo mode is :short. + # @option options [Hash] :system_env + # Environment Hash to pass to Kernel#system. + # @option options [Hash] :system_options + # Options Hash to pass to Kernel#system. def initialize(command, options = {}) @command = command @builder_info = options[:builder_info] @short_description = options[:short_description] + @system_env = options[:system_env] + @system_options = options[:system_options] end end From 800e7a51a4b7439f322bf6d997a1dd3b658f5a17 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 17 May 2017 09:14:42 -0400 Subject: [PATCH 21/69] add Environment#start_threaded_command --- lib/rscons/environment.rb | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 9894539..d8d2a3f 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -37,6 +37,7 @@ module Rscons # If a block is given, the Environment object is yielded to the block and # when the block returns, the {#process} method is automatically called. def initialize(options = {}) + @threaded_commands = Set.new @varset = VarSet.new @job_set = JobSet.new @user_deps = {} @@ -554,6 +555,7 @@ module Rscons end if rv.is_a?(ThreadedCommand) + start_threaded_command(rv) if options[:allow_delayed_execution] # Store the build operation so the post-build hooks can be called # with it when the threaded command completes. @@ -748,6 +750,31 @@ module Rscons private + # Start a threaded command in a new thread. + # + # @param tc [ThreadedCommand] + # The ThreadedCommand to start. + # + # @return [void] + def start_threaded_command(tc) + if @echo == :command + puts command_to_s(tc.command) + elsif @echo == :short + if tc.short_description + puts tc.short_description + end + end + + env_args = tc.system_env ? [tc.system_env] : [] + options_args = tc.system_options ? [tc.system_options] : [] + system_args = [*env_args, *Rscons.command_executer, *tc.command, *options_args] + + tc.thread = Thread.new do + system(*system_args) + end + @threaded_commands << tc + end + # Return a string representation of a command. # # @param command [Array] From 7534b29e2660175a74728b21fe87baf3b500d968 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 17 May 2017 09:34:19 -0400 Subject: [PATCH 22/69] add Environment#wait_for_threaded_commands, use from #run_builder if delayed execution is not allowed --- lib/rscons/environment.rb | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index d8d2a3f..21b1851 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -1,6 +1,7 @@ require "fileutils" require "set" require "shellwords" +require "thwait" module Rscons # The Environment class is the main programmatic interface to Rscons. It @@ -563,8 +564,10 @@ module Rscons else # Delayed command execution is not allowed, so we need to execute # the command and finalize the builder now. - # TODO - #rv = builder.finalize() + tc = wait_for_threaded_commands(which: [rv]) + rv = builder.finalize( + command_status: tc.thread.value, + builder_info: tc.builder_info) call_build_hooks[:post] if rv end else @@ -775,6 +778,29 @@ module Rscons @threaded_commands << tc end + # Wait for threaded commands to complete. + # + # @param options [Hash] + # Options. + # @option options [Set, Array] :which + # Which {ThreadedCommand} objects to wait for. If not specified, this + # method will wait for any. + # + # @return [ThreadedCommand] + # The {ThreadedCommand} object that is finished. + def wait_for_threaded_commands(options = {}) + raise "No threaded commands to wait for" if @threaded_commands.empty? + options[:which] ||= @threaded_commands + threads = options[:which].map(&:thread) + tw = ThreadsWait.new(*threads) + finished_thread = tw.next_wait + threaded_command = @threaded_commands.find do |tc| + tc.thread == finished_thread + end + @threaded_commands.delete(threaded_command) + threaded_command + end + # Return a string representation of a command. # # @param command [Array] From ca445f5733ebe387ee344c50e0a707ce22af9f8d Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 17 May 2017 09:36:27 -0400 Subject: [PATCH 23/69] add JobSet#size --- lib/rscons/job_set.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/rscons/job_set.rb b/lib/rscons/job_set.rb index 333b17e..911fc84 100644 --- a/lib/rscons/job_set.rb +++ b/lib/rscons/job_set.rb @@ -63,5 +63,13 @@ module Rscons @jobs.clear end + # Get the JobSet size. + # + # @return [Integer] + # JobSet size. + def size + @jobs.size + end + end end From 9cc59a35f06251a56c54fb89caad68c1fb08fe2f Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 17 May 2017 10:07:15 -0400 Subject: [PATCH 24/69] handle threaded commands in Environment#process --- lib/rscons/environment.rb | 87 +++++++++++++++++++++++++++------ spec/rscons/environment_spec.rb | 8 +-- 2 files changed, 76 insertions(+), 19 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 21b1851..0018eb9 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -284,18 +284,60 @@ module Rscons def process cache = Cache.instance begin - while job = @job_set.get_next_job_to_run + while @job_set.size > 0 + + # TODO: get_next_job_to_run needs to take into account targets still + # being processed. + job = @job_set.get_next_job_to_run + # TODO: have Cache determine when checksums may be invalid based on # file size and/or timestamp. cache.clear_checksum_cache! - result = run_builder(job[:builder], - job[:target], - job[:sources], - cache, - job[:vars]) - unless result - raise BuildError.new("Failed to build #{job[:target]}") + + if job + result = run_builder(job[:builder], + job[:target], + job[:sources], + cache, + job[:vars], + allow_delayed_execution: true) + unless result.is_a?(ThreadedCommand) + unless result + raise BuildError.new("Failed to build #{job[:target]}") + end + end end + + completed_tcs = Set.new + # First do a non-blocking wait to pick up any threads that have + # completed since last time. + loop do + if tc = wait_for_threaded_commands(nonblock: true) + completed_tcs << tc + else + break + end + end + + # If needed, do a blocking wait. + if job.nil? or @threaded_commands.size >= Rscons.n_threads + completed_tcs << wait_for_threaded_commands + end + + # Process all completed {ThreadedCommand} objects. + completed_tcs.each do |tc| + result = builder.finalize( + command_status: tc.thread.value, + builder_info: tc.builder_info) + if result + @build_hooks[:post].each do |build_hook_block| + build_hook_block.call(tc.build_operation) + end + else + raise BuildError.new("Failed to build #{tc.build_operation[:target]}") + end + end + end ensure cache.write @@ -785,20 +827,35 @@ module Rscons # @option options [Set, Array] :which # Which {ThreadedCommand} objects to wait for. If not specified, this # method will wait for any. + # @option options [Boolean] :nonblock + # Set to true to not block. # - # @return [ThreadedCommand] + # @return [ThreadedCommand, nil] # The {ThreadedCommand} object that is finished. def wait_for_threaded_commands(options = {}) - raise "No threaded commands to wait for" if @threaded_commands.empty? + if @threaded_commands.empty? + if options[:nonblock] + return nil + else + raise "No threaded commands to wait for" + end + end options[:which] ||= @threaded_commands threads = options[:which].map(&:thread) tw = ThreadsWait.new(*threads) - finished_thread = tw.next_wait - threaded_command = @threaded_commands.find do |tc| - tc.thread == finished_thread + finished_thread = + begin + tw.next_wait(options[:nonblock]) + rescue ThreadsWait::ErrNoFinishedThread + nil + end + if finished_thread + threaded_command = @threaded_commands.find do |tc| + tc.thread == finished_thread + end + @threaded_commands.delete(threaded_command) + threaded_command end - @threaded_commands.delete(threaded_command) - threaded_command end # Return a string representation of a command. diff --git a/spec/rscons/environment_spec.rb b/spec/rscons/environment_spec.rb index 802bc5b..a4094d9 100644 --- a/spec/rscons/environment_spec.rb +++ b/spec/rscons/environment_spec.rb @@ -170,7 +170,7 @@ module Rscons cache = "cache" expect(Cache).to receive(:instance).and_return(cache) allow(cache).to receive(:clear_checksum_cache!) - expect(env).to receive(:run_builder).with(anything, "a.out", ["main.c"], cache, {}).and_return(true) + expect(env).to receive(:run_builder).with(anything, "a.out", ["main.c"], cache, {}, allow_delayed_execution: true).and_return(true) expect(cache).to receive(:write) env.process @@ -184,8 +184,8 @@ module Rscons cache = "cache" expect(Cache).to receive(:instance).and_return(cache) allow(cache).to receive(:clear_checksum_cache!) - expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}).and_return("main.o") - expect(env).to receive(:run_builder).with(anything, "a.out", ["main.o"], cache, {}).and_return("a.out") + expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}, allow_delayed_execution: true).and_return("main.o") + expect(env).to receive(:run_builder).with(anything, "a.out", ["main.o"], cache, {}, allow_delayed_execution: true).and_return("a.out") expect(cache).to receive(:write) env.process @@ -199,7 +199,7 @@ module Rscons cache = "cache" expect(Cache).to receive(:instance).and_return(cache) allow(cache).to receive(:clear_checksum_cache!) - expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}).and_return(false) + expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}, allow_delayed_execution: true).and_return(false) expect(cache).to receive(:write) expect { env.process }.to raise_error BuildError, /Failed.to.build.main.o/ From 5de52620e4745871b82e02f4bf232c1bed5f6e90 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 17 May 2017 10:19:43 -0400 Subject: [PATCH 25/69] JobSet#get_next_job_to_run: take into account targets still building --- lib/rscons/environment.rb | 7 +++--- lib/rscons/job_set.rb | 48 ++++++++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 0018eb9..ef46d51 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -286,9 +286,10 @@ module Rscons begin while @job_set.size > 0 - # TODO: get_next_job_to_run needs to take into account targets still - # being processed. - job = @job_set.get_next_job_to_run + targets_still_building = @threaded_commands.map do |tc| + tc.build_operation[:target] + end + job = @job_set.get_next_job_to_run(targets_still_building) # TODO: have Cache determine when checksums may be invalid based on # file size and/or timestamp. diff --git a/lib/rscons/job_set.rb b/lib/rscons/job_set.rb index 911fc84..beab72a 100644 --- a/lib/rscons/job_set.rb +++ b/lib/rscons/job_set.rb @@ -34,28 +34,44 @@ module Rscons # # This method will remove the job from the JobSet. # + # @param targets_still_building [Array] + # Targets that are not finished building. This is used to avoid returning + # a job as available to run if it depends on one of the targets that are + # still building as a source. + # # @return [nil, Hash] # The next job to run. - def get_next_job_to_run - if @jobs.size > 0 - evaluated_targets = Set.new - attempt = lambda do |target| - evaluated_targets << target - @jobs[target][0][:sources].each do |src| - if @jobs.include?(src) and not evaluated_targets.include?(src) - return attempt[src] - end + def get_next_job_to_run(targets_still_building) + attempted_targets = Set.new + + @jobs.keys.each do |target| + attempted_targets << target + skip = false + @jobs[target][0][:sources].each do |src| + if @jobs.include?(src) and not attempted_targets.include?(src) + # Skip this target because it depends on another target later in + # the job set. + skip = true + break end - job = @jobs[target][0] - if @jobs[target].size > 1 - @jobs[target].slice!(0) - else - @jobs.delete(target) + if targets_still_building.include?(src) + # Skip this target because it depends on another target that is + # still being built. + skip = true + break end - return job end - attempt[@jobs.first.first] + next if skip + job = @jobs[target][0] + if @jobs[target].size > 1 + @jobs[target].slice!(0) + else + @jobs.delete(target) + end + return job end + + nil end # Remove all jobs from the JobSet. From 4ed584701a3a2cee17ffe956c18a2bc696e2b645 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 17 May 2017 13:50:31 -0400 Subject: [PATCH 26/69] pass setup_info to the builder's #run method --- lib/rscons/environment.rb | 12 +++++++++--- spec/rscons/environment_spec.rb | 8 ++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index ef46d51..90fde90 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -301,7 +301,8 @@ module Rscons job[:sources], cache, job[:vars], - allow_delayed_execution: true) + allow_delayed_execution: true, + setup_info: job[:setup_info]) unless result.is_a?(ThreadedCommand) unless result raise BuildError.new("Failed to build #{job[:target]}") @@ -554,11 +555,15 @@ module Rscons # @param sources [Array] List of source files. # @param cache [Cache] The Cache. # @param vars [Hash] Extra variables to pass to the builder. - # @param options [Hash] Options. + # @param options [Hash] + # @since 1.10.0 + # Options. # @option options [Boolean] :allow_delayed_execution # @since 1.10.0 # Allow a threaded command to be scheduled but not yet completed before # this method returns. + # @option options [Object] :setup_info + # Arbitrary builder info returned by Builder#setup. # # @return [String,false] Return value from the {Builder}'s +run+ method. def run_builder(builder, target, sources, cache, vars, options = {}) @@ -593,7 +598,8 @@ module Rscons sources: sources, cache: cache, env: self, - vars: vars) + vars: vars, + setup_info: options[:setup_info]) else rv = builder.run(target, sources, cache, self, vars) end diff --git a/spec/rscons/environment_spec.rb b/spec/rscons/environment_spec.rb index a4094d9..d6ac557 100644 --- a/spec/rscons/environment_spec.rb +++ b/spec/rscons/environment_spec.rb @@ -170,7 +170,7 @@ module Rscons cache = "cache" expect(Cache).to receive(:instance).and_return(cache) allow(cache).to receive(:clear_checksum_cache!) - expect(env).to receive(:run_builder).with(anything, "a.out", ["main.c"], cache, {}, allow_delayed_execution: true).and_return(true) + expect(env).to receive(:run_builder).with(anything, "a.out", ["main.c"], cache, {}, allow_delayed_execution: true, setup_info: nil).and_return(true) expect(cache).to receive(:write) env.process @@ -184,8 +184,8 @@ module Rscons cache = "cache" expect(Cache).to receive(:instance).and_return(cache) allow(cache).to receive(:clear_checksum_cache!) - expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}, allow_delayed_execution: true).and_return("main.o") - expect(env).to receive(:run_builder).with(anything, "a.out", ["main.o"], cache, {}, allow_delayed_execution: true).and_return("a.out") + expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}, allow_delayed_execution: true, setup_info: nil).and_return("main.o") + expect(env).to receive(:run_builder).with(anything, "a.out", ["main.o"], cache, {}, allow_delayed_execution: true, setup_info: nil).and_return("a.out") expect(cache).to receive(:write) env.process @@ -199,7 +199,7 @@ module Rscons cache = "cache" expect(Cache).to receive(:instance).and_return(cache) allow(cache).to receive(:clear_checksum_cache!) - expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}, allow_delayed_execution: true).and_return(false) + expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}, allow_delayed_execution: true, setup_info: nil).and_return(false) expect(cache).to receive(:write) expect { env.process }.to raise_error BuildError, /Failed.to.build.main.o/ From e694199f335a8f4459b580452168a3e288826252 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 17 May 2017 15:45:04 -0400 Subject: [PATCH 27/69] delay building targets that depend on builds registered with Environment#register_builds --- lib/rscons/environment.rb | 11 ++++++++--- lib/rscons/job_set.rb | 14 ++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 90fde90..82c7a5d 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -39,8 +39,9 @@ module Rscons # when the block returns, the {#process} method is automatically called. def initialize(options = {}) @threaded_commands = Set.new + @registered_build_dependencies = {} @varset = VarSet.new - @job_set = JobSet.new + @job_set = JobSet.new(@registered_build_dependencies) @user_deps = {} @builders = {} @build_dirs = [] @@ -328,7 +329,7 @@ module Rscons # Process all completed {ThreadedCommand} objects. completed_tcs.each do |tc| - result = builder.finalize( + result = tc.build_operation[:builder].finalize( command_status: tc.thread.value, builder_info: tc.builder_info) if result @@ -517,6 +518,8 @@ module Rscons # # @since 1.10.0 # + # @param target [String] + # The target that depends on these builds. # @param sources [Array] # List of source file(s) to build. # @param suffixes [Array] @@ -526,7 +529,8 @@ module Rscons # # @return [Array] # List of the output file name(s). - def register_builds(sources, suffixes, vars) + def register_builds(target, sources, suffixes, vars) + @registered_build_dependencies[target] ||= Set.new sources.map do |source| if source.end_with?(*suffixes) source @@ -540,6 +544,7 @@ module Rscons if builder output_fname = attempt_output_fname self.__send__(builder.name, output_fname, source, vars) + @registered_build_dependencies[target] << output_fname break end end diff --git a/lib/rscons/job_set.rb b/lib/rscons/job_set.rb index beab72a..19132de 100644 --- a/lib/rscons/job_set.rb +++ b/lib/rscons/job_set.rb @@ -5,8 +5,13 @@ module Rscons class JobSet # Create a JobSet - def initialize + # + # @param build_dependencies [Hash] + # Hash mapping targets to a set of build dependencies. A job will not be + # returned as ready to run if any of its dependencies are still building. + def initialize(build_dependencies) @jobs = {} + @build_dependencies = build_dependencies end # Add a job to the JobSet. @@ -42,13 +47,10 @@ module Rscons # @return [nil, Hash] # The next job to run. def get_next_job_to_run(targets_still_building) - attempted_targets = Set.new - @jobs.keys.each do |target| - attempted_targets << target skip = false - @jobs[target][0][:sources].each do |src| - if @jobs.include?(src) and not attempted_targets.include?(src) + (@jobs[target][0][:sources] + (@build_dependencies[target] || []).to_a).each do |src| + if @jobs.include?(src) # Skip this target because it depends on another target later in # the job set. skip = true From 267fc7124dd43d1d07c6e64e1cd7dd78a2063234 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 17 May 2017 15:52:34 -0400 Subject: [PATCH 28/69] process threaded commands until they are all complete --- lib/rscons/environment.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 82c7a5d..89e113b 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -285,7 +285,7 @@ module Rscons def process cache = Cache.instance begin - while @job_set.size > 0 + while @job_set.size > 0 or @threaded_commands.size > 0 targets_still_building = @threaded_commands.map do |tc| tc.build_operation[:target] From 63446920876862b4672beaa1e03223c044b5dbdf Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 19 May 2017 10:49:16 -0400 Subject: [PATCH 29/69] update Object builder to support parallelization; update Program and Library builders to register object builds from #setup --- lib/rscons/builders/library.rb | 41 +++++++++++++++++++++------------- lib/rscons/builders/object.rb | 39 ++++++++++++++++++++++---------- lib/rscons/builders/program.rb | 28 +++++++++++++++-------- spec/build_tests_spec.rb | 13 +++++------ 4 files changed, 77 insertions(+), 44 deletions(-) diff --git a/lib/rscons/builders/library.rb b/lib/rscons/builders/library.rb index 4545325..328764b 100644 --- a/lib/rscons/builders/library.rb +++ b/lib/rscons/builders/library.rb @@ -2,6 +2,7 @@ module Rscons module Builders # A default Rscons builder that produces a static library archive. class Library < Builder + # Return default construction variables for the builder. # # @param env [Environment] The Environment using the builder. @@ -16,28 +17,36 @@ module Rscons } end + # Set up a build operation using this builder. + # + # @param options [Hash] Builder setup options. + # + # @return [Object] + # Any object that the builder author wishes to be saved and passed back + # in to the {#run} method. + def setup(options) + target, sources, env, vars = options.values_at(:target, :sources, :env, :vars) + suffixes = env.expand_varref(["${OBJSUFFIX}", "${LIBSUFFIX}"], vars) + # Register builders to build each source to an object file or library. + env.register_builds(target, sources, suffixes, vars) + end + # Run the builder to produce a build target. # - # @param target [String] Target file name. - # @param sources [Array] Source file name(s). - # @param cache [Cache] The Cache object. - # @param env [Environment] The Environment executing the builder. - # @param vars [Hash,VarSet] Extra construction variables. + # @param options [Hash] Builder run options. # # @return [String,false] # Name of the target file on success or false on failure. - def run(target, sources, cache, env, vars) - # build sources to linkable objects - objects = env.build_sources(sources, env.expand_varref(["${OBJSUFFIX}", "${LIBSUFFIX}"], vars).flatten, cache, vars) - if objects - vars = vars.merge({ - '_TARGET' => target, - '_SOURCES' => objects, - }) - command = env.build_command("${ARCMD}", vars) - standard_build("AR #{target}", target, command, objects, env, cache) - end + def run(options) + target, sources, cache, env, vars, objects = options.values_at(:target, :sources, :cache, :env, :vars, :setup_info) + vars = vars.merge({ + '_TARGET' => target, + '_SOURCES' => objects, + }) + command = env.build_command("${ARCMD}", vars) + standard_build("AR #{target}", target, command, objects, env, cache) end + end end end diff --git a/lib/rscons/builders/object.rb b/lib/rscons/builders/object.rb index d755cf8..b9229b2 100644 --- a/lib/rscons/builders/object.rb +++ b/lib/rscons/builders/object.rb @@ -3,6 +3,7 @@ module Rscons # A default Rscons builder which knows how to produce an object file from # various types of source files. class Object < Builder + # Mapping of known sources from which to build object files. KNOWN_SUFFIXES = { "AS" => "ASSUFFIX", @@ -76,15 +77,12 @@ module Rscons # Run the builder to produce a build target. # - # @param target [String] Target file name. - # @param sources [Array] Source file name(s). - # @param cache [Cache] The Cache object. - # @param env [Environment] The Environment executing the builder. - # @param vars [Hash,VarSet] Extra construction variables. + # @param options [Hash] Builder run options. # - # @return [String,false] - # Name of the target file on success or false on failure. - def run(target, sources, cache, env, vars) + # @return [ThreadedCommand] + # Threaded command to execute. + def run(options) + target, sources, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars) vars = vars.merge({ '_TARGET' => target, '_SOURCES' => sources, @@ -96,19 +94,36 @@ module Rscons v.nil? and raise "Error: unknown input file type: #{sources.first.inspect}" end.first command = env.build_command("${#{com_prefix}CMD}", vars) - unless cache.up_to_date?(target, command, sources, env) + if cache.up_to_date?(target, command, sources, env) + target + else cache.mkdir_p(File.dirname(target)) FileUtils.rm_f(target) - return false unless env.execute("#{com_prefix} #{target}", command) - deps = sources + ThreadedCommand.new( + command, + short_description: "#{com_prefix} #{target}", + builder_info: options.merge(vars: vars, command: command)) + end + end + + # Finalize the build operation. + # + # @param options [Hash] Builder finalize options. + # + # @return [String,nil] + # Name of the target file on success or nil on failure. + def finalize(options) + if options[:command_status] + target, deps, cache, env, vars, command = options[:builder_info].values_at(:target, :sources, :cache, :env, :vars, :command) if File.exists?(vars['_DEPFILE']) deps += Environment.parse_makefile_deps(vars['_DEPFILE'], target) FileUtils.rm_f(vars['_DEPFILE']) end cache.register_build(target, command, deps.uniq, env) + target end - target end + end end end diff --git a/lib/rscons/builders/program.rb b/lib/rscons/builders/program.rb index 4785117..ce21c19 100644 --- a/lib/rscons/builders/program.rb +++ b/lib/rscons/builders/program.rb @@ -3,6 +3,7 @@ module Rscons # A default Rscons builder that knows how to link object files into an # executable program. class Program < Builder + # Return default construction variables for the builder. # # @param env [Environment] The Environment using the builder. @@ -45,20 +46,28 @@ module Rscons super(my_options) end + # Set up a build operation using this builder. + # + # @param options [Hash] Builder setup options. + # + # @return [Object] + # Any object that the builder author wishes to be saved and passed back + # in to the {#run} method. + def setup(options) + target, sources, env, vars = options.values_at(:target, :sources, :env, :vars) + suffixes = env.expand_varref(["${OBJSUFFIX}", "${LIBSUFFIX}"], vars) + # Register builders to build each source to an object file or library. + env.register_builds(target, sources, suffixes, vars) + end + # Run the builder to produce a build target. # - # @param target [String] Target file name. - # @param sources [Array] Source file name(s). - # @param cache [Cache] The Cache object. - # @param env [Environment] The Environment executing the builder. - # @param vars [Hash,VarSet] Extra construction variables. + # @param options [Hash] Builder run options. # # @return [String,false] # Name of the target file on success or false on failure. - def run(target, sources, cache, env, vars) - # build sources to linkable objects - objects = env.build_sources(sources, env.expand_varref(["${OBJSUFFIX}", "${LIBSUFFIX}"], vars).flatten, cache, vars) - return false unless objects + def run(options) + target, sources, cache, env, vars, objects = options.values_at(:target, :sources, :cache, :env, :vars, :setup_info) ld = env.expand_varref("${LD}", vars) ld = if ld != "" ld @@ -77,6 +86,7 @@ module Rscons command = env.build_command("${LDCMD}", vars) standard_build("LD #{target}", target, command, objects, env, cache) end + end end end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index ce2a91c..44a3a5f 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -806,28 +806,27 @@ EOF end expect do Rscons::Environment.new do |env| - env.Program("simple", %w[simple.c two.c]) + env.Program("simple.exe", %w[simple.c two.c]) end - end.to raise_error /Failed to build simple/ + end.to raise_error /Failed to build two\.o/ result = lines - expect(result.size).to be > 2 - expect(result[0, 2]).to eq [ + expect(Set[*result]).to eq Set[ "CC simple.o", "CC two.o", ] expect(File.exists?("simple.o")).to be_truthy expect(File.exists?("two.o")).to be_falsey - expect(File.exists?("two_sources#{Rscons::Environment.new["PROGSUFFIX"]}")).to be_falsey + expect(File.exists?("simple.exe")).to be_falsey Rscons::Cache.reset! File.open("two.c", "w") {|fh|} Rscons::Environment.new do |env| - env.Program("simple", %w[simple.c two.c]) + env.Program("simple.exe", %w[simple.c two.c]) end expect(lines).to eq [ "CC two.o", - "LD simple", + "LD simple.exe", ] end From 5cef9896acc8dd1f562d70a90b3ac5e37043db52 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 19 May 2017 10:57:30 -0400 Subject: [PATCH 30/69] consistently print failed build command if a builder fails --- lib/rscons/environment.rb | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 89e113b..8ca17bf 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -304,10 +304,8 @@ module Rscons job[:vars], allow_delayed_execution: true, setup_info: job[:setup_info]) - unless result.is_a?(ThreadedCommand) - unless result - raise BuildError.new("Failed to build #{job[:target]}") - end + unless result + raise BuildError.new("Failed to build #{job[:target]}") end end @@ -337,6 +335,9 @@ module Rscons build_hook_block.call(tc.build_operation) end else + unless @echo == :command + $stdout.write "Failed command was: #{command_to_s(tc.command)}" + end raise BuildError.new("Failed to build #{tc.build_operation[:target]}") end end @@ -393,8 +394,7 @@ module Rscons options_args = options[:options] ? [options[:options]] : [] system(*env_args, *Rscons.command_executer, *command, *options_args).tap do |result| unless result or @echo == :command - $stdout.write "Failed command was: " - puts command_to_s(command) + $stdout.write "Failed command was: #{command_to_s(command)}" end end end @@ -622,7 +622,13 @@ module Rscons rv = builder.finalize( command_status: tc.thread.value, builder_info: tc.builder_info) - call_build_hooks[:post] if rv + if rv + call_build_hooks[:post] + else + unless @echo == :command + $stdout.write "Failed command was: #{command_to_s(tc.command)}" + end + end end else call_build_hooks[:post] if rv From 2d8e08b493de63246a06f34938d07c653ce87ebd Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 19 May 2017 11:38:42 -0400 Subject: [PATCH 31/69] add -j command-line option to set number of threads --- lib/rscons/cli.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/rscons/cli.rb b/lib/rscons/cli.rb index 575754e..4776128 100644 --- a/lib/rscons/cli.rb +++ b/lib/rscons/cli.rb @@ -35,6 +35,10 @@ module Rscons rsconsfile = f end + opts.on("-j NTHREADS", "Use NTHREADS parallel jobs (local default #{Rscons.n_threads})") do |n_threads| + Rscons.n_threads = n_threads.to_i + end + opts.on_tail("--version", "Show version") do puts "Rscons version #{Rscons::VERSION}" exit 0 From 28e56251dd5f2fc5ee9fb9ba2660183b67cd3e06 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 19 May 2017 16:56:58 -0400 Subject: [PATCH 32/69] update build_tests_spec to invoke rscons in a subprocess for each test --- build_tests/build_dir/Rsconsfile | 5 + build_tests/build_dir/build_dirs_and_root.rb | 6 + build_tests/build_dir/build_hooks.rb | 12 + build_tests/build_dir/carat.rb | 8 + build_tests/build_dir/copy.rb | 3 + build_tests/build_dir/csuffix.rb | 6 + build_tests/build_dir/install.rb | 3 + build_tests/build_dir/install_directory.rb | 9 + .../build_dir/multiple_targets_same_name.rb | 5 + build_tests/build_dir/no_match_build_dir.rb | 6 + .../build_dir/post_build_hook_expansion.rb | 10 + build_tests/build_dir/slashes.rb | 6 + build_tests/clone_env/Rsconsfile | 12 + build_tests/clone_env/clone_all.rb | 15 + build_tests/custom_builder/Rsconsfile | 16 + build_tests/custom_builder/cvar_expansion.rb | 18 + build_tests/custom_builder/cvar_lambda.rb | 28 + .../custom_builder/multiple_targets.rb | 19 + build_tests/d/Rsconsfile | 3 + build_tests/header/Rsconsfile | 3 + build_tests/json_to_yaml/Rsconsfile | 15 + build_tests/library/Rsconsfile | 4 + build_tests/preprocess/Rsconsfile | 3 + build_tests/simple/Rsconsfile | 3 + .../simple/build_root_builder_no_sources.rb | 10 + .../cache_successful_builds_when_one_fails.rb | 4 + build_tests/simple/clear_targets.rb | 4 + build_tests/simple/command.rb | 4 + build_tests/simple/cvar_array.rb | 4 + build_tests/simple/directory.rb | 3 + build_tests/simple/disassemble.rb | 4 + build_tests/simple/dump.rb | 5 + build_tests/simple/link_flag_change.rb | 5 + build_tests/simple/phony_target.rb | 12 + build_tests/simple/preprocess.rb | 4 + .../simple/register_target_in_build_hook.rb | 8 + build_tests/simple/user_dependencies.rb | 4 + build_tests/simple_cc/Rsconsfile | 3 + build_tests/simple_cc/cxxsuffix.rb | 5 + build_tests/simple_cc/preprocess.rb | 4 + build_tests/two_sources/Rsconsfile | 4 + build_tests/two_sources/assuffix.rb | 7 + build_tests/two_sources/libsuffix.rb | 6 + build_tests/two_sources/objsuffix.rb | 6 + spec/build_tests_spec.rb | 980 +++++++----------- 45 files changed, 690 insertions(+), 614 deletions(-) create mode 100644 build_tests/build_dir/Rsconsfile create mode 100644 build_tests/build_dir/build_dirs_and_root.rb create mode 100644 build_tests/build_dir/build_hooks.rb create mode 100644 build_tests/build_dir/carat.rb create mode 100644 build_tests/build_dir/copy.rb create mode 100644 build_tests/build_dir/csuffix.rb create mode 100644 build_tests/build_dir/install.rb create mode 100644 build_tests/build_dir/install_directory.rb create mode 100644 build_tests/build_dir/multiple_targets_same_name.rb create mode 100644 build_tests/build_dir/no_match_build_dir.rb create mode 100644 build_tests/build_dir/post_build_hook_expansion.rb create mode 100644 build_tests/build_dir/slashes.rb create mode 100644 build_tests/clone_env/Rsconsfile create mode 100644 build_tests/clone_env/clone_all.rb create mode 100644 build_tests/custom_builder/Rsconsfile create mode 100644 build_tests/custom_builder/cvar_expansion.rb create mode 100644 build_tests/custom_builder/cvar_lambda.rb create mode 100644 build_tests/custom_builder/multiple_targets.rb create mode 100644 build_tests/d/Rsconsfile create mode 100644 build_tests/header/Rsconsfile create mode 100644 build_tests/json_to_yaml/Rsconsfile create mode 100644 build_tests/library/Rsconsfile create mode 100644 build_tests/preprocess/Rsconsfile create mode 100644 build_tests/simple/Rsconsfile create mode 100644 build_tests/simple/build_root_builder_no_sources.rb create mode 100644 build_tests/simple/cache_successful_builds_when_one_fails.rb create mode 100644 build_tests/simple/clear_targets.rb create mode 100644 build_tests/simple/command.rb create mode 100644 build_tests/simple/cvar_array.rb create mode 100644 build_tests/simple/directory.rb create mode 100644 build_tests/simple/disassemble.rb create mode 100644 build_tests/simple/dump.rb create mode 100644 build_tests/simple/link_flag_change.rb create mode 100644 build_tests/simple/phony_target.rb create mode 100644 build_tests/simple/preprocess.rb create mode 100644 build_tests/simple/register_target_in_build_hook.rb create mode 100644 build_tests/simple/user_dependencies.rb create mode 100644 build_tests/simple_cc/Rsconsfile create mode 100644 build_tests/simple_cc/cxxsuffix.rb create mode 100644 build_tests/simple_cc/preprocess.rb create mode 100644 build_tests/two_sources/Rsconsfile create mode 100644 build_tests/two_sources/assuffix.rb create mode 100644 build_tests/two_sources/libsuffix.rb create mode 100644 build_tests/two_sources/objsuffix.rb diff --git a/build_tests/build_dir/Rsconsfile b/build_tests/build_dir/Rsconsfile new file mode 100644 index 0000000..03459b3 --- /dev/null +++ b/build_tests/build_dir/Rsconsfile @@ -0,0 +1,5 @@ +Rscons::Environment.new do |env| + env.append('CPPPATH' => Dir['src/**/*/'].sort) + env.build_dir(%r{^src/([^/]+)/}, 'build_\\1/') + env.Program('build_dir.exe', Dir['src/**/*.c']) +end diff --git a/build_tests/build_dir/build_dirs_and_root.rb b/build_tests/build_dir/build_dirs_and_root.rb new file mode 100644 index 0000000..e70df71 --- /dev/null +++ b/build_tests/build_dir/build_dirs_and_root.rb @@ -0,0 +1,6 @@ +env = Rscons::Environment.new do |env| + env.append('CPPPATH' => Dir['src/**/*/'].sort) + env.build_dir("src", "build") + env.build_root = "build_root" + env.Program('build_dir.exe', Dir['src/**/*.c']) +end diff --git a/build_tests/build_dir/build_hooks.rb b/build_tests/build_dir/build_hooks.rb new file mode 100644 index 0000000..3646b2c --- /dev/null +++ b/build_tests/build_dir/build_hooks.rb @@ -0,0 +1,12 @@ +Rscons::Environment.new(echo: :command) do |env| + env.append('CPPPATH' => Dir['src/**/*/'].sort) + env.build_dir(%r{^src/([^/]+)/}, 'build_\\1/') + env.add_build_hook do |build_op| + if build_op[:target] =~ %r{build_one/.*\.o} + build_op[:vars]["CFLAGS"] << "-O1" + elsif build_op[:target] =~ %r{build_two/.*\.o} + build_op[:vars]["CFLAGS"] << "-O2" + end + end + env.Program('build_hook.exe', Dir['src/**/*.c'].sort) +end diff --git a/build_tests/build_dir/carat.rb b/build_tests/build_dir/carat.rb new file mode 100644 index 0000000..8f74526 --- /dev/null +++ b/build_tests/build_dir/carat.rb @@ -0,0 +1,8 @@ +Rscons::Environment.new(echo: :command) do |env| + env.append('CPPPATH' => Dir['src/**/*/'].sort) + env.build_root = "build_root" + FileUtils.mkdir_p(env.build_root) + FileUtils.mv("src/one/one.c", "build_root") + env.Object("^/one.o", "^/one.c") + env.Program("build_dir.exe", Dir['src/**/*.c'] + ["^/one.o"]) +end diff --git a/build_tests/build_dir/copy.rb b/build_tests/build_dir/copy.rb new file mode 100644 index 0000000..fce7294 --- /dev/null +++ b/build_tests/build_dir/copy.rb @@ -0,0 +1,3 @@ +Rscons::Environment.new do |env| + env.Copy("inst.exe", "copy.rb") +end diff --git a/build_tests/build_dir/csuffix.rb b/build_tests/build_dir/csuffix.rb new file mode 100644 index 0000000..9b25719 --- /dev/null +++ b/build_tests/build_dir/csuffix.rb @@ -0,0 +1,6 @@ +Rscons::Environment.new do |env| + env["CSUFFIX"] = %w[.yargh .c] + env["CFLAGS"] += %w[-x c] + env["CPPPATH"] += Dir["src/**/"] + env.Program("build_dir.exe", Dir["src/**/*.{c,yargh}"]) +end diff --git a/build_tests/build_dir/install.rb b/build_tests/build_dir/install.rb new file mode 100644 index 0000000..1e39ca8 --- /dev/null +++ b/build_tests/build_dir/install.rb @@ -0,0 +1,3 @@ +Rscons::Environment.new do |env| + env.Install("inst.exe", "install.rb") +end diff --git a/build_tests/build_dir/install_directory.rb b/build_tests/build_dir/install_directory.rb new file mode 100644 index 0000000..2381af7 --- /dev/null +++ b/build_tests/build_dir/install_directory.rb @@ -0,0 +1,9 @@ +Rscons::Environment.new do |env| + env.Directory("inst") + env.Install("inst", "install_directory.rb") + + env.Install("noexist/src", "src") + + env.Directory("exist/src") + env.Install("exist/src", "src") +end diff --git a/build_tests/build_dir/multiple_targets_same_name.rb b/build_tests/build_dir/multiple_targets_same_name.rb new file mode 100644 index 0000000..e74bbc9 --- /dev/null +++ b/build_tests/build_dir/multiple_targets_same_name.rb @@ -0,0 +1,5 @@ +Rscons::Environment.new do |env| + env["CPPPATH"] << "src/two" + env.Object("one.o", "src/one/one.c") + env.Object("one.o", "src/two/two.c") +end diff --git a/build_tests/build_dir/no_match_build_dir.rb b/build_tests/build_dir/no_match_build_dir.rb new file mode 100644 index 0000000..42c6e48 --- /dev/null +++ b/build_tests/build_dir/no_match_build_dir.rb @@ -0,0 +1,6 @@ +Rscons::Environment.new do |env| + env.append('CPPPATH' => Dir['src/**/*/'].sort) + env.build_dir("src2", "build") + env.build_root = "build_root" + env.Program('build_dir.exe', Dir['src/**/*.c']) +end diff --git a/build_tests/build_dir/post_build_hook_expansion.rb b/build_tests/build_dir/post_build_hook_expansion.rb new file mode 100644 index 0000000..26b7a33 --- /dev/null +++ b/build_tests/build_dir/post_build_hook_expansion.rb @@ -0,0 +1,10 @@ +Rscons::Environment.new do |env| + env["CPPPATH"] << "src/two" + env.Object("one.o", "src/one/one.c") + env.add_post_build_hook do |build_op| + if build_op[:target] == "one.o" + env["MODULE"] = "two" + env.Object("${MODULE}.o", "src/${MODULE}/${MODULE}.c") + end + end +end diff --git a/build_tests/build_dir/slashes.rb b/build_tests/build_dir/slashes.rb new file mode 100644 index 0000000..4b619ff --- /dev/null +++ b/build_tests/build_dir/slashes.rb @@ -0,0 +1,6 @@ +Rscons::Environment.new do |env| + env.append("CPPPATH" => Dir["src/**/*/"].sort) + env.build_dir("src/one/", "build_one/") + env.build_dir("src/two", "build_two") + env.Program("build_dir.exe", Dir["src/**/*.c"]) +end diff --git a/build_tests/clone_env/Rsconsfile b/build_tests/clone_env/Rsconsfile new file mode 100644 index 0000000..5f919b3 --- /dev/null +++ b/build_tests/clone_env/Rsconsfile @@ -0,0 +1,12 @@ +debug = Rscons::Environment.new(echo: :command) do |env| + env.build_dir('src', 'debug') + env['CFLAGS'] = '-O2' + env['CPPFLAGS'] = '-DSTRING="Debug Version"' + env.Program('program-debug.exe', Dir['src/*.c']) +end + +release = debug.clone do |env| + env["CPPFLAGS"] = '-DSTRING="Release Version"' + env.build_dir('src', 'release') + env.Program('program-release.exe', Dir['src/*.c']) +end diff --git a/build_tests/clone_env/clone_all.rb b/build_tests/clone_env/clone_all.rb new file mode 100644 index 0000000..eaeb542 --- /dev/null +++ b/build_tests/clone_env/clone_all.rb @@ -0,0 +1,15 @@ +env1 = Rscons::Environment.new(echo: :command) do |env| + env.build_dir('src', 'build') + env['CFLAGS'] = '-O2' + env.add_build_hook do |build_op| + build_op[:vars]['CPPFLAGS'] = '-DSTRING="Hello"' + end + env.add_post_build_hook do |build_op| + $stdout.puts "post #{build_op[:target]}" + end + env.Program('program.exe', Dir['src/*.c']) +end + +env2 = env1.clone(clone: :all) do |env| + env.Program('program2.exe', Dir['src/*.c']) +end diff --git a/build_tests/custom_builder/Rsconsfile b/build_tests/custom_builder/Rsconsfile new file mode 100644 index 0000000..fb32ba1 --- /dev/null +++ b/build_tests/custom_builder/Rsconsfile @@ -0,0 +1,16 @@ +class MySource < Rscons::Builder + def run(target, sources, cache, env, vars) + File.open(target, 'w') do |fh| + fh.puts < ['-Dmake_lib']) +end diff --git a/build_tests/preprocess/Rsconsfile b/build_tests/preprocess/Rsconsfile new file mode 100644 index 0000000..5a5e9e4 --- /dev/null +++ b/build_tests/preprocess/Rsconsfile @@ -0,0 +1,3 @@ +Rscons::Environment.new do |env| + env.Preprocess("pp", "foo.h") +end diff --git a/build_tests/simple/Rsconsfile b/build_tests/simple/Rsconsfile new file mode 100644 index 0000000..9775f9a --- /dev/null +++ b/build_tests/simple/Rsconsfile @@ -0,0 +1,3 @@ +Rscons::Environment.new do |env| + env.Program('simple', Dir['*.c']) +end diff --git a/build_tests/simple/build_root_builder_no_sources.rb b/build_tests/simple/build_root_builder_no_sources.rb new file mode 100644 index 0000000..86a74f2 --- /dev/null +++ b/build_tests/simple/build_root_builder_no_sources.rb @@ -0,0 +1,10 @@ +class TestBuilder < Rscons::Builder + def run(target, sources, cache, env, vars) + target + end +end +Rscons::Environment.new do |env| + env.build_root = "build" + env.add_builder(TestBuilder.new) + env.TestBuilder("file") +end diff --git a/build_tests/simple/cache_successful_builds_when_one_fails.rb b/build_tests/simple/cache_successful_builds_when_one_fails.rb new file mode 100644 index 0000000..d68520c --- /dev/null +++ b/build_tests/simple/cache_successful_builds_when_one_fails.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new do |env| + env.Object("simple.o", "simple.c") + env.Object("two.o", "two.c") +end diff --git a/build_tests/simple/clear_targets.rb b/build_tests/simple/clear_targets.rb new file mode 100644 index 0000000..c42ad92 --- /dev/null +++ b/build_tests/simple/clear_targets.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new do |env| + env.Program("simple.exe", "simple.c") + env.clear_targets +end diff --git a/build_tests/simple/command.rb b/build_tests/simple/command.rb new file mode 100644 index 0000000..7e80141 --- /dev/null +++ b/build_tests/simple/command.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new(echo: :command) do |env| + env["LD"] = "gcc" + env.Program('simple.exe', Dir['*.c']) +end diff --git a/build_tests/simple/cvar_array.rb b/build_tests/simple/cvar_array.rb new file mode 100644 index 0000000..f5de90e --- /dev/null +++ b/build_tests/simple/cvar_array.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new do |env| + env["sources"] = Dir["*.c"].sort + env.Program("simple.exe", "${sources}") +end diff --git a/build_tests/simple/directory.rb b/build_tests/simple/directory.rb new file mode 100644 index 0000000..9031c36 --- /dev/null +++ b/build_tests/simple/directory.rb @@ -0,0 +1,3 @@ +Rscons::Environment.new do |env| + env.Directory("teh_dir") +end diff --git a/build_tests/simple/disassemble.rb b/build_tests/simple/disassemble.rb new file mode 100644 index 0000000..113e163 --- /dev/null +++ b/build_tests/simple/disassemble.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new do |env| + env.Object("simple.o", "simple.c") + env.Disassemble("simple.txt", "simple.o") +end diff --git a/build_tests/simple/dump.rb b/build_tests/simple/dump.rb new file mode 100644 index 0000000..df80212 --- /dev/null +++ b/build_tests/simple/dump.rb @@ -0,0 +1,5 @@ +env = Rscons::Environment.new do |env| + env["CFLAGS"] += %w[-O2 -fomit-frame-pointer] + env[:foo] = :bar +end +env.dump diff --git a/build_tests/simple/link_flag_change.rb b/build_tests/simple/link_flag_change.rb new file mode 100644 index 0000000..585f941 --- /dev/null +++ b/build_tests/simple/link_flag_change.rb @@ -0,0 +1,5 @@ +Rscons::Environment.new(echo: :command) do |env| + env["LD"] = "gcc" + env["LIBPATH"] += ["libdir"] + env.Program('simple.exe', Dir['*.c']) +end diff --git a/build_tests/simple/phony_target.rb b/build_tests/simple/phony_target.rb new file mode 100644 index 0000000..af83d86 --- /dev/null +++ b/build_tests/simple/phony_target.rb @@ -0,0 +1,12 @@ +Rscons::Environment.new do |env| + env.build_root = "build" + env.add_builder(:Checker) do |target, sources, cache, env, vars| + unless cache.up_to_date?(target, :Checker, sources, env) + puts "Checker #{sources.first}" if env.echo != :off + cache.register_build(target, :Checker, sources, env) + end + target + end + env.Program("simple.exe", "simple.c") + env.Checker(:checker, "simple.exe") +end diff --git a/build_tests/simple/preprocess.rb b/build_tests/simple/preprocess.rb new file mode 100644 index 0000000..793206d --- /dev/null +++ b/build_tests/simple/preprocess.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new do |env| + env.Preprocess("simplepp.c", "simple.c") + env.Program("simple.exe", "simplepp.c") +end diff --git a/build_tests/simple/register_target_in_build_hook.rb b/build_tests/simple/register_target_in_build_hook.rb new file mode 100644 index 0000000..73d44db --- /dev/null +++ b/build_tests/simple/register_target_in_build_hook.rb @@ -0,0 +1,8 @@ +Rscons::Environment.new do |env| + env.Program("simple.exe", Dir["*.c"]) + env.add_build_hook do |build_op| + if build_op[:target].end_with?(".o") + env.Disassemble("#{build_op[:target]}.txt", build_op[:target]) + end + end +end diff --git a/build_tests/simple/user_dependencies.rb b/build_tests/simple/user_dependencies.rb new file mode 100644 index 0000000..910a694 --- /dev/null +++ b/build_tests/simple/user_dependencies.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new do |env| + program = env.Program("simple.exe", Dir["*.c"]) + env.depends(program, "program.ld") +end diff --git a/build_tests/simple_cc/Rsconsfile b/build_tests/simple_cc/Rsconsfile new file mode 100644 index 0000000..b3239ab --- /dev/null +++ b/build_tests/simple_cc/Rsconsfile @@ -0,0 +1,3 @@ +Rscons::Environment.new do |env| + env.Program('simple.exe', Dir['*.cc']) +end diff --git a/build_tests/simple_cc/cxxsuffix.rb b/build_tests/simple_cc/cxxsuffix.rb new file mode 100644 index 0000000..eff56f6 --- /dev/null +++ b/build_tests/simple_cc/cxxsuffix.rb @@ -0,0 +1,5 @@ +Rscons::Environment.new do |env| + env["CXXSUFFIX"] = %w[.cccc .cc] + env["CXXFLAGS"] += %w[-x c++] + env.Program("simple.exe", Dir["*.cc"] + Dir["*.cccc"]) +end diff --git a/build_tests/simple_cc/preprocess.rb b/build_tests/simple_cc/preprocess.rb new file mode 100644 index 0000000..bef6a82 --- /dev/null +++ b/build_tests/simple_cc/preprocess.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new do |env| + env.Preprocess("simplepp.cc", "simple.cc") + env.Program("simple.exe", "simplepp.cc") +end diff --git a/build_tests/two_sources/Rsconsfile b/build_tests/two_sources/Rsconsfile new file mode 100644 index 0000000..b30faa1 --- /dev/null +++ b/build_tests/two_sources/Rsconsfile @@ -0,0 +1,4 @@ +Rscons::Environment.new(echo: :command) do |env| + env.Object("one.o", "one.c", 'CPPFLAGS' => ['-DONE']) + env.Program('two_sources.exe', ['one.o', 'two.c']) +end diff --git a/build_tests/two_sources/assuffix.rb b/build_tests/two_sources/assuffix.rb new file mode 100644 index 0000000..a1f5597 --- /dev/null +++ b/build_tests/two_sources/assuffix.rb @@ -0,0 +1,7 @@ +Rscons::Environment.new do |env| + env["ASSUFFIX"] = %w[.ssss .sss] + env["CFLAGS"] += %w[-S] + env.Object("one.ssss", "one.c", "CPPFLAGS" => ["-DONE"]) + env.Object("two.sss", "two.c") + env.Program("two_sources.exe", %w[one.ssss two.sss], "ASFLAGS" => env["ASFLAGS"] + %w[-x assembler]) +end diff --git a/build_tests/two_sources/libsuffix.rb b/build_tests/two_sources/libsuffix.rb new file mode 100644 index 0000000..58e0a9f --- /dev/null +++ b/build_tests/two_sources/libsuffix.rb @@ -0,0 +1,6 @@ +Rscons::Environment.new() do |env| + env["LIBSUFFIX"] = %w[.aaaa .aaa] + env.Library("one.aaaa", "one.c", "CPPFLAGS" => ["-DONE"]) + env.Library("two.aaa", "two.c") + env.Program("two_sources.exe", %w[one.aaaa two.aaa]) +end diff --git a/build_tests/two_sources/objsuffix.rb b/build_tests/two_sources/objsuffix.rb new file mode 100644 index 0000000..6be2f16 --- /dev/null +++ b/build_tests/two_sources/objsuffix.rb @@ -0,0 +1,6 @@ +Rscons::Environment.new do |env| + env["OBJSUFFIX"] = %w[.oooo .ooo] + env.Object("one.oooo", "one.c", "CPPFLAGS" => ["-DONE"]) + env.Object("two.ooo", "two.c") + env.Program("two_sources.exe", %w[one.oooo two.ooo]) +end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 44a3a5f..3d150e1 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -1,18 +1,9 @@ require 'fileutils' - -class Dir - class << self - alias_method :orig_bracket, :[] - end - def self.[](*args) - orig_bracket(*args).sort - end -end +require "open3" +require "set" describe Rscons do - BUILD_TEST_RUN_DIR = "build_test_run" - def rm_rf(dir) FileUtils.rm_rf(dir) if File.exists?(dir) @@ -33,28 +24,23 @@ describe Rscons do end before(:all) do - rm_rf(BUILD_TEST_RUN_DIR) + @statics = {} + @build_test_run_dir = "build_test_run" + @run_results = Struct.new(:stdout, :stderr, :status) @owd = Dir.pwd - end - - before(:each) do - @saved_stdout = $stdout - $stdout = StringIO.new - @saved_stderr = $stderr - $stderr = StringIO.new + rm_rf(@build_test_run_dir) end after(:each) do - $stdout = @saved_stdout - $stderr = @saved_stderr Dir.chdir(@owd) - rm_rf(BUILD_TEST_RUN_DIR) + rm_rf(@build_test_run_dir) end def test_dir(build_test_directory) - FileUtils.cp_r("build_tests/#{build_test_directory}", BUILD_TEST_RUN_DIR) - Dir.chdir(BUILD_TEST_RUN_DIR) - @saved_stderr.reopen(".stderr") + Dir.chdir(@owd) + rm_rf(@build_test_run_dir) + FileUtils.cp_r("build_tests/#{build_test_directory}", @build_test_run_dir) + Dir.chdir(@build_test_run_dir) end def file_sub(fname) @@ -68,11 +54,42 @@ describe Rscons do end end - def lines - rv = ($stdout.string + $stderr.string).lines.map(&:rstrip) - $stdout.string = "" - $stderr.string = "" - rv + def run_test(options = {}) + rsconsfile = options[:rsconsfile] || "Rsconsfile" + rscons_args = options[:rscons_args] || [] + command = %w[rscons -f _build_test.rb] + rscons_args + @statics[:build_test_id] ||= 0 + @statics[:build_test_id] += 1 + command_name = "b#{@statics[:build_test_id]}" + rsconsfile_contents = File.read(rsconsfile) + File.open("_build_test.rb", "w") do |fh| + fh.puts < Dir['src/**/*/']) - env.build_dir(%r{^src/([^/]+)/}, 'build_\\1/') - env.Program('build_dir', Dir['src/**/*.c']) - end - expect(`./build_dir`).to eq "Hello from two()\n" + result = run_test + expect(result.stderr).to eq "" + expect(`./build_dir.exe`).to eq "Hello from two()\n" expect(File.exists?('build_one/one.o')).to be_truthy expect(File.exists?('build_two/two.o')).to be_truthy end it "supports trailing slashes at the end of build_dir sources and destinations" do test_dir("build_dir") - Rscons::Environment.new do |env| - env.append("CPPPATH" => Dir["src/**/*/"]) - env.build_dir("src/one/", "build_one/") - env.build_dir("src/two", "build_two") - env.Program("build_dir", Dir["src/**/*.c"]) - end - expect(`./build_dir`).to eq "Hello from two()\n" + result = run_test(rsconsfile: "slashes.rb") + expect(result.stderr).to eq "" + expect(`./build_dir.exe`).to eq "Hello from two()\n" expect(File.exists?("build_one/one.o")).to be_truthy expect(File.exists?("build_two/two.o")).to be_truthy end it 'uses build directories before build root' do test_dir('build_dir') - env = Rscons::Environment.new do |env| - env.append('CPPPATH' => Dir['src/**/*/']) - env.build_dir("src", "build") - env.build_root = "build_root" - env.Program('build_dir', Dir['src/**/*.c']) - end - expect(lines).to eq ["CC build/one/one.o", "CC build/two/two.o", "LD build_dir#{env["PROGSUFFIX"]}"] + result = run_test(rsconsfile: "build_dirs_and_root.rb") + expect(result.stderr).to eq "" + expect(Set[*lines(result.stdout)]).to eq Set[ + "CC build/one/one.o", + "CC build/two/two.o", + "LD build_dir.exe", + ] end it 'uses build_root if no build directories match' do test_dir('build_dir') - Rscons::Environment.new do |env| - env.append('CPPPATH' => Dir['src/**/*/']) - env.build_dir("src2", "build") - env.build_root = "build_root" - env.Program('build_dir.exe', Dir['src/**/*.c']) - end - expect(lines).to eq ["CC build_root/src/one/one.o", "CC build_root/src/two/two.o", "LD build_dir.exe"] + result = run_test(rsconsfile: "no_match_build_dir.rb") + expect(result.stderr).to eq "" + expect(Set[*lines(result.stdout)]).to eq Set[ + "CC build_root/src/one/one.o", + "CC build_root/src/two/two.o", + "LD build_dir.exe", + ] end it "expands target and source paths starting with ^/ to be relative to the build root" do test_dir('build_dir') - env = Rscons::Environment.new(echo: :command) do |env| - env.append('CPPPATH' => Dir['src/**/*/']) - env.build_root = "build_root" - FileUtils.mkdir_p(env.build_root) - FileUtils.mv("src/one/one.c", "build_root") - env.Object("^/one.o", "^/one.c") - env.Program('build_dir', Dir['src/**/*.c'] + ["^/one.o"]) - end - expect(lines).to eq [ + result = run_test(rsconsfile: "carat.rb") + expect(result.stderr).to eq "" + expect(Set[*lines(result.stdout)]).to eq Set[ %q{gcc -c -o build_root/one.o -MMD -MF build_root/one.mf -Isrc/one/ -Isrc/two/ build_root/one.c}, %q{gcc -c -o build_root/src/two/two.o -MMD -MF build_root/src/two/two.mf -Isrc/one/ -Isrc/two/ src/two/two.c}, - %Q{gcc -o build_dir#{env["PROGSUFFIX"]} build_root/src/two/two.o build_root/one.o}, + %Q{gcc -o build_dir.exe build_root/src/two/two.o build_root/one.o}, ] end it 'supports simple builders' do test_dir('json_to_yaml') - Rscons::Environment.new do |env| - require 'json' - require 'yaml' - env.add_builder(:JsonToYaml) do |target, sources, cache, env, vars| - unless cache.up_to_date?(target, :JsonToYaml, sources, env) - cache.mkdir_p(File.dirname(target)) - File.open(target, 'w') do |f| - f.write(YAML.dump(JSON.load(IO.read(sources.first)))) - end - cache.register_build(target, :JsonToYaml, sources, env) - end - target - end - env.JsonToYaml('foo.yml','foo.json') - end + result = run_test + expect(result.stderr).to eq "" expect(File.exists?('foo.yml')).to be_truthy expect(IO.read('foo.yml')).to eq("---\nkey: value\n") end it 'cleans built files' do test_dir('build_dir') - Rscons::Environment.new do |env| - env.append('CPPPATH' => Dir['src/**/*/']) - env.build_dir(%r{^src/([^/]+)/}, 'build_\\1/') - env.Program('build_dir', Dir['src/**/*.c']) - end - expect(`./build_dir`).to eq "Hello from two()\n" + result = run_test + expect(result.stderr).to eq "" + expect(`./build_dir.exe`).to eq "Hello from two()\n" expect(File.exists?('build_one/one.o')).to be_truthy expect(File.exists?('build_two/two.o')).to be_truthy - Rscons.clean + result = run_test(rscons_args: %w[-c]) expect(File.exists?('build_one/one.o')).to be_falsey expect(File.exists?('build_two/two.o')).to be_falsey expect(File.exists?('build_one')).to be_falsey expect(File.exists?('build_two')).to be_falsey + expect(File.exists?('build_dir.exe')).to be_falsey expect(File.exists?('src/one/one.c')).to be_truthy end it 'does not clean created directories if other non-rscons-generated files reside there' do test_dir('build_dir') - Rscons::Environment.new do |env| - env.append('CPPPATH' => Dir['src/**/*/']) - env.build_dir(%r{^src/([^/]+)/}, 'build_\\1/') - env.Program('build_dir', Dir['src/**/*.c']) - end - expect(`./build_dir`).to eq "Hello from two()\n" + result = run_test + expect(result.stderr).to eq "" + expect(`./build_dir.exe`).to eq "Hello from two()\n" expect(File.exists?('build_one/one.o')).to be_truthy expect(File.exists?('build_two/two.o')).to be_truthy File.open('build_two/tmp', 'w') { |fh| fh.puts "dum" } - Rscons.clean + result = run_test(rscons_args: %w[-c]) expect(File.exists?('build_one/one.o')).to be_falsey expect(File.exists?('build_two/two.o')).to be_falsey expect(File.exists?('build_one')).to be_falsey expect(File.exists?('build_two')).to be_truthy + expect(File.exists?('build_dir.exe')).to be_falsey expect(File.exists?('src/one/one.c')).to be_truthy end it 'allows Ruby classes as custom builders to be used to construct files' do test_dir('custom_builder') - class MySource < Rscons::Builder - def run(target, sources, cache, env, vars) - File.open(target, 'w') do |fh| - fh.puts < ['-DONE']) - env.Program('two_sources', ['one.o', 'two.c']) - end - expect(lines).to eq [ + result = run_test + expect(result.stderr).to eq "" + expect(Set[*lines(result.stdout)]).to eq Set[ 'gcc -c -o one.o -MMD -MF one.mf -DONE one.c', 'gcc -c -o two.o -MMD -MF two.mf two.c', - "gcc -o two_sources#{env["PROGSUFFIX"]} one.o two.o", + "gcc -o two_sources.exe one.o two.o", ] - expect(File.exists?("two_sources#{env["PROGSUFFIX"]}")).to be_truthy - expect(`./two_sources`).to eq "This is a C program with two sources.\n" + expect(File.exists?("two_sources.exe")).to be_truthy + expect(`./two_sources.exe`).to eq "This is a C program with two sources.\n" end it 'builds a static library archive' do test_dir('library') - env = Rscons::Environment.new(echo: :command) do |env| - env.Program('library', ['lib.a', 'three.c']) - env.Library("lib.a", ['one.c', 'two.c'], 'CPPFLAGS' => ['-Dmake_lib']) - end - expect(lines).to eq [ + result = run_test + expect(result.stderr).to eq "" + expect(Set[*lines(result.stdout)]).to eq Set[ 'gcc -c -o one.o -MMD -MF one.mf -Dmake_lib one.c', 'gcc -c -o two.o -MMD -MF two.mf -Dmake_lib two.c', 'ar rcs lib.a one.o two.o', 'gcc -c -o three.o -MMD -MF three.mf three.c', - "gcc -o library#{env["PROGSUFFIX"]} lib.a three.o", + "gcc -o library.exe lib.a three.o", ] - expect(File.exists?("library#{env["PROGSUFFIX"]}")).to be_truthy + expect(File.exists?("library.exe")).to be_truthy expect(`ar t lib.a`).to eq "one.o\ntwo.o\n" end it 'supports build hooks to override construction variables' do test_dir("build_dir") - env = Rscons::Environment.new(echo: :command) do |env| - env.append('CPPPATH' => Dir['src/**/*/']) - env.build_dir(%r{^src/([^/]+)/}, 'build_\\1/') - env.add_build_hook do |build_op| - if build_op[:target] =~ %r{build_one/.*\.o} - build_op[:vars]["CFLAGS"] << "-O1" - elsif build_op[:target] =~ %r{build_two/.*\.o} - build_op[:vars]["CFLAGS"] << "-O2" - end - end - env.Program('build_hook.exe', Dir['src/**/*.c']) - end - expect(`./build_hook.exe`).to eq "Hello from two()\n" - expect(lines).to match_array [ + result = run_test(rsconsfile: "build_hooks.rb") + expect(result.stderr).to eq "" + expect(Set[*lines(result.stdout)]).to eq Set[ 'gcc -c -o build_one/one.o -MMD -MF build_one/one.mf -Isrc/one/ -Isrc/two/ -O1 src/one/one.c', 'gcc -c -o build_two/two.o -MMD -MF build_two/two.mf -Isrc/one/ -Isrc/two/ -O2 src/two/two.c', 'gcc -o build_hook.exe build_one/one.o build_two/two.o', ] + expect(`./build_hook.exe`).to eq "Hello from two()\n" end it 'rebuilds when user-specified dependencies change' do test_dir('simple') - env = Rscons::Environment.new do |env| - env.Program('simple.exe', Dir['*.c']).depends("file.ld") - File.open("file.ld", "w") do |fh| - fh.puts("foo") - end - end - expect(lines).to eq ["CC simple.o", "LD simple.exe"] + + File.open("program.ld", "w") {|fh| fh.puts("1")} + result = run_test(rsconsfile: "user_dependencies.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq ["CC simple.o", "LD simple.exe"] expect(File.exists?('simple.o')).to be_truthy expect(`./simple.exe`).to eq "This is a simple C program\n" - e2 = Rscons::Environment.new do |env| - program = env.Program('simple.exe', Dir['*.c']) - env.depends(program, "file.ld") - File.open("file.ld", "w") do |fh| - fh.puts("bar") - end - end - expect(lines).to eq ["LD simple.exe"] - e3 = Rscons::Environment.new do |env| - env.Program('simple.exe', Dir['*.c']) - File.unlink("file.ld") - end - expect(lines).to eq ["LD simple.exe"] - Rscons::Environment.new do |env| - env.Program('simple.exe', Dir['*.c']) - end - expect(lines).to eq [] + + File.open("program.ld", "w") {|fh| fh.puts("2")} + result = run_test(rsconsfile: "user_dependencies.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq(["LD simple.exe"]) + + File.unlink("program.ld") + result = run_test(rsconsfile: "user_dependencies.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq ["LD simple.exe"] + + result = run_test(rsconsfile: "user_dependencies.rb") + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" end unless ENV["omit_gdc_tests"] it "supports building D sources" do test_dir("d") - Rscons::Environment.new(echo: :command) do |env| - env.Program("hello-d.exe", Dir["*.d"]) - end - expect(lines).to eq [ + result = run_test + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ "gdc -c -o main.o main.d", "gdc -o hello-d.exe main.o", ] @@ -534,246 +423,157 @@ EOF it "supports disassembling object files" do test_dir("simple") - Rscons::Environment.new do |env| - env.Object("simple.o", "simple.c") - env.Disassemble("simple.txt", "simple.o") - end + result = run_test(rsconsfile: "disassemble.rb") + expect(result.stderr).to eq "" expect(File.exists?("simple.txt")).to be_truthy expect(File.read("simple.txt")).to match /Disassembly of section .text:/ end it "supports preprocessing C sources" do test_dir("simple") - Rscons::Environment.new do |env| - env.Preprocess("simplepp.c", "simple.c") - env.Program("simple", "simplepp.c") - end + result = run_test(rsconsfile: "preprocess.rb") + expect(result.stderr).to eq "" expect(File.read("simplepp.c")).to match /# \d+ "simple.c"/ - expect(`./simple`).to eq "This is a simple C program\n" + expect(`./simple.exe`).to eq "This is a simple C program\n" end it "supports preprocessing C++ sources" do test_dir("simple_cc") - Rscons::Environment.new do |env| - env.Preprocess("simplepp.cc", "simple.cc") - env.Program("simple", "simplepp.cc") - end + result = run_test(rsconsfile: "preprocess.rb") + expect(result.stderr).to eq "" expect(File.read("simplepp.cc")).to match /# \d+ "simple.cc"/ - expect(`./simple`).to eq "This is a simple C++ program\n" + expect(`./simple.exe`).to eq "This is a simple C++ program\n" end it "supports invoking builders with no sources and a build_root defined" do - class TestBuilder < Rscons::Builder - def run(target, sources, cache, env, vars) - target - end - end test_dir("simple") - Rscons::Environment.new do |env| - env.build_root = "build" - env.add_builder(TestBuilder.new) - env.TestBuilder("file") - end + result = run_test(rsconsfile: "build_root_builder_no_sources.rb") + expect(result.stderr).to eq "" end it "expands construction variables in builder target and sources before invoking the builder" do test_dir('custom_builder') - class MySource < Rscons::Builder - def run(target, sources, cache, env, vars) - File.open(target, 'w') do |fh| - fh.puts < ["-DONE"]) - env.Object("two.ooo", "two.c") - env.Program("two_sources", %w[one.oooo two.ooo]) - end - expect(File.exists?("two_sources#{env["PROGSUFFIX"]}")).to be_truthy - expect(`./two_sources`).to eq "This is a C program with two sources.\n" + result = run_test(rsconsfile: "objsuffix.rb") + expect(result.stderr).to eq "" + expect(File.exists?("two_sources.exe")).to be_truthy + expect(File.exists?("one.oooo")).to be_truthy + expect(File.exists?("two.ooo")).to be_truthy + expect(`./two_sources.exe`).to eq "This is a C program with two sources.\n" end it "supports multiple values for LIBSUFFIX" do test_dir("two_sources") - env = Rscons::Environment.new() do |env| - env["LIBSUFFIX"] = %w[.aaaa .aaa] - env.Library("one.aaaa", "one.c", "CPPFLAGS" => ["-DONE"]) - env.Library("two.aaa", "two.c") - env.Program("two_sources", %w[one.aaaa two.aaa]) - end - expect(File.exists?("two_sources#{env["PROGSUFFIX"]}")).to be_truthy - expect(`./two_sources`).to eq "This is a C program with two sources.\n" + result = run_test(rsconsfile: "libsuffix.rb") + expect(result.stderr).to eq "" + expect(File.exists?("two_sources.exe")).to be_truthy + expect(`./two_sources.exe`).to eq "This is a C program with two sources.\n" end it "supports multiple values for ASSUFFIX" do test_dir("two_sources") - env = Rscons::Environment.new() do |env| - env["ASSUFFIX"] = %w[.ssss .sss] - env["CFLAGS"] += %w[-S] - env.Object("one.ssss", "one.c", "CPPFLAGS" => ["-DONE"]) - env.Object("two.sss", "two.c") - env.Program("two_sources", %w[one.ssss two.sss], "ASFLAGS" => env["ASFLAGS"] + %w[-x assembler]) - end - expect(lines).to eq([ + result = run_test(rsconsfile: "assuffix.rb") + expect(result.stderr).to eq "" + expect(Set[*lines(result.stdout)]).to eq Set[ "CC one.ssss", "CC two.sss", "AS one.o", "AS two.o", - "LD two_sources#{env["PROGSUFFIX"]}", - ]) - expect(File.exists?("two_sources#{env["PROGSUFFIX"]}")).to be_truthy - expect(`./two_sources`).to eq "This is a C program with two sources.\n" + "LD two_sources.exe", + ] + expect(File.exists?("two_sources.exe")).to be_truthy + expect(`./two_sources.exe`).to eq "This is a C program with two sources.\n" end it "supports dumping an Environment's construction variables" do test_dir("simple") - env = Rscons::Environment.new do |env| - env["CFLAGS"] += %w[-O2 -fomit-frame-pointer] - env[:foo] = :bar - end - env.dump - result = lines - expect(result.include?(%{:foo => :bar})).to be_truthy - expect(result.include?(%{CFLAGS => ["-O2", "-fomit-frame-pointer"]})).to be_truthy - expect(result.include?(%{CPPPATH => []})).to be_truthy + result = run_test(rsconsfile: "dump.rb") + expect(result.stderr).to eq "" + slines = lines(result.stdout) + expect(slines.include?(%{:foo => :bar})).to be_truthy + expect(slines.include?(%{CFLAGS => ["-O2", "-fomit-frame-pointer"]})).to be_truthy + expect(slines.include?(%{CPPPATH => []})).to be_truthy end it "considers deep dependencies when deciding whether to rerun Preprocess builder" do test_dir("preprocess") - env = Rscons::Environment.new do |env| - env.Preprocess("pp", "foo.h") - end + + result = run_test + expect(result.stderr).to eq "" + expect(result.stdout).to eq("Preprocess pp\n") expect(File.read("pp")).to match(%r{xyz42abc}m) - expect(lines).to eq(["Preprocess pp"]) - env.Preprocess("pp", "foo.h") - env.process - expect(lines).to eq([]) + + result = run_test + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" + File.open("bar.h", "w") do |fh| fh.puts "#define BAR abc88xyz" end - $ttt = true - env.Preprocess("pp", "foo.h") - env.process - expect(lines).to eq(["Preprocess pp"]) + result = run_test + expect(result.stderr).to eq "" + expect(result.stdout).to eq("Preprocess pp\n") expect(File.read("pp")).to match(%r{abc88xyz}m) end it "allows construction variable references which expand to arrays in sources of a build target" do test_dir("simple") - Rscons::Environment.new do |env| - env["sources"] = Dir["*.c"] - env.Program("simple", "${sources}") - end + result = run_test(rsconsfile: "cvar_array.rb") + expect(result.stderr).to eq "" expect(File.exists?("simple.o")).to be_truthy - expect(`./simple`).to eq "This is a simple C program\n" + expect(`./simple.exe`).to eq "This is a simple C program\n" end it "supports registering multiple build targets with the same target path" do test_dir("build_dir") - Rscons::Environment.new do |env| - env["CPPPATH"] << "src/two" - env.Object("one.o", "src/one/one.c") - env.Object("one.o", "src/two/two.c") - end + result = run_test(rsconsfile: "multiple_targets_same_name.rb") + expect(result.stderr).to eq "" expect(File.exists?("one.o")).to be_truthy - expect(lines).to eq([ + expect(lines(result.stdout)).to eq([ "CC one.o", "CC one.o", ]) @@ -781,19 +581,11 @@ EOF it "expands target and source paths when builders are registered in build hooks" do test_dir("build_dir") - Rscons::Environment.new do |env| - env["CPPPATH"] << "src/two" - env.Object("one.o", "src/one/one.c") - env.add_post_build_hook do |build_op| - if build_op[:target] == "one.o" - env["MODULE"] = "two" - env.Object("${MODULE}.o", "src/${MODULE}/${MODULE}.c") - end - end - end + result = run_test(rsconsfile: "post_build_hook_expansion.rb") + expect(result.stderr).to eq "" expect(File.exists?("one.o")).to be_truthy expect(File.exists?("two.o")).to be_truthy - expect(lines).to eq([ + expect(lines(result.stdout)).to eq([ "CC one.o", "CC two.o", ]) @@ -804,208 +596,168 @@ EOF File.open("two.c", "w") do |fh| fh.puts("FOO") end - expect do - Rscons::Environment.new do |env| - env.Program("simple.exe", %w[simple.c two.c]) - end - end.to raise_error /Failed to build two\.o/ - result = lines - expect(Set[*result]).to eq Set[ - "CC simple.o", - "CC two.o", - ] + result = run_test(rsconsfile: "cache_successful_builds_when_one_fails.rb", + rscons_args: %w[-j1]) + expect(result.stderr).to match /FOO/ expect(File.exists?("simple.o")).to be_truthy expect(File.exists?("two.o")).to be_falsey - expect(File.exists?("simple.exe")).to be_falsey - - Rscons::Cache.reset! File.open("two.c", "w") {|fh|} - Rscons::Environment.new do |env| - env.Program("simple.exe", %w[simple.c two.c]) - end - expect(lines).to eq [ + result = run_test(rsconsfile: "cache_successful_builds_when_one_fails.rb", + rscons_args: %w[-j1]) + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ "CC two.o", - "LD simple.exe", ] end context "Directory builder" do it "creates the requested directory" do test_dir("simple") - Rscons::Environment.new do |env| - env.Directory("teh_dir") - end + result = run_test(rsconsfile: "directory.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq(["Directory teh_dir"]) expect(File.directory?("teh_dir")).to be_truthy - expect(lines).to eq(["Directory teh_dir"]) end it "succeeds when the requested directory already exists" do test_dir("simple") FileUtils.mkdir("teh_dir") - Rscons::Environment.new do |env| - env.Directory("teh_dir") - end + result = run_test(rsconsfile: "directory.rb") + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" expect(File.directory?("teh_dir")).to be_truthy - expect(lines).to eq([]) end it "fails when the target path is a file" do test_dir("simple") FileUtils.touch("teh_dir") - expect do - Rscons::Environment.new do |env| - env.Directory("teh_dir") - end - end.to raise_error /Failed to build teh_dir/ - expect(lines).to eq([ - "Error: `teh_dir' already exists and is not a directory", - ]) + result = run_test(rsconsfile: "directory.rb") + expect(result.stderr).to match %r{Error: `teh_dir' already exists and is not a directory} end end context "Install buildler" do - let(:base_env) do - test_dir("build_dir") - Rscons::Environment.new do |env| - env["CPPPATH"] = Dir["src/**/"] - env["sources"] = Dir["src/**/*.c"] - env.Program("simple.exe", "${sources}") - end - end - it "copies a file to the target file name" do - env = base_env.clone do |env| - lines - env.Install("inst.exe", "simple.exe") - end - expect(lines).to eq(["Install inst.exe"]) - env.Install("inst.exe", "simple.exe") - env.process - expect(lines).to eq([]) + test_dir("build_dir") + + result = run_test(rsconsfile: "install.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq ["Install inst.exe"] + + result = run_test(rsconsfile: "install.rb") + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" + expect(File.exists?("inst.exe")).to be_truthy - expect(File.read("inst.exe", mode: "rb")).to eq(File.read("simple.exe", mode: "rb")) + expect(File.read("inst.exe", mode: "rb")).to eq(File.read("install.rb", mode: "rb")) + FileUtils.rm("inst.exe") - env.Install("inst.exe", "simple.exe") - env.process - expect(lines).to eq(["Install inst.exe"]) + result = run_test(rsconsfile: "install.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq ["Install inst.exe"] end it "operates the same as a Copy builder" do - env = base_env.clone do |env| - lines - env.Copy("inst.exe", "simple.exe") - end - expect(lines).to eq(["Copy inst.exe"]) - env.Copy("inst.exe", "simple.exe") - env.process - expect(lines).to eq([]) + test_dir("build_dir") + + result = run_test(rsconsfile: "copy.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq ["Copy inst.exe"] + + result = run_test(rsconsfile: "copy.rb") + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" + expect(File.exists?("inst.exe")).to be_truthy - expect(File.read("inst.exe", mode: "rb")).to eq(File.read("simple.exe", mode: "rb")) + expect(File.read("inst.exe", mode: "rb")).to eq(File.read("copy.rb", mode: "rb")) + FileUtils.rm("inst.exe") - env.Copy("inst.exe", "simple.exe") - env.process - expect(lines).to eq(["Copy inst.exe"]) + result = run_test(rsconsfile: "copy.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq ["Copy inst.exe"] end it "copies a file to the target directory name" do - env = base_env.clone do |env| - lines - env.Directory("inst") - env.Install("inst", "simple.exe") - end - expect(lines).to eq([ - "Directory inst", - "Install inst", - ]) - env.Install("inst", "simple.exe") - env.process - expect(lines).to eq([]) - expect(File.exists?("inst/simple.exe")).to be_truthy - expect(File.read("inst/simple.exe", mode: "rb")).to eq(File.read("simple.exe", mode: "rb")) + test_dir "build_dir" + + result = run_test(rsconsfile: "install_directory.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to include("Install inst") + expect(File.exists?("inst/install_directory.rb")).to be_truthy + expect(File.read("inst/install_directory.rb", mode: "rb")).to eq(File.read("install_directory.rb", mode: "rb")) + + result = run_test(rsconsfile: "install_directory.rb") + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" end it "copies a directory to the non-existent target directory name" do - env = base_env.clone do |env| - lines - env.Install("dist/src", "src") - end - expect(lines).to eq(["Install dist/src"]) - env.Install("dist/src", "src") - env.process - expect(lines).to eq([]) + test_dir "build_dir" + result = run_test(rsconsfile: "install_directory.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to include("Install noexist/src") %w[src/one/one.c src/two/two.c src/two/two.h].each do |f| - expect(File.exists?("dist/#{f}")).to be_truthy - expect(File.read("dist/#{f}", mode: "rb")).to eq(File.read(f, mode: "rb")) + expect(File.exists?("noexist/#{f}")).to be_truthy + expect(File.read("noexist/#{f}", mode: "rb")).to eq(File.read(f, mode: "rb")) end end it "copies a directory to the existent target directory name" do - env = base_env.clone do |env| - lines - env.Directory("dist/src") - env.Install("dist/src", "src") - end - expect(lines).to eq([ - "Directory dist/src", - "Install dist/src", - ]) - env.Install("dist/src", "src") - env.process - expect(lines).to eq([]) + test_dir "build_dir" + result = run_test(rsconsfile: "install_directory.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to include("Install exist/src") %w[src/one/one.c src/two/two.c src/two/two.h].each do |f| - expect(File.exists?("dist/#{f}")).to be_truthy - expect(File.read("dist/#{f}", mode: "rb")).to eq(File.read(f, mode: "rb")) + expect(File.exists?("exist/#{f}")).to be_truthy + expect(File.read("exist/#{f}", mode: "rb")).to eq(File.read(f, mode: "rb")) end - FileUtils.rm("dist/src/two/two.c") - env.Install("dist/src", "src") - env.process - expect(lines).to eq(["Install dist/src"]) end end context "phony targets" do [false, true].each do |with_build_root| - it "allows specifying a Symbol as a target name and reruns the builder if the sources or command have changed" do - test_dir("simple") - env = Rscons::Environment.new do |env| - env.build_root = "bld" if with_build_root - env.add_builder(:Checker) do |target, sources, cache, env, vars| - unless cache.up_to_date?(target, :Checker, sources, env) - puts "Checker #{sources.first}" if env.echo != :off - cache.register_build(target, :Checker, sources, env) - end - target + context "with#{with_build_root ? "" : "out"} build root" do + it "allows specifying a Symbol as a target name and reruns the builder if the sources or command have changed" do + test_dir("simple") + + FileUtils.cp("phony_target.rb", "phony_target2.rb") + unless with_build_root + file_sub("phony_target2.rb") {|line| line.sub(/.*build_root.*/, "")} end - env.Program("simple.exe", "simple.c") - env.Checker(:checker, "simple.exe") - end - expect(lines).to eq(["CC #{with_build_root ? 'bld/' : ''}simple.o", "LD simple.exe", "Checker simple.exe"]) + result = run_test(rsconsfile: "phony_target2.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq([ + "CC #{with_build_root ? "build/" : ""}simple.o", + "LD simple.exe", + "Checker simple.exe", + ]) - env.clone do |env| - env.Checker(:checker, "simple.exe") - end - expect(lines).to eq([]) + result = run_test(rsconsfile: "phony_target2.rb") + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" - File.open("simple.exe", "w") do |fh| - fh.puts "Changed simple.exe" + FileUtils.cp("phony_target.rb", "phony_target2.rb") + file_sub("phony_target2.rb") {|line| line.sub(/.*Program.*/, "")} + File.open("simple.exe", "w") do |fh| + fh.puts "Changed simple.exe" + end + result = run_test(rsconsfile: "phony_target2.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq([ + "Checker simple.exe", + ]) end - env.clone do |env| - env.Checker(:checker, "simple.exe") - end - expect(lines).to eq(["Checker simple.exe"]) end end end context "Environment#clear_targets" do it "clears registered targets" do - test_dir('header') - env = Rscons::Environment.new do |env| - env.Program('header', Dir['*.c']) - env.clear_targets - end - expect(lines).to eq [] + test_dir("simple") + result = run_test(rsconsfile: "clear_targets.rb") + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" end end From bfbbc19728ed0828bfacc6e2a301aa334af6226d Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Mon, 22 May 2017 11:19:55 -0400 Subject: [PATCH 33/69] fix simplecov setup to get coverage for integration build tests --- spec/build_tests_spec.rb | 15 ++++++++++----- spec/spec_helper.rb | 2 ++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 3d150e1..8ab2d4b 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -57,12 +57,11 @@ describe Rscons do def run_test(options = {}) rsconsfile = options[:rsconsfile] || "Rsconsfile" rscons_args = options[:rscons_args] || [] - command = %w[rscons -f _build_test.rb] + rscons_args + command = %W[ruby -I. -r _simplecov_setup #{@owd}/bin/rscons -f #{rsconsfile}] + rscons_args @statics[:build_test_id] ||= 0 @statics[:build_test_id] += 1 command_name = "b#{@statics[:build_test_id]}" - rsconsfile_contents = File.read(rsconsfile) - File.open("_build_test.rb", "w") do |fh| + File.open("_simplecov_setup.rb", "w") do |fh| fh.puts < Date: Mon, 22 May 2017 15:55:18 -0400 Subject: [PATCH 34/69] migrate some Cache tests to integration tests --- build_tests/simple/Rsconsfile | 2 +- build_tests/simple/cache_command_change.rb | 4 + .../simple/cache_dep_checksum_change.rb | 3 + build_tests/simple/cache_new_dep1.rb | 6 + build_tests/simple/cache_new_dep2.rb | 5 + build_tests/simple/cache_user_dep.rb | 5 + build_tests/two_sources/cache_strict_deps.rb | 19 ++ spec/build_tests_spec.rb | 171 +++++++++++++++- spec/rscons/cache_spec.rb | 188 ------------------ 9 files changed, 213 insertions(+), 190 deletions(-) create mode 100644 build_tests/simple/cache_command_change.rb create mode 100644 build_tests/simple/cache_dep_checksum_change.rb create mode 100644 build_tests/simple/cache_new_dep1.rb create mode 100644 build_tests/simple/cache_new_dep2.rb create mode 100644 build_tests/simple/cache_user_dep.rb create mode 100644 build_tests/two_sources/cache_strict_deps.rb diff --git a/build_tests/simple/Rsconsfile b/build_tests/simple/Rsconsfile index 9775f9a..06b9b79 100644 --- a/build_tests/simple/Rsconsfile +++ b/build_tests/simple/Rsconsfile @@ -1,3 +1,3 @@ Rscons::Environment.new do |env| - env.Program('simple', Dir['*.c']) + env.Program('simple.exe', Dir['*.c']) end diff --git a/build_tests/simple/cache_command_change.rb b/build_tests/simple/cache_command_change.rb new file mode 100644 index 0000000..ad6ec2d --- /dev/null +++ b/build_tests/simple/cache_command_change.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new do |env| + env["LIBS"] += ["c"] + env.Program('simple.exe', Dir['*.c']) +end diff --git a/build_tests/simple/cache_dep_checksum_change.rb b/build_tests/simple/cache_dep_checksum_change.rb new file mode 100644 index 0000000..dc12bd5 --- /dev/null +++ b/build_tests/simple/cache_dep_checksum_change.rb @@ -0,0 +1,3 @@ +Rscons::Environment.new do |env| + env.Copy("simple.copy", "simple.c") +end diff --git a/build_tests/simple/cache_new_dep1.rb b/build_tests/simple/cache_new_dep1.rb new file mode 100644 index 0000000..0a2307a --- /dev/null +++ b/build_tests/simple/cache_new_dep1.rb @@ -0,0 +1,6 @@ +Rscons::Environment.new do |env| + env.Object("simple.o", "simple.c") + env.process + env["LDCMD"] = %w[gcc -o ${_TARGET} simple.o] + env.Program('simple.exe', []) +end diff --git a/build_tests/simple/cache_new_dep2.rb b/build_tests/simple/cache_new_dep2.rb new file mode 100644 index 0000000..2308920 --- /dev/null +++ b/build_tests/simple/cache_new_dep2.rb @@ -0,0 +1,5 @@ +Rscons::Environment.new do |env| + env.Object("simple.o", "simple.c") + env["LDCMD"] = %w[gcc -o ${_TARGET} simple.o] + env.Program('simple.exe', ["simple.o"]) +end diff --git a/build_tests/simple/cache_user_dep.rb b/build_tests/simple/cache_user_dep.rb new file mode 100644 index 0000000..db7e028 --- /dev/null +++ b/build_tests/simple/cache_user_dep.rb @@ -0,0 +1,5 @@ +Rscons::Environment.new do |env| + env.Program("simple.exe", "simple.c") + user_deps = File.read("user_deps", mode: "rb").split(" ") + env.depends("simple.exe", *user_deps) +end diff --git a/build_tests/two_sources/cache_strict_deps.rb b/build_tests/two_sources/cache_strict_deps.rb new file mode 100644 index 0000000..a20ad83 --- /dev/null +++ b/build_tests/two_sources/cache_strict_deps.rb @@ -0,0 +1,19 @@ +class StrictBuilder < Rscons::Builder + def run(options) + target, sources, cache, env = options.values_at(:target, :sources, :cache, :env) + command = %W[gcc -o #{target}] + sources.sort + unless cache.up_to_date?(target, command, sources, env, strict_deps: true) + return false unless env.execute("StrictBuilder #{target}", command) + cache.register_build(target, command, sources, env) + end + target + end +end + +Rscons::Environment.new(echo: :command) do |env| + env.add_builder(StrictBuilder.new) + env.Object("one.o", "one.c", "CCFLAGS" => %w[-DONE]) + env.Object("two.o", "two.c") + sources = File.read("sources", mode: "rb").split(" ") + env.StrictBuilder("program.exe", sources) +end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 8ab2d4b..6c27976 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -106,7 +106,7 @@ EOF result = run_test expect(result.stderr).to eq "" expect(File.exists?('simple.o')).to be_truthy - expect(`./simple`).to eq "This is a simple C program\n" + expect(`./simple.exe`).to eq "This is a simple C program\n" end it 'prints commands as they are executed' do @@ -766,4 +766,173 @@ EOF end end + context "Cache management" do + it "prints a warning when the cache is corrupt" do + test_dir("simple") + File.open(Rscons::Cache::CACHE_FILE, "w") do |fh| + fh.puts("[1]") + end + result = run_test + expect(result.stderr).to match /Warning.*was corrupt. Contents:/ + end + + it "removes the cache file on a clean operation" do + test_dir("simple") + result = run_test + expect(result.stderr).to eq "" + expect(File.exists?(Rscons::Cache::CACHE_FILE)).to be_truthy + result = run_test(rscons_args: %w[-c]) + expect(result.stderr).to eq "" + expect(File.exists?(Rscons::Cache::CACHE_FILE)).to be_falsey + end + + it "forces a build when the target file does not exist and is not in the cache" do + test_dir("simple") + expect(File.exists?("simple.exe")).to be_falsey + result = run_test + expect(result.stderr).to eq "" + expect(File.exists?("simple.exe")).to be_truthy + end + + it "forces a build when the target file does exist but is not in the cache" do + test_dir("simple") + File.open("simple.exe", "wb") do |fh| + fh.write("hi") + end + result = run_test + expect(result.stderr).to eq "" + expect(File.exists?("simple.exe")).to be_truthy + expect(File.read("simple.exe", mode: "rb")).to_not eq "hi" + end + + it "forces a build when the target file exists and is in the cache but has changed since cached" do + test_dir("simple") + result = run_test + expect(result.stderr).to eq "" + File.open("simple.exe", "wb") do |fh| + fh.write("hi") + end + test_dir("simple") + result = run_test + expect(result.stderr).to eq "" + expect(File.exists?("simple.exe")).to be_truthy + expect(File.read("simple.exe", mode: "rb")).to_not eq "hi" + end + + it "forces a build when the command changes" do + test_dir("simple") + + result = run_test + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "CC simple.o", + "LD simple.exe", + ] + + result = run_test(rsconsfile: "cache_command_change.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "LD simple.exe", + ] + end + + it "forces a build when there is a new dependency" do + test_dir("simple") + + result = run_test(rsconsfile: "cache_new_dep1.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "CC simple.o", + "LD simple.exe", + ] + + result = run_test(rsconsfile: "cache_new_dep2.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "LD simple.exe", + ] + end + + it "forces a build when a dependency's checksum has changed" do + test_dir("simple") + + result = run_test(rsconsfile: "cache_dep_checksum_change.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq ["Copy simple.copy"] + File.open("simple.c", "wb") do |fh| + fh.write("hi") + end + + result = run_test(rsconsfile: "cache_dep_checksum_change.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq ["Copy simple.copy"] + end + + it "forces a rebuild with strict_deps=true when dependency order changes" do + test_dir("two_sources") + + File.open("sources", "wb") do |fh| + fh.write("one.o two.o") + end + result = run_test(rsconsfile: "cache_strict_deps.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to include "gcc -o program.exe one.o two.o" + + result = run_test(rsconsfile: "cache_strict_deps.rb") + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" + + File.open("sources", "wb") do |fh| + fh.write("two.o one.o") + end + result = run_test(rsconsfile: "cache_strict_deps.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to include "gcc -o program.exe one.o two.o" + end + + it "forces a rebuild when there is a new user dependency" do + test_dir("simple") + + File.open("foo", "wb") {|fh| fh.write("hi")} + File.open("user_deps", "wb") {|fh| fh.write("")} + result = run_test(rsconsfile: "cache_user_dep.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "CC simple.o", + "LD simple.exe", + ] + + File.open("user_deps", "wb") {|fh| fh.write("foo")} + result = run_test(rsconsfile: "cache_user_dep.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "LD simple.exe", + ] + end + + it "forces a rebuild when a user dependency file checksum has changed" do + test_dir("simple") + + File.open("foo", "wb") {|fh| fh.write("hi")} + File.open("user_deps", "wb") {|fh| fh.write("foo")} + result = run_test(rsconsfile: "cache_user_dep.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "CC simple.o", + "LD simple.exe", + ] + + result = run_test(rsconsfile: "cache_user_dep.rb") + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" + + File.open("foo", "wb") {|fh| fh.write("hi2")} + result = run_test(rsconsfile: "cache_user_dep.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "LD simple.exe", + ] + end + end + end diff --git a/spec/rscons/cache_spec.rb b/spec/rscons/cache_spec.rb index 4f5d915..4c15764 100644 --- a/spec/rscons/cache_spec.rb +++ b/spec/rscons/cache_spec.rb @@ -13,194 +13,6 @@ module Rscons end end - describe "#initialize" do - context "when corrupt" do - it "prints a warning and defaults to an empty hash" do - expect(JSON).to receive(:load).and_return("string") - expect($stderr).to receive(:puts).with(/Warning:.*was.corrupt/) - c = Cache.instance - c.send(:initialize!) - expect(c.instance_variable_get(:@cache).is_a?(Hash)).to be_truthy - end - end - end - - describe "#clear" do - it "removes the cache file" do - expect(FileUtils).to receive(:rm_f).with(Cache::CACHE_FILE) - allow(JSON).to receive(:load) {{}} - Cache.instance.clear - end - end - - describe "#write" do - it "fills in 'version' and write to file" do - cache = {} - fh = $stdout - expect(fh).to receive(:puts) - expect(File).to receive(:open).and_yield(fh) - build_from(cache).write - expect(cache["version"]).to eq Rscons::VERSION - end - end - - describe "#up_to_date?" do - empty_env = "env" - before do - allow(empty_env).to receive(:get_user_deps) { nil } - end - - it "returns false when target file does not exist" do - expect(File).to receive(:exists?).with("target").and_return(false) - expect(build_from({}).up_to_date?("target", "command", [], empty_env)).to be_falsey - end - - it "returns false when target is not registered in the cache" do - expect(File).to receive(:exists?).with("target").and_return(true) - expect(build_from({}).up_to_date?("target", "command", [], empty_env)).to be_falsey - end - - it "returns false when the target's checksum does not match" do - _cache = {"targets" => {"target" => {"checksum" => "abc"}}} - cache = build_from(_cache) - expect(File).to receive(:exists?).with("target").and_return(true) - expect(cache).to receive(:calculate_checksum).with("target").and_return("def") - expect(cache.up_to_date?("target", "command", [], empty_env)).to be_falsey - end - - it "returns false when the build command has changed" do - _cache = {"targets" => {"target" => {"checksum" => "abc", "command" => Digest::MD5.hexdigest("old command".inspect)}}} - cache = build_from(_cache) - expect(File).to receive(:exists?).with("target").and_return(true) - expect(cache).to receive(:calculate_checksum).with("target").and_return("abc") - expect(cache.up_to_date?("target", "command", [], empty_env)).to be_falsey - end - - it "returns false when there is a new dependency" do - _cache = {"targets" => {"target" => {"checksum" => "abc", - "command" => Digest::MD5.hexdigest("command".inspect), - "deps" => [{"fname" => "dep.1"}]}}} - cache = build_from(_cache) - expect(File).to receive(:exists?).with("target").and_return(true) - expect(cache).to receive(:calculate_checksum).with("target").and_return("abc") - expect(cache.up_to_date?("target", "command", ["dep.1", "dep.2"], empty_env)).to be_falsey - end - - it "returns false when a dependency's checksum has changed" do - _cache = {"targets" => {"target" => {"checksum" => "abc", - "command" => Digest::MD5.hexdigest("command".inspect), - "deps" => [{"fname" => "dep.1", - "checksum" => "dep.1.chk"}, - {"fname" => "dep.2", - "checksum" => "dep.2.chk"}, - {"fname" => "extra.dep", - "checksum" => "extra.dep.chk"}], - "user_deps" => []}}} - cache = build_from(_cache) - expect(File).to receive(:exists?).with("target").and_return(true) - expect(cache).to receive(:calculate_checksum).with("target").and_return("abc") - expect(cache).to receive(:calculate_checksum).with("dep.1").and_return("dep.1.chk") - expect(cache).to receive(:calculate_checksum).with("dep.2").and_return("dep.2.changed") - expect(cache.up_to_date?("target", "command", ["dep.1", "dep.2"], empty_env)).to be_falsey - end - - it "returns false with strict_deps=true when cache has an extra dependency" do - _cache = {"targets" => {"target" => {"checksum" => "abc", - "command" => Digest::MD5.hexdigest("command".inspect), - "deps" => [{"fname" => "dep.1", - "checksum" => "dep.1.chk"}, - {"fname" => "dep.2", - "checksum" => "dep.2.chk"}, - {"fname" => "extra.dep", - "checksum" => "extra.dep.chk"}], - "user_deps" => []}}} - cache = build_from(_cache) - expect(File).to receive(:exists?).with("target").and_return(true) - expect(cache).to receive(:calculate_checksum).with("target").and_return("abc") - expect(cache.up_to_date?("target", "command", ["dep.1", "dep.2"], empty_env, strict_deps: true)).to be_falsey - end - - it "returns false when there is a new user dependency" do - _cache = {"targets" => {"target" => {"checksum" => "abc", - "command" => Digest::MD5.hexdigest("command".inspect), - "deps" => [{"fname" => "dep.1"}], - "user_deps" => []}}} - cache = build_from(_cache) - env = "env" - expect(env).to receive(:get_user_deps).with("target").and_return(["file.ld"]) - expect(File).to receive(:exists?).with("target").and_return(true) - expect(cache).to receive(:calculate_checksum).with("target").and_return("abc") - expect(cache.up_to_date?("target", "command", ["dep.1"], env)).to be_falsey - end - - it "returns false when a user dependency checksum has changed" do - _cache = {"targets" => {"target" => {"checksum" => "abc", - "command" => Digest::MD5.hexdigest("command".inspect), - "deps" => [{"fname" => "dep.1", - "checksum" => "dep.1.chk"}, - {"fname" => "dep.2", - "checksum" => "dep.2.chk"}, - {"fname" => "extra.dep", - "checksum" => "extra.dep.chk"}], - "user_deps" => [{"fname" => "user.dep", - "checksum" => "user.dep.chk"}]}}} - cache = build_from(_cache) - env = "env" - expect(env).to receive(:get_user_deps).with("target").and_return(["user.dep"]) - expect(File).to receive(:exists?).with("target").and_return(true) - expect(cache).to receive(:calculate_checksum).with("target").and_return("abc") - expect(cache).to receive(:calculate_checksum).with("dep.1").and_return("dep.1.chk") - expect(cache).to receive(:calculate_checksum).with("dep.2").and_return("dep.2.chk") - expect(cache).to receive(:calculate_checksum).with("extra.dep").and_return("extra.dep.chk") - expect(cache).to receive(:calculate_checksum).with("user.dep").and_return("INCORRECT") - expect(cache.up_to_date?("target", "command", ["dep.1", "dep.2"], env)).to be_falsey - end - - it "returns true when no condition for false is met" do - _cache = {"targets" => {"target" => {"checksum" => "abc", - "command" => Digest::MD5.hexdigest("command".inspect), - "deps" => [{"fname" => "dep.1", - "checksum" => "dep.1.chk"}, - {"fname" => "dep.2", - "checksum" => "dep.2.chk"}, - {"fname" => "extra.dep", - "checksum" => "extra.dep.chk"}], - "user_deps" => []}}} - cache = build_from(_cache) - expect(File).to receive(:exists?).with("target").and_return(true) - expect(cache).to receive(:calculate_checksum).with("target").and_return("abc") - expect(cache).to receive(:calculate_checksum).with("dep.1").and_return("dep.1.chk") - expect(cache).to receive(:calculate_checksum).with("dep.2").and_return("dep.2.chk") - expect(cache).to receive(:calculate_checksum).with("extra.dep").and_return("extra.dep.chk") - expect(cache.up_to_date?("target", "command", ["dep.1", "dep.2"], empty_env)).to be_truthy - end - end - - describe "#register_build" do - it "stores the given information in the cache" do - _cache = {} - cache = build_from(_cache) - env = "env" - expect(env).to receive(:get_user_deps).with("the target").and_return(["user.dep"]) - expect(cache).to receive(:calculate_checksum).with("the target").and_return("the checksum") - expect(cache).to receive(:calculate_checksum).with("dep 1").and_return("dep 1 checksum") - expect(cache).to receive(:calculate_checksum).with("dep 2").and_return("dep 2 checksum") - expect(cache).to receive(:calculate_checksum).with("user.dep").and_return("user.dep checksum") - cache.register_build("the target", "the command", ["dep 1", "dep 2"], env) - cached_target = cache.instance_variable_get(:@cache)["targets"]["the target"] - expect(cached_target).to_not be_nil - expect(cached_target["command"]).to eq Digest::MD5.hexdigest("the command".inspect) - expect(cached_target["checksum"]).to eq "the checksum" - expect(cached_target["deps"]).to eq [ - {"fname" => "dep 1", "checksum" => "dep 1 checksum"}, - {"fname" => "dep 2", "checksum" => "dep 2 checksum"}, - ] - expect(cached_target["user_deps"]).to eq [ - {"fname" => "user.dep", "checksum" => "user.dep checksum"}, - ] - end - end - describe "#targets" do it "returns a list of targets that are cached" do cache = {"targets" => {"t1" => {}, "t2" => {}, "t3" => {}}} From 7b3bffd329ce57d5f30a94e34b54723d9864a775 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Mon, 22 May 2017 16:25:49 -0400 Subject: [PATCH 35/69] update some non-integration-test specs --- spec/rscons/builders/library_spec.rb | 14 ++++++- spec/rscons/builders/object_spec.rb | 23 +++++++++-- spec/rscons/builders/program_spec.rb | 14 ++++++- spec/rscons/environment_spec.rb | 62 +--------------------------- 4 files changed, 45 insertions(+), 68 deletions(-) diff --git a/spec/rscons/builders/library_spec.rb b/spec/rscons/builders/library_spec.rb index c8d82f1..6885322 100644 --- a/spec/rscons/builders/library_spec.rb +++ b/spec/rscons/builders/library_spec.rb @@ -6,12 +6,22 @@ module Rscons it "supports overriding AR construction variable" do expect(subject).to receive(:standard_build).with("AR prog.a", "prog.a", ["sp-ar", "rcs", "prog.a", "prog.o"], ["prog.o"], env, :cache) - subject.run("prog.a", ["prog.o"], :cache, env, "AR" => "sp-ar") + subject.run( + target: "prog.a", + sources: ["prog.o"], + cache: :cache, + env: env, + vars: {"AR" => "sp-ar"}) end it "supports overriding ARCMD construction variable" do expect(subject).to receive(:standard_build).with("AR prog.a", "prog.a", ["special", "AR!", "prog.o"], ["prog.o"], env, :cache) - subject.run("prog.a", ["prog.o"], :cache, env, "ARCMD" => ["special", "AR!", "${_SOURCES}"]) + subject.run( + target: "prog.a", + sources: ["prog.o"], + cache: :cache, + env: env, + vars: {"ARCMD" => ["special", "AR!", "${_SOURCES}"]}) end end end diff --git a/spec/rscons/builders/object_spec.rb b/spec/rscons/builders/object_spec.rb index 7845eab..358d829 100644 --- a/spec/rscons/builders/object_spec.rb +++ b/spec/rscons/builders/object_spec.rb @@ -13,7 +13,12 @@ module Rscons expect(File).to receive(:exists?).and_return(false) expect(cache).to receive(:register_build) - subject.run("mod.o", ["mod.c"], cache, env, "CCCMD" => ["llc", "${_SOURCES}"]) + subject.run( + target: "mod.o", + sources: ["mod.c"], + cache: cache, + env: env, + vars: {"CCCMD" => ["llc", "${_SOURCES}"]}) end it "supports overriding DEPFILESUFFIX construction variable" do @@ -24,11 +29,23 @@ module Rscons expect(File).to receive(:exists?).with("f.d").and_return(false) expect(cache).to receive(:register_build) - subject.run("f.o", ["in.c"], cache, env, "DEPFILESUFFIX" => ".d") + subject.run( + target: "f.o", + sources: ["in.c"], + cache: cache, + env: env, + vars: {"DEPFILESUFFIX" => ".d"}) end it "raises an error when given a source file with an unknown suffix" do - expect { subject.run("mod.o", ["mod.xyz"], :cache, env, {}) }.to raise_error /unknown input file type: "mod.xyz"/ + expect do + subject.run( + target: "mod.o", + sources: ["mod.xyz"], + cache: :cache, + env: env, + vars: {}) + end.to raise_error /unknown input file type: "mod.xyz"/ end end end diff --git a/spec/rscons/builders/program_spec.rb b/spec/rscons/builders/program_spec.rb index ec35081..7a42009 100644 --- a/spec/rscons/builders/program_spec.rb +++ b/spec/rscons/builders/program_spec.rb @@ -6,12 +6,22 @@ module Rscons it "supports overriding CC construction variable" do expect(subject).to receive(:standard_build).with("LD prog", "prog", ["sp-c++", "-o", "prog", "prog.o"], ["prog.o"], env, :cache) - subject.run("prog", ["prog.o"], :cache, env, "CC" => "sp-c++") + subject.run( + target: "prog", + sources: ["prog.o"], + cache: :cache, + env: env, + vars: {"CC" => "sp-c++"}) end it "supports overriding LDCMD construction variable" do expect(subject).to receive(:standard_build).with("LD prog.exe", "prog.exe", ["special", "LD!", "prog.o"], ["prog.o"], env, :cache) - subject.run("prog.exe", ["prog.o"], :cache, env, "LDCMD" => ["special", "LD!", "${_SOURCES}"]) + subject.run( + target: "prog.exe", + sources: ["prog.o"], + cache: :cache, + env: env, + vars: {"LDCMD" => ["special", "LD!", "${_SOURCES}"]}) end end end diff --git a/spec/rscons/environment_spec.rb b/spec/rscons/environment_spec.rb index d6ac557..9efd325 100644 --- a/spec/rscons/environment_spec.rb +++ b/spec/rscons/environment_spec.rb @@ -162,65 +162,6 @@ module Rscons end end - describe "#process" do - it "runs builders for all of the targets specified" do - env = Environment.new - env.Program("a.out", "main.c") - - cache = "cache" - expect(Cache).to receive(:instance).and_return(cache) - allow(cache).to receive(:clear_checksum_cache!) - expect(env).to receive(:run_builder).with(anything, "a.out", ["main.c"], cache, {}, allow_delayed_execution: true, setup_info: nil).and_return(true) - expect(cache).to receive(:write) - - env.process - end - - it "builds dependent targets first" do - env = Environment.new - env.Program("a.out", "main.o") - env.Object("main.o", "other.cc") - - cache = "cache" - expect(Cache).to receive(:instance).and_return(cache) - allow(cache).to receive(:clear_checksum_cache!) - expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}, allow_delayed_execution: true, setup_info: nil).and_return("main.o") - expect(env).to receive(:run_builder).with(anything, "a.out", ["main.o"], cache, {}, allow_delayed_execution: true, setup_info: nil).and_return("a.out") - expect(cache).to receive(:write) - - env.process - end - - it "raises a BuildError when building fails" do - env = Environment.new - env.Program("a.out", "main.o") - env.Object("main.o", "other.cc") - - cache = "cache" - expect(Cache).to receive(:instance).and_return(cache) - allow(cache).to receive(:clear_checksum_cache!) - expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}, allow_delayed_execution: true, setup_info: nil).and_return(false) - expect(cache).to receive(:write) - - expect { env.process }.to raise_error BuildError, /Failed.to.build.main.o/ - end - - it "writes the cache when the Builder raises an exception" do - env = Environment.new - env.Object("module.o", "module.c") - - cache = "cache" - expect(Cache).to receive(:instance).and_return(cache) - expect(cache).to receive(:clear_checksum_cache!) - allow(env).to receive(:run_builder) do |builder, target, sources, cache, vars| - raise "Ruby exception thrown by builder" - end - expect(cache).to receive(:write) - - expect { env.process }.to raise_error RuntimeError, /Ruby exception thrown by builder/ - end - end - describe "#build_command" do it "returns a command based on the variables in the Environment" do env = Environment.new @@ -264,8 +205,7 @@ module Rscons env = Environment.new(echo: :short) expect(env).to receive(:puts).with("short desc") expect(env).to receive(:system).with(*Rscons.command_executer, "a", "command").and_return(false) - expect($stdout).to receive(:write).with("Failed command was: ") - expect(env).to receive(:puts).with("a command") + expect($stdout).to receive(:puts).with("Failed command was: a command") env.execute("short desc", ["a", "command"]) end end From 2be738be4f162004fa2764df11c0b10fb748075e Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Mon, 22 May 2017 16:26:12 -0400 Subject: [PATCH 36/69] Including trailing newline when printing failed command --- lib/rscons/environment.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 8ca17bf..ae83bd4 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -336,7 +336,7 @@ module Rscons end else unless @echo == :command - $stdout.write "Failed command was: #{command_to_s(tc.command)}" + $stdout.puts "Failed command was: #{command_to_s(tc.command)}" end raise BuildError.new("Failed to build #{tc.build_operation[:target]}") end @@ -394,7 +394,7 @@ module Rscons options_args = options[:options] ? [options[:options]] : [] system(*env_args, *Rscons.command_executer, *command, *options_args).tap do |result| unless result or @echo == :command - $stdout.write "Failed command was: #{command_to_s(command)}" + $stdout.puts "Failed command was: #{command_to_s(command)}" end end end @@ -626,7 +626,7 @@ module Rscons call_build_hooks[:post] else unless @echo == :command - $stdout.write "Failed command was: #{command_to_s(tc.command)}" + $stdout.puts "Failed command was: #{command_to_s(tc.command)}" end end end From 7e707e7e3b04fb297ba52cc5f800a726be995424 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Mon, 22 May 2017 16:38:57 -0400 Subject: [PATCH 37/69] convert Object builder specs to integration tests --- build_tests/simple/error_unknown_suffix.rb | 6 +++ build_tests/simple/override_cccmd.rb | 4 ++ build_tests/simple/override_depfilesuffix.rb | 4 ++ spec/build_tests_spec.rb | 25 ++++++++++ spec/rscons/builders/object_spec.rb | 52 -------------------- 5 files changed, 39 insertions(+), 52 deletions(-) create mode 100644 build_tests/simple/error_unknown_suffix.rb create mode 100644 build_tests/simple/override_cccmd.rb create mode 100644 build_tests/simple/override_depfilesuffix.rb delete mode 100644 spec/rscons/builders/object_spec.rb diff --git a/build_tests/simple/error_unknown_suffix.rb b/build_tests/simple/error_unknown_suffix.rb new file mode 100644 index 0000000..d8bfe75 --- /dev/null +++ b/build_tests/simple/error_unknown_suffix.rb @@ -0,0 +1,6 @@ +Rscons::Environment.new do |env| + File.open("foo.xyz", "wb") do |fh| + fh.puts("hi") + end + env.Object("foo.o", "foo.xyz") +end diff --git a/build_tests/simple/override_cccmd.rb b/build_tests/simple/override_cccmd.rb new file mode 100644 index 0000000..3b5a02e --- /dev/null +++ b/build_tests/simple/override_cccmd.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new(echo: :command) do |env| + env.Object("simple.o", "simple.c", + "CCCMD" => %w[${CC} -c -o ${_TARGET} -Dfoobar ${_SOURCES}]) +end diff --git a/build_tests/simple/override_depfilesuffix.rb b/build_tests/simple/override_depfilesuffix.rb new file mode 100644 index 0000000..1547a46 --- /dev/null +++ b/build_tests/simple/override_depfilesuffix.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new(echo: :command) do |env| + env["DEPFILESUFFIX"] = ".deppy" + env.Object("simple.o", "simple.c") +end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 6c27976..d60f406 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -935,4 +935,29 @@ EOF end end + context "Object builder" do + it "allows overriding CCCMD construction variable" do + test_dir("simple") + result = run_test(rsconsfile: "override_cccmd.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "gcc -c -o simple.o -Dfoobar simple.c", + ] + end + + it "allows overriding DEPFILESUFFIX construction variable" do + test_dir("simple") + result = run_test(rsconsfile: "override_depfilesuffix.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "gcc -c -o simple.o -MMD -MF simple.deppy simple.c", + ] + end + + it "raises an error when given a source file with an unknown suffix" do + test_dir("simple") + result = run_test(rsconsfile: "error_unknown_suffix.rb") + expect(result.stderr).to match /unknown input file type: "foo.xyz"/ + end + end end diff --git a/spec/rscons/builders/object_spec.rb b/spec/rscons/builders/object_spec.rb deleted file mode 100644 index 358d829..0000000 --- a/spec/rscons/builders/object_spec.rb +++ /dev/null @@ -1,52 +0,0 @@ -module Rscons - module Builders - describe Object do - let(:env) {Environment.new} - let(:cache) {double(Cache)} - subject {Object.new} - - it "supports overriding CCCMD construction variable" do - expect(cache).to receive(:up_to_date?).and_return(false) - expect(cache).to receive(:mkdir_p) - expect(FileUtils).to receive(:rm_f) - expect(env).to receive(:execute).with("CC mod.o", ["llc", "mod.c"]).and_return(true) - expect(File).to receive(:exists?).and_return(false) - expect(cache).to receive(:register_build) - - subject.run( - target: "mod.o", - sources: ["mod.c"], - cache: cache, - env: env, - vars: {"CCCMD" => ["llc", "${_SOURCES}"]}) - end - - it "supports overriding DEPFILESUFFIX construction variable" do - expect(cache).to receive(:up_to_date?).and_return(false) - expect(cache).to receive(:mkdir_p) - expect(FileUtils).to receive(:rm_f) - expect(env).to receive(:execute).with(anything, %w[gcc -c -o f.o -MMD -MF f.d in.c]).and_return(true) - expect(File).to receive(:exists?).with("f.d").and_return(false) - expect(cache).to receive(:register_build) - - subject.run( - target: "f.o", - sources: ["in.c"], - cache: cache, - env: env, - vars: {"DEPFILESUFFIX" => ".d"}) - end - - it "raises an error when given a source file with an unknown suffix" do - expect do - subject.run( - target: "mod.o", - sources: ["mod.xyz"], - cache: :cache, - env: env, - vars: {}) - end.to raise_error /unknown input file type: "mod.xyz"/ - end - end - end -end From 957fd8c86d279ccc6926b970386fb3d4750b2e8e Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Mon, 22 May 2017 16:54:10 -0400 Subject: [PATCH 38/69] convert Library builder specs to integration tests --- build_tests/library/override_arcmd.rb | 4 ++++ spec/build_tests_spec.rb | 9 +++++++++ spec/rscons/builders/library_spec.rb | 28 --------------------------- 3 files changed, 13 insertions(+), 28 deletions(-) create mode 100644 build_tests/library/override_arcmd.rb delete mode 100644 spec/rscons/builders/library_spec.rb diff --git a/build_tests/library/override_arcmd.rb b/build_tests/library/override_arcmd.rb new file mode 100644 index 0000000..29e39d3 --- /dev/null +++ b/build_tests/library/override_arcmd.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new(echo: :command) do |env| + env["ARCMD"] = %w[ar rcf ${_TARGET} ${_SOURCES}] + env.Library("lib.a", Dir["*.c"].sort) +end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index d60f406..b8e87c5 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -960,4 +960,13 @@ EOF expect(result.stderr).to match /unknown input file type: "foo.xyz"/ end end + + context "Library builder" do + it "allows overriding ARCMD construction variable" do + test_dir("library") + result = run_test(rsconsfile: "override_arcmd.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to include "ar rcf lib.a one.o three.o two.o" + end + end end diff --git a/spec/rscons/builders/library_spec.rb b/spec/rscons/builders/library_spec.rb deleted file mode 100644 index 6885322..0000000 --- a/spec/rscons/builders/library_spec.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Rscons - module Builders - describe Library do - let(:env) {Environment.new} - subject {Library.new} - - it "supports overriding AR construction variable" do - expect(subject).to receive(:standard_build).with("AR prog.a", "prog.a", ["sp-ar", "rcs", "prog.a", "prog.o"], ["prog.o"], env, :cache) - subject.run( - target: "prog.a", - sources: ["prog.o"], - cache: :cache, - env: env, - vars: {"AR" => "sp-ar"}) - end - - it "supports overriding ARCMD construction variable" do - expect(subject).to receive(:standard_build).with("AR prog.a", "prog.a", ["special", "AR!", "prog.o"], ["prog.o"], env, :cache) - subject.run( - target: "prog.a", - sources: ["prog.o"], - cache: :cache, - env: env, - vars: {"ARCMD" => ["special", "AR!", "${_SOURCES}"]}) - end - end - end -end From c7bba77581918e1856d4e3f748e039f9041c8b8d Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Mon, 22 May 2017 16:57:35 -0400 Subject: [PATCH 39/69] remove Program builder non-integration specs --- spec/rscons/builders/program_spec.rb | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 spec/rscons/builders/program_spec.rb diff --git a/spec/rscons/builders/program_spec.rb b/spec/rscons/builders/program_spec.rb deleted file mode 100644 index 7a42009..0000000 --- a/spec/rscons/builders/program_spec.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Rscons - module Builders - describe Program do - let(:env) {Environment.new} - subject {Program.new} - - it "supports overriding CC construction variable" do - expect(subject).to receive(:standard_build).with("LD prog", "prog", ["sp-c++", "-o", "prog", "prog.o"], ["prog.o"], env, :cache) - subject.run( - target: "prog", - sources: ["prog.o"], - cache: :cache, - env: env, - vars: {"CC" => "sp-c++"}) - end - - it "supports overriding LDCMD construction variable" do - expect(subject).to receive(:standard_build).with("LD prog.exe", "prog.exe", ["special", "LD!", "prog.o"], ["prog.o"], env, :cache) - subject.run( - target: "prog.exe", - sources: ["prog.o"], - cache: :cache, - env: env, - vars: {"LDCMD" => ["special", "LD!", "${_SOURCES}"]}) - end - end - end -end From 1509c951764d9c5d686385a3aa4b24adab4b5bc3 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 14:05:29 -0400 Subject: [PATCH 40/69] remove Cache.reset! --- lib/rscons/cache.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/rscons/cache.rb b/lib/rscons/cache.rb index 9c71339..13cf1fd 100644 --- a/lib/rscons/cache.rb +++ b/lib/rscons/cache.rb @@ -62,11 +62,6 @@ module Rscons def instance @instance ||= Cache.new end - - # Reset the cache (for unit/integration test purposes) - def reset! - @instance = nil - end end # Create a Cache object and load in the previous contents from the cache From a99e6e81b2899c500f4d69db393bad48d1fbd098 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 14:10:45 -0400 Subject: [PATCH 41/69] simplify Environment#process looping --- lib/rscons/environment.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index ae83bd4..8a4ee5d 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -312,12 +312,8 @@ module Rscons completed_tcs = Set.new # First do a non-blocking wait to pick up any threads that have # completed since last time. - loop do - if tc = wait_for_threaded_commands(nonblock: true) - completed_tcs << tc - else - break - end + while tc = wait_for_threaded_commands(nonblock: true) + completed_tcs << tc end # If needed, do a blocking wait. From 15e5d1542427c794de853b35ff50d49c4ca33504 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 14:12:38 -0400 Subject: [PATCH 42/69] Avoid Environment#process race condition Race condition could have occurred if the last threaded command was gathered in the non-blocking wait and then a blocking wait was started because there was no job to run. Do not do a blocking wait in this case. --- lib/rscons/environment.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 8a4ee5d..58b68b0 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -317,7 +317,7 @@ module Rscons end # If needed, do a blocking wait. - if job.nil? or @threaded_commands.size >= Rscons.n_threads + if (completed_tcs.empty? and job.nil?) or @threaded_commands.size >= Rscons.n_threads completed_tcs << wait_for_threaded_commands end From 2ffdf82d9a036c3f1f38e392f603682a9e375b30 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 14:24:14 -0400 Subject: [PATCH 43/69] add multi-threading build test --- build_tests/simple/threading.rb | 28 ++++++++++++++++++++++++++++ spec/build_tests_spec.rb | 17 +++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 build_tests/simple/threading.rb diff --git a/build_tests/simple/threading.rb b/build_tests/simple/threading.rb new file mode 100644 index 0000000..fba5998 --- /dev/null +++ b/build_tests/simple/threading.rb @@ -0,0 +1,28 @@ +class ThreadedTestBuilder < Rscons::Builder + def run(options) + command = ["ruby", "-e", %[sleep 1]] + Rscons::ThreadedCommand.new( + command, + short_description: "ThreadedTestBuilder #{options[:target]}") + end + def finalize(options) + true + end +end + +class NonThreadedTestBuilder < Rscons::Builder + def run(options) + puts "NonThreadedTestBuilder #{options[:target]}" + sleep 1 + options[:target] + end +end + +Rscons::Environment.new do |env| + env.add_builder(ThreadedTestBuilder.new) + env.add_builder(NonThreadedTestBuilder.new) + env.ThreadedTestBuilder("a") + env.ThreadedTestBuilder("b") + env.ThreadedTestBuilder("c") + env.NonThreadedTestBuilder("d") +end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index b8e87c5..b33e226 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -969,4 +969,21 @@ EOF expect(lines(result.stdout)).to include "ar rcf lib.a one.o three.o two.o" end end + + context "multi-threading" do + it "waits for subcommands in threads for builders that support threaded commands" do + test_dir("simple") + start_time = Time.new + result = run_test(rsconsfile: "threading.rb", rscons_args: %w[-j 4]) + expect(result.stderr).to eq "" + expect(Set[*lines(result.stdout)]).to eq Set[ + "ThreadedTestBuilder a", + "ThreadedTestBuilder b", + "ThreadedTestBuilder c", + "NonThreadedTestBuilder d", + ] + elapsed = Time.new - start_time + expect(elapsed).to be < 3 + end + end end From 18c6c6987165cb9387cddd70aef17d785ca7d07d Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 15:24:52 -0400 Subject: [PATCH 44/69] do not use ThreadsWait to wait for threads; it does not work properly --- lib/rscons/environment.rb | 45 +++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 58b68b0..dea7ea6 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -1,7 +1,6 @@ require "fileutils" require "set" require "shellwords" -require "thwait" module Rscons # The Environment class is the main programmatic interface to Rscons. It @@ -847,23 +846,9 @@ module Rscons # @return [ThreadedCommand, nil] # The {ThreadedCommand} object that is finished. def wait_for_threaded_commands(options = {}) - if @threaded_commands.empty? - if options[:nonblock] - return nil - else - raise "No threaded commands to wait for" - end - end options[:which] ||= @threaded_commands threads = options[:which].map(&:thread) - tw = ThreadsWait.new(*threads) - finished_thread = - begin - tw.next_wait(options[:nonblock]) - rescue ThreadsWait::ErrNoFinishedThread - nil - end - if finished_thread + if finished_thread = find_finished_thread(threads, options[:nonblock]) threaded_command = @threaded_commands.find do |tc| tc.thread == finished_thread end @@ -872,6 +857,34 @@ module Rscons end end + # Check if any of the requested threads are finished. + # + # @param threads [Array] + # The threads to check. + # @param nonblock [Boolean] + # Whether to be non-blocking. If true, nil will be returned if no thread + # is finished. If false, the method will wait until one of the threads + # is finished. + # + # @return [Thread, nil] + # The finished thread, if any. + def find_finished_thread(threads, nonblock) + if nonblock + threads.find do |thread| + !thread.alive? + end + else + if threads.empty? + raise "No threads to wait for" + end + loop do + thread = find_finished_thread(threads, true) + return thread if thread + sleep 0.1 + end + end + end + # Return a string representation of a command. # # @param command [Array] From 599d10b50f79757b8fce5ab655642d289d192490 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 15:32:55 -0400 Subject: [PATCH 45/69] add integration test for overriding PROGSUFFIX --- build_tests/simple/progsuffix.rb | 4 ++++ spec/build_tests_spec.rb | 10 ++++++++++ 2 files changed, 14 insertions(+) create mode 100644 build_tests/simple/progsuffix.rb diff --git a/build_tests/simple/progsuffix.rb b/build_tests/simple/progsuffix.rb new file mode 100644 index 0000000..363efdf --- /dev/null +++ b/build_tests/simple/progsuffix.rb @@ -0,0 +1,4 @@ +Rscons::Environment.new do |env| + env["PROGSUFFIX"] = ".out" + env.Program("simple", Dir["*.c"]) +end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index b33e226..7ba5e95 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -616,6 +616,16 @@ EOF ] end + it "allows overriding progsuffix" do + test_dir("simple") + result = run_test(rsconsfile: "progsuffix.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "CC simple.o", + "LD simple.out", + ] + end + context "Directory builder" do it "creates the requested directory" do test_dir("simple") From c9946da19380031890807e08df1828f6cdecabd4 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 15:55:28 -0400 Subject: [PATCH 46/69] integration tests: only pass -f when a specific rsconsfile is specified --- spec/build_tests_spec.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 7ba5e95..0b6141c 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -55,9 +55,14 @@ describe Rscons do end def run_test(options = {}) - rsconsfile = options[:rsconsfile] || "Rsconsfile" + rsconsfile_args = + if options[:rsconsfile] + %W[-f #{options[:rsconsfile]}] + else + [] + end rscons_args = options[:rscons_args] || [] - command = %W[ruby -I. -r _simplecov_setup #{@owd}/bin/rscons -f #{rsconsfile}] + rscons_args + command = %W[ruby -I. -r _simplecov_setup #{@owd}/bin/rscons] + rsconsfile_args + rscons_args @statics[:build_test_id] ||= 0 @statics[:build_test_id] += 1 command_name = "b#{@statics[:build_test_id]}" From 032e6d5d0acdf36f50caf8e41598d0c768d9f420 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 16:02:07 -0400 Subject: [PATCH 47/69] add some integration tests for the CLI --- spec/build_tests_spec.rb | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 0b6141c..903c85b 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -1001,4 +1001,37 @@ EOF expect(elapsed).to be < 3 end end + + context "CLI" do + it "shows the version number and exits with --version argument" do + test_dir("simple") + result = run_test(rscons_args: %w[--version]) + expect(result.stderr).to eq "" + expect(result.status).to eq 0 + expect(result.stdout).to match /version #{Rscons::VERSION}/ + end + + it "shows CLI help and exits with --help argument" do + test_dir("simple") + result = run_test(rscons_args: %w[--help]) + expect(result.stderr).to eq "" + expect(result.status).to eq 0 + expect(result.stdout).to match /Usage:/ + end + + it "prints an error and exits with an error status when a default Rsconsfile cannot be found" do + test_dir("simple") + FileUtils.rm_f("Rsconsfile") + result = run_test + expect(result.stderr).to match /Could not find the Rsconsfile to execute/ + expect(result.status).to_not eq 0 + end + + it "prints an error and exits with an error status when the given Rsconsfile cannot be read" do + test_dir("simple") + result = run_test(rsconsfile: "nonexistent") + expect(result.stderr).to match /Cannot read nonexistent/ + expect(result.status).to_not eq 0 + end + end end From 0d46d616dd289ef3d498e2fa97f72032d0be0847 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 16:09:29 -0400 Subject: [PATCH 48/69] change user dependency test to use BuildTarget#depends --- build_tests/simple/cache_user_dep.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tests/simple/cache_user_dep.rb b/build_tests/simple/cache_user_dep.rb index db7e028..4a04718 100644 --- a/build_tests/simple/cache_user_dep.rb +++ b/build_tests/simple/cache_user_dep.rb @@ -1,5 +1,5 @@ Rscons::Environment.new do |env| - env.Program("simple.exe", "simple.c") + target = env.Program("simple.exe", "simple.c") user_deps = File.read("user_deps", mode: "rb").split(" ") - env.depends("simple.exe", *user_deps) + target.depends(*user_deps) end From d315f86a360eb4484c9b6a00665da58f0f3e3cf9 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 16:20:41 -0400 Subject: [PATCH 49/69] remove unit tests for Environment#run_builder, #build_sources --- spec/rscons/environment_spec.rb | 36 --------------------------------- 1 file changed, 36 deletions(-) diff --git a/spec/rscons/environment_spec.rb b/spec/rscons/environment_spec.rb index 9efd325..96a36f9 100644 --- a/spec/rscons/environment_spec.rb +++ b/spec/rscons/environment_spec.rb @@ -254,42 +254,6 @@ module Rscons end end - describe "#build_sources" do - class ABuilder < Builder - def produces?(target, source, env) - target =~ /\.ab_out$/ and source =~ /\.ab_in$/ - end - end - - it "finds and invokes a builder to produce output files with the requested suffixes" do - cache = "cache" - env = Environment.new - env.add_builder(ABuilder.new) - expect(env.builders["Object"]).to receive(:run).with("mod.o", ["mod.c"], cache, env, anything).and_return("mod.o") - expect(env.builders["ABuilder"]).to receive(:run).with("mod2.ab_out", ["mod2.ab_in"], cache, env, anything).and_return("mod2.ab_out") - expect(env.build_sources(["precompiled.o", "mod.c", "mod2.ab_in"], [".o", ".ab_out"], cache, {})).to eq ["precompiled.o", "mod.o", "mod2.ab_out"] - end - end - - describe "#run_builder" do - it "modifies the construction variables using given build hooks and invokes the builder" do - env = Environment.new - env.add_build_hook do |build_op| - if build_op[:sources].first =~ %r{src/special} - build_op[:vars]["CFLAGS"] += ["-O3", "-DSPECIAL"] - end - end - allow(env.builders["Object"]).to receive(:run) do |target, sources, cache, env, vars| - expect(vars["CFLAGS"]).to eq [] - end - env.run_builder(env.builders["Object"], "build/normal/module.o", ["src/normal/module.c"], "cache", {}) - allow(env.builders["Object"]).to receive(:run) do |target, sources, cache, env, vars| - expect(vars["CFLAGS"]).to eq ["-O3", "-DSPECIAL"] - end - env.run_builder(env.builders["Object"], "build/special/module.o", ["src/special/module.c"], "cache", {}) - end - end - describe "#shell" do it "executes the given shell command and returns the results" do env = Environment.new From 8a1dfb07853fa7e758ddb2eb94a36480bb562be7 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 16:24:45 -0400 Subject: [PATCH 50/69] do not catch NameError when looking up Builder#run method --- lib/rscons/environment.rb | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index dea7ea6..8c5ce66 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -584,15 +584,10 @@ module Rscons # Invoke pre-build hooks. call_build_hooks[:pre] - use_new_run_method_signature = - begin - builder.method(:run).arity == 1 - rescue NameError - false - end - # Call the builder's #run method. - if use_new_run_method_signature + if builder.method(:run).arity == 5 + rv = builder.run(target, sources, cache, self, vars) + else rv = builder.run( target: target, sources: sources, @@ -600,8 +595,6 @@ module Rscons env: self, vars: vars, setup_info: options[:setup_info]) - else - rv = builder.run(target, sources, cache, self, vars) end if rv.is_a?(ThreadedCommand) From e58b8bd109d22c3bc6a50c33ed187e5e01d45e9f Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 16:25:03 -0400 Subject: [PATCH 51/69] add integration test for Environment#run_builder --- build_tests/simple/run_builder.rb | 12 ++++++++++++ spec/build_tests_spec.rb | 10 ++++++++++ 2 files changed, 22 insertions(+) create mode 100644 build_tests/simple/run_builder.rb diff --git a/build_tests/simple/run_builder.rb b/build_tests/simple/run_builder.rb new file mode 100644 index 0000000..83fc5f9 --- /dev/null +++ b/build_tests/simple/run_builder.rb @@ -0,0 +1,12 @@ +class MyObject < Rscons::Builder + def run(options) + target, sources, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars) + env.run_builder(env.builders["Object"], target, sources, cache, vars) + end +end + +Rscons::Environment.new do |env| + env.add_builder(MyObject.new) + env.MyObject("simple.o", "simple.c") + env.Program("simple.exe", "simple.o") +end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 903c85b..269d401 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -631,6 +631,16 @@ EOF ] end + it "allows a builder to call Environment#run_builder in a non-threaded manner" do + test_dir("simple") + result = run_test(rsconsfile: "run_builder.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "CC simple.o", + "LD simple.exe", + ] + end + context "Directory builder" do it "creates the requested directory" do test_dir("simple") From adcee373dffd32df5ef3fef562e74ecdb1096bef Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 16:28:13 -0400 Subject: [PATCH 52/69] add integration test for Environment#build_sources --- build_tests/simple/build_sources.rb | 14 ++++++++++++++ spec/build_tests_spec.rb | 10 ++++++++++ 2 files changed, 24 insertions(+) create mode 100644 build_tests/simple/build_sources.rb diff --git a/build_tests/simple/build_sources.rb b/build_tests/simple/build_sources.rb new file mode 100644 index 0000000..3db4ac5 --- /dev/null +++ b/build_tests/simple/build_sources.rb @@ -0,0 +1,14 @@ +class MyProgram < Rscons::Builder + def run(options) + target, sources, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars) + objects = env.build_sources(sources, [".o"], cache, vars) + command = %W[gcc -o #{target}] + objects + return false unless env.execute("#{name} #{target}", command) + target + end +end + +Rscons::Environment.new do |env| + env.add_builder(MyProgram.new) + env.MyProgram("simple.exe", "simple.c") +end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 269d401..74a24a7 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -641,6 +641,16 @@ EOF ] end + it "allows a builder to call Environment#build_sources in a non-threaded manner" do + test_dir("simple") + result = run_test(rsconsfile: "build_sources.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "CC simple.o", + "MyProgram simple.exe", + ] + end + context "Directory builder" do it "creates the requested directory" do test_dir("simple") From 221cde7e5f41c3959dacadd07ebd7a875d71e185 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 16:45:21 -0400 Subject: [PATCH 53/69] update run_builder, build_sources integration tests for more coverage --- build_tests/simple/build_sources.rb | 10 +++++++++- spec/build_tests_spec.rb | 11 +++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/build_tests/simple/build_sources.rb b/build_tests/simple/build_sources.rb index 3db4ac5..ba19d82 100644 --- a/build_tests/simple/build_sources.rb +++ b/build_tests/simple/build_sources.rb @@ -10,5 +10,13 @@ end Rscons::Environment.new do |env| env.add_builder(MyProgram.new) - env.MyProgram("simple.exe", "simple.c") + env.Object("simple.o", "simple.c") + File.open("two.c", "wb") do |fh| + fh.puts <<-EOF + void two(void) + { + } + EOF + end + env.MyProgram("simple.exe", ["simple.o", "two.c"]) end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 74a24a7..67487b1 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -647,10 +647,21 @@ EOF expect(result.stderr).to eq "" expect(lines(result.stdout)).to eq [ "CC simple.o", + "CC two.o", "MyProgram simple.exe", ] end + it "prints the failed build command for a threaded builder when called via Environment#run_builder without delayed execution" do + test_dir("simple") + File.open("simple.c", "wb") do |fh| + fh.write("FOOBAR") + end + result = run_test(rsconsfile: "run_builder.rb") + expect(result.stderr).to match /Failed to build/ + expect(result.stdout).to match /Failed command was: gcc/ + end + context "Directory builder" do it "creates the requested directory" do test_dir("simple") From 2655c119e43b4f8aa89a0f0b97fa62af1fefb1fd Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 23 May 2017 16:47:00 -0400 Subject: [PATCH 54/69] add spec for Environment#find_finished_thread to fully cover --- spec/rscons/environment_spec.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/spec/rscons/environment_spec.rb b/spec/rscons/environment_spec.rb index 96a36f9..a11dde1 100644 --- a/spec/rscons/environment_spec.rb +++ b/spec/rscons/environment_spec.rb @@ -326,6 +326,13 @@ module Rscons end end + describe "#find_finished_thread" do + it "raises an error if called with nonblock=false and no threads to wait for" do + env = Environment.new + expect {env.__send__(:find_finished_thread, [], false)}.to raise_error /No threads to wait for/ + end + end + describe ".parse_makefile_deps" do it 'handles dependencies on one line' do expect(File).to receive(:read).with('makefile').and_return(< Date: Wed, 24 May 2017 14:32:43 -0400 Subject: [PATCH 55/69] pass entire build_operation Hash to Builder#run --- lib/rscons/environment.rb | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 8c5ce66..4a8335d 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -572,8 +572,10 @@ module Rscons builder: builder, target: target, sources: sources, - vars: vars, + cache: cache, env: self, + vars: vars, + setup_info: options[:setup_info] } call_build_hooks = lambda do |sec| @build_hooks[sec].each do |build_hook_block| @@ -588,13 +590,7 @@ module Rscons if builder.method(:run).arity == 5 rv = builder.run(target, sources, cache, self, vars) else - rv = builder.run( - target: target, - sources: sources, - cache: cache, - env: self, - vars: vars, - setup_info: options[:setup_info]) + rv = builder.run(build_operation) end if rv.is_a?(ThreadedCommand) From 1af3c5c9a4a8880f3f3ed7408b2e7e4b86757749 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 24 May 2017 15:13:59 -0400 Subject: [PATCH 56/69] pass Builder#run options into Builder#finalize also --- lib/rscons/builder.rb | 22 ++++++++++++++++------ lib/rscons/builders/object.rb | 9 +++++---- lib/rscons/environment.rb | 31 ++++++++++++++++++++----------- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/lib/rscons/builder.rb b/lib/rscons/builder.rb index 9bf8898..cee9016 100644 --- a/lib/rscons/builder.rb +++ b/lib/rscons/builder.rb @@ -145,19 +145,29 @@ module Rscons # Finalize a build operation. # - # This method is called after the {#run} method if the {#run} method does - # not return an error. + # This method is called after the {#run} method if the {#run} method + # returns a {ThreadedCommand} object. # # @param options [Hash] # Options. + # @option options [String] :target + # Target file name. + # @option options [Array] :sources + # Source file name(s). + # @option options [Cache] :cache + # The Cache object. + # @option options [Environment] :env + # The Environment executing the builder. + # @option options [Hash,VarSet] :vars + # Extra construction variables. + # @option options [Object] :setup_info + # Whatever value was returned from this builder's {#setup} method call. # @option options [true,false,nil] :command_status # If the {#run} method returns a {ThreadedCommand}, this field will # contain the return value from executing the command with # Kernel.system(). - # @option options [Object] :builder_info - # If the {#run} method returns a {ThreadedCommand}, this field will - # contain the value passed in to the :builder_info field of the - # {ThreadedCommand} object. + # @option options [ThreadedCommand] :tc + # The {ThreadedCommand} object that was returned by the #run method. # # @return [String,false] # Name of the target file on success or false on failure. diff --git a/lib/rscons/builders/object.rb b/lib/rscons/builders/object.rb index b9229b2..fe1671f 100644 --- a/lib/rscons/builders/object.rb +++ b/lib/rscons/builders/object.rb @@ -88,6 +88,8 @@ module Rscons '_SOURCES' => sources, '_DEPFILE' => Rscons.set_suffix(target, env.expand_varref("${DEPFILESUFFIX}", vars)), }) + # Store vars back into options so new keys are accessible in #finalize. + options[:vars] = vars com_prefix = KNOWN_SUFFIXES.find do |compiler, suffix_var| sources.first.end_with?(*env.expand_varref("${#{suffix_var}}")) end.tap do |v| @@ -101,8 +103,7 @@ module Rscons FileUtils.rm_f(target) ThreadedCommand.new( command, - short_description: "#{com_prefix} #{target}", - builder_info: options.merge(vars: vars, command: command)) + short_description: "#{com_prefix} #{target}") end end @@ -114,12 +115,12 @@ module Rscons # Name of the target file on success or nil on failure. def finalize(options) if options[:command_status] - target, deps, cache, env, vars, command = options[:builder_info].values_at(:target, :sources, :cache, :env, :vars, :command) + target, deps, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars) if File.exists?(vars['_DEPFILE']) deps += Environment.parse_makefile_deps(vars['_DEPFILE'], target) FileUtils.rm_f(vars['_DEPFILE']) end - cache.register_build(target, command, deps.uniq, env) + cache.register_build(target, options[:tc].command, deps.uniq, env) target end end diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 4a8335d..0246592 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -322,9 +322,7 @@ module Rscons # Process all completed {ThreadedCommand} objects. completed_tcs.each do |tc| - result = tc.build_operation[:builder].finalize( - command_status: tc.thread.value, - builder_info: tc.builder_info) + result = finalize_builder(tc) if result @build_hooks[:post].each do |build_hook_block| build_hook_block.call(tc.build_operation) @@ -594,18 +592,15 @@ module Rscons end if rv.is_a?(ThreadedCommand) + # Store the build operation so the post-build hooks can be called + # with it when the threaded command completes. + rv.build_operation = build_operation start_threaded_command(rv) - if options[:allow_delayed_execution] - # Store the build operation so the post-build hooks can be called - # with it when the threaded command completes. - rv.build_operation = build_operation - else + unless options[:allow_delayed_execution] # Delayed command execution is not allowed, so we need to execute # the command and finalize the builder now. tc = wait_for_threaded_commands(which: [rv]) - rv = builder.finalize( - command_status: tc.thread.value, - builder_info: tc.builder_info) + rv = finalize_builder(tc) if rv call_build_hooks[:post] else @@ -885,6 +880,20 @@ module Rscons command.map { |c| c =~ /\s/ ? "'#{c}'" : c }.join(' ') end + # Call a builder's #finalize method after a ThreadedCommand terminates. + # + # @param tc [ThreadedCommand] + # The ThreadedCommand returned from the builder's #run method. + # + # @return [String, false] + # Result of Builder#finalize. + def finalize_builder(tc) + tc.build_operation[:builder].finalize( + tc.build_operation.merge( + command_status: tc.thread.value, + tc: tc)) + end + # Parse dependencies for a given target from a Makefile. # # This method is used internally by Rscons builders. From f815952ab3acf15857f7e5571a50c965b315478e Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 24 May 2017 15:41:39 -0400 Subject: [PATCH 57/69] add Builder#standard_threaded_build, #standard_finalize parallelize Program builder command --- lib/rscons/builder.rb | 53 +++++++++++++++++++++++++++++++++- lib/rscons/builders/object.rb | 10 +------ lib/rscons/builders/program.rb | 14 ++++++++- 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/lib/rscons/builder.rb b/lib/rscons/builder.rb index cee9016..fb8a8ad 100644 --- a/lib/rscons/builder.rb +++ b/lib/rscons/builder.rb @@ -148,6 +148,8 @@ module Rscons # This method is called after the {#run} method if the {#run} method # returns a {ThreadedCommand} object. # + # @since 1.10.0 + # # @param options [Hash] # Options. # @option options [String] :target @@ -175,7 +177,7 @@ module Rscons end # Check if the cache is up to date for the target and if not execute the - # build command. + # build command. This method does not support parallelization. # # @param short_cmd_string [String] # Short description of build action to be printed when env.echo == @@ -200,5 +202,54 @@ module Rscons end target end + + # Check if the cache is up to date for the target and if not create a + # {ThreadedCommand} object to execute the build command in a thread. + # + # @since 1.10.0 + # + # @param short_cmd_string [String] + # Short description of build action to be printed when env.echo == + # :short. + # @param target [String] Name of the target file. + # @param command [Array] + # The command to execute to build the target. + # @param sources [Array] Source file name(s). + # @param env [Environment] The Environment executing the builder. + # @param cache [Cache] The Cache object. + # + # @return [String,ThreadedCommand] + # The name of the target if it is already up to date or the + # {ThreadedCommand} object created to update it. + def standard_threaded_build(short_cmd_string, target, command, sources, env, cache) + if cache.up_to_date?(target, command, sources, env) + target + else + unless Rscons.phony_target?(target) + cache.mkdir_p(File.dirname(target)) + FileUtils.rm_f(target) + end + ThreadedCommand.new( + command, + short_description: short_cmd_string) + end + end + + # Register build results from a {ThreadedCommand} with the cache. + # + # @since 1.10.0 + # + # @param options [Hash] + # Builder finalize options. + # + # @return [String, nil] + # The target name on success or nil on failure. + def standard_finalize(options) + if options[:command_status] + target, sources, cache, env = options.values_at(:target, :sources, :cache, :env) + cache.register_build(target, options[:tc].command, sources, env) + target + end + end end end diff --git a/lib/rscons/builders/object.rb b/lib/rscons/builders/object.rb index fe1671f..7650608 100644 --- a/lib/rscons/builders/object.rb +++ b/lib/rscons/builders/object.rb @@ -96,15 +96,7 @@ module Rscons v.nil? and raise "Error: unknown input file type: #{sources.first.inspect}" end.first command = env.build_command("${#{com_prefix}CMD}", vars) - if cache.up_to_date?(target, command, sources, env) - target - else - cache.mkdir_p(File.dirname(target)) - FileUtils.rm_f(target) - ThreadedCommand.new( - command, - short_description: "#{com_prefix} #{target}") - end + standard_threaded_build("#{com_prefix} #{target}", target, command, sources, env, cache) end # Finalize the build operation. diff --git a/lib/rscons/builders/program.rb b/lib/rscons/builders/program.rb index ce21c19..d558858 100644 --- a/lib/rscons/builders/program.rb +++ b/lib/rscons/builders/program.rb @@ -83,8 +83,20 @@ module Rscons '_SOURCES' => objects, 'LD' => ld, }) + options[:sources] = objects command = env.build_command("${LDCMD}", vars) - standard_build("LD #{target}", target, command, objects, env, cache) + standard_threaded_build("LD #{target}", target, command, objects, env, cache) + end + + # Finalize a build. + # + # @param options [Hash] + # Finalize options. + # + # @return [String, nil] + # The target name on success or nil on failure. + def finalize(options) + standard_finalize(options) end end From f3bf325c88013ff7d519bc979d9bf5bbce7753d1 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 24 May 2017 15:58:11 -0400 Subject: [PATCH 58/69] add a little more leeway in the multi-threading timing test --- spec/build_tests_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 67487b1..debddaf 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -1029,7 +1029,7 @@ EOF "NonThreadedTestBuilder d", ] elapsed = Time.new - start_time - expect(elapsed).to be < 3 + expect(elapsed).to be < 4 end end From df420fdf5c2608db2113a375f81e9415676b5685 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 24 May 2017 16:02:18 -0400 Subject: [PATCH 59/69] use ThreadsWait (only for blocking thread waits) --- lib/rscons/environment.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index 0246592..cc8cbd2 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -1,6 +1,7 @@ require "fileutils" require "set" require "shellwords" +require "thwait" module Rscons # The Environment class is the main programmatic interface to Rscons. It @@ -861,11 +862,7 @@ module Rscons if threads.empty? raise "No threads to wait for" end - loop do - thread = find_finished_thread(threads, true) - return thread if thread - sleep 0.1 - end + ThreadsWait.new(*threads).next_wait end end From b45420811744e2a95b09a8eb9956a918c73e768f Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 24 May 2017 16:10:58 -0400 Subject: [PATCH 60/69] fully parallelize the Library builder --- lib/rscons/builders/library.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/rscons/builders/library.rb b/lib/rscons/builders/library.rb index 328764b..a8b2ad9 100644 --- a/lib/rscons/builders/library.rb +++ b/lib/rscons/builders/library.rb @@ -43,8 +43,20 @@ module Rscons '_TARGET' => target, '_SOURCES' => objects, }) + options[:sources] = objects command = env.build_command("${ARCMD}", vars) - standard_build("AR #{target}", target, command, objects, env, cache) + standard_threaded_build("AR #{target}", target, command, objects, env, cache) + end + + # Finalize a build. + # + # @param options [Hash] + # Finalize options. + # + # @return [String, nil] + # The target name on success or nil on failure. + def finalize(options) + standard_finalize(options) end end From 83226e894db47123653e7b8313a5f74a79506e2f Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 24 May 2017 16:16:53 -0400 Subject: [PATCH 61/69] add backwards-compatibility test for Builder#standard_build --- build_tests/simple/standard_build.rb | 17 ++++++++ spec/build_tests_spec.rb | 65 ++++++++++++++++------------ 2 files changed, 54 insertions(+), 28 deletions(-) create mode 100644 build_tests/simple/standard_build.rb diff --git a/build_tests/simple/standard_build.rb b/build_tests/simple/standard_build.rb new file mode 100644 index 0000000..9d8fa06 --- /dev/null +++ b/build_tests/simple/standard_build.rb @@ -0,0 +1,17 @@ +class MyCommand < Rscons::Builder + def run(target, sources, cache, env, vars) + vars = vars.merge({ + "_TARGET" => target, + "_SOURCES" => sources, + }) + command = env.build_command("${CMD}", vars) + cmd_desc = vars["CMD_DESC"] || "MyCommand" + standard_build("#{cmd_desc} #{target}", target, command, sources, env, cache) + end +end + +Rscons::Environment.new do |env| + env.add_builder(MyCommand.new) + command = %w[gcc -c -o ${_TARGET} ${_SOURCES}] + env.MyCommand("simple.o", "simple.c", "CMD" => command) +end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index debddaf..db468ee 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -631,35 +631,44 @@ EOF ] end - it "allows a builder to call Environment#run_builder in a non-threaded manner" do - test_dir("simple") - result = run_test(rsconsfile: "run_builder.rb") - expect(result.stderr).to eq "" - expect(lines(result.stdout)).to eq [ - "CC simple.o", - "LD simple.exe", - ] - end - - it "allows a builder to call Environment#build_sources in a non-threaded manner" do - test_dir("simple") - result = run_test(rsconsfile: "build_sources.rb") - expect(result.stderr).to eq "" - expect(lines(result.stdout)).to eq [ - "CC simple.o", - "CC two.o", - "MyProgram simple.exe", - ] - end - - it "prints the failed build command for a threaded builder when called via Environment#run_builder without delayed execution" do - test_dir("simple") - File.open("simple.c", "wb") do |fh| - fh.write("FOOBAR") + context "backward compatibility" do + it "allows a builder to call Environment#run_builder in a non-threaded manner" do + test_dir("simple") + result = run_test(rsconsfile: "run_builder.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "CC simple.o", + "LD simple.exe", + ] + end + + it "allows a builder to call Environment#build_sources in a non-threaded manner" do + test_dir("simple") + result = run_test(rsconsfile: "build_sources.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq [ + "CC simple.o", + "CC two.o", + "MyProgram simple.exe", + ] + end + + it "prints the failed build command for a threaded builder when called via Environment#run_builder without delayed execution" do + test_dir("simple") + File.open("simple.c", "wb") do |fh| + fh.write("FOOBAR") + end + result = run_test(rsconsfile: "run_builder.rb") + expect(result.stderr).to match /Failed to build/ + expect(result.stdout).to match /Failed command was: gcc/ + end + + it "supports builders that call Builder#standard_build" do + test_dir("simple") + result = run_test(rsconsfile: "standard_build.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq ["MyCommand simple.o"] end - result = run_test(rsconsfile: "run_builder.rb") - expect(result.stderr).to match /Failed to build/ - expect(result.stdout).to match /Failed command was: gcc/ end context "Directory builder" do From a11fab43cbd84cfce84ac35e2a831ca0cc28c1bd Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Wed, 24 May 2017 16:29:17 -0400 Subject: [PATCH 62/69] Update spec task to preserve simplecov coverage info for previous full spec runs when doing a partial spec run --- Rakefile.rb | 1 + spec/build_tests_spec.rb | 8 +++++++- spec/spec_helper.rb | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Rakefile.rb b/Rakefile.rb index 2b47783..9c69011 100644 --- a/Rakefile.rb +++ b/Rakefile.rb @@ -15,6 +15,7 @@ CLOBBER.include %w[pkg] RSpec::Core::RakeTask.new(:spec, :example_string) do |task, args| if args.example_string + ENV["partial_specs"] = "1" task.rspec_opts = %W[-e "#{args.example_string}" -f documentation] end end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index db468ee..dee85de 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -65,7 +65,13 @@ describe Rscons do command = %W[ruby -I. -r _simplecov_setup #{@owd}/bin/rscons] + rsconsfile_args + rscons_args @statics[:build_test_id] ||= 0 @statics[:build_test_id] += 1 - command_name = "b#{@statics[:build_test_id]}" + command_prefix = + if ENV["partial_specs"] + "p" + else + "b" + end + command_name = "#{command_prefix}#{@statics[:build_test_id]}" File.open("_simplecov_setup.rb", "w") do |fh| fh.puts < Date: Thu, 25 May 2017 14:59:51 -0400 Subject: [PATCH 63/69] fully parallelize the Disassemble builder --- lib/rscons/builders/disassemble.rb | 24 ++++++++++++++++++++---- spec/build_tests_spec.rb | 5 +++++ spec/rscons/builders/disassemble_spec.rb | 17 ----------------- 3 files changed, 25 insertions(+), 21 deletions(-) delete mode 100644 spec/rscons/builders/disassemble_spec.rb diff --git a/lib/rscons/builders/disassemble.rb b/lib/rscons/builders/disassemble.rb index b5ba813..ddbecae 100644 --- a/lib/rscons/builders/disassemble.rb +++ b/lib/rscons/builders/disassemble.rb @@ -2,6 +2,7 @@ module Rscons module Builders # The Disassemble builder produces a disassembly listing of a source file. class Disassemble < Builder + # Return default construction variables for the builder. # # @param env [Environment] The Environment using the builder. @@ -28,13 +29,28 @@ module Rscons def run(target, sources, cache, env, vars) vars = vars.merge("_SOURCES" => sources) command = env.build_command("${DISASM_CMD}", vars) - unless cache.up_to_date?(target, command, sources, env) + if cache.up_to_date?(target, command, sources, env) + target + else cache.mkdir_p(File.dirname(target)) - return false unless env.execute("Disassemble #{target}", command, options: {out: target}) - cache.register_build(target, command, sources, env) + ThreadedCommand.new( + command, + short_description: "Disassemble #{target}", + system_options: {out: target}) end - target end + + # Finalize a build. + # + # @param options [Hash] + # Finalize options. + # + # @return [String, nil] + # The target name on success or nil on failure. + def finalize(options) + standard_finalize(options) + end + end end end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index dee85de..db51d95 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -439,10 +439,15 @@ EOF it "supports disassembling object files" do test_dir("simple") + result = run_test(rsconsfile: "disassemble.rb") expect(result.stderr).to eq "" expect(File.exists?("simple.txt")).to be_truthy expect(File.read("simple.txt")).to match /Disassembly of section .text:/ + + result = run_test(rsconsfile: "disassemble.rb") + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" end it "supports preprocessing C sources" do diff --git a/spec/rscons/builders/disassemble_spec.rb b/spec/rscons/builders/disassemble_spec.rb deleted file mode 100644 index d807741..0000000 --- a/spec/rscons/builders/disassemble_spec.rb +++ /dev/null @@ -1,17 +0,0 @@ -module Rscons - module Builders - describe Disassemble do - let(:env) {Environment.new} - subject {Disassemble.new} - - it "supports overriding DISASM_CMD construction variable" do - cache = "cache" - allow(cache).to receive(:up_to_date?) { false } - allow(cache).to receive(:mkdir_p) { } - allow(cache).to receive(:register_build) { } - expect(env).to receive(:execute).with("Disassemble a_file.txt", ["my_disasm", "a_file.exe"], anything).and_return(true) - subject.run("a_file.txt", ["a_file.exe"], cache, env, "DISASM_CMD" => ["my_disasm", "${_SOURCES}"]) - end - end - end -end From 178940cd5dd70ff5988ac01fa6cb550c6ad9b9a4 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Thu, 25 May 2017 15:28:01 -0400 Subject: [PATCH 64/69] fully parallelize the CFile builder --- build_tests/cfile/Rsconsfile | 4 +++ build_tests/cfile/error_unknown_extension.rb | 3 +++ build_tests/cfile/lexer.l | 4 +++ build_tests/cfile/parser.y | 9 +++++++ lib/rscons/builders/cfile.rb | 15 ++++++++++- spec/build_tests_spec.rb | 24 +++++++++++++++++ spec/rscons/builders/cfile_spec.rb | 28 -------------------- 7 files changed, 58 insertions(+), 29 deletions(-) create mode 100644 build_tests/cfile/Rsconsfile create mode 100644 build_tests/cfile/error_unknown_extension.rb create mode 100644 build_tests/cfile/lexer.l create mode 100644 build_tests/cfile/parser.y delete mode 100644 spec/rscons/builders/cfile_spec.rb diff --git a/build_tests/cfile/Rsconsfile b/build_tests/cfile/Rsconsfile new file mode 100644 index 0000000..41308ec --- /dev/null +++ b/build_tests/cfile/Rsconsfile @@ -0,0 +1,4 @@ +Rscons::Environment.new do |env| + env.CFile("lexer.c", "lexer.l") + env.CFile("parser.c", "parser.y") +end diff --git a/build_tests/cfile/error_unknown_extension.rb b/build_tests/cfile/error_unknown_extension.rb new file mode 100644 index 0000000..ad62f52 --- /dev/null +++ b/build_tests/cfile/error_unknown_extension.rb @@ -0,0 +1,3 @@ +Rscons::Environment.new do |env| + env.CFile("file.c", "foo.bar") +end diff --git a/build_tests/cfile/lexer.l b/build_tests/cfile/lexer.l new file mode 100644 index 0000000..d0c6f0d --- /dev/null +++ b/build_tests/cfile/lexer.l @@ -0,0 +1,4 @@ +%{ +%} + +%% diff --git a/build_tests/cfile/parser.y b/build_tests/cfile/parser.y new file mode 100644 index 0000000..1657847 --- /dev/null +++ b/build_tests/cfile/parser.y @@ -0,0 +1,9 @@ +%{ +%} + +%token ONE + +%% + +one: ONE + ; diff --git a/lib/rscons/builders/cfile.rb b/lib/rscons/builders/cfile.rb index 7534628..b935337 100644 --- a/lib/rscons/builders/cfile.rb +++ b/lib/rscons/builders/cfile.rb @@ -7,6 +7,7 @@ module Rscons # env.CFile("parser.tab.cc", "parser.yy") # env.CFile("lex.yy.cc", "parser.ll") class CFile < Builder + # Return default construction variables for the builder. # # @param env [Environment] The Environment using the builder. @@ -48,8 +49,20 @@ module Rscons raise "Unknown source file #{sources.first.inspect} for CFile builder" end command = env.build_command("${#{cmd}_CMD}", vars) - standard_build("#{cmd} #{target}", target, command, sources, env, cache) + standard_threaded_build("#{cmd} #{target}", target, command, sources, env, cache) end + + # Finalize a build. + # + # @param options [Hash] + # Finalize options. + # + # @return [String, nil] + # The target name on success or nil on failure. + def finalize(options) + standard_finalize(options) + end + end end end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index db51d95..af71f26 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -682,6 +682,30 @@ EOF end end + context "CFile builder" do + it "builds a .c file using flex and bison" do + test_dir("cfile") + + result = run_test + expect(result.stderr).to eq "" + expect(Set[*lines(result.stdout)]).to eq Set[ + "LEX lexer.c", + "YACC parser.c", + ] + + result = run_test + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" + end + + it "raises an error when an unknown source file is specified" do + test_dir("cfile") + result = run_test(rsconsfile: "error_unknown_extension.rb") + expect(result.stderr).to match /Unknown source file .foo.bar. for CFile builder/ + expect(result.status).to_not eq 0 + end + end + context "Directory builder" do it "creates the requested directory" do test_dir("simple") diff --git a/spec/rscons/builders/cfile_spec.rb b/spec/rscons/builders/cfile_spec.rb deleted file mode 100644 index 93ac23a..0000000 --- a/spec/rscons/builders/cfile_spec.rb +++ /dev/null @@ -1,28 +0,0 @@ -module Rscons - module Builders - describe CFile do - let(:env) {Environment.new} - subject {CFile.new} - - it "invokes bison to create a .c file from a .y file" do - expect(subject).to receive(:standard_build).with("YACC parser.c", "parser.c", ["bison", "-d", "-o", "parser.c", "parser.y"], ["parser.y"], env, :cache) - subject.run("parser.c", ["parser.y"], :cache, env, {}) - end - - it "invokes a custom lexer to create a .cc file from a .ll file" do - env["LEX"] = "custom_lex" - expect(subject).to receive(:standard_build).with("LEX lexer.cc", "lexer.cc", ["custom_lex", "-o", "lexer.cc", "parser.ll"], ["parser.ll"], env, :cache) - subject.run("lexer.cc", ["parser.ll"], :cache, env, {}) - end - - it "supports overriding construction variables" do - expect(subject).to receive(:standard_build).with("LEX lexer.c", "lexer.c", ["hi", "parser.l"], ["parser.l"], env, :cache) - subject.run("lexer.c", ["parser.l"], :cache, env, "LEX_CMD" => ["hi", "${_SOURCES}"]) - end - - it "raises an error when an unknown source file is specified" do - expect {subject.run("file.c", ["foo.bar"], :cache, env, {})}.to raise_error /Unknown source file .foo.bar. for CFile builder/ - end - end - end -end From 80a80a7cb067145ca0f574833c0e5bb670859494 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Thu, 25 May 2017 15:54:33 -0400 Subject: [PATCH 65/69] fully parallelize the Preprocess builder --- lib/rscons/builders/object.rb | 11 +++--- lib/rscons/builders/preprocess.rb | 42 +++++++++++++--------- spec/rscons/builders/preprocess_spec.rb | 48 ------------------------- 3 files changed, 31 insertions(+), 70 deletions(-) delete mode 100644 spec/rscons/builders/preprocess_spec.rb diff --git a/lib/rscons/builders/object.rb b/lib/rscons/builders/object.rb index 7650608..798d56b 100644 --- a/lib/rscons/builders/object.rb +++ b/lib/rscons/builders/object.rb @@ -79,8 +79,9 @@ module Rscons # # @param options [Hash] Builder run options. # - # @return [ThreadedCommand] - # Threaded command to execute. + # @return [String, ThreadedCommand] + # Target file name if target is up to date or a {ThreadedCommand} + # to execute to build the target. def run(options) target, sources, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars) vars = vars.merge({ @@ -88,14 +89,14 @@ module Rscons '_SOURCES' => sources, '_DEPFILE' => Rscons.set_suffix(target, env.expand_varref("${DEPFILESUFFIX}", vars)), }) - # Store vars back into options so new keys are accessible in #finalize. - options[:vars] = vars com_prefix = KNOWN_SUFFIXES.find do |compiler, suffix_var| sources.first.end_with?(*env.expand_varref("${#{suffix_var}}")) end.tap do |v| v.nil? and raise "Error: unknown input file type: #{sources.first.inspect}" end.first command = env.build_command("${#{com_prefix}CMD}", vars) + # Store vars back into options so new keys are accessible in #finalize. + options[:vars] = vars standard_threaded_build("#{com_prefix} #{target}", target, command, sources, env, cache) end @@ -103,7 +104,7 @@ module Rscons # # @param options [Hash] Builder finalize options. # - # @return [String,nil] + # @return [String, nil] # Name of the target file on success or nil on failure. def finalize(options) if options[:command_status] diff --git a/lib/rscons/builders/preprocess.rb b/lib/rscons/builders/preprocess.rb index 6bc8165..3b9d6f3 100644 --- a/lib/rscons/builders/preprocess.rb +++ b/lib/rscons/builders/preprocess.rb @@ -20,15 +20,13 @@ module Rscons # Run the builder to produce a build target. # - # @param target [String] Target file name. - # @param sources [Array] Source file name(s). - # @param cache [Cache] The Cache object. - # @param env [Environment] The Environment executing the builder. - # @param vars [Hash,VarSet] Extra construction variables. + # @param options [Hash] Builder run options. # - # @return [String,false] - # Name of the target file on success or false on failure. - def run(target, sources, cache, env, vars) + # @return [String, ThreadedCommand] + # Target file name if target is up to date or a {ThreadedCommand} + # to execute to build the target. + def run(options) + target, sources, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars) if sources.find {|s| s.end_with?(*env.expand_varref("${CXXSUFFIX}", vars))} pp_cc = "${CXX}" depgen = "${CXXDEPGEN}" @@ -42,17 +40,27 @@ module Rscons "_SOURCES" => sources, "_DEPFILE" => Rscons.set_suffix(target, env.expand_varref("${DEPFILESUFFIX}", vars))) command = env.build_command("${CPP_CMD}", vars) - unless cache.up_to_date?(target, command, sources, env) - cache.mkdir_p(File.dirname(target)) - return false unless env.execute("Preprocess #{target}", command) - deps = sources - if File.exists?(vars["_DEPFILE"]) - deps += Environment.parse_makefile_deps(vars["_DEPFILE"], nil) - FileUtils.rm_f(vars["_DEPFILE"]) + # Store vars back into options so new keys are accessible in #finalize. + options[:vars] = vars + standard_threaded_build("#{name} #{target}", target, command, sources, env, cache) + end + + # Finalize the build operation. + # + # @param options [Hash] Builder finalize options. + # + # @return [String, nil] + # Name of the target file on success or nil on failure. + def finalize(options) + if options[:command_status] + target, deps, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars) + if File.exists?(vars['_DEPFILE']) + deps += Environment.parse_makefile_deps(vars['_DEPFILE'], nil) + FileUtils.rm_f(vars['_DEPFILE']) end - cache.register_build(target, command, deps.uniq, env) + cache.register_build(target, options[:tc].command, deps.uniq, env) + target end - target end end diff --git a/spec/rscons/builders/preprocess_spec.rb b/spec/rscons/builders/preprocess_spec.rb deleted file mode 100644 index 0f56936..0000000 --- a/spec/rscons/builders/preprocess_spec.rb +++ /dev/null @@ -1,48 +0,0 @@ -module Rscons - module Builders - describe Preprocess do - - let(:env) {Environment.new} - subject {Preprocess.new} - - it "supports overriding CC construction variable" do - cache = double(Cache) - command = %w[my_cpp -E -MMD -MF module.mf -o module.pp module.c] - expect(cache).to receive(:up_to_date?).with("module.pp", command, %w[module.c], env).and_return(false) - expect(cache).to receive(:mkdir_p).with(".") - expect(env).to receive(:execute).with("Preprocess module.pp", command).and_return(true) - expect(File).to receive(:exists?).with("module.mf").and_return(true) - expect(Environment).to receive(:parse_makefile_deps).with("module.mf", nil).and_return(%w[module.c one.h two.h]) - expect(FileUtils).to receive(:rm_f).with("module.mf") - expect(cache).to receive(:register_build).with("module.pp", command, %w[module.c one.h two.h], env) - - expect(subject.run("module.pp", ["module.c"], cache, env, "CC" => "my_cpp")).to eq("module.pp") - end - - it "supports overriding CPP_CMD construction variable" do - cache = double(Cache) - command = %w[my_cpp module.c] - expect(cache).to receive(:up_to_date?).with("module.pp", command, %w[module.c], env).and_return(false) - expect(cache).to receive(:mkdir_p).with(".") - expect(env).to receive(:execute).with("Preprocess module.pp", command).and_return(true) - expect(File).to receive(:exists?).with("module.mf").and_return(true) - expect(Environment).to receive(:parse_makefile_deps).with("module.mf", nil).and_return(%w[module.c one.h two.h]) - expect(FileUtils).to receive(:rm_f).with("module.mf") - expect(cache).to receive(:register_build).with("module.pp", command, %w[module.c one.h two.h], env) - - expect(subject.run("module.pp", ["module.c"], cache, env, "CPP_CMD" => ["my_cpp", "${_SOURCES}"])).to eq("module.pp") - end - - it "returns false if executing the preprocessor fails" do - cache = double(Cache) - command = %w[gcc -E -MMD -MF module.mf -o module.pp module.c] - expect(cache).to receive(:up_to_date?).with("module.pp", command, %w[module.c], env).and_return(false) - expect(cache).to receive(:mkdir_p).with(".") - expect(env).to receive(:execute).with("Preprocess module.pp", command).and_return(false) - - expect(subject.run("module.pp", ["module.c"], cache, env, {})).to eq(false) - end - - end - end -end From 68468b6422a0f2bdbdadbdc1146f914c1ff17e91 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Thu, 25 May 2017 16:03:05 -0400 Subject: [PATCH 66/69] fully parallelize the Command builder --- build_tests/simple/command_builder.rb | 7 +++++++ lib/rscons/builders/command.rb | 26 ++++++++++++++++++-------- spec/build_tests_spec.rb | 15 +++++++++++++++ spec/rscons/builders/command_spec.rb | 18 ------------------ 4 files changed, 40 insertions(+), 26 deletions(-) create mode 100644 build_tests/simple/command_builder.rb delete mode 100755 spec/rscons/builders/command_spec.rb diff --git a/build_tests/simple/command_builder.rb b/build_tests/simple/command_builder.rb new file mode 100644 index 0000000..5b1c02d --- /dev/null +++ b/build_tests/simple/command_builder.rb @@ -0,0 +1,7 @@ +Rscons::Environment.new do |env| + command = %W[gcc -o ${_TARGET} ${_SOURCES}] + env.Command("simple.exe", + "simple.c", + "CMD" => command, + "CMD_DESC" => "BuildIt") +end diff --git a/lib/rscons/builders/command.rb b/lib/rscons/builders/command.rb index 797d7ed..5403e37 100644 --- a/lib/rscons/builders/command.rb +++ b/lib/rscons/builders/command.rb @@ -9,16 +9,14 @@ module Rscons # env.Command("docs.html", "docs.md", # CMD => %w[pandoc -fmarkdown -thtml -o${_TARGET} ${_SOURCES}]) class Command < Builder + # Run the builder to produce a build target. # - # @param target [String] Target file name. - # @param sources [Array] Source file name(s). - # @param cache [Cache] The Cache object. - # @param env [Environment] The Environment executing the builder. - # @param vars [Hash,VarSet] Extra construction variables. + # @param options [Hash] Builder run options. # - # @return [String,false] - # Name of the target file on success or false on failure. + # @return [String, ThreadedCommand] + # Target file name if target is up to date or a {ThreadedCommand} + # to execute to build the target. def run(target, sources, cache, env, vars) vars = vars.merge({ "_TARGET" => target, @@ -26,8 +24,20 @@ module Rscons }) command = env.build_command("${CMD}", vars) cmd_desc = vars["CMD_DESC"] || "Command" - standard_build("#{cmd_desc} #{target}", target, command, sources, env, cache) + standard_threaded_build("#{cmd_desc} #{target}", target, command, sources, env, cache) end + + # Finalize a build. + # + # @param options [Hash] + # Finalize options. + # + # @return [String, nil] + # The target name on success or nil on failure. + def finalize(options) + standard_finalize(options) + end + end end end diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index af71f26..2e7ff1e 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -706,6 +706,21 @@ EOF end end + context "Command builder" do + it "allows executing an arbitrary command" do + test_dir('simple') + + result = run_test(rsconsfile: "command_builder.rb") + expect(result.stderr).to eq "" + expect(lines(result.stdout)).to eq ["BuildIt simple.exe"] + expect(`./simple.exe`).to eq "This is a simple C program\n" + + result = run_test(rsconsfile: "command_builder.rb") + expect(result.stderr).to eq "" + expect(result.stdout).to eq "" + end + end + context "Directory builder" do it "creates the requested directory" do test_dir("simple") diff --git a/spec/rscons/builders/command_spec.rb b/spec/rscons/builders/command_spec.rb deleted file mode 100755 index 96ae96c..0000000 --- a/spec/rscons/builders/command_spec.rb +++ /dev/null @@ -1,18 +0,0 @@ -module Rscons - module Builders - describe Command do - - let(:command) { ['pandoc', '-fmarkdown', '-thtml', '-o${_TARGET}', '${_SOURCES}'] } - let(:env) {Environment.new} - subject {Command.new} - - it "invokes the command to build the target" do - expected_cmd = ['pandoc', '-fmarkdown', '-thtml', '-ofoo.html', 'foo.md'] - expect(subject).to receive(:standard_build).with("PANDOC foo.html", "foo.html", expected_cmd, ["foo.md"], env, :cache) - subject.run("foo.html", ["foo.md"], :cache, env, - "CMD" => command, "CMD_DESC" => "PANDOC") - end - - end - end -end From 05bbea6fa166359b39c84e082a0e84cb42eaaa5f Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Thu, 25 May 2017 17:00:21 -0400 Subject: [PATCH 67/69] add Environment#build_after --- build_tests/custom_builder/build_after.rb | 8 +++++ build_tests/custom_builder/gen.rb | 9 ++++++ lib/rscons/environment.rb | 36 +++++++++++++++++++++++ spec/build_tests_spec.rb | 6 ++++ 4 files changed, 59 insertions(+) create mode 100644 build_tests/custom_builder/build_after.rb create mode 100644 build_tests/custom_builder/gen.rb diff --git a/build_tests/custom_builder/build_after.rb b/build_tests/custom_builder/build_after.rb new file mode 100644 index 0000000..60259dc --- /dev/null +++ b/build_tests/custom_builder/build_after.rb @@ -0,0 +1,8 @@ +Rscons::Environment.new do |env| + env.Command("inc.c", + [], + "CMD" => %w[ruby gen.rb ${_TARGET}], + "CMD_DESC" => "Generating") + env.build_after("program.o", "inc.c") + env.Program("program.exe", ["program.c", "inc.c"]) +end diff --git a/build_tests/custom_builder/gen.rb b/build_tests/custom_builder/gen.rb new file mode 100644 index 0000000..79310fd --- /dev/null +++ b/build_tests/custom_builder/gen.rb @@ -0,0 +1,9 @@ +sleep 1.0 +c_file = ARGV.first +h_file = File.basename(c_file, ".c") + ".h" +File.open(c_file, "w") do |fh| + fh.puts +end +File.open(h_file, "w") do |fh| + fh.puts("#define THE_VALUE 191") +end diff --git a/lib/rscons/environment.rb b/lib/rscons/environment.rb index cc8cbd2..805ed86 100644 --- a/lib/rscons/environment.rb +++ b/lib/rscons/environment.rb @@ -459,6 +459,42 @@ module Rscons @user_deps[target] = (@user_deps[target] + user_deps).uniq end + # Manually record the given target(s) as needing to be built after the + # given prerequisite(s). + # + # For example, consider a builder registered to generate gen.c which also + # generates gen.h as a side-effect. If program.c includes gen.h, then it + # should not be compiled before gen.h has been generated. When using + # multiple threads to build, Rscons may attempt to compile program.c before + # gen.h has been generated because it does not know that gen.h will be + # generated along with gen.c. One way to prevent that situation would be + # to first process the Environment with just the code-generation builders + # in place and then register the compilation builders. Another way is to + # use this method to record that a certain target should not be built until + # another has completed. For example, for the situation previously + # described: + # env.build_after("program.o", "gen.c") + # + # @since 1.10.0 + # + # @param targets [String, Array] + # Target files to wait to build until the prerequisites are finished + # building. + # @param prerequisites [String, Array] + # Files that must be built before building the specified targets. + # + # @return [void] + def build_after(targets, prerequisites) + targets = Array(targets) + prerequisites = Array(prerequisites) + targets.each do |target| + @registered_build_dependencies[target] ||= Set.new + prerequisites.each do |prerequisite| + @registered_build_dependencies[target] << prerequisite + end + end + end + # Return the list of user dependencies for a given target. # # @param target [String] Target file name. diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 2e7ff1e..4ff375a 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -1090,6 +1090,12 @@ EOF elapsed = Time.new - start_time expect(elapsed).to be < 4 end + + it "allows the user to specify that a target be built after another" do + test_dir("custom_builder") + result = run_test(rsconsfile: "build_after.rb", rscons_args: %w[-j 4]) + expect(result.stderr).to eq "" + end end context "CLI" do From b5826bd7f494dc2f687f5aa3e111921502fc0717 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Thu, 25 May 2017 17:06:34 -0400 Subject: [PATCH 68/69] use new Builder#run signature for Command builder --- lib/rscons/builders/command.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/rscons/builders/command.rb b/lib/rscons/builders/command.rb index 5403e37..fcf5d7b 100644 --- a/lib/rscons/builders/command.rb +++ b/lib/rscons/builders/command.rb @@ -17,7 +17,8 @@ module Rscons # @return [String, ThreadedCommand] # Target file name if target is up to date or a {ThreadedCommand} # to execute to build the target. - def run(target, sources, cache, env, vars) + def run(options) + target, sources, cache, env, vars = options.values_at(:target, :sources, :cache, :env, :vars) vars = vars.merge({ "_TARGET" => target, "_SOURCES" => sources, From c7943bc2cc3ff8b6e9705fb1f1c7e16f856fc7e8 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Mon, 29 May 2017 10:57:27 -0400 Subject: [PATCH 69/69] ignore .bundle --- .gitignore | 3 ++- spec/spec_helper.rb | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 88e31de..2009ee7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ +/.bundle/ /.yardoc/ /_yardoc/ +/build_test_run/ /coverage/ /doc/ /pkg/ -/build_test_run/ diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 942a4c7..f4f7e27 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -2,6 +2,7 @@ require "simplecov" SimpleCov.start do add_filter "/spec/" + add_filter "/.bundle/" if ENV["partial_specs"] command_name "RSpec-partial" else