handle threaded commands in Environment#process

lib/rscons/environment.rb

@@ -284,18 +284,60 @@ module Rscons
     def process
       cache = Cache.instance
       begin
-        while job = @job_set.get_next_job_to_run
+        while @job_set.size > 0
+
+          # TODO: get_next_job_to_run needs to take into account targets still
+          # being processed.
+          job = @job_set.get_next_job_to_run
+
           # TODO: have Cache determine when checksums may be invalid based on
           # file size and/or timestamp.
           cache.clear_checksum_cache!
-          result = run_builder(job[:builder],
+
-                               job[:target],
+          if job
-                               job[:sources],
+            result = run_builder(job[:builder],
-                               cache,
+                                 job[:target],
-                               job[:vars])
+                                 job[:sources],
-          unless result
+                                 cache,
-            raise BuildError.new("Failed to build #{job[:target]}")
+                                 job[:vars],
+                                 allow_delayed_execution: true)
+            unless result.is_a?(ThreadedCommand)
+              unless result
+                raise BuildError.new("Failed to build #{job[:target]}")
+              end
+            end
           end
+
+          completed_tcs = Set.new
+          # First do a non-blocking wait to pick up any threads that have
+          # completed since last time.
+          loop do
+            if tc = wait_for_threaded_commands(nonblock: true)
+              completed_tcs << tc
+            else
+              break
+            end
+          end
+
+          # If needed, do a blocking wait.
+          if job.nil? or @threaded_commands.size >= Rscons.n_threads
+            completed_tcs << wait_for_threaded_commands
+          end
+
+          # Process all completed {ThreadedCommand} objects.
+          completed_tcs.each do |tc|
+            result = builder.finalize(
+              command_status: tc.thread.value,
+              builder_info: tc.builder_info)
+            if result
+              @build_hooks[:post].each do |build_hook_block|
+                build_hook_block.call(tc.build_operation)
+              end
+            else
+              raise BuildError.new("Failed to build #{tc.build_operation[:target]}")
+            end
+          end
+
         end
       ensure
         cache.write
@@ -785,20 +827,35 @@ module Rscons
     # @option options [Set<ThreadedCommand>, Array<ThreadedCommand>] :which
     #   Which {ThreadedCommand} objects to wait for. If not specified, this
     #   method will wait for any.
+    # @option options [Boolean] :nonblock
+    #   Set to true to not block.
     #
-    # @return [ThreadedCommand]
+    # @return [ThreadedCommand, nil]
     #   The {ThreadedCommand} object that is finished.
     def wait_for_threaded_commands(options = {})
-      raise "No threaded commands to wait for" if @threaded_commands.empty?
+      if @threaded_commands.empty?
+        if options[:nonblock]
+          return nil
+        else
+          raise "No threaded commands to wait for"
+        end
+      end
       options[:which] ||= @threaded_commands
       threads = options[:which].map(&:thread)
       tw = ThreadsWait.new(*threads)
-      finished_thread = tw.next_wait
+      finished_thread =
-      threaded_command = @threaded_commands.find do |tc|
+        begin
-        tc.thread == finished_thread
+          tw.next_wait(options[:nonblock])
+        rescue ThreadsWait::ErrNoFinishedThread
+          nil
+        end
+      if finished_thread
+        threaded_command = @threaded_commands.find do |tc|
+          tc.thread == finished_thread
+        end
+        @threaded_commands.delete(threaded_command)
+        threaded_command
       end
-      @threaded_commands.delete(threaded_command)
-      threaded_command
     end
 
     # Return a string representation of a command.

spec/rscons/environment_spec.rb

@@ -170,7 +170,7 @@ module Rscons
         cache = "cache"
         expect(Cache).to receive(:instance).and_return(cache)
         allow(cache).to receive(:clear_checksum_cache!)
-        expect(env).to receive(:run_builder).with(anything, "a.out", ["main.c"], cache, {}).and_return(true)
+        expect(env).to receive(:run_builder).with(anything, "a.out", ["main.c"], cache, {}, allow_delayed_execution: true).and_return(true)
         expect(cache).to receive(:write)
 
         env.process
@@ -184,8 +184,8 @@ module Rscons
         cache = "cache"
         expect(Cache).to receive(:instance).and_return(cache)
         allow(cache).to receive(:clear_checksum_cache!)
-        expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}).and_return("main.o")
+        expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}, allow_delayed_execution: true).and_return("main.o")
-        expect(env).to receive(:run_builder).with(anything, "a.out", ["main.o"], cache, {}).and_return("a.out")
+        expect(env).to receive(:run_builder).with(anything, "a.out", ["main.o"], cache, {}, allow_delayed_execution: true).and_return("a.out")
         expect(cache).to receive(:write)
 
         env.process
@@ -199,7 +199,7 @@ module Rscons
         cache = "cache"
         expect(Cache).to receive(:instance).and_return(cache)
         allow(cache).to receive(:clear_checksum_cache!)
-        expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}).and_return(false)
+        expect(env).to receive(:run_builder).with(anything, "main.o", ["other.cc"], cache, {}, allow_delayed_execution: true).and_return(false)
         expect(cache).to receive(:write)
 
         expect { env.process }.to raise_error BuildError, /Failed.to.build.main.o/