v1.19.1

This reverts commit 01ca86de8b.

CHANGELOG.md

@@ -1,5 +1,34 @@
 ## ChangeLog
 
+### v1.19.1
+
+#### Fixes
+
+- #123 - Install builder: remove target file before copying to avoid errors due to overwriting read-only files
+
+### v1.19.0
+
+#### New Features
+
+- #117 - ruby 2.7 compatibility
+
+#### Fixes
+
+- #115 - Check that user dependencies exist before running builders.
+
+### v1.18.0
+
+#### Performance Improvements
+
+- Backport 2.x VarSet performance improvements to 1.x branch.
+- Remove Cache dirty-tracking
+- Reduce recursion of VarSet#expand_varref by expanding vars earlier.
+
+#### Fixes
+
+- #73 - fix inconsistent type handling in VarSet#expand_varref
+- #75 - avoid mkdir() race conditions
+
 ### v1.17.0
 
 #### New Features

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rscons (1.17.0)
+    rscons (1.19.1)
       json (>= 1.8, < 3.0)
 
 GEM
@@ -9,28 +9,28 @@ GEM
   specs:
     diff-lcs (1.3)
     docile (1.1.5)
-    json (2.1.0)
-    rake (12.3.1)
-    rdoc (5.1.0)
-    rspec (3.6.0)
-      rspec-core (~> 3.6.0)
-      rspec-expectations (~> 3.6.0)
-      rspec-mocks (~> 3.6.0)
-    rspec-core (3.6.0)
-      rspec-support (~> 3.6.0)
-    rspec-expectations (3.6.0)
+    json (2.3.0)
+    rake (13.0.1)
+    rdoc (6.2.1)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.6.0)
-    rspec-mocks (3.6.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.6.0)
-    rspec-support (3.6.0)
-    simplecov (0.15.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
+    simplecov (0.15.1)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    yard (0.9.12)
+    yard (0.9.24)
 
 PLATFORMS
   ruby
@@ -41,8 +41,8 @@ DEPENDENCIES
   rdoc
   rscons!
   rspec
-  simplecov
+  simplecov (~> 0.15.0)
   yard
 
 BUNDLED WITH
-   1.16.6
+   2.1.4

build_tests/build_dir/install_read_only.rb

@@ -0,0 +1,3 @@
+Rscons::Environment.new do |env|
+  env.Install("dest", "install.rb")
+end

build_tests/simple/new_dep

@@ -0,0 +1 @@
+# This file is referenced by cache_debugging.rb

lib/rscons.rb

@@ -5,6 +5,7 @@ require_relative "rscons/cache"
 require_relative "rscons/environment"
 require_relative "rscons/job_set"
 require_relative "rscons/threaded_command"
+require_relative "rscons/util"
 require_relative "rscons/varset"
 require_relative "rscons/version"
 

lib/rscons/builders/install.rb

@@ -47,6 +47,7 @@ module Rscons
               printed_message = true
             end
             cache.mkdir_p(File.dirname(dest))
+            FileUtils.rm_f(dest)
             FileUtils.cp(src, dest, :preserve => true)
           end
           cache.register_build(dest, :Copy, [src], env)

lib/rscons/cache.rb

@@ -85,18 +85,15 @@ module Rscons
       @lookup_checksums = {}
     end
 
-    # Write the cache to disk to be loaded next time.
+    # Write the cache to disk.
     #
     # @return [void]
     def write
-      if @dirty || (@cache["version"] != VERSION)
       @cache["version"] = VERSION
       File.open(CACHE_FILE, "w") do |fh|
         fh.puts(JSON.dump(@cache))
       end
     end
-      @dirty = false
-    end
 
     # Check if target(s) are up to date.
     #
@@ -249,7 +246,6 @@ module Rscons
             }
           end,
         }
-        @dirty = true
       end
     end
 
@@ -272,9 +268,8 @@ module Rscons
         next if parts[i] == ""
         subpath = File.join(*parts[0, i + 1])
         unless File.exists?(subpath)
-          FileUtils.mkdir(subpath)
+          FileUtils.mkdir_p(subpath)
           @cache["directories"][subpath] = true
-          @dirty = true
         end
       end
     end
@@ -287,6 +282,16 @@ module Rscons
       @cache["directories"].keys
     end
 
+    # Return a file's checksum, or the previously calculated checksum for
+    # the same file.
+    #
+    # @param file [String] The file name.
+    #
+    # @return [String] The file's checksum.
+    def lookup_checksum(file)
+      @lookup_checksums[file] || calculate_checksum(file)
+    end
+
     private
 
     # Return a String key based on the target name to use in the on-disk cache.
@@ -315,17 +320,6 @@ module Rscons
       @cache["targets"] ||= {}
       @cache["directories"] ||= {}
       @lookup_checksums = {}
-      @dirty = false
-    end
-
-    # Return a file's checksum, or the previously calculated checksum for
-    # the same file.
-    #
-    # @param file [String] The file name.
-    #
-    # @return [String] The file's checksum.
-    def lookup_checksum(file)
-      @lookup_checksums[file] || calculate_checksum(file)
     end
 
     # Calculate and return a file's checksum.

lib/rscons/environment.rb

@@ -1,7 +1,6 @@
 require "fileutils"
 require "set"
 require "shellwords"
-require "thwait"
 
 module Rscons
   # The Environment class is the main programmatic interface to Rscons. It
@@ -280,6 +279,24 @@ module Rscons
       @varset.send(:[], *args)
     end
 
+    # Access the value of a construction variable.
+    #
+    # @since 1.18.0
+    #
+    # This method is similar to #[] but does not make a copy-on-access copy of
+    # the variable accessed. This means that the returned value is NOT safe to
+    # be modified by the caller. Thus the caller must guarantee that it does
+    # not modify the returned value.
+    #
+    # @param key [String, Symbol]
+    #   The construction variable name.
+    #
+    # @return [Object]
+    #   The construction variable's value.
+    def get_var(key)
+      @varset.get_var(key)
+    end
+
     # Set a construction variable's value.
     #
     # @see VarSet#[]=
@@ -323,6 +340,7 @@ module Rscons
           cache.clear_checksum_cache!
 
           if job
+            validate_user_deps(job[:target], @user_deps[job[:target]], cache)
             result = run_builder(job[:builder],
                                  job[:target],
                                  job[:sources],
@@ -782,7 +800,7 @@ module Rscons
             append["LDFLAGS", ["-arch", val]]
           end
           skip = true
-        elsif word =~ /^#{self["CPPDEFPREFIX"]}(.*)$/
+        elsif word =~ /^#{get_var("CPPDEFPREFIX")}(.*)$/
           handle["CPPDEFINES", $1]
         elsif word == "-include"
           if val = words[i + 1]
@@ -795,11 +813,11 @@ module Rscons
             append["LDFLAGS", ["-isysroot", val]]
           end
           skip = true
-        elsif word =~ /^#{self["INCPREFIX"]}(.*)$/
+        elsif word =~ /^#{get_var("INCPREFIX")}(.*)$/
           handle["CPPPATH", $1]
-        elsif word =~ /^#{self["LIBLINKPREFIX"]}(.*)$/
+        elsif word =~ /^#{get_var("LIBLINKPREFIX")}(.*)$/
           handle["LIBS", $1]
-        elsif word =~ /^#{self["LIBDIRPREFIX"]}(.*)$/
+        elsif word =~ /^#{get_var("LIBDIRPREFIX")}(.*)$/
           handle["LIBPATH", $1]
         elsif word == "-mno-cygwin"
           append["CCFLAGS", [word]]
@@ -848,7 +866,7 @@ module Rscons
     # @return [void]
     def merge_flags(flags)
       flags.each_pair do |key, val|
-        if self[key].is_a?(Array) and val.is_a?(Array)
+        if self.get_var(key).is_a?(Array) and val.is_a?(Array)
           self[key] += val
         else
           self[key] = val
@@ -993,10 +1011,7 @@ module Rscons
           !thread.alive?
         end
       else
-        if threads.empty?
-          raise "No threads to wait for"
-        end
-        ThreadsWait.new(*threads).next_wait
+        Util.wait_for_thread(*threads)
       end
     end
 
@@ -1089,5 +1104,25 @@ module Rscons
       end
       deps
     end
+
+    # Ensures that user dependencies exist with valid checksums in the
+    # cache.  Raises an exception if any dependency is invalid.
+    #
+    # @param target [String]
+    #   Target to be built
+    # @param user_deps [Array<String>, nil]
+    #   User dependencies of the target
+    # @param cache [Cache]
+    #   Rscons cache instance
+    #
+    # @return [void]
+    def validate_user_deps(target, user_deps, cache)
+      return if user_deps.nil?
+      user_deps.each do |dep|
+        if cache.lookup_checksum(dep) == ""
+          raise "User dependency #{dep} of target #{target} is invalid"
+        end
+      end
+    end
   end
 end

lib/rscons/util.rb

@@ -0,0 +1,34 @@
+module Rscons
+  # A collection of stand-alone utility methods.
+  module Util
+    class << self
+
+      # Wait for any of a number of threads to complete.
+      #
+      # @param threads [Array<Thread>]
+      #   Threads to wait for.
+      #
+      # @return [Thread]
+      #   The Thread that completed.
+      def wait_for_thread(*threads)
+        if threads.empty?
+          raise "No threads to wait for"
+        end
+        queue = Queue.new
+        threads.each do |thread|
+          # Create a wait thread for each thread we're waiting for.
+          Thread.new do
+            begin
+              thread.join
+            ensure
+              queue.push(thread)
+            end
+          end
+        end
+        # Wait for any thread to complete.
+        queue.pop
+      end
+
+    end
+  end
+end

lib/rscons/varset.rb

@@ -11,11 +11,13 @@ module Rscons
       append(vars)
     end
 
-    # Access the value of variable.
+    # Access the value of a variable.
     #
-    # @param key [String, Symbol] The variable name.
+    # @param key [String, Symbol]
+    #   The variable name.
     #
-    # @return [Object] The variable's value.
+    # @return [Object]
+    #   The variable's value.
     def [](key)
       if @my_vars.include?(key)
         @my_vars[key]
@@ -30,11 +32,38 @@ module Rscons
       end
     end
 
+    # Access the value of a variable.
+    #
+    # This method is similar to #[] but does not make a copy-on-access copy of
+    # the variable accessed. This means that the returned value is NOT safe to
+    # be modified by the caller. Thus the caller must guarantee that it does
+    # not modify the returned value.
+    #
+    # @param key [String, Symbol]
+    #   The variable name.
+    #
+    # @return [Object]
+    #   The variable's value.
+    def get_var(key)
+      if @my_vars.include?(key)
+        @my_vars[key]
+      else
+        @coa_vars.each do |coa_vars|
+          if coa_vars.include?(key)
+            return coa_vars[key]
+          end
+        end
+        nil
+      end
+    end
+
     # Assign a value to a variable.
     #
-    # @param key [String, Symbol] The variable name.
+    # @param key [String, Symbol]
+    #   The variable name.
     #
-    # @param val [Object] The value to set.
+    # @param val [Object]
+    #   The value to set.
     def []=(key, val)
       @my_vars[key] = val
     end
@@ -94,36 +123,42 @@ module Rscons
     # @return [nil, String, Array, Symbol, TrueClass, FalseClass]
     #   Expanded value with "$!{var}" variable references replaced.
     def expand_varref(varref, lambda_args)
-      if varref.is_a?(String)
+      case varref
+      when String
         if varref =~ /^(.*)\$\{([^}]+)\}(.*)$/
           prefix, varname, suffix = $1, $2, $3
-          varval = expand_varref(self[varname], lambda_args)
-          if varval.is_a?(String) or varval.nil?
-            expand_varref("#{prefix}#{varval}#{suffix}", lambda_args)
-          elsif varval.is_a?(Array)
-            varval.map {|vv| expand_varref("#{prefix}#{vv}#{suffix}", lambda_args)}.flatten
+          prefix = expand_varref(prefix, lambda_args) unless prefix.empty?
+          varval = expand_varref(get_var(varname), lambda_args)
+          # suffix needs no expansion since the regex matches the last occurence
+          case varval
+          when Array
+            if prefix.is_a?(Array)
+              varval.map {|vv| prefix.map {|p| "#{p}#{vv}#{suffix}"}}.flatten
             else
-            raise "I do not know how to expand a variable reference to a #{varval.class.name} (from #{varname.inspect} => #{self[varname].inspect})"
+              varval.map {|vv| "#{prefix}#{vv}#{suffix}"}
+            end
+          when String, Symbol, true, false, nil
+            if prefix.is_a?(Array)
+              prefix.map {|p| "#{p}#{varval}#{suffix}"}
+            else
+              "#{prefix}#{varval}#{suffix}"
+            end
+          else
+            raise "Unknown construction variable type: #{varval.class} (from #{varname.inspect} => #{get_var(varname).inspect})"
           end
         else
           varref
         end
-      elsif varref.is_a?(Array)
+      when Array
         varref.map do |ent|
           expand_varref(ent, lambda_args)
         end.flatten
-      elsif varref.is_a?(Proc)
+      when Symbol, true, false, nil
+        varref
+      when Proc
         expand_varref(varref[*lambda_args], lambda_args)
-      elsif varref.nil?
-        nil
-      elsif varref.is_a?(Symbol)
-        varref
-      elsif varref.is_a?(TrueClass)
-        varref
-      elsif varref.is_a?(FalseClass)
-        varref
       else
-        raise "Unknown varref type: #{varref.class} (#{varref.inspect})"
+        raise "Unknown construction variable type: #{varref.class} (#{varref.inspect})"
       end
     end
 
@@ -188,16 +223,16 @@ module Rscons
     #
     # @return [Object] Deep copied value.
     def deep_dup(obj)
-      obj_class = obj.class
-      if obj_class == Hash
+      case obj
+      when String
+        obj.dup
+      when Array
+        obj.map { |v| deep_dup(v) }
+      when Hash
         obj.reduce({}) do |result, (k, v)|
           result[k] = deep_dup(v)
           result
         end
-      elsif obj_class == Array
-        obj.map { |v| deep_dup(v) }
-      elsif obj_class == String
-        obj.dup
       else
         obj
       end

lib/rscons/version.rb

@@ -1,4 +1,4 @@
 module Rscons
   # gem version
-  VERSION = "1.17.0"
+  VERSION = "1.19.1"
 end

rscons.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency "rspec"
   gem.add_development_dependency "rake"
-  gem.add_development_dependency "simplecov"
+  gem.add_development_dependency "simplecov", "~> 0.15.0"
   gem.add_development_dependency "yard"
   gem.add_development_dependency "rdoc"
 end

spec/build_tests_spec.rb

@@ -74,6 +74,8 @@ describe Rscons do
     command_name = "#{command_prefix}#{@statics[:build_test_id]}"
     File.open("_simplecov_setup.rb", "w") do |fh|
       fh.puts <<EOF
+require "bundler"
+Bundler.setup
 require "simplecov"
 class MyFormatter
   def format(*args)
@@ -95,7 +97,7 @@ ENV["TERM"] = nil
 EOF
     end
     stdout, stderr, status = nil, nil, nil
-    Bundler.with_clean_env do
+    Bundler.with_unbundled_env do
       stdout, stderr, status = Open3.capture3(*command)
     end
     # Remove output lines generated as a result of the test environment
@@ -415,14 +417,14 @@ EOF
     expect(result.stderr).to eq ""
     expect(lines(result.stdout)).to eq(["LD simple.exe"])
 
-    File.unlink("program.ld")
-    result = run_test(rsconsfile: "user_dependencies.rb")
-    expect(result.stderr).to eq ""
-    expect(lines(result.stdout)).to eq ["LD simple.exe"]
-
     result = run_test(rsconsfile: "user_dependencies.rb")
     expect(result.stderr).to eq ""
     expect(result.stdout).to eq ""
+
+    File.unlink("program.ld")
+    result = run_test(rsconsfile: "user_dependencies.rb")
+    expect(result.stderr).to match /User dependency program\.ld of target simple\.exe is invalid/
+    expect(result.status).to_not eq 0
   end
 
   unless ENV["omit_gdc_tests"]
@@ -1000,6 +1002,22 @@ EOF
       expect(lines(result.stdout)).to eq ["Install inst.exe"]
     end
 
+    it "overwrites a read-only file" do
+      test_dir("build_dir")
+
+      FileUtils.chmod("a=r", "install.rb")
+      result = run_test(rsconsfile: "install_read_only.rb")
+      expect(result.stderr).to eq ""
+      expect(lines(result.stdout)).to eq ["Install dest"]
+
+      FileUtils.rm_f("install.rb")
+      File.binwrite("install.rb", "hi")
+      FileUtils.chmod("a=r", "install.rb")
+      result = run_test(rsconsfile: "install_read_only.rb")
+      expect(result.stderr).to eq ""
+      expect(lines(result.stdout)).to eq ["Install dest"]
+    end
+
     it "operates the same as a Copy builder" do
       test_dir("build_dir")
 

spec/rscons/cache_spec.rb

@@ -26,13 +26,13 @@ module Rscons
         cache = build_from(_cache)
         expect(File).to receive(:exists?).with("one").and_return(true)
         expect(File).to receive(:exists?).with("one/two").and_return(false)
-        expect(FileUtils).to receive(:mkdir).with("one/two")
+        expect(FileUtils).to receive(:mkdir_p).with("one/two")
         expect(File).to receive(:exists?).with("one/two/three").and_return(false)
-        expect(FileUtils).to receive(:mkdir).with("one/two/three")
+        expect(FileUtils).to receive(:mkdir_p).with("one/two/three")
         expect(File).to receive(:exists?).with("one").and_return(true)
         expect(File).to receive(:exists?).with("one/two").and_return(true)
         expect(File).to receive(:exists?).with("one/two/four").and_return(false)
-        expect(FileUtils).to receive(:mkdir).with("one/two/four")
+        expect(FileUtils).to receive(:mkdir_p).with("one/two/four")
         cache.mkdir_p("one/two/three")
         cache.mkdir_p("one\\two\\four")
         expect(cache.directories).to eq ["one/two", "one/two/three", "one/two/four"]
@@ -43,7 +43,7 @@ module Rscons
         cache = build_from(_cache)
         expect(File).to receive(:exists?).with("/one").and_return(true)
         expect(File).to receive(:exists?).with("/one/two").and_return(false)
-        expect(FileUtils).to receive(:mkdir).with("/one/two")
+        expect(FileUtils).to receive(:mkdir_p).with("/one/two")
         cache.mkdir_p("/one/two")
         expect(cache.directories).to eq ["/one/two"]
       end

spec/rscons/environment_spec.rb

@@ -184,7 +184,7 @@ module Rscons
         env["foo"] = {}
         expect(env.expand_varref(["-p${path}", "${flags}"])).to eq ["-pdir1", "-pdir2", "-x", "-y", "-z"]
         expect(env.expand_varref("foo")).to eq "foo"
-        expect {env.expand_varref("${foo}")}.to raise_error /Unknown.varref.type/
+        expect {env.expand_varref("${foo}")}.to raise_error /Unknown.construction.variable.type/
         expect(env.expand_varref("${specialflag}")).to eq "-z"
         expect(env.expand_varref("${path}")).to eq ["dir1", "dir2"]
       end

spec/rscons/varset_spec.rb

@@ -166,13 +166,26 @@ module Rscons
         expect(v.expand_varref(false, :lambda_args)).to eq(false)
       end
       it "raises an error when given an invalid argument" do
-        expect { v.expand_varref({a: :b}, :lambda_args) }.to raise_error /Unknown varref type: Hash/
+        expect { v.expand_varref({a: :b}, :lambda_args) }.to raise_error /Unknown construction variable type: Hash/
       end
       it "raises an error when an expanded variable is an unexpected type" do
-        expect(v).to receive(:[]).at_least(1).times.with("bad").and_return("bad_val")
+        expect(v).to receive(:get_var).at_least(1).times.with("bad").and_return("bad_val")
         expect(v).to receive(:expand_varref).with("bad_val", :lambda_args).and_return({a: :b})
         expect(v).to receive(:expand_varref).and_call_original
-        expect { v.expand_varref("${bad}", :lambda_args) }.to raise_error /I do not know how to expand a variable reference to a Hash/
+        expect { v.expand_varref("${bad}", :lambda_args) }.to raise_error /Unknown construction variable type: Hash/
+      end
+      it "expands symbols within a string outside an array" do
+        v['var'] = :a_symbol
+        expect(v.expand_varref("this is ${var}", :lambda_args)).to eq "this is a_symbol"
+      end
+      it "expands booleans within a string outside an array" do
+        v['var'] = false
+        expect(v.expand_varref("this is ${var}", :lambda_args)).to eq "this is false"
+      end
+      it "expands symbols and booleans in an array" do
+        v['var'] = [:a_symbol, false]
+        expanded = v.expand_varref("this is ${var}", :lambda_args)
+        expect(expanded).to eq(["this is a_symbol", "this is false"])
       end
     end