josh/rscons
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

wip for precompile phase for D

Browse Source
This commit is contained in:
Josh Holtrop 2026-01-18 21:23:50 -05:00
parent 67c135f610
commit 86fcebbda7
10 changed files with 101 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build_tests/d/Rsconscript
View File

@ -3,5 +3,6 @@ configure do
end end
env(echo: :command) do |env| env(echo: :command) do |env|
env.Program("hello-d.exe", glob("*.d")) env["D_IMPORT_PATH"] << "src"
env.Program("hello-d.exe", glob("src/*.d"))
end end

3
build_tests/d/build-ldc2.rb
View File

@ -3,5 +3,6 @@ configure do
end end
env(echo: :command) do |env| env(echo: :command) do |env|
env.Program("hello-d.exe", glob("*.d")) env["D_IMPORT_PATH"] << "src"
env.Program("hello-d.exe", glob("src/*.d"))
end end

5
build_tests/d/link_objects.rb
View File

@ -1,5 +1,6 @@
env(echo: :command) do |env| env(echo: :command) do |env|
env.Object("main.o", "main.d") env["LD_LIBRARY_PATH"] << "src"
env.Object("mod.o", "mod.d") env.Object("main.o", "src/main.d")
env.Object("mod.o", "src/mod.d")
env.Program("hello-d.exe", ["main.o", "mod.o"]) env.Program("hello-d.exe", ["main.o", "mod.o"])
end end

0
build_tests/d/main.d → build_tests/d/src/main.d
View File

0
build_tests/d/mod.d → build_tests/d/src/mod.d
View File

11
lib/rscons/builder_set.rb
View File

@ -27,8 +27,9 @@ module Rscons
# env.Directory("dest") # env.Directory("dest")
# env.Install("dest", "bin") # env.Install("dest", "bin")
# env.Install("dest", "share") # env.Install("dest", "share")
self[builder.target] ||= [] target = Util.absolute_path(builder.target)
self[builder.target] << builder self[target] ||= []
self[target] << builder
end end
# Return the number of remaining build steps. # Return the number of remaining build steps.
@ -55,9 +56,14 @@ module Rscons
def get_next_builder_to_run(targets_still_building) def get_next_builder_to_run(targets_still_building)
to_build = self.find do |target, builders| to_build = self.find do |target, builders|
deps = builders.first.sources + (@build_dependencies[target] || []).to_a deps = builders.first.sources + (@build_dependencies[target] || []).to_a
deps.map! {|dep| Util.absolute_path(dep)}
# All dependencies must have been built for this target to be ready to # All dependencies must have been built for this target to be ready to
# build. # build.
deps.all? do |dep| deps.all? do |dep|
puts "BuilderSet: #{target}: dep #{dep}"
puts " still building? #{targets_still_building.include?(dep)}"
puts " self.include? #{self.include?(dep)}"
puts " @side_effects.include? #{@side_effects.include?(dep)}"
!(targets_still_building.include?(dep) || !(targets_still_building.include?(dep) ||
self.include?(dep) || self.include?(dep) ||
@side_effects.include?(dep)) @side_effects.include?(dep))
@ -66,6 +72,7 @@ module Rscons
if to_build if to_build
target, builders = *to_build target, builders = *to_build
puts "Selecting #{target.inspect} to build"
builder = builders.first builder = builders.first
if builders.size > 1 if builders.size > 1
builders.slice!(0) builders.slice!(0)

48
lib/rscons/cache.rb
View File

@ -142,14 +142,16 @@ module Rscons
# - each cached dependency file's current checksum matches the checksum # - each cached dependency file's current checksum matches the checksum
# stored in the cache file # stored in the cache file
def up_to_date?(targets, command, deps, env, options = {}) def up_to_date?(targets, command, deps, env, options = {})
deps.map! {|dep| Util.absolute_path(dep)}
Array(targets).each do |target| Array(targets).each do |target|
target = Util.absolute_path(target)
cache_key = get_cache_key(target) cache_key = get_cache_key(target)
unless Rscons.phony_target?(target) unless Rscons.phony_target?(target)
# target file must exist on disk # target file must exist on disk
unless File.exist?(target) unless File.exist?(target)
if options[:debug] if options[:debug] || ENV["RSCONS_CACHE_DEBUG"]
puts "Target #{target} needs rebuilding because it does not exist on disk" puts "Cache: up_to_date?: Target #{target} needs rebuilding because it does not exist on disk"
end end
return false return false
end end
@ -157,8 +159,8 @@ module Rscons
# target must be registered in the cache # target must be registered in the cache
unless @cache["targets"].has_key?(cache_key) unless @cache["targets"].has_key?(cache_key)
if options[:debug] if options[:debug] || ENV["RSCONS_CACHE_DEBUG"]
puts "Target #{target} needs rebuilding because there is no cached build information for it" puts "Cache: up_to_date?: Target #{target} needs rebuilding because there is no cached build information for it"
end end
return false return false
end end
@ -166,8 +168,8 @@ module Rscons
unless Rscons.phony_target?(target) unless Rscons.phony_target?(target)
# target must have the same checksum as when it was built last # target must have the same checksum as when it was built last
unless @cache["targets"][cache_key]["checksum"] == lookup_checksum(target) unless @cache["targets"][cache_key]["checksum"] == lookup_checksum(target)
if options[:debug] if options[:debug] || ENV["RSCONS_CACHE_DEBUG"]
puts "Target #{target} needs rebuilding because it has been changed on disk since being built last" puts "Cache: up_to_date?: Target #{target} needs rebuilding because it has been changed on disk since being built last"
end end
return false return false
end end
@ -175,8 +177,8 @@ module Rscons
# command used to build target must be identical # command used to build target must be identical
unless @cache["targets"][cache_key]["command"] == Digest::MD5.hexdigest(command.inspect) unless @cache["targets"][cache_key]["command"] == Digest::MD5.hexdigest(command.inspect)
if options[:debug] if options[:debug] || ENV["RSCONS_CACHE_DEBUG"]
puts "Target #{target} needs rebuilding because the command used to build it has changed" puts "Cache: up_to_date?: Target #{target} needs rebuilding because the command used to build it has changed"
end end
return false return false
end end
@ -186,16 +188,16 @@ module Rscons
if options[:strict_deps] if options[:strict_deps]
# depedencies passed in must exactly equal those in the cache # depedencies passed in must exactly equal those in the cache
unless deps == cached_deps_fnames unless deps == cached_deps_fnames
if options[:debug] if options[:debug] || ENV["RSCONS_CACHE_DEBUG"]
puts "Target #{target} needs rebuilding because the :strict_deps option is given and the set of dependencies does not match the previous set of dependencies" puts "Cache: up_to_date?: Target #{target} needs rebuilding because the :strict_deps option is given and the set of dependencies does not match the previous set of dependencies"
end end
return false return false
end end
else else
# all dependencies passed in must exist in cache (but cache may have more) # all dependencies passed in must exist in cache (but cache may have more)
unless (Set.new(deps) - Set.new(cached_deps_fnames)).empty? unless (Set.new(deps) - Set.new(cached_deps_fnames)).empty?
if options[:debug] if options[:debug] || ENV["RSCONS_CACHE_DEBUG"]
puts "Target #{target} needs rebuilding because there are new dependencies" puts "Cache: up_to_date?: Target #{target} needs rebuilding because there are new dependencies"
end end
return false return false
end end
@ -203,11 +205,12 @@ module Rscons
# set of user dependencies must match # set of user dependencies must match
user_deps = env.get_user_deps(target) || [] user_deps = env.get_user_deps(target) || []
user_deps.map! {|dep| Util.absolute_path(dep)}
cached_user_deps = @cache["targets"][cache_key]["user_deps"] || [] cached_user_deps = @cache["targets"][cache_key]["user_deps"] || []
cached_user_deps_fnames = cached_user_deps.map { |dc| dc["fname"] } cached_user_deps_fnames = cached_user_deps.map { |dc| dc["fname"] }
unless user_deps == cached_user_deps_fnames unless user_deps == cached_user_deps_fnames
if options[:debug] if options[:debug] || ENV["RSCONS_CACHE_DEBUG"]
puts "Target #{target} needs rebuilding because the set of user-specified dependency files has changed" puts "Cache: up_to_date?: Target #{target} needs rebuilding because the set of user-specified dependency files has changed"
end end
return false return false
end end
@ -215,12 +218,14 @@ module Rscons
# all cached dependencies must have their checksums match # all cached dependencies must have their checksums match
(cached_deps + cached_user_deps).each do |dep_cache| (cached_deps + cached_user_deps).each do |dep_cache|
unless dep_cache["checksum"] == lookup_checksum(dep_cache["fname"]) unless dep_cache["checksum"] == lookup_checksum(dep_cache["fname"])
if options[:debug] if options[:debug] || ENV["RSCONS_CACHE_DEBUG"]
puts "Target #{target} needs rebuilding because dependency file #{dep_cache["fname"]} has changed" puts "Cache: up_to_date?: Target #{target} needs rebuilding because dependency file #{dep_cache["fname"]} has changed"
end end
return false return false
end end
end end
puts "Cache: up_to_date?: Target #{target} is up to date" if ENV["RSCONS_CACHE_DEBUG"]
end end
true true
@ -247,22 +252,26 @@ module Rscons
# @return [void] # @return [void]
def register_build(targets, command, deps, env, options = {}) def register_build(targets, command, deps, env, options = {})
Array(targets).each do |target| Array(targets).each do |target|
target = Util.absolute_path(target)
target_checksum = target_checksum =
if options[:side_effect] or Rscons.phony_target?(target) if options[:side_effect] or Rscons.phony_target?(target)
"" ""
else else
calculate_checksum(target) calculate_checksum(target)
end end
puts "Cache: register_build(#{target}, #{target_checksum})"
@cache["targets"][get_cache_key(target)] = { @cache["targets"][get_cache_key(target)] = {
"command" => Digest::MD5.hexdigest(command.inspect), "command" => Digest::MD5.hexdigest(command.inspect),
"checksum" => target_checksum, "checksum" => target_checksum,
"deps" => deps.map do |dep| "deps" => deps.map do |dep|
dep = Util.absolute_path(dep)
{ {
"fname" => dep, "fname" => dep,
"checksum" => lookup_checksum(dep), "checksum" => lookup_checksum(dep),
} }
end, end,
"user_deps" => (env.get_user_deps(target) || []).map do |dep| "user_deps" => (env.get_user_deps(target) || []).map do |dep|
dep = Util.absolute_path(dep)
{ {
"fname" => dep, "fname" => dep,
"checksum" => lookup_checksum(dep), "checksum" => lookup_checksum(dep),
@ -381,6 +390,8 @@ module Rscons
# #
# @return [String] The file's checksum. # @return [String] The file's checksum.
def lookup_checksum(file) def lookup_checksum(file)
file = Util.absolute_path(file)
puts "Cache: lookup_checksum(#{file})" if ENV["RSCONS_CACHE_DEBUG"]
@lookup_checksums[file] || calculate_checksum(file) @lookup_checksums[file] || calculate_checksum(file)
end end
@ -390,7 +401,10 @@ module Rscons
# #
# @return [String] The file's checksum. # @return [String] The file's checksum.
def calculate_checksum(file) def calculate_checksum(file)
@lookup_checksums[file] = Digest::MD5.hexdigest(File.read(file, mode: "rb")) rescue "" file = Util.absolute_path(file)
cs = Digest::MD5.hexdigest(File.read(file, mode: "rb")) rescue ""
puts "Cache: calculate_checksum(#{file}) = #{cs}" if ENV["RSCONS_CACHE_DEBUG"]
@lookup_checksums[file] = cs
end end
end end

24
lib/rscons/environment.rb
View File

@ -400,7 +400,9 @@ module Rscons
if target.is_a?(Builder) if target.is_a?(Builder)
target = target.target target = target.target
end end
target = expand(target.to_s) unless Rscons.phony_target?(target)
target = Util.absolute_path(expand(target))
end
user_deps = user_deps.map do |ud| user_deps = user_deps.map do |ud|
if ud.is_a?(Builder) if ud.is_a?(Builder)
ud = ud.target ud = ud.target
@ -408,9 +410,9 @@ module Rscons
expand(ud) expand(ud)
end end
@user_deps[target] ||= [] @user_deps[target] ||= []
(@user_deps[target] + user_deps).each do |ud| user_deps.each do |ud|
unless Rscons.phony_target?(ud) || @user_deps[target].include?(ud) unless @user_deps[target].include?(ud)
@user_deps[target] << ud @user_deps[target] << Util.absolute_path(ud)
end end
end end
build_after(target, user_deps) build_after(target, user_deps)
@ -443,14 +445,16 @@ module Rscons
targets = Array(targets) targets = Array(targets)
prerequisites = Array(prerequisites) prerequisites = Array(prerequisites)
targets.each do |target| targets.each do |target|
target = expand(target) unless Rscons.phony_target?(target)
target = Util.absolute_path(expand(target))
end
@registered_build_dependencies[target] ||= Set.new @registered_build_dependencies[target] ||= Set.new
prerequisites.each do |prerequisite| prerequisites.each do |prerequisite|
if prerequisite.is_a?(Builder) if prerequisite.is_a?(Builder)
prerequisite = prerequisite.target prerequisite = prerequisite.target
end end
prerequisite = expand(prerequisite) prerequisite = expand(prerequisite)
@registered_build_dependencies[target] << prerequisite @registered_build_dependencies[target] << Util.absolute_path(prerequisite)
end end
end end
end end
@ -465,7 +469,7 @@ module Rscons
# #
# @return [void] # @return [void]
def produces(target, *side_effects) def produces(target, *side_effects)
target = expand(target) target = Util.absolute_path(expand(target))
@builder_sets.reverse.each do |builder_set| @builder_sets.reverse.each do |builder_set|
if builders = builder_set[target] if builders = builder_set[target]
builders.last.produces(*side_effects) builders.last.produces(*side_effects)
@ -640,7 +644,9 @@ module Rscons
index = self["D_IMPORT_PATH"].find_index {|ip| ip == import_path} index = self["D_IMPORT_PATH"].find_index {|ip| ip == import_path}
self["D_IMPORT_PATH"].insert(index, precompile_path) self["D_IMPORT_PATH"].insert(index, precompile_path)
end end
self.Precompile("#{precompile_path}/#{module_name.gsub(".", "/")}.di", source) pctarget = "#{precompile_path}/#{module_name.gsub(".", "/")}.di"
self.Precompile(pctarget, source)
self.depends(barrier_target, pctarget)
end end
barrier_target barrier_target
end end
@ -775,7 +781,7 @@ module Rscons
# If no builder was found to run yet and there are threads available, try # If no builder was found to run yet and there are threads available, try
# to get a runnable builder from the builder set. # to get a runnable builder from the builder set.
targets_still_building = @threads.reduce([]) do |result, (thread, obj)| targets_still_building = @threads.reduce([]) do |result, (thread, obj)|
result << builder_for_thread(thread).target result << Util.absolute_path(builder_for_thread(thread).target)
end end
if @builder_sets.size > 0 if @builder_sets.size > 0
if builder = @builder_sets[0].get_next_builder_to_run(targets_still_building) if builder = @builder_sets[0].get_next_builder_to_run(targets_still_building)

17
lib/rscons/util.rb
View File

@ -16,6 +16,23 @@ module Rscons
end end
end end
# Get the absolute path for a given path.
#
# If the path is a phony target, leave it as is.
#
# @param path [String, Symbol]
# Input path.
#
# @return [String, Symbol]
# Output path.
def absolute_path(path)
if Rscons.phony_target?(path)
path
else
File.expand_path(path)
end
end
# Remove any stale .di files from the precompile path. # Remove any stale .di files from the precompile path.
# #
# @param pc_path [String] # @param pc_path [String]

44
spec/build_tests_spec.rb
View File

@ -671,9 +671,9 @@ EOF
result = run_rscons result = run_rscons
expect(result.stderr).to eq "" expect(result.stderr).to eq ""
slines = lines(result.stdout) slines = lines(result.stdout)
verify_lines(slines, [%r{gdc -c -o build/o/main.d.o -MMD -MF build/o/main.d.o.mf main.d}]) verify_lines(slines, [%r{gdc -c -o build/o/src/main.d.o .* src/main.d}])
verify_lines(slines, [%r{gdc -c -o build/o/mod.d.o -MMD -MF build/o/mod.d.o.mf mod.d}]) verify_lines(slines, [%r{gdc -c -o build/o/src/mod.d.o .* src/mod.d}])
verify_lines(slines, [%r{gdc -o hello-d.exe build/o/main.d.o build/o/mod.d.o}]) verify_lines(slines, [%r{gdc -o hello-d.exe build/o/src/main.d.o build/o/src/mod.d.o}])
expect(`./hello-d.exe`.rstrip).to eq "Hello from D, value is 42!" expect(`./hello-d.exe`.rstrip).to eq "Hello from D, value is 42!"
end end
end end
@ -683,9 +683,9 @@ EOF
result = run_rscons(args: %w[-f build-ldc2.rb]) result = run_rscons(args: %w[-f build-ldc2.rb])
expect(result.stderr).to eq "" expect(result.stderr).to eq ""
slines = lines(result.stdout) slines = lines(result.stdout)
verify_lines(slines, [%r{ldc2 -c -of build/o/main.d.o(bj)? -deps=build/o/main.d.o(bj)?.mf main.d}]) verify_lines(slines, [%r{ldc2 -c -of build/o/src/main.d.o(bj)? -deps=build/o/src/main.d.o(bj)?.mf.* src/main.d}])
verify_lines(slines, [%r{ldc2 -c -of build/o/mod.d.o(bj)? -deps=build/o/mod.d.o(bj)?.mf mod.d}]) verify_lines(slines, [%r{ldc2 -c -of build/o/src/mod.d.o(bj)? -deps=build/o/src/mod.d.o(bj)?.mf.* src/mod.d}])
verify_lines(slines, [%r{ldc2 -of hello-d.exe build/o/main.d.o(bj)? build/o/mod.d.o(bj)?}]) verify_lines(slines, [%r{ldc2 -of hello-d.exe build/o/src/main.d.o(bj)? build/o/src/mod.d.o(bj)?}])
expect(`./hello-d.exe`.rstrip).to eq "Hello from D, value is 42!" expect(`./hello-d.exe`.rstrip).to eq "Hello from D, value is 42!"
end end
@ -694,20 +694,20 @@ EOF
result = run_rscons(args: %w[-f build-ldc2.rb]) result = run_rscons(args: %w[-f build-ldc2.rb])
expect(result.stderr).to eq "" expect(result.stderr).to eq ""
slines = lines(result.stdout) slines = lines(result.stdout)
verify_lines(slines, [%r{ldc2 -c -of build/o/main.d.o(bj)? -deps=build/o/main.d.o(bj)?.mf main.d}]) verify_lines(slines, [%r{ldc2 -c -of build/o/src/main.d.o(bj)? -deps=build/o/src/main.d.o(bj)?.mf.* src/main.d}])
verify_lines(slines, [%r{ldc2 -c -of build/o/mod.d.o(bj)? -deps=build/o/mod.d.o(bj)?.mf mod.d}]) verify_lines(slines, [%r{ldc2 -c -of build/o/src/mod.d.o(bj)? -deps=build/o/src/mod.d.o(bj)?.mf.* src/mod.d}])
verify_lines(slines, [%r{ldc2 -of hello-d.exe build/o/main.d.o(bj)? build/o/mod.d.o(bj)?}]) verify_lines(slines, [%r{ldc2 -of hello-d.exe build/o/src/main.d.o(bj)? build/o/src/mod.d.o(bj)?}])
expect(`./hello-d.exe`.rstrip).to eq "Hello from D, value is 42!" expect(`./hello-d.exe`.rstrip).to eq "Hello from D, value is 42!"
contents = File.read("mod.d", mode: "rb").sub("42", "33") contents = File.read("src/mod.d", mode: "rb").sub("42", "33")
File.open("mod.d", "wb") do |fh| File.open("src/mod.d", "wb") do |fh|
fh.write(contents) fh.write(contents)
end end
result = run_rscons(args: %w[-f build-ldc2.rb]) result = run_rscons(args: %w[-f build-ldc2.rb])
expect(result.stderr).to eq "" expect(result.stderr).to eq ""
slines = lines(result.stdout) slines = lines(result.stdout)
verify_lines(slines, [%r{ldc2 -c -of build/o/main.d.o(bj)? -deps=build/o/main.d.o(bj)?.mf main.d}]) verify_lines(slines, [%r{ldc2 -c -of build/o/src/main.d.o(bj)? -deps=build/o/src/main.d.o(bj)?.mf.* src/main.d}])
verify_lines(slines, [%r{ldc2 -c -of build/o/mod.d.o(bj)? -deps=build/o/mod.d.o(bj)?.mf mod.d}]) verify_lines(slines, [%r{ldc2 -c -of build/o/src/mod.d.o(bj)? -deps=build/o/src/mod.d.o(bj)?.mf.* src/mod.d}])
verify_lines(slines, [%r{ldc2 -of hello-d.exe build/o/main.d.o(bj)? build/o/mod.d.o(bj)?}]) verify_lines(slines, [%r{ldc2 -of hello-d.exe build/o/src/main.d.o(bj)? build/o/src/mod.d.o(bj)?}])
expect(`./hello-d.exe`.rstrip).to eq "Hello from D, value is 33!" expect(`./hello-d.exe`.rstrip).to eq "Hello from D, value is 33!"
end end
@ -726,18 +726,18 @@ EOF
result = run_rscons result = run_rscons
expect(result.stderr).to eq "" expect(result.stderr).to eq ""
slines = lines(result.stdout) slines = lines(result.stdout)
verify_lines(slines, [%r{gdc -c -o build/o/main.d.o -MMD -MF build/o/main.d.o.mf main.d}]) verify_lines(slines, [%r{gdc -c -o build/o/src/main.d.o -MMD -MF build/o/src/main.d.o.mf.* src/main.d}])
verify_lines(slines, [%r{gdc -c -o build/o/mod.d.o -MMD -MF build/o/mod.d.o.mf mod.d}]) verify_lines(slines, [%r{gdc -c -o build/o/src/mod.d.o -MMD -MF build/o/src/mod.d.o.mf.* src/mod.d}])
verify_lines(slines, [%r{gdc -o hello-d.exe build/o/main.d.o build/o/mod.d.o}]) verify_lines(slines, [%r{gdc -o hello-d.exe build/o/src/main.d.o build/o/src/mod.d.o}])
expect(`./hello-d.exe`.rstrip).to eq "Hello from D, value is 42!" expect(`./hello-d.exe`.rstrip).to eq "Hello from D, value is 42!"
fcontents = File.read("mod.d", mode: "rb").sub("42", "33") fcontents = File.read("src/mod.d", mode: "rb").sub("42", "33")
File.open("mod.d", "wb") {|fh| fh.write(fcontents)} File.open("src/mod.d", "wb") {|fh| fh.write(fcontents)}
result = run_rscons result = run_rscons
expect(result.stderr).to eq "" expect(result.stderr).to eq ""
slines = lines(result.stdout) slines = lines(result.stdout)
verify_lines(slines, [%r{gdc -c -o build/o/main.d.o -MMD -MF build/o/main.d.o.mf main.d}]) verify_lines(slines, [%r{gdc -c -o build/o/src/main.d.o -MMD -MF build/o/src/main.d.o.mf.* src/main.d}])
verify_lines(slines, [%r{gdc -c -o build/o/mod.d.o -MMD -MF build/o/mod.d.o.mf mod.d}]) verify_lines(slines, [%r{gdc -c -o build/o/src/mod.d.o -MMD -MF build/o/src/mod.d.o.mf.* src/mod.d}])
verify_lines(slines, [%r{gdc -o hello-d.exe build/o/main.d.o build/o/mod.d.o}]) verify_lines(slines, [%r{gdc -o hello-d.exe build/o/src/main.d.o build/o/src/mod.d.o}])
expect(`./hello-d.exe`.rstrip).to eq "Hello from D, value is 33!" expect(`./hello-d.exe`.rstrip).to eq "Hello from D, value is 33!"
end end