josh/rscons
1
0 Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Issue #7 - change .rsconscache from using YAML to JSON to speed up parsing

Browse Source
This commit is contained in:
josh committed 2014-04-14 16:52:21 -04:00
1 parent d8dda291a5
commit c61380b354
2 files changed
+126 -125
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/rscons/cache.rb
+66 -65
View File
@@ -1,59 +1,59 @@
require 'yaml'
require 'fileutils'
require 'digest/md5'
require 'set'
require 'rscons/version'
require "digest/md5"
require "fileutils"
require "json"
require "set"
require "rscons/version"
module Rscons
# The Cache class keeps track of file checksums, build target commands and
# dependencies in a YAML file which persists from one invocation to the next.
# dependencies in a JSON file which persists from one invocation to the next.
# Example cache:
# {
# version: '1.2.3',
# targets: {
# 'program' => {
# 'checksum' => 'A1B2C3D4',
# 'command' => ['gcc', '-o', 'program', 'program.o'],
# 'deps' => [
# "version" => "1.2.3",
# "targets" => {
# "program" => {
# "checksum" => "A1B2C3D4",
# "command" => ["gcc", "-o", "program", "program.o"],
# "deps" => [
# {
# 'fname' => 'program.o',
# 'checksum' => '87654321',
# "fname" => "program.o",
# "checksum" => "87654321",
# },
# ],
# 'user_deps' => [
# "user_deps" => [
# {
# 'fname' => 'lscript.ld',
# 'checksum' => '77551133',
# "fname" => "lscript.ld",
# "checksum" => "77551133",
# },
# ],
# },
# 'program.o' => {
# 'checksum' => '87654321',
# 'command' => ['gcc', '-c', '-o', 'program.o', 'program.c'],
# 'deps' => [
# "program.o" => {
# "checksum" => "87654321",
# "command" => ["gcc", "-c", "-o", "program.o", "program.c"],
# "deps" => [
# {
# 'fname' => 'program.c',
# 'checksum' => '456789ABC',
# "fname" => "program.c",
# "checksum" => "456789ABC",
# },
# {
# 'fname' => 'program.h',
# 'checksum' => '7979764643',
# "fname" => "program.h",
# "checksum" => "7979764643",
# },
# ],
# 'user_deps' => [],
# "user_deps" => [],
# }
# },
# directories: {
# 'build' => true,
# 'build/one' => true,
# 'build/two' => true,
# "directories" => {
# "build" => true,
# "build/one" => true,
# "build/two" => true,
# },
# }
class Cache
#### Constants
# Name of the file to store cache information in
CACHE_FILE = '.rsconscache'
CACHE_FILE = ".rsconscache"
#### Class Methods
@@ -67,21 +67,21 @@ module Rscons
# Create a Cache object and load in the previous contents from the cache
# file.
def initialize
@cache = YAML.load(File.read(CACHE_FILE)) rescue {}
@cache = JSON.load(File.read(CACHE_FILE)) rescue {}
unless @cache.is_a?(Hash)
$stderr.puts "Warning: #{CACHE_FILE} was corrupt. Contents:\n#{@cache.inspect}"
@cache = {}
end
@cache[:targets] ||= {}
@cache[:directories] ||= {}
@cache["targets"] ||= {}
@cache["directories"] ||= {}
@lookup_checksums = {}
end
# Write the cache to disk to be loaded next time.
def write
@cache[:version] = VERSION
File.open(CACHE_FILE, 'w') do |fh|
fh.puts(YAML.dump(@cache))
@cache["version"] = VERSION
File.open(CACHE_FILE, "w") do |fh|
fh.puts(JSON.dump(@cache))
end
end
@@ -90,11 +90,12 @@ module Rscons
# @param command [String, Array] The command used to build the target.
# @param deps [Array] List of the target's dependency files.
# @param env [Environment] The Rscons::Environment.
# @param options [Hash] Optional options. Can contain the following keys:
# :strict_deps::
# Only consider a target up to date if its list of dependencies is
# exactly equal (including order) to the cached list of dependencies
# @return true if the targets are all up to date, meaning that,
# @param options [Hash] Optional options.
# @option options [Boolean] :strict_deps
# Only consider a target up to date if its list of dependencies is
# exactly equal (including order) to the cached list of dependencies
# @return [Boolean]
# True value if the targets are all up to date, meaning that,
# for each target:
# - the target exists on disk
# - the cache has information for the target
@@ -111,16 +112,16 @@ module Rscons
return false unless File.exists?(target)
# target must be registered in the cache
return false unless @cache[:targets].has_key?(target)
return false unless @cache["targets"].has_key?(target)
# target must have the same checksum as when it was built last
return false unless @cache[:targets][target][:checksum] == lookup_checksum(target)
return false unless @cache["targets"][target]["checksum"] == lookup_checksum(target)
# command used to build target must be identical
return false unless @cache[:targets][target][:command] == command
return false unless @cache["targets"][target]["command"] == command
cached_deps = @cache[:targets][target][:deps] || []
cached_deps_fnames = cached_deps.map { |dc| dc[:fname] }
cached_deps = @cache["targets"][target]["deps"] || []
cached_deps_fnames = cached_deps.map { |dc| dc["fname"] }
if options[:strict_deps]
# depedencies passed in must exactly equal those in the cache
return false unless deps == cached_deps_fnames
@@ -131,13 +132,13 @@ module Rscons
# set of user dependencies must match
user_deps = env.get_user_deps(target) || []
cached_user_deps = @cache[:targets][target][:user_deps] || []
cached_user_deps_fnames = cached_user_deps.map { |dc| dc[:fname] }
cached_user_deps = @cache["targets"][target]["user_deps"] || []
cached_user_deps_fnames = cached_user_deps.map { |dc| dc["fname"] }
return false unless user_deps == cached_user_deps_fnames
# all cached dependencies must have their checksums match
(cached_deps + cached_user_deps).each do |dep_cache|
return false unless dep_cache[:checksum] == lookup_checksum(dep_cache[:fname])
return false unless dep_cache["checksum"] == lookup_checksum(dep_cache["fname"])
end
end
@@ -148,22 +149,22 @@ module Rscons
# @param targets [String, Array] The name of the target(s) built.
# @param command [String, Array] The command used to build the target.
# @param deps [Array] List of dependencies for the target.
# @param env [Environment] The Rscons::Environment.
# @param env [Environment] The {Rscons::Environment}.
def register_build(targets, command, deps, env)
Array(targets).each do |target|
@cache[:targets][target.encode(__ENCODING__)] = {
command: command,
checksum: calculate_checksum(target),
deps: deps.map do |dep|
@cache["targets"][target.encode(__ENCODING__)] = {
"command" => command,
"checksum" => calculate_checksum(target),
"deps" => deps.map do |dep|
{
fname: dep.encode(__ENCODING__),
checksum: lookup_checksum(dep),
"fname" => dep.encode(__ENCODING__),
"checksum" => lookup_checksum(dep),
}
end,
user_deps: (env.get_user_deps(target) || []).map do |dep|
"user_deps" => (env.get_user_deps(target) || []).map do |dep|
{
fname: dep.encode(__ENCODING__),
checksum: lookup_checksum(dep),
"fname" => dep.encode(__ENCODING__),
"checksum" => lookup_checksum(dep),
}
end,
}
@@ -172,25 +173,25 @@ module Rscons
# Return a list of targets that have been built
def targets
@cache[:targets].keys
@cache["targets"].keys
end
# Make any needed directories and record the ones that are created for
# removal upon a "clean" operation.
def mkdir_p(path)
parts = path.split(/[\\\/]/)
(0..parts.size-1).each do |i|
parts.each_index do |i|
subpath = File.join(*parts[0, i + 1]).encode(__ENCODING__)
unless File.exists?(subpath)
FileUtils.mkdir(subpath)
@cache[:directories][subpath] = true
@cache["directories"][subpath] = true
end
end
end
# Return a list of directories which were created as a part of the build
def directories
@cache[:directories].keys
@cache["directories"].keys
end
# Private Instance Methods
@@ -206,7 +207,7 @@ module Rscons
# Calculate and return a file's checksum
# @param file [String] The file name.
def calculate_checksum(file)
@lookup_checksums[file] = Digest::MD5.hexdigest(File.read(file, mode: 'rb')).encode(__ENCODING__) rescue ''
@lookup_checksums[file] = Digest::MD5.hexdigest(File.read(file, mode: "rb")).encode(__ENCODING__) rescue ""
end
end
end