From c61380b3549967fd485ca29e971143e456c930f5 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Mon, 14 Apr 2014 16:52:21 -0400 Subject: [PATCH] Issue #7 - change .rsconscache from using YAML to JSON to speed up parsing --- lib/rscons/cache.rb | 131 +++++++++++++++++++------------------- spec/rscons/cache_spec.rb | 120 +++++++++++++++++----------------- 2 files changed, 126 insertions(+), 125 deletions(-) diff --git a/lib/rscons/cache.rb b/lib/rscons/cache.rb index c0a003b..d9d3e0b 100644 --- a/lib/rscons/cache.rb +++ b/lib/rscons/cache.rb @@ -1,59 +1,59 @@ -require 'yaml' -require 'fileutils' -require 'digest/md5' -require 'set' -require 'rscons/version' +require "digest/md5" +require "fileutils" +require "json" +require "set" +require "rscons/version" module Rscons # The Cache class keeps track of file checksums, build target commands and - # dependencies in a YAML file which persists from one invocation to the next. + # dependencies in a JSON file which persists from one invocation to the next. # Example cache: # { - # version: '1.2.3', - # targets: { - # 'program' => { - # 'checksum' => 'A1B2C3D4', - # 'command' => ['gcc', '-o', 'program', 'program.o'], - # 'deps' => [ + # "version" => "1.2.3", + # "targets" => { + # "program" => { + # "checksum" => "A1B2C3D4", + # "command" => ["gcc", "-o", "program", "program.o"], + # "deps" => [ # { - # 'fname' => 'program.o', - # 'checksum' => '87654321', + # "fname" => "program.o", + # "checksum" => "87654321", # }, # ], - # 'user_deps' => [ + # "user_deps" => [ # { - # 'fname' => 'lscript.ld', - # 'checksum' => '77551133', + # "fname" => "lscript.ld", + # "checksum" => "77551133", # }, # ], # }, - # 'program.o' => { - # 'checksum' => '87654321', - # 'command' => ['gcc', '-c', '-o', 'program.o', 'program.c'], - # 'deps' => [ + # "program.o" => { + # "checksum" => "87654321", + # "command" => ["gcc", "-c", "-o", "program.o", "program.c"], + # "deps" => [ # { - # 'fname' => 'program.c', - # 'checksum' => '456789ABC', + # "fname" => "program.c", + # "checksum" => "456789ABC", # }, # { - # 'fname' => 'program.h', - # 'checksum' => '7979764643', + # "fname" => "program.h", + # "checksum" => "7979764643", # }, # ], - # 'user_deps' => [], + # "user_deps" => [], # } # }, - # directories: { - # 'build' => true, - # 'build/one' => true, - # 'build/two' => true, + # "directories" => { + # "build" => true, + # "build/one" => true, + # "build/two" => true, # }, # } class Cache #### Constants # Name of the file to store cache information in - CACHE_FILE = '.rsconscache' + CACHE_FILE = ".rsconscache" #### Class Methods @@ -67,21 +67,21 @@ module Rscons # Create a Cache object and load in the previous contents from the cache # file. def initialize - @cache = YAML.load(File.read(CACHE_FILE)) rescue {} + @cache = JSON.load(File.read(CACHE_FILE)) rescue {} unless @cache.is_a?(Hash) $stderr.puts "Warning: #{CACHE_FILE} was corrupt. Contents:\n#{@cache.inspect}" @cache = {} end - @cache[:targets] ||= {} - @cache[:directories] ||= {} + @cache["targets"] ||= {} + @cache["directories"] ||= {} @lookup_checksums = {} end # Write the cache to disk to be loaded next time. def write - @cache[:version] = VERSION - File.open(CACHE_FILE, 'w') do |fh| - fh.puts(YAML.dump(@cache)) + @cache["version"] = VERSION + File.open(CACHE_FILE, "w") do |fh| + fh.puts(JSON.dump(@cache)) end end @@ -90,11 +90,12 @@ module Rscons # @param command [String, Array] The command used to build the target. # @param deps [Array] List of the target's dependency files. # @param env [Environment] The Rscons::Environment. - # @param options [Hash] Optional options. Can contain the following keys: - # :strict_deps:: - # Only consider a target up to date if its list of dependencies is - # exactly equal (including order) to the cached list of dependencies - # @return true if the targets are all up to date, meaning that, + # @param options [Hash] Optional options. + # @option options [Boolean] :strict_deps + # Only consider a target up to date if its list of dependencies is + # exactly equal (including order) to the cached list of dependencies + # @return [Boolean] + # True value if the targets are all up to date, meaning that, # for each target: # - the target exists on disk # - the cache has information for the target @@ -111,16 +112,16 @@ module Rscons return false unless File.exists?(target) # target must be registered in the cache - return false unless @cache[:targets].has_key?(target) + return false unless @cache["targets"].has_key?(target) # target must have the same checksum as when it was built last - return false unless @cache[:targets][target][:checksum] == lookup_checksum(target) + return false unless @cache["targets"][target]["checksum"] == lookup_checksum(target) # command used to build target must be identical - return false unless @cache[:targets][target][:command] == command + return false unless @cache["targets"][target]["command"] == command - cached_deps = @cache[:targets][target][:deps] || [] - cached_deps_fnames = cached_deps.map { |dc| dc[:fname] } + cached_deps = @cache["targets"][target]["deps"] || [] + cached_deps_fnames = cached_deps.map { |dc| dc["fname"] } if options[:strict_deps] # depedencies passed in must exactly equal those in the cache return false unless deps == cached_deps_fnames @@ -131,13 +132,13 @@ module Rscons # set of user dependencies must match user_deps = env.get_user_deps(target) || [] - cached_user_deps = @cache[:targets][target][:user_deps] || [] - cached_user_deps_fnames = cached_user_deps.map { |dc| dc[:fname] } + cached_user_deps = @cache["targets"][target]["user_deps"] || [] + cached_user_deps_fnames = cached_user_deps.map { |dc| dc["fname"] } return false unless user_deps == cached_user_deps_fnames # all cached dependencies must have their checksums match (cached_deps + cached_user_deps).each do |dep_cache| - return false unless dep_cache[:checksum] == lookup_checksum(dep_cache[:fname]) + return false unless dep_cache["checksum"] == lookup_checksum(dep_cache["fname"]) end end @@ -148,22 +149,22 @@ module Rscons # @param targets [String, Array] The name of the target(s) built. # @param command [String, Array] The command used to build the target. # @param deps [Array] List of dependencies for the target. - # @param env [Environment] The Rscons::Environment. + # @param env [Environment] The {Rscons::Environment}. def register_build(targets, command, deps, env) Array(targets).each do |target| - @cache[:targets][target.encode(__ENCODING__)] = { - command: command, - checksum: calculate_checksum(target), - deps: deps.map do |dep| + @cache["targets"][target.encode(__ENCODING__)] = { + "command" => command, + "checksum" => calculate_checksum(target), + "deps" => deps.map do |dep| { - fname: dep.encode(__ENCODING__), - checksum: lookup_checksum(dep), + "fname" => dep.encode(__ENCODING__), + "checksum" => lookup_checksum(dep), } end, - user_deps: (env.get_user_deps(target) || []).map do |dep| + "user_deps" => (env.get_user_deps(target) || []).map do |dep| { - fname: dep.encode(__ENCODING__), - checksum: lookup_checksum(dep), + "fname" => dep.encode(__ENCODING__), + "checksum" => lookup_checksum(dep), } end, } @@ -172,25 +173,25 @@ module Rscons # Return a list of targets that have been built def targets - @cache[:targets].keys + @cache["targets"].keys end # Make any needed directories and record the ones that are created for # removal upon a "clean" operation. def mkdir_p(path) parts = path.split(/[\\\/]/) - (0..parts.size-1).each do |i| + parts.each_index do |i| subpath = File.join(*parts[0, i + 1]).encode(__ENCODING__) unless File.exists?(subpath) FileUtils.mkdir(subpath) - @cache[:directories][subpath] = true + @cache["directories"][subpath] = true end end end # Return a list of directories which were created as a part of the build def directories - @cache[:directories].keys + @cache["directories"].keys end # Private Instance Methods @@ -206,7 +207,7 @@ module Rscons # Calculate and return a file's checksum # @param file [String] The file name. def calculate_checksum(file) - @lookup_checksums[file] = Digest::MD5.hexdigest(File.read(file, mode: 'rb')).encode(__ENCODING__) rescue '' + @lookup_checksums[file] = Digest::MD5.hexdigest(File.read(file, mode: "rb")).encode(__ENCODING__) rescue "" end end end diff --git a/spec/rscons/cache_spec.rb b/spec/rscons/cache_spec.rb index 2e50b61..6a48d1f 100644 --- a/spec/rscons/cache_spec.rb +++ b/spec/rscons/cache_spec.rb @@ -5,7 +5,7 @@ module Rscons end def build_from(cache) - YAML.should_receive(:load).and_return(cache) + JSON.should_receive(:load).and_return(cache) Cache.new end @@ -19,7 +19,7 @@ module Rscons describe "#initialize" do context "when corrupt" do it "prints a warning and defaults to an empty hash" do - YAML.should_receive(:load).and_return("string") + JSON.should_receive(:load).and_return("string") $stderr.should_receive(:puts).with(/Warning:.*was.corrupt/) Cache.new.instance_variable_get(:@cache).is_a?(Hash).should be_true end @@ -27,13 +27,13 @@ module Rscons end describe "#write" do - it "should fill in :version and write to file" do + it "should fill in 'version' and write to file" do cache = {} fh = $stdout fh.should_receive(:puts) File.should_receive(:open).and_yield(fh) build_from(cache).write - cache[:version].should == Rscons::VERSION + cache["version"].should == Rscons::VERSION end end @@ -54,7 +54,7 @@ module Rscons end it "returns false when the target's checksum does not match" do - _cache = {targets: {"target" => {checksum: "abc"}}} + _cache = {"targets" => {"target" => {"checksum" => "abc"}}} cache = build_from(_cache) File.should_receive(:exists?).with("target").and_return(true) cache.should_receive(:calculate_checksum).with("target").and_return("def") @@ -62,7 +62,7 @@ module Rscons end it "returns false when the build command has changed" do - _cache = {targets: {"target" => {checksum: "abc", command: "old command"}}} + _cache = {"targets" => {"target" => {"checksum" => "abc", "command" => "old command"}}} cache = build_from(_cache) File.should_receive(:exists?).with("target").and_return(true) cache.should_receive(:calculate_checksum).with("target").and_return("abc") @@ -70,9 +70,9 @@ module Rscons end it "returns false when there is a new dependency" do - _cache = {targets: {"target" => {checksum: "abc", - command: "command", - deps: [{fname: "dep.1"}]}}} + _cache = {"targets" => {"target" => {"checksum" => "abc", + "command" => "command", + "deps" => [{"fname" => "dep.1"}]}}} cache = build_from(_cache) File.should_receive(:exists?).with("target").and_return(true) cache.should_receive(:calculate_checksum).with("target").and_return("abc") @@ -80,15 +80,15 @@ module Rscons end it "returns false when a dependency's checksum has changed" do - _cache = {targets: {"target" => {checksum: "abc", - command: "command", - deps: [{fname: "dep.1", - checksum: "dep.1.chk"}, - {fname: "dep.2", - checksum: "dep.2.chk"}, - {fname: "extra.dep", - checksum: "extra.dep.chk"}], - user_deps: []}}} + _cache = {"targets" => {"target" => {"checksum" => "abc", + "command" => "command", + "deps" => [{"fname" => "dep.1", + "checksum" => "dep.1.chk"}, + {"fname" => "dep.2", + "checksum" => "dep.2.chk"}, + {"fname" => "extra.dep", + "checksum" => "extra.dep.chk"}], + "user_deps" => []}}} cache = build_from(_cache) File.should_receive(:exists?).with("target").and_return(true) cache.should_receive(:calculate_checksum).with("target").and_return("abc") @@ -98,15 +98,15 @@ module Rscons end it "returns false with strict_deps=true when cache has an extra dependency" do - _cache = {targets: {"target" => {checksum: "abc", - command: "command", - deps: [{fname: "dep.1", - checksum: "dep.1.chk"}, - {fname: "dep.2", - checksum: "dep.2.chk"}, - {fname: "extra.dep", - checksum: "extra.dep.chk"}], - user_deps: []}}} + _cache = {"targets" => {"target" => {"checksum" => "abc", + "command" => "command", + "deps" => [{"fname" => "dep.1", + "checksum" => "dep.1.chk"}, + {"fname" => "dep.2", + "checksum" => "dep.2.chk"}, + {"fname" => "extra.dep", + "checksum" => "extra.dep.chk"}], + "user_deps" => []}}} cache = build_from(_cache) File.should_receive(:exists?).with("target").and_return(true) cache.should_receive(:calculate_checksum).with("target").and_return("abc") @@ -114,10 +114,10 @@ module Rscons end it "returns false when there is a new user dependency" do - _cache = {targets: {"target" => {checksum: "abc", - command: "command", - deps: [{fname: "dep.1"}], - user_deps: []}}} + _cache = {"targets" => {"target" => {"checksum" => "abc", + "command" => "command", + "deps" => [{"fname" => "dep.1"}], + "user_deps" => []}}} cache = build_from(_cache) env = "env" env.should_receive(:get_user_deps).with("target").and_return(["file.ld"]) @@ -127,16 +127,16 @@ module Rscons end it "returns false when a user dependency checksum has changed" do - _cache = {targets: {"target" => {checksum: "abc", - command: "command", - deps: [{fname: "dep.1", - checksum: "dep.1.chk"}, - {fname: "dep.2", - checksum: "dep.2.chk"}, - {fname: "extra.dep", - checksum: "extra.dep.chk"}], - user_deps: [{fname: "user.dep", - checksum: "user.dep.chk"}]}}} + _cache = {"targets" => {"target" => {"checksum" => "abc", + "command" => "command", + "deps" => [{"fname" => "dep.1", + "checksum" => "dep.1.chk"}, + {"fname" => "dep.2", + "checksum" => "dep.2.chk"}, + {"fname" => "extra.dep", + "checksum" => "extra.dep.chk"}], + "user_deps" => [{"fname" => "user.dep", + "checksum" => "user.dep.chk"}]}}} cache = build_from(_cache) env = "env" env.should_receive(:get_user_deps).with("target").and_return(["user.dep"]) @@ -150,15 +150,15 @@ module Rscons end it "returns true when no condition for false is met" do - _cache = {targets: {"target" => {checksum: "abc", - command: "command", - deps: [{fname: "dep.1", - checksum: "dep.1.chk"}, - {fname: "dep.2", - checksum: "dep.2.chk"}, - {fname: "extra.dep", - checksum: "extra.dep.chk"}], - user_deps: []}}} + _cache = {"targets" => {"target" => {"checksum" => "abc", + "command" => "command", + "deps" => [{"fname" => "dep.1", + "checksum" => "dep.1.chk"}, + {"fname" => "dep.2", + "checksum" => "dep.2.chk"}, + {"fname" => "extra.dep", + "checksum" => "extra.dep.chk"}], + "user_deps" => []}}} cache = build_from(_cache) File.should_receive(:exists?).with("target").and_return(true) cache.should_receive(:calculate_checksum).with("target").and_return("abc") @@ -180,23 +180,23 @@ module Rscons cache.should_receive(:calculate_checksum).with("dep 2").and_return("dep 2 checksum") cache.should_receive(:calculate_checksum).with("user.dep").and_return("user.dep checksum") cache.register_build("the target", "the command", ["dep 1", "dep 2"], env) - cached_target = cache.instance_variable_get(:@cache)[:targets]["the target"] + cached_target = cache.instance_variable_get(:@cache)["targets"]["the target"] cached_target.should_not be_nil - cached_target[:command].should == "the command" - cached_target[:checksum].should == "the checksum" - cached_target[:deps].should == [ - {fname: "dep 1", checksum: "dep 1 checksum"}, - {fname: "dep 2", checksum: "dep 2 checksum"}, + cached_target["command"].should == "the command" + cached_target["checksum"].should == "the checksum" + cached_target["deps"].should == [ + {"fname" => "dep 1", "checksum" => "dep 1 checksum"}, + {"fname" => "dep 2", "checksum" => "dep 2 checksum"}, ] - cached_target[:user_deps].should == [ - {fname: "user.dep", checksum: "user.dep checksum"}, + cached_target["user_deps"].should == [ + {"fname" => "user.dep", "checksum" => "user.dep checksum"}, ] end end describe "#targets" do it "returns a list of targets that are cached" do - cache = {targets: {"t1" => {}, "t2" => {}, "t3" => {}}} + cache = {"targets" => {"t1" => {}, "t2" => {}, "t3" => {}}} build_from(cache).targets.should == ["t1", "t2", "t3"] end end @@ -222,7 +222,7 @@ module Rscons describe "#directories" do it "returns a list of directories that are cached" do - _cache = {directories: {"dir1" => true, "dir2" => true}} + _cache = {"directories" => {"dir1" => true, "dir2" => true}} build_from(_cache).directories.should == ["dir1", "dir2"] end end