From 8b2387f7a30d7ab69549a9a2e0c036305175c958 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Sat, 5 Feb 2022 16:51:52 -0500 Subject: [PATCH] Add download script method - close #152 --- build_tests/typical/download.rb | 45 ++++++++++++++++++++++++++ lib/rscons.rb | 2 ++ lib/rscons/script.rb | 56 +++++++++++++++++++++++++++++++++ spec/build_tests_spec.rb | 47 +++++++++++++++++++++++++++ 4 files changed, 150 insertions(+) create mode 100644 build_tests/typical/download.rb diff --git a/build_tests/typical/download.rb b/build_tests/typical/download.rb new file mode 100644 index 0000000..69ff063 --- /dev/null +++ b/build_tests/typical/download.rb @@ -0,0 +1,45 @@ +default do + download "https://github.com/holtrop/rscons/releases/download/v2.3.0/rscons", + "rscons-2.3.0", + sha256sum: "27a6e0f65b446d0e862d357a3ecd2904ebdfb7a9d2c387f08fb687793ac8adf8" + mtime = File.stat("rscons-2.3.0").mtime + sleep 1 + download "https://github.com/holtrop/rscons/releases/download/v2.3.0/rscons", + "rscons-2.3.0", + sha256sum: "27a6e0f65b446d0e862d357a3ecd2904ebdfb7a9d2c387f08fb687793ac8adf8" + unless mtime == File.stat("rscons-2.3.0").mtime + raise "mtime changed" + end +end + +task "badchecksum" do + download "https://github.com/holtrop/rscons/releases/download/v2.3.0/rscons", + "rscons-2.3.0", + sha256sum: "badbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbad0" + puts "continued" +end + +task "nochecksum" do + File.binwrite("rscons-2.3.0", "hi") + download "https://github.com/holtrop/rscons/releases/download/v2.3.0/rscons", + "rscons-2.3.0" +end + +task "redirectlimit" do + download "http://github.com/holtrop/rscons/releases/download/v2.3.0/rscons", + "rscons-2.3.0", + sha256sum: "27a6e0f65b446d0e862d357a3ecd2904ebdfb7a9d2c387f08fb687793ac8adf8", + redirect_limit: 0 +end + +task "badurl" do + download "https://github.com/holtrop/rscons/releases/download/v2.3.0/rscons.nope", + "rscons-2.3.0", + sha256sum: "27a6e0f65b446d0e862d357a3ecd2904ebdfb7a9d2c387f08fb687793ac8adf8" +end + +task "badhost" do + download "http://ksfjliasjlaskdmflaskfmalisfjsd.com/foo", + "foo", + sha256sum: "27a6e0f65b446d0e862d357a3ecd2904ebdfb7a9d2c387f08fb687793ac8adf8" +end diff --git a/lib/rscons.rb b/lib/rscons.rb index bfd14a1..28ea715 100644 --- a/lib/rscons.rb +++ b/lib/rscons.rb @@ -1,3 +1,5 @@ +require "net/http" +require "digest" require_relative "rscons/ansi" require_relative "rscons/application" require_relative "rscons/basic_environment" diff --git a/lib/rscons/script.rb b/lib/rscons/script.rb index 799a264..fc68cda 100644 --- a/lib/rscons/script.rb +++ b/lib/rscons/script.rb @@ -37,6 +37,62 @@ module Rscons end.sort end + # Download a file. + # + # @param url [String] + # URL. + # @param dest [String] + # Path to where to save the file. + # @param options [Hash] + # Options. + # @option options [String] :sha256sum + # Expected file checksum. + # @option options [Integer] :redirect_limit + # Maximum number of times to allow HTTP redirection (default 5). + # + # @return [void] + def download(url, dest, options = {}) + options[:redirect_limit] ||= 5 + unless options[:redirected] + if File.exist?(dest) && options[:sha256sum] + if Digest::SHA2.hexdigest(File.binread(dest)) == options[:sha256sum] + # Destination file already exists and has the expected checksum. + return + end + end + end + uri = URI(url) + use_ssl = url.start_with?("https://") + response = nil + socketerror_message = "" + digest = Digest::SHA2.new + begin + Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http| + File.open(dest, "wb") do |fh| + response = http.get(uri.request_uri) do |data| + fh.write(data) + digest << data + end + end + end + rescue SocketError => e + raise RsconsError.new("Error downloading #{dest}: #{e.message}") + end + if response.is_a?(Net::HTTPRedirection) + if options[:redirect_limit] == 0 + raise RsconsError.new("Redirect limit reached when downloading #{dest}") + else + return download(response["location"], dest, options.merge(redirect_limit: options[:redirect_limit] - 1, redirected: true)) + end + end + unless response.is_a?(Net::HTTPSuccess) + raise RsconsError.new("Error downloading #{dest}") + end + if options[:sha256sum] && options[:sha256sum] != digest.hexdigest + raise RsconsError.new("Unexpected checksum on #{dest}") + end + end + # Construct a task parameter. # # @param name [String] diff --git a/spec/build_tests_spec.rb b/spec/build_tests_spec.rb index 6aa82ed..b453fd5 100644 --- a/spec/build_tests_spec.rb +++ b/spec/build_tests_spec.rb @@ -2922,4 +2922,51 @@ EOF expect(result.stdout).to_not match /^\s*two\b/ end + context "download script method" do + it "downloads the specified file unless it already exists with the expected checksum" do + test_dir "typical" + result = run_rscons(args: %w[-f download.rb]) + expect(result.stderr).to eq "" + expect(result.status).to eq 0 + expect(File.exist?("rscons-2.3.0")).to be_truthy + end + + it "downloads the specified file if no checksum is given" do + test_dir "typical" + result = run_rscons(args: %w[-f download.rb nochecksum]) + expect(result.stderr).to eq "" + expect(result.status).to eq 0 + expect(File.exist?("rscons-2.3.0")).to be_truthy + expect(File.binread("rscons-2.3.0").size).to be > 100 + end + + it "exits with an error if the downloaded file checksum does not match the given checksum" do + test_dir "typical" + result = run_rscons(args: %w[-f download.rb badchecksum]) + expect(result.stderr).to match /Unexpected checksum on rscons-2.3.0/ + expect(result.status).to_not eq 0 + end + + it "exits with an error if the redirect limit is reached" do + test_dir "typical" + result = run_rscons(args: %w[-f download.rb redirectlimit]) + expect(result.stderr).to match /Redirect limit reached when downloading rscons-2.3.0/ + expect(result.status).to_not eq 0 + end + + it "exits with an error if the download results in an error" do + test_dir "typical" + result = run_rscons(args: %w[-f download.rb badurl]) + expect(result.stderr).to match /Error downloading rscons-2.3.0/ + expect(result.status).to_not eq 0 + end + + it "exits with an error if the download results in a socket error" do + test_dir "typical" + result = run_rscons(args: %w[-f download.rb badhost]) + expect(result.stderr).to match /Error downloading foo: .*ksfjlias/ + expect(result.status).to_not eq 0 + end + end + end