Class: CurlDownloadStrategy

Inherits:
AbstractFileDownloadStrategy show all
Includes:
Utils::Curl
Defined in:
download_strategy/curl_download_strategy.rb

Overview

Strategy for downloading files using curl.

Constant Summary collapse

URLMetadata =

This constant is part of a private API. This constant may only be used in the Homebrew/brew repository. Third parties should avoid using this constant if possible, as it may be removed or changed without warning.

url, basename, time, file_size, content_type, is_redirection

T.type_alias { [String, String, T.nilable(Time), T.nilable(Integer), T.nilable(String), T::Boolean] }

Constants inherited from AbstractDownloadStrategy

AbstractDownloadStrategy::HOMEBREW_CONTROLLED_STRATEGIES

Instance Attribute Summary collapse

Attributes inherited from AbstractDownloadStrategy

#cache, #url

Instance Method Summary collapse

Methods included from Utils::Curl

clear_path_cache, curl, curl_args, curl_check_http_content, curl_download, curl_executable, curl_headers, curl_http_content_headers_and_checksum, curl_output, curl_path, curl_response_follow_redirections, curl_response_last_location, curl_supports_fail_with_body?, curl_supports_tls13?, curl_version, curl_with_workarounds, http_status_ok?, https_redirect_curl_args, insecure_redirect?, no_insecure_redirect_curl_args, parse_curl_output, parse_curl_response, strip_progress_bar, url_protected_by_cloudflare?, url_protected_by_incapsula?

Methods included from SystemCommand::Mixin

#system_command, #system_command!

Methods included from Utils::Output::Mixin

#issue_reporting_message, #odebug, #odeprecated, #odie, #odisabled, #ofail, #oh1, #oh1_title, #ohai, #ohai_title, #onoe, #opoo, #opoo_outside_github_actions, #opoo_without_github_actions_annotation, #pretty_deprecated, #pretty_disabled, #pretty_duration, #pretty_install_status, #pretty_installed, #pretty_uninstalled, #pretty_unmarked, #pretty_upgradable, #pretty_warning

Methods inherited from AbstractFileDownloadStrategy

#basename, #cached_location, #create_symlink_to_cached_download, #fetched_size, #parse_basename, #symlink_location, #temporary_path

Methods inherited from AbstractDownloadStrategy

#basename, #cached_location, expand_deferred_environment_for?, #fetched_size, #ohai, #quiet!, #quiet?, #source_modified_time, #source_revision, #stage

Methods included from Context

current, current=, #debug?, #deferred_environment_expansion?, #quiet?, #verbose?, #with_context

Constructor Details

#initialize(url, name, version, **meta) ⇒ void

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

Parameters:



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'download_strategy/curl_download_strategy.rb', line 17

def initialize(url, name, version, **meta)
  @try_partial = T.let(true, T::Boolean)
  @expand_deferred_environment = T.let(false, T::Boolean)
  @mirrors = T.let(meta.fetch(:mirrors, []), T::Array[String])
  @file_size = T.let(nil, T.nilable(Integer))
  @last_modified = T.let(nil, T.nilable(Time))

  # Merge `:header` with `:headers`.
  if (header = meta.delete(:header))
    meta[:headers] ||= []
    meta[:headers] << header
  end

  super
end

Instance Attribute Details

#mirrors ⇒ Array<String> (readonly)

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

Returns:



14
15
16
 
# File 'download_strategy/curl_download_strategy.rb', line 14

def mirrors
  @mirrors
end

Instance Method Details

#_curl_args ⇒ Array<String>

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

Curl options to be always passed to curl, with raw head calls (curl --head) or with actual fetch.

Returns:



158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
 
# File 'download_strategy/curl_download_strategy.rb', line 158

def _curl_args
  args = []

  args += ["-b", meta.fetch(:cookies).map { |k, v| "#{k}=#{v}" }.join(";")] if meta.key?(:cookies)

  args += ["-e", meta.fetch(:referer)] if meta.key?(:referer)

  args += ["--user", meta.fetch(:user)] if meta.key?(:user)

  if meta.fetch(:headers, []).any? { |header| header.include?(EnvSensitive::DEFERRED_PLACEHOLDER_PREFIX) }
    args += ["--max-redirs", "0"]
  end

  args += expand_deferred_environment_args(meta.fetch(:headers, [])).flat_map { |h| ["--header", h.strip] }

  args
end

#allow_deferred_environment_expansion! ⇒ void

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

This method returns an undefined value.



151
152
153
 
# File 'download_strategy/curl_download_strategy.rb', line 151

def allow_deferred_environment_expansion!
  @expand_deferred_environment = true
end

#clear_cache ⇒ void

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

This method returns an undefined value.



139
140
141
142
 
# File 'download_strategy/curl_download_strategy.rb', line 139

def clear_cache
  super
  rm_rf(temporary_path)
end

#fetch(timeout: nil) ⇒ void

This method returns an undefined value.

Download and cache the file at AbstractFileDownloadStrategy#cached_location.

Parameters:

  • timeout (Float, Integer, nil) (defaults to: nil) 


37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'download_strategy/curl_download_strategy.rb', line 37

def fetch(timeout: nil)
  end_time = Time.now + timeout if timeout

  download_lock = DownloadLock.new(temporary_path)
  begin
    download_lock.lock_or_wait(quiet: quiet?, timeout: Utils::Timer.remaining(end_time))

    urls = [url, *mirrors]

    if (domain = Homebrew::EnvConfig.artifact_domain)
      artifact_urls = urls.map do |u|
        u.sub(%r{^https?://#{GitHubPackages::URL_DOMAIN}/}o, "#{domain.chomp("/")}/")
      end

      urls = if Homebrew::EnvConfig.artifact_domain_no_fallback?
        artifact_urls
      else
        # Interleave: try artifact domain first, then original for each URL that was rewritten.
        combined = []
        artifact_urls.zip(urls).each do |artifact_url, original_url|
          combined << artifact_url
          combined << original_url if original_url != artifact_url
        end
        combined
      end
    end

    begin
      url = T.must(urls.shift)

      ohai "Downloading #{url}"

      cached_location_valid = cached_location.exist?

      resolved_url, _, last_modified, @file_size, content_type, is_redirection = begin
        resolve_url_basename_time_file_size(url, timeout: Utils::Timer.remaining!(end_time))
      rescue ErrorDuringExecution
        raise unless cached_location_valid
      end
      @last_modified = last_modified

      # Caller-supplied headers (e.g. tokens) are no longer valid after a
      # redirect to another host, and `Authorization` after any redirect.
      if is_redirection
        if resolved_url && URI(url).host != URI(resolved_url).host
          meta.delete(:headers)
        else
          meta[:headers]&.delete_if { |header| header.start_with?("Authorization") }
        end
      end

      # The cached location is no longer fresh if either:
      # - Last-Modified value is newer than the file's timestamp
      # - Content-Length value is different than the file's size
      if cached_location_valid && (!content_type.is_a?(String) || !content_type.start_with?("text/"))
        if last_modified && last_modified > cached_location.mtime
          ohai "Ignoring #{cached_location}",
               "Cached modified time #{cached_location.mtime.iso8601} is before " \
               "Last-Modified header: #{last_modified.iso8601}"
          cached_location_valid = false
        end
        if @file_size&.nonzero? && @file_size != cached_location.size
          ohai "Ignoring #{cached_location}",
               "Cached size #{cached_location.size} differs from " \
               "Content-Length header: #{@file_size}"
          cached_location_valid = false
        end
      end

      if cached_location_valid
        puts "Already downloaded: #{cached_location}"
      else
        begin
          _fetch(url:, resolved_url: T.must(resolved_url), timeout: Utils::Timer.remaining!(end_time))
        rescue ErrorDuringExecution => e
          clean_stderr = strip_progress_bar(Tty.collapse_carriage_returns(e.stderr)).strip
          raise CurlDownloadStrategyError.new(url, clean_stderr)
        end
        cached_location.dirname.mkpath
        temporary_path.rename(cached_location.to_s)
      end

      create_symlink_to_cached_download(cached_location)
    rescue CurlDownloadStrategyError
      raise if urls.empty?

      puts "Trying a mirror..."
      retry
    rescue Timeout::Error => e
      raise Timeout::Error, "Timed out downloading #{self.url}: #{e}"
    end
  ensure
    download_lock.unlock(unlink: true)
  end
end

#resolved_time_file_size(timeout: nil) ⇒ Array<([Time, nil], Integer)>

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

Parameters:

  • timeout (Float, Integer, nil) (defaults to: nil)

Returns:



145
146
147
148
 
# File 'download_strategy/curl_download_strategy.rb', line 145

def resolved_time_file_size(timeout: nil)
  _, _, time, file_size, = resolve_url_basename_time_file_size(url, timeout:)
  [time, T.must(file_size)]
end

#total_size ⇒ Integer?

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

Returns:



134
135
136
 
# File 'download_strategy/curl_download_strategy.rb', line 134

def total_size
  @file_size
end