Class: CurlDownloadStrategy

Inherits:
AbstractFileDownloadStrategy show all
Includes:
Utils::Curl
Defined in:
download_strategy/curl_download_strategy.rb

Overview

Strategy for downloading files using curl.

Constant Summary collapse

URLMetadata =

This constant is part of a private API. This constant may only be used in the Homebrew/brew repository. Third parties should avoid using this constant if possible, as it may be removed or changed without warning.

url, basename, time, file_size, content_type, is_redirection

T.type_alias { [String, String, T.nilable(Time), T.nilable(Integer), T.nilable(String), T::Boolean] }

Constants inherited from AbstractDownloadStrategy

AbstractDownloadStrategy::HOMEBREW_CONTROLLED_STRATEGIES

Instance Attribute Summary collapse

Attributes inherited from AbstractDownloadStrategy

#cache, #url

Instance Method Summary collapse

Methods included from Utils::Curl

clear_path_cache, curl, curl_args, curl_check_http_content, curl_download, curl_executable, curl_headers, curl_http_content_headers_and_checksum, curl_output, curl_path, curl_response_follow_redirections, curl_response_last_location, curl_supports_fail_with_body?, curl_supports_tls13?, curl_version, curl_with_workarounds, http_status_ok?, https_redirect_curl_args, insecure_redirect?, no_insecure_redirect_curl_args, parse_curl_output, parse_curl_response, strip_progress_bar, url_protected_by_cloudflare?, url_protected_by_incapsula?

Methods included from SystemCommand::Mixin

#command_sandbox, #system_command, #system_command!

Methods included from Utils::Output::Mixin

#issue_reporting_message, #odebug, #odeprecated, #odie, #odisabled, #ofail, #oh1, #oh1_title, #ohai, #ohai_title, #onoe, #opoo, #opoo_once, #opoo_outside_github_actions, #opoo_without_github_actions_annotation, #pretty_cannot_install, #pretty_deprecated, #pretty_disabled, #pretty_duration, #pretty_install_status, #pretty_installed, #pretty_uninstalled, #pretty_unmarked, #pretty_upgradable, #pretty_warning

Methods inherited from AbstractFileDownloadStrategy

#basename, #cached_location, #create_symlink_to_cached_download, #fetched_size, #parse_basename, #symlink_location, #temporary_path

Methods inherited from AbstractDownloadStrategy

#basename, #cached_location, #chdir, expand_deferred_environment_for?, #fetched_size, #ohai, #quiet!, #quiet?, #source_modified_time, #source_revision, #stage

Methods included from Context

current, current=, #debug?, #deferred_environment_expansion?, #quiet?, #verbose?, #with_context

Constructor Details

#initialize(url, name, version, **meta) ⇒ void

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

Parameters:



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'download_strategy/curl_download_strategy.rb', line 19

def initialize(url, name, version, **meta)
  @try_partial = T.let(true, T::Boolean)
  @expand_deferred_environment = T.let(false, T::Boolean)
  @mirrors = T.let(meta.fetch(:mirrors, []), T::Array[String])
  @file_size = T.let(nil, T.nilable(Integer))
  @last_modified = T.let(nil, T.nilable(Time))
  @fetch_end_time = T.let(nil, T.nilable(Time))

  # Merge `:header` with `:headers`.
  if (header = meta.delete(:header))
    meta[:headers] ||= []
    meta[:headers] << header
  end

  super
end

Instance Attribute Details

#mirrors ⇒ Array<String> (readonly)

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

Returns:



16
17
18
 
# File 'download_strategy/curl_download_strategy.rb', line 16

def mirrors
  @mirrors
end

Instance Method Details

#_curl_args ⇒ Array<String>

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

Curl options to be always passed to curl, with raw head calls (curl --head) or with actual fetch.

Returns:



181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'download_strategy/curl_download_strategy.rb', line 181

def _curl_args
  args = []

  args += ["-b", meta.fetch(:cookies).map { |k, v| "#{k}=#{v}" }.join(";")] if meta.key?(:cookies)

  args += ["-e", meta.fetch(:referer)] if meta.key?(:referer)

  args += ["--user", meta.fetch(:user)] if meta.key?(:user)

  if meta.fetch(:headers, []).any? { |header| header.include?(EnvSensitive::DEFERRED_PLACEHOLDER_PREFIX) }
    args += ["--max-redirs", "0"]
  end

  args += expand_deferred_environment_args(meta.fetch(:headers, [])).flat_map { |h| ["--header", h.strip] }

  args
end

#allow_deferred_environment_expansion! ⇒ void

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

This method returns an undefined value.



174
175
176
 
# File 'download_strategy/curl_download_strategy.rb', line 174

def allow_deferred_environment_expansion!
  @expand_deferred_environment = true
end

#clear_cache ⇒ void

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

This method returns an undefined value.



160
161
162
163
 
# File 'download_strategy/curl_download_strategy.rb', line 160

def clear_cache
  super
  rm_rf(temporary_path)
end

#fetch(timeout: nil) ⇒ void

This method returns an undefined value.

Download and cache the file at AbstractFileDownloadStrategy#cached_location.

Parameters:

  • timeout (Float, Integer, nil) (defaults to: nil) 


40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'download_strategy/curl_download_strategy.rb', line 40

def fetch(timeout: nil)
  @fetch_end_time = Time.now + timeout if timeout

  download_lock = DownloadLock.new(temporary_path)
  begin
    download_lock.lock_or_wait(quiet: quiet?, timeout: Utils::Timer.remaining(@fetch_end_time))

    urls = [url, *mirrors]

    if (domain = Homebrew::EnvConfig.artifact_domain)
      domain = domain.chomp("/")
      # If the artifact domain already contains the Docker Registry API's
      # `/v2/` path (e.g. an OCI registry proxying ghcr.io under a repository
      # prefix: https://mirror.example.com/v2/ghcr-io), skip the `v2/` from
      # the original URL rather than producing a duplicate `/v2/`.
      # Keep this in sync with the portable-ruby URL handling in
      # Library/Homebrew/cmd/vendor-install.sh.
      domain_contains_v2 = %r{\Ahttps?://[^/]+/v2(?:/|\z)}.match?(domain)

      artifact_urls = urls.map do |u|
        if domain_contains_v2
          u.sub(%r{^https?://#{GitHubPackages::URL_DOMAIN}/v2/}o, "#{domain}/")
        else
          u.sub(%r{^https?://#{GitHubPackages::URL_DOMAIN}/}o, "#{domain}/")
        end
      end

      urls = if Homebrew::EnvConfig.artifact_domain_no_fallback?
        artifact_urls
      else
        # Interleave: try artifact domain first, then original for each URL that was rewritten.
        combined = []
        artifact_urls.zip(urls).each do |artifact_url, original_url|
          combined << artifact_url
          combined << original_url if original_url != artifact_url
        end
        combined
      end
    end

    begin
      url = urls.shift
      raise "No URLs left to download #{name} from" if url.nil?

      ohai "Downloading #{url}"

      cached_location_valid = cached_location.exist?

      resolved_url, _, last_modified, @file_size, content_type, is_redirection = begin
        resolve_url_basename_time_file_size(url, timeout: Utils::Timer.remaining!(@fetch_end_time))
      rescue ErrorDuringExecution
        raise unless cached_location_valid
      end
      @last_modified = last_modified

      # Caller-supplied headers (e.g. tokens) are no longer valid after a
      # redirect to another host, and `Authorization` after any redirect.
      if is_redirection
        if resolved_url && URI(url).host != URI(resolved_url).host
          meta.delete(:headers)
        else
          meta[:headers]&.delete_if { |header| header.start_with?("Authorization") }
        end
      end

      # The cached location is no longer fresh if either:
      # - Last-Modified value is newer than the file's timestamp
      # - Content-Length value is different than the file's size
      if cached_location_valid && (!content_type.is_a?(String) || !content_type.start_with?("text/"))
        if last_modified && last_modified > cached_location.mtime
          ohai "Ignoring #{cached_location}",
               "Cached modified time #{cached_location.mtime.iso8601} is before " \
               "Last-Modified header: #{last_modified.iso8601}"
          cached_location_valid = false
        end
        if @file_size&.nonzero? && @file_size != cached_location.size
          ohai "Ignoring #{cached_location}",
               "Cached size #{cached_location.size} differs from " \
               "Content-Length header: #{@file_size}"
          cached_location_valid = false
        end
      end

      if cached_location_valid
        puts "Already downloaded: #{cached_location}"
      else
        raise "Could not resolve #{url}" if resolved_url.nil?

        begin
          _fetch(url:, resolved_url:, timeout: Utils::Timer.remaining!(@fetch_end_time))
        rescue ErrorDuringExecution => e
          clean_stderr = strip_progress_bar(Tty.collapse_carriage_returns(e.stderr)).strip
          raise CurlDownloadStrategyError.new(url, clean_stderr)
        end
        cached_location.dirname.mkpath
        temporary_path.rename(cached_location.to_s)
      end

      create_symlink_to_cached_download(cached_location)
    rescue CurlDownloadStrategyError
      raise if urls.empty?

      puts "Trying a mirror..."
      retry
    end
  ensure
    download_lock.unlock(unlink: true)
  end
rescue Timeout::Error => e
  raise Timeout::Error, "Timed out downloading #{self.url}: #{e}"
ensure
  @fetch_end_time = nil
end

#resolved_time_file_size(timeout: nil) ⇒ Array<([Time, nil], Integer)>

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

Parameters:

  • timeout (Float, Integer, nil) (defaults to: nil)

Returns:



166
167
168
169
170
171
 
# File 'download_strategy/curl_download_strategy.rb', line 166

def resolved_time_file_size(timeout: nil)
  _, _, time, file_size, = resolve_url_basename_time_file_size(url, timeout:)
  raise "Could not determine the file size of #{url}" if file_size.nil?

  [time, file_size]
end

#total_size ⇒ Integer?

This method is part of a private API. This method may only be used in the Homebrew/brew repository. Third parties should avoid using this method if possible, as it may be removed or changed without warning.

Returns:



155
156
157
 
# File 'download_strategy/curl_download_strategy.rb', line 155

def total_size
  @file_size
end