| Class | Gem::RemoteFetcher |
| In: |
lib/rubygems/remote_fetcher.rb
|
| Parent: | Object |
RemoteFetcher handles the details of fetching gems and gem information from a remote source.
| BuiltinSSLCerts | = | File.expand_path("./ssl_certs/*.pem", File.dirname(__FILE__)) |
Cached RemoteFetcher instance.
# File lib/rubygems/remote_fetcher.rb, line 42
42: def self.fetcher
43: @fetcher ||= self.new Gem.configuration[:http_proxy]
44: end
Initialize a remote fetcher using the source URI and possible proxy information.
proxy
variable setting
HTTP_PROXY_PASS)
# File lib/rubygems/remote_fetcher.rb, line 57
57: def initialize(proxy = nil)
58: require 'net/http'
59: require 'stringio'
60: require 'time'
61: require 'uri'
62:
63: Socket.do_not_reverse_lookup = true
64:
65: @connections = {}
66: @requests = Hash.new 0
67: @proxy_uri =
68: case proxy
69: when :no_proxy then nil
70: when nil then get_proxy_from_env
71: when URI::HTTP then proxy
72: else URI.parse(proxy)
73: end
74: @user_agent = user_agent
75: end
# File lib/rubygems/remote_fetcher.rb, line 367
367: def add_rubygems_trusted_certs(store)
368: Dir.glob(BuiltinSSLCerts).each do |ssl_cert_file|
369: store.add_file ssl_cert_file
370: end
371: end
# File lib/rubygems/remote_fetcher.rb, line 344
344: def configure_connection_for_https(connection)
345: require 'net/https'
346:
347: connection.use_ssl = true
348: connection.verify_mode =
349: Gem.configuration.ssl_verify_mode || OpenSSL::SSL::VERIFY_PEER
350:
351: store = OpenSSL::X509::Store.new
352:
353: if Gem.configuration.ssl_ca_cert
354: if File.directory? Gem.configuration.ssl_ca_cert
355: store.add_path Gem.configuration.ssl_ca_cert
356: else
357: store.add_file Gem.configuration.ssl_ca_cert
358: end
359: else
360: store.set_default_paths
361: add_rubygems_trusted_certs(store)
362: end
363:
364: connection.cert_store = store
365: end
Creates or an HTTP connection based on uri, or retrieves an existing connection, using a proxy if needed.
# File lib/rubygems/remote_fetcher.rb, line 306
306: def connection_for(uri)
307: net_http_args = [uri.host, uri.port]
308:
309: if @proxy_uri then
310: net_http_args += [
311: @proxy_uri.host,
312: @proxy_uri.port,
313: @proxy_uri.user,
314: @proxy_uri.password
315: ]
316: end
317:
318: connection_id = [Thread.current.object_id, *net_http_args].join ':'
319: @connections[connection_id] ||= Net::HTTP.new(*net_http_args)
320: connection = @connections[connection_id]
321:
322: if https?(uri) and !connection.started? then
323: configure_connection_for_https(connection)
324:
325: # Don't refactor this with the else branch. We don't want the
326: # http-only code path to not depend on anything in OpenSSL.
327: #
328: begin
329: connection.start
330: rescue OpenSSL::SSL::SSLError, Errno::EHOSTDOWN => e
331: raise FetchError.new(e.message, uri)
332: end
333: else
334: begin
335: connection.start unless connection.started?
336: rescue Errno::EHOSTDOWN => e
337: raise FetchError.new(e.message, uri)
338: end
339: end
340:
341: connection
342: end
# File lib/rubygems/remote_fetcher.rb, line 373
373: def correct_for_windows_path(path)
374: if path[0].chr == '/' && path[1].chr =~ /[a-z]/i && path[2].chr == ':'
375: path = path[1..-1]
376: else
377: path
378: end
379: end
Moves the gem spec from source_uri to the cache dir unless it is already there. If the source_uri is local the gem cache dir copy is always replaced.
# File lib/rubygems/remote_fetcher.rb, line 100
100: def download(spec, source_uri, install_dir = Gem.dir)
101: Gem.ensure_gem_subdirectories(install_dir) rescue nil
102:
103: if File.writable?(install_dir)
104: cache_dir = File.join install_dir, "cache"
105: else
106: cache_dir = File.join Gem.user_dir, "cache"
107: end
108:
109: gem_file_name = File.basename spec.cache_file
110: local_gem_path = File.join cache_dir, gem_file_name
111:
112: FileUtils.mkdir_p cache_dir rescue nil unless File.exist? cache_dir
113:
114: # Always escape URI's to deal with potential spaces and such
115: unless URI::Generic === source_uri
116: source_uri = URI.parse(URI.const_defined?(:DEFAULT_PARSER) ?
117: URI::DEFAULT_PARSER.escape(source_uri.to_s) :
118: URI.escape(source_uri.to_s))
119: end
120:
121: scheme = source_uri.scheme
122:
123: # URI.parse gets confused by MS Windows paths with forward slashes.
124: scheme = nil if scheme =~ /^[a-z]$/i
125:
126: case scheme
127: when 'http', 'https' then
128: unless File.exist? local_gem_path then
129: begin
130: say "Downloading gem #{gem_file_name}" if
131: Gem.configuration.really_verbose
132:
133: remote_gem_path = source_uri + "gems/#{gem_file_name}"
134:
135: gem = self.fetch_path remote_gem_path
136: rescue Gem::RemoteFetcher::FetchError
137: raise if spec.original_platform == spec.platform
138:
139: alternate_name = "#{spec.original_name}.gem"
140:
141: say "Failed, downloading gem #{alternate_name}" if
142: Gem.configuration.really_verbose
143:
144: remote_gem_path = source_uri + "gems/#{alternate_name}"
145:
146: gem = self.fetch_path remote_gem_path
147: end
148:
149: File.open local_gem_path, 'wb' do |fp|
150: fp.write gem
151: end
152: end
153: when 'file' then
154: begin
155: path = source_uri.path
156: path = File.dirname(path) if File.extname(path) == '.gem'
157:
158: remote_gem_path = correct_for_windows_path(File.join(path, 'gems', gem_file_name))
159:
160: FileUtils.cp(remote_gem_path, local_gem_path)
161: rescue Errno::EACCES
162: local_gem_path = source_uri.to_s
163: end
164:
165: say "Using local gem #{local_gem_path}" if
166: Gem.configuration.really_verbose
167: when nil then # TODO test for local overriding cache
168: source_path = if Gem.win_platform? && source_uri.scheme &&
169: !source_uri.path.include?(':') then
170: "#{source_uri.scheme}:#{source_uri.path}"
171: else
172: source_uri.path
173: end
174:
175: source_path = unescape source_path
176:
177: begin
178: FileUtils.cp source_path, local_gem_path unless
179: File.identical?(source_path, local_gem_path)
180: rescue Errno::EACCES
181: local_gem_path = source_uri.to_s
182: end
183:
184: say "Using local gem #{local_gem_path}" if
185: Gem.configuration.really_verbose
186: else
187: raise Gem::InstallError, "unsupported URI scheme #{source_uri.scheme}"
188: end
189:
190: local_gem_path
191: end
Given a name and requirement, downloads this gem into cache and returns the filename. Returns nil if the gem cannot be located.
# File lib/rubygems/remote_fetcher.rb, line 84
84: def download_to_cache dependency
85: found = Gem::SpecFetcher.fetcher.fetch dependency, true, true,
86: dependency.prerelease?
87:
88: return if found.empty?
89:
90: spec, source_uri = found.sort_by { |(s,_)| s.version }.last
91:
92: download spec, source_uri
93: end
# File lib/rubygems/remote_fetcher.rb, line 258
258: def escape(str)
259: return unless str
260: @uri_parser ||= uri_escaper
261: @uri_parser.escape str
262: end
File Fetcher. Dispatched by fetch_path. Use it instead.
# File lib/rubygems/remote_fetcher.rb, line 196
196: def fetch_file uri, *_
197: Gem.read_binary correct_for_windows_path uri.path
198: end
HTTP Fetcher. Dispatched by fetch_path. Use it instead.
# File lib/rubygems/remote_fetcher.rb, line 203
203: def fetch_http uri, last_modified = nil, head = false, depth = 0
204: fetch_type = head ? Net::HTTP::Head : Net::HTTP::Get
205: response = request uri, fetch_type, last_modified
206:
207: case response
208: when Net::HTTPOK, Net::HTTPNotModified then
209: head ? response : response.body
210: when Net::HTTPMovedPermanently, Net::HTTPFound, Net::HTTPSeeOther,
211: Net::HTTPTemporaryRedirect then
212: raise FetchError.new('too many redirects', uri) if depth > 10
213:
214: location = URI.parse response['Location']
215:
216: if https?(uri) && !https?(location)
217: raise FetchError.new("redirecting to non-https resource: #{location}", uri)
218: end
219:
220: fetch_http(location, last_modified, head, depth + 1)
221: else
222: raise FetchError.new("bad response #{response.message} #{response.code}", uri)
223: end
224: end
Downloads uri and returns it as a String.
# File lib/rubygems/remote_fetcher.rb, line 231
231: def fetch_path(uri, mtime = nil, head = false)
232: uri = URI.parse uri unless URI::Generic === uri
233:
234: raise ArgumentError, "bad uri: #{uri}" unless uri
235: raise ArgumentError, "uri scheme is invalid: #{uri.scheme.inspect}" unless
236: uri.scheme
237:
238: data = send "fetch_#{uri.scheme}", uri, mtime, head
239: data = Gem.gunzip data if data and not head and uri.to_s =~ /gz$/
240: data
241: rescue FetchError
242: raise
243: rescue Timeout::Error
244: raise FetchError.new('timed out', uri.to_s)
245: rescue IOError, SocketError, SystemCallError => e
246: raise FetchError.new("#{e.class}: #{e}", uri.to_s)
247: end
Returns the size of uri in bytes.
# File lib/rubygems/remote_fetcher.rb, line 252
252: def fetch_size(uri) # TODO: phase this out
253: response = fetch_path(uri, nil, true)
254:
255: response['content-length'].to_i
256: end
Returns an HTTP proxy URI if one is set in the environment variables.
# File lib/rubygems/remote_fetcher.rb, line 279
279: def get_proxy_from_env
280: env_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
281:
282: return nil if env_proxy.nil? or env_proxy.empty?
283:
284: uri = URI.parse(normalize_uri(env_proxy))
285:
286: if uri and uri.user.nil? and uri.password.nil? then
287: # Probably we have http_proxy_* variables?
288: uri.user = escape(ENV['http_proxy_user'] || ENV['HTTP_PROXY_USER'])
289: uri.password = escape(ENV['http_proxy_pass'] || ENV['HTTP_PROXY_PASS'])
290: end
291:
292: uri
293: end
# File lib/rubygems/remote_fetcher.rb, line 513
513: def https?(uri)
514: uri.scheme.downcase == 'https'
515: end
Normalize the URI by adding "http://" if it is missing.
# File lib/rubygems/remote_fetcher.rb, line 298
298: def normalize_uri(uri)
299: (uri =~ /^(https?|ftp|file):/) ? uri : "http://#{uri}"
300: end
Read the data from the (source based) URI, but if it is a file:// URI, read from the filesystem instead.
# File lib/rubygems/remote_fetcher.rb, line 385
385: def open_uri_or_path(uri, last_modified = nil, head = false, depth = 0)
386: raise "NO: Use fetch_path instead"
387: # TODO: deprecate for fetch_path
388: end
Performs a Net::HTTP request of type request_class on uri returning a Net::HTTP response object. request maintains a table of persistent connections to reduce connect overhead.
# File lib/rubygems/remote_fetcher.rb, line 395
395: def request(uri, request_class, last_modified = nil)
396: request = request_class.new uri.request_uri
397:
398: unless uri.nil? || uri.user.nil? || uri.user.empty? then
399: request.basic_auth uri.user, uri.password
400: end
401:
402: request.add_field 'User-Agent', @user_agent
403: request.add_field 'Connection', 'keep-alive'
404: request.add_field 'Keep-Alive', '30'
405:
406: if last_modified then
407: last_modified = last_modified.utc
408: request.add_field 'If-Modified-Since', last_modified.rfc2822
409: end
410:
411: yield request if block_given?
412:
413: connection = connection_for uri
414:
415: retried = false
416: bad_response = false
417:
418: begin
419: @requests[connection.object_id] += 1
420:
421: say "#{request.method} #{uri}" if
422: Gem.configuration.really_verbose
423:
424: file_name = File.basename(uri.path)
425: # perform download progress reporter only for gems
426: if request.response_body_permitted? && file_name =~ /\.gem$/
427: reporter = ui.download_reporter
428: response = connection.request(request) do |incomplete_response|
429: if Net::HTTPOK === incomplete_response
430: reporter.fetch(file_name, incomplete_response.content_length)
431: downloaded = 0
432: data = ''
433:
434: incomplete_response.read_body do |segment|
435: data << segment
436: downloaded += segment.length
437: reporter.update(downloaded)
438: end
439: reporter.done
440: if incomplete_response.respond_to? :body=
441: incomplete_response.body = data
442: else
443: incomplete_response.instance_variable_set(:@body, data)
444: end
445: end
446: end
447: else
448: response = connection.request request
449: end
450:
451: say "#{response.code} #{response.message}" if
452: Gem.configuration.really_verbose
453:
454: rescue Net::HTTPBadResponse
455: say "bad response" if Gem.configuration.really_verbose
456:
457: reset connection
458:
459: raise FetchError.new('too many bad responses', uri) if bad_response
460:
461: bad_response = true
462: retry
463: # HACK work around EOFError bug in Net::HTTP
464: # NOTE Errno::ECONNABORTED raised a lot on Windows, and make impossible
465: # to install gems.
466: rescue EOFError, Timeout::Error,
467: Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE
468:
469: requests = @requests[connection.object_id]
470: say "connection reset after #{requests} requests, retrying" if
471: Gem.configuration.really_verbose
472:
473: raise FetchError.new('too many connection resets', uri) if retried
474:
475: reset connection
476:
477: retried = true
478: retry
479: end
480:
481: response
482: end
Resets HTTP connection connection.
# File lib/rubygems/remote_fetcher.rb, line 487
487: def reset(connection)
488: @requests.delete connection.object_id
489:
490: connection.finish
491: connection.start
492: end
# File lib/rubygems/remote_fetcher.rb, line 264
264: def unescape(str)
265: return unless str
266: @uri_parser ||= uri_escaper
267: @uri_parser.unescape str
268: end
# File lib/rubygems/remote_fetcher.rb, line 270
270: def uri_escaper
271: URI::Parser.new
272: rescue NameError
273: URI
274: end
# File lib/rubygems/remote_fetcher.rb, line 494
494: def user_agent
495: ua = "RubyGems/#{Gem::VERSION} #{Gem::Platform.local}"
496:
497: ruby_version = RUBY_VERSION
498: ruby_version += 'dev' if RUBY_PATCHLEVEL == -1
499:
500: ua << " Ruby/#{ruby_version} (#{RUBY_RELEASE_DATE}"
501: if RUBY_PATCHLEVEL >= 0 then
502: ua << " patchlevel #{RUBY_PATCHLEVEL}"
503: elsif defined?(RUBY_REVISION) then
504: ua << " revision #{RUBY_REVISION}"
505: end
506: ua << ")"
507:
508: ua << " #{RUBY_ENGINE}" if defined?(RUBY_ENGINE) and RUBY_ENGINE != 'ruby'
509:
510: ua
511: end