Mercurial > core / rust/lib/dl/src/lib.rs
changeset 78: |
966f92770ddf |
parent: |
97c99e44a22f
|
author: |
ellis <ellis@rwest.io> |
date: |
Sun, 03 Dec 2023 23:25:08 -0500 |
permissions: |
-rw-r--r-- |
description: |
lisp groveling and rust fmt |
1 //! Easy file downloading 2 #![deny(rust_2018_idioms)] 4 use anyhow::{Context, Result}; 8 pub use crate::errors::*; 10 /// User agent header value for HTTP request. 11 const USER_AGENT: &str = concat!("cc-install/", env!("CARGO_PKG_VERSION")); 13 #[derive(Debug, Copy, Clone)] 19 #[derive(Debug, Copy, Clone)] 25 #[derive(Debug, Copy, Clone)] 27 ResumingPartialDownload, 28 /// Received the Content-Length of the to-be downloaded data. 29 DownloadContentLengthReceived(u64), 30 /// Received some data. 31 DownloadDataReceived(&'a [u8]), 34 fn download_with_backend( 38 callback: &dyn Fn(Event<'_>) -> Result<()>, 41 Backend::Curl => curl::download(url, resume_from, callback), 42 Backend::Reqwest(tls) => { 43 reqwest_be::download(url, resume_from, callback, tls) 48 type DownloadCallback<'a> = &'a dyn Fn(Event<'_>) -> Result<()>; 50 pub fn download_to_path_with_backend( 54 resume_from_partial: bool, 55 callback: Option<DownloadCallback<'_>>, 59 fs::{remove_file, OpenOptions}, 60 io::{Read, Seek, SeekFrom, Write}, 64 let (file, resume_from) = if resume_from_partial { 65 let possible_partial = OpenOptions::new().read(true).open(path); 67 let downloaded_so_far = if let Ok(mut partial) = possible_partial { 68 if let Some(cb) = callback { 69 cb(Event::ResumingPartialDownload)?; 71 let mut buf = vec![0; 32768]; 72 let mut downloaded_so_far = 0; 74 let n = partial.read(&mut buf)?; 75 downloaded_so_far += n as u64; 79 cb(Event::DownloadDataReceived(&buf[..n]))?; 84 let file_info = partial.metadata()?; 91 let mut possible_partial = OpenOptions::new() 95 .context("error opening file for download")?; 97 possible_partial.seek(SeekFrom::End(0))?; 99 (possible_partial, downloaded_so_far) 106 .context("error creating file for download")?, 111 let file = RefCell::new(file); 113 download_with_backend(backend, url, resume_from, &|event| { 114 if let Event::DownloadDataReceived(data) = event { 118 .context("unable to write download to disk")?; 121 Some(cb) => cb(event), 129 .context("unable to sync download to disk")?; 134 // TODO: We currently clear up the cached download on any error, should we 135 // restrict it to a subset? 136 if let Err(file_err) = 137 remove_file(path).context("cleaning up cached downloads") 146 #[cfg(all(not(feature = "reqwest-backend"), not(feature = "curl-backend")))] 147 compile_error!("Must enable at least one backend"); 149 /// Download via libcurl; encrypt with the native (or OpenSSl) TLS 150 /// stack via libcurl 151 #[cfg(feature = "curl-backend")] 153 use std::{cell::RefCell, str, time::Duration}; 155 use anyhow::{Context, Result}; 156 use curl::easy::Easy; 160 use crate::errors::*; 165 callback: &dyn Fn(Event<'_>) -> Result<()>, 167 // Fetch either a cached libcurl handle (which will preserve open 168 // connections) or create a new one if it isn't listed. 170 // Once we've acquired it, reset the lifetime from 'static to our local 172 thread_local!(static EASY: RefCell<Easy> = RefCell::new(Easy::new())); 174 let mut handle = handle.borrow_mut(); 176 handle.url(url.as_ref())?; 177 handle.follow_location(true)?; 178 handle.useragent(super::USER_AGENT)?; 181 handle.resume_from(resume_from)?; 183 // an error here indicates that the range header isn't supported by 184 // underlying curl, so there's nothing to "clear" - safe to 185 // ignore this error. 186 let _ = handle.resume_from(0); 189 // Take at most 30s to connect 190 handle.connect_timeout(Duration::new(30, 0))?; 193 let cberr = RefCell::new(None); 194 let mut transfer = handle.transfer(); 196 // Data callback for libcurl which is called with data that's 197 // downloaded. We just feed it into our hasher and also write it out 199 transfer.write_function(|data| { 200 match callback(Event::DownloadDataReceived(data)) { 201 Ok(()) => Ok(data.len()), 203 *cberr.borrow_mut() = Some(e); 209 // Listen for headers and parse out a `Content-Length` 210 // (case-insensitive) if it comes so we know how much we're 212 transfer.header_function(|header| { 213 if let Ok(data) = str::from_utf8(header) { 214 let prefix = "content-length: "; 215 if data.to_ascii_lowercase().starts_with(prefix) { 216 if let Ok(s) = data[prefix.len()..].trim().parse::<u64>() { 217 let msg = Event::DownloadContentLengthReceived(s + resume_from); 218 match callback(msg) { 221 *cberr.borrow_mut() = Some(e); 231 // If an error happens check to see if we had a filesystem error up 232 // in `cberr`, but we always want to punt it up. 233 transfer.perform().or_else(|e| { 234 // If the original error was generated by one of our 235 // callbacks, return it. 236 match cberr.borrow_mut().take() { 237 Some(cberr) => Err(cberr), 239 // Otherwise, return the error from curl 240 if e.is_file_couldnt_read_file() { 241 Err(e).context(DownloadError::FileNotFound) 243 Err(e).context("error during download")? 250 // If we didn't get a 20x or 0 ("OK" for files) then return an error 251 let code = handle.response_code()?; 255 return Err(DownloadError::HttpStatus(code).into()); 264 #[cfg(feature = "reqwest-backend")] 267 not(feature = "reqwest-rustls-tls"), 268 not(feature = "reqwest-default-tls") 270 compile_error!("Must select a reqwest TLS backend"); 272 use std::{io, time::Duration}; 274 use anyhow::{anyhow, Context, Result}; 276 feature = "reqwest-rustls-tls", 277 feature = "reqwest-default-tls" 279 use once_cell::sync::Lazy; 281 blocking::{Client, ClientBuilder, Response}, 286 use super::{Event, TlsBackend}; 287 use crate::errors::*; 292 callback: &dyn Fn(Event<'_>) -> Result<()>, 295 // Short-circuit reqwest for the "file:" URL scheme 296 if download_from_file_url(url, resume_from, callback)? { 300 let mut res = request(url, resume_from, tls) 301 .context("failed to make network request")?; 303 if !res.status().is_success() { 304 let code: u16 = res.status().into(); 305 return Err(anyhow!(DownloadError::HttpStatus(u32::from(code)))); 308 let buffer_size = 0x10000; 309 let mut buffer = vec![0u8; buffer_size]; 311 if let Some(len) = res.headers().get(header::CONTENT_LENGTH) { 312 // TODO possible issues during unwrap? 313 let len = len.to_str().unwrap().parse::<u64>().unwrap() + resume_from; 314 callback(Event::DownloadContentLengthReceived(len))?; 318 let bytes_read = io::Read::read(&mut res, &mut buffer)?; 321 callback(Event::DownloadDataReceived(&buffer[0..bytes_read]))?; 328 fn client_generic() -> ClientBuilder { 331 .user_agent(super::USER_AGENT) 332 .proxy(Proxy::custom(env_proxy)) 333 .timeout(Duration::from_secs(30)) 336 #[cfg(feature = "reqwest-rustls-tls")] 337 static CLIENT_RUSTLS_TLS: Lazy<Client> = Lazy::new(|| { 338 let catcher = || client_generic().use_rustls_tls().build(); 341 // It's OK. This is the same as what is happening in curl. 343 // The curl::Easy::new() internally assert!s that the initialized 344 // Easy is not null. Inside reqwest, the errors here would be from 345 // the TLS library returning a null pointer as well. 349 #[cfg(feature = "reqwest-default-tls")] 350 static CLIENT_DEFAULT_TLS: Lazy<Client> = Lazy::new(|| { 351 let catcher = || client_generic().build(); 354 // It's OK. This is the same as what is happening in curl. 356 // The curl::Easy::new() internally assert!s that the initialized 357 // Easy is not null. Inside reqwest, the errors here would be from 358 // the TLS library returning a null pointer as well. 362 fn env_proxy(url: &Url) -> Option<Url> { 363 env_proxy::for_url(url).to_url() 370 ) -> Result<Response, DownloadError> { 371 let client: &Client = match backend { 372 #[cfg(feature = "reqwest-rustls-tls")] 373 TlsBackend::Rustls => &CLIENT_RUSTLS_TLS, 374 #[cfg(not(feature = "reqwest-rustls-tls"))] 375 TlsBackend::Rustls => { 376 return Err(DownloadError::BackendUnavailable("reqwest rustls")); 378 #[cfg(feature = "reqwest-default-tls")] 379 TlsBackend::Default => &CLIENT_DEFAULT_TLS, 380 #[cfg(not(feature = "reqwest-default-tls"))] 381 TlsBackend::Default => { 382 return Err(DownloadError::BackendUnavailable("reqwest default TLS")); 385 let mut req = client.get(url.as_str()); 387 if resume_from != 0 { 388 req = req.header(header::RANGE, format!("bytes={resume_from}-")); 394 fn download_from_file_url( 397 callback: &dyn Fn(Event<'_>) -> Result<()>, 401 // The file scheme is mostly for use by tests to mock the dist server 402 if url.scheme() == "file" { 403 let src = url.to_file_path().map_err(|_| { 404 DownloadError::Message(format!("bogus file url: '{url}'")) 407 // Because some of rustup's logic depends on checking 408 // the error when a downloaded file doesn't exist, make 409 // the file case return the same error value as the 411 return Err(anyhow!(DownloadError::FileNotFound)); 415 fs::File::open(src).context("unable to open downloaded file")?; 416 io::Seek::seek(&mut f, io::SeekFrom::Start(resume_from))?; 418 let mut buffer = vec![0u8; 0x10000]; 420 let bytes_read = io::Read::read(&mut f, &mut buffer)?; 424 callback(Event::DownloadDataReceived(&buffer[0..bytes_read]))?; 434 #[cfg(not(feature = "curl-backend"))] 437 use anyhow::{anyhow, Result}; 440 use crate::errors::*; 446 _callback: &dyn Fn(Event<'_>) -> Result<()>, 448 Err(anyhow!(DownloadError::BackendUnavailable("curl"))) 452 #[cfg(not(feature = "reqwest-backend"))] 455 use anyhow::{anyhow, Result}; 457 use super::{Event, TlsBackend}; 458 use crate::errors::*; 464 _callback: &dyn Fn(Event<'_>) -> Result<()>, 467 Err(anyhow!(DownloadError::BackendUnavailable("reqwest")))