Mercurial > core / rust/lib/dl/src/lib.rs

 //! Easy file downloading
 #![deny(rust_2018_idioms)]
 
 use anyhow::{Context, Result};
 use std::path::Path;
 use url::Url;
 mod errors;
 pub use crate::errors::*;
 
 /// User agent header value for HTTP request.
 const USER_AGENT: &str = concat!("cc-install/", env!("CARGO_PKG_VERSION"));
 
 #[derive(Debug, Copy, Clone)]
 pub enum Backend {
   Curl,
   Reqwest(TlsBackend),
 }
 
 #[derive(Debug, Copy, Clone)]
 pub enum TlsBackend {
   Rustls,
   Default,
 }
 
 #[derive(Debug, Copy, Clone)]
 pub enum Event<'a> {
   ResumingPartialDownload,
   /// Received the Content-Length of the to-be downloaded data.
   DownloadContentLengthReceived(u64),
   /// Received some data.
   DownloadDataReceived(&'a [u8]),
 }
 
 fn download_with_backend(
   backend: Backend,
   url: &Url,
   resume_from: u64,
   callback: &dyn Fn(Event<'_>) -> Result<()>,
 ) -> Result<()> {
   match backend {
     Backend::Curl => curl::download(url, resume_from, callback),
     Backend::Reqwest(tls) => {
       reqwest_be::download(url, resume_from, callback, tls)
     }
   }
 }
 
 type DownloadCallback<'a> = &'a dyn Fn(Event<'_>) -> Result<()>;
 
 pub fn download_to_path_with_backend(
   backend: Backend,
   url: &Url,
   path: &Path,
   resume_from_partial: bool,
   callback: Option<DownloadCallback<'_>>,
 ) -> Result<()> {
   use std::{
     cell::RefCell,
     fs::{remove_file, OpenOptions},
     io::{Read, Seek, SeekFrom, Write},
   };
 
   || -> Result<()> {
     let (file, resume_from) = if resume_from_partial {
       let possible_partial = OpenOptions::new().read(true).open(path);
 
       let downloaded_so_far = if let Ok(mut partial) = possible_partial {
         if let Some(cb) = callback {
           cb(Event::ResumingPartialDownload)?;
 
           let mut buf = vec![0; 32768];
           let mut downloaded_so_far = 0;
           loop {
             let n = partial.read(&mut buf)?;
             downloaded_so_far += n as u64;
             if n == 0 {
               break;
             }
             cb(Event::DownloadDataReceived(&buf[..n]))?;
           }
 
           downloaded_so_far
         } else {
           let file_info = partial.metadata()?;
           file_info.len()
         }
       } else {
         0
       };
 
       let mut possible_partial = OpenOptions::new()
         .write(true)
         .create(true)
         .open(path)
         .context("error opening file for download")?;
 
       possible_partial.seek(SeekFrom::End(0))?;
 
       (possible_partial, downloaded_so_far)
     } else {
       (
         OpenOptions::new()
           .write(true)
           .create(true)
           .open(path)
           .context("error creating file for download")?,
         0,
       )
     };
 
     let file = RefCell::new(file);
 
     download_with_backend(backend, url, resume_from, &|event| {
       if let Event::DownloadDataReceived(data) = event {
         file
           .borrow_mut()
           .write_all(data)
           .context("unable to write download to disk")?;
       }
       match callback {
         Some(cb) => cb(event),
         None => Ok(()),
       }
     })?;
 
     file
       .borrow_mut()
       .sync_data()
       .context("unable to sync download to disk")?;
 
     Ok(())
   }()
   .map_err(|e| {
     // TODO: We currently clear up the cached download on any error, should we
     // restrict it to a subset?
     if let Err(file_err) =
       remove_file(path).context("cleaning up cached downloads")
     {
       file_err.context(e)
     } else {
       e
     }
   })
 }
 
 #[cfg(all(not(feature = "reqwest-backend"), not(feature = "curl-backend")))]
 compile_error!("Must enable at least one backend");
 
 /// Download via libcurl; encrypt with the native (or OpenSSl) TLS
 /// stack via libcurl
 #[cfg(feature = "curl-backend")]
 pub mod curl {
   use std::{cell::RefCell, str, time::Duration};
 
   use anyhow::{Context, Result};
   use curl::easy::Easy;
   use url::Url;
 
   use super::Event;
   use crate::errors::*;
 
   pub fn download(
     url: &Url,
     resume_from: u64,
     callback: &dyn Fn(Event<'_>) -> Result<()>,
   ) -> Result<()> {
     // Fetch either a cached libcurl handle (which will preserve open
     // connections) or create a new one if it isn't listed.
     //
     // Once we've acquired it, reset the lifetime from 'static to our local
     // scope.
     thread_local!(static EASY: RefCell<Easy> = RefCell::new(Easy::new()));
     EASY.with(|handle| {
       let mut handle = handle.borrow_mut();
 
       handle.url(url.as_ref())?;
       handle.follow_location(true)?;
       handle.useragent(super::USER_AGENT)?;
 
       if resume_from > 0 {
         handle.resume_from(resume_from)?;
       } else {
         // an error here indicates that the range header isn't supported by
         // underlying curl, so there's nothing to "clear" - safe to
         // ignore this error.
         let _ = handle.resume_from(0);
       }
 
       // Take at most 30s to connect
       handle.connect_timeout(Duration::new(30, 0))?;
 
       {
         let cberr = RefCell::new(None);
         let mut transfer = handle.transfer();
 
         // Data callback for libcurl which is called with data that's
         // downloaded. We just feed it into our hasher and also write it out
         // to disk.
         transfer.write_function(|data| {
           match callback(Event::DownloadDataReceived(data)) {
             Ok(()) => Ok(data.len()),
             Err(e) => {
               *cberr.borrow_mut() = Some(e);
               Ok(0)
             }
           }
         })?;
 
         // Listen for headers and parse out a `Content-Length`
         // (case-insensitive) if it comes so we know how much we're
         // downloading.
         transfer.header_function(|header| {
           if let Ok(data) = str::from_utf8(header) {
             let prefix = "content-length: ";
             if data.to_ascii_lowercase().starts_with(prefix) {
               if let Ok(s) = data[prefix.len()..].trim().parse::<u64>() {
                 let msg = Event::DownloadContentLengthReceived(s + resume_from);
                 match callback(msg) {
                   Ok(()) => (),
                   Err(e) => {
                     *cberr.borrow_mut() = Some(e);
                     return false;
                   }
                 }
               }
             }
           }
           true
         })?;
 
         // If an error happens check to see if we had a filesystem error up
         // in `cberr`, but we always want to punt it up.
         transfer.perform().or_else(|e| {
           // If the original error was generated by one of our
           // callbacks, return it.
           match cberr.borrow_mut().take() {
             Some(cberr) => Err(cberr),
             None => {
               // Otherwise, return the error from curl
               if e.is_file_couldnt_read_file() {
                 Err(e).context(DownloadError::FileNotFound)
               } else {
                 Err(e).context("error during download")?
               }
             }
           }
         })?;
       }
 
       // If we didn't get a 20x or 0 ("OK" for files) then return an error
       let code = handle.response_code()?;
       match code {
         0 | 200..=299 => {}
         _ => {
           return Err(DownloadError::HttpStatus(code).into());
         }
       };
 
       Ok(())
     })
   }
 }
 
 #[cfg(feature = "reqwest-backend")]
 pub mod reqwest_be {
   #[cfg(all(
     not(feature = "reqwest-rustls-tls"),
     not(feature = "reqwest-default-tls")
   ))]
   compile_error!("Must select a reqwest TLS backend");
 
   use std::{io, time::Duration};
 
   use anyhow::{anyhow, Context, Result};
   #[cfg(any(
     feature = "reqwest-rustls-tls",
     feature = "reqwest-default-tls"
   ))]
   use once_cell::sync::Lazy;
   use reqwest::{
     blocking::{Client, ClientBuilder, Response},
     header, Proxy,
   };
   use url::Url;
 
   use super::{Event, TlsBackend};
   use crate::errors::*;
 
   pub fn download(
     url: &Url,
     resume_from: u64,
     callback: &dyn Fn(Event<'_>) -> Result<()>,
     tls: TlsBackend,
   ) -> Result<()> {
     // Short-circuit reqwest for the "file:" URL scheme
     if download_from_file_url(url, resume_from, callback)? {
       return Ok(());
     }
 
     let mut res = request(url, resume_from, tls)
       .context("failed to make network request")?;
 
     if !res.status().is_success() {
       let code: u16 = res.status().into();
       return Err(anyhow!(DownloadError::HttpStatus(u32::from(code))));
     }
 
     let buffer_size = 0x10000;
     let mut buffer = vec![0u8; buffer_size];
 
     if let Some(len) = res.headers().get(header::CONTENT_LENGTH) {
       // TODO possible issues during unwrap?
       let len = len.to_str().unwrap().parse::<u64>().unwrap() + resume_from;
       callback(Event::DownloadContentLengthReceived(len))?;
     }
 
     loop {
       let bytes_read = io::Read::read(&mut res, &mut buffer)?;
 
       if bytes_read != 0 {
         callback(Event::DownloadDataReceived(&buffer[0..bytes_read]))?;
       } else {
         return Ok(());
       }
     }
   }
 
   fn client_generic() -> ClientBuilder {
     Client::builder()
       .gzip(false)
       .user_agent(super::USER_AGENT)
       .proxy(Proxy::custom(env_proxy))
       .timeout(Duration::from_secs(30))
   }
 
   #[cfg(feature = "reqwest-rustls-tls")]
   static CLIENT_RUSTLS_TLS: Lazy<Client> = Lazy::new(|| {
     let catcher = || client_generic().use_rustls_tls().build();
 
     // woah, an unwrap?!
     // It's OK. This is the same as what is happening in curl.
     //
     // The curl::Easy::new() internally assert!s that the initialized
     // Easy is not null. Inside reqwest, the errors here would be from
     // the TLS library returning a null pointer as well.
     catcher().unwrap()
   });
 
   #[cfg(feature = "reqwest-default-tls")]
   static CLIENT_DEFAULT_TLS: Lazy<Client> = Lazy::new(|| {
     let catcher = || client_generic().build();
 
     // woah, an unwrap?!
     // It's OK. This is the same as what is happening in curl.
     //
     // The curl::Easy::new() internally assert!s that the initialized
     // Easy is not null. Inside reqwest, the errors here would be from
     // the TLS library returning a null pointer as well.
     catcher().unwrap()
   });
 
   fn env_proxy(url: &Url) -> Option<Url> {
     env_proxy::for_url(url).to_url()
   }
 
   fn request(
     url: &Url,
     resume_from: u64,
     backend: TlsBackend,
   ) -> Result<Response, DownloadError> {
     let client: &Client = match backend {
       #[cfg(feature = "reqwest-rustls-tls")]
       TlsBackend::Rustls => &CLIENT_RUSTLS_TLS,
       #[cfg(not(feature = "reqwest-rustls-tls"))]
       TlsBackend::Rustls => {
         return Err(DownloadError::BackendUnavailable("reqwest rustls"));
       }
       #[cfg(feature = "reqwest-default-tls")]
       TlsBackend::Default => &CLIENT_DEFAULT_TLS,
       #[cfg(not(feature = "reqwest-default-tls"))]
       TlsBackend::Default => {
         return Err(DownloadError::BackendUnavailable("reqwest default TLS"));
       }
     };
     let mut req = client.get(url.as_str());
 
     if resume_from != 0 {
       req = req.header(header::RANGE, format!("bytes={resume_from}-"));
     }
 
     Ok(req.send()?)
   }
 
   fn download_from_file_url(
     url: &Url,
     resume_from: u64,
     callback: &dyn Fn(Event<'_>) -> Result<()>,
   ) -> Result<bool> {
     use std::fs;
 
     // The file scheme is mostly for use by tests to mock the dist server
     if url.scheme() == "file" {
       let src = url.to_file_path().map_err(|_| {
         DownloadError::Message(format!("bogus file url: '{url}'"))
       })?;
       if !src.is_file() {
         // Because some of rustup's logic depends on checking
         // the error when a downloaded file doesn't exist, make
         // the file case return the same error value as the
         // network case.
         return Err(anyhow!(DownloadError::FileNotFound));
       }
 
       let mut f =
         fs::File::open(src).context("unable to open downloaded file")?;
       io::Seek::seek(&mut f, io::SeekFrom::Start(resume_from))?;
 
       let mut buffer = vec![0u8; 0x10000];
       loop {
         let bytes_read = io::Read::read(&mut f, &mut buffer)?;
         if bytes_read == 0 {
           break;
         }
         callback(Event::DownloadDataReceived(&buffer[0..bytes_read]))?;
       }
 
       Ok(true)
     } else {
       Ok(false)
     }
   }
 }
 
 #[cfg(not(feature = "curl-backend"))]
 pub mod curl {
 
   use anyhow::{anyhow, Result};
 
   use super::Event;
   use crate::errors::*;
   use url::Url;
 
   pub fn download(
     _url: &Url,
     _resume_from: u64,
     _callback: &dyn Fn(Event<'_>) -> Result<()>,
   ) -> Result<()> {
     Err(anyhow!(DownloadError::BackendUnavailable("curl")))
   }
 }
 
 #[cfg(not(feature = "reqwest-backend"))]
 pub mod reqwest_be {
 
   use anyhow::{anyhow, Result};
 
   use super::{Event, TlsBackend};
   use crate::errors::*;
   use url::Url;
 
   pub fn download(
     _url: &Url,
     _resume_from: u64,
     _callback: &dyn Fn(Event<'_>) -> Result<()>,
     _tls: TlsBackend,
   ) -> Result<()> {
     Err(anyhow!(DownloadError::BackendUnavailable("reqwest")))
   }
 }