From 79788ae5d2320407e96b8ac4a0389e5c44fa1cc4 Mon Sep 17 00:00:00 2001 From: Adrian Wannenmacher Date: Fri, 29 Sep 2023 19:25:03 +0200 Subject: [PATCH] begin rewrite --- Cargo.toml | 20 ++-- src/caching_downloader.rs | 115 -------------------- src/cli.rs | 11 ++ src/config.rs | 117 -------------------- src/main.rs | 222 ++------------------------------------ src/persistent_state.rs | 123 --------------------- src/target.rs | 69 ------------ src/target_list.rs | 146 ------------------------- 8 files changed, 26 insertions(+), 797 deletions(-) delete mode 100644 src/caching_downloader.rs create mode 100644 src/cli.rs delete mode 100644 src/config.rs delete mode 100644 src/persistent_state.rs delete mode 100644 src/target.rs delete mode 100644 src/target_list.rs diff --git a/Cargo.toml b/Cargo.toml index 4aec95c..21a54f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "listload" -version = "0.1.0" +version = "0.2.0" edition = "2021" license = "EUPL-1.2" authors = ["Adrian Wannenmacher "] @@ -12,14 +12,10 @@ keywords = ["download", "http", "https"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -anyhow = "1.0.71" -clap = { version = "4.3.8", features = ["cargo", "derive", "env", "wrap_help"] } -directories = "5.0.1" -downloader = { version = "0.2.7", default-features = false, features = ["rustls-tls"] } -indoc = "2.0.1" -rand = "0.8.5" -regex = "1.8.4" -serde = { version = "1.0.164", features = ["derive"] } -serde_json = "1.0.99" -toml = "0.7.5" -url = { version = "2.4.0", features = ["serde"] } +anyhow = "1.0.75" +clap = { version = "4.4.6", features = ["env", "derive", "wrap_help", "cargo", "unicode"] } +human-panic = "1.2.1" +reqwest = { version = "0.11.20", features = ["brotli", "deflate", "gzip"] } +serde = { version = "1.0.188", features = ["derive"] } +serde_json = "1.0.107" +tokio = { version = "1.32.0", features = ["full"] } diff --git a/src/caching_downloader.rs b/src/caching_downloader.rs deleted file mode 100644 index cca2241..0000000 --- a/src/caching_downloader.rs +++ /dev/null @@ -1,115 +0,0 @@ -use std::{ - collections::HashMap, - fs, mem, - path::{Path, PathBuf}, -}; - -use anyhow::{ensure, Context}; -use downloader::{Download, DownloadSummary, Downloader, Error}; -use rand::random; - -pub struct CachingDownloader { - inner: Downloader, - cache: PathBuf, - base: PathBuf, -} - -impl CachingDownloader { - /// - /// This downloader will put all downloaded files into a cache directory, before moving them to - /// their actual target location. If a download fails, any corresponding file will not be - /// touched. - /// - /// The `inner` is the downloader used to perform the actual download. The `cache` directory is - /// the location to put the downloaded files in. Individual downloads will create subdirectories - /// in it, to avoid conflicts. The `base` is the path relative file names are relative to. - pub fn new(inner: Downloader, cache: &Path, base: &Path) -> anyhow::Result { - let cache_ready = cache.is_dir(); - ensure!( - cache_ready || !cache.exists(), - "cache directory is neither directory nor nonexistant ({})", - cache.display() - ); - - if !cache_ready { - fs::create_dir_all(cache).context(format!( - "failed to create cache directory ({})", - cache.display() - ))?; - } - - ensure!(base.is_dir(), "base directory doesn't exist"); - - Ok(Self { - inner, - cache: cache.to_path_buf(), - base: base.to_path_buf(), - }) - } - - pub fn download( - &mut self, - downloads: &mut [Download], - partiton: Option<&str>, - ) -> anyhow::Result>> { - let partition = partiton - .map(ToString::to_string) - .unwrap_or_else(|| format!("{:0>16x}", random::())); - let cache = self.cache.join(partition); - - ensure!(!cache.exists(), "cache partition exists"); - fs::create_dir(&cache).context("failed to create cache partition")?; - - let mut mapping: HashMap<_, _> = downloads - .iter_mut() - .enumerate() - .map(|(counter, down)| (cache.join(format!("{counter:0>16x}")), down)) - .map(|(cache, down)| (cache.clone(), mem::replace(&mut down.file_name, cache))) - .collect(); - - let mut results: Vec<_> = self - .inner - .download(downloads) - .context("all downloads failed")? - .into_iter() - .map(|r| handle_file(&mut mapping, r, &self.base)) - .collect(); - - for (leftover, _) in mapping { - if let Err(err) = - fs::remove_file(leftover).context("failed to delete leftover cache file") - { - results.push(Err(err)); - } - } - - if let Err(err) = fs::remove_dir(cache).context("failed to delete cache partition") { - results.push(Err(err)); - } - - Ok(results) - } -} - -fn handle_file( - mapping: &mut HashMap, - summary: Result, - base: &Path, -) -> anyhow::Result { - let mut summary = summary.context("download failed")?; - - let cache = mapping - .remove(&summary.file_name) - .context("unknown target location")?; - let mut cache = base.join(cache); - - mem::swap(&mut summary.file_name, &mut cache); - - fs::hard_link(&cache, &summary.file_name) - .or_else(|_| fs::copy(&cache, &summary.file_name).map(|_| ())) - .context("failed to copy downloaded file to target location")?; - - fs::remove_file(cache).context("failed to remove cached file")?; - - Ok(summary) -} diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..71baa37 --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,11 @@ +use clap::{Parser, Subcommand}; + +#[derive(Debug, Parser)] +#[clap(author, version, about)] +pub struct CLI { + #[clap(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +pub enum Command {} diff --git a/src/config.rs b/src/config.rs deleted file mode 100644 index bd9e49e..0000000 --- a/src/config.rs +++ /dev/null @@ -1,117 +0,0 @@ -use std::{ - fmt::Display, - fs::File, - io::{BufReader, Read}, - path::{Path, PathBuf}, - time::Duration, -}; - -use anyhow::{anyhow, Context}; -use downloader::Downloader; -use serde::{Deserialize, Serialize}; - -use crate::{caching_downloader::CachingDownloader, PROJ_DIRS, USER_DIRS}; - -/// Global configuration values. -#[derive(Debug, Deserialize, Serialize)] -#[serde(deny_unknown_fields, default)] -pub struct Config { - /// The base directory used for resolving relative paths. Defaults to the home directory. - pub base_directory: PathBuf, - /// The number of parallel downloads. Defaults to 32. - pub parallel_downloads: u16, - /// The number of retries. Defaults to 3. - pub retries: u16, - /// The timeout for establishing a connection in seconds. Defaults to 15. - pub timeout_connection: u64, - /// The timeout for downloading a file in seconds. Defaults to 30. - pub timeout_download: u64, - /// The user agent to use for HTTP communication. Default is not stabilized. - pub user_agent: String, -} - -impl Config { - /// Read the configuration from the default file, or revert to default values if unavailable. - pub fn read_from_default_file() -> anyhow::Result { - let dirs = PROJ_DIRS.get().expect("directories not initialized"); - - let mut path = dirs.config_dir().to_path_buf(); - path.push("config.toml"); - - if path.is_file() { - Self::read_from_file(&path) - } else if !path.exists() { - Ok(Self::default()) - } else { - Err(anyhow!( - "default configuration file is neither file nor nonexistent" - )) - } - } - - /// Read the configuration from the specified file. - pub fn read_from_file(path: &Path) -> anyhow::Result { - let file = File::open(path).context("failed to open config file")?; - let mut buf = BufReader::new(file); - - let mut data = String::new(); - buf.read_to_string(&mut data) - .context("failed to read config data")?; - - toml::from_str(&data).context("failed to parse config file") - } - - /// Create a [`Downloader`] with the configuration applied. - pub fn downloader(&self) -> anyhow::Result { - Downloader::builder() - .user_agent(&self.user_agent) - .connect_timeout(Duration::from_secs(self.timeout_connection)) - .timeout(Duration::from_secs(self.timeout_download)) - .parallel_requests(self.parallel_downloads) - .retries(self.retries) - .download_folder(&self.base_directory) - .build() - .context("failed to build downloader") - } - - pub fn default_caching_downloader(&self) -> anyhow::Result { - let dirs = PROJ_DIRS.get().expect("directories not initialized"); - self.caching_downloader(dirs.cache_dir()) - } - - pub fn caching_downloader(&self, cache: &Path) -> anyhow::Result { - self.downloader() - .and_then(|d| CachingDownloader::new(d, cache, &self.base_directory)) - } -} - -impl Default for Config { - fn default() -> Self { - let dirs = USER_DIRS.get().expect("directories not initialized"); - - Self { - base_directory: dirs.home_dir().to_path_buf(), - parallel_downloads: 32, - retries: 3, - timeout_connection: 15, - timeout_download: 30, - user_agent: format!( - "{} {}.{}", - env!("CARGO_PKG_NAME"), - env!("CARGO_PKG_VERSION_MAJOR"), - env!("CARGO_PKG_VERSION_MINOR") - ), - } - } -} - -impl Display for Config { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!(f, "base directory: {}", self.base_directory.display())?; - writeln!(f, "parallel downloads: {}", self.parallel_downloads)?; - writeln!(f, "retries: {}", self.retries)?; - writeln!(f, "timeout connection: {}s", self.timeout_connection)?; - writeln!(f, "timeout download: {}s", self.timeout_download)?; - write!(f, "user agent: {}", self.user_agent) - } -} diff --git a/src/main.rs b/src/main.rs index 89f8734..b7469ec 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,219 +1,11 @@ -use std::{path::PathBuf, sync::OnceLock}; +use clap::Parser; -use anyhow::Context; -use clap::{Parser, Subcommand}; -use config::Config; -use directories::{ProjectDirs, UserDirs}; -use persistent_state::PersistentState; -use target::Target; -use target_list::TargetList; -use url::Url; +mod cli; -mod caching_downloader; -mod config; -mod persistent_state; -mod target; -mod target_list; +#[tokio::main] +async fn main() -> anyhow::Result<()> { + human_panic::setup_panic!(); + let cli = cli::CLI::parse(); -static USER_DIRS: OnceLock = OnceLock::new(); -static PROJ_DIRS: OnceLock = OnceLock::new(); - -fn main() -> anyhow::Result<()> { - let cli = Cli::parse(); - - // initialize dirs - let user_dirs = UserDirs::new().context("failed to discover user directiories")?; - let proj_dirs = ProjectDirs::from("dev", "TFLD", "ListLoad") - .context("failed to discover program directories")?; - - USER_DIRS - .set(user_dirs) - .ok() - .context("failed to initialize user directories")?; - PROJ_DIRS - .set(proj_dirs) - .ok() - .context("failedto initialize program directories")?; - - if let Cmd::License = cli.cmd { - println!("{}", include_str!("../LICENSE")); - return Ok(()); - } - - // prepare for operation - let cfg = Config::read_from_default_file().context("failed to load config")?; - let mut downloader = cfg - .default_caching_downloader() - .context("failed to create downloader")?; - let mut persistent = - PersistentState::read_from_default_file().context("failed to load persistent state")?; - - match cli.cmd { - Cmd::Config => { - println!("{cfg}"); - } - Cmd::PersistentState => { - println!("{persistent}"); - } - Cmd::Download { name } => { - let name = name - .as_deref() - .or(persistent.list()) - .context("no list specified or selected")?; - let list = TargetList::load(name).context("failed to load target list")?; - let mut downloads = list.downloads(); - - for res in downloader.download(&mut downloads, None)? { - if let Err(err) = res { - eprintln!("{:?}", err); - } - } - } - Cmd::List { cmd } => match cmd { - ListCommand::Create { - name, - keep_current_selected: keep_current_active, - comment, - } => { - if TargetList::exists(&name) { - eprintln!("list already exists"); - } else { - TargetList::new(&name, comment.as_deref()) - .context("failed to create target list")?; - } - - if !keep_current_active { - persistent.set_list(&name); - } - } - ListCommand::Select { name } => { - if name == "none" { - persistent.clear_list(); - } else if TargetList::exists(&name) { - persistent.set_list(&name); - } else { - eprintln!("list doesn't exist"); - } - } - }, - Cmd::Target { cmd } => { - let list = persistent.list().context("no list selected")?; - let mut list = TargetList::load(list).context("failed to load list")?; - - match cmd { - TargetCommand::Create { - file, - url, - comment, - keep_current_selected, - } => { - let target = - Target::new(url, &file, comment.as_deref()).context("invalid target")?; - list.add_target(target); - - if !keep_current_selected { - persistent.set_target(list.len_targets() - 1); - } - } - TargetCommand::Select { index } => { - if index < list.len_targets() { - persistent.set_target(index); - } else { - eprintln!("target doesn't exist"); - } - } - } - - list.save().context("failed to save list")?; - } - Cmd::License => { - panic!("late command"); - } - } - - persistent.save_to_default_file() -} - -#[derive(Parser)] -#[clap(about, author, version)] -struct Cli { - #[clap(subcommand)] - cmd: Cmd, -} - -#[derive(Subcommand)] -enum Cmd { - /// Print the current configuration. - Config, - /// Print the EUPL 1.2, under which this program is licensed. - License, - /// Print the current persistent state. - PersistentState, - /// Download a target list. - Download { - /// The name of the target list. Defaults to the selected list. - name: Option, - }, - /// Create, select, modify, … a target list. - List { - #[clap(subcommand)] - cmd: ListCommand, - }, - /// Create, select, modify, … a target in the selected target list. - Target { - #[clap(subcommand)] - cmd: TargetCommand, - }, -} - -#[derive(Subcommand)] -enum ListCommand { - /// Create a new target list. - Create { - /// A unique name for the target list. - /// - /// The name must start with a lowercase letter (`a-z`). After that, it consists of at least - /// one lowercase letter (`a-z`) or number (`0-9`). It may also contain nonconsecutive - /// underscores (`_`), but must not end with one. The name must not be `none`. - /// - /// Valid examples: default, version15, my_4_funny_pictures - /// - /// Invalid examples: none, 14, _hi, hi_, h__i - name: String, - /// A comment to remember what the target list is meant to do. - #[clap(long, short)] - comment: Option, - /// Don't select the newly created target list. - #[clap(long, short)] - keep_current_selected: bool, - }, - /// Select an existing target list. - Select { - /// The unique name of the target list. - /// - /// The special value `none` unselects the current selection. - name: String, - }, -} - -#[derive(Subcommand)] -enum TargetCommand { - /// Create a new target. - Create { - /// The local file name. - file: PathBuf, - /// A list of URLs the file is available at. - url: Vec, - /// A comment to remember why the target is in the list. - #[clap(long, short)] - comment: Option, - /// Don't select the newly created target. - #[clap(long, short)] - keep_current_selected: bool, - }, - /// Select an existing target. - Select { - /// The index of the target. - index: usize, - }, + Ok(()) } diff --git a/src/persistent_state.rs b/src/persistent_state.rs deleted file mode 100644 index 1eb4f8c..0000000 --- a/src/persistent_state.rs +++ /dev/null @@ -1,123 +0,0 @@ -use std::{ - fmt::Display, - fs::{create_dir_all, File}, - io::{BufReader, BufWriter, Read, Write}, - path::Path, -}; - -use anyhow::{bail, ensure, Context}; -use indoc::writedoc; -use serde::{Deserialize, Serialize}; - -use crate::PROJ_DIRS; - -const PERSISTENT_FILE: &str = "persistent_state.json"; - -#[derive(Debug, Default, Deserialize, Serialize)] -#[serde(deny_unknown_fields, default)] -pub struct PersistentState { - list: Option, - target: Option, -} - -impl PersistentState { - pub fn list(&self) -> Option<&str> { - self.list.as_deref() - } - - pub fn set_list(&mut self, list: &str) { - self.list = Some(list.to_string()); - self.target = None; - } - - pub fn clear_list(&mut self) { - self.list = None; - self.target = None; - } - - pub fn target(&self) -> Option { - self.target - } - - pub fn set_target(&mut self, index: usize) { - if self.list.is_some() { - self.target = Some(index); - } - } -} - -impl PersistentState { - pub fn read_from_default_file() -> anyhow::Result { - let dirs = PROJ_DIRS.get().expect("directories not initialized"); - - let mut path = dirs.preference_dir().to_path_buf(); - path.push(PERSISTENT_FILE); - - if path.is_file() { - Self::read_from_file(&path) - } else if !path.exists() { - Ok(Self::default()) - } else { - bail!("persistent state file is neither file nor nonexistant") - } - } - - pub fn read_from_file(path: &Path) -> anyhow::Result { - File::open(path) - .context("failed to open persistent state file") - .map(BufReader::new) - .and_then(Self::read_from) - } - - pub fn read_from(reader: impl Read) -> anyhow::Result { - serde_json::from_reader(reader).context("failed to parse persistent state file") - } - - pub fn save_to_default_file(&self) -> anyhow::Result<()> { - let dirs = PROJ_DIRS.get().expect("directories not initialized"); - - let mut path = dirs.preference_dir().to_path_buf(); - ensure!( - path.is_dir() || !path.exists(), - "preference directory is neither directory nor nonexistent" - ); - - if !path.exists() { - create_dir_all(&path).context("failed to create program preference directory")?; - } - - path.push(PERSISTENT_FILE); - ensure!( - path.is_file() || !path.exists(), - "persistent state file is neither file nor nonexistant" - ); - - self.save_to_file(&path) - } - - pub fn save_to_file(&self, path: &Path) -> anyhow::Result<()> { - File::create(path) - .context("failed to create persistent state file") - .map(BufWriter::new) - .and_then(|w| self.save_to(w)) - } - - pub fn save_to(&self, writer: impl Write) -> anyhow::Result<()> { - serde_json::to_writer(writer, &self).context("failed to write persistent state file") - } -} - -impl Display for PersistentState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writedoc!( - f, - " - current list: {} - current target: {}", - self.list.as_ref().map(AsRef::as_ref).unwrap_or("none"), - self.target - .map(|t| t.to_string()) - .unwrap_or("none".to_string()) - ) - } -} diff --git a/src/target.rs b/src/target.rs deleted file mode 100644 index 991c6d7..0000000 --- a/src/target.rs +++ /dev/null @@ -1,69 +0,0 @@ -use std::{ - fmt::Display, - path::{Path, PathBuf}, -}; - -use anyhow::ensure; -use downloader::Download; -use serde::{Deserialize, Serialize}; -use url::Url; - -#[derive(Debug, Deserialize, Serialize)] -pub struct Target { - urls: Vec, - file: PathBuf, - comment: Option, -} - -impl Target { - pub fn new(urls: Vec, file: &Path, comment: Option<&str>) -> anyhow::Result { - ensure!(!urls.is_empty(), "at least one url is required"); - ensure!( - file.is_file() || !file.exists(), - "file must be file or nonexistent" - ); - - for url in &urls { - let scheme = url.scheme(); - ensure!(scheme == "http" || scheme == "https", "url is not http(s)"); - } - - Ok(Self { - urls, - file: file.to_path_buf(), - comment: comment.map(ToString::to_string), - }) - } -} - -impl Display for Target { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}{}", - self.comment - .clone() - .map(|c| format!("{c}: ")) - .unwrap_or_default(), - self.file.display() - ) - } -} - -impl From<&Target> for Download { - fn from(value: &Target) -> Self { - match value.urls.len() { - 0 => panic!("target without url"), - 1 => Download::new(value.urls[0].as_str()), - _ => Download::new_mirrored( - value - .urls - .iter() - .map(|u| u.as_str()) - .collect::>() - .as_ref(), - ), - } - .file_name(&value.file) - } -} diff --git a/src/target_list.rs b/src/target_list.rs deleted file mode 100644 index 332558c..0000000 --- a/src/target_list.rs +++ /dev/null @@ -1,146 +0,0 @@ -use std::{ - fs::{create_dir_all, File, OpenOptions}, - io::{BufReader, BufWriter, Seek, SeekFrom}, - path::{Path, PathBuf}, -}; - -use anyhow::{ensure, Context}; -use downloader::Download; -use regex::Regex; -use serde::{Deserialize, Serialize}; - -use crate::{target::Target, PROJ_DIRS}; - -#[derive(Debug)] -pub struct TargetList { - inner: InnerTargetList, - file: File, -} - -impl TargetList { - pub fn name(&self) -> &str { - &self.inner.name - } - - pub fn add_target(&mut self, target: Target) { - self.inner.targets.push(target); - } - - pub fn len_targets(&self) -> usize { - self.inner.targets.len() - } - - pub fn downloads(&self) -> Vec { - self.inner.targets.iter().map(|t| t.into()).collect() - } -} - -impl TargetList { - fn list_dir() -> PathBuf { - let dirs = PROJ_DIRS.get().expect("directories not initialized"); - let mut path = dirs.data_dir().to_path_buf(); - path.push("lists"); - path - } - - fn list_file(dir: &Path, name: &str) -> PathBuf { - let mut path = dir.to_path_buf(); - path.push(name); - path.set_extension("json"); - path - } - - pub fn exists(name: &str) -> bool { - Self::exists_in(name, &Self::list_dir()) - } - - pub fn exists_in(name: &str, directory: &Path) -> bool { - Self::list_file(directory, name).is_file() - } - - pub fn new(name: &str, comment: Option<&str>) -> anyhow::Result { - let dir = Self::list_dir(); - ensure!( - dir.is_dir() || !dir.exists(), - "list directory is neither a directory nor nonexistant" - ); - - if !dir.exists() { - create_dir_all(&dir).context("failed to create list directory")?; - } - - Self::new_in(name, comment, &dir) - } - - pub fn new_in(name: &str, comment: Option<&str>, directory: &Path) -> anyhow::Result { - ensure!(directory.is_dir(), "directory isn't a directory"); - - ensure!(name != "none", "name is \"none\""); - let name_regex = Regex::new("^[a-z](_?[a-z0-9])+$").expect("correct name regex"); - ensure!(name_regex.is_match(name), "name is forbidden"); - - let path = Self::list_file(directory, name); - ensure!(!path.is_file(), "target list file already exists"); - ensure!( - !path.exists(), - "target list file is neither a file nor nonexistent", - ); - - let file = File::create(path).context("failed to create target list file")?; - - let mut ret = Self { - file, - inner: InnerTargetList { - name: name.to_string(), - comment: comment.map(ToString::to_string), - targets: Vec::new(), - }, - }; - ret.save().context("failed to save no target list file")?; - - Ok(ret) - } - - pub fn load(name: &str) -> anyhow::Result { - Self::load_in(name, &Self::list_dir()) - } - - pub fn load_in(name: &str, directory: &Path) -> anyhow::Result { - ensure!(directory.is_dir(), "directory isn't a directory"); - - let path = Self::list_file(directory, name); - let file = OpenOptions::new() - .read(true) - .write(true) - .create(false) - .append(false) - .truncate(false) - .open(path) - .context("failed to open target list file")?; - - let inner: InnerTargetList = serde_json::from_reader(BufReader::new(&file)) - .context("failed to parse target list file")?; - ensure!(inner.name == name, "list name missmatch"); - - Ok(Self { inner, file }) - } - - pub fn save(&mut self) -> anyhow::Result<()> { - let _ = self - .file - .set_len(0) - .context("failed to clear target list file"); - self.file - .seek(SeekFrom::Start(0)) - .context("failed to rewind target list file")?; - serde_json::to_writer(BufWriter::new(&self.file), &self.inner) - .context("failed to save target list") - } -} - -#[derive(Debug, Deserialize, Serialize)] -struct InnerTargetList { - name: String, - comment: Option, - targets: Vec, -}