0
0
mirror of https://github.com/TeFiLeDo/listload.git synced 2024-11-23 12:46:17 +01:00

begin rewrite

This commit is contained in:
Adrian Wannenmacher 2023-09-29 19:25:03 +02:00
parent 18a09882ed
commit 79788ae5d2
Signed by: tfld
GPG Key ID: 19D986ECB1E492D5
8 changed files with 26 additions and 797 deletions

View File

@ -1,6 +1,6 @@
[package] [package]
name = "listload" name = "listload"
version = "0.1.0" version = "0.2.0"
edition = "2021" edition = "2021"
license = "EUPL-1.2" license = "EUPL-1.2"
authors = ["Adrian Wannenmacher <tfld@tfld.dev>"] authors = ["Adrian Wannenmacher <tfld@tfld.dev>"]
@ -12,14 +12,10 @@ keywords = ["download", "http", "https"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
anyhow = "1.0.71" anyhow = "1.0.75"
clap = { version = "4.3.8", features = ["cargo", "derive", "env", "wrap_help"] } clap = { version = "4.4.6", features = ["env", "derive", "wrap_help", "cargo", "unicode"] }
directories = "5.0.1" human-panic = "1.2.1"
downloader = { version = "0.2.7", default-features = false, features = ["rustls-tls"] } reqwest = { version = "0.11.20", features = ["brotli", "deflate", "gzip"] }
indoc = "2.0.1" serde = { version = "1.0.188", features = ["derive"] }
rand = "0.8.5" serde_json = "1.0.107"
regex = "1.8.4" tokio = { version = "1.32.0", features = ["full"] }
serde = { version = "1.0.164", features = ["derive"] }
serde_json = "1.0.99"
toml = "0.7.5"
url = { version = "2.4.0", features = ["serde"] }

View File

@ -1,115 +0,0 @@
use std::{
collections::HashMap,
fs, mem,
path::{Path, PathBuf},
};
use anyhow::{ensure, Context};
use downloader::{Download, DownloadSummary, Downloader, Error};
use rand::random;
pub struct CachingDownloader {
inner: Downloader,
cache: PathBuf,
base: PathBuf,
}
impl CachingDownloader {
///
/// This downloader will put all downloaded files into a cache directory, before moving them to
/// their actual target location. If a download fails, any corresponding file will not be
/// touched.
///
/// The `inner` is the downloader used to perform the actual download. The `cache` directory is
/// the location to put the downloaded files in. Individual downloads will create subdirectories
/// in it, to avoid conflicts. The `base` is the path relative file names are relative to.
pub fn new(inner: Downloader, cache: &Path, base: &Path) -> anyhow::Result<Self> {
let cache_ready = cache.is_dir();
ensure!(
cache_ready || !cache.exists(),
"cache directory is neither directory nor nonexistant ({})",
cache.display()
);
if !cache_ready {
fs::create_dir_all(cache).context(format!(
"failed to create cache directory ({})",
cache.display()
))?;
}
ensure!(base.is_dir(), "base directory doesn't exist");
Ok(Self {
inner,
cache: cache.to_path_buf(),
base: base.to_path_buf(),
})
}
pub fn download(
&mut self,
downloads: &mut [Download],
partiton: Option<&str>,
) -> anyhow::Result<Vec<anyhow::Result<DownloadSummary>>> {
let partition = partiton
.map(ToString::to_string)
.unwrap_or_else(|| format!("{:0>16x}", random::<u64>()));
let cache = self.cache.join(partition);
ensure!(!cache.exists(), "cache partition exists");
fs::create_dir(&cache).context("failed to create cache partition")?;
let mut mapping: HashMap<_, _> = downloads
.iter_mut()
.enumerate()
.map(|(counter, down)| (cache.join(format!("{counter:0>16x}")), down))
.map(|(cache, down)| (cache.clone(), mem::replace(&mut down.file_name, cache)))
.collect();
let mut results: Vec<_> = self
.inner
.download(downloads)
.context("all downloads failed")?
.into_iter()
.map(|r| handle_file(&mut mapping, r, &self.base))
.collect();
for (leftover, _) in mapping {
if let Err(err) =
fs::remove_file(leftover).context("failed to delete leftover cache file")
{
results.push(Err(err));
}
}
if let Err(err) = fs::remove_dir(cache).context("failed to delete cache partition") {
results.push(Err(err));
}
Ok(results)
}
}
fn handle_file(
mapping: &mut HashMap<PathBuf, PathBuf>,
summary: Result<DownloadSummary, Error>,
base: &Path,
) -> anyhow::Result<DownloadSummary> {
let mut summary = summary.context("download failed")?;
let cache = mapping
.remove(&summary.file_name)
.context("unknown target location")?;
let mut cache = base.join(cache);
mem::swap(&mut summary.file_name, &mut cache);
fs::hard_link(&cache, &summary.file_name)
.or_else(|_| fs::copy(&cache, &summary.file_name).map(|_| ()))
.context("failed to copy downloaded file to target location")?;
fs::remove_file(cache).context("failed to remove cached file")?;
Ok(summary)
}

11
src/cli.rs Normal file
View File

@ -0,0 +1,11 @@
use clap::{Parser, Subcommand};
#[derive(Debug, Parser)]
#[clap(author, version, about)]
pub struct CLI {
#[clap(subcommand)]
command: Command,
}
#[derive(Debug, Subcommand)]
pub enum Command {}

View File

@ -1,117 +0,0 @@
use std::{
fmt::Display,
fs::File,
io::{BufReader, Read},
path::{Path, PathBuf},
time::Duration,
};
use anyhow::{anyhow, Context};
use downloader::Downloader;
use serde::{Deserialize, Serialize};
use crate::{caching_downloader::CachingDownloader, PROJ_DIRS, USER_DIRS};
/// Global configuration values.
#[derive(Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields, default)]
pub struct Config {
/// The base directory used for resolving relative paths. Defaults to the home directory.
pub base_directory: PathBuf,
/// The number of parallel downloads. Defaults to 32.
pub parallel_downloads: u16,
/// The number of retries. Defaults to 3.
pub retries: u16,
/// The timeout for establishing a connection in seconds. Defaults to 15.
pub timeout_connection: u64,
/// The timeout for downloading a file in seconds. Defaults to 30.
pub timeout_download: u64,
/// The user agent to use for HTTP communication. Default is not stabilized.
pub user_agent: String,
}
impl Config {
/// Read the configuration from the default file, or revert to default values if unavailable.
pub fn read_from_default_file() -> anyhow::Result<Self> {
let dirs = PROJ_DIRS.get().expect("directories not initialized");
let mut path = dirs.config_dir().to_path_buf();
path.push("config.toml");
if path.is_file() {
Self::read_from_file(&path)
} else if !path.exists() {
Ok(Self::default())
} else {
Err(anyhow!(
"default configuration file is neither file nor nonexistent"
))
}
}
/// Read the configuration from the specified file.
pub fn read_from_file(path: &Path) -> anyhow::Result<Self> {
let file = File::open(path).context("failed to open config file")?;
let mut buf = BufReader::new(file);
let mut data = String::new();
buf.read_to_string(&mut data)
.context("failed to read config data")?;
toml::from_str(&data).context("failed to parse config file")
}
/// Create a [`Downloader`] with the configuration applied.
pub fn downloader(&self) -> anyhow::Result<Downloader> {
Downloader::builder()
.user_agent(&self.user_agent)
.connect_timeout(Duration::from_secs(self.timeout_connection))
.timeout(Duration::from_secs(self.timeout_download))
.parallel_requests(self.parallel_downloads)
.retries(self.retries)
.download_folder(&self.base_directory)
.build()
.context("failed to build downloader")
}
pub fn default_caching_downloader(&self) -> anyhow::Result<CachingDownloader> {
let dirs = PROJ_DIRS.get().expect("directories not initialized");
self.caching_downloader(dirs.cache_dir())
}
pub fn caching_downloader(&self, cache: &Path) -> anyhow::Result<CachingDownloader> {
self.downloader()
.and_then(|d| CachingDownloader::new(d, cache, &self.base_directory))
}
}
impl Default for Config {
fn default() -> Self {
let dirs = USER_DIRS.get().expect("directories not initialized");
Self {
base_directory: dirs.home_dir().to_path_buf(),
parallel_downloads: 32,
retries: 3,
timeout_connection: 15,
timeout_download: 30,
user_agent: format!(
"{} {}.{}",
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION_MAJOR"),
env!("CARGO_PKG_VERSION_MINOR")
),
}
}
}
impl Display for Config {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
writeln!(f, "base directory: {}", self.base_directory.display())?;
writeln!(f, "parallel downloads: {}", self.parallel_downloads)?;
writeln!(f, "retries: {}", self.retries)?;
writeln!(f, "timeout connection: {}s", self.timeout_connection)?;
writeln!(f, "timeout download: {}s", self.timeout_download)?;
write!(f, "user agent: {}", self.user_agent)
}
}

View File

@ -1,219 +1,11 @@
use std::{path::PathBuf, sync::OnceLock}; use clap::Parser;
use anyhow::Context; mod cli;
use clap::{Parser, Subcommand};
use config::Config;
use directories::{ProjectDirs, UserDirs};
use persistent_state::PersistentState;
use target::Target;
use target_list::TargetList;
use url::Url;
mod caching_downloader; #[tokio::main]
mod config; async fn main() -> anyhow::Result<()> {
mod persistent_state; human_panic::setup_panic!();
mod target; let cli = cli::CLI::parse();
mod target_list;
static USER_DIRS: OnceLock<UserDirs> = OnceLock::new(); Ok(())
static PROJ_DIRS: OnceLock<ProjectDirs> = OnceLock::new();
fn main() -> anyhow::Result<()> {
let cli = Cli::parse();
// initialize dirs
let user_dirs = UserDirs::new().context("failed to discover user directiories")?;
let proj_dirs = ProjectDirs::from("dev", "TFLD", "ListLoad")
.context("failed to discover program directories")?;
USER_DIRS
.set(user_dirs)
.ok()
.context("failed to initialize user directories")?;
PROJ_DIRS
.set(proj_dirs)
.ok()
.context("failedto initialize program directories")?;
if let Cmd::License = cli.cmd {
println!("{}", include_str!("../LICENSE"));
return Ok(());
}
// prepare for operation
let cfg = Config::read_from_default_file().context("failed to load config")?;
let mut downloader = cfg
.default_caching_downloader()
.context("failed to create downloader")?;
let mut persistent =
PersistentState::read_from_default_file().context("failed to load persistent state")?;
match cli.cmd {
Cmd::Config => {
println!("{cfg}");
}
Cmd::PersistentState => {
println!("{persistent}");
}
Cmd::Download { name } => {
let name = name
.as_deref()
.or(persistent.list())
.context("no list specified or selected")?;
let list = TargetList::load(name).context("failed to load target list")?;
let mut downloads = list.downloads();
for res in downloader.download(&mut downloads, None)? {
if let Err(err) = res {
eprintln!("{:?}", err);
}
}
}
Cmd::List { cmd } => match cmd {
ListCommand::Create {
name,
keep_current_selected: keep_current_active,
comment,
} => {
if TargetList::exists(&name) {
eprintln!("list already exists");
} else {
TargetList::new(&name, comment.as_deref())
.context("failed to create target list")?;
}
if !keep_current_active {
persistent.set_list(&name);
}
}
ListCommand::Select { name } => {
if name == "none" {
persistent.clear_list();
} else if TargetList::exists(&name) {
persistent.set_list(&name);
} else {
eprintln!("list doesn't exist");
}
}
},
Cmd::Target { cmd } => {
let list = persistent.list().context("no list selected")?;
let mut list = TargetList::load(list).context("failed to load list")?;
match cmd {
TargetCommand::Create {
file,
url,
comment,
keep_current_selected,
} => {
let target =
Target::new(url, &file, comment.as_deref()).context("invalid target")?;
list.add_target(target);
if !keep_current_selected {
persistent.set_target(list.len_targets() - 1);
}
}
TargetCommand::Select { index } => {
if index < list.len_targets() {
persistent.set_target(index);
} else {
eprintln!("target doesn't exist");
}
}
}
list.save().context("failed to save list")?;
}
Cmd::License => {
panic!("late command");
}
}
persistent.save_to_default_file()
}
#[derive(Parser)]
#[clap(about, author, version)]
struct Cli {
#[clap(subcommand)]
cmd: Cmd,
}
#[derive(Subcommand)]
enum Cmd {
/// Print the current configuration.
Config,
/// Print the EUPL 1.2, under which this program is licensed.
License,
/// Print the current persistent state.
PersistentState,
/// Download a target list.
Download {
/// The name of the target list. Defaults to the selected list.
name: Option<String>,
},
/// Create, select, modify, … a target list.
List {
#[clap(subcommand)]
cmd: ListCommand,
},
/// Create, select, modify, … a target in the selected target list.
Target {
#[clap(subcommand)]
cmd: TargetCommand,
},
}
#[derive(Subcommand)]
enum ListCommand {
/// Create a new target list.
Create {
/// A unique name for the target list.
///
/// The name must start with a lowercase letter (`a-z`). After that, it consists of at least
/// one lowercase letter (`a-z`) or number (`0-9`). It may also contain nonconsecutive
/// underscores (`_`), but must not end with one. The name must not be `none`.
///
/// Valid examples: default, version15, my_4_funny_pictures
///
/// Invalid examples: none, 14, _hi, hi_, h__i
name: String,
/// A comment to remember what the target list is meant to do.
#[clap(long, short)]
comment: Option<String>,
/// Don't select the newly created target list.
#[clap(long, short)]
keep_current_selected: bool,
},
/// Select an existing target list.
Select {
/// The unique name of the target list.
///
/// The special value `none` unselects the current selection.
name: String,
},
}
#[derive(Subcommand)]
enum TargetCommand {
/// Create a new target.
Create {
/// The local file name.
file: PathBuf,
/// A list of URLs the file is available at.
url: Vec<Url>,
/// A comment to remember why the target is in the list.
#[clap(long, short)]
comment: Option<String>,
/// Don't select the newly created target.
#[clap(long, short)]
keep_current_selected: bool,
},
/// Select an existing target.
Select {
/// The index of the target.
index: usize,
},
} }

View File

@ -1,123 +0,0 @@
use std::{
fmt::Display,
fs::{create_dir_all, File},
io::{BufReader, BufWriter, Read, Write},
path::Path,
};
use anyhow::{bail, ensure, Context};
use indoc::writedoc;
use serde::{Deserialize, Serialize};
use crate::PROJ_DIRS;
const PERSISTENT_FILE: &str = "persistent_state.json";
#[derive(Debug, Default, Deserialize, Serialize)]
#[serde(deny_unknown_fields, default)]
pub struct PersistentState {
list: Option<String>,
target: Option<usize>,
}
impl PersistentState {
pub fn list(&self) -> Option<&str> {
self.list.as_deref()
}
pub fn set_list(&mut self, list: &str) {
self.list = Some(list.to_string());
self.target = None;
}
pub fn clear_list(&mut self) {
self.list = None;
self.target = None;
}
pub fn target(&self) -> Option<usize> {
self.target
}
pub fn set_target(&mut self, index: usize) {
if self.list.is_some() {
self.target = Some(index);
}
}
}
impl PersistentState {
pub fn read_from_default_file() -> anyhow::Result<Self> {
let dirs = PROJ_DIRS.get().expect("directories not initialized");
let mut path = dirs.preference_dir().to_path_buf();
path.push(PERSISTENT_FILE);
if path.is_file() {
Self::read_from_file(&path)
} else if !path.exists() {
Ok(Self::default())
} else {
bail!("persistent state file is neither file nor nonexistant")
}
}
pub fn read_from_file(path: &Path) -> anyhow::Result<Self> {
File::open(path)
.context("failed to open persistent state file")
.map(BufReader::new)
.and_then(Self::read_from)
}
pub fn read_from(reader: impl Read) -> anyhow::Result<Self> {
serde_json::from_reader(reader).context("failed to parse persistent state file")
}
pub fn save_to_default_file(&self) -> anyhow::Result<()> {
let dirs = PROJ_DIRS.get().expect("directories not initialized");
let mut path = dirs.preference_dir().to_path_buf();
ensure!(
path.is_dir() || !path.exists(),
"preference directory is neither directory nor nonexistent"
);
if !path.exists() {
create_dir_all(&path).context("failed to create program preference directory")?;
}
path.push(PERSISTENT_FILE);
ensure!(
path.is_file() || !path.exists(),
"persistent state file is neither file nor nonexistant"
);
self.save_to_file(&path)
}
pub fn save_to_file(&self, path: &Path) -> anyhow::Result<()> {
File::create(path)
.context("failed to create persistent state file")
.map(BufWriter::new)
.and_then(|w| self.save_to(w))
}
pub fn save_to(&self, writer: impl Write) -> anyhow::Result<()> {
serde_json::to_writer(writer, &self).context("failed to write persistent state file")
}
}
impl Display for PersistentState {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
writedoc!(
f,
"
current list: {}
current target: {}",
self.list.as_ref().map(AsRef::as_ref).unwrap_or("none"),
self.target
.map(|t| t.to_string())
.unwrap_or("none".to_string())
)
}
}

View File

@ -1,69 +0,0 @@
use std::{
fmt::Display,
path::{Path, PathBuf},
};
use anyhow::ensure;
use downloader::Download;
use serde::{Deserialize, Serialize};
use url::Url;
#[derive(Debug, Deserialize, Serialize)]
pub struct Target {
urls: Vec<Url>,
file: PathBuf,
comment: Option<String>,
}
impl Target {
pub fn new(urls: Vec<Url>, file: &Path, comment: Option<&str>) -> anyhow::Result<Self> {
ensure!(!urls.is_empty(), "at least one url is required");
ensure!(
file.is_file() || !file.exists(),
"file must be file or nonexistent"
);
for url in &urls {
let scheme = url.scheme();
ensure!(scheme == "http" || scheme == "https", "url is not http(s)");
}
Ok(Self {
urls,
file: file.to_path_buf(),
comment: comment.map(ToString::to_string),
})
}
}
impl Display for Target {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}{}",
self.comment
.clone()
.map(|c| format!("{c}: "))
.unwrap_or_default(),
self.file.display()
)
}
}
impl From<&Target> for Download {
fn from(value: &Target) -> Self {
match value.urls.len() {
0 => panic!("target without url"),
1 => Download::new(value.urls[0].as_str()),
_ => Download::new_mirrored(
value
.urls
.iter()
.map(|u| u.as_str())
.collect::<Vec<_>>()
.as_ref(),
),
}
.file_name(&value.file)
}
}

View File

@ -1,146 +0,0 @@
use std::{
fs::{create_dir_all, File, OpenOptions},
io::{BufReader, BufWriter, Seek, SeekFrom},
path::{Path, PathBuf},
};
use anyhow::{ensure, Context};
use downloader::Download;
use regex::Regex;
use serde::{Deserialize, Serialize};
use crate::{target::Target, PROJ_DIRS};
#[derive(Debug)]
pub struct TargetList {
inner: InnerTargetList,
file: File,
}
impl TargetList {
pub fn name(&self) -> &str {
&self.inner.name
}
pub fn add_target(&mut self, target: Target) {
self.inner.targets.push(target);
}
pub fn len_targets(&self) -> usize {
self.inner.targets.len()
}
pub fn downloads(&self) -> Vec<Download> {
self.inner.targets.iter().map(|t| t.into()).collect()
}
}
impl TargetList {
fn list_dir() -> PathBuf {
let dirs = PROJ_DIRS.get().expect("directories not initialized");
let mut path = dirs.data_dir().to_path_buf();
path.push("lists");
path
}
fn list_file(dir: &Path, name: &str) -> PathBuf {
let mut path = dir.to_path_buf();
path.push(name);
path.set_extension("json");
path
}
pub fn exists(name: &str) -> bool {
Self::exists_in(name, &Self::list_dir())
}
pub fn exists_in(name: &str, directory: &Path) -> bool {
Self::list_file(directory, name).is_file()
}
pub fn new(name: &str, comment: Option<&str>) -> anyhow::Result<Self> {
let dir = Self::list_dir();
ensure!(
dir.is_dir() || !dir.exists(),
"list directory is neither a directory nor nonexistant"
);
if !dir.exists() {
create_dir_all(&dir).context("failed to create list directory")?;
}
Self::new_in(name, comment, &dir)
}
pub fn new_in(name: &str, comment: Option<&str>, directory: &Path) -> anyhow::Result<Self> {
ensure!(directory.is_dir(), "directory isn't a directory");
ensure!(name != "none", "name is \"none\"");
let name_regex = Regex::new("^[a-z](_?[a-z0-9])+$").expect("correct name regex");
ensure!(name_regex.is_match(name), "name is forbidden");
let path = Self::list_file(directory, name);
ensure!(!path.is_file(), "target list file already exists");
ensure!(
!path.exists(),
"target list file is neither a file nor nonexistent",
);
let file = File::create(path).context("failed to create target list file")?;
let mut ret = Self {
file,
inner: InnerTargetList {
name: name.to_string(),
comment: comment.map(ToString::to_string),
targets: Vec::new(),
},
};
ret.save().context("failed to save no target list file")?;
Ok(ret)
}
pub fn load(name: &str) -> anyhow::Result<Self> {
Self::load_in(name, &Self::list_dir())
}
pub fn load_in(name: &str, directory: &Path) -> anyhow::Result<Self> {
ensure!(directory.is_dir(), "directory isn't a directory");
let path = Self::list_file(directory, name);
let file = OpenOptions::new()
.read(true)
.write(true)
.create(false)
.append(false)
.truncate(false)
.open(path)
.context("failed to open target list file")?;
let inner: InnerTargetList = serde_json::from_reader(BufReader::new(&file))
.context("failed to parse target list file")?;
ensure!(inner.name == name, "list name missmatch");
Ok(Self { inner, file })
}
pub fn save(&mut self) -> anyhow::Result<()> {
let _ = self
.file
.set_len(0)
.context("failed to clear target list file");
self.file
.seek(SeekFrom::Start(0))
.context("failed to rewind target list file")?;
serde_json::to_writer(BufWriter::new(&self.file), &self.inner)
.context("failed to save target list")
}
}
#[derive(Debug, Deserialize, Serialize)]
struct InnerTargetList {
name: String,
comment: Option<String>,
targets: Vec<Target>,
}