stream files to disk
- Greatly reduced memory usage while downloading - Added progressbar, download rate - Added async runtime - Tweaks to allow usage of async download function
This commit is contained in:
parent
f609a53377
commit
dacaf322d4
1003
Cargo.lock
generated
1003
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@ -23,8 +23,11 @@ serde = { version = "1.0.189", features = ["derive"] }
|
||||
serde_json = "1.0.107"
|
||||
rand = "0.8.5"
|
||||
semver = "1.0.20"
|
||||
zip = "0.6.6"
|
||||
colored = "2.0.4"
|
||||
reqwest = { version = "0.11.22", features = ["stream"] }
|
||||
futures-util = "0.3.29"
|
||||
indicatif = "0.17.7"
|
||||
tokio = {version="1.33.0", features = ["rt-multi-thread", "macros"]}
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
steamlocate = "2.0.0-alpha.0"
|
||||
|
52
src/http_async.rs
Normal file
52
src/http_async.rs
Normal file
@ -0,0 +1,52 @@
|
||||
use std::cmp::min;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use colored::*;
|
||||
use futures_util::StreamExt;
|
||||
use indicatif::ProgressBar;
|
||||
use reqwest::Client;
|
||||
|
||||
use crate::misc;
|
||||
|
||||
pub async fn download_file(
|
||||
client: &Client,
|
||||
pb: &ProgressBar,
|
||||
url: &str,
|
||||
path: &PathBuf,
|
||||
size: u64,
|
||||
) -> Result<(), String> {
|
||||
let res = client
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.or(Err(format!("Failed to GET from '{}'", &url)))?;
|
||||
// Fix for CF shenanigans
|
||||
let total_size = res.content_length().unwrap_or(size);
|
||||
pb.set_length(total_size);
|
||||
pb.println(format!(
|
||||
"[{}] {} ({})",
|
||||
"Downloading".bright_yellow(),
|
||||
misc::cute_path(path),
|
||||
misc::human_readable_bytes(total_size)
|
||||
));
|
||||
pb.set_message(misc::cute_path(path));
|
||||
|
||||
let mut file =
|
||||
File::create(path).or(Err(format!("Failed to create file '{}'", path.display())))?;
|
||||
let mut downloaded: u64 = 0;
|
||||
let mut stream = res.bytes_stream();
|
||||
|
||||
while let Some(item) = stream.next().await {
|
||||
let chunk = item.or(Err("Error while downloading file"))?;
|
||||
file.write_all(&chunk)
|
||||
.or(Err("Error while writing to file"))?;
|
||||
let new = min(downloaded + (chunk.len() as u64), total_size);
|
||||
downloaded = new;
|
||||
pb.set_position(new);
|
||||
}
|
||||
|
||||
pb.set_message("");
|
||||
Ok(())
|
||||
}
|
77
src/main.rs
77
src/main.rs
@ -2,6 +2,7 @@ mod config;
|
||||
mod github;
|
||||
mod global;
|
||||
mod http;
|
||||
mod http_async;
|
||||
mod iw4x;
|
||||
mod misc;
|
||||
mod self_update;
|
||||
@ -13,6 +14,8 @@ use structs::*;
|
||||
use colored::*;
|
||||
#[cfg(windows)]
|
||||
use mslnk::ShellLink;
|
||||
|
||||
use indicatif::ProgressBar;
|
||||
use std::{borrow::Cow, collections::HashMap, fs, path::Path, path::PathBuf};
|
||||
#[cfg(windows)]
|
||||
use steamlocate::SteamDir;
|
||||
@ -94,14 +97,14 @@ fn setup_desktop_links(path: &Path, game: &Game) {
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn auto_install(path: &Path, game: &Game) {
|
||||
async fn auto_install(path: &Path, game: &Game<'_>) {
|
||||
setup_client_links(game, path);
|
||||
setup_desktop_links(path, game);
|
||||
update(game, path, false, false);
|
||||
update(game, path, false, false).await;
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn windows_launcher_install(games: &Vec<Game>) {
|
||||
async fn windows_launcher_install(games: &Vec<Game<'_>>) {
|
||||
println!(
|
||||
"{}",
|
||||
"No game specified/found. Checking for installed Steam games..".yellow()
|
||||
@ -115,7 +118,7 @@ fn windows_launcher_install(games: &Vec<Game>) {
|
||||
println!("Found game in current directory.");
|
||||
println!("Installing AlterWare client for {}.", id);
|
||||
let game = games.iter().find(|&g| g.app_id == *id).unwrap();
|
||||
auto_install(path, game);
|
||||
auto_install(path, game).await;
|
||||
println!("Installation complete. Please run the launcher again or use a shortcut to launch the game.");
|
||||
std::io::stdin().read_line(&mut String::new()).unwrap();
|
||||
std::process::exit(0);
|
||||
@ -132,7 +135,7 @@ fn windows_launcher_install(games: &Vec<Game>) {
|
||||
let input: u32 = misc::stdin().parse().unwrap();
|
||||
|
||||
if input == 0 {
|
||||
return manual_install(games);
|
||||
return manual_install(games).await;
|
||||
}
|
||||
|
||||
for (id, path) in installed_games.iter() {
|
||||
@ -146,7 +149,7 @@ fn windows_launcher_install(games: &Vec<Game>) {
|
||||
fs::copy(launcher_path, target_path).unwrap();
|
||||
println!("Launcher copied to {}", path.display());
|
||||
}
|
||||
auto_install(path, game);
|
||||
auto_install(path, game).await;
|
||||
println!("Installation complete. Please run the launcher again or use a shortcut to launch the game.");
|
||||
std::io::stdin().read_line(&mut String::new()).unwrap();
|
||||
break;
|
||||
@ -154,7 +157,7 @@ fn windows_launcher_install(games: &Vec<Game>) {
|
||||
}
|
||||
std::process::exit(0);
|
||||
} else {
|
||||
manual_install(games);
|
||||
manual_install(games).await;
|
||||
}
|
||||
}
|
||||
|
||||
@ -171,10 +174,10 @@ fn prompt_client_selection(games: &[Game]) -> String {
|
||||
String::from(games[input].client[0])
|
||||
}
|
||||
|
||||
fn manual_install(games: &[Game]) {
|
||||
async fn manual_install(games: &[Game<'_>]) {
|
||||
let selection = prompt_client_selection(games);
|
||||
let game = games.iter().find(|&g| g.client[0] == selection).unwrap();
|
||||
update(game, &std::env::current_dir().unwrap(), false, false);
|
||||
update(game, &std::env::current_dir().unwrap(), false, false).await;
|
||||
println!("Installation complete. Please run the launcher again or use a shortcut to launch the game.");
|
||||
std::io::stdin().read_line(&mut String::new()).unwrap();
|
||||
std::process::exit(0);
|
||||
@ -192,12 +195,15 @@ fn total_download_size(cdn_info: &Vec<CdnFile>, remote_dir: &str) -> u64 {
|
||||
size
|
||||
}
|
||||
|
||||
fn update_dir(
|
||||
async fn update_dir(
|
||||
cdn_info: &Vec<CdnFile>,
|
||||
remote_dir: &str,
|
||||
dir: &Path,
|
||||
hashes: &mut HashMap<String, String>,
|
||||
pb: &ProgressBar,
|
||||
) {
|
||||
misc::pb_style_download(pb, false);
|
||||
|
||||
let remote_dir_pre = format!("{}/", remote_dir);
|
||||
|
||||
let mut files_to_download: Vec<CdnFile> = vec![];
|
||||
@ -220,7 +226,11 @@ fn update_dir(
|
||||
if sha1_local != sha1_remote {
|
||||
files_to_download.push(file.clone());
|
||||
} else {
|
||||
println!("[{}] {}", "Checked".bright_blue(), file_path.display());
|
||||
pb.println(format!(
|
||||
"[{}] {}",
|
||||
"Checked".bright_blue(),
|
||||
misc::cute_path(&file_path)
|
||||
));
|
||||
}
|
||||
} else {
|
||||
files_to_download.push(file.clone());
|
||||
@ -228,39 +238,45 @@ fn update_dir(
|
||||
}
|
||||
|
||||
if files_to_download.is_empty() {
|
||||
println!(
|
||||
pb.println(format!(
|
||||
"[{}] No files to download for {}",
|
||||
"Info".bright_magenta(),
|
||||
remote_dir
|
||||
);
|
||||
));
|
||||
return;
|
||||
}
|
||||
println!(
|
||||
pb.println(format!(
|
||||
"[{}] Downloading outdated or missing files for {}, {}",
|
||||
"Info".bright_magenta(),
|
||||
remote_dir,
|
||||
misc::human_readable_bytes(total_download_size(&files_to_download, remote_dir))
|
||||
);
|
||||
));
|
||||
|
||||
misc::pb_style_download(pb, true);
|
||||
let client = reqwest::Client::new();
|
||||
for file in files_to_download {
|
||||
let file_name = &file.name.replace(&remote_dir_pre, "");
|
||||
let file_path = dir.join(file_name);
|
||||
println!(
|
||||
"[{}] {} ({})",
|
||||
"Downloading".bright_yellow(),
|
||||
file_path.display(),
|
||||
misc::human_readable_bytes(file.size as u64)
|
||||
);
|
||||
if let Some(parent) = file_path.parent() {
|
||||
if !parent.exists() {
|
||||
fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
}
|
||||
http::download_file(&format!("{}/{}", MASTER, file.name), &file_path);
|
||||
http_async::download_file(
|
||||
&client,
|
||||
pb,
|
||||
&format!("{}/{}", MASTER, file.name),
|
||||
&file_path,
|
||||
file.size as u64,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
hashes.insert(file_name.to_owned(), file.hash.to_lowercase());
|
||||
}
|
||||
misc::pb_style_download(pb, false);
|
||||
}
|
||||
|
||||
fn update(game: &Game, dir: &Path, bonus_content: bool, force: bool) {
|
||||
async fn update(game: &Game<'_>, dir: &Path, bonus_content: bool, force: bool) {
|
||||
let cdn_info: Vec<CdnFile> = serde_json::from_str(&http::get_body_string(
|
||||
format!("{}/files.json", MASTER).as_str(),
|
||||
))
|
||||
@ -278,7 +294,8 @@ fn update(game: &Game, dir: &Path, bonus_content: bool, force: bool) {
|
||||
}
|
||||
}
|
||||
|
||||
update_dir(&cdn_info, game.engine, dir, &mut hashes);
|
||||
let pb = ProgressBar::new(0);
|
||||
update_dir(&cdn_info, game.engine, dir, &mut hashes, &pb).await;
|
||||
|
||||
if game.engine == "iw4" {
|
||||
iw4x::update(dir);
|
||||
@ -286,10 +303,12 @@ fn update(game: &Game, dir: &Path, bonus_content: bool, force: bool) {
|
||||
|
||||
if bonus_content && !game.bonus.is_empty() {
|
||||
for bonus in game.bonus.iter() {
|
||||
update_dir(&cdn_info, bonus, dir, &mut hashes);
|
||||
update_dir(&cdn_info, bonus, dir, &mut hashes, &pb).await;
|
||||
}
|
||||
}
|
||||
|
||||
pb.finish();
|
||||
|
||||
let mut hash_file_content = String::new();
|
||||
for (file, hash) in hashes.iter() {
|
||||
hash_file_content.push_str(&format!("{} {}\n", hash, file));
|
||||
@ -338,7 +357,8 @@ fn arg_remove_value(args: &mut Vec<String>, arg: &str) {
|
||||
};
|
||||
}
|
||||
|
||||
fn main() {
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
#[cfg(windows)]
|
||||
setup_env();
|
||||
|
||||
@ -484,7 +504,8 @@ fn main() {
|
||||
install_path.as_path(),
|
||||
cfg.download_bonus_content,
|
||||
cfg.force_update,
|
||||
);
|
||||
)
|
||||
.await;
|
||||
if !cfg.update_only {
|
||||
launch(&install_path.join(format!("{}.exe", c)), &cfg.args);
|
||||
}
|
||||
@ -494,7 +515,7 @@ fn main() {
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
windows_launcher_install(&games);
|
||||
windows_launcher_install(&games).await;
|
||||
|
||||
#[cfg(not(windows))]
|
||||
manual_install(&games);
|
||||
|
23
src/misc.rs
23
src/misc.rs
@ -1,6 +1,10 @@
|
||||
use std::{fs, path::PathBuf};
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use colored::Colorize;
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
|
||||
pub fn get_file_sha1(path: &PathBuf) -> String {
|
||||
let mut sha1 = sha1_smol::Sha1::new();
|
||||
@ -35,5 +39,20 @@ pub fn human_readable_bytes(bytes: u64) -> String {
|
||||
bytes /= 1024.0;
|
||||
i += 1;
|
||||
}
|
||||
format!("{:.2} {}", bytes, units[i])
|
||||
format!("{:.2}{}", bytes, units[i])
|
||||
}
|
||||
|
||||
pub fn pb_style_download(pb: &ProgressBar, state: bool) {
|
||||
if state {
|
||||
pb.set_style(
|
||||
ProgressStyle::with_template("{spinner:.magenta} {msg:.magenta} > {bytes}/{total_bytes} | {bytes_per_sec} | {eta}")
|
||||
.unwrap(),
|
||||
);
|
||||
} else {
|
||||
pb.set_style(ProgressStyle::with_template("{spinner:.magenta} {msg}").unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cute_path(path: &Path) -> String {
|
||||
path.to_str().unwrap().replace('\\', "/")
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user