stream files to disk

- Greatly reduced memory usage while downloading
- Added progressbar, download rate
- Added async runtime
- Tweaks to allow usage of async download function

Cargo.toml

@@ -23,8 +23,11 @@ serde = { version = "1.0.189", features = ["derive"] }
 serde_json = "1.0.107"
 rand = "0.8.5"
 semver = "1.0.20"
-zip = "0.6.6"
 colored = "2.0.4"
+reqwest = { version = "0.11.22", features = ["stream"] }
+futures-util = "0.3.29"
+indicatif = "0.17.7"
+tokio = {version="1.33.0", features = ["rt-multi-thread", "macros"]}
 
 [target.'cfg(windows)'.dependencies]
 steamlocate = "2.0.0-alpha.0"

src/http_async.rs

@@ -0,0 +1,52 @@
+use std::cmp::min;
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+
+use colored::*;
+use futures_util::StreamExt;
+use indicatif::ProgressBar;
+use reqwest::Client;
+
+use crate::misc;
+
+pub async fn download_file(
+    client: &Client,
+    pb: &ProgressBar,
+    url: &str,
+    path: &PathBuf,
+    size: u64,
+) -> Result<(), String> {
+    let res = client
+        .get(url)
+        .send()
+        .await
+        .or(Err(format!("Failed to GET from '{}'", &url)))?;
+    // Fix for CF shenanigans
+    let total_size = res.content_length().unwrap_or(size);
+    pb.set_length(total_size);
+    pb.println(format!(
+        "[{}] {} ({})",
+        "Downloading".bright_yellow(),
+        misc::cute_path(path),
+        misc::human_readable_bytes(total_size)
+    ));
+    pb.set_message(misc::cute_path(path));
+
+    let mut file =
+        File::create(path).or(Err(format!("Failed to create file '{}'", path.display())))?;
+    let mut downloaded: u64 = 0;
+    let mut stream = res.bytes_stream();
+
+    while let Some(item) = stream.next().await {
+        let chunk = item.or(Err("Error while downloading file"))?;
+        file.write_all(&chunk)
+            .or(Err("Error while writing to file"))?;
+        let new = min(downloaded + (chunk.len() as u64), total_size);
+        downloaded = new;
+        pb.set_position(new);
+    }
+
+    pb.set_message("");
+    Ok(())
+}

src/main.rs

@@ -2,6 +2,7 @@ mod config;
 mod github;
 mod global;
 mod http;
+mod http_async;
 mod iw4x;
 mod misc;
 mod self_update;
@@ -13,6 +14,8 @@ use structs::*;
 use colored::*;
 #[cfg(windows)]
 use mslnk::ShellLink;
+
+use indicatif::ProgressBar;
 use std::{borrow::Cow, collections::HashMap, fs, path::Path, path::PathBuf};
 #[cfg(windows)]
 use steamlocate::SteamDir;
@@ -94,14 +97,14 @@ fn setup_desktop_links(path: &Path, game: &Game) {
 }
 
 #[cfg(windows)]
-fn auto_install(path: &Path, game: &Game) {
+async fn auto_install(path: &Path, game: &Game<'_>) {
     setup_client_links(game, path);
     setup_desktop_links(path, game);
-    update(game, path, false, false);
+    update(game, path, false, false).await;
 }
 
 #[cfg(windows)]
-fn windows_launcher_install(games: &Vec<Game>) {
+async fn windows_launcher_install(games: &Vec<Game<'_>>) {
     println!(
         "{}",
         "No game specified/found. Checking for installed Steam games..".yellow()
@@ -115,7 +118,7 @@ fn windows_launcher_install(games: &Vec<Game>) {
                 println!("Found game in current directory.");
                 println!("Installing AlterWare client for {}.", id);
                 let game = games.iter().find(|&g| g.app_id == *id).unwrap();
-                auto_install(path, game);
+                auto_install(path, game).await;
                 println!("Installation complete. Please run the launcher again or use a shortcut to launch the game.");
                 std::io::stdin().read_line(&mut String::new()).unwrap();
                 std::process::exit(0);
@@ -132,7 +135,7 @@ fn windows_launcher_install(games: &Vec<Game>) {
         let input: u32 = misc::stdin().parse().unwrap();
 
         if input == 0 {
-            return manual_install(games);
+            return manual_install(games).await;
         }
 
         for (id, path) in installed_games.iter() {
@@ -146,7 +149,7 @@ fn windows_launcher_install(games: &Vec<Game>) {
                     fs::copy(launcher_path, target_path).unwrap();
                     println!("Launcher copied to {}", path.display());
                 }
-                auto_install(path, game);
+                auto_install(path, game).await;
                 println!("Installation complete. Please run the launcher again or use a shortcut to launch the game.");
                 std::io::stdin().read_line(&mut String::new()).unwrap();
                 break;
@@ -154,7 +157,7 @@ fn windows_launcher_install(games: &Vec<Game>) {
         }
         std::process::exit(0);
     } else {
-        manual_install(games);
+        manual_install(games).await;
     }
 }
 
@@ -171,10 +174,10 @@ fn prompt_client_selection(games: &[Game]) -> String {
     String::from(games[input].client[0])
 }
 
-fn manual_install(games: &[Game]) {
+async fn manual_install(games: &[Game<'_>]) {
     let selection = prompt_client_selection(games);
     let game = games.iter().find(|&g| g.client[0] == selection).unwrap();
-    update(game, &std::env::current_dir().unwrap(), false, false);
+    update(game, &std::env::current_dir().unwrap(), false, false).await;
     println!("Installation complete. Please run the launcher again or use a shortcut to launch the game.");
     std::io::stdin().read_line(&mut String::new()).unwrap();
     std::process::exit(0);
@@ -192,12 +195,15 @@ fn total_download_size(cdn_info: &Vec<CdnFile>, remote_dir: &str) -> u64 {
     size
 }
 
-fn update_dir(
+async fn update_dir(
     cdn_info: &Vec<CdnFile>,
     remote_dir: &str,
     dir: &Path,
     hashes: &mut HashMap<String, String>,
+    pb: &ProgressBar,
 ) {
+    misc::pb_style_download(pb, false);
+
     let remote_dir_pre = format!("{}/", remote_dir);
 
     let mut files_to_download: Vec<CdnFile> = vec![];
@@ -220,7 +226,11 @@ fn update_dir(
             if sha1_local != sha1_remote {
                 files_to_download.push(file.clone());
             } else {
-                println!("[{}]     {}", "Checked".bright_blue(), file_path.display());
+                pb.println(format!(
+                    "[{}]     {}",
+                    "Checked".bright_blue(),
+                    misc::cute_path(&file_path)
+                ));
             }
         } else {
             files_to_download.push(file.clone());
@@ -228,39 +238,45 @@ fn update_dir(
     }
 
     if files_to_download.is_empty() {
-        println!(
+        pb.println(format!(
             "[{}]        No files to download for {}",
             "Info".bright_magenta(),
             remote_dir
-        );
+        ));
         return;
     }
-    println!(
+    pb.println(format!(
         "[{}]        Downloading outdated or missing files for {}, {}",
         "Info".bright_magenta(),
         remote_dir,
         misc::human_readable_bytes(total_download_size(&files_to_download, remote_dir))
-    );
+    ));
+
+    misc::pb_style_download(pb, true);
+    let client = reqwest::Client::new();
     for file in files_to_download {
         let file_name = &file.name.replace(&remote_dir_pre, "");
         let file_path = dir.join(file_name);
-        println!(
-            "[{}] {} ({})",
-            "Downloading".bright_yellow(),
-            file_path.display(),
-            misc::human_readable_bytes(file.size as u64)
-        );
         if let Some(parent) = file_path.parent() {
             if !parent.exists() {
                 fs::create_dir_all(parent).unwrap();
             }
         }
-        http::download_file(&format!("{}/{}", MASTER, file.name), &file_path);
+        http_async::download_file(
+            &client,
+            pb,
+            &format!("{}/{}", MASTER, file.name),
+            &file_path,
+            file.size as u64,
+        )
+        .await
+        .unwrap();
         hashes.insert(file_name.to_owned(), file.hash.to_lowercase());
     }
+    misc::pb_style_download(pb, false);
 }
 
-fn update(game: &Game, dir: &Path, bonus_content: bool, force: bool) {
+async fn update(game: &Game<'_>, dir: &Path, bonus_content: bool, force: bool) {
     let cdn_info: Vec<CdnFile> = serde_json::from_str(&http::get_body_string(
         format!("{}/files.json", MASTER).as_str(),
     ))
@@ -278,7 +294,8 @@ fn update(game: &Game, dir: &Path, bonus_content: bool, force: bool) {
         }
     }
 
-    update_dir(&cdn_info, game.engine, dir, &mut hashes);
+    let pb = ProgressBar::new(0);
+    update_dir(&cdn_info, game.engine, dir, &mut hashes, &pb).await;
 
     if game.engine == "iw4" {
         iw4x::update(dir);
@@ -286,10 +303,12 @@ fn update(game: &Game, dir: &Path, bonus_content: bool, force: bool) {
 
     if bonus_content && !game.bonus.is_empty() {
         for bonus in game.bonus.iter() {
-            update_dir(&cdn_info, bonus, dir, &mut hashes);
+            update_dir(&cdn_info, bonus, dir, &mut hashes, &pb).await;
         }
     }
 
+    pb.finish();
+
     let mut hash_file_content = String::new();
     for (file, hash) in hashes.iter() {
         hash_file_content.push_str(&format!("{} {}\n", hash, file));
@@ -338,7 +357,8 @@ fn arg_remove_value(args: &mut Vec<String>, arg: &str) {
     };
 }
 
-fn main() {
+#[tokio::main]
+async fn main() {
     #[cfg(windows)]
     setup_env();
 
@@ -484,7 +504,8 @@ fn main() {
                     install_path.as_path(),
                     cfg.download_bonus_content,
                     cfg.force_update,
-                );
+                )
+                .await;
                 if !cfg.update_only {
                     launch(&install_path.join(format!("{}.exe", c)), &cfg.args);
                 }
@@ -494,7 +515,7 @@ fn main() {
     }
 
     #[cfg(windows)]
-    windows_launcher_install(&games);
+    windows_launcher_install(&games).await;
 
     #[cfg(not(windows))]
     manual_install(&games);

src/misc.rs

@@ -1,6 +1,10 @@
-use std::{fs, path::PathBuf};
+use std::{
+    fs,
+    path::{Path, PathBuf},
+};
 
 use colored::Colorize;
+use indicatif::{ProgressBar, ProgressStyle};
 
 pub fn get_file_sha1(path: &PathBuf) -> String {
     let mut sha1 = sha1_smol::Sha1::new();
@@ -35,5 +39,20 @@ pub fn human_readable_bytes(bytes: u64) -> String {
         bytes /= 1024.0;
         i += 1;
     }
-    format!("{:.2} {}", bytes, units[i])
+    format!("{:.2}{}", bytes, units[i])
+}
+
+pub fn pb_style_download(pb: &ProgressBar, state: bool) {
+    if state {
+        pb.set_style(
+            ProgressStyle::with_template("{spinner:.magenta} {msg:.magenta} > {bytes}/{total_bytes} | {bytes_per_sec} | {eta}")
+                .unwrap(),
+        );
+    } else {
+        pb.set_style(ProgressStyle::with_template("{spinner:.magenta} {msg}").unwrap());
+    }
+}
+
+pub fn cute_path(path: &Path) -> String {
+    path.to_str().unwrap().replace('\\', "/")
 }