Optimize moddownload

deps/protobuf

@@ -1 +1 @@
-Subproject commit 74638a253ea76baa0aed78491b92d157326295a6
+Subproject commit 14e74f6a21f2726d25e0e679c59d569f6bc8fe8e

src/Components/Modules/Download.cpp

@@ -11,11 +11,11 @@ namespace Components
 	{
 		if (Download::CLDownload.Running) return;
 
-		Download::CLDownload.Mutex.lock();
+		Localization::SetTemp("MPUI_EST_TIME_LEFT", Utils::String::FormatTimeSpan(0));
-		Download::CLDownload.Progress = "Downloading mod";
+		Localization::SetTemp("MPUI_PROGRESS_DL", "(0/0) %");
-		Download::CLDownload.Mutex.unlock();
+		Localization::SetTemp("MPUI_TRANS_RATE", "0.0 MB/s");
 
-		Command::Execute("openmenu popup_reconnectingtoparty", true);
+		Command::Execute("openmenu mod_download_popmenu", true);
 
 		Download::CLDownload.Running = true;
 		Download::CLDownload.Mod = mod;
@@ -35,6 +35,8 @@ namespace Components
 
 		if (!error.empty() || !listData.is_array()) return false;
 
+		download->TotalBytes = 0;
+
 		for (auto& file : listData.array_items())
 		{
 			if (!file.is_object()) return false;
@@ -53,6 +55,7 @@ namespace Components
 			if (!fileEntry.Name.empty())
 			{
 				download->Files.push_back(fileEntry);
+				download->TotalBytes += fileEntry.Size;
 			}
 		}
 
@@ -75,12 +78,32 @@ namespace Components
 
 		if (ev == MG_EV_RECV)
 		{
-			fDownload->receivedBytes += static_cast<size_t>(*reinterpret_cast<int*>(ev_data));
+			size_t bytes = static_cast<size_t>(*reinterpret_cast<int*>(ev_data));
+			fDownload->receivedBytes += bytes;
+			fDownload->download->DownBytes += bytes;
+			fDownload->download->TimeStampBytes += bytes;
 
-			double progress = (100.0 / fDownload->file.Size) * fDownload->receivedBytes;
+			double progress = (100.0 / fDownload->download->TotalBytes) * fDownload->download->DownBytes;
-			fDownload->download->Mutex.lock();
+			Localization::SetTemp("MPUI_PROGRESS_DL", fmt::sprintf("(%d/%d) %d%%", fDownload->index + 1, fDownload->download->Files.size(), static_cast<unsigned int>(progress)));
-			fDownload->download->Progress = fmt::sprintf("Downloading file (%d/%d) %s %d%%", fDownload->index + 1, fDownload->download->Files.size(), fDownload->file.Name.data(), static_cast<unsigned int>(progress));
+
-			fDownload->download->Mutex.unlock();
+			int delta = Game::Sys_Milliseconds() - fDownload->download->LastTimeStamp;
+			if (delta > 300)
+			{
+				bool doFormat = fDownload->download->LastTimeStamp != 0;
+				fDownload->download->LastTimeStamp = Game::Sys_Milliseconds();
+
+				size_t dataLeft = fDownload->download->TotalBytes - fDownload->download->DownBytes;
+				double timeLeftD = ((1.0 * dataLeft) / fDownload->download->TimeStampBytes) * delta;
+				int timeLeft = static_cast<int>(timeLeftD);
+
+				if (doFormat)
+				{
+					Localization::SetTemp("MPUI_EST_TIME_LEFT", Utils::String::FormatTimeSpan(timeLeft));
+					Localization::SetTemp("MPUI_TRANS_RATE", Utils::String::FormatBandwidth(fDownload->download->TimeStampBytes, delta));
+				}
+
+				fDownload->download->TimeStampBytes = 0;
+			}
 		}
 
 		if (ev == MG_EV_HTTP_REPLY)
@@ -98,10 +121,6 @@ namespace Components
 
 		auto file = download->Files[index];
 
-		download->Mutex.lock();
-		download->Progress = fmt::sprintf("Downloading file (%d/%d) %s 0%%", index + 1, download->Files.size(), file.Name.data());
-		download->Mutex.unlock();
-
 		std::string path = download->Mod + "/" + file.Name;
 		if (Utils::IO::FileExists(path))
 		{
@@ -109,6 +128,7 @@ namespace Components
 
 			if (data.size() == file.Size && Utils::String::DumpHex(Utils::Cryptography::SHA256::Compute(data), "") == file.Hash)
 			{
+				download->TotalBytes += file.Size;
 				return true;
 			}
 		}
@@ -122,6 +142,8 @@ namespace Components
 		fDownload.downloading = true;
 		fDownload.receivedBytes = 0;
 
+		Utils::String::Replace(url, " ", "%20");
+
 		download->Valid = true;
 		mg_mgr_init(&download->Mgr, &fDownload);
 		mg_connect_http(&download->Mgr, Download::DownloadHandler, url.data(), NULL, NULL);
@@ -154,6 +176,8 @@ namespace Components
 
 		if (list.empty())
 		{
+			if (download->TerminateThread) return;
+
 			download->Thread.detach();
 			download->Clear();
 
@@ -169,6 +193,8 @@ namespace Components
 
 		if (!Download::ParseModList(download, list))
 		{
+			if (download->TerminateThread) return;
+
 			download->Thread.detach();
 			download->Clear();
 
@@ -182,21 +208,27 @@ namespace Components
 
 		if (download->TerminateThread) return;
 
+		static std::string mod = download->Mod;
+
 		for (unsigned int i = 0; i < download->Files.size(); ++i)
 		{
 			if (download->TerminateThread) return;
 
 			if (!Download::DownloadFile(download, i))
 			{
-				Dvar::Var("partyend_reason").Set(fmt::sprintf("Failed to download file: %s!", download->Files[i].Name.data()));
+				if (download->TerminateThread) return;
 
+				mod = fmt::sprintf("Failed to download file: %s!", download->Files[i].Name.data());
 				download->Thread.detach();
 				download->Clear();
 
 				QuickPatch::Once([] ()
 				{
+					Dvar::Var("partyend_reason").Set(mod);
+					mod.clear();
+
 					Localization::ClearTemp();
-					Command::Execute("closemenu popup_reconnectingtoparty");
+					Command::Execute("closemenu mod_download_popmenu");
 					Command::Execute("openmenu menu_xboxlive_partyended");
 				});
 
@@ -206,8 +238,6 @@ namespace Components
 
 		if (download->TerminateThread) return;
 
-		static std::string mod = download->Mod;
-
 		download->Thread.detach();
 		download->Clear();
 		
@@ -221,7 +251,7 @@ namespace Components
 			mod.clear();
 
 			Localization::ClearTemp();
-			Command::Execute("closemenu popup_reconnectingtoparty", true);
+			Command::Execute("closemenu mod_download_popmenu", true);
 
 			if (Dvar::Var("cl_modVidRestart").Get<bool>())
 			{
@@ -346,6 +376,8 @@ namespace Components
 			Utils::String::Replace(url, "\\", "/");
 			url = url.substr(6);
 
+			Utils::String::Replace(url, "%20", " ");
+
 			if (url.find_first_of("/") != std::string::npos || (!Utils::String::EndsWith(url, ".iwd") && url != "mod.ff") || strstr(url.data(), "_svr_") != NULL)
 			{
 				Download::Forbid(nc);
@@ -542,14 +574,9 @@ namespace Components
 		}
 		else
 		{
-			QuickPatch::OnFrame([]
+			UIScript::Add("mod_download_cancel", [] ()
 			{
-				if (Download::CLDownload.Running)
+				Download::CLDownload.Clear();
-				{
-					Download::CLDownload.Mutex.lock();
-					Localization::SetTemp("MENU_RECONNECTING_TO_PARTY", Download::CLDownload.Progress);
-					Download::CLDownload.Mutex.unlock();
-				}
 			});
 		}
 	}

src/Components/Modules/Download.hpp

@@ -16,7 +16,7 @@ namespace Components
 		class ClientDownload
 		{
 		public:
-			ClientDownload() : Valid(false), Running(false), TerminateThread(false) {}
+			ClientDownload() : Valid(false), Running(false), TerminateThread(false), TotalBytes(0), DownBytes(0), LastTimeStamp(0), TimeStampBytes(0) {}
 			~ClientDownload() { this->Clear(); }
 
 			bool Running;
@@ -25,10 +25,13 @@ namespace Components
 			mg_mgr Mgr;
 			Network::Address Target;
 			std::string Mod;
-			std::mutex Mutex;
 			std::thread Thread;
-			std::string Progress;
 
+			size_t TotalBytes;
+			size_t DownBytes;
+
+			int LastTimeStamp;
+			size_t TimeStampBytes;
 
 			class File
 			{
@@ -58,10 +61,6 @@ namespace Components
 					this->Valid = false;
 					mg_mgr_free(&(this->Mgr));
 				}
-
-				this->Mutex.lock();
-				this->Progress.clear();
-				this->Mutex.unlock();
 			}
 		};
 
@@ -71,10 +70,13 @@ namespace Components
 			ClientDownload* download;
 			ClientDownload::File file;
 
+			int timestamp;
 			bool downloading;
 			unsigned int index;
 			std::string buffer;
 			size_t receivedBytes;
+
+
 		};
 
 		static mg_mgr Mgr;

src/Components/Modules/Localization.cpp

@@ -2,6 +2,7 @@
 
 namespace Components
 {
+	std::mutex Localization::LocalizeMutex;
 	Dvar::Var Localization::UseLocalization;
 	Utils::Memory::Allocator Localization::MemAllocator;
 	std::map<std::string, Game::LocalizedEntry*> Localization::LocalizeMap;
@@ -9,6 +10,8 @@ namespace Components
 
 	void Localization::Set(std::string key, std::string value)
 	{
+		std::lock_guard<std::mutex> _(Localization::LocalizeMutex);
+
 		if (Localization::LocalizeMap.find(key) != Localization::LocalizeMap.end())
 		{
 			Game::LocalizedEntry* entry = Localization::LocalizeMap[key];
@@ -46,6 +49,7 @@ namespace Components
 		if (!Localization::UseLocalization.Get<bool>()) return key;
 
 		Game::LocalizedEntry* entry = nullptr;
+		std::lock_guard<std::mutex> _(Localization::LocalizeMutex);
 
 		if (Localization::TempLocalizeMap.find(key) != Localization::TempLocalizeMap.end())
 		{
@@ -56,7 +60,12 @@ namespace Components
 			entry = Localization::LocalizeMap[key];
 		}
 
-		if (!entry || !entry->value) entry = Game::DB_FindXAssetHeader(Game::XAssetType::ASSET_TYPE_LOCALIZE, key).localize;
+		if (!entry || !entry->value)
+		{
+			Localization::LocalizeMutex.unlock();
+			entry = Game::DB_FindXAssetHeader(Game::XAssetType::ASSET_TYPE_LOCALIZE, key).localize;
+			Localization::LocalizeMutex.lock();
+		}
 
 		if (entry && entry->value)
 		{
@@ -68,6 +77,8 @@ namespace Components
 
 	void Localization::SetTemp(std::string key, std::string value)
 	{
+		std::lock_guard<std::mutex> _(Localization::LocalizeMutex);
+
 		if (Localization::TempLocalizeMap.find(key) != Localization::TempLocalizeMap.end())
 		{
 			Game::LocalizedEntry* entry = Localization::TempLocalizeMap[key];
@@ -100,6 +111,8 @@ namespace Components
 
 	void Localization::ClearTemp()
 	{
+		std::lock_guard<std::mutex> _(Localization::LocalizeMutex);
+
 		for (auto i = Localization::TempLocalizeMap.begin(); i != Localization::TempLocalizeMap.end(); ++i)
 		{
 			if (i->second)
@@ -131,6 +144,7 @@ namespace Components
 		AssetHandler::OnFind(Game::XAssetType::ASSET_TYPE_LOCALIZE, [] (Game::XAssetType, std::string filename)
 		{
 			Game::XAssetHeader header = { 0 };
+			std::lock_guard<std::mutex> _(Localization::LocalizeMutex);
 
 			if (Localization::TempLocalizeMap.find(filename) != Localization::TempLocalizeMap.end())
 			{

src/Components/Modules/Localization.hpp

@@ -17,6 +17,7 @@ namespace Components
 		static void ClearTemp();
 
 	private:
+		static std::mutex LocalizeMutex;
 		static Utils::Memory::Allocator MemAllocator;
 		static std::map<std::string, Game::LocalizedEntry*> LocalizeMap;
 		static std::map<std::string, Game::LocalizedEntry*> TempLocalizeMap;

src/Components/Modules/Menus.cpp

@@ -674,6 +674,7 @@ namespace Components
 		Menus::Add("ui_mp/stats_reset.menu");
 		Menus::Add("ui_mp/stats_unlock.menu");
 		Menus::Add("ui_mp/security_increase_popmenu.menu");
+		Menus::Add("ui_mp/mod_download_popmenu.menu");
 	}
 
 	Menus::~Menus()

src/Utils/String.cpp

@@ -119,6 +119,28 @@ namespace Utils
 			return fmt::sprintf("%02d:%02d:%02d", hoursTotal, minutes, seconds);
 		}
 
+		std::string FormatBandwidth(size_t bytes, int milliseconds)
+		{
+			static char* sizes[] = 
+			{
+				"B",
+				"KB",
+				"MB",
+				"GB",
+				"TB"
+			};
+
+			double bytesPerSecond = (1000.0 / milliseconds) * bytes;
+
+			int i = 0;
+			for (i = 0; bytesPerSecond > 1000 && i < ARRAY_SIZE(sizes); ++i) // 1024 or 1000?
+			{
+				bytesPerSecond /= 1000;
+			}
+
+			return fmt::sprintf("%.2f %s/s", static_cast<float>(bytesPerSecond), sizes[i]);
+		}
+
 		// Encodes a given string in Base64
 		std::string EncodeBase64(const char* input, const unsigned long inputSize) 
 		{

src/Utils/String.hpp

@@ -35,6 +35,7 @@ namespace Utils
 		std::string &Trim(std::string &s);
 
 		std::string FormatTimeSpan(int milliseconds);
+		std::string FormatBandwidth(size_t bytes, int milliseconds);
 
 		std::string DumpHex(std::string data, std::string separator = " ");