Merge pull request #13 from faroukbmiled/refactored

From bs4 to API for item updater

boiiiwd_package/info.json

@@ -0,0 +1,34 @@
+{
+    "response": {
+        "result": 1,
+        "resultcount": 1,
+        "publishedfiledetails": [
+            {
+                "publishedfileid": "770675911",
+                "result": 1,
+                "creator": "76561198294963547",
+                "creator_app_id": 455130,
+                "consumer_app_id": 311210,
+                "filename": "",
+                "file_size": 0,
+                "file_url": "",
+                "hcontent_file": "3845203201590843337",
+                "preview_url": "https://steamuserimages-a.akamaihd.net/ugc/262722490466018040/361198F0A8D4F1479CD8F75E7A4A78354592EAF3/",
+                "hcontent_preview": "262722490466018040",
+                "title": "Zombies Life Timer",
+                "description": "On-screen text that tells you how long you've been alive in Zombies. Only works in user-created maps.",
+                "time_created": 1474934901,
+                "time_updated": 1474934901,
+                "visibility": 0,
+                "banned": 0,
+                "ban_reason": "",
+                "subscriptions": 4900,
+                "favorited": 33,
+                "lifetime_subscriptions": 10835,
+                "lifetime_favorited": 46,
+                "views": 9038,
+                "tags": []
+            }
+        ]
+    }
+}

boiiiwd_package/src/helpers.py

@@ -25,8 +25,8 @@ def save_config(name, value):
         config.write(config_file)
 
 def check_custom_theme(theme_name):
-    if os.path.exists(os.path.join(application_path, theme_name)):
-        return os.path.join(application_path, theme_name)
+    if os.path.exists(os.path.join(APPLICATION_PATH, theme_name)):
+        return os.path.join(APPLICATION_PATH, theme_name)
     else:
         try: return os.path.join(RESOURCES_DIR, theme_name)
         except: return os.path.join(RESOURCES_DIR, "boiiiwd_theme.json")
@@ -171,20 +171,10 @@ def initialize_steam(master):
 
 @if_internet_available
 def valid_id(workshop_id):
-    url = f"https://steamcommunity.com/sharedfiles/filedetails/?id={workshop_id}"
-    response = requests.get(url)
-    response.raise_for_status()
-    content = response.text
-    soup = BeautifulSoup(content, "html.parser")
-
-    try:
-        soup.find("div", class_="rightDetailsBlock").text.strip()
-        soup.find("div", class_="workshopItemTitle").text.strip()
-        soup.find("div", class_="detailsStatRight").text.strip()
-        stars_div = soup.find("div", class_="fileRatingDetails")
-        stars_div.find("img")["src"]
+    data = item_steam_api(workshop_id)
+    if "consumer_app_id" in data['response']['publishedfiledetails'][0]:
         return True
-    except:
+    else:
         return False
 
 def convert_speed(speed_bytes):
@@ -200,7 +190,7 @@ def convert_speed(speed_bytes):
 def create_default_config():
     config = configparser.ConfigParser()
     config["Settings"] = {
-        "SteamCMDPath": application_path,
+        "SteamCMDPath": APPLICATION_PATH,
         "DestinationFolder": "",
         "checkforupdtes": "on",
         "console": "off"
@@ -211,7 +201,7 @@ def create_default_config():
 def get_steamcmd_path():
     config = configparser.ConfigParser()
     config.read(CONFIG_FILE_PATH)
-    return config.get("Settings", "SteamCMDPath", fallback=application_path)
+    return config.get("Settings", "SteamCMDPath", fallback=APPLICATION_PATH)
 
 def extract_json_data(json_path, key):
     with open(json_path, 'r') as json_file:
@@ -227,26 +217,13 @@ def convert_bytes_to_readable(size_in_bytes, no_symb=None):
         size_in_bytes /= 1024.0
 
 def get_workshop_file_size(workshop_id, raw=None):
-    url = f"https://steamcommunity.com/sharedfiles/filedetails/?id={workshop_id}&searchtext="
-    response = requests.get(url)
-    soup = BeautifulSoup(response.text, "html.parser")
-    file_size_element = soup.find("div", class_="detailsStatRight")
-
+    data = item_steam_api(workshop_id)
     try:
+        file_size_in_bytes = data['response']['publishedfiledetails'][0]['file_size']
         if raw:
-            file_size_text = file_size_element.get_text(strip=True)
-            file_size_text = file_size_text.replace(",", "")
-            file_size_in_mb = float(file_size_text.replace(" MB", ""))
-            file_size_in_bytes = int(file_size_in_mb * 1024 * 1024)
             return convert_bytes_to_readable(file_size_in_bytes)
-
-        if file_size_element:
-            file_size_text = file_size_element.get_text(strip=True)
-            file_size_text = file_size_text.replace(",", "")
-            file_size_in_mb = float(file_size_text.replace(" MB", ""))
-            file_size_in_bytes = int(file_size_in_mb * 1024 * 1024)
+        else:
             return file_size_in_bytes
-        return None
     except:
         return None
 
@@ -351,16 +328,9 @@ def reset_steamcmd(no_warn=None):
 
 def get_item_name(id):
     try:
-        url = f"https://steamcommunity.com/sharedfiles/filedetails/?id={id}"
-        response = requests.get(url)
-        response.raise_for_status()
-        content = response.text
-
-        soup = BeautifulSoup(content, "html.parser")
-
+        data = item_steam_api(id)
         try:
-            map_name = soup.find("div", class_="workshopItemTitle").text.strip()
-            name = map_name[:32] + "..." if len(map_name) > 32 else map_name
+            name = data['response']['publishedfiledetails'][0]['title']
             return name
         except:
             return True
@@ -381,14 +351,9 @@ def check_item_date(down_date, date_updated):
         except ValueError:
             download_datetime = datetime.strptime(down_date + f", {current_year}", date_format_with_added_year)
 
-        try:
-            upload_datetime = datetime.strptime(date_updated, date_format_with_year)
-        except ValueError:
-            upload_datetime = datetime.strptime(date_updated + f", {current_year}", date_format_with_added_year)
-
-        if upload_datetime >= download_datetime:
+        if date_updated >= download_datetime:
             return True
-        elif upload_datetime < download_datetime:
+        elif date_updated < download_datetime:
             return False
     except:
         return False
@@ -414,4 +379,39 @@ def save_window_size_to_registry(width, height, x, y):
     except Exception as e:
         print(f"Error saving to registry: {e}")
 
+def item_steam_api(id):
+    try:
+        url = ITEM_INFO_API
+        data = {
+            "itemcount": 1,
+            "publishedfileids[0]": int(id),
+        }
+        info = requests.post(url, data=data)
+        return info.json()
+
+    except Exception as e:
+        print(e)
+        return False
+
+def get_item_dates(ids):
+    try:
+        data = {
+            "itemcount": len(ids),
+        }
+        for i, id in enumerate(ids):
+            data[f"publishedfileids[{i}]"] = int(id)
+
+        info = requests.post(ITEM_INFO_API, data=data)
+        response_data = info.json()
+
+        if "response" in response_data:
+            item_details = response_data["response"]["publishedfiledetails"]
+            return {item["publishedfileid"]: item["time_updated"] for item in item_details}
+
+        return {}
+
+    except Exception as e:
+        print(e)
+        return {}
+
 # End helper functions

boiiiwd_package/src/imports.py

@@ -1,6 +1,5 @@
 import configparser
 import io
-import json
 import math
 import os
 import re
@@ -11,15 +10,18 @@ import threading
 import time
 import webbrowser
 import zipfile
+
 from datetime import datetime
 from pathlib import Path
 from tkinter import END, Event, Menu
 
 import customtkinter as ctk
+import ujson as json
 import psutil
 import requests
 import winreg
 from bs4 import BeautifulSoup
+
 from CTkMessagebox import CTkMessagebox
 from PIL import Image
 
@@ -32,15 +34,16 @@ if getattr(sys, 'frozen', False):
     # If the application is run as a bundle, the PyInstaller bootloader
     # extends the sys module by a flag frozen=True and sets the app
     # path into variable _MEIPASS'.
-    application_path = os.path.dirname(sys.executable)
+    APPLICATION_PATH = os.path.dirname(sys.executable)
 else:
-    application_path = os.path.dirname(os.path.abspath(__file__))
+    APPLICATION_PATH = os.path.dirname(os.path.abspath(__file__))
 
 CONFIG_FILE_PATH = "config.ini"
 GITHUB_REPO = "faroukbmiled/BOIIIWD"
+ITEM_INFO_API = "https://api.steampowered.com/ISteamRemoteStorage/GetPublishedFileDetails/v1/"
 LATEST_RELEASE_URL = "https://github.com/faroukbmiled/BOIIIWD/releases/latest/download/Release.zip"
 LIBRARY_FILE = "boiiiwd_library.json"
 RESOURCES_DIR = os.path.join(os.path.dirname(__file__), '..', 'resources')
 UPDATER_FOLDER = "update"
 REGISTRY_KEY_PATH = r"Software\BOIIIWD"
-VERSION = "v0.3.1"
+VERSION = "v0.3.2"

boiiiwd_package/src/library_tab.py

@@ -256,7 +256,7 @@ class LibraryTab(ctk.CTkScrollableFrame):
         folders_to_process = [mods_folder, maps_folder]
         ui_items_to_add = []
 
-        items_file = os.path.join(application_path, LIBRARY_FILE)
+        items_file = os.path.join(APPLICATION_PATH, LIBRARY_FILE)
         if not self.is_valid_json_format(items_file):
             try: self.rename_invalid_json_file(items_file)
             except: pass
@@ -409,7 +409,7 @@ class LibraryTab(ctk.CTkScrollableFrame):
                             creation_timestamp = zone_path.stat().st_mtime
                             date_added = datetime.fromtimestamp(creation_timestamp).strftime("%d %b, %Y @ %I:%M%p")
 
-                        items_file = os.path.join(application_path, LIBRARY_FILE)
+                        items_file = os.path.join(APPLICATION_PATH, LIBRARY_FILE)
 
                         item_info = {
                             "id": workshop_id,
@@ -425,7 +425,7 @@ class LibraryTab(ctk.CTkScrollableFrame):
             show_message("Error updating json file", f"Error while updating library json file\n{e}")
 
     def remove_item(self, item, folder, id):
-        items_file = os.path.join(application_path, LIBRARY_FILE)
+        items_file = os.path.join(APPLICATION_PATH, LIBRARY_FILE)
         for label, button, button_view_list in zip(self.label_list, self.button_list, self.button_view_list):
             if item == label.cget("text"):
                 self.added_folders.remove(os.path.basename(folder))
@@ -622,7 +622,7 @@ class LibraryTab(ctk.CTkScrollableFrame):
                              url, workshop_id, invalid_warn, folder, description ,online,offline_date=None):
         def main_thread():
             try:
-                items_file = os.path.join(application_path, LIBRARY_FILE)
+                items_file = os.path.join(APPLICATION_PATH, LIBRARY_FILE)
                 top = ctk.CTkToplevel(self)
                 if os.path.exists(os.path.join(RESOURCES_DIR, "ryuk.ico")):
                     top.after(210, lambda: top.iconbitmap(os.path.join(RESOURCES_DIR, "ryuk.ico")))
@@ -849,51 +849,40 @@ class LibraryTab(ctk.CTkScrollableFrame):
         # Needed to refresh item that needs updates
         self.to_update.clear()
 
-        def if_id_needs_update(item_id, item_date, text):
+        def if_ids_need_update(item_ids, item_dates, texts):
             try:
-                headers = {'Cache-Control': 'no-cache'}
-                url = f"https://steamcommunity.com/sharedfiles/filedetails/?id={item_id}"
-                response = requests.get(url, headers=headers)
-                response.raise_for_status()
-                content = response.text
-                soup = BeautifulSoup(content, "html.parser")
-                details_stats_container = soup.find("div", class_="detailsStatsContainerRight")
-                details_stat_elements = details_stats_container.find_all("div", class_="detailsStatRight")
-                try:
-                    date_updated = details_stat_elements[2].text.strip()
-                except:
-                    try:
-                        date_updated = details_stat_elements[1].text.strip()
-                    except:
-                        return False
+                item_data = get_item_dates(item_ids)
 
-                if check_item_date(item_date, date_updated):
-                    self.to_update.add(text + f" | Updated: {date_updated}")
-                    return True
-                else:
-                    return False
+                for item_id, date_updated in item_data.items():
+                    item_date = item_dates[item_id]
+                    date_updated = datetime.fromtimestamp(date_updated)
+
+                    if check_item_date(item_date, date_updated):
+                        date_updated = date_updated.strftime("%d %b @ %I:%M%p, %Y")
+                        self.to_update.add(texts[item_id] + f" | Updated: {date_updated}")
 
             except Exception as e:
-                show_message("Error", f"Error occured\n{e}", icon="cancel")
-                return
+                show_message("Error", f"Error occurred\n{e}", icon="cancel")
 
         def check_for_update():
             try:
                 lib_data = None
 
-                if not os.path.exists(os.path.join(application_path, LIBRARY_FILE)):
-                    show_message("Error checking for item updates! -> Setting is on", "Please visit library tab at least once with the correct boiii path!, you also need to have at lease 1 item!")
+                if not os.path.exists(os.path.join(APPLICATION_PATH, LIBRARY_FILE)):
+                    show_message("Error checking for item updates! -> Setting is on", "Please visit library tab at least once with the correct boiii path!, you also need to have at least 1 item!")
                     return
 
-                with open(LIBRARY_FILE, 'r') as file:
+                with open(os.path.join(APPLICATION_PATH, LIBRARY_FILE), 'r') as file:
                     lib_data = json.load(file)
 
-                for item in lib_data:
-                    item_id = item["id"]
-                    item_date = item["date"]
-                    if_id_needs_update(item_id, item_date, item["text"])
+                item_ids = [item["id"] for item in lib_data]
+                item_dates = {item["id"]: item["date"] for item in lib_data}
+                texts = {item["id"]: item["text"] for item in lib_data}
+
+                if_ids_need_update(item_ids, item_dates, texts)
+
             except:
-                show_message("Error checking for item updates!", "Please visit library tab at least once with the correct boiii path!, you also need to have at lease 1 item!")
+                show_message("Error checking for item updates!", "Please visit the library tab at least once with the correct boiii path!, you also need to have at least 1 item!")
                 return
 
         check_for_update()
@@ -916,7 +905,8 @@ class LibraryTab(ctk.CTkScrollableFrame):
             top.title("Item updater - List of Items with Updates - Click to select 1 or more")
             longest_text_length = max(len(text) for text in self.to_update)
             window_width = longest_text_length * 6 + 5
-            top.geometry(f"{window_width}x450")
+            _, _, x, y = get_window_size_from_registry()
+            top.geometry(f"{window_width}x450+{x}+{y}")
             top.attributes('-topmost', 'true')
             top.resizable(True, True)
             selected_id_list = []

boiiiwd_package/src/main.py

@@ -430,7 +430,7 @@ class BOIIIWD(ctk.CTk):
     def load_configs(self):
         if os.path.exists(CONFIG_FILE_PATH):
             destination_folder = check_config("DestinationFolder", "")
-            steamcmd_path = check_config("SteamCMDPath", application_path)
+            steamcmd_path = check_config("SteamCMDPath", APPLICATION_PATH)
             new_appearance_mode = check_config("appearance", "Dark")
             new_scaling = check_config("scaling", 1.0)
             self.edit_destination_folder.delete(0, "end")
@@ -453,7 +453,7 @@ class BOIIIWD(ctk.CTk):
             scaling_int = math.trunc(scaling_float)
             self.settings_tab.scaling_optionemenu.set(f"{scaling_int}%")
             self.edit_steamcmd_path.delete(0, "end")
-            self.edit_steamcmd_path.insert(0, application_path)
+            self.edit_steamcmd_path.insert(0, APPLICATION_PATH)
             create_default_config()
 
     def help_queue_text_func(self, event=None):
@@ -522,11 +522,11 @@ class BOIIIWD(ctk.CTk):
     @if_internet_available
     def download_steamcmd(self):
         self.edit_steamcmd_path.delete(0, "end")
-        self.edit_steamcmd_path.insert(0, application_path)
+        self.edit_steamcmd_path.insert(0, APPLICATION_PATH)
         save_config("DestinationFolder" ,self.edit_destination_folder.get())
         save_config("SteamCMDPath" ,self.edit_steamcmd_path.get())
         steamcmd_url = "https://steamcdn-a.akamaihd.net/client/installer/steamcmd.zip"
-        steamcmd_zip_path = os.path.join(application_path, "steamcmd.zip")
+        steamcmd_zip_path = os.path.join(APPLICATION_PATH, "steamcmd.zip")
 
         try:
             response = requests.get(steamcmd_url)
@@ -536,7 +536,7 @@ class BOIIIWD(ctk.CTk):
                 zip_file.write(response.content)
 
             with zipfile.ZipFile(steamcmd_zip_path, "r") as zip_ref:
-                zip_ref.extractall(application_path)
+                zip_ref.extractall(APPLICATION_PATH)
 
             if check_steamcmd():
                 os.remove(fr"{steamcmd_zip_path}")
@@ -1286,7 +1286,7 @@ class BOIIIWD(ctk.CTk):
                     if os.path.exists(json_file_path):
                         self.label_speed.configure(text="Installing...")
                         mod_type = extract_json_data(json_file_path, "Type")
-                        items_file = os.path.join(application_path, LIBRARY_FILE)
+                        items_file = os.path.join(APPLICATION_PATH, LIBRARY_FILE)
                         item_exists,_ = self.library_tab.item_exists_in_file(items_file, workshop_id)
 
                         if item_exists:
@@ -1552,7 +1552,7 @@ class BOIIIWD(ctk.CTk):
                 if os.path.exists(json_file_path):
                     self.label_speed.configure(text="Installing...")
                     mod_type = extract_json_data(json_file_path, "Type")
-                    items_file = os.path.join(application_path, LIBRARY_FILE)
+                    items_file = os.path.join(APPLICATION_PATH, LIBRARY_FILE)
                     item_exists,_ = self.library_tab.item_exists_in_file(items_file, workshop_id)
 
                     if item_exists:

boiiiwd_package/src/settings_tab.py

@@ -174,7 +174,7 @@ class SettingsTab(ctk.CTkFrame):
                     if response == "No":
                         return
                     elif response == "Ok":
-                        os.system(f"notepad {os.path.join(application_path, 'config.ini')}")
+                        os.system(f"notepad {os.path.join(APPLICATION_PATH, 'config.ini')}")
                     else:
                         return
                 self.after(0, callback)
@@ -363,7 +363,7 @@ class SettingsTab(ctk.CTkFrame):
         if setting == "theme":
             theme_config = check_config("theme", "boiiiwd_theme.json")
 
-            if os.path.exists(os.path.join(application_path, theme_config)):
+            if os.path.exists(os.path.join(APPLICATION_PATH, theme_config)):
                 return "Custom"
 
             if theme_config == "boiiiwd_theme.json":
@@ -377,11 +377,11 @@ class SettingsTab(ctk.CTkFrame):
             return 1 if check_config(setting, fallback) == "on" else 0
 
     def boiiiwd_custom_theme(self, disable_only=None):
-        file_to_rename = os.path.join(application_path, "boiiiwd_theme.json")
+        file_to_rename = os.path.join(APPLICATION_PATH, "boiiiwd_theme.json")
         if os.path.exists(file_to_rename):
             timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
             new_name = f"boiiiwd_theme_{timestamp}.json"
-            os.rename(file_to_rename, os.path.join(application_path, new_name))
+            os.rename(file_to_rename, os.path.join(APPLICATION_PATH, new_name))
 
             if not disable_only:
                 show_message("Preset file renamed", "Custom preset disabled, file has been renmaed\n* Restart the app to take effect", icon="info")
@@ -389,14 +389,15 @@ class SettingsTab(ctk.CTkFrame):
             if disable_only:
                 return
             try:
-                shutil.copy(os.path.join(RESOURCES_DIR, check_config("theme", "boiiiwd_theme.json")), os.path.join(application_path, "boiiiwd_theme.json"))
+                shutil.copy(os.path.join(RESOURCES_DIR, check_config("theme", "boiiiwd_theme.json")), os.path.join(APPLICATION_PATH, "boiiiwd_theme.json"))
             except:
-                shutil.copy(os.path.join(RESOURCES_DIR, "boiiiwd_theme.json"), os.path.join(application_path, "boiiiwd_theme.json"))
+                shutil.copy(os.path.join(RESOURCES_DIR, "boiiiwd_theme.json"), os.path.join(APPLICATION_PATH, "boiiiwd_theme.json"))
             show_message("Preset file created", "You can now edit boiiiwd_theme.json in the current directory to your liking\n* Edits will apply next time you open boiiiwd\n* Program will always take boiiiwd_theme.json as the first theme option if found\n* Click on this button again to disable your custom theme or just rename boiiiwd_theme.json", icon="info")
 
     def settings_check_for_updates(self):
         check_for_updates_func(self, ignore_up_todate=False)
 
+    # make this rename to {id}_duplicate as a fallback
     def rename_all_folders(self, option):
         boiiiFolder = main_app.app.edit_destination_folder.get()
         maps_folder = os.path.join(boiiiFolder, "mods")
@@ -436,14 +437,20 @@ class SettingsTab(ctk.CTkFrame):
                     folder_to_rename = os.path.join(folder_path, folder_name)
                     new_folder_name = new_name
                     while new_folder_name in processed_names:
-                        new_folder_name += f"_{publisher_id}"
+                        if option == "PublisherID":
+                            new_folder_name += f"_duplicated"
+                        else:
+                            new_folder_name += f"_{publisher_id}"
                         if folder_name == new_folder_name:
                             rename_flag = False
                             break
                     new_path = os.path.join(folder_path, new_folder_name)
 
                     while os.path.exists(new_path):
-                        new_folder_name += f"_{publisher_id}"
+                        if option == "PublishedID":
+                            new_folder_name += f"_duplicated"
+                        else:
+                            new_folder_name += f"_{publisher_id}"
                         if folder_name == new_folder_name:
                             rename_flag = False
                             break
@@ -618,7 +625,7 @@ class SettingsTab(ctk.CTkFrame):
                             if os.path.exists(json_file_path):
                                 workshop_id = extract_json_data(json_file_path, "PublisherID")
                                 mod_type = extract_json_data(json_file_path, "Type")
-                                items_file = os.path.join(application_path, LIBRARY_FILE)
+                                items_file = os.path.join(APPLICATION_PATH, LIBRARY_FILE)
                                 item_exists,_ = main_app.app.library_tab.item_exists_in_file(items_file, workshop_id)
 
                                 if item_exists:

boiiiwd_package/src/update_window.py

@@ -81,7 +81,7 @@ class UpdateWindow(ctk.CTkToplevel):
 
     def update_progress_bar(self):
         try:
-            update_dir = os.path.join(application_path, UPDATER_FOLDER)
+            update_dir = os.path.join(APPLICATION_PATH, UPDATER_FOLDER)
             response = requests.get(LATEST_RELEASE_URL, stream=True)
             response.raise_for_status()
             current_exe = sys.argv[0]