Rim/IW4MAdminDBParser
Rim/IW4MAdminDBParser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Add sorting functionality when parsing, introduce vacuuming to signifigantly lower file sizes

Browse Source
This commit is contained in:
Ahrimdon 2024-03-03 04:18:32 -05:00
parent b22d2f3b00
commit e1da54b127
1 changed files with 37 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
parse_db.py
View File

@ -14,6 +14,7 @@ import re
def setup_argparser():
parser = argparse.ArgumentParser(description="Accurately parses IW4MAdmin's database into a consolidated, easy-to-read format.")
parser.add_argument('-s', '--sort', type=str, choices=['ASC', 'DESC'], default='DESC', help="Sort order for the query results (ASC or DESC).")
parser.add_argument('-p', '--path', type=str, default="Database.db", help="Path to the IW4MAdmin database file.")
parser.add_argument('-o', '--output', type=str, help="Output directory for the parsed database file.")
parser.add_argument('-t', '--time', action='store_true', help="Time the script's execution.")
@ -98,18 +99,23 @@ def main():
'Servers': servers_table,
}
sort_order = args.sort
# Process each table according to the configuration
for table_name, func in table_functions.items():
limit = config.get('tables', {}).get(table_name)
if limit is not None:
print(f"Processing {table_name} with limit: {limit}")
func(existing_cur, new_cur, limit=limit) # Assuming each function can accept a limit parameter
func(existing_cur, new_cur, limit=limit, sort_order=sort_order) # Assuming each function can accept a limit parameter
else:
print(f"Processing {table_name} without limit")
func(existing_cur, new_cur)
func(existing_cur, new_cur, sort_order=sort_order)
# Commit and close
new_conn.commit() # Commit changes before vacuuming
new_conn.execute("VACUUM;")
new_conn.commit()
existing_conn.close()
new_conn.close()
@ -117,7 +123,7 @@ def main():
end_time = time.time() # End timing
print(f"Script execution time: {end_time - start_time:.4f} seconds") # Print execution time
def audit_log_table(existing_cur, new_cur, limit=None):
def audit_log_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute("""
CREATE TABLE "AuditLog" (
"ChangeHistoryId" INTEGER NOT NULL,
@ -131,7 +137,7 @@ def audit_log_table(existing_cur, new_cur, limit=None):
)
""")
query = """
query = f"""
SELECT
EFChangeHistory.ChangeHistoryId,
EFChangeHistory.TypeOfChange,
@ -142,6 +148,7 @@ def audit_log_table(existing_cur, new_cur, limit=None):
EFChangeHistory.TargetEntityId
FROM
EFChangeHistory
ORDER BY ChangeHistoryId {sort_order}
"""
if limit:
query += f" LIMIT {limit}"
@ -196,7 +203,7 @@ def audit_log_table(existing_cur, new_cur, limit=None):
new_row = (row[0], type_of_change, row[2], row[3], row[4], client_name_map[origin_entity_id], client_name_map[target_entity_id])
new_cur.execute("INSERT INTO \"AuditLog\" (ChangeHistoryId, TypeOfChange, Time, Data, Command, Origin, Target) VALUES (?, ?, ?, ?, ?, ?, ?)", new_row)
def client_connection_history_table(existing_cur, new_cur, limit=None):
def client_connection_history_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute("""
CREATE TABLE "ClientConnectionHistory" (
"ConnectionId" INTEGER NOT NULL,
@ -208,7 +215,7 @@ def client_connection_history_table(existing_cur, new_cur, limit=None):
)
""")
query = """
query = f"""
SELECT
EFClientConnectionHistory.ClientConnectionId,
EFClientConnectionHistory.ClientId,
@ -217,6 +224,7 @@ def client_connection_history_table(existing_cur, new_cur, limit=None):
EFClientConnectionHistory.ServerId
FROM
EFClientConnectionHistory
ORDER BY ClientConnectionId {sort_order}
"""
if limit:
query += f" LIMIT {limit}"
@ -263,7 +271,7 @@ def client_connection_history_table(existing_cur, new_cur, limit=None):
new_row = (row[0], client_name, row[2], connection_type, server_hostname)
new_cur.execute("INSERT INTO ClientConnectionHistory (ConnectionId, Client, ConnectionTime, ConnectionType, Server) VALUES (?, ?, ?, ?, ?)", new_row)
def client_messages_table(existing_cur, new_cur, limit=None):
def client_messages_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute("""
CREATE TABLE "ClientMessages" (
"MessageId" INTEGER NOT NULL,
@ -275,7 +283,7 @@ def client_messages_table(existing_cur, new_cur, limit=None):
)
""")
query = """
query = f"""
SELECT
EFClientMessages.MessageId,
EFClientMessages.ClientId,
@ -284,6 +292,7 @@ def client_messages_table(existing_cur, new_cur, limit=None):
EFClientMessages.ServerId
FROM
EFClientMessages
ORDER BY MessageId {sort_order}
"""
if limit:
query += f" LIMIT {limit}"
@ -330,7 +339,7 @@ def client_messages_table(existing_cur, new_cur, limit=None):
new_row = (row[0], client_name, row[2], row[3], server_hostname)
new_cur.execute("INSERT INTO ClientMessages (MessageId, Client, Message, TimeSent, Server) VALUES (?, ?, ?, ?, ?)", new_row)
def clients_table(existing_cur, new_cur, limit=None):
def clients_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute("""
CREATE TABLE "Clients" (
"Connections" INTEGER NOT NULL,
@ -345,7 +354,7 @@ def clients_table(existing_cur, new_cur, limit=None):
)
""")
query = """
query = f"""
SELECT
EFClients.Connections,
EFClients.CurrentAliasId,
@ -360,6 +369,7 @@ def clients_table(existing_cur, new_cur, limit=None):
EFClients
JOIN
EFAlias ON EFClients.CurrentAliasId = EFAlias.AliasId
ORDER BY LastConnection {sort_order}
"""
if limit:
query += f" LIMIT {limit}"
@ -405,7 +415,7 @@ def clients_table(existing_cur, new_cur, limit=None):
new_row = (connections, client_name, first_connection, game, last_connection, level, masked, total_connection_time, ip_address)
new_cur.execute("INSERT INTO Clients (Connections, Name, FirstConnection, Game, LastConnection, Level, Masked, TotalConnectionTime, IP) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", new_row)
def ip_address_table(existing_cur, new_cur, limit=None):
def ip_address_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
def fetch_client_info(src_cur):
src_cur.execute("""
SELECT Name, SearchableIPAddress, DateAdded FROM EFAlias
@ -433,7 +443,7 @@ def ip_address_table(existing_cur, new_cur, limit=None):
) VALUES (?, ?, ?)
""", client_info)
def inbox_messages_modified_table(existing_cur, new_cur, limit=None):
def inbox_messages_modified_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute("""
CREATE TABLE InboxMessagesModified (
InboxMessageId INTEGER PRIMARY KEY,
@ -470,7 +480,7 @@ def inbox_messages_modified_table(existing_cur, new_cur, limit=None):
VALUES (?, ?, ?, ?, ?, ?, ?)
""", (msg_id, created, origin, target, server_id, message, read))
def maps_table(existing_cur, new_cur, limit=None):
def maps_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute("""
CREATE TABLE IF NOT EXISTS "Maps" (
"MapId" INTEGER NOT NULL,
@ -481,11 +491,12 @@ def maps_table(existing_cur, new_cur, limit=None):
)
""")
query = """
query = f"""
SELECT
MapId, CreatedDateTime, Name, Game
FROM
EFMaps
ORDER BY MapId {sort_order}
"""
if limit:
query += f" LIMIT {limit}"
@ -505,7 +516,7 @@ def maps_table(existing_cur, new_cur, limit=None):
) VALUES (?, ?, ?, ?)
""", modified_rows)
def metadata_table(existing_cur, new_cur, limit=None):
def metadata_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute("""
CREATE TABLE "Metadata" (
"MetaId" INTEGER NOT NULL,
@ -516,7 +527,7 @@ def metadata_table(existing_cur, new_cur, limit=None):
)
""")
query = """
query = f"""
SELECT
EFMeta.MetaId,
EFMeta.ClientId,
@ -525,6 +536,7 @@ def metadata_table(existing_cur, new_cur, limit=None):
EFMeta.Value
FROM
EFMeta
ORDER BY MetaId {sort_order}
"""
if limit:
query += f" LIMIT {limit}"
@ -576,7 +588,7 @@ def metadata_table(existing_cur, new_cur, limit=None):
new_row = (meta_id, client_name, created, key, value)
new_cur.execute("INSERT INTO Metadata (MetaId, Name, Timestamp, Note, Value) VALUES (?, ?, ?, ?, ?)", new_row)
def penalties_table(existing_cur, new_cur, limit=None):
def penalties_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute("""
CREATE TABLE "Penalties" (
"PenaltyId" INTEGER NOT NULL,
@ -591,7 +603,7 @@ def penalties_table(existing_cur, new_cur, limit=None):
)
""")
query = """
query = f"""
SELECT
EFPenalties.PenaltyId,
EFPenalties.AutomatedOffense,
@ -604,6 +616,7 @@ def penalties_table(existing_cur, new_cur, limit=None):
EFPenalties."When"
FROM
EFPenalties
ORDER BY PenaltyId {sort_order}
"""
if limit:
query += f" LIMIT {limit}"
@ -695,7 +708,7 @@ def penalties_table(existing_cur, new_cur, limit=None):
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (penalty_id, automated_offense, expires, evaded_offense, offender_name, offense, punisher_name, penalty_type, timestamp))
def penalty_identifiers_table(existing_cur, new_cur, limit=None):
def penalty_identifiers_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute("""
CREATE TABLE "PenaltyIdentifiers" (
"PenaltyIdentifierId" INTEGER NOT NULL,
@ -705,7 +718,7 @@ def penalty_identifiers_table(existing_cur, new_cur, limit=None):
)
""")
query = """
query = f"""
SELECT
EFPenaltyIdentifiers.PenaltyIdentifierId,
EFPenaltyIdentifiers.PenaltyId,
@ -713,6 +726,7 @@ def penalty_identifiers_table(existing_cur, new_cur, limit=None):
EFPenaltyIdentifiers.NetworkId
FROM
EFPenaltyIdentifiers
ORDER BY PenaltyIdentifierId {sort_order}
"""
if limit:
query += f" LIMIT {limit}"
@ -754,7 +768,7 @@ def penalty_identifiers_table(existing_cur, new_cur, limit=None):
VALUES (?, ?, ?, ?)
""", (penalty_identifier_id, penalty_id, created, client_name))
def servers_table(existing_cur, new_cur, limit=None):
def servers_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute("""
CREATE TABLE "Servers" (
"ServerId" INTEGER NOT NULL,
@ -767,9 +781,10 @@ def servers_table(existing_cur, new_cur, limit=None):
)
""")
query = """
query = f"""
SELECT ServerId, Active, Port, Endpoint, GameName, HostName, IsPasswordProtected
FROM EFServers
ORDER BY ServerId {sort_order}
"""
if limit:
query += f" LIMIT {limit}"