Rim/IW4MAdminDBParser
Rim/IW4MAdminDBParser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Add sorting functionality when parsing, introduce vacuuming to signifigantly lower file sizes

Browse Source
This commit is contained in:
Ahrimdon 2024-03-03 04:18:32 -05:00
parent b22d2f3b00
commit e1da54b127
1 changed files with 37 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
parse_db.py
View File

@ -14,6 +14,7 @@ import re
def setup_argparser(): def setup_argparser():
parser = argparse.ArgumentParser(description="Accurately parses IW4MAdmin's database into a consolidated, easy-to-read format.") parser = argparse.ArgumentParser(description="Accurately parses IW4MAdmin's database into a consolidated, easy-to-read format.")
parser.add_argument('-s', '--sort', type=str, choices=['ASC', 'DESC'], default='DESC', help="Sort order for the query results (ASC or DESC).")
parser.add_argument('-p', '--path', type=str, default="Database.db", help="Path to the IW4MAdmin database file.") parser.add_argument('-p', '--path', type=str, default="Database.db", help="Path to the IW4MAdmin database file.")
parser.add_argument('-o', '--output', type=str, help="Output directory for the parsed database file.") parser.add_argument('-o', '--output', type=str, help="Output directory for the parsed database file.")
parser.add_argument('-t', '--time', action='store_true', help="Time the script's execution.") parser.add_argument('-t', '--time', action='store_true', help="Time the script's execution.")
@ -98,18 +99,23 @@ def main():
'Servers': servers_table, 'Servers': servers_table,
} }
sort_order = args.sort
# Process each table according to the configuration # Process each table according to the configuration
for table_name, func in table_functions.items(): for table_name, func in table_functions.items():
limit = config.get('tables', {}).get(table_name) limit = config.get('tables', {}).get(table_name)
if limit is not None: if limit is not None:
print(f"Processing {table_name} with limit: {limit}") print(f"Processing {table_name} with limit: {limit}")
func(existing_cur, new_cur, limit=limit) # Assuming each function can accept a limit parameter func(existing_cur, new_cur, limit=limit, sort_order=sort_order) # Assuming each function can accept a limit parameter
else: else:
print(f"Processing {table_name} without limit") print(f"Processing {table_name} without limit")
func(existing_cur, new_cur) func(existing_cur, new_cur, sort_order=sort_order)
# Commit and close # Commit and close
new_conn.commit() # Commit changes before vacuuming
new_conn.execute("VACUUM;")
new_conn.commit() new_conn.commit()
existing_conn.close() existing_conn.close()
new_conn.close() new_conn.close()
@ -117,7 +123,7 @@ def main():
end_time = time.time() # End timing end_time = time.time() # End timing
print(f"Script execution time: {end_time - start_time:.4f} seconds") # Print execution time print(f"Script execution time: {end_time - start_time:.4f} seconds") # Print execution time
def audit_log_table(existing_cur, new_cur, limit=None): def audit_log_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute(""" new_cur.execute("""
CREATE TABLE "AuditLog" ( CREATE TABLE "AuditLog" (
"ChangeHistoryId" INTEGER NOT NULL, "ChangeHistoryId" INTEGER NOT NULL,
@ -131,7 +137,7 @@ def audit_log_table(existing_cur, new_cur, limit=None):
) )
""") """)
query = """ query = f"""
SELECT SELECT
EFChangeHistory.ChangeHistoryId, EFChangeHistory.ChangeHistoryId,
EFChangeHistory.TypeOfChange, EFChangeHistory.TypeOfChange,
@ -142,6 +148,7 @@ def audit_log_table(existing_cur, new_cur, limit=None):
EFChangeHistory.TargetEntityId EFChangeHistory.TargetEntityId
FROM FROM
EFChangeHistory EFChangeHistory
ORDER BY ChangeHistoryId {sort_order}
""" """
if limit: if limit:
query += f" LIMIT {limit}" query += f" LIMIT {limit}"
@ -196,7 +203,7 @@ def audit_log_table(existing_cur, new_cur, limit=None):
new_row = (row[0], type_of_change, row[2], row[3], row[4], client_name_map[origin_entity_id], client_name_map[target_entity_id]) new_row = (row[0], type_of_change, row[2], row[3], row[4], client_name_map[origin_entity_id], client_name_map[target_entity_id])
new_cur.execute("INSERT INTO \"AuditLog\" (ChangeHistoryId, TypeOfChange, Time, Data, Command, Origin, Target) VALUES (?, ?, ?, ?, ?, ?, ?)", new_row) new_cur.execute("INSERT INTO \"AuditLog\" (ChangeHistoryId, TypeOfChange, Time, Data, Command, Origin, Target) VALUES (?, ?, ?, ?, ?, ?, ?)", new_row)
def client_connection_history_table(existing_cur, new_cur, limit=None): def client_connection_history_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute(""" new_cur.execute("""
CREATE TABLE "ClientConnectionHistory" ( CREATE TABLE "ClientConnectionHistory" (
"ConnectionId" INTEGER NOT NULL, "ConnectionId" INTEGER NOT NULL,
@ -208,7 +215,7 @@ def client_connection_history_table(existing_cur, new_cur, limit=None):
) )
""") """)
query = """ query = f"""
SELECT SELECT
EFClientConnectionHistory.ClientConnectionId, EFClientConnectionHistory.ClientConnectionId,
EFClientConnectionHistory.ClientId, EFClientConnectionHistory.ClientId,
@ -217,6 +224,7 @@ def client_connection_history_table(existing_cur, new_cur, limit=None):
EFClientConnectionHistory.ServerId EFClientConnectionHistory.ServerId
FROM FROM
EFClientConnectionHistory EFClientConnectionHistory
ORDER BY ClientConnectionId {sort_order}
""" """
if limit: if limit:
query += f" LIMIT {limit}" query += f" LIMIT {limit}"
@ -263,7 +271,7 @@ def client_connection_history_table(existing_cur, new_cur, limit=None):
new_row = (row[0], client_name, row[2], connection_type, server_hostname) new_row = (row[0], client_name, row[2], connection_type, server_hostname)
new_cur.execute("INSERT INTO ClientConnectionHistory (ConnectionId, Client, ConnectionTime, ConnectionType, Server) VALUES (?, ?, ?, ?, ?)", new_row) new_cur.execute("INSERT INTO ClientConnectionHistory (ConnectionId, Client, ConnectionTime, ConnectionType, Server) VALUES (?, ?, ?, ?, ?)", new_row)
def client_messages_table(existing_cur, new_cur, limit=None): def client_messages_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute(""" new_cur.execute("""
CREATE TABLE "ClientMessages" ( CREATE TABLE "ClientMessages" (
"MessageId" INTEGER NOT NULL, "MessageId" INTEGER NOT NULL,
@ -275,7 +283,7 @@ def client_messages_table(existing_cur, new_cur, limit=None):
) )
""") """)
query = """ query = f"""
SELECT SELECT
EFClientMessages.MessageId, EFClientMessages.MessageId,
EFClientMessages.ClientId, EFClientMessages.ClientId,
@ -284,6 +292,7 @@ def client_messages_table(existing_cur, new_cur, limit=None):
EFClientMessages.ServerId EFClientMessages.ServerId
FROM FROM
EFClientMessages EFClientMessages
ORDER BY MessageId {sort_order}
""" """
if limit: if limit:
query += f" LIMIT {limit}" query += f" LIMIT {limit}"
@ -330,7 +339,7 @@ def client_messages_table(existing_cur, new_cur, limit=None):
new_row = (row[0], client_name, row[2], row[3], server_hostname) new_row = (row[0], client_name, row[2], row[3], server_hostname)
new_cur.execute("INSERT INTO ClientMessages (MessageId, Client, Message, TimeSent, Server) VALUES (?, ?, ?, ?, ?)", new_row) new_cur.execute("INSERT INTO ClientMessages (MessageId, Client, Message, TimeSent, Server) VALUES (?, ?, ?, ?, ?)", new_row)
def clients_table(existing_cur, new_cur, limit=None): def clients_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute(""" new_cur.execute("""
CREATE TABLE "Clients" ( CREATE TABLE "Clients" (
"Connections" INTEGER NOT NULL, "Connections" INTEGER NOT NULL,
@ -345,7 +354,7 @@ def clients_table(existing_cur, new_cur, limit=None):
) )
""") """)
query = """ query = f"""
SELECT SELECT
EFClients.Connections, EFClients.Connections,
EFClients.CurrentAliasId, EFClients.CurrentAliasId,
@ -360,6 +369,7 @@ def clients_table(existing_cur, new_cur, limit=None):
EFClients EFClients
JOIN JOIN
EFAlias ON EFClients.CurrentAliasId = EFAlias.AliasId EFAlias ON EFClients.CurrentAliasId = EFAlias.AliasId
ORDER BY LastConnection {sort_order}
""" """
if limit: if limit:
query += f" LIMIT {limit}" query += f" LIMIT {limit}"
@ -405,7 +415,7 @@ def clients_table(existing_cur, new_cur, limit=None):
new_row = (connections, client_name, first_connection, game, last_connection, level, masked, total_connection_time, ip_address) new_row = (connections, client_name, first_connection, game, last_connection, level, masked, total_connection_time, ip_address)
new_cur.execute("INSERT INTO Clients (Connections, Name, FirstConnection, Game, LastConnection, Level, Masked, TotalConnectionTime, IP) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", new_row) new_cur.execute("INSERT INTO Clients (Connections, Name, FirstConnection, Game, LastConnection, Level, Masked, TotalConnectionTime, IP) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", new_row)
def ip_address_table(existing_cur, new_cur, limit=None): def ip_address_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
def fetch_client_info(src_cur): def fetch_client_info(src_cur):
src_cur.execute(""" src_cur.execute("""
SELECT Name, SearchableIPAddress, DateAdded FROM EFAlias SELECT Name, SearchableIPAddress, DateAdded FROM EFAlias
@ -433,7 +443,7 @@ def ip_address_table(existing_cur, new_cur, limit=None):
) VALUES (?, ?, ?) ) VALUES (?, ?, ?)
""", client_info) """, client_info)
def inbox_messages_modified_table(existing_cur, new_cur, limit=None): def inbox_messages_modified_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute(""" new_cur.execute("""
CREATE TABLE InboxMessagesModified ( CREATE TABLE InboxMessagesModified (
InboxMessageId INTEGER PRIMARY KEY, InboxMessageId INTEGER PRIMARY KEY,
@ -470,7 +480,7 @@ def inbox_messages_modified_table(existing_cur, new_cur, limit=None):
VALUES (?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?)
""", (msg_id, created, origin, target, server_id, message, read)) """, (msg_id, created, origin, target, server_id, message, read))
def maps_table(existing_cur, new_cur, limit=None): def maps_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute(""" new_cur.execute("""
CREATE TABLE IF NOT EXISTS "Maps" ( CREATE TABLE IF NOT EXISTS "Maps" (
"MapId" INTEGER NOT NULL, "MapId" INTEGER NOT NULL,
@ -481,11 +491,12 @@ def maps_table(existing_cur, new_cur, limit=None):
) )
""") """)
query = """ query = f"""
SELECT SELECT
MapId, CreatedDateTime, Name, Game MapId, CreatedDateTime, Name, Game
FROM FROM
EFMaps EFMaps
ORDER BY MapId {sort_order}
""" """
if limit: if limit:
query += f" LIMIT {limit}" query += f" LIMIT {limit}"
@ -505,7 +516,7 @@ def maps_table(existing_cur, new_cur, limit=None):
) VALUES (?, ?, ?, ?) ) VALUES (?, ?, ?, ?)
""", modified_rows) """, modified_rows)
def metadata_table(existing_cur, new_cur, limit=None): def metadata_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute(""" new_cur.execute("""
CREATE TABLE "Metadata" ( CREATE TABLE "Metadata" (
"MetaId" INTEGER NOT NULL, "MetaId" INTEGER NOT NULL,
@ -516,7 +527,7 @@ def metadata_table(existing_cur, new_cur, limit=None):
) )
""") """)
query = """ query = f"""
SELECT SELECT
EFMeta.MetaId, EFMeta.MetaId,
EFMeta.ClientId, EFMeta.ClientId,
@ -525,6 +536,7 @@ def metadata_table(existing_cur, new_cur, limit=None):
EFMeta.Value EFMeta.Value
FROM FROM
EFMeta EFMeta
ORDER BY MetaId {sort_order}
""" """
if limit: if limit:
query += f" LIMIT {limit}" query += f" LIMIT {limit}"
@ -576,7 +588,7 @@ def metadata_table(existing_cur, new_cur, limit=None):
new_row = (meta_id, client_name, created, key, value) new_row = (meta_id, client_name, created, key, value)
new_cur.execute("INSERT INTO Metadata (MetaId, Name, Timestamp, Note, Value) VALUES (?, ?, ?, ?, ?)", new_row) new_cur.execute("INSERT INTO Metadata (MetaId, Name, Timestamp, Note, Value) VALUES (?, ?, ?, ?, ?)", new_row)
def penalties_table(existing_cur, new_cur, limit=None): def penalties_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute(""" new_cur.execute("""
CREATE TABLE "Penalties" ( CREATE TABLE "Penalties" (
"PenaltyId" INTEGER NOT NULL, "PenaltyId" INTEGER NOT NULL,
@ -591,7 +603,7 @@ def penalties_table(existing_cur, new_cur, limit=None):
) )
""") """)
query = """ query = f"""
SELECT SELECT
EFPenalties.PenaltyId, EFPenalties.PenaltyId,
EFPenalties.AutomatedOffense, EFPenalties.AutomatedOffense,
@ -604,6 +616,7 @@ def penalties_table(existing_cur, new_cur, limit=None):
EFPenalties."When" EFPenalties."When"
FROM FROM
EFPenalties EFPenalties
ORDER BY PenaltyId {sort_order}
""" """
if limit: if limit:
query += f" LIMIT {limit}" query += f" LIMIT {limit}"
@ -695,7 +708,7 @@ def penalties_table(existing_cur, new_cur, limit=None):
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (penalty_id, automated_offense, expires, evaded_offense, offender_name, offense, punisher_name, penalty_type, timestamp)) """, (penalty_id, automated_offense, expires, evaded_offense, offender_name, offense, punisher_name, penalty_type, timestamp))
def penalty_identifiers_table(existing_cur, new_cur, limit=None): def penalty_identifiers_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute(""" new_cur.execute("""
CREATE TABLE "PenaltyIdentifiers" ( CREATE TABLE "PenaltyIdentifiers" (
"PenaltyIdentifierId" INTEGER NOT NULL, "PenaltyIdentifierId" INTEGER NOT NULL,
@ -705,7 +718,7 @@ def penalty_identifiers_table(existing_cur, new_cur, limit=None):
) )
""") """)
query = """ query = f"""
SELECT SELECT
EFPenaltyIdentifiers.PenaltyIdentifierId, EFPenaltyIdentifiers.PenaltyIdentifierId,
EFPenaltyIdentifiers.PenaltyId, EFPenaltyIdentifiers.PenaltyId,
@ -713,6 +726,7 @@ def penalty_identifiers_table(existing_cur, new_cur, limit=None):
EFPenaltyIdentifiers.NetworkId EFPenaltyIdentifiers.NetworkId
FROM FROM
EFPenaltyIdentifiers EFPenaltyIdentifiers
ORDER BY PenaltyIdentifierId {sort_order}
""" """
if limit: if limit:
query += f" LIMIT {limit}" query += f" LIMIT {limit}"
@ -754,7 +768,7 @@ def penalty_identifiers_table(existing_cur, new_cur, limit=None):
VALUES (?, ?, ?, ?) VALUES (?, ?, ?, ?)
""", (penalty_identifier_id, penalty_id, created, client_name)) """, (penalty_identifier_id, penalty_id, created, client_name))
def servers_table(existing_cur, new_cur, limit=None): def servers_table(existing_cur, new_cur, limit=None, sort_order='DESC'):
new_cur.execute(""" new_cur.execute("""
CREATE TABLE "Servers" ( CREATE TABLE "Servers" (
"ServerId" INTEGER NOT NULL, "ServerId" INTEGER NOT NULL,
@ -767,9 +781,10 @@ def servers_table(existing_cur, new_cur, limit=None):
) )
""") """)
query = """ query = f"""
SELECT ServerId, Active, Port, Endpoint, GameName, HostName, IsPasswordProtected SELECT ServerId, Active, Port, Endpoint, GameName, HostName, IsPasswordProtected
FROM EFServers FROM EFServers
ORDER BY ServerId {sort_order}
""" """
if limit: if limit:
query += f" LIMIT {limit}" query += f" LIMIT {limit}"