Archives
/
Database-Merging-SQLite3
mirror of https://github.com/gopherchucks/Database-Merging-SQLite3
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4 from whattheserver/patch-1

Browse Source 
pep fixes
master
gopherchucks 4 years ago committed by GitHub
parent cfc33d252a 0a4547b12c
commit 02240eb638
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 59 additions and 52 deletions
Show Stats Download Patch File Download Diff File

111
mergeScript.py
Unescape Escape View File

@ -17,37 +17,40 @@ from time import gmtime, strftime
############################ Global Variables ##################################
################################################################################
dbCount = 0 # Variable to count the number of databases
listDB = [] # Variable to store the names of the databases
listTable = [] # Variable to store table names
dbCount = 0 # Variable to count the number of databases
listDB = [] # Variable to store the names of the databases
listTable = [] # Variable to store table names
############################ Function Definitions ##############################
################################################################################
# Attaches a database to the currently connected database
#
# @param dbName the name of the database file (i.e. "example.db")
# @param db_name the name of the database file (i.e. "example.db")
# @return none
def attachDatabase( dbName ):
def attach_database(db_name):
global dbCount
global listDB
print(("Attaching database: %s") % dbName)
curs.execute("ATTACH DATABASE ? as ? ;", (dbName, 'db' + str(dbCount)))
print("Attaching database: %s" % db_name)
curs.execute("ATTACH DATABASE ? as ? ;", (db_name, 'db' + str(dbCount)))
listDB.append('db' + str(dbCount))
dbCount += 1
# Closes the current database connection
#
# @return none
def closeConnection():
def close_connection():
curs.close()
conn.close()
# Gets the table names of a database
#
# @param dbName the name of the database file (i.e. "example.db")
# @param db_name the name of the database file (i.e. "example.db")
# @return a string array of the table names
def getTableNames():
def get_table_names():
curs.execute("SELECT name FROM sqlite_master WHERE type='table';")
temp = curs.fetchall()
tables = []
@ -55,27 +58,29 @@ def getTableNames():
tables.append(temp[i][0])
return tables
# Gets the column names of a table
#
# @param dbName the name of the database file (i.e. "example.db")
# @param db_name the name of the database file (i.e. "example.db")
# @return a string array of the column names - strips primary ids column
def getColumnNames( tableName ):
curs.execute("PRAGMA table_info(%s);" % str(tableName))
def get_column_names(table_name):
curs.execute("PRAGMA table_info(%s);" % str(table_name))
temp = curs.fetchall()
columns = []
for i in range(0, len(temp)):
if (("id" in temp[i][1]) | ("ID" in temp[i][1])):
if ("id" in temp[i][1]) | ("ID" in temp[i][1]):
continue
else:
columns.append(temp[i][1])
return columns
# Compares two lists to see if they have identical data
#
# @param list1 the first list parameter for comparison
# @param list2 the second list parameter for comparison
# @return will return a boolean (0 lists !=, 1 lists ==)
def compareLists( list1, list2 ):
def compare_lists(list1, list2):
if len(list1) != len(list2):
return 0
else:
@ -84,32 +89,34 @@ def compareLists( list1, list2 ):
return 0
return 1
# Converts a list of string objects to a string of comma separated items.
#
# @param listObj the list to convert
# @return a string containing the list items - separated
# by commas.
def listToString( listObj ):
listString = ""
for i in range(0, len(listObj)):
if (i == (len(listObj) - 1)):
listString = listString + listObj[i]
def list_to_string(list_obj):
list_string = ""
for i in range(0, len(list_obj)):
if i == (len(list_obj) - 1):
list_string = list_string + list_obj[i]
else:
listString = listString + listObj[i] + ", "
return listString
list_string = list_string + list_obj[i] + ", "
return list_string
# Merges a table from an attached database to the source table
#
# @param tableName the name of the table to merge
# @param columnNames the names of the columns to include in the merge
# @param dbNameTableName the name of the attached database and the table
# i.e. "databaseName.tableName"
# @param table_name the name of the table to merge
# @param column_names the names of the columns to include in the merge
# @param db_name_table_name the name of the attached database and the table
# i.e. "db_name.table_name"
# @return none
def mergeTable( tableName, columnNames, dbName ):
dbNameTableName = dbName + "." + tableName
def merge_table(table_name, column_names, db_name):
db_name_table_name = db_name + "." + table_name
try:
curs.execute("INSERT INTO %s (%s) SELECT %s FROM %s;" %
(tableName, columnNames, columnNames, dbNameTableName))
(table_name, column_names, column_names, db_name_table_name))
conn.commit()
except:
pass
@ -119,12 +126,12 @@ def mergeTable( tableName, columnNames, dbName ):
################################################################################
mainDB = '' # This is where the main database is
# referenced. Where all items will be
# merged to.
# referenced. Where all items will be
# merged to.
otherDBs = []
if (len(otherDBs) == 0):
if len(otherDBs) == 0:
print("ERROR: No databases have been added for merging.")
sys.exit()
@ -132,10 +139,10 @@ if (len(otherDBs) == 0):
################################################################################
# Initialize Connection and get main list of tables
conn = sqlite3.connect(mainDB) # Connect to the main database
curs = conn.cursor() # Connect a cursor
listTable = getTableNames() # Get the table names
closeConnection()
conn = sqlite3.connect(mainDB) # Connect to the main database
curs = conn.cursor() # Connect a cursor
listTable = get_table_names() # Get the table names
close_connection()
# Compare databases
startTime = time.time()
@ -144,26 +151,26 @@ print("Comparing databases. Started at: " + strftime("%H:%M", gmtime()))
for i in range(0, len(otherDBs)):
conn = sqlite3.connect(otherDBs[i])
curs = conn.cursor()
temp = getTableNames() # Get the current list of tables
if (len(listTable) > len(temp)):
temp = get_table_names() # Get the current list of tables
if len(listTable) > len(temp):
print("Table is missing from non-primary database: %s" % otherDBs[i])
print("Database will NOT BE MERGED with the main database.")
otherDBs.remove(otherDBs[i]) # Remove the table to avoid errors
otherDBs.remove(otherDBs[i]) # Remove the table to avoid errors
continue
if (len(listTable) < len(temp)):
if len(listTable) < len(temp):
print("Extra table(s) in non-primary database: %s" % otherDBs[i])
print("TABLES that are NOT in main database will NOT be added.")
if (listTable != temp):
if listTable != temp:
print("Tables do not match in non-primary database: %s" % otherDBs[i])
print("The database will NOT BE MERGED with the main database.")
otherDBs.remove(otherDBs[i]) # Remove the table to avoid errors
otherDBs.remove(otherDBs[i]) # Remove the table to avoid errors
continue
closeConnection()
close_connection()
if (len(otherDBs) == 0):
if len(otherDBs) == 0:
print("ERROR: No databases to merge. Databases were either removed due to \
inconsistencies, or databases were not added properly.")
sys.exit()
@ -175,19 +182,19 @@ print("Finished comparing databases. Time elapsed: %.3f" % (time.time() -
startTime = time.time()
print("Merging databases. Started at: " + strftime("%H:%M", gmtime()))
conn = sqlite3.connect(mainDB) # Attach main database
curs = conn.cursor() # Attach cursor
conn = sqlite3.connect(mainDB) # Attach main database
curs = conn.cursor() # Attach cursor
for i in range(0, len(otherDBs)):
attachDatabase(otherDBs[i]) # Attach other databases
attach_database(otherDBs[i]) # Attach other databases
# Merge databases
for i in range(0, len(listDB)):
for j in range(0, len(listTable)):
columns = listToString(getColumnNames(listTable[j])) # get columns
mergeTable(listTable[j], columns, listDB[i])
columns = list_to_string(get_column_names(listTable[j])) # get columns
merge_table(listTable[j], columns, listDB[i])
conn.commit() # Commit changes
closeConnection() # Close connection
conn.commit() # Commit changes
close_connection() # Close connection
print("Databases finished merging. Time elapsed: %.3f" % (time.time() -
startTime))
startTime))

Write Preview
Loading…
Cancel
Save