j3g
/
dbfp_pub
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

MOD: improved the table schema comparison, difference # count added, cleaned up old commented code

This commit is contained in:
JohnE 2015-09-03 01:31:39 -07:00
parent 7fa40e6c9c
commit 82bf211e4c
2 changed files with 42 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
dbfp.py
View File

@ -10,29 +10,6 @@ from libs import toolbox
def main(): def main():
parseArgs() parseArgs()
# db = fingerprint.DBSchema()
# if (filein and filejson):
# db.scanDBFile(filein)
# db.compareDB(filejson)
# elif (filein):
# retVal = db.scanDBFile(filein)
# if (retVal > 0):
# fh = open(fileout, "w")
# if verbose:
# db.debugFingerprint()
# if title:
# db.setTitle(title)
# if notes:
# db.setNotes(notes)
# db.writeFingerprint(fh)
# fh.close()
# else:
# print db.getErrorString(retVal)
# elif (filejson):
# db.importJson(filejson)
# db.debugFingerprint()
# #
def compareFingerprintDir(filein, filejson, fpdir): def compareFingerprintDir(filein, filejson, fpdir):
db = fingerprint.DBSchema() db = fingerprint.DBSchema()
@ -71,8 +48,8 @@ def parseArgs():
timestr = time.strftime('%Y-%m-%d_%H%M%S', time.localtime(time.time())) timestr = time.strftime('%Y-%m-%d_%H%M%S', time.localtime(time.time()))
parser = argparse.ArgumentParser(description='Fingerprint a sqlite database based on its schema') parser = argparse.ArgumentParser(description='Fingerprint a sqlite database based on its schema')
parser.add_argument('-f', '--file', required=False) parser.add_argument('-f', '--file', required=False)
parser.add_argument('-fd', '--fpdir', required=False) parser.add_argument('-fd', '--fpdir', required=False, help="path to dirctory of fingerprint files")
parser.add_argument('-j', '--json', required=False) parser.add_argument('-fp', '--fp', required=False, help="fingerprint file to use in comparison")
parser.add_argument('-an', '--app_name', required=False) parser.add_argument('-an', '--app_name', required=False)
parser.add_argument('-av', '--app_version', required=False) parser.add_argument('-av', '--app_version', required=False)
parser.add_argument('-n', '--notes', required=False) parser.add_argument('-n', '--notes', required=False)
@ -80,25 +57,22 @@ def parseArgs():
# parser.add_argument('-t', '--title', required=False) # parser.add_argument('-t', '--title', required=False)
args = parser.parse_args() args = parser.parse_args()
if (args.file is None) and (args.json is None): if (args.file is None) and (args.fp is None):
parser.print_help() parser.print_help()
exit() exit()
# compare a sqlite database file to all fingerprints # compare a sqlite database file to all fingerprints
if (args.json and args.fpdir): if (args.fp and args.fpdir):
compareFingerprints(args.file, args.json, args.fpdir) compareFingerprints(args.file, args.fp, args.fpdir)
# compare a sqlite database file to a fingerprint # compare a sqlite database file to a fingerprint
if (args.json): if (args.fp):
compareFingerprint(args.file, args.json) compareFingerprint(args.file, args.fp)
# create a fingerprint from the sqlite file # create a fingerprint from the sqlite file
filename = toolbox.ToolBox.parseFilename(args.file) filename = toolbox.ToolBox.parseFilename(args.file)
fileout = filename + "_" + timestr + '.json' fileout = filename + "_" + timestr + '.json'
createFingerprint(args.file, fileout, args.verbose, args.app_name, args.app_version, args.notes) createFingerprint(args.file, fileout, args.verbose, args.app_name, args.app_version, args.notes)
# return (args.file, fileout, args.json, args.verbose, args.title, args.notes,
# args.fpdir)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

55
libs/fingerprint.py
View File

@ -79,13 +79,14 @@ class DBSchema:
def compareDB(self, filejson): def compareDB(self, filejson):
self.tablesJson = self.__importJsonDBSchema(filejson) self.tablesJson = self.__importJsonDBSchema(filejson)
result = self.__DBSchemaCompare() result = self.__DBSchemaCompare()
print "[ Percetage == {}]".format(result)
return result return result
# import fingerprint from a json file # import fingerprint from a json file
def __importJsonDBSchema(self, filejson): def __importJsonDBSchema(self, file_json):
tables = {} tables = {}
try: try:
fh = open(filein, "r") fh = open(file_json, "r")
jsonData = json.load(fh) jsonData = json.load(fh)
dbmt = jsonData['db-metadata'] dbmt = jsonData['db-metadata']
tb = jsonData['tables'] tb = jsonData['tables']
@ -97,7 +98,7 @@ class DBSchema:
tables[table_name] = newTable tables[table_name] = newTable
except Exception, e: except Exception, e:
print "ERROR: problem loading json file: " + filein print "ERROR: problem loading json file: " + file_json
print e print e
return tables return tables
@ -109,48 +110,58 @@ class DBSchema:
# look for table, if exists, compare each # look for table, if exists, compare each
# if exists, compare each field # if exists, compare each field
# else, add to unknown tables...or do a fuzzy compare (look at number of fields, field names) # else, add to unknown tables...or do a fuzzy compare (look at number of fields, field names)
diff_num = 0
diff_total = 0
all_total = 0
for tableName in self.tablesJson.keys(): for tableName in self.tablesJson.keys():
table = self.tables[tableName] table = self.tables[tableName]
print "[[ Comparing Table: " + tableName + " ]]" print "[[ Comparing Table: " + tableName + " ]]"
if (table): if (table):
self.__CompareTable(self.tablesJson[tableName], table) if not (self.tablesJson[tableName].hash() == table.hash()):
(total, diff_num) = self.__CompareTable(self.tablesJson[tableName], table)
all_total += total
diff_total += diff_num
else: else:
self.__FuzzyTable() self.__FuzzyTable()
percentage = 0
return if (diff_total > 0):
percentage = float(diff_total) / all_total
return percentage
# #
# Compare the Table Definitions. # Compare the Table Definitions.
# Compare Table 1 (Json table) to Table 2 # Compare Table 1 (Json table) to Table 2
# #
def __CompareTable(self, tb1, tb2): def __CompareTable(self, tb1, tb2):
fieldsTotalCount = 0 fields_total_count = 0
fieldsErrorCount = 0 fields_diff_count = 0
propTotalCount = 0 prop_total_count = 0
propErrorCount = 0 prop_error_count = 0
totals = 0
diff_total = 0
fields1 = tb1.fields fields1 = tb1.fields
fields2 = tb2.fields fields2 = tb2.fields
for field in fields1.keys(): for field in fields1.keys():
field1 = fields1[field] field1 = fields1[field]
fieldsTotalCount += 1 fields_total_count += 1
if (fields2.has_key(field)): if (fields2.has_key(field)):
field2 = fields1[field] field2 = fields1[field]
for properties in field1.keys(): for properties in field1.keys():
propTotalCount += 1 prop_total_count += 1
if not field2.has_key(properties): if not field2.has_key(properties):
propErrorCount += 1 prop_error_count += 1
else: else:
fieldsErrorCount += 1 fields_diff_count += 1
if (propErrorCount == 0 and fieldsErrorCount == 0): if (prop_error_count == 0 and fields_diff_count == 0):
print "100% compatible" print "100% compatible"
else: else:
totals = propTotalCount + fieldsTotalCount totals = prop_total_count + fields_total_count
errors = propErrorCount + fieldsErrorCount diff_total = prop_error_count + fields_diff_count
print "Table difference found: " + str(errors) print "Table difference found: " + str(diff_total)
#print str((errors/totals) * 100) + '% compatible total == ' + str(totals) + " errors == " + str(errors) #print str((diff_total/totals) * 100) + '% compatible total == ' + str(totals) + " diff_total == " + str(diff_total)
return (totals, diff_total)
# look at un-identified tables and try to match fields by their properties # look at un-identified tables and try to match fields by their properties
def __FuzzyTable(): def __FuzzyTable():
@ -433,4 +444,8 @@ class TableDefinition:
def SQLstr(self): def SQLstr(self):
return self.sqlStr return self.sqlStr
#
def hash(self):
return self.sqlStrHash