NEW: added new table to the index with app info for each database, population of the index is complete too
This commit is contained in:
JohnE
parent
fe504e33c9
commit
0d6c52f2d4
|
|
@ -37,7 +37,7 @@ class FingerprintDB:
|
|||
# version of the scanner used to create the fingerprint
|
||||
scanner_ver = "0.98"
|
||||
# version of the json file format, this # is inserted in the json fingerprint file and can be used to determine what is supported at the time of that version
|
||||
format_ver = "0.91"
|
||||
format_ver = "0.92"
|
||||
|
||||
#
|
||||
def __init__(self):
|
||||
|
|
@ -47,13 +47,20 @@ class FingerprintDB:
|
|||
self.tables = {}
|
||||
self.db_hash = None
|
||||
self.table_hashes = None
|
||||
# fingerprint metadata
|
||||
self.db_name = ""
|
||||
self.app_name = ""
|
||||
self.app_ver = ""
|
||||
self.notes = ""
|
||||
self.filein = ""
|
||||
self.init = False
|
||||
# fingerprint metadata
|
||||
self.metadata = {}
|
||||
self.metadata['db-name'] = ""
|
||||
self.metadata['app-name'] = ""
|
||||
self.metadata['app-ver'] = ""
|
||||
self.metadata['notes'] = ""
|
||||
self.metadata['scan-date'] = ""
|
||||
# self.db_name = ""
|
||||
# self.app_name = ""
|
||||
# self.app_ver = ""
|
||||
# self.notes = ""
|
||||
# self.scan_date = ""
|
||||
|
||||
#
|
||||
def scanDBFile(self, filein):
|
||||
|
|
@ -66,9 +73,9 @@ class FingerprintDB:
|
|||
|
||||
try:
|
||||
# extract file name from path+filename
|
||||
self.db_name = ToolBox.parseFilenameIncExt(filein)
|
||||
self.metadata['db-name'] = ToolBox.parseFilenameIncExt(filein)
|
||||
except:
|
||||
self.db_name = filein
|
||||
self.metadata['db-name'] = filein
|
||||
try:
|
||||
# read database schema, parse the schema
|
||||
self.__readDatabase()
|
||||
|
|
@ -173,6 +180,8 @@ class FingerprintDB:
|
|||
dbmt = jsonData['db-metadata']
|
||||
dbht = jsonData['db-metadata-hashes']
|
||||
dbmd5 = jsonData['db-metadata-md5']
|
||||
metadata = jsonData['_file-metadata']
|
||||
|
||||
all_tables = tb.keys()
|
||||
for table_name in all_tables:
|
||||
#logging.info("[[ Table <" + table_name + "> imported ]]")
|
||||
|
|
@ -183,6 +192,7 @@ class FingerprintDB:
|
|||
self.tables = tables
|
||||
self.db_hash = dbmd5
|
||||
self.table_hashes = dbht
|
||||
self.metadata = metadata
|
||||
except Exception as ex:
|
||||
logging.error("ERROR: problem loading json file: \n{}\n{}".format(file_json, ex))
|
||||
|
||||
|
|
@ -276,11 +286,6 @@ class FingerprintDB:
|
|||
def __readDatabase(self):
|
||||
flag = False
|
||||
rows = self.cur.execute(self.sqlmaster)
|
||||
# row = self.cur.fetchone()
|
||||
# newTable = TableSchema()
|
||||
# newTable.loadTable(row[0], row[1])
|
||||
# self.table_names.append(newTable.name())
|
||||
# self.tables[newTable.name()] = newTable
|
||||
|
||||
for row in rows:
|
||||
flag = True
|
||||
|
|
@ -329,10 +334,10 @@ class FingerprintDB:
|
|||
mhash['format-ver'] = self.format_ver
|
||||
mhash['scanner-ver'] = self.scanner_ver
|
||||
mhash['scanner-name'] = 'dbfp'
|
||||
mhash['dn-name'] = self.db_name
|
||||
mhash['app-name'] = self.app_name
|
||||
mhash['app-ver'] = self.app_ver
|
||||
mhash['notes'] = self.notes
|
||||
mhash['db-name'] = self.metadata['db-name']
|
||||
mhash['app-name'] = self.metadata['app-name']
|
||||
mhash['app-ver'] = self.metadata['app-ver']
|
||||
mhash['notes'] = self.metadata['notes']
|
||||
|
||||
# tables
|
||||
tables = self.tables.keys()
|
||||
|
|
@ -365,15 +370,15 @@ class FingerprintDB:
|
|||
|
||||
#
|
||||
def setAppName(self, name):
|
||||
self.app_name = name
|
||||
self.metadata['app-name'] = name
|
||||
|
||||
#
|
||||
def setAppVer(self, version):
|
||||
self.app_ver = version
|
||||
self.metadata['app-ver'] = version
|
||||
|
||||
#
|
||||
def setNotes(self, notes):
|
||||
self.notes = notes
|
||||
self.metadata['notes'] = notes
|
||||
|
||||
#
|
||||
def getErrorString(self, errorCode):
|
||||
|
|
|
|||
|
|
@ -154,17 +154,16 @@ class FingerprintIndex:
|
|||
''')
|
||||
self.db_conn.execute('''
|
||||
CREATE TABLE metadata (
|
||||
md5_db TEXT PRIMARY KEY,
|
||||
md5_db TEXT,
|
||||
app_name TEXT,
|
||||
app_ver TEXT,
|
||||
file_name TEXT,
|
||||
db_file TEXT,
|
||||
fp_file TEXT,
|
||||
scan_date NUMERIC);
|
||||
scan_date TEXT);
|
||||
''')
|
||||
logging.info("Successfully created index table")
|
||||
except Exception as ex:
|
||||
raise
|
||||
# raise FingerprintIndexWrite("Error creating index file\n{}".format(ex))
|
||||
|
||||
#
|
||||
def __populateIndex(self, fp_dir):
|
||||
|
|
@ -184,12 +183,13 @@ class FingerprintIndex:
|
|||
db.importJson(fq_file)
|
||||
self.__insertMod_md5_all(db.db_hash, db.table_hashes.values(), file)
|
||||
self.__insertMod_md5_tables(db.table_hashes.values(), file)
|
||||
self.__insertMod_metadata(db, file)
|
||||
finCount = finCount+1
|
||||
if ((finCount % 5) == 0):
|
||||
self.db_conn.commit()
|
||||
self.db_conn.commit()
|
||||
except Exception as ex:
|
||||
logging.error(ex)
|
||||
self.db_conn.rollback()
|
||||
failCount = failCount+1
|
||||
logging.error(ex)
|
||||
except Exception as ex:
|
||||
logging.error(ex)
|
||||
finally:
|
||||
|
|
@ -200,7 +200,7 @@ class FingerprintIndex:
|
|||
#
|
||||
def __insertMod_md5_all(self, md5_db, md5_list, filename):
|
||||
try:
|
||||
logging.info("INSERT INTO md5_all VALUES({}, {}, {}, 1)".format(md5_db, ','.join(md5_list), filename))
|
||||
#logging.info("INSERT INTO md5_all VALUES({}, {}, {}, 1)".format(md5_db, ','.join(md5_list), filename))
|
||||
self.db_conn.execute(
|
||||
'''
|
||||
INSERT INTO md5_all VALUES(?, ?, ?, ?)
|
||||
|
|
@ -210,7 +210,7 @@ class FingerprintIndex:
|
|||
(fp_list, fp_count) = self.__selectFileList(md5_db)
|
||||
fp_list += ","+filename
|
||||
fp_count += 1
|
||||
# logging.info("fp_list=={}".format(fp_list))
|
||||
#logging.info("fp_list=={}".format(fp_list))
|
||||
self.db_conn.execute(
|
||||
'''
|
||||
UPDATE md5_all SET fp_list=?, fp_count=? WHERE md5_db=?
|
||||
|
|
@ -243,6 +243,23 @@ class FingerprintIndex:
|
|||
except Exception as ex:
|
||||
raise FingerprintIndexWrite("Error inserting a row\n{}".format(ex))
|
||||
|
||||
#
|
||||
def __insertMod_metadata(self, db, filename):
|
||||
# insert the md5 of the table schemas
|
||||
# print "*****"
|
||||
# print "***** {}".format(db.metadata)
|
||||
# print "***** {}".format(db.metadata['fart'])
|
||||
# print "***** {}".format(db.metadata['app_name'])
|
||||
try:
|
||||
self.db_conn.execute(
|
||||
'''
|
||||
INSERT INTO metadata VALUES(?, ?, ?, ?, ?, ?)
|
||||
''', [db.db_hash, db.metadata['app-name'], db.metadata['app-ver'], db.metadata['db-name'], filename, db.metadata['scan-date']])
|
||||
# ''', ["abc", "efg", "hij", "klm", "nop", "qrs"])
|
||||
except Exception as ex:
|
||||
print ex
|
||||
raise
|
||||
|
||||
#
|
||||
def __selectFileList(self, md5_db):
|
||||
try:
|
||||
|
|
|
|||
Loading…
Reference in New Issue