NEW: added hashing to fingerprint, added time/date

dbfp.py

@@ -3,8 +3,8 @@
 #
 import argparse
 import time
-
 from libs import fingerprint
+from libs import toolbox
 
 
 def main():
@@ -46,7 +46,8 @@ def parseArgs():
     exit()
 
   if (args.file):
-    fileout = args.file + "_" + timestr + '.json'
+    filename = toolbox.ToolBox.parseFilename(args.file)
+    fileout = filename + "_" + timestr + '.json'
 
   return (args.file, fileout, args.json, args.verbose)
 

libs/fingerprint.py

@@ -4,10 +4,12 @@
 import re
 import json
 import sqlite3
+import hashlib
+import time
+from libs import toolbox
 
 delimeter = "|"
 
-
 #
 # Database Schema
 # The SQLite database schema is stored in page 1 of the database (root page).
@@ -51,8 +53,11 @@ class DBSchema:
     except Exception, e:
       print e
       return -2
-
+    # extract file name from path+filename
-    self.dbName = filein
+    try:
+      self.dbName = toolbox.ToolBox.parseFilenameIncExt(filein)
+    except:
+      self.dbName = filein
     # read database schema
     try:
       self.__readDatabase()
@@ -110,7 +115,6 @@ class DBSchema:
 
     return
 
-
   #
   # Compare the Table Definitions. 
   # Compare Table 1 (Json table) to Table 2
@@ -183,27 +187,31 @@ class DBSchema:
   def writeFingerprint(self, filehandle):
     ahash  = {}
     thash  = {}
-    mhash  = {}
-    dhash  = {}
     dmhash = {}
-    ahash['tables'] = thash
+    shash  = {}
-    ahash['file-metadata'] = mhash
+    mhash  = {}
-    ahash['db-config']     = dhash
+    ahash['tables']        = thash
+    ahash['sql-hashes']    = shash
     ahash['db-metadata']   = dmhash
+    ahash['file-metadata'] = mhash
 
-    # metadata
+    try:
-    mhash['scanner-name'] = 'dbfp'
+      timestr = time.strftime('%Y-%m-%d_%H%M%S', time.localtime(time.time()))
-    mhash['scanner-ver']  = self.scanner_ver
+    except:
+      timestr = ""
+
+    mhash['scan-date']    = timestr
     mhash['format-ver']   = self.format_ver
-
+    mhash['scanner-ver']  = self.scanner_ver
-    # database configuration information
+    mhash['scanner-name'] = 'dbfp'
-    dhash['dn-name'] = self.dbName
+    mhash['dn-name']      = self.dbName
 
     # tables
-    keys = self.tables.keys()
+    tables = self.tables.keys()
-    for key in keys:
+    for table in tables:
-      thash[key] = self.tables[key].fields
+      thash[table]  = self.tables[table].fields
-      dmhash[key] = self.tables[key].SQLstr()
+      dmhash[table] = self.tables[table].SQLstr()
+      shash[table]  = self.tables[table].sqlStrHash
 
     json.dump(ahash, filehandle, sort_keys=True, indent=4)
 
@@ -215,8 +223,6 @@ class DBSchema:
     elif (errorCode == -3):
       retval = "ERROR: problem reading database"
     return retval
-
-
 #
 #
 #
@@ -227,29 +233,41 @@ class TableDefinition:
 
   tableschemaregex = r'\((.*)\)'
 
-
+  #
   def __init__(self):
-    self.tableName = ""
+    self.tableName  = ""
-    self.sqlStr    = ""
+    self.sqlStr     = ""
-    self.fields    = {}
+    self.sqlStrHash = ""
+    self.fields     = {}
     self.primarykeyFlag = False
     self.uniqueFlag     = False
 
-
+  #
   def loadTable(self, tableName, sqlStr):
     self.tableName = tableName
     self.sqlStr = sqlStr
+
+    # hash the sql create string for quicker fingerprint matching
+    try:
+      m = hashlib.md5()
+      m.update(self.sqlStr)
+      self.sqlStrHash = m.hexdigest()
+    except:
+      print 'WARN: problem hashing sql string: "{}"'.format(self.sqlStr)
+    # parse the create string into a structured hash table
     results = re.search(self.tableschemaregex, sqlStr)
     if results:
       colstr = results.group(1)
-      print "[[ TABLE: <" + tableName + "> ]]"
       columns = colstr.split(',')
       for col in columns:
         newField = self.__parseCreateStr(col.strip())
         if newField:
+          print "[[ TABLE: <{}> ] imported]".format(tableName)
           self.fields[newField['name']] = newField
+        else:
+          print "[[ TABLE: <{}> ] failed]".format(tableName)
 
-
+  #
   def importTable(self, tbName, sqlStr, fields):
     self.tableName = tbName
     self.sqlStr = sqlStr
@@ -355,25 +373,26 @@ class TableDefinition:
             self.uniqueFlag = False
         return False
         
-      print 'WARN: field definition not recognized: "' + sqltext + '"'
+      print 'WARN: field definition not recognized: "{}"'.format(sqltext)
     except Exception, e:
+      print 'WARN: problem parsing sql create text: "{}"'.format(sqltext)
       return None
 
     return None
 
-
+  #
   def fields(self):
     return self.fields
 
-
+  #
   def toJSON(self):
     print json.dumps(self.fields)
 
-
+  #
   def toFile(self, filehandle):
     json.dump(self.fields, filehandle, sort_keys=True, indent=4)
 
-
+  #
   def __str__(self):
     global delimeter
     retstr = ""
@@ -381,15 +400,15 @@ class TableDefinition:
 
     return retstr    
 
-
+  #
   def name(self):
     return self.tableName
 
-
+  #
   def setSQLstr(self, str):
     return self.sqlStr
 
-
+  #
   def SQLstr(self):
     return self.sqlStr