WIP: md5 create index

dbfp.py

@@ -21,20 +21,20 @@ def main():
 
 #
 def compareFingerprintDir(filein, fpdir):
-  db = fingerprint.DBSchema()
+  db = fingerprint.FingerprintDB()
   fp = FingerprintIndex()
-  fp.createIndex()
+  fp.openIndex()
 
 #
 def compareFingerprint(filein, filejson):
-  db = fingerprint.DBSchema()
+  db = fingerprint.FingerprintDB()
   db.scanDBFile(filein)
   percent = db.compareDB(filejson)
   print "Percent match: {}".format(str(percent))
 
 #
 def createFingerprint(filein, verbose, app_name, app_ver, notes):
-  db = fingerprint.DBSchema()
+  db = fingerprint.FingerprintDB()
   retVal = db.scanDBFile(filein)
   if (retVal > 0):
     if verbose:
@@ -76,7 +76,7 @@ def __createFingerprint(dir_name):
   except:
     # not finding a databases folder is normal, not all apps use sqlite
     return
-  db = fingerprint.DBSchema()
+  db = fingerprint.FingerprintDB()
   fdir = FP_BASE_DIR + os.path.sep + dir_name
   if (not mkdir(fdir)):
     logging.error('Error creating directory "{}"'.format(fdir))

libs/exceptions.py

@@ -3,6 +3,9 @@
 #
 
 class FingerprintWrite(Exception):
-  """Problem writing the fingerprint to a file"""
+  """Error writing the fingerprint to a file"""
   pass
 
+class FingerprintIndexWrite(Exception):
+  """Error creating an index file"""
+  pass

libs/fingerprint.py

@@ -25,7 +25,7 @@ delimeter = "|"
 #   sql text
 # );
 #
-class DBSchema:
+class FingerprintDB:
   """
   This class represents a complete database schema
   Helper functions:
@@ -92,7 +92,7 @@ class DBSchema:
       finally:
         fh.close()
     except Exception, ex:
-      print ex
+      logging.error(ex)
       raise FingerprintWrite("Problem writing the fingerprint to a file")
 
   #
@@ -139,7 +139,7 @@ class DBSchema:
       all_tables = tb.keys()
       for table_name in all_tables:
         print "[[ Table <" + table_name + "> imported ]]"
-        newTable = TableDefinition()
+        newTable = TableSchema()
         newTable.importTable(table_name, tb[table_name], dbmt[table_name], dbht[table_name])
         tables[table_name] = newTable
 
@@ -228,7 +228,7 @@ class DBSchema:
   # sqlmaster = "SELECT name, sql FROM sqlite_master WHERE type='table'"
   def __readDatabase(self):
     for row in self.cur.execute(self.sqlmaster):
-      newTable = TableDefinition()
+      newTable = TableSchema()
       newTable.loadTable(row[0], row[1])
       self.tableNames.append(newTable.name())
       self.tables[newTable.name()] = newTable
@@ -308,7 +308,7 @@ class DBSchema:
 #
 #
 #
-class TableDefinition:
+class TableSchema:
   """
   This class represents the definition of database table
   """

libs/fingerprint_index.py

@@ -3,27 +3,43 @@
 #  [ md5_all, md5_list, file_name ]
 # 
 #
+import os
+import sys
 import logging
 import sqlite3 as sql
-from libs import fingerprint
+from libs.fingerprint import FingerprintDB
+
+# prefixed with "_" so that it will be listed first and visible
+INDEX_FILENAME = '_index_dpfp.db'
 
 
 class FingerprintIndex:
   """
-  Create and use an index to 
+  Class handling an index of fingerprints for effeciently locating a fingerprint
   """
 
   #
   def __init__(self):
+    self.db_conn = None
     return
 
-  def openIndex(self):
+  def openIndex(self, fpdir):
-    pass
+    fq_fpidx = fpdir + os.path.sep + INDEX_FILENAME
-
-  def createIndex(self):
     try:
-      conn = sql.connect('test123.db')
+      if (os.path.isfile(fq_fpidx)):
-      conn.execute('''
+        self.db_conn = sql.connect(fq_fpidx)
+    except:
+      self.db_conn = None
+      logging.info("No index file found, creating index now...")
+    # create the Index file, also populate it with entrieds from the fingerprint  
+    self.creatIndex(fpdir)
+
+  #
+  def createIndex(self, fpdir):
+    fq_fpidx = fpdir + os.path.sep + INDEX_FILENAME
+    try:
+      self.db_conn = sql.connect(fq_fpidx)
+      self.db_conn.execute('''
           CREATE TABLE md5_index (
           md5_all TEXT PRIMARY KEY,
           md5_list TEXT,
@@ -33,10 +49,35 @@ class FingerprintIndex:
     except:
       logging.error("Error creating index table")
     finally:
-      if conn:
+      if self.db_conn:
-        conn.close()
+        self.db_conn.close()
+        self.db_conn = None
+        raise FingerprintIndexWrite("Error creating an index file")
 
-  def checkIntegrity(self):
+    self.__populateIndex(fpdir) 
+
+
+  def __populateIndex(self, fpdir):
+    #
+    # read each file, pull md5, add row to database
+    failCount = 0
+    finCount = 0
+    try:
+      db = FingerprintDB()
+      files = os.listdir(fpdir)
+      for file in Files:
+        try:
+          db.scanDBFile(fpdir)
+          finCount = finCount+1
+        except:
+          failCount = failCount+1
+    except:
+
+    logging.info("Completed populating the index. Completed: {}  Failed: {} ".format(str(finCount), str(failCount)))
+    pass
+
+  #
+  def __checkIntegrity(self):
     """ Sanity check the number of files against the index rows """
     pass