WIP: optimising the fingerprint lookup/comparison

dbfp.py

@@ -61,20 +61,19 @@ def indexFingerprints(fp_dir):
 
 #
 def compareFPIndex(fp_dir, db_in, json_in):
-  try:
+  # try:
     db = FingerprintDB()
     if (db_in):
       db.scanDBFile(db_in)
     else:
       db.importJson(json_in)
 
-    logging.info("MD5 DB == {}".format(db.getMD5DB()))
-    logging.info("MD5 TB == {}".format(db.getMD5Tables()))
 
     fp = FingerprintIndex()
     fp.openIndex(fp_dir)
 
     # search for fingerprints with exact database match
+    logging.info("Searching for MD5 DB: {}".format(db.getMD5DB()))
     fp_ret = fp.findFP(db.getMD5DB())
     if (fp_ret):
       print "Database matche(s) found"
@@ -85,11 +84,11 @@ def compareFPIndex(fp_dir, db_in, json_in):
     # search for fingerprints with similar tables
     else:
       logging.info("Searching for md5 tables: {}".format(db.getMD5Tables()))
-      fp_list = fp.findFPTables(db.getMD5Tables())
+      fp_list = fp.findFPTables(db.getMD5Tables().values())
       results = []
       for fp in fp_list:
         fq_fp = fp_dir + os.path.sep + fp
-        print "[ OPEN fingerprint ] [ {} ]".format(fq_fp)
+        logging.info("Comparing fingerprint: {}".format(fq_fp))
         percent = db.compareDB(fq_fp)
         results.append(percent)
 
@@ -97,9 +96,9 @@ def compareFPIndex(fp_dir, db_in, json_in):
       print "RESULTS: {}".format(results)
       results.sort()
       print "RESULTS: {}".format(results)
-  except Exception as ex:
-    print "Error comparing fingerprint"
-    print ex
+  # except Exception as ex:
+  #   print "Error comparing fingerprint"
+  #   print ex
 
 #
 def androidPull():

docs/devnotes

@@ -3,7 +3,7 @@
 
 Action Items from the code review:
 
-1) Change table name from "metadata" to "app_details"
+xx1) Change table name from "metadata" to "app_details"
 
 1) Regression Testing of all Features
 

libs/fingerprint_index.py

@@ -8,6 +8,7 @@ import os
 import sys
 import logging
 import sqlite3 as sql
+from operator import itemgetter
 from libs.exceptions import FingerprintIndexWrite
 from libs.exceptions import FingerprintIndexOpen
 from libs.fingerprint import FingerprintDB
@@ -80,17 +81,33 @@ class FingerprintIndex:
       return rows
     return None
 
-  # return an a list of json fingerprint files to open
+  # return an a sort array with the most likely fingerprints ordered first
   def findFPTables(self, md5_tables):
     retval = {}
     for md5_table in md5_tables:
+      logging.debug("findFPTables::md5_table=={}".format(md5_table))
       rows = self.__qTableMD5(md5_table)
       for row in rows:
+        logging.debug("findFPTables::row=={}\n".format(row[0]))
         fp_list = row[0]
         fps = fp_list.split(',')
         for fp in fps:
-          retval[fp] = 1
-    return retval.keys()
+          if (retval.has_key(fp)):
+            retval[fp] = retval[fp] + 1
+          else:
+            retval[fp] = 1
+    logging.debug("findFPTables::retval=={}\n".format(retval))
+
+    # retval_sorted = sorted(retval.items(), key==lambda x: x[1])
+    retval_sorted = sorted(retval.items(), key=itemgetter(1))
+    # retval_sorted = []
+    # vals = retval.values()
+    # vals.sort()
+    # for val in vals:
+    #   retval_sorted.append(retval[val])
+    logging.debug("findFPTables::retval_sorted=={}\n".format(retval_sorted))    
+    return retval_sorted
+    # return retval.keys()
 
   #
   def findDB(self, md5_db):