Merge branch 'md5func' into fpsave

dbfp.py

@@ -121,6 +121,33 @@ def androidData(data_dir):
   
   print "COMPLETED: created {} fingerprints\n".format(str(fin_count))
 
+#
+def queryMD5(fp_dir, md5_db):
+  try:
+    fp = FingerprintIndex()
+    fp.openIndex(fp_dir)
+    results = fp.findDB(md5_db)
+    print results
+  except Exception as ex:
+    print "ERROR: {}".format(ex)
+
+#
+def insertFP(db_file, fp_file, fp_idx_dir):
+  try:
+    dbfp = FingerprintDB()
+    fpidx = FingerprintIndex()
+    fpidx.openIndex(fp_idx_dir)
+    if (db_file):
+      dbfp.scanDBFile(db_file)
+      # db.debugFingerprint()
+      fpidx.insertFP(dbfp, db_file)
+    elif (fp_file):
+      dbfp.importJson(fp_file)
+      fpidx.insertFP(dbfp, fp_file)
+    print "Insert suceessful"
+  except Exception as ex:
+    print ex
+
 # in_dir:  fully qualified directory path to find sqlite files
 def __createFingerprint(in_dir, out_dir, dir_name):
   fin_count = 0
@@ -175,10 +202,12 @@ def parseArgs():
   parser.add_argument('-ad', '--android_dir', required=False, help="path to a directory with android folder structure sqlite files")
   parser.add_argument('-dd', '--data_dir', required=False, help="path to a directory to search for sqlite files")
   parser.add_argument('-idx', '--index_fingerprints', required=False, help="path to a directory with sqlite files")
+  parser.add_argument('-md5', required=False, help="md5 hash to query the index`")
   parser.add_argument('-an', '--app_name',  required=False)
   parser.add_argument('-av', '--app_version', required=False)
   parser.add_argument('-n', '--notes', required=False)
-  parser.add_argument('-android_pull', required=False, action='store_true', help="automated pull of applications from a physical android phone")
+  parser.add_argument('-idxf', action='store_true', help="add a fingerprint to the index")
+  parser.add_argument('-android_pull', action='store_true', help="automated pull of applications from a physical android phone")
   parser.add_argument('-v', '--verbose', action='store_true', help="will set logging level to INFO")
   parser.add_argument('-vv', '--vverbose', action='store_true', help="will set logging level to DEBUG")
   parser.add_argument('-l', '--logging', action='store_true', help="will supercede the -v option and send all logging to a file, logging.DEBUG")
@@ -195,10 +224,14 @@ def parseArgs():
   else:
     logging.basicConfig(level=logging.CRITICAL)
   
-  if (args.database and args.fingerprint):
+  if args.idxf and args.fpdir and (args.database or args.fingerprint):
+    insertFP(args.database, args.fingerprint, args.fpdir)
+  elif (args.database and args.fingerprint):
     compareFingerprint(args.database, args.fingerprint)      
   elif (args.database and args.fpdir):
     compareFingerprintDir(args.database, args.fpdir)
+  elif (args.fpdir and args.md5):
+    queryMD5(args.fpdir, args.md5)
   elif (args.android_dir):
     androidData(args.android_dir)
   elif (args.index_fingerprints):

docs/devnotes

@@ -3,24 +3,29 @@
 
 Action Items from the code review:
 
--Create a document describing the index file and include an example
+-Add function to query the index for a specific MD5 table (database schema)
+
+-Add a feature to add a fingerprint to the existing index (it currently recreates an index)
 
 -Add a table to the Index to list all the applications that have a fingerprint (include the app version)
   -more functionality can result from this information in the index
 
+-Create a document describing the index file and include an example
 -Create an example of the FingerprintDB class usage with a standalone tool
 
--Add a feature to add a fingerprint to the existing index (it currently recreates an index)
-
--Add function to query the index for a specific MD5 table (database schema)
-
 -Add automated app version discovery to the android pull feature
-
-
 Get App Version
 http://stackoverflow.com/questions/11942762/get-application-version-name-using-adb
 
 
+
+com.google.android.gms__node.db
+0b48447805d645966439e1b4042d2625
+
+"sqlite_sequence": "079355c84d8b3b1511a504e08aab7fd2"
+
+
+
 more testing
 
 

libs/exceptions.py

@@ -10,6 +10,10 @@ class FingerprintIndexOpen(Exception):
   """Error opening an index file"""
   pass
 
+class FingerprintIndexInsert(Exception):
+  """Error creating an index file"""
+  pass
+
 class FingerprintIndexWrite(Exception):
   """Error creating an index file"""
   pass

libs/fingerprint.py

@@ -78,6 +78,11 @@ class FingerprintDB:
       logging.error(ex)
       return -3
 
+    # create and index of table hashes
+    self.table_hashes = {}
+    for key in self.tables.keys():
+      self.table_hashes[key] = self.tables[key].hash()
+
     # flag is used to determine if the class has data
     self.init = True
     self.filein = filein
@@ -144,14 +149,15 @@ class FingerprintDB:
     return self.db_hash
 
   #
-  def getMD5Tables(self):
+  # def getMD5Tables(self):
-    if (self.table_hashes):
+  #   if (self.table_hashes):
-      return self.table_hashes
+  #     return self.table_hashes
 
-    self.table_hashes = []
+  #   self.table_hashes = {}
-    for key in self.tables.keys():
+  #   for key in self.tables.keys():
-      self.table_hashes.append(self.tables[key].hash())
+  #     self.table_hashes[key] = self.tables[key].hash()
-    return self.table_hashes
+  #     # self.table_hashes.append(self.tables[key].hash())
+  #   return self.table_hashes
 
   #
   def __importJsonDBSchema(self, file_json):

libs/fingerprint_index.py

@@ -29,12 +29,16 @@ class FingerprintIndex:
 
   #
   def openIndex(self, fp_dir):
-    fq_fpidx = fp_dir + os.path.sep + INDEX_FILENAME
+    if os.path.isdir(fp_dir):
+      fq_fpidx = fp_dir + os.path.sep + INDEX_FILENAME
+    else:
+      fq_fpidx = fp_dir
+
     try:
       if (os.path.isfile(fq_fpidx)):
         self.db_conn = sql.connect(fq_fpidx)
-        logging.info("DB Open SUCCESSFUL")
         self.cur = self.db_conn.cursor()
+        logging.info("DB Open SUCCESSFUL")
       else:
         logging.info("No index file found, creating index now...")
         self.__createIndex(fp_dir)
@@ -55,9 +59,7 @@ class FingerprintIndex:
   #
   def findFP(self, md5_db, md5_tables):
     rows = self.__qDatabaseMD5(md5_db)    
-    # rowcount will be -1 if nothing was returned
+    if len(rows) > 0:
-    if rows.rowcount > 0:
-      #print "***** __qDatabaseMD5 *****\n{}\n".format(rows)
       return rows
 
     for md5_table in md5_tables:
@@ -72,12 +74,18 @@ class FingerprintIndex:
     return retval.keys()
 
   #
-  def findTable(self, md5_db):
+  def findDB(self, md5_db):
     rows = self.__qDatabaseMD5(md5_db)
-    # rowcount will be -1 if nothing was returned
+    return rows
-    if rows.rowcount > 0:
+
-      #print "***** __qDatabaseMD5 *****\n{}\n".format(rows)
+  def insertFP(self, dbfp, file_name):
-      return rows
+    try:
+      self.__insertMod_md5_all(dbfp.db_hash, dbfp.table_hashes.values(), file_name)
+      self.__insertMod_md5_tables(dbfp.table_hashes.values(), file_name)
+      self.db_conn.commit()
+    except Exception as ex:
+      logging.error(ex)
+      raise FingerprintIndexOpen("Error inserting fingerprint into index file\n")
 
   #
   def __qDatabaseMD5(self, md5_db):
@@ -88,10 +96,10 @@ class FingerprintIndex:
         WHERE md5_db=?
         ''', [md5_db])
 
+      results = []
       for row in rows:
-        # normalize the data, go from tuple to array
+        results.append((row[0], row[1], row[2]))
-        # https://docs.python.org/2/library/sqlite3.html
+      return results
-      return rows
     except Exception as ex:
       logging.error(ex)
 
@@ -143,8 +151,6 @@ class FingerprintIndex:
           # only parese files with .json eextension
           if not re.search(r'.*\.json', file):
             naCount = naCount+1
-            pass
-          #print file
           fq_file = fp_dir + os.path.sep + file
           db.importJson(fq_file)
           self.__insertMod_md5_all(db.db_hash, db.table_hashes.values(), file)
@@ -164,7 +170,7 @@ class FingerprintIndex:
   #
   def __insertMod_md5_all(self, md5_db, md5_list, filename):
     try:
-      # logging.info("INSERT INTO md5_index VALUES(?, ?, ?): {}; {}; {}".format(md5_all, str(md5_list), filename))
+      logging.info("INSERT INTO md5_all VALUES({}, {}, {}, 1)".format(md5_db, ','.join(md5_list), filename))
       self.db_conn.execute(
         '''
         INSERT INTO md5_all VALUES(?, ?, ?, ?)