diff --git a/dbfp.py b/dbfp.py index ca531de..3c78ca4 100644 --- a/dbfp.py +++ b/dbfp.py @@ -121,6 +121,33 @@ def androidData(data_dir): print "COMPLETED: created {} fingerprints\n".format(str(fin_count)) +# +def queryMD5(fp_dir, md5_db): + try: + fp = FingerprintIndex() + fp.openIndex(fp_dir) + results = fp.findDB(md5_db) + print results + except Exception as ex: + print "ERROR: {}".format(ex) + +# +def insertFP(db_file, fp_file, fp_idx_dir): + try: + dbfp = FingerprintDB() + fpidx = FingerprintIndex() + fpidx.openIndex(fp_idx_dir) + if (db_file): + dbfp.scanDBFile(db_file) + # db.debugFingerprint() + fpidx.insertFP(dbfp, db_file) + elif (fp_file): + dbfp.importJson(fp_file) + fpidx.insertFP(dbfp, fp_file) + print "Insert suceessful" + except Exception as ex: + print ex + # in_dir: fully qualified directory path to find sqlite files def __createFingerprint(in_dir, out_dir, dir_name): fin_count = 0 @@ -175,10 +202,12 @@ def parseArgs(): parser.add_argument('-ad', '--android_dir', required=False, help="path to a directory with android folder structure sqlite files") parser.add_argument('-dd', '--data_dir', required=False, help="path to a directory to search for sqlite files") parser.add_argument('-idx', '--index_fingerprints', required=False, help="path to a directory with sqlite files") + parser.add_argument('-md5', required=False, help="md5 hash to query the index`") parser.add_argument('-an', '--app_name', required=False) parser.add_argument('-av', '--app_version', required=False) parser.add_argument('-n', '--notes', required=False) - parser.add_argument('-android_pull', required=False, action='store_true', help="automated pull of applications from a physical android phone") + parser.add_argument('-idxf', action='store_true', help="add a fingerprint to the index") + parser.add_argument('-android_pull', action='store_true', help="automated pull of applications from a physical android phone") parser.add_argument('-v', '--verbose', action='store_true', help="will set logging level to INFO") parser.add_argument('-vv', '--vverbose', action='store_true', help="will set logging level to DEBUG") parser.add_argument('-l', '--logging', action='store_true', help="will supercede the -v option and send all logging to a file, logging.DEBUG") @@ -195,16 +224,20 @@ def parseArgs(): else: logging.basicConfig(level=logging.CRITICAL) - if (args.database and args.fingerprint): + if args.idxf and args.fpdir and (args.database or args.fingerprint): + insertFP(args.database, args.fingerprint, args.fpdir) + elif (args.database and args.fingerprint): compareFingerprint(args.database, args.fingerprint) elif (args.database and args.fpdir): compareFingerprintDir(args.database, args.fpdir) + elif (args.fpdir and args.md5): + queryMD5(args.fpdir, args.md5) elif (args.android_dir): androidData(args.android_dir) elif (args.index_fingerprints): indexFingerprints(args.index_fingerprints) elif (args.android_pull): - androidPull() + androidPull() elif (args.database): createFingerprint(args.database, args.app_name, args.app_version, args.notes) else: diff --git a/docs/devnotes b/docs/devnotes index 4e153d3..216196b 100644 --- a/docs/devnotes +++ b/docs/devnotes @@ -3,24 +3,29 @@ Action Items from the code review: --Create a document describing the index file and include an example +-Add function to query the index for a specific MD5 table (database schema) + +-Add a feature to add a fingerprint to the existing index (it currently recreates an index) -Add a table to the Index to list all the applications that have a fingerprint (include the app version) -more functionality can result from this information in the index +-Create a document describing the index file and include an example -Create an example of the FingerprintDB class usage with a standalone tool --Add a feature to add a fingerprint to the existing index (it currently recreates an index) - --Add function to query the index for a specific MD5 table (database schema) - -Add automated app version discovery to the android pull feature - - Get App Version http://stackoverflow.com/questions/11942762/get-application-version-name-using-adb + +com.google.android.gms__node.db +0b48447805d645966439e1b4042d2625 + +"sqlite_sequence": "079355c84d8b3b1511a504e08aab7fd2" + + + more testing diff --git a/libs/exceptions.py b/libs/exceptions.py index 3633d65..28b5a56 100644 --- a/libs/exceptions.py +++ b/libs/exceptions.py @@ -10,6 +10,10 @@ class FingerprintIndexOpen(Exception): """Error opening an index file""" pass +class FingerprintIndexInsert(Exception): + """Error creating an index file""" + pass + class FingerprintIndexWrite(Exception): """Error creating an index file""" pass diff --git a/libs/fingerprint.py b/libs/fingerprint.py index ec44aff..bdf1117 100644 --- a/libs/fingerprint.py +++ b/libs/fingerprint.py @@ -78,6 +78,11 @@ class FingerprintDB: logging.error(ex) return -3 + # create and index of table hashes + self.table_hashes = {} + for key in self.tables.keys(): + self.table_hashes[key] = self.tables[key].hash() + # flag is used to determine if the class has data self.init = True self.filein = filein @@ -144,14 +149,15 @@ class FingerprintDB: return self.db_hash # - def getMD5Tables(self): - if (self.table_hashes): - return self.table_hashes + # def getMD5Tables(self): + # if (self.table_hashes): + # return self.table_hashes - self.table_hashes = [] - for key in self.tables.keys(): - self.table_hashes.append(self.tables[key].hash()) - return self.table_hashes + # self.table_hashes = {} + # for key in self.tables.keys(): + # self.table_hashes[key] = self.tables[key].hash() + # # self.table_hashes.append(self.tables[key].hash()) + # return self.table_hashes # def __importJsonDBSchema(self, file_json): diff --git a/libs/fingerprint_index.py b/libs/fingerprint_index.py index 4ad95f8..d52cdcf 100644 --- a/libs/fingerprint_index.py +++ b/libs/fingerprint_index.py @@ -29,12 +29,16 @@ class FingerprintIndex: # def openIndex(self, fp_dir): - fq_fpidx = fp_dir + os.path.sep + INDEX_FILENAME + if os.path.isdir(fp_dir): + fq_fpidx = fp_dir + os.path.sep + INDEX_FILENAME + else: + fq_fpidx = fp_dir + try: if (os.path.isfile(fq_fpidx)): self.db_conn = sql.connect(fq_fpidx) - logging.info("DB Open SUCCESSFUL") self.cur = self.db_conn.cursor() + logging.info("DB Open SUCCESSFUL") else: logging.info("No index file found, creating index now...") self.__createIndex(fp_dir) @@ -54,10 +58,8 @@ class FingerprintIndex: # def findFP(self, md5_db, md5_tables): - rows = self.__qDatabaseMD5(md5_db) - # rowcount will be -1 if nothing was returned - if rows.rowcount > 0: - #print "***** __qDatabaseMD5 *****\n{}\n".format(rows) + rows = self.__qDatabaseMD5(md5_db) + if len(rows) > 0: return rows for md5_table in md5_tables: @@ -72,12 +74,18 @@ class FingerprintIndex: return retval.keys() # - def findTable(self, md5_db): + def findDB(self, md5_db): rows = self.__qDatabaseMD5(md5_db) - # rowcount will be -1 if nothing was returned - if rows.rowcount > 0: - #print "***** __qDatabaseMD5 *****\n{}\n".format(rows) - return rows + return rows + + def insertFP(self, dbfp, file_name): + try: + self.__insertMod_md5_all(dbfp.db_hash, dbfp.table_hashes.values(), file_name) + self.__insertMod_md5_tables(dbfp.table_hashes.values(), file_name) + self.db_conn.commit() + except Exception as ex: + logging.error(ex) + raise FingerprintIndexOpen("Error inserting fingerprint into index file\n") # def __qDatabaseMD5(self, md5_db): @@ -88,10 +96,10 @@ class FingerprintIndex: WHERE md5_db=? ''', [md5_db]) + results = [] for row in rows: - # normalize the data, go from tuple to array - # https://docs.python.org/2/library/sqlite3.html - return rows + results.append((row[0], row[1], row[2])) + return results except Exception as ex: logging.error(ex) @@ -143,8 +151,6 @@ class FingerprintIndex: # only parese files with .json eextension if not re.search(r'.*\.json', file): naCount = naCount+1 - pass - #print file fq_file = fp_dir + os.path.sep + file db.importJson(fq_file) self.__insertMod_md5_all(db.db_hash, db.table_hashes.values(), file) @@ -164,7 +170,7 @@ class FingerprintIndex: # def __insertMod_md5_all(self, md5_db, md5_list, filename): try: - # logging.info("INSERT INTO md5_index VALUES(?, ?, ?): {}; {}; {}".format(md5_all, str(md5_list), filename)) + logging.info("INSERT INTO md5_all VALUES({}, {}, {}, 1)".format(md5_db, ','.join(md5_list), filename)) self.db_conn.execute( ''' INSERT INTO md5_all VALUES(?, ?, ?, ?)