From dbdfadbe5a510862825306ee3afdc414c568917b Mon Sep 17 00:00:00 2001 From: JohnE Date: Thu, 18 Feb 2016 19:54:15 -0800 Subject: [PATCH 1/5] MOD: merged changes from Nathan' --- docs/devnotes | 11 ++++------- libs/fingerprint_index.py | 6 +++++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/devnotes b/docs/devnotes index 4d0cb64..ab9491f 100644 --- a/docs/devnotes +++ b/docs/devnotes @@ -3,20 +3,17 @@ Action Items from the code review: --Create a document describing the index file and include an example +-Add function to query the index for a specific MD5 table (database schema) + +-Add a feature to add a fingerprint to the existing index (it currently recreates an index) -Add a table to the Index to list all the applications that have a fingerprint (include the app version) -more functionality can result from this information in the index +-Create a document describing the index file and include an example -Create an example of the FingerprintDB class usage with a standalone tool --Add a feature to add a fingerprint to the existing index (it currently recreates an index) - --Add function to query the index for a specific MD5 table (database schema) - -Add automated app version discovery to the android pull feature - - Get App Version http://stackoverflow.com/questions/11942762/get-application-version-name-using-adb diff --git a/libs/fingerprint_index.py b/libs/fingerprint_index.py index 4ad95f8..089705a 100644 --- a/libs/fingerprint_index.py +++ b/libs/fingerprint_index.py @@ -29,7 +29,11 @@ class FingerprintIndex: # def openIndex(self, fp_dir): - fq_fpidx = fp_dir + os.path.sep + INDEX_FILENAME + if os.path.isdir(fp_dir): + fq_fpidx = fp_dir + os.path.sep + INDEX_FILENAME + else: + fq_fpidx = fp_dir + try: if (os.path.isfile(fq_fpidx)): self.db_conn = sql.connect(fq_fpidx) From 817530d135718364ba4ba2081dddb1878d6f44f5 Mon Sep 17 00:00:00 2001 From: JohnE Date: Thu, 18 Feb 2016 23:09:34 -0800 Subject: [PATCH 2/5] MOD: notes --- docs/devnotes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/devnotes b/docs/devnotes index ab9491f..2cbd0ec 100644 --- a/docs/devnotes +++ b/docs/devnotes @@ -18,6 +18,8 @@ Get App Version http://stackoverflow.com/questions/11942762/get-application-version-name-using-adb + + [ Regression Testing ] android pull, data dir, comparison From 0239696efbd8f96c7638683f602e1abba3a253c9 Mon Sep 17 00:00:00 2001 From: JohnE Date: Fri, 19 Feb 2016 01:33:52 -0800 Subject: [PATCH 3/5] NEW: md5 serach complete --- dbfp.py | 16 +++++++++++++++- docs/devnotes | 4 ++++ libs/fingerprint_index.py | 19 +++++++------------ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/dbfp.py b/dbfp.py index ca531de..80e49aa 100644 --- a/dbfp.py +++ b/dbfp.py @@ -121,6 +121,17 @@ def androidData(data_dir): print "COMPLETED: created {} fingerprints\n".format(str(fin_count)) +# +def queryMD5(fp_dir, md5_db): + try: + fp = FingerprintIndex() + fp.openIndex(fp_dir) + results = fp.findDB(md5_db) + print results + except Exception as ex: + print "ERROR: {}".format(ex) + + # in_dir: fully qualified directory path to find sqlite files def __createFingerprint(in_dir, out_dir, dir_name): fin_count = 0 @@ -175,6 +186,7 @@ def parseArgs(): parser.add_argument('-ad', '--android_dir', required=False, help="path to a directory with android folder structure sqlite files") parser.add_argument('-dd', '--data_dir', required=False, help="path to a directory to search for sqlite files") parser.add_argument('-idx', '--index_fingerprints', required=False, help="path to a directory with sqlite files") + parser.add_argument('-md5', required=False, help="md5 hash to query the index`") parser.add_argument('-an', '--app_name', required=False) parser.add_argument('-av', '--app_version', required=False) parser.add_argument('-n', '--notes', required=False) @@ -199,12 +211,14 @@ def parseArgs(): compareFingerprint(args.database, args.fingerprint) elif (args.database and args.fpdir): compareFingerprintDir(args.database, args.fpdir) + elif (args.fpdir and args.md5): + queryMD5(args.fpdir, args.md5) elif (args.android_dir): androidData(args.android_dir) elif (args.index_fingerprints): indexFingerprints(args.index_fingerprints) elif (args.android_pull): - androidPull() + androidPull() elif (args.database): createFingerprint(args.database, args.app_name, args.app_version, args.notes) else: diff --git a/docs/devnotes b/docs/devnotes index 2cbd0ec..acb0b36 100644 --- a/docs/devnotes +++ b/docs/devnotes @@ -19,6 +19,10 @@ http://stackoverflow.com/questions/11942762/get-application-version-name-using-a +com.google.android.gms__node.db +0b48447805d645966439e1b4042d2625 + + [ Regression Testing ] android pull, data dir, comparison diff --git a/libs/fingerprint_index.py b/libs/fingerprint_index.py index 089705a..7f447ab 100644 --- a/libs/fingerprint_index.py +++ b/libs/fingerprint_index.py @@ -58,10 +58,8 @@ class FingerprintIndex: # def findFP(self, md5_db, md5_tables): - rows = self.__qDatabaseMD5(md5_db) - # rowcount will be -1 if nothing was returned - if rows.rowcount > 0: - #print "***** __qDatabaseMD5 *****\n{}\n".format(rows) + rows = self.__qDatabaseMD5(md5_db) + if len(rows) > 0: return rows for md5_table in md5_tables: @@ -76,12 +74,9 @@ class FingerprintIndex: return retval.keys() # - def findTable(self, md5_db): + def findDB(self, md5_db): rows = self.__qDatabaseMD5(md5_db) - # rowcount will be -1 if nothing was returned - if rows.rowcount > 0: - #print "***** __qDatabaseMD5 *****\n{}\n".format(rows) - return rows + return rows # def __qDatabaseMD5(self, md5_db): @@ -92,10 +87,10 @@ class FingerprintIndex: WHERE md5_db=? ''', [md5_db]) + results = [] for row in rows: - # normalize the data, go from tuple to array - # https://docs.python.org/2/library/sqlite3.html - return rows + results.append((row[0], row[1], row[2])) + return results except Exception as ex: logging.error(ex) From 3afbd57d78372423b3a15d162ac46dac0991df80 Mon Sep 17 00:00:00 2001 From: JohnE Date: Fri, 19 Feb 2016 23:13:25 -0800 Subject: [PATCH 4/5] MOD: insert working with an exception --- dbfp.py | 19 +++++++++++++++++-- docs/devnotes | 2 ++ libs/exceptions.py | 4 ++++ libs/fingerprint.py | 20 +++++++++++++------- libs/fingerprint_index.py | 12 ++++++++++-- 5 files changed, 46 insertions(+), 11 deletions(-) diff --git a/dbfp.py b/dbfp.py index 80e49aa..f3d5634 100644 --- a/dbfp.py +++ b/dbfp.py @@ -131,6 +131,18 @@ def queryMD5(fp_dir, md5_db): except Exception as ex: print "ERROR: {}".format(ex) +# +def insertFP(db_file, fp_file, fp_idx_dir): + dbfp = FingerprintDB() + fpidx = FingerprintIndex() + fpidx.openIndex(fp_idx_dir) + if (db_file): + dbfp.scanDBFile(db_file) + # db.debugFingerprint() + fpidx.insertFP(dbfp, db_file) + elif (fp_file): + db.importJson(fp_file) + fpidx.insertFP(dbfp, fp_file) # in_dir: fully qualified directory path to find sqlite files def __createFingerprint(in_dir, out_dir, dir_name): @@ -190,7 +202,8 @@ def parseArgs(): parser.add_argument('-an', '--app_name', required=False) parser.add_argument('-av', '--app_version', required=False) parser.add_argument('-n', '--notes', required=False) - parser.add_argument('-android_pull', required=False, action='store_true', help="automated pull of applications from a physical android phone") + parser.add_argument('-idxf', action='store_true', help="add a fingerprint to the index") + parser.add_argument('-android_pull', action='store_true', help="automated pull of applications from a physical android phone") parser.add_argument('-v', '--verbose', action='store_true', help="will set logging level to INFO") parser.add_argument('-vv', '--vverbose', action='store_true', help="will set logging level to DEBUG") parser.add_argument('-l', '--logging', action='store_true', help="will supercede the -v option and send all logging to a file, logging.DEBUG") @@ -207,7 +220,9 @@ def parseArgs(): else: logging.basicConfig(level=logging.CRITICAL) - if (args.database and args.fingerprint): + if args.idxf and args.fpdir and (args.database or args.fingerprint): + insertFP(args.database, args.fingerprint, args.fpdir) + elif (args.database and args.fingerprint): compareFingerprint(args.database, args.fingerprint) elif (args.database and args.fpdir): compareFingerprintDir(args.database, args.fpdir) diff --git a/docs/devnotes b/docs/devnotes index acb0b36..8249ffc 100644 --- a/docs/devnotes +++ b/docs/devnotes @@ -22,6 +22,8 @@ http://stackoverflow.com/questions/11942762/get-application-version-name-using-a com.google.android.gms__node.db 0b48447805d645966439e1b4042d2625 +"sqlite_sequence": "079355c84d8b3b1511a504e08aab7fd2" + [ Regression Testing ] diff --git a/libs/exceptions.py b/libs/exceptions.py index 3633d65..28b5a56 100644 --- a/libs/exceptions.py +++ b/libs/exceptions.py @@ -10,6 +10,10 @@ class FingerprintIndexOpen(Exception): """Error opening an index file""" pass +class FingerprintIndexInsert(Exception): + """Error creating an index file""" + pass + class FingerprintIndexWrite(Exception): """Error creating an index file""" pass diff --git a/libs/fingerprint.py b/libs/fingerprint.py index ec44aff..bdf1117 100644 --- a/libs/fingerprint.py +++ b/libs/fingerprint.py @@ -78,6 +78,11 @@ class FingerprintDB: logging.error(ex) return -3 + # create and index of table hashes + self.table_hashes = {} + for key in self.tables.keys(): + self.table_hashes[key] = self.tables[key].hash() + # flag is used to determine if the class has data self.init = True self.filein = filein @@ -144,14 +149,15 @@ class FingerprintDB: return self.db_hash # - def getMD5Tables(self): - if (self.table_hashes): - return self.table_hashes + # def getMD5Tables(self): + # if (self.table_hashes): + # return self.table_hashes - self.table_hashes = [] - for key in self.tables.keys(): - self.table_hashes.append(self.tables[key].hash()) - return self.table_hashes + # self.table_hashes = {} + # for key in self.tables.keys(): + # self.table_hashes[key] = self.tables[key].hash() + # # self.table_hashes.append(self.tables[key].hash()) + # return self.table_hashes # def __importJsonDBSchema(self, file_json): diff --git a/libs/fingerprint_index.py b/libs/fingerprint_index.py index 7f447ab..82acb60 100644 --- a/libs/fingerprint_index.py +++ b/libs/fingerprint_index.py @@ -78,6 +78,16 @@ class FingerprintIndex: rows = self.__qDatabaseMD5(md5_db) return rows + def insertFP(self, dbfp, file_name): + print "***** ***** WTF ***** *****" + try: + print "WTF222: {}".format(dbfp.table_hashes.values()) + self.__insertMod_md5_all(dbfp.db_hash, dbfp.table_hashes.values(), file_name) + self.__insertMod_md5_tables(dbfp.table_hashes.values(), file_name) + except Exception as ex: + logging.error(ex) + raise FingerprintIndexOpen("Error inserting fingerprint into index file\n") + # def __qDatabaseMD5(self, md5_db): try: @@ -142,8 +152,6 @@ class FingerprintIndex: # only parese files with .json eextension if not re.search(r'.*\.json', file): naCount = naCount+1 - pass - #print file fq_file = fp_dir + os.path.sep + file db.importJson(fq_file) self.__insertMod_md5_all(db.db_hash, db.table_hashes.values(), file) From b19750dcaa0969281ac21a92f330e17b9b89b630 Mon Sep 17 00:00:00 2001 From: JohnE Date: Sat, 20 Feb 2016 01:42:47 -0800 Subject: [PATCH 5/5] FIN: fingerprint insert complete --- dbfp.py | 24 ++++++++++++++---------- libs/fingerprint_index.py | 7 +++---- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/dbfp.py b/dbfp.py index f3d5634..3c78ca4 100644 --- a/dbfp.py +++ b/dbfp.py @@ -133,16 +133,20 @@ def queryMD5(fp_dir, md5_db): # def insertFP(db_file, fp_file, fp_idx_dir): - dbfp = FingerprintDB() - fpidx = FingerprintIndex() - fpidx.openIndex(fp_idx_dir) - if (db_file): - dbfp.scanDBFile(db_file) - # db.debugFingerprint() - fpidx.insertFP(dbfp, db_file) - elif (fp_file): - db.importJson(fp_file) - fpidx.insertFP(dbfp, fp_file) + try: + dbfp = FingerprintDB() + fpidx = FingerprintIndex() + fpidx.openIndex(fp_idx_dir) + if (db_file): + dbfp.scanDBFile(db_file) + # db.debugFingerprint() + fpidx.insertFP(dbfp, db_file) + elif (fp_file): + dbfp.importJson(fp_file) + fpidx.insertFP(dbfp, fp_file) + print "Insert suceessful" + except Exception as ex: + print ex # in_dir: fully qualified directory path to find sqlite files def __createFingerprint(in_dir, out_dir, dir_name): diff --git a/libs/fingerprint_index.py b/libs/fingerprint_index.py index 82acb60..d52cdcf 100644 --- a/libs/fingerprint_index.py +++ b/libs/fingerprint_index.py @@ -37,8 +37,8 @@ class FingerprintIndex: try: if (os.path.isfile(fq_fpidx)): self.db_conn = sql.connect(fq_fpidx) - logging.info("DB Open SUCCESSFUL") self.cur = self.db_conn.cursor() + logging.info("DB Open SUCCESSFUL") else: logging.info("No index file found, creating index now...") self.__createIndex(fp_dir) @@ -79,11 +79,10 @@ class FingerprintIndex: return rows def insertFP(self, dbfp, file_name): - print "***** ***** WTF ***** *****" try: - print "WTF222: {}".format(dbfp.table_hashes.values()) self.__insertMod_md5_all(dbfp.db_hash, dbfp.table_hashes.values(), file_name) self.__insertMod_md5_tables(dbfp.table_hashes.values(), file_name) + self.db_conn.commit() except Exception as ex: logging.error(ex) raise FingerprintIndexOpen("Error inserting fingerprint into index file\n") @@ -171,7 +170,7 @@ class FingerprintIndex: # def __insertMod_md5_all(self, md5_db, md5_list, filename): try: - # logging.info("INSERT INTO md5_index VALUES(?, ?, ?): {}; {}; {}".format(md5_all, str(md5_list), filename)) + logging.info("INSERT INTO md5_all VALUES({}, {}, {}, 1)".format(md5_db, ','.join(md5_list), filename)) self.db_conn.execute( ''' INSERT INTO md5_all VALUES(?, ?, ?, ?)