j3g
/
dbfp_pub
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'md5func' into fpsave

This commit is contained in:
JohnE 2016-02-20 01:47:00 -08:00
parent 4f08d382a8 b19750dcaa
commit 6b0ba75e0c
5 changed files with 88 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
dbfp.py
View File

@ -121,6 +121,33 @@ def androidData(data_dir):
print "COMPLETED: created {} fingerprints\n".format(str(fin_count))
#
def queryMD5(fp_dir, md5_db):
try:
fp = FingerprintIndex()
fp.openIndex(fp_dir)
results = fp.findDB(md5_db)
print results
except Exception as ex:
print "ERROR: {}".format(ex)
#
def insertFP(db_file, fp_file, fp_idx_dir):
try:
dbfp = FingerprintDB()
fpidx = FingerprintIndex()
fpidx.openIndex(fp_idx_dir)
if (db_file):
dbfp.scanDBFile(db_file)
# db.debugFingerprint()
fpidx.insertFP(dbfp, db_file)
elif (fp_file):
dbfp.importJson(fp_file)
fpidx.insertFP(dbfp, fp_file)
print "Insert suceessful"
except Exception as ex:
print ex
# in_dir: fully qualified directory path to find sqlite files
def __createFingerprint(in_dir, out_dir, dir_name):
fin_count = 0
@ -175,10 +202,12 @@ def parseArgs():
parser.add_argument('-ad', '--android_dir', required=False, help="path to a directory with android folder structure sqlite files")
parser.add_argument('-dd', '--data_dir', required=False, help="path to a directory to search for sqlite files")
parser.add_argument('-idx', '--index_fingerprints', required=False, help="path to a directory with sqlite files")
parser.add_argument('-md5', required=False, help="md5 hash to query the index`")
parser.add_argument('-an', '--app_name', required=False)
parser.add_argument('-av', '--app_version', required=False)
parser.add_argument('-n', '--notes', required=False)
parser.add_argument('-android_pull', required=False, action='store_true', help="automated pull of applications from a physical android phone")
parser.add_argument('-idxf', action='store_true', help="add a fingerprint to the index")
parser.add_argument('-android_pull', action='store_true', help="automated pull of applications from a physical android phone")
parser.add_argument('-v', '--verbose', action='store_true', help="will set logging level to INFO")
parser.add_argument('-vv', '--vverbose', action='store_true', help="will set logging level to DEBUG")
parser.add_argument('-l', '--logging', action='store_true', help="will supercede the -v option and send all logging to a file, logging.DEBUG")
@ -195,16 +224,20 @@ def parseArgs():
else:
logging.basicConfig(level=logging.CRITICAL)
if (args.database and args.fingerprint):
if args.idxf and args.fpdir and (args.database or args.fingerprint):
insertFP(args.database, args.fingerprint, args.fpdir)
elif (args.database and args.fingerprint):
compareFingerprint(args.database, args.fingerprint)
elif (args.database and args.fpdir):
compareFingerprintDir(args.database, args.fpdir)
elif (args.fpdir and args.md5):
queryMD5(args.fpdir, args.md5)
elif (args.android_dir):
androidData(args.android_dir)
elif (args.index_fingerprints):
indexFingerprints(args.index_fingerprints)
elif (args.android_pull):
androidPull()
androidPull()
elif (args.database):
createFingerprint(args.database, args.app_name, args.app_version, args.notes)
else:

19
docs/devnotes
View File

@ -3,24 +3,29 @@
Action Items from the code review:
-Create a document describing the index file and include an example
-Add function to query the index for a specific MD5 table (database schema)
-Add a feature to add a fingerprint to the existing index (it currently recreates an index)
-Add a table to the Index to list all the applications that have a fingerprint (include the app version)
-more functionality can result from this information in the index
-Create a document describing the index file and include an example
-Create an example of the FingerprintDB class usage with a standalone tool
-Add a feature to add a fingerprint to the existing index (it currently recreates an index)
-Add function to query the index for a specific MD5 table (database schema)
-Add automated app version discovery to the android pull feature
Get App Version
http://stackoverflow.com/questions/11942762/get-application-version-name-using-adb
com.google.android.gms__node.db
0b48447805d645966439e1b4042d2625
"sqlite_sequence": "079355c84d8b3b1511a504e08aab7fd2"
more testing

4
libs/exceptions.py
View File

@ -10,6 +10,10 @@ class FingerprintIndexOpen(Exception):
"""Error opening an index file"""
pass
class FingerprintIndexInsert(Exception):
"""Error creating an index file"""
pass
class FingerprintIndexWrite(Exception):
"""Error creating an index file"""
pass

20
libs/fingerprint.py
View File

@ -78,6 +78,11 @@ class FingerprintDB:
logging.error(ex)
return -3
# create and index of table hashes
self.table_hashes = {}
for key in self.tables.keys():
self.table_hashes[key] = self.tables[key].hash()
# flag is used to determine if the class has data
self.init = True
self.filein = filein
@ -144,14 +149,15 @@ class FingerprintDB:
return self.db_hash
#
def getMD5Tables(self):
if (self.table_hashes):
return self.table_hashes
# def getMD5Tables(self):
# if (self.table_hashes):
# return self.table_hashes
self.table_hashes = []
for key in self.tables.keys():
self.table_hashes.append(self.tables[key].hash())
return self.table_hashes
# self.table_hashes = {}
# for key in self.tables.keys():
# self.table_hashes[key] = self.tables[key].hash()
# # self.table_hashes.append(self.tables[key].hash())
# return self.table_hashes
#
def __importJsonDBSchema(self, file_json):

40
libs/fingerprint_index.py
View File

@ -29,12 +29,16 @@ class FingerprintIndex:
#
def openIndex(self, fp_dir):
fq_fpidx = fp_dir + os.path.sep + INDEX_FILENAME
if os.path.isdir(fp_dir):
fq_fpidx = fp_dir + os.path.sep + INDEX_FILENAME
else:
fq_fpidx = fp_dir
try:
if (os.path.isfile(fq_fpidx)):
self.db_conn = sql.connect(fq_fpidx)
logging.info("DB Open SUCCESSFUL")
self.cur = self.db_conn.cursor()
logging.info("DB Open SUCCESSFUL")
else:
logging.info("No index file found, creating index now...")
self.__createIndex(fp_dir)
@ -54,10 +58,8 @@ class FingerprintIndex:
#
def findFP(self, md5_db, md5_tables):
rows = self.__qDatabaseMD5(md5_db)
# rowcount will be -1 if nothing was returned
if rows.rowcount > 0:
#print "***** __qDatabaseMD5 *****\n{}\n".format(rows)
rows = self.__qDatabaseMD5(md5_db)
if len(rows) > 0:
return rows
for md5_table in md5_tables:
@ -72,12 +74,18 @@ class FingerprintIndex:
return retval.keys()
#
def findTable(self, md5_db):
def findDB(self, md5_db):
rows = self.__qDatabaseMD5(md5_db)
# rowcount will be -1 if nothing was returned
if rows.rowcount > 0:
#print "***** __qDatabaseMD5 *****\n{}\n".format(rows)
return rows
return rows
def insertFP(self, dbfp, file_name):
try:
self.__insertMod_md5_all(dbfp.db_hash, dbfp.table_hashes.values(), file_name)
self.__insertMod_md5_tables(dbfp.table_hashes.values(), file_name)
self.db_conn.commit()
except Exception as ex:
logging.error(ex)
raise FingerprintIndexOpen("Error inserting fingerprint into index file\n")
#
def __qDatabaseMD5(self, md5_db):
@ -88,10 +96,10 @@ class FingerprintIndex:
WHERE md5_db=?
''', [md5_db])
results = []
for row in rows:
# normalize the data, go from tuple to array
# https://docs.python.org/2/library/sqlite3.html
return rows
results.append((row[0], row[1], row[2]))
return results
except Exception as ex:
logging.error(ex)
@ -143,8 +151,6 @@ class FingerprintIndex:
# only parese files with .json eextension
if not re.search(r'.*\.json', file):
naCount = naCount+1
pass
#print file
fq_file = fp_dir + os.path.sep + file
db.importJson(fq_file)
self.__insertMod_md5_all(db.db_hash, db.table_hashes.values(), file)
@ -164,7 +170,7 @@ class FingerprintIndex:
#
def __insertMod_md5_all(self, md5_db, md5_list, filename):
try:
# logging.info("INSERT INTO md5_index VALUES(?, ?, ?): {}; {}; {}".format(md5_all, str(md5_list), filename))
logging.info("INSERT INTO md5_all VALUES({}, {}, {}, 1)".format(md5_db, ','.join(md5_list), filename))
self.db_conn.execute(
'''
INSERT INTO md5_all VALUES(?, ?, ?, ?)