292 lines
9.5 KiB
Python
292 lines
9.5 KiB
Python
#
|
|
#
|
|
#
|
|
import os
|
|
import argparse
|
|
import logging
|
|
from subprocess import Popen, PIPE, check_call
|
|
# our libs
|
|
from libs import android
|
|
from libs.toolbox import ToolBox
|
|
from libs.fingerprint import FingerprintDB
|
|
from libs.fingerprint_index import FingerprintIndex
|
|
|
|
BASE_DIR = "data"
|
|
FP_BASE_DIR = "fingerprints"
|
|
|
|
|
|
def main():
|
|
parseArgs()
|
|
|
|
#
|
|
def compareFingerprint(file_in, file_json):
|
|
db = FingerprintDB()
|
|
db.scanDBFile(file_in)
|
|
percent = db.compareDB(file_json)
|
|
print "Percent match: {}".format(str(percent))
|
|
|
|
#
|
|
def createFingerprint(file_in, app_name, app_ver, notes):
|
|
print "Reading database file: {}".format(file_in)
|
|
try:
|
|
db = FingerprintDB()
|
|
db.scanDBFile(file_in)
|
|
if app_name:
|
|
db.setAppName(app_name)
|
|
if app_ver:
|
|
db.setAppVer(app_ver)
|
|
if notes:
|
|
db.setNotes(notes)
|
|
filename = db.writeFingerprint()
|
|
print "Fingerprint generated: {}".format(filename)
|
|
except Exception as ex:
|
|
print ex
|
|
|
|
#
|
|
def indexFingerprints(fp_dir):
|
|
db = FingerprintDB()
|
|
fp = FingerprintIndex()
|
|
try:
|
|
fp.openIndex(fp_dir)
|
|
print "WARN: index already exists. remove, then reindex"
|
|
except:
|
|
try:
|
|
retVal = fp.createIndex(fp_dir)
|
|
print "Index created:"
|
|
print " {} fingerprints processed.".format(retVal[0])
|
|
print " {} fingprint processing errors.".format(retVal[1])
|
|
print " {} files skipped (N/A, not a fingerprint file)".format(retVal[2])
|
|
except Exception as ex:
|
|
print ex
|
|
|
|
#
|
|
def compareFPIndex(fp_dir, db_in, json_in):
|
|
try:
|
|
db = FingerprintDB()
|
|
if (db_in):
|
|
db.scanDBFile(db_in)
|
|
else:
|
|
db.importJson(json_in)
|
|
|
|
fp = FingerprintIndex()
|
|
fp.openIndex(fp_dir)
|
|
|
|
# search for fingerprints with exact database match
|
|
logging.info("Searching for MD5 DB: {}".format(db.getMD5DB()))
|
|
fp_ret = fp.findFP(db.getMD5DB())
|
|
if (fp_ret):
|
|
print "Database matche(s) found"
|
|
print "RESULTS:"
|
|
for fp_list in fp_ret:
|
|
for fp in fp_list[1].split(','):
|
|
print fp
|
|
# search for fingerprints with similar tables
|
|
else:
|
|
print "[ Table percent match: ]"
|
|
logging.info("Searching for md5 tables: {}".format(db.getMD5Tables()))
|
|
fp_list = fp.findFPTables(db.getMD5Tables().values())
|
|
for fp in fp_list:
|
|
fq_fp = fp_dir + os.path.sep + fp[0]
|
|
logging.info("Comparing fingerprint: {}".format(fq_fp))
|
|
percent = db.compareDB(fq_fp)
|
|
print "{:2.2f}%: {}".format(percent, fp[0])
|
|
except Exception as ex:
|
|
print "ERROR: error occured while comparing fingerprint"
|
|
print ex
|
|
|
|
#
|
|
def androidPull():
|
|
fin_count = 0
|
|
print "Android pull started..."
|
|
ap = android.AndroidAppPull()
|
|
isRoot = ap.isADBRoot();
|
|
if (not isRoot):
|
|
print "ERROR: adb is not running as root, exec 'adb root'"
|
|
return
|
|
|
|
if (not mkdir(BASE_DIR)):
|
|
print "ERROR creating directory: {}".format(BASE_DIR)
|
|
return
|
|
if (not mkdir(FP_BASE_DIR)):
|
|
print "ERROR creating directory: {}".format(FP_BASE_DIR)
|
|
return
|
|
|
|
dir_names = ap.getAppsDir()
|
|
for dir_name in dir_names:
|
|
print "Processing directory [{}]".format(dir_name)
|
|
ap.pullApp(BASE_DIR, dir_name)
|
|
fq_dir = BASE_DIR + os.path.sep + dir_name
|
|
count = __createFingerprint(fq_dir, FP_BASE_DIR, dir_name)
|
|
fin_count += count
|
|
print "Fingerprints created: {}".format(str(count))
|
|
|
|
print "\nTotal Fingerprints created: {}".format(str(fin_count))
|
|
|
|
#
|
|
def androidData(data_dir):
|
|
dir_names = []
|
|
try:
|
|
dirs = os.listdir(data_dir)
|
|
print "Opening directory: [{}], [{} folders found]".format(data_dir, len(dirs))
|
|
except Exception, ex:
|
|
print "ERROR opening Android Data Directory (-dd): {}\n{}".format(data_dir, ex)
|
|
return
|
|
|
|
out_dir = FP_BASE_DIR + "_" + ToolBox.getTimestampStr()
|
|
if (not mkdir(out_dir)):
|
|
print "ERROR creating directory: {}".format(FP_BASE_DIR)
|
|
return
|
|
|
|
fin_count = 0
|
|
for ddir in dirs:
|
|
in_dir = data_dir + os.path.sep + ddir
|
|
count = __createFingerprint(in_dir, out_dir, ddir)
|
|
fin_count += count
|
|
|
|
print "COMPLETED: created {} fingerprints".format(str(fin_count))
|
|
|
|
#
|
|
def queryMD5(fp_dir, md5_db):
|
|
try:
|
|
fp = FingerprintIndex()
|
|
fp.openIndex(fp_dir)
|
|
results = fp.queryAppDetails(md5_db)
|
|
for row in results:
|
|
print "[{}]\nDB: {}\nFP: {}\nDate: {}\n".format(row[0], row[2], row[3], row[4])
|
|
except Exception as ex:
|
|
print "ERROR: {}".format(ex)
|
|
|
|
#
|
|
def insertFP(db_file, fp_file, fp_idx_dir):
|
|
try:
|
|
dbfp = FingerprintDB()
|
|
fpidx = FingerprintIndex()
|
|
fpidx.openIndex(fp_idx_dir)
|
|
if (db_file):
|
|
dbfp.scanDBFile(db_file)
|
|
# db.debugFingerprint()
|
|
fpidx.insertFP(dbfp, db_file)
|
|
elif (fp_file):
|
|
dbfp.importJson(fp_file)
|
|
fpidx.insertFP(dbfp, fp_file)
|
|
print "Insert suceessful"
|
|
except Exception as ex:
|
|
print ex
|
|
|
|
# in_dir: fully qualified directory path to find sqlite files
|
|
def __createFingerprint(in_dir, out_dir, dir_name):
|
|
fin_count = 0
|
|
try:
|
|
db_dir = in_dir + os.path.sep + "databases"
|
|
#logging.info("in_dir=={}".format(db_dir))
|
|
files = os.listdir(db_dir)
|
|
except:
|
|
# not finding a databases folder is normal, not all apps use sqlite
|
|
return fin_count
|
|
for filein in files:
|
|
try:
|
|
db = FingerprintDB()
|
|
ddir = db_dir + os.path.sep + filein
|
|
logging.info('Parsing file "{}"'.format(ddir))
|
|
db.scanDBFile(ddir)
|
|
fname = dir_name + "__" + filein + "__dbfp" + ".json"
|
|
fq_name = out_dir + os.path.sep + fname
|
|
db.setAppName(dir_name)
|
|
db.writeFingerprintFile(fq_name)
|
|
fin_count += 1
|
|
except Exception as ex:
|
|
# log error, but move on in hopes of writing more fingerprints
|
|
logging.error(ex)
|
|
return fin_count
|
|
|
|
#
|
|
def __getFileName():
|
|
'''standardize on a file name, use timestamp? '''
|
|
pass
|
|
|
|
#
|
|
def mkdir(fdir):
|
|
retval = False
|
|
try:
|
|
check_call(["mkdir", fdir])
|
|
retval = True
|
|
except:
|
|
print 'ERROR: problem creating directory "{}"'.format(fdir)
|
|
return retval
|
|
|
|
#
|
|
def parseArgs():
|
|
print '***** ***** ***** *****'
|
|
print ' DB Fingerprint'
|
|
print '***** ***** ***** *****\n'
|
|
parser = argparse.ArgumentParser(description="Fingerprint a sqlite database based on its schema")
|
|
parser.add_argument('-db', '--database', required=False, help="path to file to be fingerprinted")
|
|
parser.add_argument('-fd', '--fpdir', required=False, help="path to directory of fingerprint files, compare each file")
|
|
parser.add_argument('-fp', '--fingerprint', required=False, help="fingerprint file to use in comparison")
|
|
parser.add_argument('-ad', '--android_dir', required=False, help="path to a directory with android folder structure sqlite files")
|
|
parser.add_argument('-dd', '--data_dir', required=False, help="path to a directory to search for sqlite files")
|
|
# parser.add_argument('-idx', '--index_fingerprints', required=False, help="path to a directory with sqlite files, index fingerprints if no other args given")
|
|
parser.add_argument('-an', '--app_name', required=False)
|
|
parser.add_argument('-av', '--app_version', required=False)
|
|
parser.add_argument('-n', '--notes', required=False)
|
|
parser.add_argument('-idx', action='store_true', help="add a fingerprint to the index")
|
|
parser.add_argument('-md5', required=False, help="md5 hash to query the index`")
|
|
parser.add_argument('-android_pull', action='store_true', help="automated pull of applications from a physical android phone")
|
|
parser.add_argument('-v', '--verbose', action='store_true', help="will set logging level to INFO")
|
|
parser.add_argument('-vv', '--vverbose', action='store_true', help="will set logging level to DEBUG")
|
|
parser.add_argument('-l', '--logging', action='store_true', help="will supercede the -v option and send all logging to a file, logging.DEBUG")
|
|
# parser.add_argument('-t', '--title', required=False)
|
|
args = parser.parse_args()
|
|
|
|
if (args.logging):
|
|
logging.basicConfig(filename='dbfp.log', level=logging.DEBUG)
|
|
|
|
if (args.verbose):
|
|
logging.basicConfig(level=logging.INFO)
|
|
elif (args.vverbose):
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
else:
|
|
logging.basicConfig(level=logging.CRITICAL)
|
|
|
|
if args.fpdir and args.idx and (args.database or args.fingerprint):
|
|
insertFP(args.database, args.fingerprint, args.fpdir)
|
|
elif args.fpdir and args.idx:
|
|
indexFingerprints(args.fpdir)
|
|
elif (args.database and args.fingerprint):
|
|
compareFingerprint(args.database, args.fingerprint)
|
|
elif (args.fpdir and (args.database or args.fingerprint)):
|
|
compareFPIndex(args.fpdir, args.database, args.fingerprint)
|
|
elif (args.fpdir and args.md5):
|
|
queryMD5(args.fpdir, args.md5)
|
|
elif (args.android_dir):
|
|
androidData(args.android_dir)
|
|
elif (args.android_pull):
|
|
androidPull()
|
|
elif (args.database):
|
|
createFingerprint(args.database, args.app_name, args.app_version, args.notes)
|
|
else:
|
|
print 'Create fingerprint:'
|
|
print ' dbfp.py -db <database_file>\n'
|
|
print 'Create fingerprint index:'
|
|
print ' dbfp.py -fd <fingerprint_dir> -idx\n'
|
|
print 'Add fingerprint to index:'
|
|
print ' dbfp.py -fp <fingerprint_file> -idx (-db <database_file> | -fp <fingerprint_file>)\n'
|
|
print 'Compare fingerprint to a database file:'
|
|
print ' dbfp.py -fp <fingerprint_file> -db <database_file>\n'
|
|
print 'Lookup fingerprint from index:'
|
|
print ' dbfp.py -fd <fingerprint_dir> -fp <fingerprint_file>)\n'
|
|
print 'Lookup database from index:'
|
|
print ' dbfp.py -fd <fingerprint_dir> -db <database_file>\n'
|
|
print 'Lookup MD5 hash from index:'
|
|
print ' dbfp.py -fd <fingerprint_dir> -md5 <md5_hash_string>\n'
|
|
print 'Android App pull and fingerprint:'
|
|
print ' dbfp.py -android_pull'
|
|
print '\n***** ***** ***** *****\n'
|
|
parser.print_help()
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
print
|