
[[ TODO ]]

[ MD5 Index ]
-modify fingerprint compare feature to use the index

-add "-idxf" "--index_force" to force a re-index
  -first give info about current index
  -next, create a new index
  -give info about new index
"152 fingerprints in the index"
"Creating a new index. Moved previous to backup"
"Creating... FINISHED."
"168 fingerprints in the new index"

[ Android Data ]
-Processing of android data
  -removed verbose messsage for each fingerprint
  -fix error message to look good

[ Module Cleanup ]
-remove print statements (from modules)
-add logging statements
-add exception throwing
-add unit testing
  -test md5 sum create of tables
  -test md5 sum create of entire db (tables concatenated)

[ Incorporate with another Product ]
-add module to other python project
-add unit test to other python project


[[ Code Snippets ]]
      self.db_conn.execute('''
          CREATE TABLE md5_index (
          md5_all TEXT PRIMARY KEY,
          md5_list TEXT,
          file_name TEXT);
        ''')



[[ Errors ]]

INFO:root:fp_dir==fingerprints_2015-12-02_131904
INFO:root:No index file found, creating index now...
INFO:root:Successfully created index table
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  7d150935b5bd1d5cfeec415cecb75d8c
md5_list:  21737d16afae9f32f4f7bd55d8bf7a41,70bd01ed9d384374d4ee30a11d861901,fed036842e5577dc74dd01dc956ac9f4,079355c84d8b3b1511a504e08aab7fd2,a081eb58c6f3ea13cd5acdc9a7f9fffe,e9debddbaa3691cc3f0c9bd019a7f55b,4187eaa7d583e9ad442f84842f00f6f3,6a6a57be1f1e3867a3b91aee3141fc6a,97cbd6dd02a2184e14b0fedde14d38b0,7cfffad6145c96ef445be6023d5f2014,c9fb401dd9be58b01e201fd73ee9ce09,80aa2267979381e66869eed9619714be,ba739eb03730e563915f2f76b26ced51
filename:  com.android.email__EmailProvider.db__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  7d150935b5bd1d5cfeec415cecb75d8c
md5_list:  21737d16afae9f32f4f7bd55d8bf7a41,70bd01ed9d384374d4ee30a11d861901,fed036842e5577dc74dd01dc956ac9f4,079355c84d8b3b1511a504e08aab7fd2,a081eb58c6f3ea13cd5acdc9a7f9fffe,e9debddbaa3691cc3f0c9bd019a7f55b,4187eaa7d583e9ad442f84842f00f6f3,6a6a57be1f1e3867a3b91aee3141fc6a,97cbd6dd02a2184e14b0fedde14d38b0,7cfffad6145c96ef445be6023d5f2014,c9fb401dd9be58b01e201fd73ee9ce09,80aa2267979381e66869eed9619714be,ba739eb03730e563915f2f76b26ced51
filename:  com.android.email__EmailProviderBackup.db__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  25ce931f5ca9156b5c517b3376993e85
md5_list:  d719302e78c7571056e30486ddc0354c,9cca82eefd2721ba34390d2f2f638c16,ba739eb03730e563915f2f76b26ced51
filename:  com.android.inputmethod.latin__pendingUpdates__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  e35fbefe86fad99613f8dbe1e3c2c1cb
md5_list:  079355c84d8b3b1511a504e08aab7fd2,8968b80db1f6b44ddad9eb920e556aed,a9a7ef56e0983ef11672942c38c6deed,cc2d681f809c584db305d6794e5b0d35,eba241cd21a03178062d567b2a6c2c05,8bfe27c783fb07c0b3faa359eeb1b76d,95859cafc58f0650d7b44eb645acda60,f606e98bb3514d47f85db02f13e35eff,c94437a91586b046f0b0b3f5912a06f3,12614dc36cb6594b219a44b987589b59,e0deadfa8ebacb0e94eeef6c88950101,bf46d5cdbb0d8a5da846c054e866ff7f,d80d9ea5eb4a0df9b81105923d33da31,c222b7aaee2ff274a74ade1a9e1c6ed8,fa82ea61ddf6943d5279639ef9762c7b,ded7727effe7b8518d6a49fc2100dda3,481cc0b131875c7e45607fdde90f6cd0,50135107f57c79c1ea1b65f38a486fe0,341e23c6864093d94e4eb4933610530a,a0476eb5bf00773a0038cb8929f8e0a2,52992214cdf6d9d84618317deeb451fa,3e5f37504dfb2e217e904ef1e203bd8f,5b716d15eac62cac37b2a5b3df45299f,010e30bef85117885584fdd47da7be67,2d578ea22c45f608b70e9b5118ce1315,a5af6db2b1eb8ee13b78e4f259b94552,580b4c7ddec986334c56d4d5562f0d4e,9ef99127e55eefb301a9860a49ef8dc8,bb4300d3aa332aab1f5956b6c76610b2,0d80f652e203843a542a8d3813cb6456,5ca43b5bc0aa3425c1560e74b5687c38,b87dbd41332fefed918f722392a413f1,b018ea44c267b279a7591580d7152e2d,52c527eda8ffff058be84722b9de2759,03c5cff3d4ea1586ea81287046b9adbd,ba739eb03730e563915f2f76b26ced51
filename:  com.android.providers.contacts__profile.db-journal__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  e35fbefe86fad99613f8dbe1e3c2c1cb
md5_list:  079355c84d8b3b1511a504e08aab7fd2,8968b80db1f6b44ddad9eb920e556aed,a9a7ef56e0983ef11672942c38c6deed,cc2d681f809c584db305d6794e5b0d35,eba241cd21a03178062d567b2a6c2c05,8bfe27c783fb07c0b3faa359eeb1b76d,95859cafc58f0650d7b44eb645acda60,f606e98bb3514d47f85db02f13e35eff,c94437a91586b046f0b0b3f5912a06f3,12614dc36cb6594b219a44b987589b59,e0deadfa8ebacb0e94eeef6c88950101,bf46d5cdbb0d8a5da846c054e866ff7f,d80d9ea5eb4a0df9b81105923d33da31,c222b7aaee2ff274a74ade1a9e1c6ed8,fa82ea61ddf6943d5279639ef9762c7b,ded7727effe7b8518d6a49fc2100dda3,481cc0b131875c7e45607fdde90f6cd0,50135107f57c79c1ea1b65f38a486fe0,341e23c6864093d94e4eb4933610530a,a0476eb5bf00773a0038cb8929f8e0a2,52992214cdf6d9d84618317deeb451fa,3e5f37504dfb2e217e904ef1e203bd8f,5b716d15eac62cac37b2a5b3df45299f,010e30bef85117885584fdd47da7be67,2d578ea22c45f608b70e9b5118ce1315,a5af6db2b1eb8ee13b78e4f259b94552,580b4c7ddec986334c56d4d5562f0d4e,9ef99127e55eefb301a9860a49ef8dc8,bb4300d3aa332aab1f5956b6c76610b2,0d80f652e203843a542a8d3813cb6456,5ca43b5bc0aa3425c1560e74b5687c38,b87dbd41332fefed918f722392a413f1,b018ea44c267b279a7591580d7152e2d,52c527eda8ffff058be84722b9de2759,03c5cff3d4ea1586ea81287046b9adbd,ba739eb03730e563915f2f76b26ced51
filename:  com.android.providers.contacts__profile.db__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  4521a05a93bb297f6c7238e63b770a40
md5_list:  079355c84d8b3b1511a504e08aab7fd2,3e4d4f86832fea605af6218081e8e486,85230651f7dedcbf722f3fcf76e979bc,98940ebaa95c6a082401d231db5bd454,ba739eb03730e563915f2f76b26ced51
filename:  com.facebook.orca__analytics_db2__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  c519033dd1b15689d01de6ea56204c34
md5_list:  ff1446387e16d08af6d10e9b7aee1d01,383d3b94eb3d4deb28216a0a88cf798c,fb572352c0f5aa6b4287aedd62950160,1c7ce35ab1145781807ff1e79dddd4c1,586b3d990301e62d03c3cf0c460fe090,1829660b11f6712a2957588c1da9d716,fa749920613c7e4bd49ebbbfd1720bc6,214b6222b52b25d5c65a7938d29d5c43,1fa8123b312b54bb0fdaed71fa1db74f,474cca0cbce4bc56cde84486493cb4c4,079355c84d8b3b1511a504e08aab7fd2,89709f2881041f3f2382663fa00610a3,b7b50cd354aac9d826b6a46a4cc1ecdb,eb9f03627aaa22c71bb4a9d95d5f1b39,b570b9b7cde71ce3008a6df0c6882205,58ff373730fec5f3b21d6b251f0e8f93,a07d17f6016c1904183d33e1181994eb,79623bbd16cc0c97ad4eabd76393b09c,cd4a006ec1290275b09c3b9af266b061,ba739eb03730e563915f2f76b26ced51
filename:  com.google.android.gms__context_johnegq_gmail.com.db__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  dc9a78db95e556c9a08090db92c78b4e
md5_list:  81cef0b7868db82186d659eb188d92e8,a7ab77c5a21f710f0c09cb21b94af96d,079355c84d8b3b1511a504e08aab7fd2,ba739eb03730e563915f2f76b26ced51
filename:  com.google.android.youtube__downloads.db__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  d0969bba9d6efa5935cac2d88abddea7
md5_list:  b212357f91c49f299bbc278f90db06ba,079355c84d8b3b1511a504e08aab7fd2,162c915fb22c0efd19d6988215982a1a,2d5e7b14fcfc7d6becaedd80a5724950,a7ab77c5a21f710f0c09cb21b94af96d,81cef0b7868db82186d659eb188d92e8,4d0ec6feac74df5bdee3e8688fcffcea,ba739eb03730e563915f2f76b26ced51
filename:  com.google.android.youtube__preload_videos_tasks.db__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  d4cbd6335e4891cde0a7c5ab28847e15
md5_list:  6920fe03ed1e32dc2b08592580fcd7ce,079355c84d8b3b1511a504e08aab7fd2,ea056637e399ff6e84ef01791313bd10,ba739eb03730e563915f2f76b26ced51
filename:  com.Slack__mixpanel__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  9e6cedc9ff7cff7bd4bcdbb76d74d034
md5_list:  ee1f29f07849214cabbc6a4b48f1af10,a8a390c36422bcb2f63d107d36987837,8124f58a95e54f46e27bd5a1eb363cf6,90c680a8c2e707724a7e661bde608deb,dd0636ab41d06b4e1cf4cd6dd848e339,c6e8a5aa07c0543610444061bbd49937,c421cbe1139b8392754e921f52e85048,32af69b2bd8de612cd0dc9393b4e3125,079355c84d8b3b1511a504e08aab7fd2,9b657052533890deecf647ab66426467,7f1e068560e0187f605c42bf9b733e61,bf0465c97b7600454d418c61de68f8f6,aa7c460450c970534ac334864389fd5c,f73d57f2d9aca05ab4c373bfe2a04a2d,eac5d51b3d0eb791d84f02c4d99d3f97,065711d082a075ed964e1a2dcd474ea4,838ccd38acf503cda66539b85d08feaa,a967332f54d3e7e75456bc66844bba81,fe3f1c13421a99c71ac642668d987b9d,d5c233876273e87f5bf60a9f49cfd0d7,61215a6364756841bf16b952679cdc05,8821b12d367ade0c97d1c807681b5aa7,ba739eb03730e563915f2f76b26ced51,303c17c219547a7bb76c4a71128fdb7d,48437b1cde43eca6f6113a0333438888,447059096b4e8821c2e333d2da3e197a,9fde65ab6299575fdb37b0bc422f502e,c75177dbab227c8ae5e7b4d11f4322e3,3ec4f0a46c0727d859e03c49577f3bb7,4b8e99285639efadda863e2026e60493,979de9ef3d6e62cfef7f2419c8ea5da6,41e146ad98459f74453ff1c4287d3e58,3bb14149a773fe7b070f859f9bb91621,b09f88b3f43154d2ca18008051377366,17c8a878b0b99cbea5a7ae41c65b0555,dcaf248ac4f2699b023b381bbe61545e,6322a3e781f2d9264aa4e467ca785a74,836227512bf361329b3a8e95a0b7d52d,1f4726eab8ec9189d71f190afdf61bd0,77ee1b60fb2bb116325c928bafb412cc,34fef7e3d70b061e15b7efc25869da40,d236fad4191e5ba1418872dc78f7db72,132838240f9b360364b1f4903044ab61,a52fafbf4871f1e38529a538fcba88e4,50f11ccd0579aeda53f38904e78e0ddb
filename:  com.twitter.android__14243322-scribe.db__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  d4cbd6335e4891cde0a7c5ab28847e15
md5_list:  6920fe03ed1e32dc2b08592580fcd7ce,079355c84d8b3b1511a504e08aab7fd2,ea056637e399ff6e84ef01791313bd10,ba739eb03730e563915f2f76b26ced51
filename:  com.viber.voip__mixpanel__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  bab2395831d16181e1ef6b828f1a67e8
md5_list:  a6ceedecf7fabdfaba525b40c659e2d6,ba739eb03730e563915f2f76b26ced51
filename:  org.smssecure.smssecure___jobqueue-SMSSecureJobs__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  bab2395831d16181e1ef6b828f1a67e8
md5_list:  a6ceedecf7fabdfaba525b40c659e2d6,ba739eb03730e563915f2f76b26ced51
filename:  org.thoughtcrime.securesms___jobqueue-TextSecureJobs__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  cbda118295c225126928bc6c89d23d90
md5_list:  3f6827d42448a984d03f4a7c031ca22d,a6ceedecf7fabdfaba525b40c659e2d6,ba739eb03730e563915f2f76b26ced51
filename:  org.thoughtcrime.securesms__canonical_address.db__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  2dd632d99af71e58bedd9dac47a9eee0
md5_list:  f06a8e20243c3517e732e1996d81cbee,079355c84d8b3b1511a504e08aab7fd2,2f1cd3ec169f81f477aed964866cb033,d661f16cd6b94695eb06190c12e6f070,fe7f6ea84bf8ea1cb59f60528fcc71bf,ba739eb03730e563915f2f76b26ced51
filename:  tv.periscope.android__com.localytics.android.c685a165210ff465709ae8d359bef6c90f17a12f344206db16d6f2f7f129e34f.analytics.sqlite__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  fc3340b05ddebed7003d4b98ee8e0a6d
md5_list:  f06a8e20243c3517e732e1996d81cbee,2aa49e6195c3d4f0a3346cac9dae6cd8,079355c84d8b3b1511a504e08aab7fd2,433b737da03ddb1dbc5900729e9848b5,0f0965f5677afc5c31bff7e71dcab317,2f1cd3ec169f81f477aed964866cb033,d661f16cd6b94695eb06190c12e6f070,8d27531bc1968246f6a0cf822c3bd980,bb588a3e91d8a6d8b7a602289723d403,fe7f6ea84bf8ea1cb59f60528fcc71bf,ba739eb03730e563915f2f76b26ced51
filename:  tv.periscope.android__com.localytics.android.c685a165210ff465709ae8d359bef6c90f17a12f344206db16d6f2f7f129e34f.in-app.sqlite__dbfp.json
ERROR:root:Error inserting a row
UNIQUE constraint failed: md5_index.md5_all
ERROR:root:md5_all:  b6afb87fe25b289135b5a858a379bc99
md5_list:  f06a8e20243c3517e732e1996d81cbee,2aa49e6195c3d4f0a3346cac9dae6cd8,079355c84d8b3b1511a504e08aab7fd2,433b737da03ddb1dbc5900729e9848b5,0f0965f5677afc5c31bff7e71dcab317,2f1cd3ec169f81f477aed964866cb033,d661f16cd6b94695eb06190c12e6f070,8d27531bc1968246f6a0cf822c3bd980,bb588a3e91d8a6d8b7a602289723d403,622d589d10fb633aeba8bc2a1411dfe5,fe7f6ea84bf8ea1cb59f60528fcc71bf,ba739eb03730e563915f2f76b26ced51
filename:  tv.periscope.android__com.localytics.android.c685a165210ff465709ae8d359bef6c90f17a12f344206db16d6f2f7f129e34f.profile.sqlite__dbfp.json
INFO:root:Completed populating the index. Completed: 137  Failed: 19 
INFO:root:Successfully populated the index