NEW: added new table to the index with app info for each database, population of the index is complete too

libs/fingerprint.py

@@ -37,7 +37,7 @@ class FingerprintDB:
   # version of the scanner used to create the fingerprint
   scanner_ver = "0.98"
   # version of the json file format, this # is inserted in the json fingerprint file and can be used to determine what is supported at the time of that version
-  format_ver  = "0.91"
+  format_ver  = "0.92"
 
   #
   def __init__(self):
@@ -47,13 +47,20 @@ class FingerprintDB:
     self.tables = {}
     self.db_hash = None
     self.table_hashes = None
-    # fingerprint metadata
-    self.db_name = ""
-    self.app_name = ""
-    self.app_ver = ""
-    self.notes = ""
     self.filein = ""
     self.init = False
+    # fingerprint metadata
+    self.metadata = {}
+    self.metadata['db-name'] = ""
+    self.metadata['app-name'] = ""
+    self.metadata['app-ver'] = ""
+    self.metadata['notes'] = ""
+    self.metadata['scan-date'] = ""
+    # self.db_name = ""
+    # self.app_name = ""
+    # self.app_ver = ""
+    # self.notes = ""
+    # self.scan_date = ""
 
   #
   def scanDBFile(self, filein):
@@ -66,9 +73,9 @@ class FingerprintDB:
 
     try:
       # extract file name from path+filename
-      self.db_name = ToolBox.parseFilenameIncExt(filein)
+      self.metadata['db-name'] = ToolBox.parseFilenameIncExt(filein)
     except:
-      self.db_name = filein
+      self.metadata['db-name'] = filein
     try:
       # read database schema, parse the schema
       self.__readDatabase()
@@ -173,6 +180,8 @@ class FingerprintDB:
       dbmt = jsonData['db-metadata']
       dbht = jsonData['db-metadata-hashes']
       dbmd5 = jsonData['db-metadata-md5']
+      metadata = jsonData['_file-metadata']
+
       all_tables = tb.keys()
       for table_name in all_tables:
         #logging.info("[[ Table <" + table_name + "> imported ]]")
@@ -183,6 +192,7 @@ class FingerprintDB:
       self.tables       = tables
       self.db_hash      = dbmd5
       self.table_hashes = dbht
+      self.metadata     = metadata
     except Exception as ex:
       logging.error("ERROR: problem loading json file: \n{}\n{}".format(file_json, ex))
 
@@ -276,11 +286,6 @@ class FingerprintDB:
   def __readDatabase(self):
     flag = False
     rows = self.cur.execute(self.sqlmaster)
-    # row = self.cur.fetchone()
-    # newTable = TableSchema()
-    # newTable.loadTable(row[0], row[1])
-    # self.table_names.append(newTable.name())
-    # self.tables[newTable.name()] = newTable
 
     for row in rows:
       flag = True
@@ -329,10 +334,10 @@ class FingerprintDB:
     mhash['format-ver']   = self.format_ver
     mhash['scanner-ver']  = self.scanner_ver
     mhash['scanner-name'] = 'dbfp'
-    mhash['dn-name']      = self.db_name
+    mhash['db-name']      = self.metadata['db-name']
-    mhash['app-name']     = self.app_name
+    mhash['app-name']     = self.metadata['app-name']
-    mhash['app-ver']      = self.app_ver
+    mhash['app-ver']      = self.metadata['app-ver']
-    mhash['notes']        = self.notes
+    mhash['notes']        = self.metadata['notes']
 
     # tables
     tables = self.tables.keys()
@@ -365,15 +370,15 @@ class FingerprintDB:
 
   #
   def setAppName(self, name):
-    self.app_name = name
+    self.metadata['app-name'] = name
 
   #
   def setAppVer(self, version):
-    self.app_ver = version
+    self.metadata['app-ver'] = version
 
   #
   def setNotes(self, notes):
-    self.notes = notes
+    self.metadata['notes'] = notes
 
   #
   def getErrorString(self, errorCode):

libs/fingerprint_index.py

@@ -154,17 +154,16 @@ class FingerprintIndex:
         ''')
       self.db_conn.execute('''
           CREATE TABLE metadata (
-          md5_db TEXT PRIMARY KEY,
+          md5_db TEXT,
           app_name TEXT,
           app_ver TEXT,
-          file_name TEXT,
+          db_file TEXT,
           fp_file TEXT,
-          scan_date NUMERIC);
+          scan_date TEXT);
         ''')
       logging.info("Successfully created index table")
     except Exception as ex:
       raise
-      # raise FingerprintIndexWrite("Error creating index file\n{}".format(ex))
 
   #
   def __populateIndex(self, fp_dir):
@@ -184,12 +183,13 @@ class FingerprintIndex:
           db.importJson(fq_file)
           self.__insertMod_md5_all(db.db_hash, db.table_hashes.values(), file)
           self.__insertMod_md5_tables(db.table_hashes.values(), file)
+          self.__insertMod_metadata(db, file)
           finCount = finCount+1
-          if ((finCount % 5) == 0):
+          self.db_conn.commit()
-            self.db_conn.commit()
         except Exception as ex:
-          logging.error(ex)
+          self.db_conn.rollback()
           failCount = failCount+1
+          logging.error(ex)
     except Exception as ex:
       logging.error(ex)
     finally:
@@ -200,7 +200,7 @@ class FingerprintIndex:
   #
   def __insertMod_md5_all(self, md5_db, md5_list, filename):
     try:
-      logging.info("INSERT INTO md5_all VALUES({}, {}, {}, 1)".format(md5_db, ','.join(md5_list), filename))
+      #logging.info("INSERT INTO md5_all VALUES({}, {}, {}, 1)".format(md5_db, ','.join(md5_list), filename))
       self.db_conn.execute(
         '''
         INSERT INTO md5_all VALUES(?, ?, ?, ?)
@@ -210,7 +210,7 @@ class FingerprintIndex:
         (fp_list, fp_count) = self.__selectFileList(md5_db)
         fp_list += ","+filename
         fp_count += 1
-        # logging.info("fp_list=={}".format(fp_list))
+        #logging.info("fp_list=={}".format(fp_list))
         self.db_conn.execute(
           '''
           UPDATE md5_all SET fp_list=?, fp_count=? WHERE md5_db=?
@@ -243,6 +243,23 @@ class FingerprintIndex:
       except Exception as ex:
         raise FingerprintIndexWrite("Error inserting a row\n{}".format(ex))
 
+  #
+  def __insertMod_metadata(self, db, filename):
+    # insert the md5 of the table schemas
+    # print "*****"
+    # print "***** {}".format(db.metadata)
+    # print "***** {}".format(db.metadata['fart'])
+    # print "***** {}".format(db.metadata['app_name'])
+    try:
+      self.db_conn.execute(
+        '''
+        INSERT INTO metadata VALUES(?, ?, ?, ?, ?, ?)
+        ''', [db.db_hash, db.metadata['app-name'], db.metadata['app-ver'], db.metadata['db-name'], filename, db.metadata['scan-date']])
+        # ''', ["abc", "efg", "hij", "klm", "nop", "qrs"])
+    except Exception as ex:
+      print ex
+      raise
+
   #
   def __selectFileList(self, md5_db):
     try: