j3g
/
dbfp_pub
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

MOD: updated the processing of sql create statements to handle more syntax styles

This commit is contained in:
JohnE 2015-09-23 12:21:54 -07:00
parent 5bc90d7407
commit 5d4d351248
1 changed files with 45 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
libs/fingerprint.py
View File

@ -278,7 +278,7 @@ class TableDefinition:
self.tableName = tableName self.tableName = tableName
self.sqlStr = sqlStr self.sqlStr = sqlStr
print "[[ TABLE: <{}> ] loading]".format(tableName) print "[[ TABLE: <{}> ] processing...]".format(tableName)
# hash the sql create string for quicker fingerprint matching # hash the sql create string for quicker fingerprint matching
try: try:
m = hashlib.md5() m = hashlib.md5()
@ -319,18 +319,30 @@ class TableDefinition:
def __parseCreateStr(self, sqltext): def __parseCreateStr(self, sqltext):
try: try:
newField = {} newField = {}
# photo_id INTEGER REFERENCES data(_id)
# name_raw_contact_id INTEGER REFERENCES raw_contacts(_id) # use for debug purposes
results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+REFERENCESS\s+(\W+)', sqltext) # print "sqltext=={}".format(sqltext)
# raw_contact_id INTEGER REFERENCES raw_contacts(_id) NOT NULL
results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+REFERENCES\s+(.*)\s+NOT.NULL', sqltext, re.IGNORECASE)
if results: if results:
newField['name'] = results.group(1) newField['name'] = results.group(1)
newField['datatype'] = results.group(2) newField['datatype'] = results.group(2)
newField['references'] = 1 newField['references'] = True
newField['referencesdata'] = results.group(3)
newField['notnull'] = True
return newField
# photo_id INTEGER REFERENCES data(_id)
# name_raw_contact_id INTEGER REFERENCES raw_contacts(_id)
results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+REFERENCES\s+(.*)', sqltext, re.IGNORECASE)
if results:
newField['name'] = results.group(1)
newField['datatype'] = results.group(2)
newField['references'] = True
newField['referencesdata'] = results.group(3) newField['referencesdata'] = results.group(3)
return newField return newField
# pinned INTEGER NOT NULL DEFAULT 2147483647 # pinned INTEGER NOT NULL DEFAULT 2147483647
# send_to_voicemail INTEGER NOT NULL DEFAULT 0 # send_to_voicemail INTEGER NOT NULL DEFAULT 0
results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+NOT NULL\s+DEFAULT\s+(\w+)', sqltext) results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+NOT.NULL\s+DEFAULT\s+(\w+)', sqltext, re.IGNORECASE)
if results: if results:
newField['name'] = results.group(1) newField['name'] = results.group(1)
newField['datatype'] = results.group(2) newField['datatype'] = results.group(2)
@ -339,15 +351,14 @@ class TableDefinition:
return newField return newField
# pinned INTEGER DEFAULT 2147483647 # pinned INTEGER DEFAULT 2147483647
# send_to_voicemail INTEGER DEFAULT 0 # send_to_voicemail INTEGER DEFAULT 0
results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+DEFAULT\s+(\w+)', sqltext) results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+DEFAULT\s+(\w+)', sqltext, re.IGNORECASE)
if results: if results:
newField['name'] = results.group(1) newField['name'] = results.group(1)
newField['datatype'] = results.group(2) newField['datatype'] = results.group(2)
newField['notnull'] = True
newField['default'] = results.group(3) newField['default'] = results.group(3)
return newField return newField
# _id INTEGER PRIMARY KEY AUTOINCREMENT # _id INTEGER PRIMARY KEY AUTOINCREMENT
results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+PRIMARY KEY\s+AUTOINCREMENT', sqltext) results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+PRIMARY.KEY\s+AUTOINCREMENT', sqltext, re.IGNORECASE)
if results: if results:
newField['name'] = results.group(1) newField['name'] = results.group(1)
newField['datatype'] = results.group(2) newField['datatype'] = results.group(2)
@ -355,54 +366,67 @@ class TableDefinition:
newField['autoincrement'] = True newField['autoincrement'] = True
return newField return newField
# _id INTEGER PRIMARY KEY # _id INTEGER PRIMARY KEY
results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+PRIMARY KEY', sqltext) results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+PRIMARY.KEY', sqltext, re.IGNORECASE)
if results: if results:
newField['name'] = results.group(1) newField['name'] = results.group(1)
newField['datatype'] = results.group(2) newField['datatype'] = results.group(2)
newField['primarykey'] = True newField['primarykey'] = True
return newField return newField
# FileID INTEGER NOT NULL # FileID INTEGER NOT NULL
results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+NOT NULL', sqltext) results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)\s+NOT.NULL', sqltext, re.IGNORECASE)
if results: if results:
newField['name'] = results.group(1) newField['name'] = results.group(1)
newField['datatype'] = results.group(2) newField['datatype'] = results.group(2)
newField['notnull'] = True newField['notnull'] = True
return newField return newField
# PRIMARY KEY (field_name, # PRIMARY KEY (field_name,
results = re.match(r'PRIMARY KEY \((?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\,?', sqltext) results = re.match(r'PRIMARY.KEY\s+\((?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\,?', sqltext, re.IGNORECASE)
if results: if results:
field = self.fields[results.group(1)] field = self.fields[results.group(1)]
field['primarykey'] = True field['primarykey'] = True
self.primarykeyFlag = True self.primarykeyFlag = True
return False return False
# UNIQUE(field_name, # UNIQUE(field_name,
results = re.match(r'UNIQUE\((?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\,?', sqltext) results = re.match(r'UNIQUE\((?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\,?', sqltext, re.IGNORECASE)
if results: if results:
field = self.fields[results.group(1)] field = self.fields[results.group(1)]
field['unique'] = True field['unique'] = True
self.uniqueFlag = True; self.uniqueFlag = True;
return False return False
# custom_ringtone TEXT # custom_ringtone TEXT
results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)', sqltext) results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\s+(\w+)', sqltext, re.IGNORECASE)
if results: if results:
newField['name'] = results.group(1) newField['name'] = results.group(1)
newField['datatype'] = results.group(2) newField['datatype'] = results.group(2)
return newField return newField
# field_name) # field_name)
# results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*(\)?)', sqltext) results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\)', sqltext, re.IGNORECASE)
results = re.match(r'(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*\)', sqltext)
if results: if results:
field = self.fields[results.group(1)] field = self.fields[results.group(1)]
if (self.primarykeyFlag): if (self.primarykeyFlag):
field['primarykey'] = True
if (field): if (field):
self.primarykeyFlag = False field['primarykey'] = True
self.primarykeyFlag = False
elif (self.uniqueFlag): elif (self.uniqueFlag):
field['unique'] = True
if (field): if (field):
self.uniqueFlag = False field['unique'] = True
self.uniqueFlag = False
return False return False
# field_name
results = re.match(r'^(?:[`|\"|\'])*(\w+)(?:[`|\"|\'])*$', sqltext, re.IGNORECASE)
if results:
if (self.primarykeyFlag):
field = self.fields[results.group(1)]
field['primarykey'] = True
elif (self.uniqueFlag):
field = self.fields[results.group(1)]
field['unique'] = True
else:
newField['name'] = results.group(1)
newField['datatype'] = "INTEGER"
return newField
return False
print 'WARN: field definition not recognized: "{}"'.format(sqltext) print 'WARN: field definition not recognized: "{}"'.format(sqltext)
except Exception, e: except Exception, e:
print 'WARN: problem parsing sql create text: "{}"'.format(sqltext) print 'WARN: problem parsing sql create text: "{}"'.format(sqltext)