Bitmessage/PyBitmessage-2024-12-13
Archived
2
0
Fork 0
Code Issues 168 Pull Requests 53 Actions Packages Projects Releases 5 Wiki Activity

Read from and write to SQLite database in binary mode #2245

Open
kashikoibumi wants to merge 2 commits from kashikoibumi/binary into v0.6
pull from: kashikoibumi/binary
merge into: Bitmessage:v0.6
Conversation 0 Commits 2 Files Changed 33 +97 -50
12 changed files with 97 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 0f858bca89 - Show all commits

24
src/addresses.py
View File

@ -187,7 +187,7 @@ def decodeAddress(address):
integer = decodeBase58(address)
if integer == 0:
status = 'invalidcharacters'
return status, 0, 0, ''
return status, 0, 0, b''
# after converting to hex, the string will be prepended
# with a 0x and appended with a L in python2
hexdata = hex(integer)[2:].rstrip('L')
@ -200,23 +200,23 @@ def decodeAddress(address):
if checksum != double_sha512(data[:-4])[0:4]:
status = 'checksumfailed'
return status, 0, 0, ''
return status, 0, 0, b''
try:
addressVersionNumber, bytesUsedByVersionNumber = decodeVarint(data[:9])
except varintDecodeError as e:
logger.error(str(e))
status = 'varintmalformed'
return status, 0, 0, ''
return status, 0, 0, b''
if addressVersionNumber > 4:
logger.error('cannot decode address version numbers this high')
status = 'versiontoohigh'
return status, 0, 0, ''
return status, 0, 0, b''
elif addressVersionNumber == 0:
logger.error('cannot decode address version numbers of zero.')
status = 'versiontoohigh'
return status, 0, 0, ''
return status, 0, 0, b''
try:
streamNumber, bytesUsedByStreamNumber = \
@ -224,7 +224,7 @@ def decodeAddress(address):
except varintDecodeError as e:
logger.error(str(e))
status = 'varintmalformed'
return status, 0, 0, ''
return status, 0, 0, b''
status = 'success'
if addressVersionNumber == 1:
@ -242,21 +242,21 @@ def decodeAddress(address):
return status, addressVersionNumber, streamNumber, \
b'\x00\x00' + embeddedRipeData
elif len(embeddedRipeData) < 18:
return 'ripetooshort', 0, 0, ''
return 'ripetooshort', 0, 0, b''
elif len(embeddedRipeData) > 20:
return 'ripetoolong', 0, 0, ''
return 'otherproblem', 0, 0, ''
return 'ripetoolong', 0, 0, b''
return 'otherproblem', 0, 0, b''
elif addressVersionNumber == 4:
embeddedRipeData = \
data[bytesUsedByVersionNumber + bytesUsedByStreamNumber:-4]
if embeddedRipeData[0:1] == b'\x00':
# In order to enforce address non-malleability, encoded
# RIPE data must have NULL bytes removed from the front
return 'encodingproblem', 0, 0, ''
return 'encodingproblem', 0, 0, b''
elif len(embeddedRipeData) > 20:
return 'ripetoolong', 0, 0, ''
return 'ripetoolong', 0, 0, b''
elif len(embeddedRipeData) < 4:
return 'ripetooshort', 0, 0, ''
return 'ripetooshort', 0, 0, b''
x00string = b'\x00' * (20 - len(embeddedRipeData))
return status, addressVersionNumber, streamNumber, \
x00string + embeddedRipeData

44
src/bitmessageqt/__init__.py
View File

@ -552,6 +552,8 @@ class MyForm(settingsmixin.SMainWindow):
"GROUP BY toaddress, folder")
for row in queryreturn:
toaddress, folder, cnt = row
toaddress = toaddress.decode("utf-8", "replace")
folder = folder.decode("utf-8", "replace")
total += cnt
if toaddress in db and folder in db[toaddress]:
db[toaddress][folder] = cnt
@ -1044,6 +1046,8 @@ class MyForm(settingsmixin.SMainWindow):
normalUnread = {}
broadcastsUnread = {}
for addr, fld, count in queryReturn:
addr = addr.decode("utf-8", "replace")
fld = fld.decode("utf-8", "replace")
try:
normalUnread[addr][fld] = count
except KeyError:
@ -1211,11 +1215,11 @@ class MyForm(settingsmixin.SMainWindow):
items = [
MessageList_AddressWidget(
toAddress, unicode(acct.toLabel, 'utf-8'), not read),
toAddress, acct.toLabel, not read),
MessageList_AddressWidget(
fromAddress, unicode(acct.fromLabel, 'utf-8'), not read),
fromAddress, acct.fromLabel, not read),
MessageList_SubjectWidget(
str(subject), unicode(acct.subject, 'utf-8', 'replace'),
subject, acct.subject,
not read),
MessageList_TimeWidget(
l10n.formatTimestamp(received), not read, received, msgid)
@ -1243,7 +1247,14 @@ class MyForm(settingsmixin.SMainWindow):
xAddress, account, "sent", where, what, False)
for row in queryreturn:
self.addMessageListItemSent(tableWidget, *row)
r = []
r.append(row[0].decode("utf-8", "replace")) # toaddress
r.append(row[1].decode("utf-8", "replace")) # fromaddress
r.append(row[2].decode("utf-8", "replace")) # subject
r.append(row[3].decode("utf-8", "replace")) # status
r.append(row[3]) # ackdata
r.append(row[4]) # lastactiontime
self.addMessageListItemSent(tableWidget, *r)
tableWidget.horizontalHeader().setSortIndicator(
3, QtCore.Qt.DescendingOrder)
@ -1284,6 +1295,10 @@ class MyForm(settingsmixin.SMainWindow):
for row in queryreturn:
toAddress, fromAddress, subject, _, msgid, received, read = row
toAddress = toAddress.decode("utf-8", "replace")
fromAddress = fromAddress.decode("utf-8", "replace")
subject = subject.decode("utf-8", "replace")
received = received.decode("utf-8", "replace")
self.addMessageListItemInbox(
tableWidget, toAddress, fromAddress, subject,
msgid, received, read)
@ -1369,6 +1384,7 @@ class MyForm(settingsmixin.SMainWindow):
SELECT msgid, toaddress, read FROM inbox where folder='inbox'
''')
for msgid, toAddress, read in queryreturn:
toAddress = toAddress.decode("utf-8", "replace")
if not read:
# increment the unread subscriptions if True (1)
@ -1447,7 +1463,7 @@ class MyForm(settingsmixin.SMainWindow):
# Adapters and converters for QT <-> sqlite
def sqlInit(self):
register_adapter(QtCore.QByteArray, str)
register_adapter(QtCore.QByteArray, bytes)
def indicatorInit(self):
"""
@ -1995,9 +2011,9 @@ class MyForm(settingsmixin.SMainWindow):
def rerenderAddressBook(self):
def addRow (address, label, type):
self.ui.tableWidgetAddressBook.insertRow(0)
newItem = Ui_AddressBookWidgetItemLabel(address, unicode(label, 'utf-8'), type)
newItem = Ui_AddressBookWidgetItemLabel(address, label, type)
self.ui.tableWidgetAddressBook.setItem(0, 0, newItem)
newItem = Ui_AddressBookWidgetItemAddress(address, unicode(label, 'utf-8'), type)
newItem = Ui_AddressBookWidgetItemAddress(address, label, type)
self.ui.tableWidgetAddressBook.setItem(0, 1, newItem)
oldRows = {}
@ -2015,6 +2031,8 @@ class MyForm(settingsmixin.SMainWindow):
queryreturn = sqlQuery('SELECT label, address FROM subscriptions WHERE enabled = 1')
for row in queryreturn:
label, address = row
label = label.decode("utf-8", "replace")
address = address.decode("utf-8", "replace")
newRows[address] = [label, AccountMixin.SUBSCRIPTION]
# chans
for address in config.addresses(True):
@ -2025,6 +2043,8 @@ class MyForm(settingsmixin.SMainWindow):
queryreturn = sqlQuery('SELECT * FROM addressbook')
for row in queryreturn:
label, address = row
label = label.decode("utf-8", "replace")
address = address.decode("utf-8", "replace")
newRows[address] = [label, AccountMixin.NORMAL]
completerList = []
@ -2038,7 +2058,7 @@ class MyForm(settingsmixin.SMainWindow):
self.ui.tableWidgetAddressBook.removeRow(oldRows[address][2])
for address in newRows:
addRow(address, newRows[address][0], newRows[address][1])
completerList.append(unicode(newRows[address][0], encoding="UTF-8") + " <" + address + ">")
completerList.append(newRows[address][0] + " <" + address + ">")
# sort
self.ui.tableWidgetAddressBook.sortByColumn(
@ -2273,6 +2293,7 @@ class MyForm(settingsmixin.SMainWindow):
if queryreturn != []:
for row in queryreturn:
toLabel, = row
toLabel = toLabel.decode("utf-8", "replace")
self.displayNewSentMessage(
toAddress, toLabel, fromAddress, subject, message, ackdata)
@ -2920,6 +2941,7 @@ class MyForm(settingsmixin.SMainWindow):
if queryreturn != []:
for row in queryreturn:
messageText, = row
messageText = messageText.decode("utf-8", "replace")
lines = messageText.split('\n')
totalLines = len(lines)
@ -3043,6 +3065,7 @@ class MyForm(settingsmixin.SMainWindow):
if queryreturn != []:
for row in queryreturn:
messageAtCurrentInboxRow, = row
messageAtCurrentInboxRow = messageAtCurrentInboxRow.decode("utf-8", "replace")
acct.parseMessage(
toAddressAtCurrentInboxRow, fromAddressAtCurrentInboxRow,
tableWidget.item(currentInboxRow, 2).subject,
@ -3274,6 +3297,7 @@ class MyForm(settingsmixin.SMainWindow):
if queryreturn != []:
for row in queryreturn:
message, = row
message = message.decode("utf-8", "replace")
defaultFilename = "".join(x for x in subjectAtCurrentInboxRow if x.isalnum()) + '.txt'
filename = QtGui.QFileDialog.getSaveFileName(
@ -3285,7 +3309,7 @@ class MyForm(settingsmixin.SMainWindow):
return
try:
f = open(filename, 'w')
f.write(message)
f.write(message.encode("utf-8", "replace"))
f.close()
except Exception:
logger.exception('Message not saved', exc_info=True)
@ -4020,6 +4044,7 @@ class MyForm(settingsmixin.SMainWindow):
queryreturn = sqlQuery('''SELECT status FROM sent where ackdata=?''', ackData)
for row in queryreturn:
status, = row
status = status.decode("utf-8", "replace")
if status == 'toodifficult':
self.popMenuSent.addAction(self.actionForceSend)
@ -4124,6 +4149,7 @@ class MyForm(settingsmixin.SMainWindow):
try:
message = queryreturn[-1][0]
message = message.decode("utf-8", "replace")
except NameError:
message = ""
except IndexError:

17
src/bitmessageqt/account.py
View File

@ -38,6 +38,8 @@ def getSortedSubscriptions(count=False):
ret = {}
for row in queryreturn:
label, address, enabled = row
label = label.decode("utf-8", "replace")
address = address.decode("utf-8", "replace")
ret[address] = {}
ret[address]["inbox"] = {}
ret[address]["inbox"]['label'] = label
@ -50,6 +52,8 @@ def getSortedSubscriptions(count=False):
GROUP BY inbox.fromaddress, folder''', str_broadcast_subscribers)
for row in queryreturn:
address, folder, cnt = row
address = address.decode("utf-8", "replace")
folder = folder.decode("utf-8", "replace")
if folder not in ret[address]:
ret[address][folder] = {
'label': ret[address]['inbox']['label'],
@ -137,12 +141,14 @@ class BMAccount(object):
if queryreturn != []:
for row in queryreturn:
label, = row
label = label.decode("utf-8", "replace")
else:
queryreturn = sqlQuery(
'''select label from subscriptions where address=?''', address)
if queryreturn != []:
for row in queryreturn:
label, = row
label = label.replace("utf-8", "replace")
return label
def parseMessage(self, toAddress, fromAddress, subject, message):
@ -150,9 +156,6 @@ class BMAccount(object):
self.toAddress = toAddress
self.fromAddress = fromAddress
if isinstance(subject, unicode):
self.subject = str(subject)
else:
self.subject = subject
self.message = message
self.fromLabel = self.getLabel(fromAddress)
@ -202,11 +205,11 @@ class GatewayAccount(BMAccount):
sqlExecute(
'''INSERT INTO sent VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''',
'',
self.toAddress,
self.toAddress.encode("utf-8", "replace"),
ripe,
self.fromAddress,
self.subject,
self.message,
self.fromAddress.encode("utf-8", "replace"),
self.subject.encode("utf-8", "replace"),
self.message.encode("utf-8", "replace"),
ackdata,
int(time.time()), # sentTime (this will never change)
int(time.time()), # lastActionTime

18
src/bitmessageqt/blacklist.py
View File

@ -111,10 +111,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
if isinstance(addressitem, QtGui.QTableWidgetItem):
if self.radioButtonBlacklist.isChecked():
sqlExecute('''UPDATE blacklist SET label=? WHERE address=?''',
str(item.text()), str(addressitem.text()))
item.text().encode("utf-8", "replace"), addressitem.text().encode("utf-8", "replace"))
else:
sqlExecute('''UPDATE whitelist SET label=? WHERE address=?''',
str(item.text()), str(addressitem.text()))
item.text().encode("utf-8", "replace"), addressitem.text().encode("utf-8", "replace"))
def init_blacklist_popup_menu(self, connectSignal=True):
# Popup menu for the Blacklist page
@ -171,6 +171,8 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
self.tableWidgetBlacklist.setSortingEnabled(False)
for row in queryreturn:
label, address, enabled = row
label = label.decode("utf-8", "replace")
address = address.decode("utf-8", "replace")
self.tableWidgetBlacklist.insertRow(0)
newItem = QtGui.QTableWidgetItem(unicode(label, 'utf-8'))
if not enabled:
@ -198,11 +200,11 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
sqlExecute(
'''DELETE FROM blacklist WHERE label=? AND address=?''',
str(labelAtCurrentRow), str(addressAtCurrentRow))
labelAtCurrentRow.encode("utf-8", "replace"), addressAtCurrentRow.encode("utf-8", "replace"))
else:
sqlExecute(
'''DELETE FROM whitelist WHERE label=? AND address=?''',
str(labelAtCurrentRow), str(addressAtCurrentRow))
labelAtCurrentRow.encode("utf-8", "replace"), addressAtCurrentRow.encode("utf-8", "replace"))
self.tableWidgetBlacklist.removeRow(currentRow)
def on_action_BlacklistClipboard(self):
@ -227,11 +229,11 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
sqlExecute(
'''UPDATE blacklist SET enabled=1 WHERE address=?''',
str(addressAtCurrentRow))
addressAtCurrentRow.encode("utf-8", "replace"))
else:
sqlExecute(
'''UPDATE whitelist SET enabled=1 WHERE address=?''',
str(addressAtCurrentRow))
addressAtCurrentRow.encode("utf-8", "replace"))
def on_action_BlacklistDisable(self):
currentRow = self.tableWidgetBlacklist.currentRow()
@ -243,10 +245,10 @@ class Blacklist(QtGui.QWidget, RetranslateMixin):
currentRow, 1).setTextColor(QtGui.QColor(128, 128, 128))
if config.get('bitmessagesettings', 'blackwhitelist') == 'black':
sqlExecute(
'''UPDATE blacklist SET enabled=0 WHERE address=?''', str(addressAtCurrentRow))
'''UPDATE blacklist SET enabled=0 WHERE address=?''', addressAtCurrentRow.encode("utf-8", "replace"))
else:
sqlExecute(
'''UPDATE whitelist SET enabled=0 WHERE address=?''', str(addressAtCurrentRow))
'''UPDATE whitelist SET enabled=0 WHERE address=?''', addressAtCurrentRow.encode("utf-8", "replace"))
def on_action_BlacklistSetAvatar(self):
self.window().on_action_SetAvatar(self.tableWidgetBlacklist)

11
src/bitmessageqt/foldertree.py
View File

@ -136,7 +136,7 @@ class AccountMixin(object):
if queryreturn != []:
for row in queryreturn:
retval, = row
retval = unicode(retval, 'utf-8')
retval = retval.decode("utf-8", "replace")
elif self.address is None or self.type == AccountMixin.ALL:
return unicode(
str(_translate("MainWindow", "All accounts")), 'utf-8')
@ -311,8 +311,8 @@ class Ui_SubscriptionWidget(Ui_AddressWidget):
if queryreturn != []:
for row in queryreturn:
retval, = row
return unicode(retval, 'utf-8', 'ignore')
return unicode(self.address, 'utf-8')
return retval.decode("utf-8", "replace")
return self.address
def setType(self):
"""Set account type"""
@ -418,7 +418,8 @@ class MessageList_AddressWidget(BMAddressWidget):
'''select label from subscriptions where address=?''', self.address)
if queryreturn:
for row in queryreturn:
newLabel = unicode(row[0], 'utf-8', 'ignore')
newLabel = row[0]
newLabel = newLabel.decode("utf-8", "replace")
self.label = newLabel
@ -456,7 +457,7 @@ class MessageList_SubjectWidget(BMTableWidgetItem):
if role == QtCore.Qt.UserRole:
return self.subject
if role == QtCore.Qt.ToolTipRole:
return escape(unicode(self.subject, 'utf-8'))
return escape(self.subject)
return super(MessageList_SubjectWidget, self).data(role)
# label (or address) alphabetically, disabled at the end

2
src/bitmessageqt/safehtmlparser.py
View File

@ -125,6 +125,8 @@ class SafeHTMLParser(HTMLParser):
def feed(self, data):
try:
data = unicode(data, 'utf-8')
except TypeError:
pass
except UnicodeDecodeError:
data = unicode(data, 'utf-8', errors='replace')
HTMLParser.feed(self, data)

2
src/class_singleCleaner.py
View File

@ -99,6 +99,8 @@ class singleCleaner(StoppableThread):
tick - state.maximumLengthOfTimeToBotherResendingMessages
)
for toAddress, ackData, status in queryreturn:
toAddress = toAddress.decode("utf-8", "replace")
status = status.decode("utf-8", "replace")
if status == 'awaitingpubkey':
self.resendPubkeyRequest(toAddress)
elif status == 'msgsent':

9
src/class_singleWorker.py
View File

@ -73,6 +73,7 @@ class singleWorker(StoppableThread):
'''SELECT DISTINCT toaddress FROM sent'''
''' WHERE (status='awaitingpubkey' AND folder='sent')''')
for toAddress, in queryreturn:
toAddress = toAddress.decode("utf-8", "replace")
toAddressVersionNumber, toStreamNumber, toRipe = \
decodeAddress(toAddress)[1:]
if toAddressVersionNumber <= 3:
@ -538,6 +539,9 @@ class singleWorker(StoppableThread):
for row in queryreturn:
fromaddress, subject, body, ackdata, TTL, encoding = row
fromaddress = fromaddress.decode("utf-8", "replace")
subject = subject.decode("utf-8", "replace")
body = body.decode("utf-8", "replace")
# status
_, addressVersionNumber, streamNumber, ripe = \
decodeAddress(fromaddress)
@ -726,6 +730,11 @@ class singleWorker(StoppableThread):
for row in queryreturn:
toaddress, fromaddress, subject, message, \
ackdata, status, TTL, retryNumber, encoding = row
toaddress = toaddress.decode("utf-8", "replace")
fromaddress = fromaddress.decode("utf-8", "replace")
subject = subject.decode("utf-8", "replace")
message = message.decode("utf-8", "replace")
status = status.decode("utf-8", "replace")
# toStatus
_, toAddressVersionNumber, toStreamNumber, toRipe = \
decodeAddress(toaddress)

6
src/class_sqlThread.py
View File

@ -38,7 +38,7 @@ class sqlThread(threading.Thread):
helper_sql.sql_available = True
config_ready.wait()
self.conn = sqlite3.connect(state.appdata + 'messages.dat')
self.conn.text_factory = str
self.conn.text_factory = bytes
self.cur = self.conn.cursor()
self.cur.execute('PRAGMA secure_delete = true')
@ -542,7 +542,7 @@ class sqlThread(threading.Thread):
shutil.move(
paths.lookupAppdataFolder() + 'messages.dat', paths.lookupExeFolder() + 'messages.dat')
self.conn = sqlite3.connect(paths.lookupExeFolder() + 'messages.dat')
self.conn.text_factory = str
self.conn.text_factory = bytes
self.cur = self.conn.cursor()
elif item == 'movemessagstoappdata':
logger.debug('the sqlThread is moving the messages.dat file to the Appdata folder.')
@ -568,7 +568,7 @@ class sqlThread(threading.Thread):
shutil.move(
paths.lookupExeFolder() + 'messages.dat', paths.lookupAppdataFolder() + 'messages.dat')
self.conn = sqlite3.connect(paths.lookupAppdataFolder() + 'messages.dat')
self.conn.text_factory = str
self.conn.text_factory = bytes
self.cur = self.conn.cursor()
elif item == 'deleteandvacuume':
self.cur.execute('''delete from inbox where folder='trash' ''')

3
src/helper_msgcoding.py
View File

@ -71,7 +71,8 @@ class MsgEncode(object):
def encodeSimple(self, message):
"""Handle simple encoding"""
self.data = 'Subject:%(subject)s\nBody:%(body)s' % message
data = 'Subject:%(subject)s\nBody:%(body)s' % message
self.data = data.encode("utf-8", "replace")
self.length = len(self.data)
def encodeTrivial(self, message):

3
src/shared.py
View File

@ -47,7 +47,7 @@ def isAddressInMySubscriptionsList(address):
"""Am I subscribed to this address?"""
queryreturn = sqlQuery(
'''select * from subscriptions where address=?''',
str(address))
address.encode("utf-8", "replace"))
return queryreturn != []
@ -136,6 +136,7 @@ def reloadBroadcastSendersForWhichImWatching():
logger.debug('reloading subscriptions...')
for row in queryreturn:
address, = row
address = address.decode("utf-8", "replace")
# status
addressVersionNumber, streamNumber, hashobj = decodeAddress(address)[1:]
if addressVersionNumber == 2:

6
src/tests/test_shared.py
View File

@ -46,7 +46,7 @@ class TestShared(unittest.TestCase):
address = sample_address
# if address is in MyAddressbook
mock_sql_query.return_value = [address]
mock_sql_query.return_value = [address.encode("utf-8", "replace")]
return_val = isAddressInMyAddressBook(address)
mock_sql_query.assert_called_once()
self.assertTrue(return_val)
@ -64,7 +64,7 @@ class TestShared(unittest.TestCase):
address = sample_address
# if address is in MySubscriptionsList
mock_sql_query.return_value = [address]
mock_sql_query.return_value = [address.encode("utf-8", "replace")]
return_val = isAddressInMySubscriptionsList(address)
self.assertTrue(return_val)
@ -78,7 +78,7 @@ class TestShared(unittest.TestCase):
def test_reloadBroadcastSendersForWhichImWatching(self, mock_sql_query):
"""Test for reload Broadcast Senders For Which Im Watching"""
mock_sql_query.return_value = [
(sample_address,),
(sample_address.encode("utf-8", "replace"),),
]
# before reload
self.assertEqual(len(MyECSubscriptionCryptorObjects), 0)