resolved conflicts

This commit is contained in:
Navjot 2019-09-27 19:20:14 +05:30
commit be8e84e4d5
No known key found for this signature in database
GPG Key ID: 9EE70AFD71357F1C
15 changed files with 249 additions and 95 deletions

View File

@ -1,4 +1,8 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""
src/plugins/indicator_libmessaging.py
=====================================
"""
import gi import gi
gi.require_version('MessagingMenu', '1.0') # noqa:E402 gi.require_version('MessagingMenu', '1.0') # noqa:E402
@ -9,6 +13,7 @@ from pybitmessage.tr import _translate
class IndicatorLibmessaging(object): class IndicatorLibmessaging(object):
"""Plugin for libmessage indicator"""
def __init__(self, form): def __init__(self, form):
try: try:
self.app = MessagingMenu.App(desktop_id='pybitmessage.desktop') self.app = MessagingMenu.App(desktop_id='pybitmessage.desktop')
@ -32,15 +37,18 @@ class IndicatorLibmessaging(object):
if self.app: if self.app:
self.app.unregister() self.app.unregister()
def activate(self, app, source): def activate(self, app, source): # pylint: disable=unused-argument
"""Activate the libmessaging indicator plugin"""
self.form.appIndicatorInbox( self.form.appIndicatorInbox(
self.new_message_item if source == 'messages' self.new_message_item if source == 'messages'
else self.new_broadcast_item else self.new_broadcast_item
) )
# show the number of unread messages and subscriptions
# on the messaging menu
def show_unread(self, draw_attention=False): def show_unread(self, draw_attention=False):
"""
show the number of unread messages and subscriptions
on the messaging menu
"""
for source, count in zip( for source, count in zip(
('messages', 'subscriptions'), ('messages', 'subscriptions'),
self.form.getUnread() self.form.getUnread()

View File

@ -1,5 +1,8 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
""" """
src/plugins/menu_qrcode.py
==========================
A menu plugin showing QR-Code for bitmessage address in modal dialog. A menu plugin showing QR-Code for bitmessage address in modal dialog.
""" """
@ -12,9 +15,10 @@ from pybitmessage.tr import _translate
# http://stackoverflow.com/questions/20452486 # http://stackoverflow.com/questions/20452486
class Image(qrcode.image.base.BaseImage): class Image(qrcode.image.base.BaseImage): # pylint: disable=abstract-method
"""Image output class for qrcode using QPainter""" """Image output class for qrcode using QPainter"""
def __init__(self, border, width, box_size):
def __init__(self, border, width, box_size): # pylint: disable=super-init-not-called
self.border = border self.border = border
self.width = width self.width = width
self.box_size = box_size self.box_size = box_size

View File

@ -1,4 +1,8 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""
src/plugins/notification_notify2.py
===================================
"""
import gi import gi
gi.require_version('Notify', '0.7') gi.require_version('Notify', '0.7')
@ -6,10 +10,13 @@ from gi.repository import Notify
Notify.init('pybitmessage') Notify.init('pybitmessage')
def connect_plugin(title, subtitle, category, label, icon): def connect_plugin(title, subtitle, category, label, icon):
"""Plugin for notify2"""
if not icon: if not icon:
icon = 'mail-message-new' if category == 2 else 'pybitmessage' icon = 'mail-message-new' if category == 2 else 'pybitmessage'
connect_plugin.notification.update(title, subtitle, icon) connect_plugin.notification.update(title, subtitle, icon)
connect_plugin.notification.show() connect_plugin.notification.show()
connect_plugin.notification = Notify.Notification.new("Init", "Init") connect_plugin.notification = Notify.Notification.new("Init", "Init")

View File

@ -1,5 +1,8 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""
src/plugins/plugin.py
===================================
"""
import pkg_resources import pkg_resources

View File

@ -1,5 +1,8 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""
src/plugins/proxyconfig_stem.py
===================================
"""
import os import os
import logging import logging
import random # noseq import random # noseq
@ -29,7 +32,7 @@ class DebugLogger(object):
# Plugin's debug or unexpected log line from tor # Plugin's debug or unexpected log line from tor
self._logger.debug(line) self._logger.debug(line)
else: else:
self._logger.log(self._levels.get(level, 10), '(tor)' + line) self._logger.log(self._levels.get(level, 10), '(tor) %s', line)
def connect_plugin(config): # pylint: disable=too-many-branches def connect_plugin(config): # pylint: disable=too-many-branches

View File

@ -1,4 +1,8 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""
src/plugins/proxyconfig_stem.py
===================================
"""
from pybitmessage.bitmessageqt import sound from pybitmessage.bitmessageqt import sound
@ -14,7 +18,8 @@ _theme = {
} }
def connect_plugin(category, label=None): def connect_plugin(category, label=None): # pylint: disable=unused-argument
"""This function implements the entry point."""
try: try:
_canberra.play(0, pycanberra.CA_PROP_EVENT_ID, _theme[category], None) _canberra.play(0, pycanberra.CA_PROP_EVENT_ID, _theme[category], None)
except (KeyError, pycanberra.CanberraException): except (KeyError, pycanberra.CanberraException):

View File

@ -1,5 +1,8 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""
src/plugins/sound_gstreamer.py
===================================
"""
import gi import gi
gi.require_version('Gst', '1.0') gi.require_version('Gst', '1.0')
from gi.repository import Gst # noqa: E402 from gi.repository import Gst # noqa: E402
@ -9,6 +12,7 @@ _player = Gst.ElementFactory.make("playbin", "player")
def connect_plugin(sound_file): def connect_plugin(sound_file):
"""Entry point for sound file"""
_player.set_state(Gst.State.NULL) _player.set_state(Gst.State.NULL)
_player.set_property("uri", "file://" + sound_file) _player.set_property("uri", "file://" + sound_file)
_player.set_state(Gst.State.PLAYING) _player.set_state(Gst.State.PLAYING)

View File

@ -1,10 +1,14 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""
src/plugins/sound_playfile.py
===================================
"""
try: try:
import winsound import winsound
def connect_plugin(sound_file): def connect_plugin(sound_file):
"""Plugin's entry point"""
winsound.PlaySound(sound_file, winsound.SND_FILENAME) winsound.PlaySound(sound_file, winsound.SND_FILENAME)
except ImportError: except ImportError:
import os import os
@ -18,7 +22,8 @@ except ImportError:
args, stdout=FNULL, stderr=subprocess.STDOUT, close_fds=True) args, stdout=FNULL, stderr=subprocess.STDOUT, close_fds=True)
def connect_plugin(sound_file): def connect_plugin(sound_file):
global play_cmd """This function implements the entry point."""
global play_cmd # pylint: disable=global-statement
ext = os.path.splitext(sound_file)[-1] ext = os.path.splitext(sound_file)[-1]
try: try:

View File

@ -1,3 +1,7 @@
"""
src/pyelliptic/__init__.py
=====================================
"""
# Copyright (C) 2010 # Copyright (C) 2010
# Author: Yann GUIBET # Author: Yann GUIBET
# Contact: <yannguibet@gmail.com> # Contact: <yannguibet@gmail.com>

View File

@ -1,5 +1,9 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""
src/pyelliptic/cipher.py
========================
"""
# Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com> # Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
# See LICENSE for details. # See LICENSE for details.
@ -7,7 +11,8 @@
from openssl import OpenSSL from openssl import OpenSSL
class Cipher: # pylint: disable=redefined-builtin
class Cipher(object):
""" """
Symmetric encryption Symmetric encryption
@ -44,30 +49,34 @@ class Cipher:
@staticmethod @staticmethod
def get_blocksize(ciphername): def get_blocksize(ciphername):
"""This Method returns cipher blocksize"""
cipher = OpenSSL.get_cipher(ciphername) cipher = OpenSSL.get_cipher(ciphername)
return cipher.get_blocksize() return cipher.get_blocksize()
@staticmethod @staticmethod
def gen_IV(ciphername): def gen_IV(ciphername):
"""Generate random initialization vector"""
cipher = OpenSSL.get_cipher(ciphername) cipher = OpenSSL.get_cipher(ciphername)
return OpenSSL.rand(cipher.get_blocksize()) return OpenSSL.rand(cipher.get_blocksize())
def update(self, input): def update(self, input):
"""Update result with more data"""
i = OpenSSL.c_int(0) i = OpenSSL.c_int(0)
buffer = OpenSSL.malloc(b"", len(input) + self.cipher.get_blocksize()) buffer = OpenSSL.malloc(b"", len(input) + self.cipher.get_blocksize())
inp = OpenSSL.malloc(input, len(input)) inp = OpenSSL.malloc(input, len(input))
if OpenSSL.EVP_CipherUpdate(self.ctx, OpenSSL.byref(buffer), if OpenSSL.EVP_CipherUpdate(self.ctx, OpenSSL.byref(buffer),
OpenSSL.byref(i), inp, len(input)) == 0: OpenSSL.byref(i), inp, len(input)) == 0:
raise Exception("[OpenSSL] EVP_CipherUpdate FAIL ...") raise Exception("[OpenSSL] EVP_CipherUpdate FAIL ...")
return buffer.raw[0:i.value] return buffer.raw[0:i.value] # pylint: disable=invalid-slice-index
def final(self): def final(self):
"""Returning the final value"""
i = OpenSSL.c_int(0) i = OpenSSL.c_int(0)
buffer = OpenSSL.malloc(b"", self.cipher.get_blocksize()) buffer = OpenSSL.malloc(b"", self.cipher.get_blocksize())
if (OpenSSL.EVP_CipherFinal_ex(self.ctx, OpenSSL.byref(buffer), if (OpenSSL.EVP_CipherFinal_ex(self.ctx, OpenSSL.byref(buffer),
OpenSSL.byref(i))) == 0: OpenSSL.byref(i))) == 0:
raise Exception("[OpenSSL] EVP_CipherFinal_ex FAIL ...") raise Exception("[OpenSSL] EVP_CipherFinal_ex FAIL ...")
return buffer.raw[0:i.value] return buffer.raw[0:i.value] # pylint: disable=invalid-slice-index
def ciphering(self, input): def ciphering(self, input):
""" """
@ -77,6 +86,7 @@ class Cipher:
return buff + self.final() return buff + self.final()
def __del__(self): def __del__(self):
# pylint: disable=protected-access
if OpenSSL._hexversion > 0x10100000 and not OpenSSL._libreSSL: if OpenSSL._hexversion > 0x10100000 and not OpenSSL._libreSSL:
OpenSSL.EVP_CIPHER_CTX_reset(self.ctx) OpenSSL.EVP_CIPHER_CTX_reset(self.ctx)
else: else:

View File

@ -1,6 +1,9 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""
src/pyelliptic/hash.py
=====================
"""
# Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com> # Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
# See LICENSE for details. # See LICENSE for details.
@ -27,9 +30,9 @@ def _equals_str(a, b):
def equals(a, b): def equals(a, b):
"""Compare two strings or bytearrays"""
if isinstance(a, str): if isinstance(a, str):
return _equals_str(a, b) return _equals_str(a, b)
else:
return _equals_bytes(a, b) return _equals_bytes(a, b)
@ -58,6 +61,7 @@ def hmac_sha512(k, m):
def pbkdf2(password, salt=None, i=10000, keylen=64): def pbkdf2(password, salt=None, i=10000, keylen=64):
"""Key derivation function using SHA256"""
if salt is None: if salt is None:
salt = OpenSSL.rand(8) salt = OpenSSL.rand(8)
p_password = OpenSSL.malloc(password, len(password)) p_password = OpenSSL.malloc(password, len(password))

View File

@ -11,16 +11,23 @@ OpenSSL = None
# !/usr/bin/env python # !/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""
src/pyelliptic/openssl.py
=====================
"""
# Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com> # Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
# See LICENSE for details. # See LICENSE for details.
# #
# Software slightly changed by Jonathan Warren <bitmessage at-symbol jonwarren.org> # Software slightly changed by Jonathan Warren <bitmessage at-symbol jonwarren.org>
# pylint: disable=protected-access
class CipherName: # pylint: disable=old-style-class
"""Method helps to get pointers, name and blocksize"""
class CipherName:
"""Class returns cipher name, pointer and blocksize"""
# pylint: disable=old-style-class
def __init__(self, name, pointer, blocksize): def __init__(self, name, pointer, blocksize):
self._name = name self._name = name
self._pointer = pointer self._pointer = pointer
@ -36,11 +43,11 @@ class CipherName: # pylint: disable=old-style-class
return self._pointer() return self._pointer()
def get_name(self): def get_name(self):
"""Method returns the name""" """This method returns cipher name"""
return self._name return self._name
def get_blocksize(self): def get_blocksize(self):
"""Method returns the blocksize""" """This method returns cipher blocksize"""
return self._blocksize return self._blocksize
@ -76,9 +83,11 @@ def get_version(library):
return (version, hexversion, cflags) return (version, hexversion, cflags)
class _OpenSSL: # pylint: disable=too-many-instance-attributes, old-style-class, too-many-statements class _OpenSSL:
"""Wrapper for OpenSSL using ctypes""" """
Wrapper for OpenSSL using ctypes
"""
# pylint: disable=too-many-statements, too-many-instance-attributes, old-style-class
def __init__(self, library): def __init__(self, library):
"""Build the wrapper""" """Build the wrapper"""
self._lib = ctypes.CDLL(library) self._lib = ctypes.CDLL(library)
@ -647,8 +656,7 @@ class _OpenSSL: # pylint: disable=too-many-instance-attributes, old-style-cl
def loadOpenSSL(): def loadOpenSSL():
"""Method find and load the OpenSSL library""" """Method find and load the OpenSSL library"""
# pylint: disable=global-statement, protected-access, too-many-branches, no-member # pylint: disable=global-statement, protected-access, too-many-branches
global OpenSSL global OpenSSL
from os import path, environ from os import path, environ
from ctypes.util import find_library from ctypes.util import find_library

View File

@ -1,21 +1,26 @@
"""
src/storage/filesystem.py
=========================
"""
from binascii import hexlify, unhexlify from binascii import hexlify, unhexlify
from os import listdir, makedirs, path, remove, rmdir from os import listdir, makedirs, path, remove, rmdir
import string import string
from threading import RLock from threading import RLock
import time import time
import traceback
from paths import lookupAppdataFolder from paths import lookupAppdataFolder
from storage import InventoryStorage, InventoryItem from storage import InventoryStorage, InventoryItem
class FilesystemInventory(InventoryStorage):
class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ancestors, abstract-method
"""Module for using filesystem (directory with files) for inventory storage"""
topDir = "inventory" topDir = "inventory"
objectDir = "objects" objectDir = "objects"
metadataFilename = "metadata" metadataFilename = "metadata"
dataFilename = "data" dataFilename = "data"
def __init__(self): def __init__(self):
super(self.__class__, self).__init__() super(FilesystemInventory, self).__init__()
self.baseDir = path.join(lookupAppdataFolder(), FilesystemInventory.topDir) self.baseDir = path.join(lookupAppdataFolder(), FilesystemInventory.topDir)
for createDir in [self.baseDir, path.join(self.baseDir, "objects")]: for createDir in [self.baseDir, path.join(self.baseDir, "objects")]:
if path.exists(createDir): if path.exists(createDir):
@ -23,65 +28,94 @@ class FilesystemInventory(InventoryStorage):
raise IOError("%s exists but it's not a directory" % (createDir)) raise IOError("%s exists but it's not a directory" % (createDir))
else: else:
makedirs(createDir) makedirs(createDir)
self.lock = RLock() # Guarantees that two receiveDataThreads don't receive and process the same message concurrently (probably sent by a malicious individual) # Guarantees that two receiveDataThreads don't receive and process the same message
# concurrently (probably sent by a malicious individual)
self.lock = RLock()
self._inventory = {} self._inventory = {}
self._load() self._load()
def __contains__(self, hash): def __contains__(self, hashval):
retval = False retval = False
for streamDict in self._inventory.values(): for streamDict in self._inventory.values():
if hash in streamDict: if hashval in streamDict:
return True return True
return False return False
def __getitem__(self, hash): def __getitem__(self, hashval):
for streamDict in self._inventory.values(): for streamDict in self._inventory.values():
try: try:
retval = streamDict[hash] retval = streamDict[hashval]
except KeyError: except KeyError:
continue continue
if retval.payload is None: if retval.payload is None:
retval = InventoryItem(retval.type, retval.stream, self.getData(hash), retval.expires, retval.tag) retval = InventoryItem(retval.type, retval.stream, self.getData(hashval), retval.expires, retval.tag)
return retval return retval
raise KeyError(hash) raise KeyError(hashval)
def __setitem__(self, hash, value): def __setitem__(self, hashval, value):
with self.lock: with self.lock:
value = InventoryItem(*value) value = InventoryItem(*value)
try: try:
makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash))) makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval)))
except OSError: except OSError:
pass pass
try: try:
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.metadataFilename), 'w') as f: with open(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashval),
FilesystemInventory.metadataFilename,
),
"w",
) as f:
f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag))) f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag)))
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.dataFilename), 'w') as f: with open(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashval),
FilesystemInventory.dataFilename,
),
"w",
) as f:
f.write(value.payload) f.write(value.payload)
except IOError: except IOError:
raise KeyError raise KeyError
try: try:
self._inventory[value.stream][hash] = value self._inventory[value.stream][hashval] = value
except KeyError: except KeyError:
self._inventory[value.stream] = {} self._inventory[value.stream] = {}
self._inventory[value.stream][hash] = value self._inventory[value.stream][hashval] = value
def delHashId(self, hash): def delHashId(self, hashval):
for stream in self._inventory.keys(): """Remove object from inventory"""
for stream in self._inventory:
try: try:
del self._inventory[stream][hash] del self._inventory[stream][hashval]
except KeyError: except KeyError:
pass pass
with self.lock: with self.lock:
try: try:
remove(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.metadataFilename)) remove(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashval),
FilesystemInventory.metadataFilename))
except IOError: except IOError:
pass pass
try: try:
remove(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.dataFilename)) remove(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashval),
FilesystemInventory.dataFilename))
except IOError: except IOError:
pass pass
try: try:
rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash))) rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval)))
except IOError: except IOError:
pass pass
@ -103,39 +137,61 @@ class FilesystemInventory(InventoryStorage):
try: try:
objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId) objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId)
try: try:
newInventory[streamNumber][hashId] = InventoryItem(objectType, streamNumber, None, expiresTime, tag) newInventory[streamNumber][hashId] = InventoryItem(
objectType, streamNumber, None, expiresTime, tag)
except KeyError: except KeyError:
newInventory[streamNumber] = {} newInventory[streamNumber] = {}
newInventory[streamNumber][hashId] = InventoryItem(objectType, streamNumber, None, expiresTime, tag) newInventory[streamNumber][hashId] = InventoryItem(
objectType, streamNumber, None, expiresTime, tag)
except KeyError: except KeyError:
print "error loading %s" % (hexlify(hashId)) print "error loading %s" % (hexlify(hashId))
pass
self._inventory = newInventory self._inventory = newInventory
# for i, v in self._inventory.items(): # for i, v in self._inventory.items():
# print "loaded stream: %s, %i items" % (i, len(v)) # print "loaded stream: %s, %i items" % (i, len(v))
def stream_list(self): def stream_list(self):
"""Return list of streams"""
return self._inventory.keys() return self._inventory.keys()
def object_list(self): def object_list(self):
"""Return inventory vectors (hashes) from a directory"""
return [unhexlify(x) for x in listdir(path.join(self.baseDir, FilesystemInventory.objectDir))] return [unhexlify(x) for x in listdir(path.join(self.baseDir, FilesystemInventory.objectDir))]
def getData(self, hashId): def getData(self, hashId):
"""Get object data"""
try: try:
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashId), FilesystemInventory.dataFilename), 'r') as f: with open(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashId),
FilesystemInventory.dataFilename,
),
"r",
) as f:
return f.read() return f.read()
except IOError: except IOError:
raise AttributeError raise AttributeError
def getMetadata(self, hashId): def getMetadata(self, hashId):
"""Get object metadata"""
try: try:
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashId), FilesystemInventory.metadataFilename), 'r') as f: with open(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashId),
FilesystemInventory.metadataFilename,
),
"r",
) as f:
objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4) objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4)
return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)] return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)]
except IOError: except IOError:
raise KeyError raise KeyError
def by_type_and_tag(self, objectType, tag): def by_type_and_tag(self, objectType, tag):
"""Get a list of objects filtered by object type and tag"""
retval = [] retval = []
for stream, streamDict in self._inventory: for stream, streamDict in self._inventory:
for hashId, item in streamDict: for hashId, item in streamDict:
@ -149,12 +205,14 @@ class FilesystemInventory(InventoryStorage):
return retval return retval
def hashes_by_stream(self, stream): def hashes_by_stream(self, stream):
"""Return inventory vectors (hashes) for a stream"""
try: try:
return self._inventory[stream].keys() return self._inventory[stream].keys()
except KeyError: except KeyError:
return [] return []
def unexpired_hashes_by_stream(self, stream): def unexpired_hashes_by_stream(self, stream):
"""Return unexpired hashes in the inventory for a particular stream"""
t = int(time.time()) t = int(time.time())
try: try:
return [x for x, value in self._inventory[stream].items() if value.expires > t] return [x for x, value in self._inventory[stream].items() if value.expires > t]
@ -162,9 +220,11 @@ class FilesystemInventory(InventoryStorage):
return [] return []
def flush(self): def flush(self):
"""Flush the inventory and create a new, empty one"""
self._load() self._load()
def clean(self): def clean(self):
"""Clean out old items from the inventory"""
minTime = int(time.time()) - (60 * 60 * 30) minTime = int(time.time()) - (60 * 60 * 30)
deletes = [] deletes = []
for stream, streamDict in self._inventory.items(): for stream, streamDict in self._inventory.items():

View File

@ -1,45 +1,59 @@
import collections """
from threading import current_thread, enumerate as threadingEnumerate, RLock src/storage/sqlite.py
import Queue =========================
"""
import sqlite3 import sqlite3
import time import time
from threading import RLock
from helper_sql import * from helper_sql import sqlQuery, SqlBulkExecute, sqlExecute
from storage import InventoryStorage, InventoryItem from storage import InventoryStorage, InventoryItem
class SqliteInventory(InventoryStorage):
def __init__(self):
super(self.__class__, self).__init__()
self._inventory = {} #of objects (like msg payloads and pubkey payloads) Does not include protocol headers (the first 24 bytes of each packet).
self._objects = {} # cache for existing objects, used for quick lookups if we have an object. This is used for example whenever we receive an inv message from a peer to check to see what items are new to us. We don't delete things out of it; instead, the singleCleaner thread clears and refills it.
self.lock = RLock() # Guarantees that two receiveDataThreads don't receive and process the same message concurrently (probably sent by a malicious individual)
def __contains__(self, hash): class SqliteInventory(InventoryStorage): # pylint: disable=too-many-ancestors
"""Inventory using SQLite"""
def __init__(self):
super(SqliteInventory, self).__init__()
# of objects (like msg payloads and pubkey payloads)
# Does not include protocol headers (the first 24 bytes of each packet).
self._inventory = {}
# cache for existing objects, used for quick lookups if we have an object.
# This is used for example whenever we receive an inv message from a peer
# to check to see what items are new to us.
# We don't delete things out of it; instead, the singleCleaner thread clears and refills it.
self._objects = {}
# Guarantees that two receiveDataThreads don't receive and process the same message concurrently
# (probably sent by a malicious individual)
self.lock = RLock()
def __contains__(self, hash_):
with self.lock: with self.lock:
if hash in self._objects: if hash_ in self._objects:
return True return True
rows = sqlQuery('SELECT streamnumber FROM inventory WHERE hash=?', sqlite3.Binary(hash)) rows = sqlQuery('SELECT streamnumber FROM inventory WHERE hash=?', sqlite3.Binary(hash_))
if not rows: if not rows:
return False return False
self._objects[hash] = rows[0][0] self._objects[hash_] = rows[0][0]
return True return True
def __getitem__(self, hash): def __getitem__(self, hash_):
with self.lock: with self.lock:
if hash in self._inventory: if hash_ in self._inventory:
return self._inventory[hash] return self._inventory[hash_]
rows = sqlQuery('SELECT objecttype, streamnumber, payload, expirestime, tag FROM inventory WHERE hash=?', sqlite3.Binary(hash)) rows = sqlQuery(
'SELECT objecttype, streamnumber, payload, expirestime, tag FROM inventory WHERE hash=?',
sqlite3.Binary(hash_))
if not rows: if not rows:
raise KeyError(hash) raise KeyError(hash_)
return InventoryItem(*rows[0]) return InventoryItem(*rows[0])
def __setitem__(self, hash, value): def __setitem__(self, hash_, value):
with self.lock: with self.lock:
value = InventoryItem(*value) value = InventoryItem(*value)
self._inventory[hash] = value self._inventory[hash_] = value
self._objects[hash] = value.stream self._objects[hash_] = value.stream
def __delitem__(self, hash): def __delitem__(self, hash_):
raise NotImplementedError raise NotImplementedError
def __iter__(self): def __iter__(self):
@ -55,18 +69,22 @@ class SqliteInventory(InventoryStorage):
def by_type_and_tag(self, objectType, tag): def by_type_and_tag(self, objectType, tag):
with self.lock: with self.lock:
values = [value for value in self._inventory.values() if value.type == objectType and value.tag == tag] values = [value for value in self._inventory.values() if value.type == objectType and value.tag == tag]
values += (InventoryItem(*value) for value in sqlQuery('SELECT objecttype, streamnumber, payload, expirestime, tag FROM inventory WHERE objecttype=? AND tag=?', objectType, sqlite3.Binary(tag))) values += (InventoryItem(*value) for value in sqlQuery(
'SELECT objecttype, streamnumber, payload, expirestime, tag \
FROM inventory WHERE objecttype=? AND tag=?', objectType, sqlite3.Binary(tag)))
return values return values
def unexpired_hashes_by_stream(self, stream): def unexpired_hashes_by_stream(self, stream):
with self.lock: with self.lock:
t = int(time.time()) t = int(time.time())
hashes = [x for x, value in self._inventory.items() if value.stream == stream and value.expires > t] hashes = [x for x, value in self._inventory.items() if value.stream == stream and value.expires > t]
hashes += (str(payload) for payload, in sqlQuery('SELECT hash FROM inventory WHERE streamnumber=? AND expirestime>?', stream, t)) hashes += (str(payload) for payload, in sqlQuery(
'SELECT hash FROM inventory WHERE streamnumber=? AND expirestime>?', stream, t))
return hashes return hashes
def flush(self): def flush(self):
with self.lock: # If you use both the inventoryLock and the sqlLock, always use the inventoryLock OUTSIDE of the sqlLock. with self.lock:
# If you use both the inventoryLock and the sqlLock, always use the inventoryLock OUTSIDE of the sqlLock.
with SqlBulkExecute() as sql: with SqlBulkExecute() as sql:
for objectHash, value in self._inventory.items(): for objectHash, value in self._inventory.items():
sql.execute('INSERT INTO inventory VALUES (?, ?, ?, ?, ?, ?)', sqlite3.Binary(objectHash), *value) sql.execute('INSERT INTO inventory VALUES (?, ?, ?, ?, ?, ?)', sqlite3.Binary(objectHash), *value)
@ -78,4 +96,3 @@ class SqliteInventory(InventoryStorage):
self._objects.clear() self._objects.clear()
for objectHash, value in self._inventory.items(): for objectHash, value in self._inventory.items():
self._objects[objectHash] = value.stream self._objects[objectHash] = value.stream

View File

@ -1,27 +1,33 @@
"""
src/storage/storage.py
======================
"""
import collections import collections
InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload expires tag') InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload expires tag')
class Storage(object): class Storage(object):
"""Base class for storing inventory (extendable for other items to store)"""
pass pass
# def __init__(self):
# super(self.__class__, self).__init__()
class InventoryStorage(Storage, collections.MutableMapping): class InventoryStorage(Storage, collections.MutableMapping):
"""Module used for inventory storage"""
def __init__(self): def __init__(self):
# super(self.__class__, self).__init__() # pylint: disable=super-init-not-called
self.numberOfInventoryLookupsPerformed = 0 self.numberOfInventoryLookupsPerformed = 0
def __contains__(self, hash): def __contains__(self, _):
raise NotImplementedError raise NotImplementedError
def __getitem__(self, hash): def __getitem__(self, _):
raise NotImplementedError raise NotImplementedError
def __setitem__(self, hash, value): def __setitem__(self, _, value):
raise NotImplementedError raise NotImplementedError
def __delitem__(self, hash): def __delitem__(self, _):
raise NotImplementedError raise NotImplementedError
def __iter__(self): def __iter__(self):
@ -31,18 +37,24 @@ class InventoryStorage(Storage, collections.MutableMapping):
raise NotImplementedError raise NotImplementedError
def by_type_and_tag(self, objectType, tag): def by_type_and_tag(self, objectType, tag):
"""Return objects filtered by object type and tag"""
raise NotImplementedError raise NotImplementedError
def unexpired_hashes_by_stream(self, stream): def unexpired_hashes_by_stream(self, stream):
"""Return unexpired inventory vectors filtered by stream"""
raise NotImplementedError raise NotImplementedError
def flush(self): def flush(self):
"""Flush cache"""
raise NotImplementedError raise NotImplementedError
def clean(self): def clean(self):
"""Free memory / perform garbage collection"""
raise NotImplementedError raise NotImplementedError
class MailboxStorage(Storage, collections.MutableMapping):
class MailboxStorage(Storage, collections.MutableMapping): # pylint: disable=abstract-method
"""Method for storing mails"""
def __init__(self): def __init__(self):
# super(self.__class__, self).__init__() # pylint: disable=super-init-not-called
pass pass