resolved conflicts
This commit is contained in:
commit
be8e84e4d5
|
@ -1,4 +1,8 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
src/plugins/indicator_libmessaging.py
|
||||||
|
=====================================
|
||||||
|
"""
|
||||||
|
|
||||||
import gi
|
import gi
|
||||||
gi.require_version('MessagingMenu', '1.0') # noqa:E402
|
gi.require_version('MessagingMenu', '1.0') # noqa:E402
|
||||||
|
@ -9,6 +13,7 @@ from pybitmessage.tr import _translate
|
||||||
|
|
||||||
|
|
||||||
class IndicatorLibmessaging(object):
|
class IndicatorLibmessaging(object):
|
||||||
|
"""Plugin for libmessage indicator"""
|
||||||
def __init__(self, form):
|
def __init__(self, form):
|
||||||
try:
|
try:
|
||||||
self.app = MessagingMenu.App(desktop_id='pybitmessage.desktop')
|
self.app = MessagingMenu.App(desktop_id='pybitmessage.desktop')
|
||||||
|
@ -32,15 +37,18 @@ class IndicatorLibmessaging(object):
|
||||||
if self.app:
|
if self.app:
|
||||||
self.app.unregister()
|
self.app.unregister()
|
||||||
|
|
||||||
def activate(self, app, source):
|
def activate(self, app, source): # pylint: disable=unused-argument
|
||||||
|
"""Activate the libmessaging indicator plugin"""
|
||||||
self.form.appIndicatorInbox(
|
self.form.appIndicatorInbox(
|
||||||
self.new_message_item if source == 'messages'
|
self.new_message_item if source == 'messages'
|
||||||
else self.new_broadcast_item
|
else self.new_broadcast_item
|
||||||
)
|
)
|
||||||
|
|
||||||
# show the number of unread messages and subscriptions
|
|
||||||
# on the messaging menu
|
|
||||||
def show_unread(self, draw_attention=False):
|
def show_unread(self, draw_attention=False):
|
||||||
|
"""
|
||||||
|
show the number of unread messages and subscriptions
|
||||||
|
on the messaging menu
|
||||||
|
"""
|
||||||
for source, count in zip(
|
for source, count in zip(
|
||||||
('messages', 'subscriptions'),
|
('messages', 'subscriptions'),
|
||||||
self.form.getUnread()
|
self.form.getUnread()
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
"""
|
"""
|
||||||
|
src/plugins/menu_qrcode.py
|
||||||
|
==========================
|
||||||
|
|
||||||
A menu plugin showing QR-Code for bitmessage address in modal dialog.
|
A menu plugin showing QR-Code for bitmessage address in modal dialog.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@ -12,9 +15,10 @@ from pybitmessage.tr import _translate
|
||||||
|
|
||||||
|
|
||||||
# http://stackoverflow.com/questions/20452486
|
# http://stackoverflow.com/questions/20452486
|
||||||
class Image(qrcode.image.base.BaseImage):
|
class Image(qrcode.image.base.BaseImage): # pylint: disable=abstract-method
|
||||||
"""Image output class for qrcode using QPainter"""
|
"""Image output class for qrcode using QPainter"""
|
||||||
def __init__(self, border, width, box_size):
|
|
||||||
|
def __init__(self, border, width, box_size): # pylint: disable=super-init-not-called
|
||||||
self.border = border
|
self.border = border
|
||||||
self.width = width
|
self.width = width
|
||||||
self.box_size = box_size
|
self.box_size = box_size
|
||||||
|
|
|
@ -1,4 +1,8 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
src/plugins/notification_notify2.py
|
||||||
|
===================================
|
||||||
|
"""
|
||||||
|
|
||||||
import gi
|
import gi
|
||||||
gi.require_version('Notify', '0.7')
|
gi.require_version('Notify', '0.7')
|
||||||
|
@ -6,10 +10,13 @@ from gi.repository import Notify
|
||||||
|
|
||||||
Notify.init('pybitmessage')
|
Notify.init('pybitmessage')
|
||||||
|
|
||||||
|
|
||||||
def connect_plugin(title, subtitle, category, label, icon):
|
def connect_plugin(title, subtitle, category, label, icon):
|
||||||
|
"""Plugin for notify2"""
|
||||||
if not icon:
|
if not icon:
|
||||||
icon = 'mail-message-new' if category == 2 else 'pybitmessage'
|
icon = 'mail-message-new' if category == 2 else 'pybitmessage'
|
||||||
connect_plugin.notification.update(title, subtitle, icon)
|
connect_plugin.notification.update(title, subtitle, icon)
|
||||||
connect_plugin.notification.show()
|
connect_plugin.notification.show()
|
||||||
|
|
||||||
|
|
||||||
connect_plugin.notification = Notify.Notification.new("Init", "Init")
|
connect_plugin.notification = Notify.Notification.new("Init", "Init")
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
src/plugins/plugin.py
|
||||||
|
===================================
|
||||||
|
"""
|
||||||
import pkg_resources
|
import pkg_resources
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
src/plugins/proxyconfig_stem.py
|
||||||
|
===================================
|
||||||
|
"""
|
||||||
import os
|
import os
|
||||||
import logging
|
import logging
|
||||||
import random # noseq
|
import random # noseq
|
||||||
|
@ -29,14 +32,14 @@ class DebugLogger(object):
|
||||||
# Plugin's debug or unexpected log line from tor
|
# Plugin's debug or unexpected log line from tor
|
||||||
self._logger.debug(line)
|
self._logger.debug(line)
|
||||||
else:
|
else:
|
||||||
self._logger.log(self._levels.get(level, 10), '(tor)' + line)
|
self._logger.log(self._levels.get(level, 10), '(tor) %s', line)
|
||||||
|
|
||||||
|
|
||||||
def connect_plugin(config): # pylint: disable=too-many-branches
|
def connect_plugin(config): # pylint: disable=too-many-branches
|
||||||
"""Run stem proxy configurator"""
|
"""Run stem proxy configurator"""
|
||||||
logwrite = DebugLogger()
|
logwrite = DebugLogger()
|
||||||
if config.safeGet('bitmessagesettings', 'sockshostname') not in (
|
if config.safeGet('bitmessagesettings', 'sockshostname') not in (
|
||||||
'localhost', '127.0.0.1', ''
|
'localhost', '127.0.0.1', ''
|
||||||
):
|
):
|
||||||
# remote proxy is choosen for outbound connections,
|
# remote proxy is choosen for outbound connections,
|
||||||
# nothing to do here, but need to set socksproxytype to SOCKS5!
|
# nothing to do here, but need to set socksproxytype to SOCKS5!
|
||||||
|
|
|
@ -1,4 +1,8 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
src/plugins/proxyconfig_stem.py
|
||||||
|
===================================
|
||||||
|
"""
|
||||||
|
|
||||||
from pybitmessage.bitmessageqt import sound
|
from pybitmessage.bitmessageqt import sound
|
||||||
|
|
||||||
|
@ -14,7 +18,8 @@ _theme = {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def connect_plugin(category, label=None):
|
def connect_plugin(category, label=None): # pylint: disable=unused-argument
|
||||||
|
"""This function implements the entry point."""
|
||||||
try:
|
try:
|
||||||
_canberra.play(0, pycanberra.CA_PROP_EVENT_ID, _theme[category], None)
|
_canberra.play(0, pycanberra.CA_PROP_EVENT_ID, _theme[category], None)
|
||||||
except (KeyError, pycanberra.CanberraException):
|
except (KeyError, pycanberra.CanberraException):
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
src/plugins/sound_gstreamer.py
|
||||||
|
===================================
|
||||||
|
"""
|
||||||
import gi
|
import gi
|
||||||
gi.require_version('Gst', '1.0')
|
gi.require_version('Gst', '1.0')
|
||||||
from gi.repository import Gst # noqa: E402
|
from gi.repository import Gst # noqa: E402
|
||||||
|
@ -9,6 +12,7 @@ _player = Gst.ElementFactory.make("playbin", "player")
|
||||||
|
|
||||||
|
|
||||||
def connect_plugin(sound_file):
|
def connect_plugin(sound_file):
|
||||||
|
"""Entry point for sound file"""
|
||||||
_player.set_state(Gst.State.NULL)
|
_player.set_state(Gst.State.NULL)
|
||||||
_player.set_property("uri", "file://" + sound_file)
|
_player.set_property("uri", "file://" + sound_file)
|
||||||
_player.set_state(Gst.State.PLAYING)
|
_player.set_state(Gst.State.PLAYING)
|
||||||
|
|
|
@ -1,10 +1,14 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
src/plugins/sound_playfile.py
|
||||||
|
===================================
|
||||||
|
"""
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import winsound
|
import winsound
|
||||||
|
|
||||||
def connect_plugin(sound_file):
|
def connect_plugin(sound_file):
|
||||||
|
"""Plugin's entry point"""
|
||||||
winsound.PlaySound(sound_file, winsound.SND_FILENAME)
|
winsound.PlaySound(sound_file, winsound.SND_FILENAME)
|
||||||
except ImportError:
|
except ImportError:
|
||||||
import os
|
import os
|
||||||
|
@ -18,7 +22,8 @@ except ImportError:
|
||||||
args, stdout=FNULL, stderr=subprocess.STDOUT, close_fds=True)
|
args, stdout=FNULL, stderr=subprocess.STDOUT, close_fds=True)
|
||||||
|
|
||||||
def connect_plugin(sound_file):
|
def connect_plugin(sound_file):
|
||||||
global play_cmd
|
"""This function implements the entry point."""
|
||||||
|
global play_cmd # pylint: disable=global-statement
|
||||||
|
|
||||||
ext = os.path.splitext(sound_file)[-1]
|
ext = os.path.splitext(sound_file)[-1]
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -1,3 +1,7 @@
|
||||||
|
"""
|
||||||
|
src/pyelliptic/__init__.py
|
||||||
|
=====================================
|
||||||
|
"""
|
||||||
# Copyright (C) 2010
|
# Copyright (C) 2010
|
||||||
# Author: Yann GUIBET
|
# Author: Yann GUIBET
|
||||||
# Contact: <yannguibet@gmail.com>
|
# Contact: <yannguibet@gmail.com>
|
||||||
|
|
|
@ -1,5 +1,9 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
src/pyelliptic/cipher.py
|
||||||
|
========================
|
||||||
|
"""
|
||||||
|
|
||||||
# Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
|
# Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
|
||||||
# See LICENSE for details.
|
# See LICENSE for details.
|
||||||
|
@ -7,7 +11,8 @@
|
||||||
from openssl import OpenSSL
|
from openssl import OpenSSL
|
||||||
|
|
||||||
|
|
||||||
class Cipher:
|
# pylint: disable=redefined-builtin
|
||||||
|
class Cipher(object):
|
||||||
"""
|
"""
|
||||||
Symmetric encryption
|
Symmetric encryption
|
||||||
|
|
||||||
|
@ -44,30 +49,34 @@ class Cipher:
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_blocksize(ciphername):
|
def get_blocksize(ciphername):
|
||||||
|
"""This Method returns cipher blocksize"""
|
||||||
cipher = OpenSSL.get_cipher(ciphername)
|
cipher = OpenSSL.get_cipher(ciphername)
|
||||||
return cipher.get_blocksize()
|
return cipher.get_blocksize()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def gen_IV(ciphername):
|
def gen_IV(ciphername):
|
||||||
|
"""Generate random initialization vector"""
|
||||||
cipher = OpenSSL.get_cipher(ciphername)
|
cipher = OpenSSL.get_cipher(ciphername)
|
||||||
return OpenSSL.rand(cipher.get_blocksize())
|
return OpenSSL.rand(cipher.get_blocksize())
|
||||||
|
|
||||||
def update(self, input):
|
def update(self, input):
|
||||||
|
"""Update result with more data"""
|
||||||
i = OpenSSL.c_int(0)
|
i = OpenSSL.c_int(0)
|
||||||
buffer = OpenSSL.malloc(b"", len(input) + self.cipher.get_blocksize())
|
buffer = OpenSSL.malloc(b"", len(input) + self.cipher.get_blocksize())
|
||||||
inp = OpenSSL.malloc(input, len(input))
|
inp = OpenSSL.malloc(input, len(input))
|
||||||
if OpenSSL.EVP_CipherUpdate(self.ctx, OpenSSL.byref(buffer),
|
if OpenSSL.EVP_CipherUpdate(self.ctx, OpenSSL.byref(buffer),
|
||||||
OpenSSL.byref(i), inp, len(input)) == 0:
|
OpenSSL.byref(i), inp, len(input)) == 0:
|
||||||
raise Exception("[OpenSSL] EVP_CipherUpdate FAIL ...")
|
raise Exception("[OpenSSL] EVP_CipherUpdate FAIL ...")
|
||||||
return buffer.raw[0:i.value]
|
return buffer.raw[0:i.value] # pylint: disable=invalid-slice-index
|
||||||
|
|
||||||
def final(self):
|
def final(self):
|
||||||
|
"""Returning the final value"""
|
||||||
i = OpenSSL.c_int(0)
|
i = OpenSSL.c_int(0)
|
||||||
buffer = OpenSSL.malloc(b"", self.cipher.get_blocksize())
|
buffer = OpenSSL.malloc(b"", self.cipher.get_blocksize())
|
||||||
if (OpenSSL.EVP_CipherFinal_ex(self.ctx, OpenSSL.byref(buffer),
|
if (OpenSSL.EVP_CipherFinal_ex(self.ctx, OpenSSL.byref(buffer),
|
||||||
OpenSSL.byref(i))) == 0:
|
OpenSSL.byref(i))) == 0:
|
||||||
raise Exception("[OpenSSL] EVP_CipherFinal_ex FAIL ...")
|
raise Exception("[OpenSSL] EVP_CipherFinal_ex FAIL ...")
|
||||||
return buffer.raw[0:i.value]
|
return buffer.raw[0:i.value] # pylint: disable=invalid-slice-index
|
||||||
|
|
||||||
def ciphering(self, input):
|
def ciphering(self, input):
|
||||||
"""
|
"""
|
||||||
|
@ -77,6 +86,7 @@ class Cipher:
|
||||||
return buff + self.final()
|
return buff + self.final()
|
||||||
|
|
||||||
def __del__(self):
|
def __del__(self):
|
||||||
|
# pylint: disable=protected-access
|
||||||
if OpenSSL._hexversion > 0x10100000 and not OpenSSL._libreSSL:
|
if OpenSSL._hexversion > 0x10100000 and not OpenSSL._libreSSL:
|
||||||
OpenSSL.EVP_CIPHER_CTX_reset(self.ctx)
|
OpenSSL.EVP_CIPHER_CTX_reset(self.ctx)
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -1,6 +1,9 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
src/pyelliptic/hash.py
|
||||||
|
=====================
|
||||||
|
"""
|
||||||
# Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
|
# Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
|
||||||
# See LICENSE for details.
|
# See LICENSE for details.
|
||||||
|
|
||||||
|
@ -27,10 +30,10 @@ def _equals_str(a, b):
|
||||||
|
|
||||||
|
|
||||||
def equals(a, b):
|
def equals(a, b):
|
||||||
|
"""Compare two strings or bytearrays"""
|
||||||
if isinstance(a, str):
|
if isinstance(a, str):
|
||||||
return _equals_str(a, b)
|
return _equals_str(a, b)
|
||||||
else:
|
return _equals_bytes(a, b)
|
||||||
return _equals_bytes(a, b)
|
|
||||||
|
|
||||||
|
|
||||||
def hmac_sha256(k, m):
|
def hmac_sha256(k, m):
|
||||||
|
@ -58,6 +61,7 @@ def hmac_sha512(k, m):
|
||||||
|
|
||||||
|
|
||||||
def pbkdf2(password, salt=None, i=10000, keylen=64):
|
def pbkdf2(password, salt=None, i=10000, keylen=64):
|
||||||
|
"""Key derivation function using SHA256"""
|
||||||
if salt is None:
|
if salt is None:
|
||||||
salt = OpenSSL.rand(8)
|
salt = OpenSSL.rand(8)
|
||||||
p_password = OpenSSL.malloc(password, len(password))
|
p_password = OpenSSL.malloc(password, len(password))
|
||||||
|
|
|
@ -11,16 +11,23 @@ OpenSSL = None
|
||||||
|
|
||||||
# !/usr/bin/env python
|
# !/usr/bin/env python
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
src/pyelliptic/openssl.py
|
||||||
|
=====================
|
||||||
|
"""
|
||||||
# Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
|
# Copyright (C) 2011 Yann GUIBET <yannguibet@gmail.com>
|
||||||
# See LICENSE for details.
|
# See LICENSE for details.
|
||||||
#
|
#
|
||||||
# Software slightly changed by Jonathan Warren <bitmessage at-symbol jonwarren.org>
|
# Software slightly changed by Jonathan Warren <bitmessage at-symbol jonwarren.org>
|
||||||
|
# pylint: disable=protected-access
|
||||||
|
|
||||||
|
|
||||||
class CipherName: # pylint: disable=old-style-class
|
|
||||||
"""Method helps to get pointers, name and blocksize"""
|
|
||||||
|
|
||||||
|
|
||||||
|
class CipherName:
|
||||||
|
"""Class returns cipher name, pointer and blocksize"""
|
||||||
|
|
||||||
|
# pylint: disable=old-style-class
|
||||||
def __init__(self, name, pointer, blocksize):
|
def __init__(self, name, pointer, blocksize):
|
||||||
self._name = name
|
self._name = name
|
||||||
self._pointer = pointer
|
self._pointer = pointer
|
||||||
|
@ -36,11 +43,11 @@ class CipherName: # pylint: disable=old-style-class
|
||||||
return self._pointer()
|
return self._pointer()
|
||||||
|
|
||||||
def get_name(self):
|
def get_name(self):
|
||||||
"""Method returns the name"""
|
"""This method returns cipher name"""
|
||||||
return self._name
|
return self._name
|
||||||
|
|
||||||
def get_blocksize(self):
|
def get_blocksize(self):
|
||||||
"""Method returns the blocksize"""
|
"""This method returns cipher blocksize"""
|
||||||
return self._blocksize
|
return self._blocksize
|
||||||
|
|
||||||
|
|
||||||
|
@ -76,9 +83,11 @@ def get_version(library):
|
||||||
return (version, hexversion, cflags)
|
return (version, hexversion, cflags)
|
||||||
|
|
||||||
|
|
||||||
class _OpenSSL: # pylint: disable=too-many-instance-attributes, old-style-class, too-many-statements
|
class _OpenSSL:
|
||||||
"""Wrapper for OpenSSL using ctypes"""
|
"""
|
||||||
|
Wrapper for OpenSSL using ctypes
|
||||||
|
"""
|
||||||
|
# pylint: disable=too-many-statements, too-many-instance-attributes, old-style-class
|
||||||
def __init__(self, library):
|
def __init__(self, library):
|
||||||
"""Build the wrapper"""
|
"""Build the wrapper"""
|
||||||
self._lib = ctypes.CDLL(library)
|
self._lib = ctypes.CDLL(library)
|
||||||
|
@ -647,8 +656,7 @@ class _OpenSSL: # pylint: disable=too-many-instance-attributes, old-style-cl
|
||||||
|
|
||||||
def loadOpenSSL():
|
def loadOpenSSL():
|
||||||
"""Method find and load the OpenSSL library"""
|
"""Method find and load the OpenSSL library"""
|
||||||
# pylint: disable=global-statement, protected-access, too-many-branches, no-member
|
# pylint: disable=global-statement, protected-access, too-many-branches
|
||||||
|
|
||||||
global OpenSSL
|
global OpenSSL
|
||||||
from os import path, environ
|
from os import path, environ
|
||||||
from ctypes.util import find_library
|
from ctypes.util import find_library
|
||||||
|
|
|
@ -1,21 +1,26 @@
|
||||||
|
"""
|
||||||
|
src/storage/filesystem.py
|
||||||
|
=========================
|
||||||
|
"""
|
||||||
from binascii import hexlify, unhexlify
|
from binascii import hexlify, unhexlify
|
||||||
from os import listdir, makedirs, path, remove, rmdir
|
from os import listdir, makedirs, path, remove, rmdir
|
||||||
import string
|
import string
|
||||||
from threading import RLock
|
from threading import RLock
|
||||||
import time
|
import time
|
||||||
import traceback
|
|
||||||
|
|
||||||
from paths import lookupAppdataFolder
|
from paths import lookupAppdataFolder
|
||||||
from storage import InventoryStorage, InventoryItem
|
from storage import InventoryStorage, InventoryItem
|
||||||
|
|
||||||
class FilesystemInventory(InventoryStorage):
|
|
||||||
|
class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ancestors, abstract-method
|
||||||
|
"""Module for using filesystem (directory with files) for inventory storage"""
|
||||||
topDir = "inventory"
|
topDir = "inventory"
|
||||||
objectDir = "objects"
|
objectDir = "objects"
|
||||||
metadataFilename = "metadata"
|
metadataFilename = "metadata"
|
||||||
dataFilename = "data"
|
dataFilename = "data"
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super(self.__class__, self).__init__()
|
super(FilesystemInventory, self).__init__()
|
||||||
self.baseDir = path.join(lookupAppdataFolder(), FilesystemInventory.topDir)
|
self.baseDir = path.join(lookupAppdataFolder(), FilesystemInventory.topDir)
|
||||||
for createDir in [self.baseDir, path.join(self.baseDir, "objects")]:
|
for createDir in [self.baseDir, path.join(self.baseDir, "objects")]:
|
||||||
if path.exists(createDir):
|
if path.exists(createDir):
|
||||||
|
@ -23,72 +28,101 @@ class FilesystemInventory(InventoryStorage):
|
||||||
raise IOError("%s exists but it's not a directory" % (createDir))
|
raise IOError("%s exists but it's not a directory" % (createDir))
|
||||||
else:
|
else:
|
||||||
makedirs(createDir)
|
makedirs(createDir)
|
||||||
self.lock = RLock() # Guarantees that two receiveDataThreads don't receive and process the same message concurrently (probably sent by a malicious individual)
|
# Guarantees that two receiveDataThreads don't receive and process the same message
|
||||||
|
# concurrently (probably sent by a malicious individual)
|
||||||
|
self.lock = RLock()
|
||||||
self._inventory = {}
|
self._inventory = {}
|
||||||
self._load()
|
self._load()
|
||||||
|
|
||||||
def __contains__(self, hash):
|
def __contains__(self, hashval):
|
||||||
retval = False
|
retval = False
|
||||||
for streamDict in self._inventory.values():
|
for streamDict in self._inventory.values():
|
||||||
if hash in streamDict:
|
if hashval in streamDict:
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def __getitem__(self, hash):
|
def __getitem__(self, hashval):
|
||||||
for streamDict in self._inventory.values():
|
for streamDict in self._inventory.values():
|
||||||
try:
|
try:
|
||||||
retval = streamDict[hash]
|
retval = streamDict[hashval]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
continue
|
continue
|
||||||
if retval.payload is None:
|
if retval.payload is None:
|
||||||
retval = InventoryItem(retval.type, retval.stream, self.getData(hash), retval.expires, retval.tag)
|
retval = InventoryItem(retval.type, retval.stream, self.getData(hashval), retval.expires, retval.tag)
|
||||||
return retval
|
return retval
|
||||||
raise KeyError(hash)
|
raise KeyError(hashval)
|
||||||
|
|
||||||
def __setitem__(self, hash, value):
|
def __setitem__(self, hashval, value):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
value = InventoryItem(*value)
|
value = InventoryItem(*value)
|
||||||
try:
|
try:
|
||||||
makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash)))
|
makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval)))
|
||||||
except OSError:
|
except OSError:
|
||||||
pass
|
pass
|
||||||
try:
|
try:
|
||||||
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.metadataFilename), 'w') as f:
|
with open(
|
||||||
|
path.join(
|
||||||
|
self.baseDir,
|
||||||
|
FilesystemInventory.objectDir,
|
||||||
|
hexlify(hashval),
|
||||||
|
FilesystemInventory.metadataFilename,
|
||||||
|
),
|
||||||
|
"w",
|
||||||
|
) as f:
|
||||||
f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag)))
|
f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag)))
|
||||||
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.dataFilename), 'w') as f:
|
with open(
|
||||||
|
path.join(
|
||||||
|
self.baseDir,
|
||||||
|
FilesystemInventory.objectDir,
|
||||||
|
hexlify(hashval),
|
||||||
|
FilesystemInventory.dataFilename,
|
||||||
|
),
|
||||||
|
"w",
|
||||||
|
) as f:
|
||||||
f.write(value.payload)
|
f.write(value.payload)
|
||||||
except IOError:
|
except IOError:
|
||||||
raise KeyError
|
raise KeyError
|
||||||
try:
|
try:
|
||||||
self._inventory[value.stream][hash] = value
|
self._inventory[value.stream][hashval] = value
|
||||||
except KeyError:
|
except KeyError:
|
||||||
self._inventory[value.stream] = {}
|
self._inventory[value.stream] = {}
|
||||||
self._inventory[value.stream][hash] = value
|
self._inventory[value.stream][hashval] = value
|
||||||
|
|
||||||
def delHashId(self, hash):
|
def delHashId(self, hashval):
|
||||||
for stream in self._inventory.keys():
|
"""Remove object from inventory"""
|
||||||
|
for stream in self._inventory:
|
||||||
try:
|
try:
|
||||||
del self._inventory[stream][hash]
|
del self._inventory[stream][hashval]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
pass
|
||||||
with self.lock:
|
with self.lock:
|
||||||
try:
|
try:
|
||||||
remove(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.metadataFilename))
|
remove(
|
||||||
|
path.join(
|
||||||
|
self.baseDir,
|
||||||
|
FilesystemInventory.objectDir,
|
||||||
|
hexlify(hashval),
|
||||||
|
FilesystemInventory.metadataFilename))
|
||||||
except IOError:
|
except IOError:
|
||||||
pass
|
pass
|
||||||
try:
|
try:
|
||||||
remove(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.dataFilename))
|
remove(
|
||||||
|
path.join(
|
||||||
|
self.baseDir,
|
||||||
|
FilesystemInventory.objectDir,
|
||||||
|
hexlify(hashval),
|
||||||
|
FilesystemInventory.dataFilename))
|
||||||
except IOError:
|
except IOError:
|
||||||
pass
|
pass
|
||||||
try:
|
try:
|
||||||
rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash)))
|
rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval)))
|
||||||
except IOError:
|
except IOError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def __iter__(self):
|
def __iter__(self):
|
||||||
elems = []
|
elems = []
|
||||||
for streamDict in self._inventory.values():
|
for streamDict in self._inventory.values():
|
||||||
elems.extend (streamDict.keys())
|
elems.extend(streamDict.keys())
|
||||||
return elems.__iter__()
|
return elems.__iter__()
|
||||||
|
|
||||||
def __len__(self):
|
def __len__(self):
|
||||||
|
@ -103,39 +137,61 @@ class FilesystemInventory(InventoryStorage):
|
||||||
try:
|
try:
|
||||||
objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId)
|
objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId)
|
||||||
try:
|
try:
|
||||||
newInventory[streamNumber][hashId] = InventoryItem(objectType, streamNumber, None, expiresTime, tag)
|
newInventory[streamNumber][hashId] = InventoryItem(
|
||||||
|
objectType, streamNumber, None, expiresTime, tag)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
newInventory[streamNumber] = {}
|
newInventory[streamNumber] = {}
|
||||||
newInventory[streamNumber][hashId] = InventoryItem(objectType, streamNumber, None, expiresTime, tag)
|
newInventory[streamNumber][hashId] = InventoryItem(
|
||||||
|
objectType, streamNumber, None, expiresTime, tag)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
print "error loading %s" % (hexlify(hashId))
|
print "error loading %s" % (hexlify(hashId))
|
||||||
pass
|
|
||||||
self._inventory = newInventory
|
self._inventory = newInventory
|
||||||
# for i, v in self._inventory.items():
|
# for i, v in self._inventory.items():
|
||||||
# print "loaded stream: %s, %i items" % (i, len(v))
|
# print "loaded stream: %s, %i items" % (i, len(v))
|
||||||
|
|
||||||
def stream_list(self):
|
def stream_list(self):
|
||||||
|
"""Return list of streams"""
|
||||||
return self._inventory.keys()
|
return self._inventory.keys()
|
||||||
|
|
||||||
def object_list(self):
|
def object_list(self):
|
||||||
|
"""Return inventory vectors (hashes) from a directory"""
|
||||||
return [unhexlify(x) for x in listdir(path.join(self.baseDir, FilesystemInventory.objectDir))]
|
return [unhexlify(x) for x in listdir(path.join(self.baseDir, FilesystemInventory.objectDir))]
|
||||||
|
|
||||||
def getData(self, hashId):
|
def getData(self, hashId):
|
||||||
|
"""Get object data"""
|
||||||
try:
|
try:
|
||||||
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashId), FilesystemInventory.dataFilename), 'r') as f:
|
with open(
|
||||||
|
path.join(
|
||||||
|
self.baseDir,
|
||||||
|
FilesystemInventory.objectDir,
|
||||||
|
hexlify(hashId),
|
||||||
|
FilesystemInventory.dataFilename,
|
||||||
|
),
|
||||||
|
"r",
|
||||||
|
) as f:
|
||||||
return f.read()
|
return f.read()
|
||||||
except IOError:
|
except IOError:
|
||||||
raise AttributeError
|
raise AttributeError
|
||||||
|
|
||||||
def getMetadata(self, hashId):
|
def getMetadata(self, hashId):
|
||||||
|
"""Get object metadata"""
|
||||||
try:
|
try:
|
||||||
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashId), FilesystemInventory.metadataFilename), 'r') as f:
|
with open(
|
||||||
|
path.join(
|
||||||
|
self.baseDir,
|
||||||
|
FilesystemInventory.objectDir,
|
||||||
|
hexlify(hashId),
|
||||||
|
FilesystemInventory.metadataFilename,
|
||||||
|
),
|
||||||
|
"r",
|
||||||
|
) as f:
|
||||||
objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4)
|
objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4)
|
||||||
return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)]
|
return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)]
|
||||||
except IOError:
|
except IOError:
|
||||||
raise KeyError
|
raise KeyError
|
||||||
|
|
||||||
def by_type_and_tag(self, objectType, tag):
|
def by_type_and_tag(self, objectType, tag):
|
||||||
|
"""Get a list of objects filtered by object type and tag"""
|
||||||
retval = []
|
retval = []
|
||||||
for stream, streamDict in self._inventory:
|
for stream, streamDict in self._inventory:
|
||||||
for hashId, item in streamDict:
|
for hashId, item in streamDict:
|
||||||
|
@ -149,12 +205,14 @@ class FilesystemInventory(InventoryStorage):
|
||||||
return retval
|
return retval
|
||||||
|
|
||||||
def hashes_by_stream(self, stream):
|
def hashes_by_stream(self, stream):
|
||||||
|
"""Return inventory vectors (hashes) for a stream"""
|
||||||
try:
|
try:
|
||||||
return self._inventory[stream].keys()
|
return self._inventory[stream].keys()
|
||||||
except KeyError:
|
except KeyError:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def unexpired_hashes_by_stream(self, stream):
|
def unexpired_hashes_by_stream(self, stream):
|
||||||
|
"""Return unexpired hashes in the inventory for a particular stream"""
|
||||||
t = int(time.time())
|
t = int(time.time())
|
||||||
try:
|
try:
|
||||||
return [x for x, value in self._inventory[stream].items() if value.expires > t]
|
return [x for x, value in self._inventory[stream].items() if value.expires > t]
|
||||||
|
@ -162,9 +220,11 @@ class FilesystemInventory(InventoryStorage):
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def flush(self):
|
def flush(self):
|
||||||
|
"""Flush the inventory and create a new, empty one"""
|
||||||
self._load()
|
self._load()
|
||||||
|
|
||||||
def clean(self):
|
def clean(self):
|
||||||
|
"""Clean out old items from the inventory"""
|
||||||
minTime = int(time.time()) - (60 * 60 * 30)
|
minTime = int(time.time()) - (60 * 60 * 30)
|
||||||
deletes = []
|
deletes = []
|
||||||
for stream, streamDict in self._inventory.items():
|
for stream, streamDict in self._inventory.items():
|
||||||
|
|
|
@ -1,45 +1,59 @@
|
||||||
import collections
|
"""
|
||||||
from threading import current_thread, enumerate as threadingEnumerate, RLock
|
src/storage/sqlite.py
|
||||||
import Queue
|
=========================
|
||||||
|
"""
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import time
|
import time
|
||||||
|
from threading import RLock
|
||||||
|
|
||||||
from helper_sql import *
|
from helper_sql import sqlQuery, SqlBulkExecute, sqlExecute
|
||||||
from storage import InventoryStorage, InventoryItem
|
from storage import InventoryStorage, InventoryItem
|
||||||
|
|
||||||
class SqliteInventory(InventoryStorage):
|
|
||||||
def __init__(self):
|
|
||||||
super(self.__class__, self).__init__()
|
|
||||||
self._inventory = {} #of objects (like msg payloads and pubkey payloads) Does not include protocol headers (the first 24 bytes of each packet).
|
|
||||||
self._objects = {} # cache for existing objects, used for quick lookups if we have an object. This is used for example whenever we receive an inv message from a peer to check to see what items are new to us. We don't delete things out of it; instead, the singleCleaner thread clears and refills it.
|
|
||||||
self.lock = RLock() # Guarantees that two receiveDataThreads don't receive and process the same message concurrently (probably sent by a malicious individual)
|
|
||||||
|
|
||||||
def __contains__(self, hash):
|
class SqliteInventory(InventoryStorage): # pylint: disable=too-many-ancestors
|
||||||
|
"""Inventory using SQLite"""
|
||||||
|
def __init__(self):
|
||||||
|
super(SqliteInventory, self).__init__()
|
||||||
|
# of objects (like msg payloads and pubkey payloads)
|
||||||
|
# Does not include protocol headers (the first 24 bytes of each packet).
|
||||||
|
self._inventory = {}
|
||||||
|
# cache for existing objects, used for quick lookups if we have an object.
|
||||||
|
# This is used for example whenever we receive an inv message from a peer
|
||||||
|
# to check to see what items are new to us.
|
||||||
|
# We don't delete things out of it; instead, the singleCleaner thread clears and refills it.
|
||||||
|
self._objects = {}
|
||||||
|
# Guarantees that two receiveDataThreads don't receive and process the same message concurrently
|
||||||
|
# (probably sent by a malicious individual)
|
||||||
|
self.lock = RLock()
|
||||||
|
|
||||||
|
def __contains__(self, hash_):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
if hash in self._objects:
|
if hash_ in self._objects:
|
||||||
return True
|
return True
|
||||||
rows = sqlQuery('SELECT streamnumber FROM inventory WHERE hash=?', sqlite3.Binary(hash))
|
rows = sqlQuery('SELECT streamnumber FROM inventory WHERE hash=?', sqlite3.Binary(hash_))
|
||||||
if not rows:
|
if not rows:
|
||||||
return False
|
return False
|
||||||
self._objects[hash] = rows[0][0]
|
self._objects[hash_] = rows[0][0]
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def __getitem__(self, hash):
|
def __getitem__(self, hash_):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
if hash in self._inventory:
|
if hash_ in self._inventory:
|
||||||
return self._inventory[hash]
|
return self._inventory[hash_]
|
||||||
rows = sqlQuery('SELECT objecttype, streamnumber, payload, expirestime, tag FROM inventory WHERE hash=?', sqlite3.Binary(hash))
|
rows = sqlQuery(
|
||||||
|
'SELECT objecttype, streamnumber, payload, expirestime, tag FROM inventory WHERE hash=?',
|
||||||
|
sqlite3.Binary(hash_))
|
||||||
if not rows:
|
if not rows:
|
||||||
raise KeyError(hash)
|
raise KeyError(hash_)
|
||||||
return InventoryItem(*rows[0])
|
return InventoryItem(*rows[0])
|
||||||
|
|
||||||
def __setitem__(self, hash, value):
|
def __setitem__(self, hash_, value):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
value = InventoryItem(*value)
|
value = InventoryItem(*value)
|
||||||
self._inventory[hash] = value
|
self._inventory[hash_] = value
|
||||||
self._objects[hash] = value.stream
|
self._objects[hash_] = value.stream
|
||||||
|
|
||||||
def __delitem__(self, hash):
|
def __delitem__(self, hash_):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def __iter__(self):
|
def __iter__(self):
|
||||||
|
@ -55,18 +69,22 @@ class SqliteInventory(InventoryStorage):
|
||||||
def by_type_and_tag(self, objectType, tag):
|
def by_type_and_tag(self, objectType, tag):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
values = [value for value in self._inventory.values() if value.type == objectType and value.tag == tag]
|
values = [value for value in self._inventory.values() if value.type == objectType and value.tag == tag]
|
||||||
values += (InventoryItem(*value) for value in sqlQuery('SELECT objecttype, streamnumber, payload, expirestime, tag FROM inventory WHERE objecttype=? AND tag=?', objectType, sqlite3.Binary(tag)))
|
values += (InventoryItem(*value) for value in sqlQuery(
|
||||||
|
'SELECT objecttype, streamnumber, payload, expirestime, tag \
|
||||||
|
FROM inventory WHERE objecttype=? AND tag=?', objectType, sqlite3.Binary(tag)))
|
||||||
return values
|
return values
|
||||||
|
|
||||||
def unexpired_hashes_by_stream(self, stream):
|
def unexpired_hashes_by_stream(self, stream):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
t = int(time.time())
|
t = int(time.time())
|
||||||
hashes = [x for x, value in self._inventory.items() if value.stream == stream and value.expires > t]
|
hashes = [x for x, value in self._inventory.items() if value.stream == stream and value.expires > t]
|
||||||
hashes += (str(payload) for payload, in sqlQuery('SELECT hash FROM inventory WHERE streamnumber=? AND expirestime>?', stream, t))
|
hashes += (str(payload) for payload, in sqlQuery(
|
||||||
|
'SELECT hash FROM inventory WHERE streamnumber=? AND expirestime>?', stream, t))
|
||||||
return hashes
|
return hashes
|
||||||
|
|
||||||
def flush(self):
|
def flush(self):
|
||||||
with self.lock: # If you use both the inventoryLock and the sqlLock, always use the inventoryLock OUTSIDE of the sqlLock.
|
with self.lock:
|
||||||
|
# If you use both the inventoryLock and the sqlLock, always use the inventoryLock OUTSIDE of the sqlLock.
|
||||||
with SqlBulkExecute() as sql:
|
with SqlBulkExecute() as sql:
|
||||||
for objectHash, value in self._inventory.items():
|
for objectHash, value in self._inventory.items():
|
||||||
sql.execute('INSERT INTO inventory VALUES (?, ?, ?, ?, ?, ?)', sqlite3.Binary(objectHash), *value)
|
sql.execute('INSERT INTO inventory VALUES (?, ?, ?, ?, ?, ?)', sqlite3.Binary(objectHash), *value)
|
||||||
|
@ -74,8 +92,7 @@ class SqliteInventory(InventoryStorage):
|
||||||
|
|
||||||
def clean(self):
|
def clean(self):
|
||||||
with self.lock:
|
with self.lock:
|
||||||
sqlExecute('DELETE FROM inventory WHERE expirestime<?',int(time.time()) - (60 * 60 * 3))
|
sqlExecute('DELETE FROM inventory WHERE expirestime<?', int(time.time()) - (60 * 60 * 3))
|
||||||
self._objects.clear()
|
self._objects.clear()
|
||||||
for objectHash, value in self._inventory.items():
|
for objectHash, value in self._inventory.items():
|
||||||
self._objects[objectHash] = value.stream
|
self._objects[objectHash] = value.stream
|
||||||
|
|
||||||
|
|
|
@ -1,27 +1,33 @@
|
||||||
|
"""
|
||||||
|
src/storage/storage.py
|
||||||
|
======================
|
||||||
|
"""
|
||||||
import collections
|
import collections
|
||||||
|
|
||||||
InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload expires tag')
|
InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload expires tag')
|
||||||
|
|
||||||
|
|
||||||
class Storage(object):
|
class Storage(object):
|
||||||
|
"""Base class for storing inventory (extendable for other items to store)"""
|
||||||
pass
|
pass
|
||||||
# def __init__(self):
|
|
||||||
# super(self.__class__, self).__init__()
|
|
||||||
|
|
||||||
class InventoryStorage(Storage, collections.MutableMapping):
|
class InventoryStorage(Storage, collections.MutableMapping):
|
||||||
|
"""Module used for inventory storage"""
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
# super(self.__class__, self).__init__()
|
# pylint: disable=super-init-not-called
|
||||||
self.numberOfInventoryLookupsPerformed = 0
|
self.numberOfInventoryLookupsPerformed = 0
|
||||||
|
|
||||||
def __contains__(self, hash):
|
def __contains__(self, _):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def __getitem__(self, hash):
|
def __getitem__(self, _):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def __setitem__(self, hash, value):
|
def __setitem__(self, _, value):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def __delitem__(self, hash):
|
def __delitem__(self, _):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def __iter__(self):
|
def __iter__(self):
|
||||||
|
@ -31,18 +37,24 @@ class InventoryStorage(Storage, collections.MutableMapping):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def by_type_and_tag(self, objectType, tag):
|
def by_type_and_tag(self, objectType, tag):
|
||||||
|
"""Return objects filtered by object type and tag"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def unexpired_hashes_by_stream(self, stream):
|
def unexpired_hashes_by_stream(self, stream):
|
||||||
|
"""Return unexpired inventory vectors filtered by stream"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def flush(self):
|
def flush(self):
|
||||||
|
"""Flush cache"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def clean(self):
|
def clean(self):
|
||||||
|
"""Free memory / perform garbage collection"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
class MailboxStorage(Storage, collections.MutableMapping):
|
|
||||||
|
class MailboxStorage(Storage, collections.MutableMapping): # pylint: disable=abstract-method
|
||||||
|
"""Method for storing mails"""
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
# super(self.__class__, self).__init__()
|
# pylint: disable=super-init-not-called
|
||||||
pass
|
pass
|
||||||
|
|
Reference in New Issue
Block a user