From fa65b17fc98a5689197beae21f3aa8be4be7f31f Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Mon, 23 Sep 2019 14:51:54 +0530 Subject: [PATCH 01/18] __init__ pylint fixes --- src/pyelliptic/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pyelliptic/__init__.py b/src/pyelliptic/__init__.py index 1d6a928f..7aa666e0 100644 --- a/src/pyelliptic/__init__.py +++ b/src/pyelliptic/__init__.py @@ -1,3 +1,7 @@ +""" +src/pyelliptic/__init__.py +===================================== +""" # Copyright (C) 2010 # Author: Yann GUIBET # Contact: From e0d81bb7e8e23c4d8612b0f8a382df3eddfd87f8 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Mon, 23 Sep 2019 17:51:58 +0530 Subject: [PATCH 02/18] cipher pylint fixes --- src/pyelliptic/cipher.py | 16 +++++++++++++--- src/pyelliptic/hash.py | 2 ++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/pyelliptic/cipher.py b/src/pyelliptic/cipher.py index bc1af6b0..d02b743a 100644 --- a/src/pyelliptic/cipher.py +++ b/src/pyelliptic/cipher.py @@ -1,5 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +""" +src/pyelliptic/cipher.py +======================== +""" # Copyright (C) 2011 Yann GUIBET # See LICENSE for details. @@ -7,7 +11,8 @@ from openssl import OpenSSL -class Cipher: +# pylint: disable=redefined-builtin +class Cipher(object): """ Symmetric encryption @@ -44,30 +49,34 @@ class Cipher: @staticmethod def get_blocksize(ciphername): + """This Method returns cipher blocksize""" cipher = OpenSSL.get_cipher(ciphername) return cipher.get_blocksize() @staticmethod def gen_IV(ciphername): + """Generate random initialization vector""" cipher = OpenSSL.get_cipher(ciphername) return OpenSSL.rand(cipher.get_blocksize()) def update(self, input): + """Update result with more data""" i = OpenSSL.c_int(0) buffer = OpenSSL.malloc(b"", len(input) + self.cipher.get_blocksize()) inp = OpenSSL.malloc(input, len(input)) if OpenSSL.EVP_CipherUpdate(self.ctx, OpenSSL.byref(buffer), OpenSSL.byref(i), inp, len(input)) == 0: raise Exception("[OpenSSL] EVP_CipherUpdate FAIL ...") - return buffer.raw[0:i.value] + return buffer.raw[0:i.value] # pylint: disable=invalid-slice-index def final(self): + """Returning the final value""" i = OpenSSL.c_int(0) buffer = OpenSSL.malloc(b"", self.cipher.get_blocksize()) if (OpenSSL.EVP_CipherFinal_ex(self.ctx, OpenSSL.byref(buffer), OpenSSL.byref(i))) == 0: raise Exception("[OpenSSL] EVP_CipherFinal_ex FAIL ...") - return buffer.raw[0:i.value] + return buffer.raw[0:i.value] # pylint: disable=invalid-slice-index def ciphering(self, input): """ @@ -77,6 +86,7 @@ class Cipher: return buff + self.final() def __del__(self): + # pylint: disable=protected-access if OpenSSL._hexversion > 0x10100000 and not OpenSSL._libreSSL: OpenSSL.EVP_CIPHER_CTX_reset(self.ctx) else: diff --git a/src/pyelliptic/hash.py b/src/pyelliptic/hash.py index f2240500..2d7064f8 100644 --- a/src/pyelliptic/hash.py +++ b/src/pyelliptic/hash.py @@ -27,6 +27,7 @@ def _equals_str(a, b): def equals(a, b): + """Compare two strings or bytearrays""" if isinstance(a, str): return _equals_str(a, b) else: @@ -58,6 +59,7 @@ def hmac_sha512(k, m): def pbkdf2(password, salt=None, i=10000, keylen=64): + """Compute the salt, key and the message with pbkdf2""" if salt is None: salt = OpenSSL.rand(8) p_password = OpenSSL.malloc(password, len(password)) From 4448e6ee7ba040c2582618a291e9cb28af57e511 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Mon, 23 Sep 2019 17:52:56 +0530 Subject: [PATCH 03/18] hash pylint fixes --- src/pyelliptic/hash.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/pyelliptic/hash.py b/src/pyelliptic/hash.py index 2d7064f8..c21dd6a4 100644 --- a/src/pyelliptic/hash.py +++ b/src/pyelliptic/hash.py @@ -1,6 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - +""" +src/pyelliptic/hash.py +===================== +""" # Copyright (C) 2011 Yann GUIBET # See LICENSE for details. @@ -30,8 +33,7 @@ def equals(a, b): """Compare two strings or bytearrays""" if isinstance(a, str): return _equals_str(a, b) - else: - return _equals_bytes(a, b) + return _equals_bytes(a, b) def hmac_sha256(k, m): @@ -59,7 +61,7 @@ def hmac_sha512(k, m): def pbkdf2(password, salt=None, i=10000, keylen=64): - """Compute the salt, key and the message with pbkdf2""" + """Key derivation function using SHA256""" if salt is None: salt = OpenSSL.rand(8) p_password = OpenSSL.malloc(password, len(password)) From a6f951d37fe05a3b6518585cfdd6b11be6bc5f68 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Tue, 24 Sep 2019 13:05:42 +0530 Subject: [PATCH 04/18] openssl pylint fixes --- src/pyelliptic/openssl.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/pyelliptic/openssl.py b/src/pyelliptic/openssl.py index ab2990e6..fcde01ec 100644 --- a/src/pyelliptic/openssl.py +++ b/src/pyelliptic/openssl.py @@ -1,10 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - +""" +src/pyelliptic/openssl.py +===================== +""" # Copyright (C) 2011 Yann GUIBET # See LICENSE for details. # # Software slightly changed by Jonathan Warren +# pylint: disable=protected-access import sys import ctypes @@ -13,6 +17,9 @@ OpenSSL = None class CipherName: + """Class returns cipher name, pointer and blocksize""" + + # pylint: disable=old-style-class def __init__(self, name, pointer, blocksize): self._name = name self._pointer = pointer @@ -24,16 +31,20 @@ class CipherName: " | Function pointer : " + str(self._pointer) def get_pointer(self): + """This method returns cipher pointer""" return self._pointer() def get_name(self): + """This method returns cipher name""" return self._name def get_blocksize(self): + """This method returns cipher blocksize""" return self._blocksize def get_version(library): + """This function return version, hexversion and cflages""" version = None hexversion = None cflags = None @@ -68,6 +79,7 @@ class _OpenSSL: """ Wrapper for OpenSSL using ctypes """ + # pylint: disable=too-many-statements, too-many-instance-attributes, old-style-class def __init__(self, library): """ Build the wrapper @@ -594,6 +606,7 @@ class _OpenSSL: """ returns the name of a elliptic curve with his id """ + # pylint: disable=redefined-builtin res = None for i in self.curves: if self.curves[i] == id: @@ -607,6 +620,7 @@ class _OpenSSL: """ OpenSSL random function """ + # pylint: disable=redefined-builtin buffer = self.malloc(0, size) # This pyelliptic library, by default, didn't check the return value of RAND_bytes. It is # evidently possible that it returned an error and not-actually-random data. However, in @@ -623,6 +637,7 @@ class _OpenSSL: """ returns a create_string_buffer (ctypes) """ + # pylint: disable=redefined-builtin buffer = None if data != 0: if sys.version_info.major == 3 and isinstance(data, type('')): @@ -634,6 +649,8 @@ class _OpenSSL: def loadOpenSSL(): + """This function finds and load the OpenSSL library""" + # pylint: disable=global-statement global OpenSSL from os import path, environ from ctypes.util import find_library From 36775ae88b87cb6a126f9f0091b1a8dd41e8e0be Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Mon, 23 Sep 2019 16:07:04 +0530 Subject: [PATCH 05/18] indicator_libmessaging pylint fixes --- src/plugins/indicator_libmessaging.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/plugins/indicator_libmessaging.py b/src/plugins/indicator_libmessaging.py index 36178663..ab2e833e 100644 --- a/src/plugins/indicator_libmessaging.py +++ b/src/plugins/indicator_libmessaging.py @@ -1,4 +1,8 @@ # -*- coding: utf-8 -*- +""" +src/plugins/indicator_libmessaging.py +===================================== +""" import gi gi.require_version('MessagingMenu', '1.0') # noqa:E402 @@ -9,6 +13,7 @@ from pybitmessage.tr import _translate class IndicatorLibmessaging(object): + """Plugin for libmessage indicator""" def __init__(self, form): try: self.app = MessagingMenu.App(desktop_id='pybitmessage.desktop') @@ -32,15 +37,18 @@ class IndicatorLibmessaging(object): if self.app: self.app.unregister() - def activate(self, app, source): + def activate(self, app, source): # pylint: disable=unused-argument + """Activate the libmessaging indicator plugin""" self.form.appIndicatorInbox( self.new_message_item if source == 'messages' else self.new_broadcast_item ) - # show the number of unread messages and subscriptions - # on the messaging menu def show_unread(self, draw_attention=False): + """ + show the number of unread messages and subscriptions + on the messaging menu + """ for source, count in zip( ('messages', 'subscriptions'), self.form.getUnread() From 4c1568a3eb4c6748c2e26c56d50a60e1c17271b7 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Mon, 23 Sep 2019 16:07:23 +0530 Subject: [PATCH 06/18] menu_qrcode pylint fixes --- src/plugins/menu_qrcode.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/plugins/menu_qrcode.py b/src/plugins/menu_qrcode.py index 7f21c6b8..2e65bd88 100644 --- a/src/plugins/menu_qrcode.py +++ b/src/plugins/menu_qrcode.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- """ +src/plugins/menu_qrcode.py +========================== + A menu plugin showing QR-Code for bitmessage address in modal dialog. """ @@ -12,9 +15,10 @@ from pybitmessage.tr import _translate # http://stackoverflow.com/questions/20452486 -class Image(qrcode.image.base.BaseImage): +class Image(qrcode.image.base.BaseImage): # pylint: disable=abstract-method """Image output class for qrcode using QPainter""" - def __init__(self, border, width, box_size): + + def __init__(self, border, width, box_size): # pylint: disable=super-init-not-called self.border = border self.width = width self.box_size = box_size From 7aa9b94c111106ea739ea3890b351a5c953d6a43 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Mon, 23 Sep 2019 16:08:00 +0530 Subject: [PATCH 07/18] notification_notify2 pylint fixes --- src/plugins/notification_notify2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/plugins/notification_notify2.py b/src/plugins/notification_notify2.py index 3fd935c4..b4cd045d 100644 --- a/src/plugins/notification_notify2.py +++ b/src/plugins/notification_notify2.py @@ -1,4 +1,8 @@ # -*- coding: utf-8 -*- +""" +src/plugins/notification_notify2.py +=================================== +""" import gi gi.require_version('Notify', '0.7') @@ -6,10 +10,13 @@ from gi.repository import Notify Notify.init('pybitmessage') + def connect_plugin(title, subtitle, category, label, icon): + """Plugin for notify2""" if not icon: icon = 'mail-message-new' if category == 2 else 'pybitmessage' connect_plugin.notification.update(title, subtitle, icon) connect_plugin.notification.show() + connect_plugin.notification = Notify.Notification.new("Init", "Init") From e50f99419fba105a56b100c89958b281ff21cc14 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Mon, 23 Sep 2019 16:08:18 +0530 Subject: [PATCH 08/18] plugin pylint fixes --- src/plugins/plugin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plugins/plugin.py b/src/plugins/plugin.py index 6601adaf..e671a73f 100644 --- a/src/plugins/plugin.py +++ b/src/plugins/plugin.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- - +""" +src/plugins/plugin.py +=================================== +""" import pkg_resources From df1994d6f3f02b833bbd690102ad8dfbbe3aa9c0 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Mon, 23 Sep 2019 16:08:34 +0530 Subject: [PATCH 09/18] proxyconfig_stem pylint fixes --- src/plugins/proxyconfig_stem.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/plugins/proxyconfig_stem.py b/src/plugins/proxyconfig_stem.py index e8b5417e..5bb9a726 100644 --- a/src/plugins/proxyconfig_stem.py +++ b/src/plugins/proxyconfig_stem.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- - +""" +src/plugins/proxyconfig_stem.py +=================================== +""" import os import logging import random # noseq @@ -29,14 +32,14 @@ class DebugLogger(object): # Plugin's debug or unexpected log line from tor self._logger.debug(line) else: - self._logger.log(self._levels.get(level, 10), '(tor)' + line) + self._logger.log(self._levels.get(level, 10), '(tor) %s', line) def connect_plugin(config): # pylint: disable=too-many-branches """Run stem proxy configurator""" logwrite = DebugLogger() if config.safeGet('bitmessagesettings', 'sockshostname') not in ( - 'localhost', '127.0.0.1', '' + 'localhost', '127.0.0.1', '' ): # remote proxy is choosen for outbound connections, # nothing to do here, but need to set socksproxytype to SOCKS5! From a86c5188c44411d173bd2a15b24b7801b8bddf8b Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Mon, 23 Sep 2019 16:08:56 +0530 Subject: [PATCH 10/18] sound_canberra pylint fixes --- src/plugins/sound_canberra.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/plugins/sound_canberra.py b/src/plugins/sound_canberra.py index 094901ed..dbb4baed 100644 --- a/src/plugins/sound_canberra.py +++ b/src/plugins/sound_canberra.py @@ -1,4 +1,8 @@ # -*- coding: utf-8 -*- +""" +src/plugins/proxyconfig_stem.py +=================================== +""" from pybitmessage.bitmessageqt import sound @@ -14,7 +18,8 @@ _theme = { } -def connect_plugin(category, label=None): +def connect_plugin(category, label=None): # pylint: disable=unused-argument + """This function implements the entry point.""" try: _canberra.play(0, pycanberra.CA_PROP_EVENT_ID, _theme[category], None) except (KeyError, pycanberra.CanberraException): From a86e43c108c4117472868f2bcecdb0f4d20df31e Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Mon, 23 Sep 2019 16:09:19 +0530 Subject: [PATCH 11/18] sound_gstreamer pylint fixes --- src/plugins/sound_gstreamer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plugins/sound_gstreamer.py b/src/plugins/sound_gstreamer.py index 062da3f9..32a0aa65 100644 --- a/src/plugins/sound_gstreamer.py +++ b/src/plugins/sound_gstreamer.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- - +""" +src/plugins/sound_gstreamer.py +=================================== +""" import gi gi.require_version('Gst', '1.0') from gi.repository import Gst # noqa: E402 @@ -9,6 +12,7 @@ _player = Gst.ElementFactory.make("playbin", "player") def connect_plugin(sound_file): + """Entry point for sound file""" _player.set_state(Gst.State.NULL) _player.set_property("uri", "file://" + sound_file) _player.set_state(Gst.State.PLAYING) From 433cb9818b045843128a9be52f1bff2be6c06239 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Mon, 23 Sep 2019 16:09:35 +0530 Subject: [PATCH 12/18] sound_playfile pylint fixes --- src/plugins/sound_playfile.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/plugins/sound_playfile.py b/src/plugins/sound_playfile.py index c8216d07..6396c319 100644 --- a/src/plugins/sound_playfile.py +++ b/src/plugins/sound_playfile.py @@ -1,10 +1,14 @@ # -*- coding: utf-8 -*- - +""" +src/plugins/sound_playfile.py +=================================== +""" try: import winsound def connect_plugin(sound_file): + """Plugin's entry point""" winsound.PlaySound(sound_file, winsound.SND_FILENAME) except ImportError: import os @@ -18,7 +22,8 @@ except ImportError: args, stdout=FNULL, stderr=subprocess.STDOUT, close_fds=True) def connect_plugin(sound_file): - global play_cmd + """This function implements the entry point.""" + global play_cmd # pylint: disable=global-statement ext = os.path.splitext(sound_file)[-1] try: From 6f910f67c0ebb45f6b7455922f33c710a515ae7b Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Tue, 24 Sep 2019 14:50:20 +0530 Subject: [PATCH 13/18] filesystem flake8 fixes --- src/storage/filesystem.py | 71 ++++++++++++++++++++++++++++++++------- src/storage/sqlite.py | 14 ++++++-- src/storage/storage.py | 5 +++ 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/src/storage/filesystem.py b/src/storage/filesystem.py index d64894a9..4f7e717d 100644 --- a/src/storage/filesystem.py +++ b/src/storage/filesystem.py @@ -3,11 +3,11 @@ from os import listdir, makedirs, path, remove, rmdir import string from threading import RLock import time -import traceback from paths import lookupAppdataFolder from storage import InventoryStorage, InventoryItem + class FilesystemInventory(InventoryStorage): topDir = "inventory" objectDir = "objects" @@ -23,7 +23,9 @@ class FilesystemInventory(InventoryStorage): raise IOError("%s exists but it's not a directory" % (createDir)) else: makedirs(createDir) - self.lock = RLock() # Guarantees that two receiveDataThreads don't receive and process the same message concurrently (probably sent by a malicious individual) + # Guarantees that two receiveDataThreads don't receive and process the same message + # concurrently (probably sent by a malicious individual) + self.lock = RLock() self._inventory = {} self._load() @@ -53,9 +55,25 @@ class FilesystemInventory(InventoryStorage): except OSError: pass try: - with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.metadataFilename), 'w') as f: + with open( + path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hash), + FilesystemInventory.metadataFilename, + ), + "w", + ) as f: f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag))) - with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.dataFilename), 'w') as f: + with open( + path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hash), + FilesystemInventory.dataFilename, + ), + "w", + ) as f: f.write(value.payload) except IOError: raise KeyError @@ -73,11 +91,21 @@ class FilesystemInventory(InventoryStorage): pass with self.lock: try: - remove(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.metadataFilename)) + remove( + path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hash), + FilesystemInventory.metadataFilename)) except IOError: pass try: - remove(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.dataFilename)) + remove( + path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hash), + FilesystemInventory.dataFilename)) except IOError: pass try: @@ -88,7 +116,7 @@ class FilesystemInventory(InventoryStorage): def __iter__(self): elems = [] for streamDict in self._inventory.values(): - elems.extend (streamDict.keys()) + elems.extend(streamDict.keys()) return elems.__iter__() def __len__(self): @@ -103,10 +131,12 @@ class FilesystemInventory(InventoryStorage): try: objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId) try: - newInventory[streamNumber][hashId] = InventoryItem(objectType, streamNumber, None, expiresTime, tag) + newInventory[streamNumber][hashId] = InventoryItem( + objectType, streamNumber, None, expiresTime, tag) except KeyError: newInventory[streamNumber] = {} - newInventory[streamNumber][hashId] = InventoryItem(objectType, streamNumber, None, expiresTime, tag) + newInventory[streamNumber][hashId] = InventoryItem( + objectType, streamNumber, None, expiresTime, tag) except KeyError: print "error loading %s" % (hexlify(hashId)) pass @@ -122,25 +152,42 @@ class FilesystemInventory(InventoryStorage): def getData(self, hashId): try: - with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashId), FilesystemInventory.dataFilename), 'r') as f: + with open( + path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hashId), + FilesystemInventory.dataFilename, + ), + "r", + ) as f: return f.read() except IOError: raise AttributeError def getMetadata(self, hashId): try: - with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashId), FilesystemInventory.metadataFilename), 'r') as f: + with open( + path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hashId), + FilesystemInventory.metadataFilename, + ), + "r", + ) as f: objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4) return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)] except IOError: raise KeyError def by_type_and_tag(self, objectType, tag): + """Get a list of objects filtered by object type and tag""" retval = [] for stream, streamDict in self._inventory: for hashId, item in streamDict: if item.type == objectType and item.tag == tag: - try: + try: if item.payload is None: item.payload = self.getData(hashId) except IOError: diff --git a/src/storage/sqlite.py b/src/storage/sqlite.py index 438cbdcb..d1ec220f 100644 --- a/src/storage/sqlite.py +++ b/src/storage/sqlite.py @@ -10,9 +10,17 @@ from storage import InventoryStorage, InventoryItem class SqliteInventory(InventoryStorage): def __init__(self): super(self.__class__, self).__init__() - self._inventory = {} #of objects (like msg payloads and pubkey payloads) Does not include protocol headers (the first 24 bytes of each packet). - self._objects = {} # cache for existing objects, used for quick lookups if we have an object. This is used for example whenever we receive an inv message from a peer to check to see what items are new to us. We don't delete things out of it; instead, the singleCleaner thread clears and refills it. - self.lock = RLock() # Guarantees that two receiveDataThreads don't receive and process the same message concurrently (probably sent by a malicious individual) + # of objects (like msg payloads and pubkey payloads) + # Does not include protocol headers (the first 24 bytes of each packet). + self._inventory = {} + # cache for existing objects, used for quick lookups if we have an object. + # This is used for example whenever we receive an inv message from a peer + # to check to see what items are new to us. + # We don't delete things out of it; instead, the singleCleaner thread clears and refills it. + self._objects = {} + # Guarantees that two receiveDataThreads don't receive and process the same message concurrently + # (probably sent by a malicious individual) + self.lock = RLock() def __contains__(self, hash): with self.lock: diff --git a/src/storage/storage.py b/src/storage/storage.py index 08c85708..050197b8 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -3,6 +3,7 @@ import collections InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload expires tag') class Storage(object): + """Base class for storing inventory (extendable for other items to store)""" pass # def __init__(self): # super(self.__class__, self).__init__() @@ -31,15 +32,19 @@ class InventoryStorage(Storage, collections.MutableMapping): raise NotImplementedError def by_type_and_tag(self, objectType, tag): + """Return objects filtered by object type and tag""" raise NotImplementedError def unexpired_hashes_by_stream(self, stream): + """Return unexpired inventory vectors filtered by stream""" raise NotImplementedError def flush(self): + """Flush cache""" raise NotImplementedError def clean(self): + """Free memory / perform garbage collection""" raise NotImplementedError class MailboxStorage(Storage, collections.MutableMapping): From ac341482d4435869d4d9988da61d1450a15f8d4c Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Tue, 24 Sep 2019 14:51:53 +0530 Subject: [PATCH 14/18] filesystem pylint fixes --- src/storage/filesystem.py | 55 ++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/src/storage/filesystem.py b/src/storage/filesystem.py index 4f7e717d..43ba03fc 100644 --- a/src/storage/filesystem.py +++ b/src/storage/filesystem.py @@ -1,3 +1,7 @@ +""" +src/storage/filesystem.py +========================= +""" from binascii import hexlify, unhexlify from os import listdir, makedirs, path, remove, rmdir import string @@ -8,14 +12,15 @@ from paths import lookupAppdataFolder from storage import InventoryStorage, InventoryItem -class FilesystemInventory(InventoryStorage): +class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ancestors, abstract-method + """Module for using filesystem (directory with files) for inventory storage""" topDir = "inventory" objectDir = "objects" metadataFilename = "metadata" dataFilename = "data" def __init__(self): - super(self.__class__, self).__init__() + super(FilesystemInventory, self).__init__() self.baseDir = path.join(lookupAppdataFolder(), FilesystemInventory.topDir) for createDir in [self.baseDir, path.join(self.baseDir, "objects")]: if path.exists(createDir): @@ -29,29 +34,29 @@ class FilesystemInventory(InventoryStorage): self._inventory = {} self._load() - def __contains__(self, hash): + def __contains__(self, hashval): retval = False for streamDict in self._inventory.values(): - if hash in streamDict: + if hashval in streamDict: return True return False - def __getitem__(self, hash): + def __getitem__(self, hashval): for streamDict in self._inventory.values(): try: - retval = streamDict[hash] + retval = streamDict[hashval] except KeyError: continue if retval.payload is None: - retval = InventoryItem(retval.type, retval.stream, self.getData(hash), retval.expires, retval.tag) + retval = InventoryItem(retval.type, retval.stream, self.getData(hashval), retval.expires, retval.tag) return retval - raise KeyError(hash) + raise KeyError(hashval) - def __setitem__(self, hash, value): + def __setitem__(self, hashval, value): with self.lock: value = InventoryItem(*value) try: - makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash))) + makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval))) except OSError: pass try: @@ -59,7 +64,7 @@ class FilesystemInventory(InventoryStorage): path.join( self.baseDir, FilesystemInventory.objectDir, - hexlify(hash), + hexlify(hashval), FilesystemInventory.metadataFilename, ), "w", @@ -69,7 +74,7 @@ class FilesystemInventory(InventoryStorage): path.join( self.baseDir, FilesystemInventory.objectDir, - hexlify(hash), + hexlify(hashval), FilesystemInventory.dataFilename, ), "w", @@ -78,15 +83,16 @@ class FilesystemInventory(InventoryStorage): except IOError: raise KeyError try: - self._inventory[value.stream][hash] = value + self._inventory[value.stream][hashval] = value except KeyError: self._inventory[value.stream] = {} - self._inventory[value.stream][hash] = value + self._inventory[value.stream][hashval] = value - def delHashId(self, hash): - for stream in self._inventory.keys(): + def delHashId(self, hashval): + """Remove object from inventory""" + for stream in self._inventory: try: - del self._inventory[stream][hash] + del self._inventory[stream][hashval] except KeyError: pass with self.lock: @@ -95,7 +101,7 @@ class FilesystemInventory(InventoryStorage): path.join( self.baseDir, FilesystemInventory.objectDir, - hexlify(hash), + hexlify(hashval), FilesystemInventory.metadataFilename)) except IOError: pass @@ -104,12 +110,12 @@ class FilesystemInventory(InventoryStorage): path.join( self.baseDir, FilesystemInventory.objectDir, - hexlify(hash), + hexlify(hashval), FilesystemInventory.dataFilename)) except IOError: pass try: - rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash))) + rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval))) except IOError: pass @@ -139,18 +145,20 @@ class FilesystemInventory(InventoryStorage): objectType, streamNumber, None, expiresTime, tag) except KeyError: print "error loading %s" % (hexlify(hashId)) - pass self._inventory = newInventory # for i, v in self._inventory.items(): # print "loaded stream: %s, %i items" % (i, len(v)) def stream_list(self): + """Return list of streams""" return self._inventory.keys() def object_list(self): + """Return inventory vectors (hashes) from a directory""" return [unhexlify(x) for x in listdir(path.join(self.baseDir, FilesystemInventory.objectDir))] def getData(self, hashId): + """Get object data""" try: with open( path.join( @@ -166,6 +174,7 @@ class FilesystemInventory(InventoryStorage): raise AttributeError def getMetadata(self, hashId): + """Get object metadata""" try: with open( path.join( @@ -196,12 +205,14 @@ class FilesystemInventory(InventoryStorage): return retval def hashes_by_stream(self, stream): + """Return inventory vectors (hashes) for a stream""" try: return self._inventory[stream].keys() except KeyError: return [] def unexpired_hashes_by_stream(self, stream): + """Return unexpired hashes in the inventory for a particular stream""" t = int(time.time()) try: return [x for x, value in self._inventory[stream].items() if value.expires > t] @@ -209,9 +220,11 @@ class FilesystemInventory(InventoryStorage): return [] def flush(self): + """Flush the inventory and create a new, empty one""" self._load() def clean(self): + """Clean out old items from the inventory""" minTime = int(time.time()) - (60 * 60 * 30) deletes = [] for stream, streamDict in self._inventory.items(): From 54ebbcb7db1f9ec0b2a0d0fe04ec189b70dd2b2a Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Wed, 25 Sep 2019 13:07:29 +0530 Subject: [PATCH 15/18] sqlite flake8 fixes --- src/storage/sqlite.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/storage/sqlite.py b/src/storage/sqlite.py index d1ec220f..eb41b5c0 100644 --- a/src/storage/sqlite.py +++ b/src/storage/sqlite.py @@ -1,12 +1,11 @@ -import collections -from threading import current_thread, enumerate as threadingEnumerate, RLock -import Queue +from threading import RLock import sqlite3 import time -from helper_sql import * +from helper_sql import sqlQuery, SqlBulkExecute, sqlExecute from storage import InventoryStorage, InventoryItem + class SqliteInventory(InventoryStorage): def __init__(self): super(self.__class__, self).__init__() @@ -36,7 +35,9 @@ class SqliteInventory(InventoryStorage): with self.lock: if hash in self._inventory: return self._inventory[hash] - rows = sqlQuery('SELECT objecttype, streamnumber, payload, expirestime, tag FROM inventory WHERE hash=?', sqlite3.Binary(hash)) + rows = sqlQuery( + 'SELECT objecttype, streamnumber, payload, expirestime, tag FROM inventory WHERE hash=?', + sqlite3.Binary(hash)) if not rows: raise KeyError(hash) return InventoryItem(*rows[0]) @@ -63,18 +64,22 @@ class SqliteInventory(InventoryStorage): def by_type_and_tag(self, objectType, tag): with self.lock: values = [value for value in self._inventory.values() if value.type == objectType and value.tag == tag] - values += (InventoryItem(*value) for value in sqlQuery('SELECT objecttype, streamnumber, payload, expirestime, tag FROM inventory WHERE objecttype=? AND tag=?', objectType, sqlite3.Binary(tag))) + values += (InventoryItem(*value) for value in sqlQuery( + 'SELECT objecttype, streamnumber, payload, expirestime, tag \ + FROM inventory WHERE objecttype=? AND tag=?', objectType, sqlite3.Binary(tag))) return values def unexpired_hashes_by_stream(self, stream): with self.lock: t = int(time.time()) hashes = [x for x, value in self._inventory.items() if value.stream == stream and value.expires > t] - hashes += (str(payload) for payload, in sqlQuery('SELECT hash FROM inventory WHERE streamnumber=? AND expirestime>?', stream, t)) + hashes += (str(payload) for payload, in sqlQuery( + 'SELECT hash FROM inventory WHERE streamnumber=? AND expirestime>?', stream, t)) return hashes def flush(self): - with self.lock: # If you use both the inventoryLock and the sqlLock, always use the inventoryLock OUTSIDE of the sqlLock. + with self.lock: + # If you use both the inventoryLock and the sqlLock, always use the inventoryLock OUTSIDE of the sqlLock. with SqlBulkExecute() as sql: for objectHash, value in self._inventory.items(): sql.execute('INSERT INTO inventory VALUES (?, ?, ?, ?, ?, ?)', sqlite3.Binary(objectHash), *value) @@ -82,8 +87,7 @@ class SqliteInventory(InventoryStorage): def clean(self): with self.lock: - sqlExecute('DELETE FROM inventory WHERE expirestime Date: Wed, 25 Sep 2019 15:16:28 +0530 Subject: [PATCH 16/18] sqlite pylint fixes --- src/storage/sqlite.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/storage/sqlite.py b/src/storage/sqlite.py index eb41b5c0..0c2b4afa 100644 --- a/src/storage/sqlite.py +++ b/src/storage/sqlite.py @@ -1,14 +1,19 @@ -from threading import RLock +""" +src/storage/sqlite.py +========================= +""" import sqlite3 import time +from threading import RLock from helper_sql import sqlQuery, SqlBulkExecute, sqlExecute from storage import InventoryStorage, InventoryItem -class SqliteInventory(InventoryStorage): +class SqliteInventory(InventoryStorage): # pylint: disable=too-many-ancestors + """Inventory using SQLite""" def __init__(self): - super(self.__class__, self).__init__() + super(SqliteInventory, self).__init__() # of objects (like msg payloads and pubkey payloads) # Does not include protocol headers (the first 24 bytes of each packet). self._inventory = {} @@ -21,34 +26,34 @@ class SqliteInventory(InventoryStorage): # (probably sent by a malicious individual) self.lock = RLock() - def __contains__(self, hash): + def __contains__(self, hash_): with self.lock: - if hash in self._objects: + if hash_ in self._objects: return True - rows = sqlQuery('SELECT streamnumber FROM inventory WHERE hash=?', sqlite3.Binary(hash)) + rows = sqlQuery('SELECT streamnumber FROM inventory WHERE hash=?', sqlite3.Binary(hash_)) if not rows: return False - self._objects[hash] = rows[0][0] + self._objects[hash_] = rows[0][0] return True - def __getitem__(self, hash): + def __getitem__(self, hash_): with self.lock: - if hash in self._inventory: - return self._inventory[hash] + if hash_ in self._inventory: + return self._inventory[hash_] rows = sqlQuery( 'SELECT objecttype, streamnumber, payload, expirestime, tag FROM inventory WHERE hash=?', - sqlite3.Binary(hash)) + sqlite3.Binary(hash_)) if not rows: - raise KeyError(hash) + raise KeyError(hash_) return InventoryItem(*rows[0]) - def __setitem__(self, hash, value): + def __setitem__(self, hash_, value): with self.lock: value = InventoryItem(*value) - self._inventory[hash] = value - self._objects[hash] = value.stream + self._inventory[hash_] = value + self._objects[hash_] = value.stream - def __delitem__(self, hash): + def __delitem__(self, hash_): raise NotImplementedError def __iter__(self): From e924e9208f3669d8ccfd8aaffb82dbbb6420b063 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Wed, 25 Sep 2019 16:39:15 +0530 Subject: [PATCH 17/18] storage flake8 fixes --- src/storage/storage.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/storage/storage.py b/src/storage/storage.py index 050197b8..4e603021 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -2,15 +2,17 @@ import collections InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload expires tag') + class Storage(object): """Base class for storing inventory (extendable for other items to store)""" pass # def __init__(self): # super(self.__class__, self).__init__() + class InventoryStorage(Storage, collections.MutableMapping): def __init__(self): -# super(self.__class__, self).__init__() + # super(self.__class__, self).__init__() self.numberOfInventoryLookupsPerformed = 0 def __contains__(self, hash): @@ -47,7 +49,8 @@ class InventoryStorage(Storage, collections.MutableMapping): """Free memory / perform garbage collection""" raise NotImplementedError + class MailboxStorage(Storage, collections.MutableMapping): def __init__(self): -# super(self.__class__, self).__init__() + # super(self.__class__, self).__init__() pass From fba2d6d8375fa6968dd1a0c01354e2f7b08ce490 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Thu, 26 Sep 2019 14:12:15 +0530 Subject: [PATCH 18/18] storage pylint fixes --- src/storage/storage.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/storage/storage.py b/src/storage/storage.py index 4e603021..482937e7 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -1,3 +1,7 @@ +""" +src/storage/storage.py +====================== +""" import collections InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload expires tag') @@ -6,25 +10,24 @@ InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload exp class Storage(object): """Base class for storing inventory (extendable for other items to store)""" pass -# def __init__(self): -# super(self.__class__, self).__init__() class InventoryStorage(Storage, collections.MutableMapping): + """Module used for inventory storage""" def __init__(self): - # super(self.__class__, self).__init__() + # pylint: disable=super-init-not-called self.numberOfInventoryLookupsPerformed = 0 - def __contains__(self, hash): + def __contains__(self, _): raise NotImplementedError - def __getitem__(self, hash): + def __getitem__(self, _): raise NotImplementedError - def __setitem__(self, hash, value): + def __setitem__(self, _, value): raise NotImplementedError - def __delitem__(self, hash): + def __delitem__(self, _): raise NotImplementedError def __iter__(self): @@ -50,7 +53,8 @@ class InventoryStorage(Storage, collections.MutableMapping): raise NotImplementedError -class MailboxStorage(Storage, collections.MutableMapping): +class MailboxStorage(Storage, collections.MutableMapping): # pylint: disable=abstract-method + """Method for storing mails""" def __init__(self): - # super(self.__class__, self).__init__() + # pylint: disable=super-init-not-called pass