From c79636863d8bae013e929c0ab842fd40d0cd13c2 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Thu, 5 Dec 2019 18:17:35 +0200 Subject: [PATCH 01/31] If tray is not available, disable settings group "Tray" and related checkboxes; set checkBoxMinimizeToTray to false by default --- src/bitmessageqt/settings.py | 39 +++++++++++++++++++++++++----------- src/bitmessageqt/settings.ui | 2 +- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/bitmessageqt/settings.py b/src/bitmessageqt/settings.py index 982328cc..7fb0ea91 100644 --- a/src/bitmessageqt/settings.py +++ b/src/bitmessageqt/settings.py @@ -47,20 +47,33 @@ class SettingsDialog(QtGui.QDialog): def adjust_from_config(self, config): """Adjust all widgets state according to config settings""" # pylint: disable=too-many-branches,too-many-statements - self.checkBoxStartOnLogon.setChecked( - config.getboolean('bitmessagesettings', 'startonlogon')) - self.checkBoxMinimizeToTray.setChecked( - config.getboolean('bitmessagesettings', 'minimizetotray')) - self.checkBoxTrayOnClose.setChecked( - config.safeGetBoolean('bitmessagesettings', 'trayonclose')) + if not self.parent.tray.isSystemTrayAvailable(): + self.groupBoxTray.setEnabled(False) + self.groupBoxTray.setTitle(_translate( + "MainWindow", "Tray (not available in your system)")) + for setting in ( + 'minimizetotray', 'trayonclose', 'startintray'): + config.set('bitmessagesettings', setting, 'false') + else: + self.checkBoxMinimizeToTray.setChecked( + config.getboolean('bitmessagesettings', 'minimizetotray')) + self.checkBoxTrayOnClose.setChecked( + config.safeGetBoolean('bitmessagesettings', 'trayonclose')) + self.checkBoxStartInTray.setChecked( + config.getboolean('bitmessagesettings', 'startintray')) + self.checkBoxHideTrayConnectionNotifications.setChecked( - config.getboolean("bitmessagesettings", "hidetrayconnectionnotifications")) + config.getboolean( + 'bitmessagesettings', 'hidetrayconnectionnotifications')) self.checkBoxShowTrayNotifications.setChecked( config.getboolean('bitmessagesettings', 'showtraynotifications')) - self.checkBoxStartInTray.setChecked( - config.getboolean('bitmessagesettings', 'startintray')) + + self.checkBoxStartOnLogon.setChecked( + config.getboolean('bitmessagesettings', 'startonlogon')) + self.checkBoxWillinglySendToMobile.setChecked( - config.safeGetBoolean('bitmessagesettings', 'willinglysendtomobile')) + config.safeGetBoolean( + 'bitmessagesettings', 'willinglysendtomobile')) self.checkBoxUseIdenticons.setChecked( config.safeGetBoolean('bitmessagesettings', 'useidenticons')) self.checkBoxReplyBelow.setChecked( @@ -82,10 +95,12 @@ class SettingsDialog(QtGui.QDialog): "MainWindow", "Start-on-login not yet supported on your OS.")) self.checkBoxMinimizeToTray.setDisabled(True) self.checkBoxMinimizeToTray.setText(_translate( - "MainWindow", "Minimize-to-tray not yet supported on your OS.")) + "MainWindow", + "Minimize-to-tray not yet supported on your OS.")) self.checkBoxShowTrayNotifications.setDisabled(True) self.checkBoxShowTrayNotifications.setText(_translate( - "MainWindow", "Tray notifications not yet supported on your OS.")) + "MainWindow", + "Tray notifications not yet supported on your OS.")) elif 'linux' in sys.platform: self.checkBoxStartOnLogon.setDisabled(True) self.checkBoxStartOnLogon.setText(_translate( diff --git a/src/bitmessageqt/settings.ui b/src/bitmessageqt/settings.ui index 963f2e64..33278e94 100644 --- a/src/bitmessageqt/settings.ui +++ b/src/bitmessageqt/settings.ui @@ -75,7 +75,7 @@ Minimize to tray - true + false From 5a35de6bca4385431905be5a59f004f7bfd2896c Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Fri, 20 Dec 2019 15:20:14 +0200 Subject: [PATCH 02/31] Fix sendOnionPeerObj() broken in 9923e97 --- src/class_singleWorker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/class_singleWorker.py b/src/class_singleWorker.py index a275b79d..d035c092 100644 --- a/src/class_singleWorker.py +++ b/src/class_singleWorker.py @@ -467,8 +467,8 @@ class singleWorker(StoppableThread): def sendOnionPeerObj(self, peer=None): """Send onionpeer object representing peer""" if not peer: # find own onionhostname - for peer_ in state.ownAddresses: - if peer_.host.endswith('.onion'): + for peer in state.ownAddresses: + if peer.host.endswith('.onion'): break else: return From 03316496b7c3380c5ac408f86d049855dbcedac6 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Sat, 14 Dec 2019 12:53:51 +0200 Subject: [PATCH 03/31] Stop UDPSocket on socket.error 101 (Network is unreachable) --- src/network/udp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/network/udp.py b/src/network/udp.py index cf694567..d8c5f340 100644 --- a/src/network/udp.py +++ b/src/network/udp.py @@ -146,6 +146,9 @@ class UDPSocket(BMProto): # pylint: disable=too-many-instance-attributes retval = self.socket.sendto( self.write_buf, ('', self.port)) except socket.error as e: - logger.error("socket error on sendato: %s", e) + logger.error("socket error on sendto: %s", e) + if e.errno == 101: + self.announcing = False + self.socket.close() retval = 0 self.slice_write_buf(retval) From 9119507b033372cdca432eb45c3d3d897455bd4b Mon Sep 17 00:00:00 2001 From: sandakersmann Date: Fri, 27 Dec 2019 18:23:02 +0100 Subject: [PATCH 04/31] Changed copyright year to 2020 --- COPYING | 2 +- LICENSE | 2 +- src/api.py | 2 +- src/bitmessagemain.py | 2 +- src/bitmessageqt/about.ui | 2 +- src/bitmessageqt/dialogs.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/COPYING b/COPYING index 2e0ae6c1..078bf213 100644 --- a/COPYING +++ b/COPYING @@ -1,5 +1,5 @@ Copyright (c) 2012-2016 Jonathan Warren -Copyright (c) 2012-2019 The Bitmessage Developers +Copyright (c) 2012-2020 The Bitmessage Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/LICENSE b/LICENSE index 3be738c0..6bb86242 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) Copyright (c) 2012-2016 Jonathan Warren -Copyright (c) 2012-2019 The Bitmessage Developers +Copyright (c) 2012-2020 The Bitmessage Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/src/api.py b/src/api.py index f9d0a518..3201fba5 100644 --- a/src/api.py +++ b/src/api.py @@ -2,7 +2,7 @@ # pylint: disable=too-many-statements # Copyright (c) 2012-2016 Jonathan Warren -# Copyright (c) 2012-2019 The Bitmessage developers +# Copyright (c) 2012-2020 The Bitmessage developers """ This is not what you run to run the Bitmessage API. Instead, enable the API diff --git a/src/bitmessagemain.py b/src/bitmessagemain.py index 176125cc..96885b5e 100755 --- a/src/bitmessagemain.py +++ b/src/bitmessagemain.py @@ -3,7 +3,7 @@ The PyBitmessage startup script """ # Copyright (c) 2012-2016 Jonathan Warren -# Copyright (c) 2012-2019 The Bitmessage developers +# Copyright (c) 2012-2020 The Bitmessage developers # Distributed under the MIT/X11 software license. See the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bitmessageqt/about.ui b/src/bitmessageqt/about.ui index a912927a..7073bbd1 100644 --- a/src/bitmessageqt/about.ui +++ b/src/bitmessageqt/about.ui @@ -46,7 +46,7 @@ - <html><head/><body><p>Copyright © 2012-2016 Jonathan Warren<br/>Copyright © 2012-2019 The Bitmessage Developers</p></body></html> + <html><head/><body><p>Copyright © 2012-2016 Jonathan Warren<br/>Copyright © 2012-2020 The Bitmessage Developers</p></body></html> Qt::AlignLeft diff --git a/src/bitmessageqt/dialogs.py b/src/bitmessageqt/dialogs.py index b4bcd2fd..c667edb1 100644 --- a/src/bitmessageqt/dialogs.py +++ b/src/bitmessageqt/dialogs.py @@ -47,7 +47,7 @@ class AboutDialog(QtGui.QDialog, RetranslateMixin): try: self.label_2.setText( self.label_2.text().replace( - '2019', str(last_commit.get('time').year) + '2020', str(last_commit.get('time').year) )) except AttributeError: pass From 168c4a5696510bb3c6256b519352a6b0413dd713 Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Tue, 3 Dec 2019 12:59:27 +0100 Subject: [PATCH 05/31] Checkdeps fix - an exception can be triggered if too many requirements are fulfilled --- checkdeps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkdeps.py b/checkdeps.py index 03782037..45dc2fc9 100755 --- a/checkdeps.py +++ b/checkdeps.py @@ -164,7 +164,7 @@ if (not compiler or prereqs) and OPSYS in PACKAGE_MANAGER: if not compiler: compilerToPackages() prereqToPackages() - if mandatory: + if prereqs and mandatory: sys.exit(1) else: print("All the dependencies satisfied, you can install PyBitmessage") From c35f48bd0bdd7581571be6ee111d6530c00246b8 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Sun, 22 Sep 2019 15:35:10 +0300 Subject: [PATCH 06/31] Fix setting socksproxytype and return in proxyconfig_stem: socksproxytype is set to SOCKS5 temporary if proxyconfig succeeds. --- src/plugins/proxyconfig_stem.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/plugins/proxyconfig_stem.py b/src/plugins/proxyconfig_stem.py index bdbfe8ca..e56aecc6 100644 --- a/src/plugins/proxyconfig_stem.py +++ b/src/plugins/proxyconfig_stem.py @@ -43,6 +43,7 @@ def connect_plugin(config): # pylint: disable=too-many-branches ): # remote proxy is choosen for outbound connections, # nothing to do here, but need to set socksproxytype to SOCKS5! + config.set('bitmessagesettings', 'socksproxytype', 'SOCKS5') logwrite( 'sockshostname is set to remote address,' ' aborting stem proxy configuration') @@ -77,6 +78,8 @@ def connect_plugin(config): # pylint: disable=too-many-branches logwrite('Started tor on port %s' % port) break + config.setTemp('bitmessagesettings', 'socksproxytype', 'SOCKS5') + if config.safeGetBoolean('bitmessagesettings', 'sockslisten'): # need a hidden service for inbound connections try: @@ -117,6 +120,5 @@ def connect_plugin(config): # pylint: disable=too-many-branches config.set( onionhostname, 'keytype', response.private_key_type) config.save() - config.set('bitmessagesettings', 'socksproxytype', 'SOCKS5') - return True + return True From 44cb975a611dcca0f40f328f44f517962f013a52 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Fri, 18 Oct 2019 17:52:00 +0300 Subject: [PATCH 07/31] Fixed bug in plugin.get_plugins(), edited docstrings --- src/plugins/__init__.py | 7 +++++++ src/plugins/plugin.py | 27 ++++++++++++++++++++------- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/plugins/__init__.py b/src/plugins/__init__.py index e69de29b..285009df 100644 --- a/src/plugins/__init__.py +++ b/src/plugins/__init__.py @@ -0,0 +1,7 @@ +""" +Simple plugin system based on setuptools +---------------------------------------- + + +""" +# .. include:: pybitmessage.plugins.plugin.rst diff --git a/src/plugins/plugin.py b/src/plugins/plugin.py index e671a73f..629de0a6 100644 --- a/src/plugins/plugin.py +++ b/src/plugins/plugin.py @@ -1,19 +1,28 @@ # -*- coding: utf-8 -*- """ -src/plugins/plugin.py -=================================== +Operating with plugins """ +import logging + import pkg_resources +logger = logging.getLogger('default') + + def get_plugins(group, point='', name=None, fallback=None): """ - Iterate through plugins (`connect_plugin` attribute of entry point) - which name starts with `point` or equals to `name`. - If `fallback` kwarg specified, plugin with that name yield last. + :param str group: plugin group + :param str point: plugin name prefix + :param name: exact plugin name + :param fallback: fallback plugin name + + Iterate through plugins (``connect_plugin`` attribute of entry point) + which name starts with ``point`` or equals to ``name``. + If ``fallback`` kwarg specified, plugin with that name yield last. """ for ep in pkg_resources.iter_entry_points('bitmessage.' + group): - if name and ep.name == name or ep.name.startswith(point): + if name and ep.name == name or not point or ep.name.startswith(point): try: plugin = ep.load().connect_plugin if ep.name == fallback: @@ -25,6 +34,8 @@ def get_plugins(group, point='', name=None, fallback=None): ValueError, pkg_resources.DistributionNotFound, pkg_resources.UnknownExtra): + logger.debug( + 'Problem while loading %s', ep.name, exc_info=True) continue try: yield _fallback @@ -33,6 +44,8 @@ def get_plugins(group, point='', name=None, fallback=None): def get_plugin(*args, **kwargs): - """Returns first available plugin `from get_plugins()` if any.""" + """ + :return: first available plugin from :func:`get_plugins` if any. + """ for plugin in get_plugins(*args, **kwargs): return plugin From 5160a68c28a3493e4c0bb55a4169cef350e2058e Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Fri, 18 Oct 2019 17:52:44 +0300 Subject: [PATCH 08/31] Moved start_proxyconfig to helper_startup; no more prints in helper_startup --- src/bitmessagemain.py | 32 ++++++---------------------- src/helper_startup.py | 49 ++++++++++++++++++++++++++++++++++++------- src/tests/core.py | 9 ++++---- 3 files changed, 51 insertions(+), 39 deletions(-) diff --git a/src/bitmessagemain.py b/src/bitmessagemain.py index 96885b5e..48ed9738 100755 --- a/src/bitmessagemain.py +++ b/src/bitmessagemain.py @@ -39,7 +39,8 @@ import shutdown from bmconfigparser import BMConfigParser from debug import logger # this should go before any threads from helper_startup import ( - isOurOperatingSystemLimitedToHavingVeryFewHalfOpenConnections + isOurOperatingSystemLimitedToHavingVeryFewHalfOpenConnections, + start_proxyconfig ) from inventory import Inventory from knownnodes import readKnownNodes @@ -168,30 +169,9 @@ def signal_handler(signum, frame): class Main(object): """Main PyBitmessage class""" - @staticmethod - def start_proxyconfig(config): - """Check socksproxytype and start any proxy configuration plugin""" - proxy_type = config.safeGet('bitmessagesettings', 'socksproxytype') - if proxy_type not in ('none', 'SOCKS4a', 'SOCKS5'): - # pylint: disable=relative-import - from plugins.plugin import get_plugin - try: - proxyconfig_start = time.time() - if not get_plugin('proxyconfig', name=proxy_type)(config): - raise TypeError - except TypeError: - logger.error( - 'Failed to run proxy config plugin %s', - proxy_type, exc_info=True) - shutdown.doCleanShutdown() - sys.exit(2) - else: - logger.info( - 'Started proxy config plugin %s in %s sec', - proxy_type, time.time() - proxyconfig_start) - - def start(self): # pylint: disable=too-many-statements, too-many-branches, too-many-locals + def start(self): """Start main application""" + # pylint: disable=too-many-statements,too-many-branches,too-many-locals _fixSocket() config = BMConfigParser() @@ -350,7 +330,7 @@ class Main(object): # start network components if networking is enabled if state.enableNetwork: - self.start_proxyconfig(config) + start_proxyconfig() BMConnectionPool() asyncoreThread = BMNetworkThread() asyncoreThread.daemon = True @@ -410,7 +390,7 @@ class Main(object): self.stop() elif not state.enableGUI: from tests import core as test_core # pylint: disable=relative-import - test_core_result = test_core.run(self) + test_core_result = test_core.run() state.enableGUI = True self.stop() test_core.cleanup() diff --git a/src/helper_startup.py b/src/helper_startup.py index 9aaad5ef..9711c339 100644 --- a/src/helper_startup.py +++ b/src/helper_startup.py @@ -2,11 +2,12 @@ Startup operations. """ # pylint: disable=too-many-branches,too-many-statements -from __future__ import print_function +import logging import os import platform import sys +import time from distutils.version import StrictVersion import defaults @@ -15,6 +16,13 @@ import paths import state from bmconfigparser import BMConfigParser +try: + from plugins.plugin import get_plugin +except ImportError: + get_plugin = None + + +logger = logging.getLogger('default') # The user may de-select Portable Mode in the settings if they want # the config files to stay in the application data folder. @@ -30,14 +38,14 @@ def loadConfig(): needToCreateKeysFile = config.safeGet( 'bitmessagesettings', 'settingsversion') is None if not needToCreateKeysFile: - print( + logger.info( 'Loading config files from directory specified' - ' on startup: %s' % state.appdata) + ' on startup: %s', state.appdata) else: config.read(paths.lookupExeFolder() + 'keys.dat') try: config.get('bitmessagesettings', 'settingsversion') - print('Loading config files from same directory as program.') + logger.info('Loading config files from same directory as program.') needToCreateKeysFile = False state.appdata = paths.lookupExeFolder() except: @@ -48,7 +56,8 @@ def loadConfig(): needToCreateKeysFile = config.safeGet( 'bitmessagesettings', 'settingsversion') is None if not needToCreateKeysFile: - print('Loading existing config files from', state.appdata) + logger.info( + 'Loading existing config files from %s', state.appdata) if needToCreateKeysFile: @@ -103,9 +112,10 @@ def loadConfig(): # Just use the same directory as the program and forget about # the appdata folder state.appdata = '' - print('Creating new config files in same directory as program.') + logger.info( + 'Creating new config files in same directory as program.') else: - print('Creating new config files in', state.appdata) + logger.info('Creating new config files in %s', state.appdata) if not os.path.exists(state.appdata): os.makedirs(state.appdata) if not sys.platform.startswith('win'): @@ -255,7 +265,7 @@ def updateConfig(): 'bitmessagesettings', 'hidetrayconnectionnotifications', 'false') if config.safeGetInt('bitmessagesettings', 'maxoutboundconnections') < 1: config.set('bitmessagesettings', 'maxoutboundconnections', '8') - print('WARNING: your maximum outbound connections must be a number.') + logger.warning('Your maximum outbound connections must be a number.') # TTL is now user-specifiable. Let's add an option to save # whatever the user selects. @@ -278,3 +288,26 @@ def isOurOperatingSystemLimitedToHavingVeryFewHalfOpenConnections(): return False except Exception: pass + + +def start_proxyconfig(): + """Check socksproxytype and start any proxy configuration plugin""" + if not get_plugin: + return + config = BMConfigParser() + proxy_type = config.safeGet('bitmessagesettings', 'socksproxytype') + if proxy_type and proxy_type not in ('none', 'SOCKS4a', 'SOCKS5'): + try: + proxyconfig_start = time.time() + if not get_plugin('proxyconfig', name=proxy_type)(config): + raise TypeError() + except TypeError: + # cannot import shutdown here ): + logger.error( + 'Failed to run proxy config plugin %s', + proxy_type, exc_info=True) + os._exit(0) # pylint: disable=protected-access + else: + logger.info( + 'Started proxy config plugin %s in %s sec', + proxy_type, time.time() - proxyconfig_start) diff --git a/src/tests/core.py b/src/tests/core.py index d2456064..84971dc5 100644 --- a/src/tests/core.py +++ b/src/tests/core.py @@ -14,6 +14,7 @@ import unittest import knownnodes import state from bmconfigparser import BMConfigParser +from helper_startup import start_proxyconfig from helper_msgcoding import MsgEncode, MsgDecode from network import asyncore_pollchoose as asyncore from network.connectionpool import BMConnectionPool @@ -21,8 +22,8 @@ from network.node import Peer from network.tcp import Socks4aBMConnection, Socks5BMConnection, TCPConnection from queues import excQueue + knownnodes_file = os.path.join(state.appdata, 'knownnodes.dat') -program = None def pickle_knownnodes(): @@ -196,14 +197,12 @@ class TestCore(unittest.TestCase): self._check_bootstrap() self._initiate_bootstrap() BMConfigParser().set('bitmessagesettings', 'socksproxytype', 'stem') - program.start_proxyconfig(BMConfigParser()) + start_proxyconfig() self._check_bootstrap() -def run(prog): +def run(): """Starts all tests defined in this module""" - global program # pylint: disable=global-statement - program = prog loader = unittest.TestLoader() loader.sortTestMethodsUsing = None suite = loader.loadTestsFromTestCase(TestCore) From e3ccc3c7c8e4d3662adb8e20e20f1d8a32687e83 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Thu, 3 Oct 2019 18:34:27 +0300 Subject: [PATCH 09/31] Support for socksproxytype plugins in Settings dialog --- src/bitmessageqt/settings.py | 64 +++++++++++++++++++----------------- src/bitmessageqt/settings.ui | 4 +-- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/src/bitmessageqt/settings.py b/src/bitmessageqt/settings.py index 7fb0ea91..8fe1ecba 100644 --- a/src/bitmessageqt/settings.py +++ b/src/bitmessageqt/settings.py @@ -1,3 +1,4 @@ +import ConfigParser import os import sys @@ -16,6 +17,7 @@ import tempfile import widgets from bmconfigparser import BMConfigParser from helper_sql import sqlExecute, sqlStoredProcedure +from helper_startup import start_proxyconfig from network.asyncore_pollchoose import set_rates from tr import _translate @@ -32,6 +34,16 @@ class SettingsDialog(QtGui.QDialog): self.net_restart_needed = False self.timer = QtCore.QTimer() + try: + import pkg_resources + except ImportError: + pass + else: + # Append proxy types defined in plugins + for ep in pkg_resources.iter_entry_points( + 'bitmessage.proxyconfig'): + self.comboBoxProxyType.addItem(ep.name) + self.lineEditMaxOutboundConnections.setValidator( QtGui.QIntValidator(0, 8, self.lineEditMaxOutboundConnections)) @@ -117,21 +129,16 @@ class SettingsDialog(QtGui.QDialog): self.checkBoxOnionOnly.setChecked( config.safeGetBoolean('bitmessagesettings', 'onionservicesonly')) - proxy_type = config.safeGet( - 'bitmessagesettings', 'socksproxytype', 'none') - if proxy_type == 'none': - self.comboBoxProxyType.setCurrentIndex(0) - self.lineEditSocksHostname.setEnabled(False) - self.lineEditSocksPort.setEnabled(False) - self.lineEditSocksUsername.setEnabled(False) - self.lineEditSocksPassword.setEnabled(False) - self.checkBoxAuthentication.setEnabled(False) - self.checkBoxSocksListen.setEnabled(False) - self.checkBoxOnionOnly.setEnabled(False) - elif proxy_type == 'SOCKS4a': - self.comboBoxProxyType.setCurrentIndex(1) - elif proxy_type == 'SOCKS5': - self.comboBoxProxyType.setCurrentIndex(2) + try: + # Get real value, not temporary + self._proxy_type = ConfigParser.SafeConfigParser.get( + config, 'bitmessagesettings', 'socksproxytype') + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self._proxy_type = 'none' + self.comboBoxProxyType.setCurrentIndex( + 0 if self._proxy_type == 'none' + else self.comboBoxProxyType.findText(self._proxy_type)) + self.comboBoxProxyTypeChanged(self.comboBoxProxyType.currentIndex()) self.lineEditSocksHostname.setText( config.get('bitmessagesettings', 'sockshostname')) @@ -219,7 +226,7 @@ class SettingsDialog(QtGui.QDialog): self.checkBoxAuthentication.setEnabled(False) self.checkBoxSocksListen.setEnabled(False) self.checkBoxOnionOnly.setEnabled(False) - elif comboBoxIndex in (1, 2): + else: self.lineEditSocksHostname.setEnabled(True) self.lineEditSocksPort.setEnabled(True) self.checkBoxAuthentication.setEnabled(True) @@ -321,27 +328,22 @@ class SettingsDialog(QtGui.QDialog): upnpThread = upnp.uPnPThread() upnpThread.start() - proxy_type = self.config.safeGet( - 'bitmessagesettings', 'socksproxytype', 'none') - if ( - proxy_type == 'none' and - self.comboBoxProxyType.currentText()[0:5] == 'SOCKS' and - shared.statusIconColor != 'red' - ): - self.net_restart_needed = True - if ( - proxy_type[0:5] == 'SOCKS' and - self.comboBoxProxyType.currentText()[0:5] != 'SOCKS' - ): + proxytype_index = self.comboBoxProxyType.currentIndex() + if proxytype_index == 0: + if self._proxy_type != 'none' and shared.statusIconColor != 'red': + self.net_restart_needed = True + elif self.comboBoxProxyType.currentText() != self._proxy_type: self.net_restart_needed = True self.parent.statusbar.clearMessage() self.config.set( 'bitmessagesettings', 'socksproxytype', - str(self.comboBoxProxyType.currentText()) - if self.comboBoxProxyType.currentText()[0:5] == 'SOCKS' - else 'none' + 'none' if self.comboBoxProxyType.currentIndex() == 0 + else str(self.comboBoxProxyType.currentText()) ) + if proxytype_index > 2: # last literal proxytype in ui + start_proxyconfig() + self.config.set('bitmessagesettings', 'socksauthentication', str( self.checkBoxAuthentication.isChecked())) self.config.set('bitmessagesettings', 'sockshostname', str( diff --git a/src/bitmessageqt/settings.ui b/src/bitmessageqt/settings.ui index 33278e94..0ffbf442 100644 --- a/src/bitmessageqt/settings.ui +++ b/src/bitmessageqt/settings.ui @@ -419,12 +419,12 @@ - SOCKS4a + SOCKS4a - SOCKS5 + SOCKS5 From 2bddae511a84fae6c4830ce7903e442c77de5673 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Mon, 21 Oct 2019 15:20:29 +0300 Subject: [PATCH 10/31] Fixed some mistakes in tor dependent tests and marked them for skipping until the finish of debug. --- requirements.txt | 1 - src/tests/core.py | 48 +++++++++++++++++++++++++++++++---------------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/requirements.txt b/requirements.txt index be429a9f..c55e5cf1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ python_prctl psutil pycrypto -stem diff --git a/src/tests/core.py b/src/tests/core.py index 84971dc5..d56076c3 100644 --- a/src/tests/core.py +++ b/src/tests/core.py @@ -14,14 +14,19 @@ import unittest import knownnodes import state from bmconfigparser import BMConfigParser -from helper_startup import start_proxyconfig from helper_msgcoding import MsgEncode, MsgDecode +from helper_startup import start_proxyconfig from network import asyncore_pollchoose as asyncore from network.connectionpool import BMConnectionPool from network.node import Peer from network.tcp import Socks4aBMConnection, Socks5BMConnection, TCPConnection from queues import excQueue +try: + import stem.version as stem_version +except ImportError: + stem_version = None + knownnodes_file = os.path.join(state.appdata, 'knownnodes.dat') @@ -171,12 +176,29 @@ class TestCore(unittest.TestCase): self.assertIsInstance(con, connection_base) self.assertNotEqual(peer.host, '127.0.0.1') return - else: # pylint: disable=useless-else-on-loop - self.fail( - 'Failed to connect during %s sec' % (time.time() - _started)) + self.fail( + 'Failed to connect during %s sec' % (time.time() - _started)) - def test_onionservicesonly(self): - """test onionservicesonly networking mode""" + def test_bootstrap(self): + """test bootstrapping""" + self._initiate_bootstrap() + self._check_bootstrap() + + @unittest.skipUnless(stem_version, 'No stem, skipping tor dependent test') + def test_bootstrap_tor(self): + """test bootstrapping with tor""" + self._initiate_bootstrap() + BMConfigParser().set('bitmessagesettings', 'socksproxytype', 'stem') + start_proxyconfig() + self._check_bootstrap() + + @unittest.skipUnless(stem_version, 'No stem, skipping tor dependent test') + def test_onionservicesonly(self): # this should start after bootstrap + """ + set onionservicesonly, wait for 3 connections and check them all + are onions + """ + BMConfigParser().set('bitmessagesettings', 'socksproxytype', 'SOCKS5') BMConfigParser().set('bitmessagesettings', 'onionservicesonly', 'true') self._initiate_bootstrap() BMConfigParser().remove_option('bitmessagesettings', 'dontconnect') @@ -185,21 +207,15 @@ class TestCore(unittest.TestCase): for n, peer in enumerate(BMConnectionPool().outboundConnections): if n > 2: return - if not peer.host.endswith('.onion'): + if ( + not peer.host.endswith('.onion') + and not peer.host.startswith('bootstrap') + ): self.fail( 'Found non onion hostname %s in outbound connections!' % peer.host) self.fail('Failed to connect to at least 3 nodes within 360 sec') - def test_bootstrap(self): - """test bootstrapping""" - self._initiate_bootstrap() - self._check_bootstrap() - self._initiate_bootstrap() - BMConfigParser().set('bitmessagesettings', 'socksproxytype', 'stem') - start_proxyconfig() - self._check_bootstrap() - def run(): """Starts all tests defined in this module""" From 52d5c1ff03707829b5055ef71015c0da369fb09d Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Tue, 22 Oct 2019 10:10:41 +0300 Subject: [PATCH 11/31] Document proxyconfig_stem --- src/plugins/proxyconfig_stem.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/plugins/proxyconfig_stem.py b/src/plugins/proxyconfig_stem.py index e56aecc6..cbc6395d 100644 --- a/src/plugins/proxyconfig_stem.py +++ b/src/plugins/proxyconfig_stem.py @@ -1,7 +1,15 @@ # -*- coding: utf-8 -*- """ -src/plugins/proxyconfig_stem.py -=================================== +Configure tor proxy and hidden service with +`stem `_ depending on *bitmessagesettings*: + + * try to start own tor instance on *socksport* if *sockshostname* + is unset or set to localhost; + * if *socksport* is already in use that instance is used only for + hidden service (if *sockslisten* is also set True); + * create ephemeral hidden service v3 if there is already *onionhostname*; + * otherwise use stem's 'BEST' version and save onion keys to the new + section using *onionhostname* as name for future use. """ import os import logging @@ -36,10 +44,16 @@ class DebugLogger(object): def connect_plugin(config): # pylint: disable=too-many-branches - """Run stem proxy configurator""" + """ + Run stem proxy configurator + + :param config: current configuration instance + :type config: :class:`pybitmessage.bmconfigparser.BMConfigParser` + :return: True if configuration was done successfully + """ logwrite = DebugLogger() - if config.safeGet('bitmessagesettings', 'sockshostname') not in ( - 'localhost', '127.0.0.1', '' + if config.safeGet('bitmessagesettings', 'sockshostname', '') not in ( + 'localhost', '127.0.0.1', '' ): # remote proxy is choosen for outbound connections, # nothing to do here, but need to set socksproxytype to SOCKS5! From 1c4d7655c3a58e067db4a872b747fe5a39c3c181 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Sat, 7 Dec 2019 21:33:30 +0200 Subject: [PATCH 12/31] Fix socksproxytype in the support message, made that a separate function getSOCKSProxyType(config) in the settings. --- src/bitmessageqt/settings.py | 24 ++++++++++++++++-------- src/bitmessageqt/support.py | 4 ++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/bitmessageqt/settings.py b/src/bitmessageqt/settings.py index 8fe1ecba..011d38ed 100644 --- a/src/bitmessageqt/settings.py +++ b/src/bitmessageqt/settings.py @@ -22,6 +22,19 @@ from network.asyncore_pollchoose import set_rates from tr import _translate +def getSOCKSProxyType(config): + """Get user socksproxytype setting from *config*""" + try: + result = ConfigParser.SafeConfigParser.get( + config, 'bitmessagesettings', 'socksproxytype') + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + return + else: + if result.lower() in ('', 'none', 'false'): + result = None + return result + + class SettingsDialog(QtGui.QDialog): """The "Settings" dialog""" def __init__(self, parent=None, firstrun=False): @@ -129,14 +142,9 @@ class SettingsDialog(QtGui.QDialog): self.checkBoxOnionOnly.setChecked( config.safeGetBoolean('bitmessagesettings', 'onionservicesonly')) - try: - # Get real value, not temporary - self._proxy_type = ConfigParser.SafeConfigParser.get( - config, 'bitmessagesettings', 'socksproxytype') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self._proxy_type = 'none' + self._proxy_type = getSOCKSProxyType(config) self.comboBoxProxyType.setCurrentIndex( - 0 if self._proxy_type == 'none' + 0 if not self._proxy_type else self.comboBoxProxyType.findText(self._proxy_type)) self.comboBoxProxyTypeChanged(self.comboBoxProxyType.currentIndex()) @@ -330,7 +338,7 @@ class SettingsDialog(QtGui.QDialog): proxytype_index = self.comboBoxProxyType.currentIndex() if proxytype_index == 0: - if self._proxy_type != 'none' and shared.statusIconColor != 'red': + if self._proxy_type and shared.statusIconColor != 'red': self.net_restart_needed = True elif self.comboBoxProxyType.currentText() != self._proxy_type: self.net_restart_needed = True diff --git a/src/bitmessageqt/support.py b/src/bitmessageqt/support.py index 2a1ddb18..d6d4543d 100644 --- a/src/bitmessageqt/support.py +++ b/src/bitmessageqt/support.py @@ -15,6 +15,7 @@ from openclpow import openclAvailable, openclEnabled import paths import proofofwork from pyelliptic.openssl import OpenSSL +from settings import getSOCKSProxyType import queues import network.stats import state @@ -118,8 +119,7 @@ def createSupportMessage(myapp): BMConfigParser().safeGet('bitmessagesettings', 'opencl') ) if openclEnabled() else "None" locale = getTranslationLanguage() - socks = BMConfigParser().safeGet( - 'bitmessagesettings', 'socksproxytype', "N/A") + socks = getSOCKSProxyType(BMConfigParser()) or "N/A" upnp = BMConfigParser().safeGet('bitmessagesettings', 'upnp', "N/A") connectedhosts = len(network.stats.connectedHostsList()) From 61f64f72c36b315ef8663176d6cc2474af4457ea Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Fri, 20 Dec 2019 10:53:31 +0200 Subject: [PATCH 13/31] Fixing port for hidden service --- src/plugins/proxyconfig_stem.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/proxyconfig_stem.py b/src/plugins/proxyconfig_stem.py index cbc6395d..494519f2 100644 --- a/src/plugins/proxyconfig_stem.py +++ b/src/plugins/proxyconfig_stem.py @@ -112,7 +112,8 @@ def connect_plugin(config): # pylint: disable=too-many-branches onionkeytype = config.safeGet(onionhostname, 'keytype') response = controller.create_ephemeral_hidden_service( - config.safeGetInt('bitmessagesettings', 'onionport', 8444), + {config.safeGetInt('bitmessagesettings', 'onionport', 8444): + config.safeGetInt('bitmessagesettings', 'port', 8444)}, key_type=(onionkeytype or 'NEW'), key_content=(onionkey or onionhostname and 'ED25519-V3' or 'BEST') ) From d9ef4a8e8dede2de1c47998ba05584d6d2a64dcf Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Fri, 27 Dec 2019 18:02:43 +0530 Subject: [PATCH 14/31] fix spelling mistakes --- LICENSE | 4 ++-- android_instruction.rst | 4 ++-- checkdeps.py | 2 +- docs/contribute.dir/develop.dir/opsec.rst | 4 ++-- docs/contribute.dir/develop.dir/overview.rst | 8 ++++---- fabfile/README.md | 4 ++-- packages/README.md | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/LICENSE b/LICENSE index 6bb86242..d6df32b5 100644 --- a/LICENSE +++ b/LICENSE @@ -22,7 +22,7 @@ SOFTWARE. ===== qidenticon.py identicon python implementation with QPixmap output by sendiulo -qidenticon.py is Licesensed under FreeBSD License. +qidenticon.py is Licensed under FreeBSD License. (http://www.freebsd.org/copyright/freebsd-license.html) Copyright 2013 "Sendiulo". All rights reserved. @@ -36,7 +36,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR I ===== based on identicon.py identicon python implementation. by Shin Adachi -identicon.py is Licesensed under FreeBSD License. +identicon.py is Licensed under FreeBSD License. (http://www.freebsd.org/copyright/freebsd-license.html) Copyright 1994-2009 Shin Adachi. All rights reserved. diff --git a/android_instruction.rst b/android_instruction.rst index fab55b55..e6c7797d 100644 --- a/android_instruction.rst +++ b/android_instruction.rst @@ -1,6 +1,6 @@ PyBitmessage(Android) -This sample aims to be as close to a real world example of a mobile. It has a more refined design and also provides a practical example of how a mobile app would interact and communicate with its adresses. +This sample aims to be as close to a real world example of a mobile. It has a more refined design and also provides a practical example of how a mobile app would interact and communicate with its addresses. Steps for trying out this sample: @@ -13,7 +13,7 @@ This sample uses the kivy as Kivy is an open source, cross-platform Python frame Kivy is written in Python and Cython, supports various input devices and has an extensive widget library. With the same codebase, you can target Windows, OS X, Linux, Android and iOS. All Kivy widgets are built with multitouch support. -Kivy in support take Buildozer which is a tool that automates the entire build process. It downloads and sets up all the prequisites for python-for-android, including the android SDK and NDK, then builds an apk that can be automatically pushed to the device. +Kivy in support take Buildozer which is a tool that automates the entire build process. It downloads and sets up all the prerequisite for python-for-android, including the android SDK and NDK, then builds an apk that can be automatically pushed to the device. Buildozer currently works only in Linux, and is an alpha release, but it already works well and can significantly simplify the apk build. diff --git a/checkdeps.py b/checkdeps.py index 45dc2fc9..c3dedc1d 100755 --- a/checkdeps.py +++ b/checkdeps.py @@ -1,6 +1,6 @@ #!/usr/bin/env python2 """ -Check dependendies and give recommendations about how to satisfy them +Check dependencies and give recommendations about how to satisfy them Limitations: diff --git a/docs/contribute.dir/develop.dir/opsec.rst b/docs/contribute.dir/develop.dir/opsec.rst index 87bf4f49..1af43668 100644 --- a/docs/contribute.dir/develop.dir/opsec.rst +++ b/docs/contribute.dir/develop.dir/opsec.rst @@ -21,12 +21,12 @@ If we are to make bold claims about protecting your privacy we should demonstrat - looking to audit - warrant canary -Digital foootprint +Digital footprint ------------------ Your internet use can reveal metadata you wouldn't expect. This can be connected with other information about you if you're not careful. - * Use separate addresses for different puprose + * Use separate addresses for different purposes * Don't make the same mistakes all the time * Your language use is unique. The more you type, the more you fingerprint yourself. The words you know and use often vs the words you don't know or use often. diff --git a/docs/contribute.dir/develop.dir/overview.rst b/docs/contribute.dir/develop.dir/overview.rst index 342c9dbb..8bbc8299 100644 --- a/docs/contribute.dir/develop.dir/overview.rst +++ b/docs/contribute.dir/develop.dir/overview.rst @@ -11,17 +11,17 @@ Bitmessage makes use of fabric_ to define tasks such as building documentation o Code style and linters ---------------------- -We aim to be PEP8 compliant but we recognise that we have a long way still to go. Currently we have style and lint exceptions specified at the most specific place we can. We are ignoring certain issues project-wide in order to avoid alert-blindess, avoid style and lint regressions and to allow continuous integration to hook into the output from the tools. While it is hoped that all new changes pass the checks, fixing some existing violations are mini-projects in themselves. Current thinking on ignorable violations is reflected in the options and comments in setup.cfg. Module and line-level lint warnings represent refactoring opportunities. +We aim to be PEP8 compliant but we recognize that we have a long way still to go. Currently we have style and lint exceptions specified at the most specific place we can. We are ignoring certain issues project-wide in order to avoid alert-blindness, avoid style and lint regressions and to allow continuous integration to hook into the output from the tools. While it is hoped that all new changes pass the checks, fixing some existing violations are mini-projects in themselves. Current thinking on ignorable violations is reflected in the options and comments in setup.cfg. Module and line-level lint warnings represent refactoring opportunities. Pull requests ------------- -There is a template at PULL_REQUEST_TEMPLATE.md that appears in the pull-request description. Please replace this text with something appropriate to your changes based off the ideas in the template. +There is a template at PULL_REQUEST_TEMPLATE.md that appears in the pull-request description. Please replace this text with something appropriate to your changes based on the ideas in the template. Bike-shedding ------------- -Beyond having well-documented, Pythonic code with static analysis tool checks, extensive test coverage and powerful devops tools, what else can we have? Without violating any linters there is room for making arbirary decisions solely for the sake of project consistency. These are the stuff of the pedant's PR comments. Rather than have such conversations in PR comments, we can lay out the result of discussion here. +Beyond having well-documented, Pythonic code with static analysis tool checks, extensive test coverage and powerful devops tools, what else can we have? Without violating any linters there is room for making arbitrary decisions solely for the sake of project consistency. These are the stuff of the pedant's PR comments. Rather than have such conversations in PR comments, we can lay out the result of discussion here. I'm putting up a strawman for each topic here, mostly based on my memory of reading related Stack Overflow articles etc. If contributors feel strongly (and we don't have anything better to do) then maybe we can convince each other to update this section. @@ -49,7 +49,7 @@ British vs American spelling Dependency graph ---------------- -These images are not very useful right now but the aim is to tweak the settings of one or more of them to be informative, and/or divide them up into smaller grapghs. +These images are not very useful right now but the aim is to tweak the settings of one or more of them to be informative, and/or divide them up into smaller graphs. To re-build them, run `fab build_docs:dep_graphs=true`. Note that the dot graph takes a lot of time. diff --git a/fabfile/README.md b/fabfile/README.md index 643aed8e..5e90147c 100644 --- a/fabfile/README.md +++ b/fabfile/README.md @@ -1,6 +1,6 @@ # Fabric -[Fabric](https://www.fabfile.org) is a Python library for performing devops tasks. You can thing of it a bit like a +[Fabric](https://www.fabfile.org) is a Python library for performing devops tasks. You can think of it a bit like a makefile on steroids for Python. Its api abstracts away the clunky way you would run shell commands in Python, check return values and manage stdio. Tasks may be targetted at particular hosts or group of hosts. @@ -46,7 +46,7 @@ Furthermore, you can use -- to run arbitrary shell commands rather than tasks: There are a number of advantages that should benefit us: - * Common tasks can be writen in Python and executed consistently + * Common tasks can be written in Python and executed consistently * Common tasks are now under source control * All developers can run the same commands, if the underlying command sequence for a task changes (after review, obv) the user does not have to care diff --git a/packages/README.md b/packages/README.md index ed2df3cc..2905ec20 100644 --- a/packages/README.md +++ b/packages/README.md @@ -15,7 +15,7 @@ OSX: https://github.com/Bitmessage/PyBitmessage/releases -Wors on OSX 10.7.5 or higher +Works on OSX 10.7.5 or higher Arch linux: From 21ae6cb9b00a17eb084ab994e5e6d4bed462f980 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Fri, 27 Dec 2019 19:42:57 +0530 Subject: [PATCH 15/31] curses fixes --- src/bitmessagecurses/__init__.py | 186 ++++++++++++++++--------------- 1 file changed, 96 insertions(+), 90 deletions(-) diff --git a/src/bitmessagecurses/__init__.py b/src/bitmessagecurses/__init__.py index 80dc3f14..d8daeef7 100644 --- a/src/bitmessagecurses/__init__.py +++ b/src/bitmessagecurses/__init__.py @@ -1,8 +1,6 @@ """ -src/bitmessagecurses/__init__.py -================================ +Bitmessage commandline interface """ - # Copyright (c) 2014 Luke Montalvo # This file adds a alternative commandline interface, feel free to critique and fork # @@ -20,21 +18,22 @@ import time from textwrap import fill from threading import Timer -from addresses import addBMIfNotPresent, decodeAddress -from bmconfigparser import BMConfigParser from dialog import Dialog -from helper_ackPayload import genAckPayload -from helper_sql import sqlExecute, sqlQuery -from inventory import Inventory import l10n import network.stats -from pyelliptic.openssl import OpenSSL import queues import shared import shutdown +from addresses import addBMIfNotPresent, decodeAddress +from bmconfigparser import BMConfigParser +from helper_ackPayload import genAckPayload +from helper_sql import sqlExecute, sqlQuery +from inventory import Inventory +# pylint: disable=global-statement -quit = False # pylint: disable=redefined-builtin + +quit_ = False menutab = 1 menu = ["Inbox", "Send", "Sent", "Your Identities", "Subscriptions", "Address Book", "Blacklist", "Network Status"] naptime = 100 @@ -61,26 +60,31 @@ bwtype = "black" BROADCAST_STR = "[Broadcast subscribers]" -class printLog: # pylint: disable=no-self-use, no-init, old-style-class +class printLog(object): """Printing logs""" + # pylint: disable=no-self-use def write(self, output): - # pylint: disable=global-statement + """Write logs""" global log log += output def flush(self): + """Flush logs""" pass -class errLog: # pylint: disable=no-self-use, no-init, old-style-class +class errLog(object): """Error logs""" + # pylint: disable=no-self-use + def write(self, output): - # pylint: disable=global-statement + """Write error logs""" global log log += "!" + output def flush(self): + """Flush error logs""" pass @@ -138,14 +142,15 @@ def scrollbox(d, text, height=None, width=None): def resetlookups(): """Reset the Inventory Lookups""" - global inventorydata # pylint: disable=global-statement + global inventorydata inventorydata = Inventory().numberOfInventoryLookupsPerformed Inventory().numberOfInventoryLookupsPerformed = 0 Timer(1, resetlookups, ()).start() -def drawtab(stdscr): # pylint: disable=too-many-branches, too-many-statements +def drawtab(stdscr): """Method for drawing different tabs""" + # pylint: disable=too-many-branches, too-many-statements if menutab in range(1, len(menu) + 1): if menutab == 1: # Inbox stdscr.addstr(3, 5, "To", curses.A_BOLD) @@ -282,12 +287,12 @@ def drawtab(stdscr): # pylint: disable=too-many-branches, too-many-statem stdscr.addstr(13, 6, "Log", curses.A_BOLD) n = log.count('\n') if n > 0: - l = log.split('\n') + lg = log.split('\n') if n > 512: - del l[:(n - 256)] + del lg[:(n - 256)] logpad.erase() - n = len(l) - for i, item in enumerate(l): + n = len(lg) + for i, item in enumerate(lg): a = 0 if item and item[0] == '!': a = curses.color_pair(1) @@ -314,7 +319,8 @@ def dialogreset(stdscr): # pylint: disable=too-many-branches, too-many-statements def handlech(c, stdscr): - # pylint: disable=redefined-outer-name, too-many-nested-blocks, too-many-locals, global-statement + """Handle character given on the command-line interface""" + # pylint: disable=redefined-outer-name, too-many-nested-blocks, too-many-locals if c != curses.ERR: global inboxcur, addrcur, sentcur, subcur, abookcur, blackcur if c in range(256): @@ -322,8 +328,8 @@ def handlech(c, stdscr): global menutab menutab = int(chr(c)) elif chr(c) == 'q': - global quit - quit = True + global quit_ + quit_ = True elif chr(c) == '\n': curses.curs_set(1) d = Dialog(dialog="dialog") @@ -363,10 +369,10 @@ def handlech(c, stdscr): inbox[inboxcur][7] = 1 else: scrollbox(d, unicode("Could not fetch message.")) - elif t == "2": # Mark unread + elif t == "2": # Mark unread sqlExecute("UPDATE inbox SET read=0 WHERE msgid=?", inbox[inboxcur][0]) inbox[inboxcur][7] = 0 - elif t == "3": # Reply + elif t == "3": # Reply curses.curs_set(1) m = inbox[inboxcur] fromaddr = m[4] @@ -375,7 +381,7 @@ def handlech(c, stdscr): if fromaddr == item[2] and item[3] != 0: ischan = True break - if not addresses[i][1]: # pylint: disable=undefined-loop-variable + if not addresses[i][1]: # pylint: disable=undefined-loop-variable scrollbox(d, unicode( "Sending address disabled, please either enable it" "or choose a different address.")) @@ -396,7 +402,7 @@ def handlech(c, stdscr): sendMessage(fromaddr, toaddr, ischan, subject, body, True) dialogreset(stdscr) - elif t == "4": # Add to Address Book + elif t == "4": # Add to Address Book addr = inbox[inboxcur][4] if addr not in [item[1] for i, item in enumerate(addrbook)]: r, t = d.inputbox("Label for address \"" + addr + "\"") @@ -409,7 +415,7 @@ def handlech(c, stdscr): addrbook.reverse() else: scrollbox(d, unicode("The selected address is already in the Address Book.")) - elif t == "5": # Save message + elif t == "5": # Save message set_background_title(d, "Save \"" + inbox[inboxcur][5] + "\" as text file") r, t = d.inputbox("Filename", init=inbox[inboxcur][5] + ".txt") if r == d.DIALOG_OK: @@ -418,12 +424,12 @@ def handlech(c, stdscr): if ret != []: for row in ret: msg, = row - fh = open(t, "a") # Open in append mode just in case + fh = open(t, "a") # Open in append mode just in case fh.write(msg) fh.close() else: scrollbox(d, unicode("Could not fetch message.")) - elif t == "6": # Move to trash + elif t == "6": # Move to trash sqlExecute("UPDATE inbox SET folder='trash' WHERE msgid=?", inbox[inboxcur][0]) del inbox[inboxcur] scrollbox(d, unicode( @@ -431,7 +437,7 @@ def handlech(c, stdscr): " \nbut the message is still on disk if you are desperate to recover it.")) elif menutab == 2: a = "" - if addresses[addrcur][3] != 0: # if current address is a chan + if addresses[addrcur][3] != 0: # if current address is a chan a = addresses[addrcur][2] sendMessage(addresses[addrcur][2], a) elif menutab == 3: @@ -467,7 +473,7 @@ def handlech(c, stdscr): scrollbox(d, unicode(ascii(msg)), 30, 80) else: scrollbox(d, unicode("Could not fetch message.")) - elif t == "2": # Move to trash + elif t == "2": # Move to trash sqlExecute( "UPDATE sent SET folder='trash' WHERE subject=? AND ackdata=?", sentbox[sentcur][4], @@ -495,7 +501,7 @@ def handlech(c, stdscr): ("6", "Delete"), ("7", "Special address behavior")]) if r == d.DIALOG_OK: - if t == "1": # Create new address + if t == "1": # Create new address set_background_title(d, "Create new address") scrollbox( d, unicode( @@ -598,12 +604,12 @@ def handlech(c, stdscr): str(passphrase), shorten)) else: scrollbox(d, unicode("Passphrases do not match")) - elif t == "2": # Send a message + elif t == "2": # Send a message a = "" - if addresses[addrcur][3] != 0: # if current address is a chan + if addresses[addrcur][3] != 0: # if current address is a chan a = addresses[addrcur][2] sendMessage(addresses[addrcur][2], a) - elif t == "3": # Rename address label + elif t == "3": # Rename address label a = addresses[addrcur][2] label = addresses[addrcur][0] r, t = d.inputbox("New address label", init=label) @@ -613,35 +619,35 @@ def handlech(c, stdscr): # Write config BMConfigParser().save() addresses[addrcur][0] = label - elif t == "4": # Enable address + elif t == "4": # Enable address a = addresses[addrcur][2] - BMConfigParser().set(a, "enabled", "true") # Set config + BMConfigParser().set(a, "enabled", "true") # Set config # Write config BMConfigParser().save() # Change color if BMConfigParser().safeGetBoolean(a, 'chan'): - addresses[addrcur][3] = 9 # orange + addresses[addrcur][3] = 9 # orange elif BMConfigParser().safeGetBoolean(a, 'mailinglist'): - addresses[addrcur][3] = 5 # magenta + addresses[addrcur][3] = 5 # magenta else: - addresses[addrcur][3] = 0 # black + addresses[addrcur][3] = 0 # black addresses[addrcur][1] = True - shared.reloadMyAddressHashes() # Reload address hashes - elif t == "5": # Disable address + shared.reloadMyAddressHashes() # Reload address hashes + elif t == "5": # Disable address a = addresses[addrcur][2] - BMConfigParser().set(a, "enabled", "false") # Set config - addresses[addrcur][3] = 8 # Set color to gray + BMConfigParser().set(a, "enabled", "false") # Set config + addresses[addrcur][3] = 8 # Set color to gray # Write config BMConfigParser().save() addresses[addrcur][1] = False - shared.reloadMyAddressHashes() # Reload address hashes - elif t == "6": # Delete address + shared.reloadMyAddressHashes() # Reload address hashes + elif t == "6": # Delete address r, t = d.inputbox("Type in \"I want to delete this address\"", width=50) if r == d.DIALOG_OK and t == "I want to delete this address": BMConfigParser().remove_section(addresses[addrcur][2]) BMConfigParser().save() del addresses[addrcur] - elif t == "7": # Special address behavior + elif t == "7": # Special address behavior a = addresses[addrcur][2] set_background_title(d, "Special address behavior") if BMConfigParser().safeGetBoolean(a, "chan"): @@ -658,9 +664,9 @@ def handlech(c, stdscr): if t == "1" and m: BMConfigParser().set(a, "mailinglist", "false") if addresses[addrcur][1]: - addresses[addrcur][3] = 0 # Set color to black + addresses[addrcur][3] = 0 # Set color to black else: - addresses[addrcur][3] = 8 # Set color to gray + addresses[addrcur][3] = 8 # Set color to gray elif t == "2" and m is False: try: mn = BMConfigParser().get(a, "mailinglistname") @@ -671,7 +677,7 @@ def handlech(c, stdscr): mn = t BMConfigParser().set(a, "mailinglist", "true") BMConfigParser().set(a, "mailinglistname", mn) - addresses[addrcur][3] = 6 # Set color to magenta + addresses[addrcur][3] = 6 # Set color to magenta # Write config BMConfigParser().save() elif menutab == 5: @@ -877,7 +883,7 @@ def sendMessage(sender="", recv="", broadcast=None, subject="", body="", reply=F 10, 60) if r != d.DIALOG_OK: - global menutab # pylint: disable=global-statement + global menutab menutab = 6 return recv = t @@ -890,7 +896,7 @@ def sendMessage(sender="", recv="", broadcast=None, subject="", body="", reply=F if r != d.DIALOG_OK: return broadcast = False - if t == "2": # Broadcast + if t == "2": # Broadcast broadcast = True if subject == "" or reply: r, t = d.inputbox("Message subject", width=60, init=subject) @@ -906,9 +912,9 @@ def sendMessage(sender="", recv="", broadcast=None, subject="", body="", reply=F if not broadcast: recvlist = [] - for i, item in enumerate(recv.replace(",", ";").split(";")): + for _, item in enumerate(recv.replace(",", ";").split(";")): recvlist.append(item.strip()) - list(set(recvlist)) # Remove exact duplicates + list(set(recvlist)) # Remove exact duplicates for addr in recvlist: if addr != "": # pylint: disable=redefined-outer-name @@ -968,16 +974,16 @@ def sendMessage(sender="", recv="", broadcast=None, subject="", body="", reply=F subject, body, ackdata, - int(time.time()), # sentTime (this will never change) - int(time.time()), # lastActionTime - 0, # sleepTill time. This will get set when the POW gets done. + int(time.time()), # sentTime (this will never change) + int(time.time()), # lastActionTime + 0, # sleepTill time. This will get set when the POW gets done. "msgqueued", - 0, # retryNumber + 0, # retryNumber "sent", - 2, # encodingType + 2, # encodingType BMConfigParser().getint('bitmessagesettings', 'ttl')) queues.workerQueue.put(("sendmessage", addr)) - else: # Broadcast + else: # Broadcast if recv == "": set_background_title(d, "Empty sender error") scrollbox(d, unicode("You must specify an address to send the message from.")) @@ -995,13 +1001,13 @@ def sendMessage(sender="", recv="", broadcast=None, subject="", body="", reply=F subject, body, ackdata, - int(time.time()), # sentTime (this will never change) - int(time.time()), # lastActionTime - 0, # sleepTill time. This will get set when the POW gets done. + int(time.time()), # sentTime (this will never change) + int(time.time()), # lastActionTime + 0, # sleepTill time. This will get set when the POW gets done. "broadcastqueued", - 0, # retryNumber - "sent", # folder - 2, # encodingType + 0, # retryNumber + "sent", # folder + 2, # encodingType BMConfigParser().getint('bitmessagesettings', 'ttl')) queues.workerQueue.put(('sendbroadcast', '')) @@ -1039,12 +1045,12 @@ def loadInbox(): fromlabel = "" if BMConfigParser().has_section(fromaddr): fromlabel = BMConfigParser().get(fromaddr, "label") - if fromlabel == "": # Check Address Book + if fromlabel == "": # Check Address Book qr = sqlQuery("SELECT label FROM addressbook WHERE address=?", fromaddr) if qr != []: for r in qr: fromlabel, = r - if fromlabel == "": # Check Subscriptions + if fromlabel == "": # Check Subscriptions qr = sqlQuery("SELECT label FROM subscriptions WHERE address=?", fromaddr) if qr != []: for r in qr: @@ -1170,7 +1176,7 @@ def loadSubscriptions(): def loadBlackWhiteList(): """load black/white list""" - global bwtype # pylint: disable=global-statement + global bwtype bwtype = BMConfigParser().get("bitmessagesettings", "blackwhitelist") if bwtype == "black": ret = sqlQuery("SELECT label, address, enabled FROM blacklist") @@ -1183,10 +1189,10 @@ def loadBlackWhiteList(): def runwrapper(): + """Main method""" sys.stdout = printlog # sys.stderr = errlog - # Load messages from database loadInbox() loadSent() loadAddrBook() @@ -1195,7 +1201,7 @@ def runwrapper(): stdscr = curses.initscr() - global logpad # pylint: disable=global-statement + global logpad logpad = curses.newpad(1024, curses.COLS) stdscr.nodelay(0) @@ -1207,26 +1213,27 @@ def runwrapper(): def run(stdscr): + """Main loop""" # Schedule inventory lookup data resetlookups() # Init color pairs if curses.has_colors(): - curses.init_pair(1, curses.COLOR_RED, curses.COLOR_BLACK) # red - curses.init_pair(2, curses.COLOR_GREEN, curses.COLOR_BLACK) # green - curses.init_pair(3, curses.COLOR_YELLOW, curses.COLOR_BLACK) # yellow - curses.init_pair(4, curses.COLOR_BLUE, curses.COLOR_BLACK) # blue - curses.init_pair(5, curses.COLOR_MAGENTA, curses.COLOR_BLACK) # magenta - curses.init_pair(6, curses.COLOR_CYAN, curses.COLOR_BLACK) # cyan - curses.init_pair(7, curses.COLOR_WHITE, curses.COLOR_BLACK) # white + curses.init_pair(1, curses.COLOR_RED, curses.COLOR_BLACK) # red + curses.init_pair(2, curses.COLOR_GREEN, curses.COLOR_BLACK) # green + curses.init_pair(3, curses.COLOR_YELLOW, curses.COLOR_BLACK) # yellow + curses.init_pair(4, curses.COLOR_BLUE, curses.COLOR_BLACK) # blue + curses.init_pair(5, curses.COLOR_MAGENTA, curses.COLOR_BLACK) # magenta + curses.init_pair(6, curses.COLOR_CYAN, curses.COLOR_BLACK) # cyan + curses.init_pair(7, curses.COLOR_WHITE, curses.COLOR_BLACK) # white if curses.can_change_color(): - curses.init_color(8, 500, 500, 500) # gray + curses.init_color(8, 500, 500, 500) # gray curses.init_pair(8, 8, 0) - curses.init_color(9, 844, 465, 0) # orange + curses.init_color(9, 844, 465, 0) # orange curses.init_pair(9, 9, 0) else: - curses.init_pair(8, curses.COLOR_WHITE, curses.COLOR_BLACK) # grayish - curses.init_pair(9, curses.COLOR_YELLOW, curses.COLOR_BLACK) # orangish + curses.init_pair(8, curses.COLOR_WHITE, curses.COLOR_BLACK) # grayish + curses.init_pair(9, curses.COLOR_YELLOW, curses.COLOR_BLACK) # orangish # Init list of address in 'Your Identities' tab configSections = BMConfigParser().addresses() @@ -1235,18 +1242,18 @@ def run(stdscr): addresses.append([BMConfigParser().get(addressInKeysFile, "label"), isEnabled, addressInKeysFile]) # Set address color if not isEnabled: - addresses[len(addresses) - 1].append(8) # gray + addresses[len(addresses) - 1].append(8) # gray elif BMConfigParser().safeGetBoolean(addressInKeysFile, 'chan'): - addresses[len(addresses) - 1].append(9) # orange + addresses[len(addresses) - 1].append(9) # orange elif BMConfigParser().safeGetBoolean(addressInKeysFile, 'mailinglist'): - addresses[len(addresses) - 1].append(5) # magenta + addresses[len(addresses) - 1].append(5) # magenta else: - addresses[len(addresses) - 1].append(0) # black + addresses[len(addresses) - 1].append(0) # black addresses.reverse() stdscr.clear() redraw(stdscr) - while quit is False: + while quit_ is False: drawtab(stdscr) handlech(stdscr.getch(), stdscr) @@ -1259,5 +1266,4 @@ def doShutdown(): shutdown.doCleanShutdown() sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ - - os._exit(0) # pylint: disable=protected-access + os._exit(0) # pylint: disable=protected-access From 4a369f70c1fe8907e84014dcd753a0d92efdc6e1 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Mon, 23 Dec 2019 15:19:03 +0530 Subject: [PATCH 16/31] formatting and docstring --- src/class_objectProcessor.py | 12 +++++++----- src/class_smtpDeliver.py | 3 +-- src/helper_sent.py | 3 ++- src/multiqueue.py | 4 ++-- src/proofofwork.py | 3 +-- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/class_objectProcessor.py b/src/class_objectProcessor.py index 2c741661..1a2f7751 100644 --- a/src/class_objectProcessor.py +++ b/src/class_objectProcessor.py @@ -149,11 +149,13 @@ class objectProcessor(threading.Thread): 'ackreceived', int(time.time()), data[readPosition:]) queues.UISignalQueue.put(( 'updateSentItemStatusByAckdata', - (data[readPosition:], - tr._translate( - "MainWindow", - "Acknowledgement of the message received %1" - ).arg(l10n.formatTimestamp())) + ( + data[readPosition:], + tr._translate( + "MainWindow", + "Acknowledgement of the message received %1" + ).arg(l10n.formatTimestamp()) + ) )) else: logger.debug('This object is not an acknowledgement bound for me.') diff --git a/src/class_smtpDeliver.py b/src/class_smtpDeliver.py index 58cd4631..14df14c9 100644 --- a/src/class_smtpDeliver.py +++ b/src/class_smtpDeliver.py @@ -1,6 +1,5 @@ """ -src/class_smtpDeliver.py -======================== +SMTP client thread for delivering emails """ # pylint: disable=unused-variable diff --git a/src/helper_sent.py b/src/helper_sent.py index 5a345fe7..bc3362e8 100644 --- a/src/helper_sent.py +++ b/src/helper_sent.py @@ -2,7 +2,8 @@ Insert values into sent table """ -from helper_sql import * +from helper_sql import sqlExecute + def insert(t): """Perform an insert into the `sent` table""" diff --git a/src/multiqueue.py b/src/multiqueue.py index 8c64d33d..d7c10847 100644 --- a/src/multiqueue.py +++ b/src/multiqueue.py @@ -1,6 +1,6 @@ """ -src/multiqueue.py -================= +A queue with multiple internal subqueues. +Elements are added into a random subqueue, and retrieval rotates """ import Queue diff --git a/src/proofofwork.py b/src/proofofwork.py index bb16951c..e43e0f02 100644 --- a/src/proofofwork.py +++ b/src/proofofwork.py @@ -1,7 +1,6 @@ # pylint: disable=too-many-branches,too-many-statements,protected-access """ -src/proofofwork.py -================== +Proof of work calculation """ import ctypes From b16515dc09de901ab2d153588d01a3bf776c5a2b Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Sat, 21 Dec 2019 15:13:03 +0530 Subject: [PATCH 17/31] arithmetic docstring and formatting --- src/pyelliptic/arithmetic.py | 28 +++++++++++++++++++++++----- src/pyelliptic/ecc.py | 3 +-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/pyelliptic/arithmetic.py b/src/pyelliptic/arithmetic.py index 95c85b93..83e634ad 100644 --- a/src/pyelliptic/arithmetic.py +++ b/src/pyelliptic/arithmetic.py @@ -1,5 +1,6 @@ -# pylint: disable=missing-docstring,too-many-function-args - +""" +Arithmetic Expressions +""" import hashlib import re @@ -11,6 +12,7 @@ G = (Gx, Gy) def inv(a, n): + """Inversion""" lm, hm = 1, 0 low, high = a % n, n while low > 1: @@ -21,6 +23,7 @@ def inv(a, n): def get_code_string(base): + """Returns string according to base value""" if base == 2: return '01' elif base == 10: @@ -36,6 +39,7 @@ def get_code_string(base): def encode(val, base, minlen=0): + """Returns the encoded string""" code_string = get_code_string(base) result = "" while val > 0: @@ -47,6 +51,7 @@ def encode(val, base, minlen=0): def decode(string, base): + """Returns the decoded string""" code_string = get_code_string(base) result = 0 if base == 16: @@ -59,10 +64,13 @@ def decode(string, base): def changebase(string, frm, to, minlen=0): + """Change base of the string""" return encode(decode(string, frm), to, minlen) def base10_add(a, b): + """Adding the numbers that are of base10""" + # pylint: disable=too-many-function-args if a is None: return b[0], b[1] if b is None: @@ -78,6 +86,7 @@ def base10_add(a, b): def base10_double(a): + """Double the numbers that are of base10""" if a is None: return None m = ((3 * a[0] * a[0] + A) * inv(2 * a[1], P)) % P @@ -87,6 +96,7 @@ def base10_double(a): def base10_multiply(a, n): + """Multiply the numbers that are of base10""" if n == 0: return G if n == 1: @@ -99,28 +109,35 @@ def base10_multiply(a, n): def hex_to_point(h): + """Converting hexadecimal to point value""" return (decode(h[2:66], 16), decode(h[66:], 16)) def point_to_hex(p): + """Converting point value to hexadecimal""" return '04' + encode(p[0], 16, 64) + encode(p[1], 16, 64) def multiply(privkey, pubkey): - return point_to_hex(base10_multiply(hex_to_point(pubkey), decode(privkey, 16))) + """Multiplying keys""" + return point_to_hex(base10_multiply( + hex_to_point(pubkey), decode(privkey, 16))) def privtopub(privkey): + """Converting key from private to public""" return point_to_hex(base10_multiply(G, decode(privkey, 16))) def add(p1, p2): + """Adding two public keys""" if len(p1) == 32: return encode(decode(p1, 16) + decode(p2, 16) % P, 16, 32) return point_to_hex(base10_add(hex_to_point(p1), hex_to_point(p2))) def hash_160(string): + """Hashed version of public key""" intermed = hashlib.sha256(string).digest() ripemd160 = hashlib.new('ripemd160') ripemd160.update(intermed) @@ -128,17 +145,18 @@ def hash_160(string): def dbl_sha256(string): + """Double hashing (SHA256)""" return hashlib.sha256(hashlib.sha256(string).digest()).digest() def bin_to_b58check(inp): + """Convert binary to base58""" inp_fmtd = '\x00' + inp leadingzbytes = len(re.match('^\x00*', inp_fmtd).group(0)) checksum = dbl_sha256(inp_fmtd)[:4] return '1' * leadingzbytes + changebase(inp_fmtd + checksum, 256, 58) -# Convert a public key (in hex) to a Bitcoin address - def pubkey_to_address(pubkey): + """Convert a public key (in hex) to a Bitcoin address""" return bin_to_b58check(hash_160(changebase(pubkey, 16, 256))) diff --git a/src/pyelliptic/ecc.py b/src/pyelliptic/ecc.py index 2de0bfe9..803a6de2 100644 --- a/src/pyelliptic/ecc.py +++ b/src/pyelliptic/ecc.py @@ -1,8 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -src/pyelliptic/ecc.py -===================== +Asymmetric cryptography using elliptic curves """ # pylint: disable=protected-access From 36c24cc09a44854203e9d3758b8f914a10d63876 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Sat, 21 Dec 2019 15:13:29 +0530 Subject: [PATCH 18/31] cipher quality fixes --- src/pyelliptic/cipher.py | 10 ++++------ src/pyelliptic/ecc.py | 19 +++++++++---------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/pyelliptic/cipher.py b/src/pyelliptic/cipher.py index d02b743a..4057e169 100644 --- a/src/pyelliptic/cipher.py +++ b/src/pyelliptic/cipher.py @@ -1,10 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -src/pyelliptic/cipher.py -======================== +Symmetric Encryption """ - # Copyright (C) 2011 Yann GUIBET # See LICENSE for details. @@ -14,7 +12,7 @@ from openssl import OpenSSL # pylint: disable=redefined-builtin class Cipher(object): """ - Symmetric encryption + Main class for encryption import pyelliptic iv = pyelliptic.Cipher.gen_IV('aes-256-cfb') @@ -67,7 +65,7 @@ class Cipher(object): if OpenSSL.EVP_CipherUpdate(self.ctx, OpenSSL.byref(buffer), OpenSSL.byref(i), inp, len(input)) == 0: raise Exception("[OpenSSL] EVP_CipherUpdate FAIL ...") - return buffer.raw[0:i.value] # pylint: disable=invalid-slice-index + return buffer.raw[0:i.value] # pylint: disable=invalid-slice-index def final(self): """Returning the final value""" @@ -76,7 +74,7 @@ class Cipher(object): if (OpenSSL.EVP_CipherFinal_ex(self.ctx, OpenSSL.byref(buffer), OpenSSL.byref(i))) == 0: raise Exception("[OpenSSL] EVP_CipherFinal_ex FAIL ...") - return buffer.raw[0:i.value] # pylint: disable=invalid-slice-index + return buffer.raw[0:i.value] # pylint: disable=invalid-slice-index def ciphering(self, input): """ diff --git a/src/pyelliptic/ecc.py b/src/pyelliptic/ecc.py index 803a6de2..a7f5a6b7 100644 --- a/src/pyelliptic/ecc.py +++ b/src/pyelliptic/ecc.py @@ -3,8 +3,7 @@ """ Asymmetric cryptography using elliptic curves """ -# pylint: disable=protected-access - +# pylint: disable=protected-access, too-many-branches, too-many-locals # Copyright (C) 2011 Yann GUIBET # See LICENSE for details. @@ -172,7 +171,8 @@ class ECC(object): if OpenSSL.EC_POINT_get_affine_coordinates_GFp( group, pub_key, pub_key_x, pub_key_y, 0) == 0: - raise Exception("[OpenSSL] EC_POINT_get_affine_coordinates_GFp FAIL ...") + raise Exception( + "[OpenSSL] EC_POINT_get_affine_coordinates_GFp FAIL ...") privkey = OpenSSL.malloc(0, OpenSSL.BN_num_bytes(priv_key)) pubkeyx = OpenSSL.malloc(0, OpenSSL.BN_num_bytes(pub_key_x)) @@ -275,7 +275,6 @@ class ECC(object): def raw_check_key(self, privkey, pubkey_x, pubkey_y, curve=None): """Check key validity, key is supplied as binary data""" - # pylint: disable=too-many-branches if curve is None: curve = self.curve elif isinstance(curve, str): @@ -323,7 +322,6 @@ class ECC(object): """ Sign the input with ECDSA method and returns the signature """ - # pylint: disable=too-many-branches,too-many-locals try: size = len(inputb) buff = OpenSSL.malloc(inputb, size) @@ -393,7 +391,6 @@ class ECC(object): Verify the signature with the input and the local public key. Returns a boolean """ - # pylint: disable=too-many-branches try: bsig = OpenSSL.malloc(sig, len(sig)) binputb = OpenSSL.malloc(inputb, len(inputb)) @@ -436,10 +433,13 @@ class ECC(object): 0, digest, dgst_len.contents, bsig, len(sig), key) if ret == -1: - return False # Fail to Check + # Fail to Check + return False if ret == 0: - return False # Bad signature ! - return True # Good + # Bad signature ! + return False + # Good + return True finally: OpenSSL.EC_KEY_free(key) @@ -487,7 +487,6 @@ class ECC(object): """ Decrypt data with ECIES method using the local private key """ - # pylint: disable=too-many-locals blocksize = OpenSSL.get_cipher(ciphername).get_blocksize() iv = data[:blocksize] i = blocksize From 814aae5166b858f85af11b51ae27b666b24c7389 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Sat, 21 Dec 2019 15:14:07 +0530 Subject: [PATCH 19/31] eccblind quality fixes --- src/pyelliptic/eccblind.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pyelliptic/eccblind.py b/src/pyelliptic/eccblind.py index 2ca2581a..5b9b045e 100644 --- a/src/pyelliptic/eccblind.py +++ b/src/pyelliptic/eccblind.py @@ -1,6 +1,7 @@ #!/usr/bin/env python """ -ECC blind signature functionality based on "An Efficient Blind Signature Scheme +ECC blind signature functionality based on +"An Efficient Blind Signature Scheme Based on the Elliptic CurveDiscrete Logarithm Problem" by Morteza Nikooghadama and Ali Zakerolhosseini , http://www.isecure-journal.com/article_39171_47f9ec605dd3918c2793565ec21fcd7a.pdf @@ -8,7 +9,6 @@ http://www.isecure-journal.com/article_39171_47f9ec605dd3918c2793565ec21fcd7a.pd # variable names are based on the math in the paper, so they don't conform # to PEP8 -# pylint: disable=invalid-name from .openssl import OpenSSL @@ -72,8 +72,7 @@ class ECCBlind(object): # pylint: disable=too-many-instance-attributes # F = (x0, y0) x0 = OpenSSL.BN_new() y0 = OpenSSL.BN_new() - OpenSSL.EC_POINT_get_affine_coordinates_GFp(group, F, x0, y0, - ctx) + OpenSSL.EC_POINT_get_affine_coordinates_GFp(group, F, x0, y0, ctx) return x0 def __init__(self, curve="secp256k1", pubkey=None): @@ -82,7 +81,8 @@ class ECCBlind(object): # pylint: disable=too-many-instance-attributes if pubkey: self.group, self.G, self.n, self.Q = pubkey else: - self.group = OpenSSL.EC_GROUP_new_by_curve_name(OpenSSL.get_curve(curve)) + self.group = OpenSSL.EC_GROUP_new_by_curve_name( + OpenSSL.get_curve(curve)) # Order n self.n = OpenSSL.BN_new() OpenSSL.EC_GROUP_get_order(self.group, self.n, self.ctx) From 8659c5313d54ea8c1f899f8a4e0bc8c02ae5dab3 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Sat, 21 Dec 2019 15:14:31 +0530 Subject: [PATCH 20/31] openssl pylint issue fixes --- src/pyelliptic/openssl.py | 127 +++++++++++++++++++++++--------------- 1 file changed, 78 insertions(+), 49 deletions(-) diff --git a/src/pyelliptic/openssl.py b/src/pyelliptic/openssl.py index 152a780c..f769f0e3 100644 --- a/src/pyelliptic/openssl.py +++ b/src/pyelliptic/openssl.py @@ -2,22 +2,20 @@ # See LICENSE for details. # # Software slightly changed by Jonathan Warren -# pylint: disable=protected-access """ This module loads openssl libs with ctypes and incapsulates needed openssl functionality in class _OpenSSL. """ - +# pylint: disable=protected-access import sys import ctypes OpenSSL = None -class CipherName: +class CipherName(object): """Class returns cipher name, pointer and blocksize""" - # pylint: disable=old-style-class def __init__(self, name, pointer, blocksize): self._name = name self._pointer = pointer @@ -73,11 +71,11 @@ def get_version(library): return (version, hexversion, cflags) -class _OpenSSL: +class _OpenSSL(object): """ Wrapper for OpenSSL using ctypes """ - # pylint: disable=too-many-statements, too-many-instance-attributes, old-style-class + # pylint: disable=too-many-statements, too-many-instance-attributes def __init__(self, library): """ Build the wrapper @@ -140,7 +138,8 @@ class _OpenSSL: self.EC_KEY_get0_group.restype = ctypes.c_void_p self.EC_KEY_get0_group.argtypes = [ctypes.c_void_p] - self.EC_POINT_get_affine_coordinates_GFp = self._lib.EC_POINT_get_affine_coordinates_GFp + self.EC_POINT_get_affine_coordinates_GFp = \ + self._lib.EC_POINT_get_affine_coordinates_GFp self.EC_POINT_get_affine_coordinates_GFp.restype = ctypes.c_int self.EC_POINT_get_affine_coordinates_GFp.argtypes = [ctypes.c_void_p, ctypes.c_void_p, @@ -163,7 +162,8 @@ class _OpenSSL: self.EC_KEY_set_group.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - self.EC_POINT_set_affine_coordinates_GFp = self._lib.EC_POINT_set_affine_coordinates_GFp + self.EC_POINT_set_affine_coordinates_GFp = \ + self._lib.EC_POINT_set_affine_coordinates_GFp self.EC_POINT_set_affine_coordinates_GFp.restype = ctypes.c_int self.EC_POINT_set_affine_coordinates_GFp.argtypes = [ctypes.c_void_p, ctypes.c_void_p, @@ -297,7 +297,8 @@ class _OpenSSL: self.EVP_CipherUpdate = self._lib.EVP_CipherUpdate self.EVP_CipherUpdate.restype = ctypes.c_int self.EVP_CipherUpdate.argtypes = [ctypes.c_void_p, - ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int] + ctypes.c_void_p, ctypes.c_void_p, + ctypes.c_void_p, ctypes.c_int] self.EVP_CipherFinal_ex = self._lib.EVP_CipherFinal_ex self.EVP_CipherFinal_ex.restype = ctypes.c_int @@ -330,12 +331,14 @@ class _OpenSSL: self.ECDSA_sign = self._lib.ECDSA_sign self.ECDSA_sign.restype = ctypes.c_int self.ECDSA_sign.argtypes = [ctypes.c_int, ctypes.c_void_p, - ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] + ctypes.c_int, ctypes.c_void_p, + ctypes.c_void_p, ctypes.c_void_p] self.ECDSA_verify = self._lib.ECDSA_verify self.ECDSA_verify.restype = ctypes.c_int self.ECDSA_verify.argtypes = [ctypes.c_int, ctypes.c_void_p, - ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p] + ctypes.c_int, ctypes.c_void_p, + ctypes.c_int, ctypes.c_void_p] if self._hexversion >= 0x10100000 and not self._libreSSL: self.EVP_MD_CTX_new = self._lib.EVP_MD_CTX_new @@ -393,7 +396,8 @@ class _OpenSSL: self.HMAC = self._lib.HMAC self.HMAC.restype = ctypes.c_void_p self.HMAC.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, - ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] + ctypes.c_void_p, ctypes.c_int, + ctypes.c_void_p, ctypes.c_void_p] try: self.PKCS5_PBKDF2_HMAC = self._lib.PKCS5_PBKDF2_HMAC @@ -530,17 +534,29 @@ class _OpenSSL: def _set_ciphers(self): self.cipher_algo = { - 'aes-128-cbc': CipherName('aes-128-cbc', self.EVP_aes_128_cbc, 16), - 'aes-256-cbc': CipherName('aes-256-cbc', self.EVP_aes_256_cbc, 16), - 'aes-128-cfb': CipherName('aes-128-cfb', self.EVP_aes_128_cfb128, 16), - 'aes-256-cfb': CipherName('aes-256-cfb', self.EVP_aes_256_cfb128, 16), - 'aes-128-ofb': CipherName('aes-128-ofb', self._lib.EVP_aes_128_ofb, 16), - 'aes-256-ofb': CipherName('aes-256-ofb', self._lib.EVP_aes_256_ofb, 16), - # 'aes-128-ctr': CipherName('aes-128-ctr', self._lib.EVP_aes_128_ctr, 16), - # 'aes-256-ctr': CipherName('aes-256-ctr', self._lib.EVP_aes_256_ctr, 16), - 'bf-cfb': CipherName('bf-cfb', self.EVP_bf_cfb64, 8), - 'bf-cbc': CipherName('bf-cbc', self.EVP_bf_cbc, 8), - 'rc4': CipherName('rc4', self.EVP_rc4, 128), # 128 is the initialisation size not block size + 'aes-128-cbc': CipherName( + 'aes-128-cbc', self.EVP_aes_128_cbc, 16), + 'aes-256-cbc': CipherName( + 'aes-256-cbc', self.EVP_aes_256_cbc, 16), + 'aes-128-cfb': CipherName( + 'aes-128-cfb', self.EVP_aes_128_cfb128, 16), + 'aes-256-cfb': CipherName( + 'aes-256-cfb', self.EVP_aes_256_cfb128, 16), + 'aes-128-ofb': CipherName( + 'aes-128-ofb', self._lib.EVP_aes_128_ofb, 16), + 'aes-256-ofb': CipherName( + 'aes-256-ofb', self._lib.EVP_aes_256_ofb, 16), + # 'aes-128-ctr': CipherName( + # 'aes-128-ctr', self._lib.EVP_aes_128_ctr, 16), + # 'aes-256-ctr': CipherName( + # 'aes-256-ctr', self._lib.EVP_aes_256_ctr, 16), + 'bf-cfb': CipherName( + 'bf-cfb', self.EVP_bf_cfb64, 8), + 'bf-cbc': CipherName( + 'bf-cbc', self.EVP_bf_cbc, 8), + # 128 is the initialisation size not block size + 'rc4': CipherName( + 'rc4', self.EVP_rc4, 128), } def _set_curves(self): @@ -600,14 +616,13 @@ class _OpenSSL: raise Exception("Unknown curve") return self.curves[name] - def get_curve_by_id(self, id): + def get_curve_by_id(self, id_): """ returns the name of a elliptic curve with his id """ - # pylint: disable=redefined-builtin res = None for i in self.curves: - if self.curves[i] == id: + if self.curves[i] == id_: res = i break if res is None: @@ -618,32 +633,31 @@ class _OpenSSL: """ OpenSSL random function """ - # pylint: disable=redefined-builtin - buffer = self.malloc(0, size) - # This pyelliptic library, by default, didn't check the return value of RAND_bytes. It is - # evidently possible that it returned an error and not-actually-random data. However, in - # tests on various operating systems, while generating hundreds of gigabytes of random - # strings of various sizes I could not get an error to occur. Also Bitcoin doesn't check - # the return value of RAND_bytes either. + buffer_ = self.malloc(0, size) + # This pyelliptic library, by default, didn't check the return value + # of RAND_bytes. It is evidently possible that it returned an error + # and not-actually-random data. However, in tests on various + # operating systems, while generating hundreds of gigabytes of random + # strings of various sizes I could not get an error to occur. + # Also Bitcoin doesn't check the return value of RAND_bytes either. # Fixed in Bitmessage version 0.4.2 (in source code on 2013-10-13) - while self.RAND_bytes(buffer, size) != 1: + while self.RAND_bytes(buffer_, size) != 1: import time time.sleep(1) - return buffer.raw + return buffer_.raw def malloc(self, data, size): """ returns a create_string_buffer (ctypes) """ - # pylint: disable=redefined-builtin - buffer = None + buffer_ = None if data != 0: if sys.version_info.major == 3 and isinstance(data, type('')): data = data.encode() - buffer = self.create_string_buffer(data, size) + buffer_ = self.create_string_buffer(data, size) else: - buffer = self.create_string_buffer(size) - return buffer + buffer_ = self.create_string_buffer(size) + return buffer_ def loadOpenSSL(): @@ -657,12 +671,24 @@ def loadOpenSSL(): if getattr(sys, 'frozen', None): if 'darwin' in sys.platform: libdir.extend([ - path.join(environ['RESOURCEPATH'], '..', 'Frameworks', 'libcrypto.dylib'), - path.join(environ['RESOURCEPATH'], '..', 'Frameworks', 'libcrypto.1.1.0.dylib'), - path.join(environ['RESOURCEPATH'], '..', 'Frameworks', 'libcrypto.1.0.2.dylib'), - path.join(environ['RESOURCEPATH'], '..', 'Frameworks', 'libcrypto.1.0.1.dylib'), - path.join(environ['RESOURCEPATH'], '..', 'Frameworks', 'libcrypto.1.0.0.dylib'), - path.join(environ['RESOURCEPATH'], '..', 'Frameworks', 'libcrypto.0.9.8.dylib'), + path.join( + environ['RESOURCEPATH'], '..', + 'Frameworks', 'libcrypto.dylib'), + path.join( + environ['RESOURCEPATH'], '..', + 'Frameworks', 'libcrypto.1.1.0.dylib'), + path.join( + environ['RESOURCEPATH'], '..', + 'Frameworks', 'libcrypto.1.0.2.dylib'), + path.join( + environ['RESOURCEPATH'], '..', + 'Frameworks', 'libcrypto.1.0.1.dylib'), + path.join( + environ['RESOURCEPATH'], '..', + 'Frameworks', 'libcrypto.1.0.0.dylib'), + path.join( + environ['RESOURCEPATH'], '..', + 'Frameworks', 'libcrypto.0.9.8.dylib'), ]) elif 'win32' in sys.platform or 'win64' in sys.platform: libdir.append(path.join(sys._MEIPASS, 'libeay32.dll')) @@ -682,7 +708,8 @@ def loadOpenSSL(): path.join(sys._MEIPASS, 'libssl.so.0.9.8'), ]) if 'darwin' in sys.platform: - libdir.extend(['libcrypto.dylib', '/usr/local/opt/openssl/lib/libcrypto.dylib']) + libdir.extend([ + 'libcrypto.dylib', '/usr/local/opt/openssl/lib/libcrypto.dylib']) elif 'win32' in sys.platform or 'win64' in sys.platform: libdir.append('libeay32.dll') else: @@ -690,7 +717,8 @@ def loadOpenSSL(): libdir.append('libssl.so') libdir.append('libcrypto.so.1.0.0') libdir.append('libssl.so.1.0.0') - if 'linux' in sys.platform or 'darwin' in sys.platform or 'bsd' in sys.platform: + if 'linux' in sys.platform or 'darwin' in sys.platform \ + or 'bsd' in sys.platform: libdir.append(find_library('ssl')) elif 'win32' in sys.platform or 'win64' in sys.platform: libdir.append(find_library('libeay32')) @@ -700,7 +728,8 @@ def loadOpenSSL(): return except: pass - raise Exception("Couldn't find and load the OpenSSL library. You must install it.") + raise Exception( + "Couldn't find and load the OpenSSL library. You must install it.") loadOpenSSL() From 624d96fbb943d18161974277d2f6e865353d4bec Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Sat, 21 Dec 2019 12:43:52 +0530 Subject: [PATCH 21/31] indicator_libmessaging docstring and formatting --- src/plugins/indicator_libmessaging.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/indicator_libmessaging.py b/src/plugins/indicator_libmessaging.py index ab2e833e..60bf5e7e 100644 --- a/src/plugins/indicator_libmessaging.py +++ b/src/plugins/indicator_libmessaging.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """ -src/plugins/indicator_libmessaging.py -===================================== +Indicator plugin using libmessaging """ import gi @@ -37,7 +36,7 @@ class IndicatorLibmessaging(object): if self.app: self.app.unregister() - def activate(self, app, source): # pylint: disable=unused-argument + def activate(self, app, source): # pylint: disable=unused-argument """Activate the libmessaging indicator plugin""" self.form.appIndicatorInbox( self.new_message_item if source == 'messages' From 7b0bf845857a309dddd4ac22535ebfb57c47e090 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Sat, 21 Dec 2019 12:44:13 +0530 Subject: [PATCH 22/31] menu_qrcode docstring and formatting --- src/plugins/menu_qrcode.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/plugins/menu_qrcode.py b/src/plugins/menu_qrcode.py index 2e65bd88..63644db6 100644 --- a/src/plugins/menu_qrcode.py +++ b/src/plugins/menu_qrcode.py @@ -1,8 +1,5 @@ # -*- coding: utf-8 -*- """ -src/plugins/menu_qrcode.py -========================== - A menu plugin showing QR-Code for bitmessage address in modal dialog. """ @@ -15,10 +12,11 @@ from pybitmessage.tr import _translate # http://stackoverflow.com/questions/20452486 -class Image(qrcode.image.base.BaseImage): # pylint: disable=abstract-method +class Image(qrcode.image.base.BaseImage): # pylint: disable=abstract-method """Image output class for qrcode using QPainter""" - def __init__(self, border, width, box_size): # pylint: disable=super-init-not-called + def __init__(self, border, width, box_size): + # pylint: disable=super-init-not-called self.border = border self.width = width self.box_size = box_size From ea50485de25b86bc430cc4e696929cba79e12863 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Sat, 21 Dec 2019 12:44:33 +0530 Subject: [PATCH 23/31] notification_notify2 pylint fixes --- src/plugins/notification_notify2.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/notification_notify2.py b/src/plugins/notification_notify2.py index b4cd045d..84ecbdde 100644 --- a/src/plugins/notification_notify2.py +++ b/src/plugins/notification_notify2.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """ -src/plugins/notification_notify2.py -=================================== +Notification plugin using notify2 """ import gi @@ -11,7 +10,7 @@ from gi.repository import Notify Notify.init('pybitmessage') -def connect_plugin(title, subtitle, category, label, icon): +def connect_plugin(title, subtitle, category, _, icon): """Plugin for notify2""" if not icon: icon = 'mail-message-new' if category == 2 else 'pybitmessage' From e24f4de40e858a9828714fbf9b0dc6e082c697ef Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Sat, 21 Dec 2019 12:45:29 +0530 Subject: [PATCH 24/31] proxyconfig_stem quality fixes --- src/plugins/proxyconfig_stem.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/plugins/proxyconfig_stem.py b/src/plugins/proxyconfig_stem.py index 494519f2..bc760ef1 100644 --- a/src/plugins/proxyconfig_stem.py +++ b/src/plugins/proxyconfig_stem.py @@ -22,9 +22,8 @@ import stem.process import stem.version -class DebugLogger(object): +class DebugLogger(object): # pylint: disable=too-few-public-methods """Safe logger wrapper for tor and plugin's logs""" - # pylint: disable=too-few-public-methods def __init__(self): self._logger = logging.getLogger('default') self._levels = { @@ -108,7 +107,8 @@ def connect_plugin(config): # pylint: disable=too-many-branches onionhostname = config.safeGet('bitmessagesettings', 'onionhostname') onionkey = config.safeGet(onionhostname, 'privsigningkey') if onionhostname and not onionkey: - logwrite('The hidden service found in config ): %s' % onionhostname) + logwrite('The hidden service found in config ): %s' % + onionhostname) onionkeytype = config.safeGet(onionhostname, 'keytype') response = controller.create_ephemeral_hidden_service( @@ -124,7 +124,8 @@ def connect_plugin(config): # pylint: disable=too-many-branches if not onionkey: logwrite('Started hidden service %s.onion' % response.service_id) - # only save new service keys if onionhostname was not set previously + # only save new service keys + # if onionhostname was not set previously if not onionhostname: onionhostname = response.service_id + '.onion' config.set( From 208090ce5d1d9fe13b7635d1404343ef2a06eede Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Sat, 21 Dec 2019 12:45:52 +0530 Subject: [PATCH 25/31] sound_canberra docstring and formatting --- src/plugins/sound_canberra.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/sound_canberra.py b/src/plugins/sound_canberra.py index dbb4baed..20bd054d 100644 --- a/src/plugins/sound_canberra.py +++ b/src/plugins/sound_canberra.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """ -src/plugins/proxyconfig_stem.py -=================================== +Sound theme plugin using pycanberra """ from pybitmessage.bitmessageqt import sound @@ -18,7 +17,7 @@ _theme = { } -def connect_plugin(category, label=None): # pylint: disable=unused-argument +def connect_plugin(category, label=None): # pylint: disable=unused-argument """This function implements the entry point.""" try: _canberra.play(0, pycanberra.CA_PROP_EVENT_ID, _theme[category], None) From 8338a9ee74f4579611bc2fc83998263f7e350cdd Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Sat, 21 Dec 2019 12:46:12 +0530 Subject: [PATCH 26/31] sound_gstreamer docstring fixes --- src/plugins/sound_gstreamer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/plugins/sound_gstreamer.py b/src/plugins/sound_gstreamer.py index 32a0aa65..8f3606dd 100644 --- a/src/plugins/sound_gstreamer.py +++ b/src/plugins/sound_gstreamer.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """ -src/plugins/sound_gstreamer.py -=================================== +Sound notification plugin using gstreamer """ import gi gi.require_version('Gst', '1.0') From a31d6c8422aff2266416cc8fef1697f2120d463a Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Sat, 21 Dec 2019 12:46:31 +0530 Subject: [PATCH 27/31] sound_playfile quality fixes --- src/plugins/sound_playfile.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/sound_playfile.py b/src/plugins/sound_playfile.py index 6396c319..e36d9922 100644 --- a/src/plugins/sound_playfile.py +++ b/src/plugins/sound_playfile.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """ -src/plugins/sound_playfile.py -=================================== +Sound notification plugin using external executable or winsound (on Windows) """ try: @@ -23,7 +22,7 @@ except ImportError: def connect_plugin(sound_file): """This function implements the entry point.""" - global play_cmd # pylint: disable=global-statement + global play_cmd # pylint: disable=global-statement ext = os.path.splitext(sound_file)[-1] try: From 81872c7f2f8f092190f2bbd3a02f9c8065343e40 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Thu, 19 Dec 2019 16:54:53 +0530 Subject: [PATCH 28/31] network code quality fixes --- src/network/networkthread.py | 5 ++- src/network/objectracker.py | 27 ++++++++----- src/network/proxy.py | 6 +-- src/network/randomtrackingdict.py | 30 +++++++------- src/network/receivequeuethread.py | 11 +++++- src/network/socks4a.py | 3 +- src/network/socks5.py | 7 ++-- src/network/stats.py | 10 +++-- src/network/tcp.py | 25 ++++++------ src/network/tls.py | 65 +++++++++++++++++++------------ src/network/udp.py | 14 +++---- 11 files changed, 117 insertions(+), 86 deletions(-) diff --git a/src/network/networkthread.py b/src/network/networkthread.py index ba560906..61ff6c09 100644 --- a/src/network/networkthread.py +++ b/src/network/networkthread.py @@ -1,3 +1,6 @@ +""" +A thread to handle network concerns +""" import network.asyncore_pollchoose as asyncore import state from network.connectionpool import BMConnectionPool @@ -6,7 +9,7 @@ from threads import StoppableThread class BMNetworkThread(StoppableThread): - """A thread to handle network concerns""" + """Main network thread""" name = "Asyncore" def run(self): diff --git a/src/network/objectracker.py b/src/network/objectracker.py index b97aee46..ca29c023 100644 --- a/src/network/objectracker.py +++ b/src/network/objectracker.py @@ -1,6 +1,5 @@ """ -src/network/objectracker.py -=========================== +Module for tracking objects """ import time from threading import RLock @@ -50,15 +49,18 @@ class ObjectTracker(object): """Init bloom filter for tracking. WIP.""" if haveBloom: # lock? - self.invBloom = BloomFilter(capacity=ObjectTracker.invInitialCapacity, - error_rate=ObjectTracker.invErrorRate) + self.invBloom = BloomFilter( + capacity=ObjectTracker.invInitialCapacity, + error_rate=ObjectTracker.invErrorRate) def initAddrBloom(self): - """Init bloom filter for tracking addrs, WIP. This either needs to be moved to addrthread.py or removed.""" + """Init bloom filter for tracking addrs, WIP. + This either needs to be moved to addrthread.py or removed.""" if haveBloom: # lock? - self.addrBloom = BloomFilter(capacity=ObjectTracker.invInitialCapacity, - error_rate=ObjectTracker.invErrorRate) + self.addrBloom = BloomFilter( + capacity=ObjectTracker.invInitialCapacity, + error_rate=ObjectTracker.invErrorRate) def clean(self): """Clean up tracking to prevent memory bloat""" @@ -71,7 +73,10 @@ class ObjectTracker(object): # release memory deadline = time.time() - ObjectTracker.trackingExpires with self.objectsNewToThemLock: - self.objectsNewToThem = {k: v for k, v in self.objectsNewToThem.iteritems() if v >= deadline} + self.objectsNewToThem = { + k: v + for k, v in self.objectsNewToThem.iteritems() + if v >= deadline} self.lastCleaned = time.time() def hasObj(self, hashid): @@ -102,10 +107,12 @@ class ObjectTracker(object): del i.objectsNewToMe[hashid] except KeyError: if streamNumber in i.streams and ( - not Dandelion().hasHash(hashid) or Dandelion().objectChildStem(hashid) == i): + not Dandelion().hasHash(hashid) or + Dandelion().objectChildStem(hashid) == i): with i.objectsNewToThemLock: i.objectsNewToThem[hashid] = time.time() - # update stream number, which we didn't have when we just received the dinv + # update stream number, + # which we didn't have when we just received the dinv # also resets expiration of the stem mode Dandelion().setHashStream(hashid, streamNumber) diff --git a/src/network/proxy.py b/src/network/proxy.py index e0bb5e78..38676d66 100644 --- a/src/network/proxy.py +++ b/src/network/proxy.py @@ -1,6 +1,5 @@ """ -src/network/proxy.py -==================== +Set proxy if avaiable otherwise exception """ # pylint: disable=protected-access import logging @@ -122,8 +121,7 @@ class Proxy(AdvancedDispatcher): BMConfigParser().safeGet( "bitmessagesettings", "socksusername"), BMConfigParser().safeGet( - "bitmessagesettings", "sockspassword") - ) + "bitmessagesettings", "sockspassword")) else: self.auth = None self.connect( diff --git a/src/network/randomtrackingdict.py b/src/network/randomtrackingdict.py index 6c3300ab..e87bf156 100644 --- a/src/network/randomtrackingdict.py +++ b/src/network/randomtrackingdict.py @@ -1,8 +1,6 @@ """ -src/randomtrackingdict.py -========================= +Track randomize ordered dict """ - import random from threading import RLock from time import time @@ -14,10 +12,12 @@ class RandomTrackingDict(object): """ Dict with randomised order and tracking. - Keeps a track of how many items have been requested from the dict, and timeouts. Resets after all objects have been - retrieved and timed out. The main purpose of this isn't as much putting related code together as performance - optimisation and anonymisation of downloading of objects from other peers. If done using a standard dict or array, - it takes too much CPU (and looks convoluted). Randomisation helps with anonymity. + Keeps a track of how many items have been requested from the dict, + and timeouts. Resets after all objects have been retrieved and timed out. + The main purpose of this isn't as much putting related code together + as performance optimisation and anonymisation of downloading of objects + from other peers. If done using a standard dict or array, it takes + too much CPU (and looks convoluted). Randomisation helps with anonymity. """ # pylint: disable=too-many-instance-attributes maxPending = 10 @@ -85,13 +85,14 @@ class RandomTrackingDict(object): def setMaxPending(self, maxPending): """ - Sets maximum number of objects that can be retrieved from the class simultaneously as long as there is no - timeout + Sets maximum number of objects that can be retrieved from the class + simultaneously as long as there is no timeout """ self.maxPending = maxPending def setPendingTimeout(self, pendingTimeout): - """Sets how long to wait for a timeout if max pending is reached (or all objects have been retrieved)""" + """Sets how long to wait for a timeout if max pending is reached + (or all objects have been retrieved)""" self.pendingTimeout = pendingTimeout def setLastObject(self): @@ -99,7 +100,8 @@ class RandomTrackingDict(object): self.lastObject = time() def randomKeys(self, count=1): - """Retrieve count random keys from the dict that haven't already been retrieved""" + """Retrieve count random keys from the dict + that haven't already been retrieved""" if self.len == 0 or ((self.pendingLen >= self.maxPending or self.pendingLen == self.len) and self.lastPoll + self.pendingTimeout > time()): @@ -109,13 +111,15 @@ class RandomTrackingDict(object): with self.lock: # reset if we've requested all # and if last object received too long time ago - if self.pendingLen == self.len and self.lastObject + self.pendingTimeout < time(): + if self.pendingLen == self.len and self.lastObject + \ + self.pendingTimeout < time(): self.pendingLen = 0 self.setLastObject() available = self.len - self.pendingLen if count > available: count = available - randomIndex = helper_random.randomsample(range(self.len - self.pendingLen), count) + randomIndex = helper_random.randomsample( + range(self.len - self.pendingLen), count) retval = [self.indexDict[i] for i in randomIndex] for i in sorted(randomIndex, reverse=True): diff --git a/src/network/receivequeuethread.py b/src/network/receivequeuethread.py index cd904065..1f5533b3 100644 --- a/src/network/receivequeuethread.py +++ b/src/network/receivequeuethread.py @@ -1,3 +1,6 @@ +""" +Process data incoming from network +""" import errno import Queue import socket @@ -10,6 +13,8 @@ from threads import StoppableThread class ReceiveQueueThread(StoppableThread): + """This thread processes data received from the network + (which is done by the asyncore thread)""" def __init__(self, num=0): super(ReceiveQueueThread, self).__init__(name="ReceiveQueue_%i" % num) @@ -32,12 +37,14 @@ class ReceiveQueueThread(StoppableThread): try: connection = BMConnectionPool().getConnectionByAddr(dest) - except KeyError: # connection object not found + # connection object not found + except KeyError: receiveDataQueue.task_done() continue try: connection.process() - except UnknownStateError: # state isn't implemented + # state isn't implemented + except UnknownStateError: pass except socket.error as err: if err.errno == errno.EBADF: diff --git a/src/network/socks4a.py b/src/network/socks4a.py index f0b234f5..42eab4b7 100644 --- a/src/network/socks4a.py +++ b/src/network/socks4a.py @@ -1,6 +1,5 @@ """ -src/network/socks4a.py -================================= +SOCKS4a proxy module """ # pylint: disable=attribute-defined-outside-init import socket diff --git a/src/network/socks5.py b/src/network/socks5.py index f0241744..fc33f4df 100644 --- a/src/network/socks5.py +++ b/src/network/socks5.py @@ -1,7 +1,5 @@ """ -src/network/socks5.py -===================== - +SOCKS5 proxy module """ # pylint: disable=attribute-defined-outside-init @@ -155,7 +153,8 @@ class Socks5(Proxy): return True def proxy_sock_name(self): - """Handle return value when using SOCKS5 for DNS resolving instead of connecting.""" + """Handle return value when using SOCKS5 + for DNS resolving instead of connecting.""" return socket.inet_ntoa(self.__proxysockname[0]) diff --git a/src/network/stats.py b/src/network/stats.py index d760ace2..82e6c87f 100644 --- a/src/network/stats.py +++ b/src/network/stats.py @@ -1,6 +1,5 @@ """ -src/network/stats.py -==================== +Network statistics """ import time @@ -34,7 +33,9 @@ def uploadSpeed(): currentTimestamp = time.time() if int(lastSentTimestamp) < int(currentTimestamp): currentSentBytes = asyncore.sentBytes - currentSentSpeed = int((currentSentBytes - lastSentBytes) / (currentTimestamp - lastSentTimestamp)) + currentSentSpeed = int( + (currentSentBytes - lastSentBytes) / ( + currentTimestamp - lastSentTimestamp)) lastSentBytes = currentSentBytes lastSentTimestamp = currentTimestamp return currentSentSpeed @@ -53,7 +54,8 @@ def downloadSpeed(): if int(lastReceivedTimestamp) < int(currentTimestamp): currentReceivedBytes = asyncore.receivedBytes currentReceivedSpeed = int( - (currentReceivedBytes - lastReceivedBytes) / (currentTimestamp - lastReceivedTimestamp)) + (currentReceivedBytes - lastReceivedBytes) / ( + currentTimestamp - lastReceivedTimestamp)) lastReceivedBytes = currentReceivedBytes lastReceivedTimestamp = currentTimestamp return currentReceivedSpeed diff --git a/src/network/tcp.py b/src/network/tcp.py index 3097765f..d611b1ca 100644 --- a/src/network/tcp.py +++ b/src/network/tcp.py @@ -1,9 +1,7 @@ +""" +TCP protocol handler +""" # pylint: disable=too-many-ancestors -""" -src/network/tcp.py -================== -""" - import logging import math import random @@ -31,7 +29,7 @@ from network.socks4a import Socks4aConnection from network.socks5 import Socks5Connection from network.tls import TLSDispatcher from node import Peer -from queues import UISignalQueue, invQueue, receiveDataQueue +from queues import invQueue, receiveDataQueue, UISignalQueue logger = logging.getLogger('default') @@ -39,7 +37,6 @@ logger = logging.getLogger('default') class TCPConnection(BMProto, TLSDispatcher): # pylint: disable=too-many-instance-attributes """ - .. todo:: Look to understand and/or fix the non-parent-init-called """ @@ -85,7 +82,8 @@ class TCPConnection(BMProto, TLSDispatcher): not protocol.checkSocksIP(self.destination.host) ) except socket.error: - pass # it's probably a hostname + # it's probably a hostname + pass self.network_group = protocol.network_group(self.destination.host) ObjectTracker.__init__(self) # pylint: disable=non-parent-init-called self.bm_proto_reset() @@ -140,10 +138,9 @@ class TCPConnection(BMProto, TLSDispatcher): if not self.isOutbound and not self.local: shared.clientHasReceivedIncomingConnections = True UISignalQueue.put(('setStatusIcon', 'green')) - UISignalQueue.put(( - 'updateNetworkStatusTab', - (self.isOutbound, True, self.destination) - )) + UISignalQueue.put( + ('updateNetworkStatusTab', ( + self.isOutbound, True, self.destination))) self.antiIntersectionDelay(True) self.fullyEstablished = True if self.isOutbound: @@ -215,8 +212,8 @@ class TCPConnection(BMProto, TLSDispatcher): bigInvList[objHash] = 0 objectCount = 0 payload = b'' - # Now let us start appending all of these hashes together. They will be - # sent out in a big inv message to our new peer. + # Now let us start appending all of these hashes together. + # They will be sent out in a big inv message to our new peer. for obj_hash, _ in bigInvList.items(): payload += obj_hash objectCount += 1 diff --git a/src/network/tls.py b/src/network/tls.py index d5c4e23a..f756591c 100644 --- a/src/network/tls.py +++ b/src/network/tls.py @@ -1,7 +1,6 @@ """ SSL/TLS negotiation. """ - import logging import os import socket @@ -10,6 +9,7 @@ import sys from network.advanceddispatcher import AdvancedDispatcher import network.asyncore_pollchoose as asyncore + from queues import receiveDataQueue import paths @@ -24,7 +24,8 @@ if sys.version_info >= (2, 7, 13): # ssl.PROTOCOL_TLS1.2 sslProtocolVersion = ssl.PROTOCOL_TLS # pylint: disable=no-member elif sys.version_info >= (2, 7, 9): - # this means any SSL/TLS. SSLv2 and 3 are excluded with an option after context is created + # this means any SSL/TLS. + # SSLv2 and 3 are excluded with an option after context is created sslProtocolVersion = ssl.PROTOCOL_SSLv23 else: # this means TLSv1, there is no way to set "TLSv1 or higher" or @@ -33,7 +34,8 @@ else: # ciphers -if ssl.OPENSSL_VERSION_NUMBER >= 0x10100000 and not ssl.OPENSSL_VERSION.startswith("LibreSSL"): +if ssl.OPENSSL_VERSION_NUMBER >= 0x10100000 and not \ + ssl.OPENSSL_VERSION.startswith("LibreSSL"): sslProtocolCiphers = "AECDH-AES256-SHA@SECLEVEL=0" else: sslProtocolCiphers = "AECDH-AES256-SHA" @@ -41,19 +43,19 @@ else: class TLSDispatcher(AdvancedDispatcher): """TLS functionality for classes derived from AdvancedDispatcher""" - # pylint: disable=too-many-instance-attributes - # pylint: disable=too-many-arguments,super-init-not-called,unused-argument - def __init__( - self, address=None, sock=None, certfile=None, keyfile=None, - server_side=False, ciphers=sslProtocolCiphers - ): + # pylint: disable=too-many-instance-attributes, too-many-arguments + # pylint: disable=super-init-not-called + def __init__(self, _=None, sock=None, certfile=None, keyfile=None, + server_side=False, ciphers=sslProtocolCiphers): self.want_read = self.want_write = True if certfile is None: - self.certfile = os.path.join(paths.codePath(), 'sslkeys', 'cert.pem') + self.certfile = os.path.join( + paths.codePath(), 'sslkeys', 'cert.pem') else: self.certfile = certfile if keyfile is None: - self.keyfile = os.path.join(paths.codePath(), 'sslkeys', 'key.pem') + self.keyfile = os.path.join( + paths.codePath(), 'sslkeys', 'key.pem') else: self.keyfile = keyfile self.server_side = server_side @@ -68,20 +70,23 @@ class TLSDispatcher(AdvancedDispatcher): # pylint: disable=attribute-defined-outside-init self.isSSL = True self.tlsStarted = True - # Once the connection has been established, it's safe to wrap the - # socket. + # Once the connection has been established, + # it's safe to wrap the socket. if sys.version_info >= (2, 7, 9): context = ssl.create_default_context( - purpose=ssl.Purpose.SERVER_AUTH if self.server_side else ssl.Purpose.CLIENT_AUTH) + purpose=ssl.Purpose.SERVER_AUTH + if self.server_side else ssl.Purpose.CLIENT_AUTH) context.set_ciphers(self.ciphers) context.set_ecdh_curve("secp256k1") context.check_hostname = False context.verify_mode = ssl.CERT_NONE # also exclude TLSv1 and TLSv1.1 in the future context.options = ssl.OP_ALL | ssl.OP_NO_SSLv2 |\ - ssl.OP_NO_SSLv3 | ssl.OP_SINGLE_ECDH_USE | ssl.OP_CIPHER_SERVER_PREFERENCE + ssl.OP_NO_SSLv3 | ssl.OP_SINGLE_ECDH_USE |\ + ssl.OP_CIPHER_SERVER_PREFERENCE self.sslSocket = context.wrap_socket( - self.socket, server_side=self.server_side, do_handshake_on_connect=False) + self.socket, server_side=self.server_side, + do_handshake_on_connect=False) else: self.sslSocket = ssl.wrap_socket( self.socket, server_side=self.server_side, @@ -115,12 +120,15 @@ class TLSDispatcher(AdvancedDispatcher): def readable(self): """Handle readable check for TLS-enabled sockets""" try: - # during TLS handshake, and after flushing write buffer, return status of last handshake attempt + # during TLS handshake, and after flushing write buffer, + # return status of last handshake attempt if self.tlsStarted and not self.tlsDone and not self.write_buf: # print "tls readable, %r" % (self.want_read) return self.want_read - # prior to TLS handshake, receiveDataThread should emulate synchronous behaviour - elif not self.fullyEstablished and (self.expectBytes == 0 or not self.write_buf_empty()): + # prior to TLS handshake, + # receiveDataThread should emulate synchronous behaviour + elif not self.fullyEstablished and ( + self.expectBytes == 0 or not self.write_buf_empty()): return False return AdvancedDispatcher.readable(self) except AttributeError: @@ -135,10 +143,14 @@ class TLSDispatcher(AdvancedDispatcher): try: # wait for write buffer flush if self.tlsStarted and not self.tlsDone and not self.write_buf: - # logger.debug("%s:%i TLS handshaking (read)", self.destination.host, self.destination.port) + # logger.debug( + # "%s:%i TLS handshaking (read)", self.destination.host, + # self.destination.port) self.tls_handshake() else: - # logger.debug("%s:%i Not TLS handshaking (read)", self.destination.host, self.destination.port) + # logger.debug( + # "%s:%i Not TLS handshaking (read)", self.destination.host, + # self.destination.port) return AdvancedDispatcher.handle_read(self) except AttributeError: return AdvancedDispatcher.handle_read(self) @@ -161,10 +173,14 @@ class TLSDispatcher(AdvancedDispatcher): try: # wait for write buffer flush if self.tlsStarted and not self.tlsDone and not self.write_buf: - # logger.debug("%s:%i TLS handshaking (write)", self.destination.host, self.destination.port) + # logger.debug( + # "%s:%i TLS handshaking (write)", self.destination.host, + # self.destination.port) self.tls_handshake() else: - # logger.debug("%s:%i Not TLS handshaking (write)", self.destination.host, self.destination.port) + # logger.debug( + # "%s:%i Not TLS handshaking (write)", self.destination.host, + # self.destination.port) return AdvancedDispatcher.handle_write(self) except AttributeError: return AdvancedDispatcher.handle_write(self) @@ -188,7 +204,8 @@ class TLSDispatcher(AdvancedDispatcher): # print "handshaking (internal)" self.sslSocket.do_handshake() except ssl.SSLError as err: - # print "%s:%i: handshake fail" % (self.destination.host, self.destination.port) + # print "%s:%i: handshake fail" % ( + # self.destination.host, self.destination.port) self.want_read = self.want_write = False if err.args[0] == ssl.SSL_ERROR_WANT_READ: # print "want read" diff --git a/src/network/udp.py b/src/network/udp.py index d8c5f340..d5f1cccd 100644 --- a/src/network/udp.py +++ b/src/network/udp.py @@ -1,13 +1,12 @@ """ -src/network/udp.py -================== +UDP protocol handler """ import logging -import time import socket +import time -import state import protocol +import state from bmproto import BMProto from node import Peer from objectracker import ObjectTracker @@ -79,7 +78,7 @@ class UDPSocket(BMProto): # pylint: disable=too-many-instance-attributes if not self.local: return True remoteport = False - for seenTime, stream, services, ip, port in addresses: + for seenTime, stream, _, ip, port in addresses: decodedIP = protocol.checkIPAddress(str(ip)) if stream not in state.streamsInWhichIAmParticipating: continue @@ -96,9 +95,8 @@ class UDPSocket(BMProto): # pylint: disable=too-many-instance-attributes "received peer discovery from %s:%i (port %i):", self.destination.host, self.destination.port, remoteport) if self.local: - state.discoveredPeers[ - Peer(self.destination.host, remoteport) - ] = time.time() + state.discoveredPeers[Peer(self.destination.host, remoteport)] = \ + time.time() return True def bm_command_portcheck(self): From 6fad4f56657c4f44d1bc30298928b12fe5011bbc Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Tue, 24 Dec 2019 18:23:22 +0530 Subject: [PATCH 29/31] filesystem quality fixes --- src/storage/filesystem.py | 70 ++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/src/storage/filesystem.py b/src/storage/filesystem.py index 43ba03fc..571a16c1 100644 --- a/src/storage/filesystem.py +++ b/src/storage/filesystem.py @@ -1,6 +1,5 @@ """ -src/storage/filesystem.py -========================= +Module for using filesystem (directory with files) for inventory storage """ from binascii import hexlify, unhexlify from os import listdir, makedirs, path, remove, rmdir @@ -12,8 +11,9 @@ from paths import lookupAppdataFolder from storage import InventoryStorage, InventoryItem -class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ancestors, abstract-method - """Module for using filesystem (directory with files) for inventory storage""" +class FilesystemInventory(InventoryStorage): + """Filesystem for inventory storage""" + # pylint: disable=too-many-ancestors, abstract-method topDir = "inventory" objectDir = "objects" metadataFilename = "metadata" @@ -21,21 +21,23 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances def __init__(self): super(FilesystemInventory, self).__init__() - self.baseDir = path.join(lookupAppdataFolder(), FilesystemInventory.topDir) + self.baseDir = path.join( + lookupAppdataFolder(), FilesystemInventory.topDir) for createDir in [self.baseDir, path.join(self.baseDir, "objects")]: if path.exists(createDir): if not path.isdir(createDir): - raise IOError("%s exists but it's not a directory" % (createDir)) + raise IOError( + "%s exists but it's not a directory" % createDir) else: makedirs(createDir) - # Guarantees that two receiveDataThreads don't receive and process the same message + # Guarantees that two receiveDataThreads + # don't receive and process the same message # concurrently (probably sent by a malicious individual) self.lock = RLock() self._inventory = {} self._load() def __contains__(self, hashval): - retval = False for streamDict in self._inventory.values(): if hashval in streamDict: return True @@ -48,7 +50,12 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances except KeyError: continue if retval.payload is None: - retval = InventoryItem(retval.type, retval.stream, self.getData(hashval), retval.expires, retval.tag) + retval = InventoryItem( + retval.type, + retval.stream, + self.getData(hashval), + retval.expires, + retval.tag) return retval raise KeyError(hashval) @@ -56,7 +63,10 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances with self.lock: value = InventoryItem(*value) try: - makedirs(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval))) + makedirs(path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hashval))) except OSError: pass try: @@ -69,7 +79,11 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances ), "w", ) as f: - f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag))) + f.write("%s,%s,%s,%s," % ( + value.type, + value.stream, + value.expires, + hexlify(value.tag))) with open( path.join( self.baseDir, @@ -115,7 +129,10 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances except IOError: pass try: - rmdir(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashval))) + rmdir(path.join( + self.baseDir, + FilesystemInventory.objectDir, + hexlify(hashval))) except IOError: pass @@ -135,7 +152,8 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances newInventory = {} for hashId in self.object_list(): try: - objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId) + objectType, streamNumber, expiresTime, tag = self.getMetadata( + hashId) try: newInventory[streamNumber][hashId] = InventoryItem( objectType, streamNumber, None, expiresTime, tag) @@ -155,7 +173,8 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances def object_list(self): """Return inventory vectors (hashes) from a directory""" - return [unhexlify(x) for x in listdir(path.join(self.baseDir, FilesystemInventory.objectDir))] + return [unhexlify(x) for x in listdir(path.join( + self.baseDir, FilesystemInventory.objectDir))] def getData(self, hashId): """Get object data""" @@ -185,15 +204,20 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances ), "r", ) as f: - objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4) - return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)] + objectType, streamNumber, expiresTime, tag = string.split( + f.read(), ",", 4)[:4] + return [ + int(objectType), + int(streamNumber), + int(expiresTime), + unhexlify(tag)] except IOError: raise KeyError def by_type_and_tag(self, objectType, tag): """Get a list of objects filtered by object type and tag""" retval = [] - for stream, streamDict in self._inventory: + for streamDict in self._inventory.values(): for hashId, item in streamDict: if item.type == objectType and item.tag == tag: try: @@ -201,7 +225,12 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances item.payload = self.getData(hashId) except IOError: continue - retval.append(InventoryItem(item.type, item.stream, item.payload, item.expires, item.tag)) + retval.append(InventoryItem( + item.type, + item.stream, + item.payload, + item.expires, + item.tag)) return retval def hashes_by_stream(self, stream): @@ -215,7 +244,8 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances """Return unexpired hashes in the inventory for a particular stream""" t = int(time.time()) try: - return [x for x, value in self._inventory[stream].items() if value.expires > t] + return [x for x, value in self._inventory[stream].items() + if value.expires > t] except KeyError: return [] @@ -227,7 +257,7 @@ class FilesystemInventory(InventoryStorage): # pylint: disable=too-many-ances """Clean out old items from the inventory""" minTime = int(time.time()) - (60 * 60 * 30) deletes = [] - for stream, streamDict in self._inventory.items(): + for streamDict in self._inventory.values(): for hashId, item in streamDict.items(): if item.expires < minTime: deletes.append(hashId) From 108b231c1c612710683ce24954a07c6db7db3450 Mon Sep 17 00:00:00 2001 From: lakshyacis Date: Tue, 24 Dec 2019 18:23:37 +0530 Subject: [PATCH 30/31] sqlite quality fixes --- src/storage/sqlite.py | 51 ++++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/src/storage/sqlite.py b/src/storage/sqlite.py index 0c2b4afa..7f41abc5 100644 --- a/src/storage/sqlite.py +++ b/src/storage/sqlite.py @@ -1,6 +1,5 @@ """ -src/storage/sqlite.py -========================= +Sqlite Inventory """ import sqlite3 import time @@ -10,7 +9,7 @@ from helper_sql import sqlQuery, SqlBulkExecute, sqlExecute from storage import InventoryStorage, InventoryItem -class SqliteInventory(InventoryStorage): # pylint: disable=too-many-ancestors +class SqliteInventory(InventoryStorage): # pylint: disable=too-many-ancestors """Inventory using SQLite""" def __init__(self): super(SqliteInventory, self).__init__() @@ -20,9 +19,11 @@ class SqliteInventory(InventoryStorage): # pylint: disable=too-many-ancestors # cache for existing objects, used for quick lookups if we have an object. # This is used for example whenever we receive an inv message from a peer # to check to see what items are new to us. - # We don't delete things out of it; instead, the singleCleaner thread clears and refills it. + # We don't delete things out of it; instead, + # the singleCleaner thread clears and refills it. self._objects = {} - # Guarantees that two receiveDataThreads don't receive and process the same message concurrently + # Guarantees that two receiveDataThreads don't receive + # and process the same message concurrently # (probably sent by a malicious individual) self.lock = RLock() @@ -30,7 +31,9 @@ class SqliteInventory(InventoryStorage): # pylint: disable=too-many-ancestors with self.lock: if hash_ in self._objects: return True - rows = sqlQuery('SELECT streamnumber FROM inventory WHERE hash=?', sqlite3.Binary(hash_)) + rows = sqlQuery( + 'SELECT streamnumber FROM inventory WHERE hash=?', + sqlite3.Binary(hash_)) if not rows: return False self._objects[hash_] = rows[0][0] @@ -41,8 +44,8 @@ class SqliteInventory(InventoryStorage): # pylint: disable=too-many-ancestors if hash_ in self._inventory: return self._inventory[hash_] rows = sqlQuery( - 'SELECT objecttype, streamnumber, payload, expirestime, tag FROM inventory WHERE hash=?', - sqlite3.Binary(hash_)) + 'SELECT objecttype, streamnumber, payload, expirestime, tag' + ' FROM inventory WHERE hash=?', sqlite3.Binary(hash_)) if not rows: raise KeyError(hash_) return InventoryItem(*rows[0]) @@ -64,35 +67,49 @@ class SqliteInventory(InventoryStorage): # pylint: disable=too-many-ancestors def __len__(self): with self.lock: - return len(self._inventory) + sqlQuery('SELECT count(*) FROM inventory')[0][0] + return len(self._inventory) + sqlQuery( + 'SELECT count(*) FROM inventory')[0][0] def by_type_and_tag(self, objectType, tag): + """Return objects filtered by object type and tag""" with self.lock: - values = [value for value in self._inventory.values() if value.type == objectType and value.tag == tag] + values = [value for value in self._inventory.values() + if value.type == objectType and value.tag == tag] values += (InventoryItem(*value) for value in sqlQuery( - 'SELECT objecttype, streamnumber, payload, expirestime, tag \ - FROM inventory WHERE objecttype=? AND tag=?', objectType, sqlite3.Binary(tag))) + 'SELECT objecttype, streamnumber, payload, expirestime, tag' + ' FROM inventory WHERE objecttype=? AND tag=?', + objectType, sqlite3.Binary(tag))) return values def unexpired_hashes_by_stream(self, stream): + """Return unexpired inventory vectors filtered by stream""" with self.lock: t = int(time.time()) - hashes = [x for x, value in self._inventory.items() if value.stream == stream and value.expires > t] + hashes = [x for x, value in self._inventory.items() + if value.stream == stream and value.expires > t] hashes += (str(payload) for payload, in sqlQuery( - 'SELECT hash FROM inventory WHERE streamnumber=? AND expirestime>?', stream, t)) + 'SELECT hash FROM inventory WHERE streamnumber=?' + ' AND expirestime>?', stream, t)) return hashes def flush(self): + """Flush cache""" with self.lock: - # If you use both the inventoryLock and the sqlLock, always use the inventoryLock OUTSIDE of the sqlLock. + # If you use both the inventoryLock and the sqlLock, + # always use the inventoryLock OUTSIDE of the sqlLock. with SqlBulkExecute() as sql: for objectHash, value in self._inventory.items(): - sql.execute('INSERT INTO inventory VALUES (?, ?, ?, ?, ?, ?)', sqlite3.Binary(objectHash), *value) + sql.execute( + 'INSERT INTO inventory VALUES (?, ?, ?, ?, ?, ?)', + sqlite3.Binary(objectHash), *value) self._inventory.clear() def clean(self): + """Free memory / perform garbage collection""" with self.lock: - sqlExecute('DELETE FROM inventory WHERE expirestime Date: Tue, 24 Dec 2019 18:23:48 +0530 Subject: [PATCH 31/31] storage quality fixes --- src/storage/storage.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/storage/storage.py b/src/storage/storage.py index 482937e7..0391979a 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -1,21 +1,22 @@ """ -src/storage/storage.py -====================== +Storing inventory items """ import collections -InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload expires tag') +InventoryItem = collections.namedtuple( + 'InventoryItem', 'type stream payload expires tag') -class Storage(object): - """Base class for storing inventory (extendable for other items to store)""" +class Storage(object): # pylint: disable=too-few-public-methods + """Base class for storing inventory + (extendable for other items to store)""" pass class InventoryStorage(Storage, collections.MutableMapping): """Module used for inventory storage""" - def __init__(self): - # pylint: disable=super-init-not-called + + def __init__(self): # pylint: disable=super-init-not-called self.numberOfInventoryLookupsPerformed = 0 def __contains__(self, _): @@ -53,8 +54,20 @@ class InventoryStorage(Storage, collections.MutableMapping): raise NotImplementedError -class MailboxStorage(Storage, collections.MutableMapping): # pylint: disable=abstract-method +class MailboxStorage(Storage, collections.MutableMapping): """Method for storing mails""" - def __init__(self): - # pylint: disable=super-init-not-called - pass + + def __delitem__(self, key): + raise NotImplementedError + + def __getitem__(self, key): + raise NotImplementedError + + def __iter__(self): + raise NotImplementedError + + def __len__(self): + raise NotImplementedError + + def __setitem__(self, key, value): + raise NotImplementedError