From 303352099565dc9a441b8d741a4e0665b8b1defd Mon Sep 17 00:00:00 2001 From: Peter Surda Date: Thu, 1 Dec 2016 16:45:04 +0100 Subject: [PATCH] Directory maintenance - rearrange obsolete build scripts - rearrange working build scripts - add build / devel scripts I use - add existing development tests (bloom filter, message encoding, thread interrupts) - add directory descriptions --- build/README.md | 2 + build/changelang.sh | 16 +++++ compiletest.py => build/compiletest.py | 0 build/mergepullrequest.sh | 11 ++++ osx.sh => build/osx.sh | 0 build/updatetranslations.sh | 22 +++++++ dev/README.md | 3 + dev/bloomfiltertest.py | 60 +++++++++++++++++++ dev/msgtest.py | 27 +++++++++ dev/powinterrupttest.py | 49 +++++++++++++++ packages/README.md | 36 +++++++++++ .../pyinstaller}/bitmessagemain.spec | 0 Makefile => packages/unmaintained/Makefile | 0 arch.sh => packages/unmaintained/arch.sh | 0 .../unmaintained/archpackage}/PKGBUILD | 0 debian.sh => packages/unmaintained/debian.sh | 0 .../unmaintained/debian}/changelog | 0 .../unmaintained/debian}/compat | 0 .../unmaintained/debian}/control | 0 .../unmaintained/debian}/copyright | 0 {debian => packages/unmaintained/debian}/docs | 0 .../unmaintained/debian}/manpages | 0 {debian => packages/unmaintained/debian}/pybm | 0 .../unmaintained/debian}/rules | 0 .../unmaintained/debian}/source/format | 0 .../debian}/source/include-binaries | 0 ebuild.sh => packages/unmaintained/ebuild.sh | 0 .../unmaintained/generate.sh | 0 puppy.sh => packages/unmaintained/puppy.sh | 0 .../unmaintained/puppypackage}/icon14.xpm | 0 .../pybitmessage-0.3.5.pet.specs | 0 rpm.sh => packages/unmaintained/rpm.sh | 0 .../rpmpackage}/pybitmessage.spec | 0 slack.sh => packages/unmaintained/slack.sh | 0 .../unmaintained/slackpackage}/doinst.sh | 0 .../unmaintained/slackpackage}/slack-desc | 0 {upstart => packages/upstart}/bitmessage.conf | 0 37 files changed, 226 insertions(+) create mode 100644 build/README.md create mode 100755 build/changelang.sh rename compiletest.py => build/compiletest.py (100%) create mode 100755 build/mergepullrequest.sh rename osx.sh => build/osx.sh (100%) create mode 100755 build/updatetranslations.sh create mode 100644 dev/README.md create mode 100644 dev/bloomfiltertest.py create mode 100644 dev/msgtest.py create mode 100644 dev/powinterrupttest.py create mode 100644 packages/README.md rename {pyinstaller => packages/pyinstaller}/bitmessagemain.spec (100%) rename Makefile => packages/unmaintained/Makefile (100%) rename arch.sh => packages/unmaintained/arch.sh (100%) rename {archpackage => packages/unmaintained/archpackage}/PKGBUILD (100%) rename debian.sh => packages/unmaintained/debian.sh (100%) rename {debian => packages/unmaintained/debian}/changelog (100%) rename {debian => packages/unmaintained/debian}/compat (100%) rename {debian => packages/unmaintained/debian}/control (100%) rename {debian => packages/unmaintained/debian}/copyright (100%) rename {debian => packages/unmaintained/debian}/docs (100%) rename {debian => packages/unmaintained/debian}/manpages (100%) rename {debian => packages/unmaintained/debian}/pybm (100%) rename {debian => packages/unmaintained/debian}/rules (100%) rename {debian => packages/unmaintained/debian}/source/format (100%) rename {debian => packages/unmaintained/debian}/source/include-binaries (100%) rename ebuild.sh => packages/unmaintained/ebuild.sh (100%) rename generate.sh => packages/unmaintained/generate.sh (100%) rename puppy.sh => packages/unmaintained/puppy.sh (100%) rename {puppypackage => packages/unmaintained/puppypackage}/icon14.xpm (100%) rename {puppypackage => packages/unmaintained/puppypackage}/pybitmessage-0.3.5.pet.specs (100%) rename rpm.sh => packages/unmaintained/rpm.sh (100%) rename {rpmpackage => packages/unmaintained/rpmpackage}/pybitmessage.spec (100%) rename slack.sh => packages/unmaintained/slack.sh (100%) rename {slackpackage => packages/unmaintained/slackpackage}/doinst.sh (100%) rename {slackpackage => packages/unmaintained/slackpackage}/slack-desc (100%) rename {upstart => packages/upstart}/bitmessage.conf (100%) diff --git a/build/README.md b/build/README.md new file mode 100644 index 00000000..248d2c41 --- /dev/null +++ b/build/README.md @@ -0,0 +1,2 @@ +This directory contains scripts that are helpful for developers when building +or maintaining PyBitmessage. diff --git a/build/changelang.sh b/build/changelang.sh new file mode 100755 index 00000000..915c5dea --- /dev/null +++ b/build/changelang.sh @@ -0,0 +1,16 @@ +export LANG=de_DE.UTF-8 +export LANGUAGE=de_DE +export LC_CTYPE="de_DE.UTF-8" +export LC_NUMERIC=de_DE.UTF-8 +export LC_TIME=de_DE.UTF-8 +export LC_COLLATE="de_DE.UTF-8" +export LC_MONETARY=de_DE.UTF-8 +export LC_MESSAGES="de_DE.UTF-8" +export LC_PAPER=de_DE.UTF-8 +export LC_NAME=de_DE.UTF-8 +export LC_ADDRESS=de_DE.UTF-8 +export LC_TELEPHONE=de_DE.UTF-8 +export LC_MEASUREMENT=de_DE.UTF-8 +export LC_IDENTIFICATION=de_DE.UTF-8 +export LC_ALL= +python2.7 src/bitmessagemain.py diff --git a/compiletest.py b/build/compiletest.py similarity index 100% rename from compiletest.py rename to build/compiletest.py diff --git a/build/mergepullrequest.sh b/build/mergepullrequest.sh new file mode 100755 index 00000000..35e87566 --- /dev/null +++ b/build/mergepullrequest.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ -z "$1" ]; then + echo "You must specify pull request number" + exit +fi + +git pull +git checkout v0.6 +git fetch origin pull/"$1"/head:"$1" +git merge --ff-only "$1" diff --git a/osx.sh b/build/osx.sh similarity index 100% rename from osx.sh rename to build/osx.sh diff --git a/build/updatetranslations.sh b/build/updatetranslations.sh new file mode 100755 index 00000000..ba5a3fdb --- /dev/null +++ b/build/updatetranslations.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if [ ! -f "$1" ]; then + echo "$1 not found, please specify the file name for source" + exit +fi + +srcdir=`mktemp -d` + +unzip "$1" -d $srcdir + +for i in $srcdir/*ts; do + o=`basename $i|cut -b3-` + o="${o,,}" + o="${o//@/_}" + echo "$i -> $o" + mv "$i" "$HOME/src/PyBitmessage/src/translations/$o" +done + +rm -rf -- $srcdir + +lrelease-qt4 "$HOME/src/PyBitmessage/src/translations/bitmessage.pro" diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 00000000..41bf59c8 --- /dev/null +++ b/dev/README.md @@ -0,0 +1,3 @@ +This directory contains code for future features, but it's not integrated into +PyBitmessage and may not work at all. Developers can look at it if they are +interested. diff --git a/dev/bloomfiltertest.py b/dev/bloomfiltertest.py new file mode 100644 index 00000000..539d00f3 --- /dev/null +++ b/dev/bloomfiltertest.py @@ -0,0 +1,60 @@ +from math import ceil +from os import stat, getenv, path +from pybloom import BloomFilter as BloomFilter1 +from pybloomfilter import BloomFilter as BloomFilter2 +import sqlite3 +from time import time + +# Ubuntu: apt-get install python-pybloomfiltermmap + +conn = sqlite3.connect(path.join(getenv("HOME"), '.config/PyBitmessage/messages.dat')) + +conn.text_factory = str +cur = conn.cursor() +rawlen = 0 +itemcount = 0 + +cur.execute('''SELECT COUNT(hash) FROM inventory''') +for row in cur.fetchall(): + itemcount = row[0] + +filtersize = 1000 * (int(itemcount / 1000) + 1) +errorrate = 1.0 / 1000.0 + +bf1 = BloomFilter1(capacity=filtersize, error_rate=errorrate) +bf2 = BloomFilter2(capacity=filtersize, error_rate=errorrate) + +item = '''SELECT hash FROM inventory''' +cur.execute(item, '') +bf1time = 0 +bf2time = 0 +for row in cur.fetchall(): + rawlen += len(row[0]) + try: + times = [time()] + bf1.add(row[0]) + times.append(time()) + bf2.add(row[0]) + times.append(time()) + bf1time += times[1] - times[0] + bf2time += times[2] - times[1] + except IndexError: + pass + +#f = open("/home/shurdeek/tmp/bloom.dat", "wb") +#sb1.tofile(f) +#f.close() + + +print "Item count: %i" % (itemcount) +print "Raw length: %i" % (rawlen) +print "Bloom filter 1 length: %i, reduction to: %.2f%%" % \ + (bf1.bitarray.buffer_info()[1], + 100.0 * bf1.bitarray.buffer_info()[1] / rawlen) +print "Bloom filter 1 capacity: %i and error rate: %.3f%%" % (bf1.capacity, 100.0 * bf1.error_rate) +print "Bloom filter 1 took %.2fs" % (bf1time) +print "Bloom filter 2 length: %i, reduction to: %.3f%%" % \ + (bf2.num_bits / 8, + 100.0 * bf2.num_bits / 8 / rawlen) +print "Bloom filter 2 capacity: %i and error rate: %.3f%%" % (bf2.capacity, 100.0 * bf2.error_rate) +print "Bloom filter 2 took %.2fs" % (bf2time) diff --git a/dev/msgtest.py b/dev/msgtest.py new file mode 100644 index 00000000..d5a8be8e --- /dev/null +++ b/dev/msgtest.py @@ -0,0 +1,27 @@ +import importlib +from os import listdir, path +from pprint import pprint +import sys +import traceback + +data = {"": "message", "subject": "subject", "body": "body"} +#data = {"": "vote", "msgid": "msgid"} +#data = {"fsck": 1} + +import messagetypes + +if __name__ == '__main__': + try: + msgType = data[""] + except KeyError: + print "Message type missing" + sys.exit(1) + else: + print "Message type: %s" % (msgType) + msgObj = messagetypes.constructObject(data) + if msgObj is None: + sys.exit(1) + try: + msgObj.process() + except: + pprint(sys.exc_info()) diff --git a/dev/powinterrupttest.py b/dev/powinterrupttest.py new file mode 100644 index 00000000..2f8cfcac --- /dev/null +++ b/dev/powinterrupttest.py @@ -0,0 +1,49 @@ +import ctypes +import hashlib +from multiprocessing import current_process +import os +import signal +from struct import unpack, pack +from threading import current_thread + +shutdown = 0 + + +def signal_handler(signal, frame): + global shutdown + print "Got signal %i in %s/%s" % (signal, current_process().name, current_thread().name) + if current_process().name != "MainProcess": + raise StopIteration("Interrupted") + if current_thread().name != "MainThread": + return + shutdown = 1 + + +def _doCPoW(target, initialHash): +# global shutdown + h = initialHash + m = target + out_h = ctypes.pointer(ctypes.create_string_buffer(h, 64)) + out_m = ctypes.c_ulonglong(m) + print "C PoW start" + for c in range(0, 200000): + print "Iter: %i" % (c) + nonce = bmpow(out_h, out_m) + if shutdown: + break + trialValue, = unpack('>Q', hashlib.sha512(hashlib.sha512(pack('>Q', nonce) + initialHash).digest()).digest()[0:8]) + if shutdown != 0: + raise StopIteration("Interrupted") + print "C PoW done" + return [trialValue, nonce] + + +signal.signal(signal.SIGINT, signal_handler) +signal.signal(signal.SIGTERM, signal_handler) + +bso = ctypes.CDLL(os.path.join("bitmsghash", "bitmsghash.so")) + +bmpow = bso.BitmessagePOW +bmpow.restype = ctypes.c_ulonglong + +_doCPoW(2**44, "") diff --git a/packages/README.md b/packages/README.md new file mode 100644 index 00000000..ed2df3cc --- /dev/null +++ b/packages/README.md @@ -0,0 +1,36 @@ +The `generate.sh` script is obsolete, but is included for historical reasons. + +Maintained packages can be obtained: + +Windows: +======== + +https://github.com/Bitmessage/PyBitmessage/releases + +Works on Windows XP or higher. + + +OSX: +==== + +https://github.com/Bitmessage/PyBitmessage/releases + +Wors on OSX 10.7.5 or higher + + +Arch linux: +=========== + +Releases matching PyBitmessage releases: + +https://aur.archlinux.org/packages/pybitmessage-git/ + +Development snapshot equivalent to the v0.6 git branch: + +https://aur.archlinux.org/packages/pybitmessage-dev-git/ + + +FreeBSD: +======== + +Use the FreeBSD ports. diff --git a/pyinstaller/bitmessagemain.spec b/packages/pyinstaller/bitmessagemain.spec similarity index 100% rename from pyinstaller/bitmessagemain.spec rename to packages/pyinstaller/bitmessagemain.spec diff --git a/Makefile b/packages/unmaintained/Makefile similarity index 100% rename from Makefile rename to packages/unmaintained/Makefile diff --git a/arch.sh b/packages/unmaintained/arch.sh similarity index 100% rename from arch.sh rename to packages/unmaintained/arch.sh diff --git a/archpackage/PKGBUILD b/packages/unmaintained/archpackage/PKGBUILD similarity index 100% rename from archpackage/PKGBUILD rename to packages/unmaintained/archpackage/PKGBUILD diff --git a/debian.sh b/packages/unmaintained/debian.sh similarity index 100% rename from debian.sh rename to packages/unmaintained/debian.sh diff --git a/debian/changelog b/packages/unmaintained/debian/changelog similarity index 100% rename from debian/changelog rename to packages/unmaintained/debian/changelog diff --git a/debian/compat b/packages/unmaintained/debian/compat similarity index 100% rename from debian/compat rename to packages/unmaintained/debian/compat diff --git a/debian/control b/packages/unmaintained/debian/control similarity index 100% rename from debian/control rename to packages/unmaintained/debian/control diff --git a/debian/copyright b/packages/unmaintained/debian/copyright similarity index 100% rename from debian/copyright rename to packages/unmaintained/debian/copyright diff --git a/debian/docs b/packages/unmaintained/debian/docs similarity index 100% rename from debian/docs rename to packages/unmaintained/debian/docs diff --git a/debian/manpages b/packages/unmaintained/debian/manpages similarity index 100% rename from debian/manpages rename to packages/unmaintained/debian/manpages diff --git a/debian/pybm b/packages/unmaintained/debian/pybm similarity index 100% rename from debian/pybm rename to packages/unmaintained/debian/pybm diff --git a/debian/rules b/packages/unmaintained/debian/rules similarity index 100% rename from debian/rules rename to packages/unmaintained/debian/rules diff --git a/debian/source/format b/packages/unmaintained/debian/source/format similarity index 100% rename from debian/source/format rename to packages/unmaintained/debian/source/format diff --git a/debian/source/include-binaries b/packages/unmaintained/debian/source/include-binaries similarity index 100% rename from debian/source/include-binaries rename to packages/unmaintained/debian/source/include-binaries diff --git a/ebuild.sh b/packages/unmaintained/ebuild.sh similarity index 100% rename from ebuild.sh rename to packages/unmaintained/ebuild.sh diff --git a/generate.sh b/packages/unmaintained/generate.sh similarity index 100% rename from generate.sh rename to packages/unmaintained/generate.sh diff --git a/puppy.sh b/packages/unmaintained/puppy.sh similarity index 100% rename from puppy.sh rename to packages/unmaintained/puppy.sh diff --git a/puppypackage/icon14.xpm b/packages/unmaintained/puppypackage/icon14.xpm similarity index 100% rename from puppypackage/icon14.xpm rename to packages/unmaintained/puppypackage/icon14.xpm diff --git a/puppypackage/pybitmessage-0.3.5.pet.specs b/packages/unmaintained/puppypackage/pybitmessage-0.3.5.pet.specs similarity index 100% rename from puppypackage/pybitmessage-0.3.5.pet.specs rename to packages/unmaintained/puppypackage/pybitmessage-0.3.5.pet.specs diff --git a/rpm.sh b/packages/unmaintained/rpm.sh similarity index 100% rename from rpm.sh rename to packages/unmaintained/rpm.sh diff --git a/rpmpackage/pybitmessage.spec b/packages/unmaintained/rpmpackage/pybitmessage.spec similarity index 100% rename from rpmpackage/pybitmessage.spec rename to packages/unmaintained/rpmpackage/pybitmessage.spec diff --git a/slack.sh b/packages/unmaintained/slack.sh similarity index 100% rename from slack.sh rename to packages/unmaintained/slack.sh diff --git a/slackpackage/doinst.sh b/packages/unmaintained/slackpackage/doinst.sh similarity index 100% rename from slackpackage/doinst.sh rename to packages/unmaintained/slackpackage/doinst.sh diff --git a/slackpackage/slack-desc b/packages/unmaintained/slackpackage/slack-desc similarity index 100% rename from slackpackage/slack-desc rename to packages/unmaintained/slackpackage/slack-desc diff --git a/upstart/bitmessage.conf b/packages/upstart/bitmessage.conf similarity index 100% rename from upstart/bitmessage.conf rename to packages/upstart/bitmessage.conf