From c5ae33dc471a2b87176cedee96e153a905f5d3a7 Mon Sep 17 00:00:00 2001 From: Dmitri Bogomolov <4glitch@gmail.com> Date: Mon, 14 Oct 2019 15:20:12 +0300 Subject: [PATCH] Made fallback.umsgpack a git submodule pointing to upstream v2.5.2 --- .gitmodules | 4 + src/fallback/__init__.py | 8 + src/fallback/umsgpack | 1 + src/fallback/umsgpack.py | 1 + src/fallback/umsgpack/__init__.py | 0 src/fallback/umsgpack/umsgpack.py | 1067 ----------------------------- src/helper_msgcoding.py | 8 +- 7 files changed, 15 insertions(+), 1074 deletions(-) create mode 100644 .gitmodules create mode 160000 src/fallback/umsgpack create mode 120000 src/fallback/umsgpack.py delete mode 100644 src/fallback/umsgpack/__init__.py delete mode 100644 src/fallback/umsgpack/umsgpack.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..e831002c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "src/fallback/umsgpack"] + path = src/fallback/umsgpack + url = https://github.com/vsergeev/u-msgpack-python.git + branch = tag/v2.5.2 diff --git a/src/fallback/__init__.py b/src/fallback/__init__.py index d45c754d..1cee04c3 100644 --- a/src/fallback/__init__.py +++ b/src/fallback/__init__.py @@ -24,3 +24,11 @@ else: if data: hasher.update(data) return hasher + +try: + import msgpack +except ImportError: + import umsgpack as msgpack + + +__all__ = ['RIPEMD160Hash', 'msgpack'] diff --git a/src/fallback/umsgpack b/src/fallback/umsgpack new file mode 160000 index 00000000..027bc3f9 --- /dev/null +++ b/src/fallback/umsgpack @@ -0,0 +1 @@ +Subproject commit 027bc3f9f45de75d570ff9239677d8b92e7fd7a1 diff --git a/src/fallback/umsgpack.py b/src/fallback/umsgpack.py new file mode 120000 index 00000000..02dac23e --- /dev/null +++ b/src/fallback/umsgpack.py @@ -0,0 +1 @@ +umsgpack/umsgpack.py \ No newline at end of file diff --git a/src/fallback/umsgpack/__init__.py b/src/fallback/umsgpack/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/fallback/umsgpack/umsgpack.py b/src/fallback/umsgpack/umsgpack.py deleted file mode 100644 index 34938614..00000000 --- a/src/fallback/umsgpack/umsgpack.py +++ /dev/null @@ -1,1067 +0,0 @@ -# u-msgpack-python v2.4.1 - v at sergeev.io -# https://github.com/vsergeev/u-msgpack-python -# -# u-msgpack-python is a lightweight MessagePack serializer and deserializer -# module, compatible with both Python 2 and 3, as well CPython and PyPy -# implementations of Python. u-msgpack-python is fully compliant with the -# latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In -# particular, it supports the new binary, UTF-8 string, and application ext -# types. -# -# MIT License -# -# Copyright (c) 2013-2016 vsergeev / Ivan (Vanya) A. Sergeev -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -""" -src/fallback/umsgpack/umsgpack.py -================================= - -u-msgpack-python v2.4.1 - v at sergeev.io -https://github.com/vsergeev/u-msgpack-python - -u-msgpack-python is a lightweight MessagePack serializer and deserializer -module, compatible with both Python 2 and 3, as well CPython and PyPy -implementations of Python. u-msgpack-python is fully compliant with the -latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In -particular, it supports the new binary, UTF-8 string, and application ext -types. - -License: MIT -""" -# pylint: disable=too-many-lines,too-many-branches,too-many-statements,global-statement,too-many-return-statements -# pylint: disable=unused-argument - -import collections -import io -import struct -import sys - -__version__ = "2.4.1" -"Module version string" - -version = (2, 4, 1) -"Module version tuple" - - -############################################################################## -# Ext Class -############################################################################## - -# Extension type for application-defined types and data -class Ext: # pylint: disable=old-style-class - """ - The Ext class facilitates creating a serializable extension object to store - an application-defined type and data byte array. - """ - - def __init__(self, type, data): - """ - Construct a new Ext object. - - Args: - type: application-defined type integer from 0 to 127 - data: application-defined data byte array - - Raises: - TypeError: - Specified ext type is outside of 0 to 127 range. - - Example: - >>> foo = umsgpack.Ext(0x05, b"\x01\x02\x03") - >>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) - '\x82\xa7awesome\xc3\xadspecial stuff\xc7\x03\x05\x01\x02\x03' - >>> bar = umsgpack.unpackb(_) - >>> print(bar["special stuff"]) - Ext Object (Type: 0x05, Data: 01 02 03) - >>> - """ - # pylint:disable=redefined-builtin - - # Application ext type should be 0 <= type <= 127 - if not isinstance(type, int) or not (type >= 0 and type <= 127): - raise TypeError("ext type out of range") - # Check data is type bytes - elif sys.version_info[0] == 3 and not isinstance(data, bytes): - raise TypeError("ext data is not type \'bytes\'") - elif sys.version_info[0] == 2 and not isinstance(data, str): - raise TypeError("ext data is not type \'str\'") - self.type = type - self.data = data - - def __eq__(self, other): - """ - Compare this Ext object with another for equality. - """ - return (isinstance(other, self.__class__) and - self.type == other.type and - self.data == other.data) - - def __ne__(self, other): - """ - Compare this Ext object with another for inequality. - """ - return not self.__eq__(other) - - def __str__(self): - """ - String representation of this Ext object. - """ - s = "Ext Object (Type: 0x%02x, Data: " % self.type - s += " ".join(["0x%02x" % ord(self.data[i:i + 1]) - for i in xrange(min(len(self.data), 8))]) - if len(self.data) > 8: - s += " ..." - s += ")" - return s - - def __hash__(self): - """ - Provide a hash of this Ext object. - """ - return hash((self.type, self.data)) - - -class InvalidString(bytes): - """Subclass of bytes to hold invalid UTF-8 strings.""" - pass - -############################################################################## -# Exceptions -############################################################################## - - -# Base Exception classes -class PackException(Exception): - "Base class for exceptions encountered during packing." - pass - - -class UnpackException(Exception): - "Base class for exceptions encountered during unpacking." - pass - - -# Packing error -class UnsupportedTypeException(PackException): - "Object type not supported for packing." - pass - - -# Unpacking error -class InsufficientDataException(UnpackException): - "Insufficient data to unpack the serialized object." - pass - - -class InvalidStringException(UnpackException): - "Invalid UTF-8 string encountered during unpacking." - pass - - -class ReservedCodeException(UnpackException): - "Reserved code encountered during unpacking." - pass - - -class UnhashableKeyException(UnpackException): - """ - Unhashable key encountered during map unpacking. - The serialized map cannot be deserialized into a Python dictionary. - """ - pass - - -class DuplicateKeyException(UnpackException): - "Duplicate key encountered during map unpacking." - pass - - -# Backwards compatibility -KeyNotPrimitiveException = UnhashableKeyException -KeyDuplicateException = DuplicateKeyException - -############################################################################# -# Exported Functions and Glob -############################################################################# - -# Exported functions and variables, set up in __init() -pack = None -packb = None -unpack = None -unpackb = None -dump = None -dumps = None -load = None -loads = None - -compatibility = False -u""" -Compatibility mode boolean. - -When compatibility mode is enabled, u-msgpack-python will serialize both -unicode strings and bytes into the old "raw" msgpack type, and deserialize the -"raw" msgpack type into bytes. This provides backwards compatibility with the -old MessagePack specification. - -Example: ->>> umsgpack.compatibility = True ->>> ->>> umsgpack.packb([u"some string", b"some bytes"]) -b'\x92\xabsome string\xaasome bytes' ->>> umsgpack.unpackb(_) -[b'some string', b'some bytes'] ->>> -""" - -############################################################################## -# Packing -############################################################################## - -# You may notice struct.pack("B", obj) instead of the simpler chr(obj) in the -# code below. This is to allow for seamless Python 2 and 3 compatibility, as -# chr(obj) has a str return type instead of bytes in Python 3, and -# struct.pack(...) has the right return type in both versions. - - -def _pack_integer(obj, fp, options): - if obj < 0: - if obj >= -32: - fp.write(struct.pack("b", obj)) - elif obj >= -2**(8 - 1): - fp.write(b"\xd0" + struct.pack("b", obj)) - elif obj >= -2**(16 - 1): - fp.write(b"\xd1" + struct.pack(">h", obj)) - elif obj >= -2**(32 - 1): - fp.write(b"\xd2" + struct.pack(">i", obj)) - elif obj >= -2**(64 - 1): - fp.write(b"\xd3" + struct.pack(">q", obj)) - else: - raise UnsupportedTypeException("huge signed int") - else: - if obj <= 127: - fp.write(struct.pack("B", obj)) - elif obj <= 2**8 - 1: - fp.write(b"\xcc" + struct.pack("B", obj)) - elif obj <= 2**16 - 1: - fp.write(b"\xcd" + struct.pack(">H", obj)) - elif obj <= 2**32 - 1: - fp.write(b"\xce" + struct.pack(">I", obj)) - elif obj <= 2**64 - 1: - fp.write(b"\xcf" + struct.pack(">Q", obj)) - else: - raise UnsupportedTypeException("huge unsigned int") - - -def _pack_nil(obj, fp, options): - fp.write(b"\xc0") - - -def _pack_boolean(obj, fp, options): - fp.write(b"\xc3" if obj else b"\xc2") - - -def _pack_float(obj, fp, options): - float_precision = options.get('force_float_precision', _float_precision) - - if float_precision == "double": - fp.write(b"\xcb" + struct.pack(">d", obj)) - elif float_precision == "single": - fp.write(b"\xca" + struct.pack(">f", obj)) - else: - raise ValueError("invalid float precision") - - -def _pack_string(obj, fp, options): - obj = obj.encode('utf-8') - if len(obj) <= 31: - fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) - elif len(obj) <= 2**8 - 1: - fp.write(b"\xd9" + struct.pack("B", len(obj)) + obj) - elif len(obj) <= 2**16 - 1: - fp.write(b"\xda" + struct.pack(">H", len(obj)) + obj) - elif len(obj) <= 2**32 - 1: - fp.write(b"\xdb" + struct.pack(">I", len(obj)) + obj) - else: - raise UnsupportedTypeException("huge string") - - -def _pack_binary(obj, fp, options): - if len(obj) <= 2**8 - 1: - fp.write(b"\xc4" + struct.pack("B", len(obj)) + obj) - elif len(obj) <= 2**16 - 1: - fp.write(b"\xc5" + struct.pack(">H", len(obj)) + obj) - elif len(obj) <= 2**32 - 1: - fp.write(b"\xc6" + struct.pack(">I", len(obj)) + obj) - else: - raise UnsupportedTypeException("huge binary string") - - -def _pack_oldspec_raw(obj, fp, options): - if len(obj) <= 31: - fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) - elif len(obj) <= 2**16 - 1: - fp.write(b"\xda" + struct.pack(">H", len(obj)) + obj) - elif len(obj) <= 2**32 - 1: - fp.write(b"\xdb" + struct.pack(">I", len(obj)) + obj) - else: - raise UnsupportedTypeException("huge raw string") - - -def _pack_ext(obj, fp, options): - if len(obj.data) == 1: - fp.write(b"\xd4" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) == 2: - fp.write(b"\xd5" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) == 4: - fp.write(b"\xd6" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) == 8: - fp.write(b"\xd7" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) == 16: - fp.write(b"\xd8" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) <= 2**8 - 1: - fp.write(b"\xc7" + - struct.pack("BB", len(obj.data), obj.type & 0xff) + obj.data) - elif len(obj.data) <= 2**16 - 1: - fp.write(b"\xc8" + - struct.pack(">HB", len(obj.data), obj.type & 0xff) + obj.data) - elif len(obj.data) <= 2**32 - 1: - fp.write(b"\xc9" + - struct.pack(">IB", len(obj.data), obj.type & 0xff) + obj.data) - else: - raise UnsupportedTypeException("huge ext data") - - -def _pack_array(obj, fp, options): - if len(obj) <= 15: - fp.write(struct.pack("B", 0x90 | len(obj))) - elif len(obj) <= 2**16 - 1: - fp.write(b"\xdc" + struct.pack(">H", len(obj))) - elif len(obj) <= 2**32 - 1: - fp.write(b"\xdd" + struct.pack(">I", len(obj))) - else: - raise UnsupportedTypeException("huge array") - - for e in obj: - pack(e, fp, **options) - - -def _pack_map(obj, fp, options): - if len(obj) <= 15: - fp.write(struct.pack("B", 0x80 | len(obj))) - elif len(obj) <= 2**16 - 1: - fp.write(b"\xde" + struct.pack(">H", len(obj))) - elif len(obj) <= 2**32 - 1: - fp.write(b"\xdf" + struct.pack(">I", len(obj))) - else: - raise UnsupportedTypeException("huge array") - - for k, v in obj.items(): - pack(k, fp, **options) - pack(v, fp, **options) - -######################################## - - -# Pack for Python 2, with 'unicode' type, 'str' type, and 'long' type -def _pack2(obj, fp, **options): - """ - Serialize a Python object into MessagePack bytes. - - Args: - obj: a Python object - fp: a .write()-supporting file-like object - - Kwargs: - ext_handlers (dict): dictionary of Ext handlers, mapping a custom type - to a callable that packs an instance of the type - into an Ext object - force_float_precision (str): "single" to force packing floats as - IEEE-754 single-precision floats, - "double" to force packing floats as - IEEE-754 double-precision floats. - - Returns: - None. - - Raises: - UnsupportedType(PackException): - Object type not supported for packing. - - Example: - >>> f = open('test.bin', 'wb') - >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) - >>> - """ - global compatibility - - ext_handlers = options.get("ext_handlers") - - if obj is None: - _pack_nil(obj, fp, options) - elif ext_handlers and obj.__class__ in ext_handlers: - _pack_ext(ext_handlers[obj.__class__](obj), fp, options) - elif isinstance(obj, bool): - _pack_boolean(obj, fp, options) - elif isinstance(obj, (int, long)): - _pack_integer(obj, fp, options) - elif isinstance(obj, float): - _pack_float(obj, fp, options) - elif compatibility and isinstance(obj, unicode): - _pack_oldspec_raw(bytes(obj), fp, options) - elif compatibility and isinstance(obj, bytes): - _pack_oldspec_raw(obj, fp, options) - elif isinstance(obj, unicode): - _pack_string(obj, fp, options) - elif isinstance(obj, str): - _pack_binary(obj, fp, options) - elif isinstance(obj, (list, tuple)): - _pack_array(obj, fp, options) - elif isinstance(obj, dict): - _pack_map(obj, fp, options) - elif isinstance(obj, Ext): - _pack_ext(obj, fp, options) - elif ext_handlers: - # Linear search for superclass - t = next((t for t in ext_handlers.keys() if isinstance(obj, t)), None) - if t: - _pack_ext(ext_handlers[t](obj), fp, options) - else: - raise UnsupportedTypeException( - "unsupported type: %s" % str(type(obj))) - else: - raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) - - -# Pack for Python 3, with unicode 'str' type, 'bytes' type, and no 'long' type -def _pack3(obj, fp, **options): - """ - Serialize a Python object into MessagePack bytes. - - Args: - obj: a Python object - fp: a .write()-supporting file-like object - - Kwargs: - ext_handlers (dict): dictionary of Ext handlers, mapping a custom type - to a callable that packs an instance of the type - into an Ext object - force_float_precision (str): "single" to force packing floats as - IEEE-754 single-precision floats, - "double" to force packing floats as - IEEE-754 double-precision floats. - - Returns: - None. - - Raises: - UnsupportedType(PackException): - Object type not supported for packing. - - Example: - >>> f = open('test.bin', 'wb') - >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) - >>> - """ - global compatibility - - ext_handlers = options.get("ext_handlers") - - if obj is None: - _pack_nil(obj, fp, options) - elif ext_handlers and obj.__class__ in ext_handlers: - _pack_ext(ext_handlers[obj.__class__](obj), fp, options) - elif isinstance(obj, bool): - _pack_boolean(obj, fp, options) - elif isinstance(obj, int): - _pack_integer(obj, fp, options) - elif isinstance(obj, float): - _pack_float(obj, fp, options) - elif compatibility and isinstance(obj, str): - _pack_oldspec_raw(obj.encode('utf-8'), fp, options) - elif compatibility and isinstance(obj, bytes): - _pack_oldspec_raw(obj, fp, options) - elif isinstance(obj, str): - _pack_string(obj, fp, options) - elif isinstance(obj, bytes): - _pack_binary(obj, fp, options) - elif isinstance(obj, (list, tuple)): - _pack_array(obj, fp, options) - elif isinstance(obj, dict): - _pack_map(obj, fp, options) - elif isinstance(obj, Ext): - _pack_ext(obj, fp, options) - elif ext_handlers: - # Linear search for superclass - t = next((t for t in ext_handlers.keys() if isinstance(obj, t)), None) - if t: - _pack_ext(ext_handlers[t](obj), fp, options) - else: - raise UnsupportedTypeException( - "unsupported type: %s" % str(type(obj))) - else: - raise UnsupportedTypeException( - "unsupported type: %s" % str(type(obj))) - - -def _packb2(obj, **options): - """ - Serialize a Python object into MessagePack bytes. - - Args: - obj: a Python object - - Kwargs: - ext_handlers (dict): dictionary of Ext handlers, mapping a custom type - to a callable that packs an instance of the type - into an Ext object - force_float_precision (str): "single" to force packing floats as - IEEE-754 single-precision floats, - "double" to force packing floats as - IEEE-754 double-precision floats. - - Returns: - A 'str' containing serialized MessagePack bytes. - - Raises: - UnsupportedType(PackException): - Object type not supported for packing. - - Example: - >>> umsgpack.packb({u"compact": True, u"schema": 0}) - '\x82\xa7compact\xc3\xa6schema\x00' - >>> - """ - fp = io.BytesIO() - _pack2(obj, fp, **options) - return fp.getvalue() - - -def _packb3(obj, **options): - """ - Serialize a Python object into MessagePack bytes. - - Args: - obj: a Python object - - Kwargs: - ext_handlers (dict): dictionary of Ext handlers, mapping a custom type - to a callable that packs an instance of the type - into an Ext object - force_float_precision (str): "single" to force packing floats as - IEEE-754 single-precision floats, - "double" to force packing floats as - IEEE-754 double-precision floats. - - Returns: - A 'bytes' containing serialized MessagePack bytes. - - Raises: - UnsupportedType(PackException): - Object type not supported for packing. - - Example: - >>> umsgpack.packb({u"compact": True, u"schema": 0}) - b'\x82\xa7compact\xc3\xa6schema\x00' - >>> - """ - fp = io.BytesIO() - _pack3(obj, fp, **options) - return fp.getvalue() - -############################################################################# -# Unpacking -############################################################################# - - -def _read_except(fp, n): - data = fp.read(n) - if len(data) < n: - raise InsufficientDataException() - return data - - -def _unpack_integer(code, fp, options): - if (ord(code) & 0xe0) == 0xe0: - return struct.unpack("b", code)[0] - elif code == b'\xd0': - return struct.unpack("b", _read_except(fp, 1))[0] - elif code == b'\xd1': - return struct.unpack(">h", _read_except(fp, 2))[0] - elif code == b'\xd2': - return struct.unpack(">i", _read_except(fp, 4))[0] - elif code == b'\xd3': - return struct.unpack(">q", _read_except(fp, 8))[0] - elif (ord(code) & 0x80) == 0x00: - return struct.unpack("B", code)[0] - elif code == b'\xcc': - return struct.unpack("B", _read_except(fp, 1))[0] - elif code == b'\xcd': - return struct.unpack(">H", _read_except(fp, 2))[0] - elif code == b'\xce': - return struct.unpack(">I", _read_except(fp, 4))[0] - elif code == b'\xcf': - return struct.unpack(">Q", _read_except(fp, 8))[0] - raise Exception("logic error, not int: 0x%02x" % ord(code)) - - -def _unpack_reserved(code, fp, options): - if code == b'\xc1': - raise ReservedCodeException( - "encountered reserved code: 0x%02x" % ord(code)) - raise Exception( - "logic error, not reserved code: 0x%02x" % ord(code)) - - -def _unpack_nil(code, fp, options): - if code == b'\xc0': - return None - raise Exception("logic error, not nil: 0x%02x" % ord(code)) - - -def _unpack_boolean(code, fp, options): - if code == b'\xc2': - return False - elif code == b'\xc3': - return True - raise Exception("logic error, not boolean: 0x%02x" % ord(code)) - - -def _unpack_float(code, fp, options): - if code == b'\xca': - return struct.unpack(">f", _read_except(fp, 4))[0] - elif code == b'\xcb': - return struct.unpack(">d", _read_except(fp, 8))[0] - raise Exception("logic error, not float: 0x%02x" % ord(code)) - - -def _unpack_string(code, fp, options): - if (ord(code) & 0xe0) == 0xa0: - length = ord(code) & ~0xe0 - elif code == b'\xd9': - length = struct.unpack("B", _read_except(fp, 1))[0] - elif code == b'\xda': - length = struct.unpack(">H", _read_except(fp, 2))[0] - elif code == b'\xdb': - length = struct.unpack(">I", _read_except(fp, 4))[0] - else: - raise Exception("logic error, not string: 0x%02x" % ord(code)) - - # Always return raw bytes in compatibility mode - global compatibility - if compatibility: - return _read_except(fp, length) - - data = _read_except(fp, length) - try: - return bytes.decode(data, 'utf-8') - except UnicodeDecodeError: - if options.get("allow_invalid_utf8"): - return InvalidString(data) - raise InvalidStringException("unpacked string is invalid utf-8") - - -def _unpack_binary(code, fp, options): - if code == b'\xc4': - length = struct.unpack("B", _read_except(fp, 1))[0] - elif code == b'\xc5': - length = struct.unpack(">H", _read_except(fp, 2))[0] - elif code == b'\xc6': - length = struct.unpack(">I", _read_except(fp, 4))[0] - else: - raise Exception("logic error, not binary: 0x%02x" % ord(code)) - - return _read_except(fp, length) - - -def _unpack_ext(code, fp, options): - if code == b'\xd4': - length = 1 - elif code == b'\xd5': - length = 2 - elif code == b'\xd6': - length = 4 - elif code == b'\xd7': - length = 8 - elif code == b'\xd8': - length = 16 - elif code == b'\xc7': - length = struct.unpack("B", _read_except(fp, 1))[0] - elif code == b'\xc8': - length = struct.unpack(">H", _read_except(fp, 2))[0] - elif code == b'\xc9': - length = struct.unpack(">I", _read_except(fp, 4))[0] - else: - raise Exception("logic error, not ext: 0x%02x" % ord(code)) - - ext = Ext(ord(_read_except(fp, 1)), _read_except(fp, length)) - - # Unpack with ext handler, if we have one - ext_handlers = options.get("ext_handlers") - if ext_handlers and ext.type in ext_handlers: - ext = ext_handlers[ext.type](ext) - - return ext - - -def _unpack_array(code, fp, options): - if (ord(code) & 0xf0) == 0x90: - length = (ord(code) & ~0xf0) - elif code == b'\xdc': - length = struct.unpack(">H", _read_except(fp, 2))[0] - elif code == b'\xdd': - length = struct.unpack(">I", _read_except(fp, 4))[0] - else: - raise Exception("logic error, not array: 0x%02x" % ord(code)) - - return [_unpack(fp, options) for _ in xrange(length)] - - -def _deep_list_to_tuple(obj): - if isinstance(obj, list): - return tuple([_deep_list_to_tuple(e) for e in obj]) - return obj - - -def _unpack_map(code, fp, options): - if (ord(code) & 0xf0) == 0x80: - length = (ord(code) & ~0xf0) - elif code == b'\xde': - length = struct.unpack(">H", _read_except(fp, 2))[0] - elif code == b'\xdf': - length = struct.unpack(">I", _read_except(fp, 4))[0] - else: - raise Exception("logic error, not map: 0x%02x" % ord(code)) - - d = {} if not options.get('use_ordered_dict') \ - else collections.OrderedDict() - for _ in xrange(length): - # Unpack key - k = _unpack(fp, options) - - if isinstance(k, list): - # Attempt to convert list into a hashable tuple - k = _deep_list_to_tuple(k) - elif not isinstance(k, collections.Hashable): - raise UnhashableKeyException( - "encountered unhashable key: %s, %s" % (str(k), str(type(k)))) - elif k in d: - raise DuplicateKeyException( - "encountered duplicate key: %s, %s" % (str(k), str(type(k)))) - - # Unpack value - v = _unpack(fp, options) - - try: - d[k] = v - except TypeError: - raise UnhashableKeyException( - "encountered unhashable key: %s" % str(k)) - return d - - -def _unpack(fp, options): - code = _read_except(fp, 1) - return _unpack_dispatch_table[code](code, fp, options) - -######################################## - - -def _unpack2(fp, **options): - """ - Deserialize MessagePack bytes into a Python object. - - Args: - fp: a .read()-supporting file-like object - - Kwargs: - ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext - type to a callable that unpacks an instance of - Ext into an object - use_ordered_dict (bool): unpack maps into OrderedDict, instead of - unordered dict (default False) - allow_invalid_utf8 (bool): unpack invalid strings into instances of - InvalidString, for access to the bytes - (default False) - - Returns: - A Python object. - - Raises: - InsufficientDataException(UnpackException): - Insufficient data to unpack the serialized object. - InvalidStringException(UnpackException): - Invalid UTF-8 string encountered during unpacking. - ReservedCodeException(UnpackException): - Reserved code encountered during unpacking. - UnhashableKeyException(UnpackException): - Unhashable key encountered during map unpacking. - The serialized map cannot be deserialized into a Python dictionary. - DuplicateKeyException(UnpackException): - Duplicate key encountered during map unpacking. - - Example: - >>> f = open('test.bin', 'rb') - >>> umsgpack.unpackb(f) - {u'compact': True, u'schema': 0} - >>> - """ - return _unpack(fp, options) - - -def _unpack3(fp, **options): - """ - Deserialize MessagePack bytes into a Python object. - - Args: - fp: a .read()-supporting file-like object - - Kwargs: - ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext - type to a callable that unpacks an instance of - Ext into an object - use_ordered_dict (bool): unpack maps into OrderedDict, instead of - unordered dict (default False) - allow_invalid_utf8 (bool): unpack invalid strings into instances of - InvalidString, for access to the bytes - (default False) - - Returns: - A Python object. - - Raises: - InsufficientDataException(UnpackException): - Insufficient data to unpack the serialized object. - InvalidStringException(UnpackException): - Invalid UTF-8 string encountered during unpacking. - ReservedCodeException(UnpackException): - Reserved code encountered during unpacking. - UnhashableKeyException(UnpackException): - Unhashable key encountered during map unpacking. - The serialized map cannot be deserialized into a Python dictionary. - DuplicateKeyException(UnpackException): - Duplicate key encountered during map unpacking. - - Example: - >>> f = open('test.bin', 'rb') - >>> umsgpack.unpackb(f) - {'compact': True, 'schema': 0} - >>> - """ - return _unpack(fp, options) - - -# For Python 2, expects a str object -def _unpackb2(s, **options): - """ - Deserialize MessagePack bytes into a Python object. - - Args: - s: a 'str' or 'bytearray' containing serialized MessagePack bytes - - Kwargs: - ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext - type to a callable that unpacks an instance of - Ext into an object - use_ordered_dict (bool): unpack maps into OrderedDict, instead of - unordered dict (default False) - allow_invalid_utf8 (bool): unpack invalid strings into instances of - InvalidString, for access to the bytes - (default False) - - Returns: - A Python object. - - Raises: - TypeError: - Packed data type is neither 'str' nor 'bytearray'. - InsufficientDataException(UnpackException): - Insufficient data to unpack the serialized object. - InvalidStringException(UnpackException): - Invalid UTF-8 string encountered during unpacking. - ReservedCodeException(UnpackException): - Reserved code encountered during unpacking. - UnhashableKeyException(UnpackException): - Unhashable key encountered during map unpacking. - The serialized map cannot be deserialized into a Python dictionary. - DuplicateKeyException(UnpackException): - Duplicate key encountered during map unpacking. - - Example: - >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') - {u'compact': True, u'schema': 0} - >>> - """ - if not isinstance(s, (str, bytearray)): - raise TypeError("packed data must be type 'str' or 'bytearray'") - return _unpack(io.BytesIO(s), options) - - -# For Python 3, expects a bytes object -def _unpackb3(s, **options): - """ - Deserialize MessagePack bytes into a Python object. - - Args: - s: a 'bytes' or 'bytearray' containing serialized MessagePack bytes - - Kwargs: - ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext - type to a callable that unpacks an instance of - Ext into an object - use_ordered_dict (bool): unpack maps into OrderedDict, instead of - unordered dict (default False) - allow_invalid_utf8 (bool): unpack invalid strings into instances of - InvalidString, for access to the bytes - (default False) - - Returns: - A Python object. - - Raises: - TypeError: - Packed data type is neither 'bytes' nor 'bytearray'. - InsufficientDataException(UnpackException): - Insufficient data to unpack the serialized object. - InvalidStringException(UnpackException): - Invalid UTF-8 string encountered during unpacking. - ReservedCodeException(UnpackException): - Reserved code encountered during unpacking. - UnhashableKeyException(UnpackException): - Unhashable key encountered during map unpacking. - The serialized map cannot be deserialized into a Python dictionary. - DuplicateKeyException(UnpackException): - Duplicate key encountered during map unpacking. - - Example: - >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') - {'compact': True, 'schema': 0} - >>> - """ - if not isinstance(s, (bytes, bytearray)): - raise TypeError("packed data must be type 'bytes' or 'bytearray'") - return _unpack(io.BytesIO(s), options) - -############################################################################# -# Module Initialization -############################################################################# - - -def __init(): - # pylint: disable=global-variable-undefined - - global pack - global packb - global unpack - global unpackb - global dump - global dumps - global load - global loads - global compatibility - global _float_precision - global _unpack_dispatch_table - global xrange - - # Compatibility mode for handling strings/bytes with the old specification - compatibility = False - - # Auto-detect system float precision - if sys.float_info.mant_dig == 53: - _float_precision = "double" - else: - _float_precision = "single" - - # Map packb and unpackb to the appropriate version - if sys.version_info[0] == 3: - pack = _pack3 - packb = _packb3 - dump = _pack3 - dumps = _packb3 - unpack = _unpack3 - unpackb = _unpackb3 - load = _unpack3 - loads = _unpackb3 - xrange = range # pylint: disable=redefined-builtin - else: - pack = _pack2 - packb = _packb2 - dump = _pack2 - dumps = _packb2 - unpack = _unpack2 - unpackb = _unpackb2 - load = _unpack2 - loads = _unpackb2 - - # Build a dispatch table for fast lookup of unpacking function - - _unpack_dispatch_table = {} - # Fix uint - for code in range(0, 0x7f + 1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer - # Fix map - for code in range(0x80, 0x8f + 1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_map - # Fix array - for code in range(0x90, 0x9f + 1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_array - # Fix str - for code in range(0xa0, 0xbf + 1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string - # Nil - _unpack_dispatch_table[b'\xc0'] = _unpack_nil - # Reserved - _unpack_dispatch_table[b'\xc1'] = _unpack_reserved - # Boolean - _unpack_dispatch_table[b'\xc2'] = _unpack_boolean - _unpack_dispatch_table[b'\xc3'] = _unpack_boolean - # Bin - for code in range(0xc4, 0xc6 + 1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_binary - # Ext - for code in range(0xc7, 0xc9 + 1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext - # Float - _unpack_dispatch_table[b'\xca'] = _unpack_float - _unpack_dispatch_table[b'\xcb'] = _unpack_float - # Uint - for code in range(0xcc, 0xcf + 1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer - # Int - for code in range(0xd0, 0xd3 + 1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer - # Fixext - for code in range(0xd4, 0xd8 + 1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext - # String - for code in range(0xd9, 0xdb + 1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string - # Array - _unpack_dispatch_table[b'\xdc'] = _unpack_array - _unpack_dispatch_table[b'\xdd'] = _unpack_array - # Map - _unpack_dispatch_table[b'\xde'] = _unpack_map - _unpack_dispatch_table[b'\xdf'] = _unpack_map - # Negative fixint - for code in range(0xe0, 0xff + 1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer - - -__init() diff --git a/src/helper_msgcoding.py b/src/helper_msgcoding.py index cc632ffa..f7e30ee2 100644 --- a/src/helper_msgcoding.py +++ b/src/helper_msgcoding.py @@ -5,17 +5,11 @@ Message encoding end decoding functions import string import zlib -try: - import msgpack -except ImportError: - try: - import umsgpack as msgpack - except ImportError: - import fallback.umsgpack.umsgpack as msgpack import messagetypes from bmconfigparser import BMConfigParser from debug import logger +from fallback import msgpack from tr import _translate BITMESSAGE_ENCODING_IGNORE = 0