cis-kuldeep/PyBitmessage-2021-04-27
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

filesystem flake8 fixes

Browse Source
This commit is contained in:
lakshyacis 2019-09-24 14:50:20 +05:30
parent 433cb9818b
commit 6f910f67c0
No known key found for this signature in database
GPG Key ID: D2C539C8EC63E9EB
3 changed files with 75 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
src/storage/filesystem.py
View File

@ -3,11 +3,11 @@ from os import listdir, makedirs, path, remove, rmdir
import string
from threading import RLock
import time
import traceback
from paths import lookupAppdataFolder
from storage import InventoryStorage, InventoryItem
class FilesystemInventory(InventoryStorage):
topDir = "inventory"
objectDir = "objects"
@ -23,7 +23,9 @@ class FilesystemInventory(InventoryStorage):
raise IOError("%s exists but it's not a directory" % (createDir))
else:
makedirs(createDir)
self.lock = RLock() # Guarantees that two receiveDataThreads don't receive and process the same message concurrently (probably sent by a malicious individual)
# Guarantees that two receiveDataThreads don't receive and process the same message
# concurrently (probably sent by a malicious individual)
self.lock = RLock()
self._inventory = {}
self._load()
@ -53,9 +55,25 @@ class FilesystemInventory(InventoryStorage):
except OSError:
pass
try:
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.metadataFilename), 'w') as f:
with open(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hash),
FilesystemInventory.metadataFilename,
),
"w",
) as f:
f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag)))
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.dataFilename), 'w') as f:
with open(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hash),
FilesystemInventory.dataFilename,
),
"w",
) as f:
f.write(value.payload)
except IOError:
raise KeyError
@ -73,11 +91,21 @@ class FilesystemInventory(InventoryStorage):
pass
with self.lock:
try:
remove(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.metadataFilename))
remove(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hash),
FilesystemInventory.metadataFilename))
except IOError:
pass
try:
remove(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.dataFilename))
remove(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hash),
FilesystemInventory.dataFilename))
except IOError:
pass
try:
@ -88,7 +116,7 @@ class FilesystemInventory(InventoryStorage):
def __iter__(self):
elems = []
for streamDict in self._inventory.values():
elems.extend (streamDict.keys())
elems.extend(streamDict.keys())
return elems.__iter__()
def __len__(self):
@ -103,10 +131,12 @@ class FilesystemInventory(InventoryStorage):
try:
objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId)
try:
newInventory[streamNumber][hashId] = InventoryItem(objectType, streamNumber, None, expiresTime, tag)
newInventory[streamNumber][hashId] = InventoryItem(
objectType, streamNumber, None, expiresTime, tag)
except KeyError:
newInventory[streamNumber] = {}
newInventory[streamNumber][hashId] = InventoryItem(objectType, streamNumber, None, expiresTime, tag)
newInventory[streamNumber][hashId] = InventoryItem(
objectType, streamNumber, None, expiresTime, tag)
except KeyError:
print "error loading %s" % (hexlify(hashId))
pass
@ -122,25 +152,42 @@ class FilesystemInventory(InventoryStorage):
def getData(self, hashId):
try:
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashId), FilesystemInventory.dataFilename), 'r') as f:
with open(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashId),
FilesystemInventory.dataFilename,
),
"r",
) as f:
return f.read()
except IOError:
raise AttributeError
def getMetadata(self, hashId):
try:
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashId), FilesystemInventory.metadataFilename), 'r') as f:
with open(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashId),
FilesystemInventory.metadataFilename,
),
"r",
) as f:
objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4)
return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)]
except IOError:
raise KeyError
def by_type_and_tag(self, objectType, tag):
"""Get a list of objects filtered by object type and tag"""
retval = []
for stream, streamDict in self._inventory:
for hashId, item in streamDict:
if item.type == objectType and item.tag == tag:
try:
try:
if item.payload is None:
item.payload = self.getData(hashId)
except IOError:

14
src/storage/sqlite.py
View File

@ -10,9 +10,17 @@ from storage import InventoryStorage, InventoryItem
class SqliteInventory(InventoryStorage):
def __init__(self):
super(self.__class__, self).__init__()
self._inventory = {} #of objects (like msg payloads and pubkey payloads) Does not include protocol headers (the first 24 bytes of each packet).
self._objects = {} # cache for existing objects, used for quick lookups if we have an object. This is used for example whenever we receive an inv message from a peer to check to see what items are new to us. We don't delete things out of it; instead, the singleCleaner thread clears and refills it.
self.lock = RLock() # Guarantees that two receiveDataThreads don't receive and process the same message concurrently (probably sent by a malicious individual)
# of objects (like msg payloads and pubkey payloads)
# Does not include protocol headers (the first 24 bytes of each packet).
self._inventory = {}
# cache for existing objects, used for quick lookups if we have an object.
# This is used for example whenever we receive an inv message from a peer
# to check to see what items are new to us.
# We don't delete things out of it; instead, the singleCleaner thread clears and refills it.
self._objects = {}
# Guarantees that two receiveDataThreads don't receive and process the same message concurrently
# (probably sent by a malicious individual)
self.lock = RLock()
def __contains__(self, hash):
with self.lock:

5
src/storage/storage.py
View File

@ -3,6 +3,7 @@ import collections
InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload expires tag')
class Storage(object):
"""Base class for storing inventory (extendable for other items to store)"""
pass
# def __init__(self):
# super(self.__class__, self).__init__()
@ -31,15 +32,19 @@ class InventoryStorage(Storage, collections.MutableMapping):
raise NotImplementedError
def by_type_and_tag(self, objectType, tag):
"""Return objects filtered by object type and tag"""
raise NotImplementedError
def unexpired_hashes_by_stream(self, stream):
"""Return unexpired inventory vectors filtered by stream"""
raise NotImplementedError
def flush(self):
"""Flush cache"""
raise NotImplementedError
def clean(self):
"""Free memory / perform garbage collection"""
raise NotImplementedError
class MailboxStorage(Storage, collections.MutableMapping):