filesystem flake8 fixes

This commit is contained in:
lakshyacis 2019-09-24 14:50:20 +05:30
parent 433cb9818b
commit 6f910f67c0
No known key found for this signature in database
GPG Key ID: D2C539C8EC63E9EB
3 changed files with 75 additions and 15 deletions

View File

@ -3,11 +3,11 @@ from os import listdir, makedirs, path, remove, rmdir
import string import string
from threading import RLock from threading import RLock
import time import time
import traceback
from paths import lookupAppdataFolder from paths import lookupAppdataFolder
from storage import InventoryStorage, InventoryItem from storage import InventoryStorage, InventoryItem
class FilesystemInventory(InventoryStorage): class FilesystemInventory(InventoryStorage):
topDir = "inventory" topDir = "inventory"
objectDir = "objects" objectDir = "objects"
@ -23,7 +23,9 @@ class FilesystemInventory(InventoryStorage):
raise IOError("%s exists but it's not a directory" % (createDir)) raise IOError("%s exists but it's not a directory" % (createDir))
else: else:
makedirs(createDir) makedirs(createDir)
self.lock = RLock() # Guarantees that two receiveDataThreads don't receive and process the same message concurrently (probably sent by a malicious individual) # Guarantees that two receiveDataThreads don't receive and process the same message
# concurrently (probably sent by a malicious individual)
self.lock = RLock()
self._inventory = {} self._inventory = {}
self._load() self._load()
@ -53,9 +55,25 @@ class FilesystemInventory(InventoryStorage):
except OSError: except OSError:
pass pass
try: try:
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.metadataFilename), 'w') as f: with open(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hash),
FilesystemInventory.metadataFilename,
),
"w",
) as f:
f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag))) f.write("%s,%s,%s,%s," % (value.type, value.stream, value.expires, hexlify(value.tag)))
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.dataFilename), 'w') as f: with open(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hash),
FilesystemInventory.dataFilename,
),
"w",
) as f:
f.write(value.payload) f.write(value.payload)
except IOError: except IOError:
raise KeyError raise KeyError
@ -73,11 +91,21 @@ class FilesystemInventory(InventoryStorage):
pass pass
with self.lock: with self.lock:
try: try:
remove(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.metadataFilename)) remove(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hash),
FilesystemInventory.metadataFilename))
except IOError: except IOError:
pass pass
try: try:
remove(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hash), FilesystemInventory.dataFilename)) remove(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hash),
FilesystemInventory.dataFilename))
except IOError: except IOError:
pass pass
try: try:
@ -103,10 +131,12 @@ class FilesystemInventory(InventoryStorage):
try: try:
objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId) objectType, streamNumber, expiresTime, tag = self.getMetadata(hashId)
try: try:
newInventory[streamNumber][hashId] = InventoryItem(objectType, streamNumber, None, expiresTime, tag) newInventory[streamNumber][hashId] = InventoryItem(
objectType, streamNumber, None, expiresTime, tag)
except KeyError: except KeyError:
newInventory[streamNumber] = {} newInventory[streamNumber] = {}
newInventory[streamNumber][hashId] = InventoryItem(objectType, streamNumber, None, expiresTime, tag) newInventory[streamNumber][hashId] = InventoryItem(
objectType, streamNumber, None, expiresTime, tag)
except KeyError: except KeyError:
print "error loading %s" % (hexlify(hashId)) print "error loading %s" % (hexlify(hashId))
pass pass
@ -122,20 +152,37 @@ class FilesystemInventory(InventoryStorage):
def getData(self, hashId): def getData(self, hashId):
try: try:
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashId), FilesystemInventory.dataFilename), 'r') as f: with open(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashId),
FilesystemInventory.dataFilename,
),
"r",
) as f:
return f.read() return f.read()
except IOError: except IOError:
raise AttributeError raise AttributeError
def getMetadata(self, hashId): def getMetadata(self, hashId):
try: try:
with open(path.join(self.baseDir, FilesystemInventory.objectDir, hexlify(hashId), FilesystemInventory.metadataFilename), 'r') as f: with open(
path.join(
self.baseDir,
FilesystemInventory.objectDir,
hexlify(hashId),
FilesystemInventory.metadataFilename,
),
"r",
) as f:
objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4) objectType, streamNumber, expiresTime, tag, undef = string.split(f.read(), ",", 4)
return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)] return [int(objectType), int(streamNumber), int(expiresTime), unhexlify(tag)]
except IOError: except IOError:
raise KeyError raise KeyError
def by_type_and_tag(self, objectType, tag): def by_type_and_tag(self, objectType, tag):
"""Get a list of objects filtered by object type and tag"""
retval = [] retval = []
for stream, streamDict in self._inventory: for stream, streamDict in self._inventory:
for hashId, item in streamDict: for hashId, item in streamDict:

View File

@ -10,9 +10,17 @@ from storage import InventoryStorage, InventoryItem
class SqliteInventory(InventoryStorage): class SqliteInventory(InventoryStorage):
def __init__(self): def __init__(self):
super(self.__class__, self).__init__() super(self.__class__, self).__init__()
self._inventory = {} #of objects (like msg payloads and pubkey payloads) Does not include protocol headers (the first 24 bytes of each packet). # of objects (like msg payloads and pubkey payloads)
self._objects = {} # cache for existing objects, used for quick lookups if we have an object. This is used for example whenever we receive an inv message from a peer to check to see what items are new to us. We don't delete things out of it; instead, the singleCleaner thread clears and refills it. # Does not include protocol headers (the first 24 bytes of each packet).
self.lock = RLock() # Guarantees that two receiveDataThreads don't receive and process the same message concurrently (probably sent by a malicious individual) self._inventory = {}
# cache for existing objects, used for quick lookups if we have an object.
# This is used for example whenever we receive an inv message from a peer
# to check to see what items are new to us.
# We don't delete things out of it; instead, the singleCleaner thread clears and refills it.
self._objects = {}
# Guarantees that two receiveDataThreads don't receive and process the same message concurrently
# (probably sent by a malicious individual)
self.lock = RLock()
def __contains__(self, hash): def __contains__(self, hash):
with self.lock: with self.lock:

View File

@ -3,6 +3,7 @@ import collections
InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload expires tag') InventoryItem = collections.namedtuple('InventoryItem', 'type stream payload expires tag')
class Storage(object): class Storage(object):
"""Base class for storing inventory (extendable for other items to store)"""
pass pass
# def __init__(self): # def __init__(self):
# super(self.__class__, self).__init__() # super(self.__class__, self).__init__()
@ -31,15 +32,19 @@ class InventoryStorage(Storage, collections.MutableMapping):
raise NotImplementedError raise NotImplementedError
def by_type_and_tag(self, objectType, tag): def by_type_and_tag(self, objectType, tag):
"""Return objects filtered by object type and tag"""
raise NotImplementedError raise NotImplementedError
def unexpired_hashes_by_stream(self, stream): def unexpired_hashes_by_stream(self, stream):
"""Return unexpired inventory vectors filtered by stream"""
raise NotImplementedError raise NotImplementedError
def flush(self): def flush(self):
"""Flush cache"""
raise NotImplementedError raise NotImplementedError
def clean(self): def clean(self):
"""Free memory / perform garbage collection"""
raise NotImplementedError raise NotImplementedError
class MailboxStorage(Storage, collections.MutableMapping): class MailboxStorage(Storage, collections.MutableMapping):