60 lines
1.4 KiB
Python
60 lines
1.4 KiB
Python
import hashlib
|
|
from threading import Semaphore
|
|
|
|
|
|
CHUNK_SIZE = 1024 * 1024 * 4 # 4 mb chunks
|
|
|
|
|
|
def hash_chunk(data):
|
|
h = hashlib.sha256()
|
|
h.update(data)
|
|
return h.hexdigest()
|
|
|
|
|
|
class FilePool(object):
|
|
def __init__(self, instances, open_function):
|
|
"""
|
|
Pool of many files handle
|
|
"""
|
|
self.instances = [open_function() for i in range(0, instances)]
|
|
self.lock = Semaphore(instances)
|
|
|
|
def get(self):
|
|
"""
|
|
Return a context-manager that contains the file object
|
|
"""
|
|
return FilePoolSlice(self)
|
|
|
|
def close(self):
|
|
for f in self.instances:
|
|
f.close()
|
|
|
|
def _get_locked(self):
|
|
"""
|
|
Acquire and return an instance from the pool
|
|
"""
|
|
self.lock.acquire()
|
|
return self.instances.pop()
|
|
|
|
def _return_locked(self, instance):
|
|
"""
|
|
Release an instance emitted by get_locked
|
|
"""
|
|
self.instances.append(instance)
|
|
self.lock.release()
|
|
|
|
|
|
class FilePoolSlice(object):
|
|
def __init__(self, pool):
|
|
self.pool = pool
|
|
self.mine = None
|
|
|
|
def __enter__(self):
|
|
#TODO don't use me more than once
|
|
self.mine = self.pool._get_locked()
|
|
self.mine.seek(0)
|
|
return self.mine
|
|
|
|
def __exit__(self, exc_type, exc_value, exc_traceback):
|
|
self.pool._return_locked(self.mine)
|