관리-도구

편집 파일: buf.py

"""Module with a simple buffer implementation using the memory manager"""
import sys

__all__ = ["SlidingWindowMapBuffer"]

class SlidingWindowMapBuffer:

"""A buffer like object which allows direct byte-wise object and slicing into
    memory of a mapped file. The mapping is controlled by the provided cursor.

The buffer is relative, that is if you map an offset, index 0 will map to the
    first byte at the offset you used during initialization or begin_access

**Note:** Although this type effectively hides the fact that there are mapped windows
    underneath, it can unfortunately not be used in any non-pure python method which
    needs a buffer or string"""
    __slots__ = (
        '_c',           # our cursor
        '_size',        # our supposed size
    )

def __init__(self, cursor=None, offset=0, size=sys.maxsize, flags=0):
        """Initalize the instance to operate on the given cursor.
        :param cursor: if not None, the associated cursor to the file you want to access
            If None, you have call begin_access before using the buffer and provide a cursor
        :param offset: absolute offset in bytes
        :param size: the total size of the mapping. Defaults to the maximum possible size
            From that point on, the __len__ of the buffer will be the given size or the file size.
            If the size is larger than the mappable area, you can only access the actually available
            area, although the length of the buffer is reported to be your given size.
            Hence it is in your own interest to provide a proper size !
        :param flags: Additional flags to be passed to os.open
        :raise ValueError: if the buffer could not achieve a valid state"""
        self._c = cursor
        if cursor and not self.begin_access(cursor, offset, size, flags):
            raise ValueError("Failed to allocate the buffer - probably the given offset is out of bounds")
        # END handle offset

def __del__(self):
        self.end_access()

def __enter__(self):
        return self

def __exit__(self, exc_type, exc_value, traceback):
        self.end_access()

def __len__(self):
        return self._size

def __getitem__(self, i):
        if isinstance(i, slice):
            return self.__getslice__(i.start or 0, i.stop or self._size)
        c = self._c
        assert c.is_valid()
        if i < 0:
            i = self._size + i
        if not c.includes_ofs(i):
            c.use_region(i, 1)
        # END handle region usage
        return c.buffer()[i - c.ofs_begin()]

def __getslice__(self, i, j):
        c = self._c
        # fast path, slice fully included - safes a concatenate operation and
        # should be the default
        assert c.is_valid()
        if i < 0:
            i = self._size + i
        if j == sys.maxsize:
            j = self._size
        if j < 0:
            j = self._size + j
        if (c.ofs_begin() <= i) and (j < c.ofs_end()):
            b = c.ofs_begin()
            return c.buffer()[i - b:j - b]
        else:
            l = j - i                 # total length
            ofs = i
            # It's fastest to keep tokens and join later, especially in py3, which was 7 times slower
            # in the previous iteration of this code
            md = list()
            while l:
                c.use_region(ofs, l)
                assert c.is_valid()
                d = c.buffer()[:l]
                ofs += len(d)
                l -= len(d)
                # Make sure we don't keep references, as c.use_region() might attempt to free resources, but
                # can't unless we use pure bytes
                if hasattr(d, 'tobytes'):
                    d = d.tobytes()
                md.append(d)
            # END while there are bytes to read
            return bytes().join(md)
        # END fast or slow path
    #{ Interface

def begin_access(self, cursor=None, offset=0, size=sys.maxsize, flags=0):
        """Call this before the first use of this instance. The method was already
        called by the constructor in case sufficient information was provided.

For more information no the parameters, see the __init__ method
        :param path: if cursor is None the existing one will be used.
        :return: True if the buffer can be used"""
        if cursor:
            self._c = cursor
        # END update our cursor

# reuse existing cursors if possible
        if self._c is not None and self._c.is_associated():
            res = self._c.use_region(offset, size, flags).is_valid()
            if res:
                # if given size is too large or default, we computer a proper size
                # If its smaller, we assume the combination between offset and size
                # as chosen by the user is correct and use it !
                # If not, the user is in trouble.
                if size > self._c.file_size():
                    size = self._c.file_size() - offset
                # END handle size
                self._size = size
            # END set size
            return res
        # END use our cursor
        return False

def end_access(self):
        """Call this method once you are done using the instance. It is automatically
        called on destruction, and should be called just in time to allow system
        resources to be freed.

Once you called end_access, you must call begin access before reusing this instance!"""
        self._size = 0
        if self._c is not None:
            self._c.unuse_region()
        # END unuse region

def cursor(self):
        """:return: the currently set cursor which provides access to the data"""
        return self._c

#}END interface