Python os.sendfile() Examples

def _sendfile_cb(self, fut, out_fd, in_fd,
                     offset, count, loop, registered):
        if registered:
            loop.remove_writer(out_fd)
        if fut.cancelled():
            return

        try:
            n = os.sendfile(out_fd, in_fd, offset, count)
            if n == 0:  # EOF reached
                n = count
        except (BlockingIOError, InterruptedError):
            n = 0
        except Exception as exc:
            fut.set_exception(exc)
            return

        if n < count:
            loop.add_writer(out_fd, self._sendfile_cb, fut, out_fd, in_fd,
                            offset + n, count - n, loop, True)
        else:
            fut.set_result(None) 

def sendfile(self, file, offset=0, count=None):
        """sendfile(file[, offset[, count]]) -> sent

        Send a file until EOF is reached by using high-performance
        os.sendfile() and return the total number of bytes which
        were sent.
        *file* must be a regular file object opened in binary mode.
        If os.sendfile() is not available (e.g. Windows) or file is
        not a regular file socket.send() will be used instead.
        *offset* tells from where to start reading the file.
        If specified, *count* is the total number of bytes to transmit
        as opposed to sending the file until EOF is reached.
        File position is updated on return or also in case of error in
        which case file.tell() can be used to figure out the number of
        bytes which were sent.
        The socket must be of SOCK_STREAM type.
        Non-blocking sockets are not supported.
        """
        try:
            return self._sendfile_use_sendfile(file, offset, count)
        except _GiveupOnSendfile:
            return self._sendfile_use_send(file, offset, count) 

def _import_sendfile():
    # By default attempt to use os.sendfile introduced in Python 3.3:
    # http://bugs.python.org/issue10882
    # ...otherwise fallback on using third-party pysendfile module:
    # https://github.com/giampaolo/pysendfile/
    if os.name == 'posix':
        try:
            return os.sendfile  # py >= 3.3
        except AttributeError:
            try:
                import sendfile as sf
                # dirty hack to detect whether old 1.2.4 version is installed
                if hasattr(sf, 'has_sf_hdtr'):
                    raise ImportError
                return sf.sendfile
            except ImportError:
                pass 

def get_repr_info(self, as_str=False, extra_info={}):
        info = OrderedDict()
        info['id'] = id(self)
        info['addr'] = "%s:%s" % (self.remote_ip, self.remote_port)
        if _is_ssl_sock(self.socket):
            info['ssl'] = True
        if self.username:
            info['user'] = self.username
        # If threads are involved sometimes "self" may be None (?!?).
        dc = getattr(self, 'data_channel', None)
        if dc is not None:
            if _is_ssl_sock(dc.socket):
                info['ssl-data'] = True
            if dc.file_obj:
                if self.data_channel.receive:
                    info['sending-file'] = dc.file_obj
                    if dc.use_sendfile():
                        info['use-sendfile(2)'] = True
                else:
                    info['receiving-file'] = dc.file_obj
                info['bytes-trans'] = dc.get_transmitted_bytes()
        info.update(extra_info)
        if as_str:
            return ', '.join(['%s=%r' % (k, v) for (k, v) in info.items()])
        return info 

def _use_sendfile(self, producer):
        if not self.cmd_channel.use_sendfile:
            debug("starting transfer not using sendfile(2) as per server "
                  "config", self)
            return False
        if not isinstance(producer, FileProducer):
            debug("starting transfer not using sendfile(2) (directory "
                  "listing)", self)
            return False
        else:
            if not hasattr(self.file_obj, "fileno"):
                debug("starting transfer not using sendfile(2) %r has no "
                      "fileno() method" % self.file_obj, self)
                return False
        if not producer.type == 'i':
            debug("starting transfer not using sendfile(2) (text file "
                  "transfer)", self)
            return False
        debug("starting transfer using sendfile()", self)
        return True 

def sendfile(self, file, offset=0, count=None):
        """sendfile(file[, offset[, count]]) -> sent

        Send a file until EOF is reached by using high-performance
        os.sendfile() and return the total number of bytes which
        were sent.
        *file* must be a regular file object opened in binary mode.
        If os.sendfile() is not available (e.g. Windows) or file is
        not a regular file socket.send() will be used instead.
        *offset* tells from where to start reading the file.
        If specified, *count* is the total number of bytes to transmit
        as opposed to sending the file until EOF is reached.
        File position is updated on return or also in case of error in
        which case file.tell() can be used to figure out the number of
        bytes which were sent.
        The socket must be of SOCK_STREAM type.
        Non-blocking sockets are not supported.

        .. versionadded:: 1.1rc4
           Added in Python 3.5, but available under all Python 3 versions in
           gevent.
        """
        return self._sendfile_use_send(file, offset, count)

    # get/set_inheritable new in 3.4 

def _import_sendfile():
    # By default attempt to use os.sendfile introduced in Python 3.3:
    # http://bugs.python.org/issue10882
    # ...otherwise fallback on using third-party pysendfile module:
    # https://github.com/giampaolo/pysendfile/
    if os.name == 'posix':
        try:
            return os.sendfile  # py >= 3.3
        except AttributeError:
            try:
                import sendfile as sf
                # dirty hack to detect whether old 1.2.4 version is installed
                if hasattr(sf, 'has_sf_hdtr'):
                    raise ImportError
                return sf.sendfile
            except ImportError:
                pass 

def _use_sendfile(self, producer):
        if not self.cmd_channel.use_sendfile:
            debug("starting transfer not using sendfile(2) as per server "
                  "config", self)
            return False
        if not isinstance(producer, FileProducer):
            debug("starting transfer not using sendfile(2) (directory "
                  "listing)", self)
            return False
        else:
            if not hasattr(self.file_obj, "fileno"):
                debug("starting transfer not using sendfile(2) %r has no "
                      "fileno() method" % self.file_obj, self)
                return False
        if not producer.type == 'i':
            debug("starting transfer not using sendfile(2) (text file "
                  "transfer)", self)
            return False
        debug("starting transfer using sendfile()", self)
        return True 

def _import_sendfile():
    # By default attempt to use os.sendfile introduced in Python 3.3:
    # http://bugs.python.org/issue10882
    # ...otherwise fallback on using third-party pysendfile module:
    # https://github.com/giampaolo/pysendfile/
    if os.name == 'posix':
        try:
            return os.sendfile  # py >= 3.3
        except AttributeError:
            try:
                import sendfile as sf
                # dirty hack to detect whether old 1.2.4 version is installed
                if hasattr(sf, 'has_sf_hdtr'):
                    raise ImportError
                return sf.sendfile
            except ImportError:
                pass
    return None 

def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]):
        """A higher level wrapper representing how an application is
        supposed to use sendfile().
        """
        while 1:
            try:
                if self.SUPPORT_HEADERS_TRAILERS:
                    return os.sendfile(sock, file, offset, nbytes, headers,
                                       trailers)
                else:
                    return os.sendfile(sock, file, offset, nbytes)
            except OSError as err:
                if err.errno == errno.ECONNRESET:
                    # disconnected
                    raise
                elif err.errno in (errno.EAGAIN, errno.EBUSY):
                    # we have to retry send data
                    continue
                else:
                    raise 

def test_offset_overflow(self):
        # specify an offset > file size
        offset = len(self.DATA) + 4096
        try:
            sent = os.sendfile(self.sockno, self.fileno, offset, 4096)
        except OSError as e:
            # Solaris can raise EINVAL if offset >= file length, ignore.
            if e.errno != errno.EINVAL:
                raise
        else:
            self.assertEqual(sent, 0)
        self.client.shutdown(socket.SHUT_RDWR)
        self.client.close()
        self.server.wait()
        data = self.server.handler_instance.get_data()
        self.assertEqual(data, b'') 

def test_headers(self):
        total_sent = 0
        sent = os.sendfile(self.sockno, self.fileno, 0, 4096,
                            headers=[b"x" * 512])
        total_sent += sent
        offset = 4096
        nbytes = 4096
        while 1:
            sent = self.sendfile_wrapper(self.sockno, self.fileno,
                                                    offset, nbytes)
            if sent == 0:
                break
            total_sent += sent
            offset += sent

        expected_data = b"x" * 512 + self.DATA
        self.assertEqual(total_sent, len(expected_data))
        self.client.close()
        self.server.wait()
        data = self.server.handler_instance.get_data()
        self.assertEqual(hash(data), hash(expected_data)) 

def _sock_sendfile_native(self, sock, file, offset, count):
        try:
            os.sendfile
        except AttributeError as exc:
            raise events.SendfileNotAvailableError(
                "os.sendfile() is not available")
        try:
            fileno = file.fileno()
        except (AttributeError, io.UnsupportedOperation) as err:
            raise events.SendfileNotAvailableError("not a regular file")
        try:
            fsize = os.fstat(fileno).st_size
        except OSError as err:
            raise events.SendfileNotAvailableError("not a regular file")
        blocksize = count if count else fsize
        if not blocksize:
            return 0  # empty file

        fut = self.create_future()
        self._sock_sendfile_native_impl(fut, None, sock, fileno,
                                        offset, count, blocksize, 0)
        return await fut 

def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]):
        """A higher level wrapper representing how an application is
        supposed to use sendfile().
        """
        while 1:
            try:
                if self.SUPPORT_HEADERS_TRAILERS:
                    return os.sendfile(sock, file, offset, nbytes, headers,
                                       trailers)
                else:
                    return os.sendfile(sock, file, offset, nbytes)
            except OSError as err:
                if err.errno == errno.ECONNRESET:
                    # disconnected
                    raise
                elif err.errno in (errno.EAGAIN, errno.EBUSY):
                    # we have to retry send data
                    continue
                else:
                    raise 

def test_offset_overflow(self):
        # specify an offset > file size
        offset = len(self.DATA) + 4096
        try:
            sent = os.sendfile(self.sockno, self.fileno, offset, 4096)
        except OSError as e:
            # Solaris can raise EINVAL if offset >= file length, ignore.
            if e.errno != errno.EINVAL:
                raise
        else:
            self.assertEqual(sent, 0)
        self.client.shutdown(socket.SHUT_RDWR)
        self.client.close()
        self.server.wait()
        data = self.server.handler_instance.get_data()
        self.assertEqual(data, b'') 

def blob_client(self, csock):
        file_id = Pyro5.socketutil.receive_data(csock, 36).decode()
        print("{0} requesting file id {1}".format(csock.getpeername(), file_id))
        is_file, data = self.find_blob_data(file_id)
        if is_file:
            if hasattr(os, "sendfile"):
                print("...from file using sendfile()")
                out_fn = csock.fileno()
                in_fn = data.fileno()
                sent = 1
                offset = 0
                while sent:
                    sent = os.sendfile(out_fn, in_fn, offset, 512000)
                    offset += sent
            else:
                print("...from file using plain old read(); your os doesn't have sendfile()")
                while True:
                    chunk = data.read(512000)
                    if not chunk:
                        break
                    csock.sendall(chunk)
        else:
            print("...from memory")
            csock.sendall(data)
        csock.close() 

def sendfile(self) -> None:
        assert self.transport is not None
        out_socket = self.transport.get_extra_info('socket').dup()
        out_socket.setblocking(False)
        out_fd = out_socket.fileno()

        loop = self.loop
        data = b''.join(self._sendfile_buffer)
        try:
            await loop.sock_sendall(out_socket, data)
            if not self._do_sendfile(out_fd):
                fut = loop.create_future()
                fut.add_done_callback(partial(self._done_fut, out_fd))
                loop.add_writer(out_fd, self._sendfile_cb, fut, out_fd)
                await fut
        except asyncio.CancelledError:
            raise
        except Exception:
            server_logger.debug('Socket error')
            self.transport.close()
        finally:
            out_socket.close()

        await super().write_eof() 

def sendfile_all(self, fileno, sockno, offset, nbytes):
        # Send file in at most 1GB blocks as some operating
        # systems can have problems with sending files in blocks
        # over 2GB.

        BLKSIZE = 0x3FFFFFFF

        if nbytes > BLKSIZE:
            for m in range(0, nbytes, BLKSIZE):
                self.sendfile_all(fileno, sockno, offset, min(nbytes, BLKSIZE))
                offset += BLKSIZE
                nbytes -= BLKSIZE
        else:
            sent = 0
            sent += sendfile(sockno, fileno, offset + sent, nbytes - sent)
            while sent != nbytes:
                sent += sendfile(sockno, fileno, offset + sent, nbytes - sent) 

def sendfile(self, file, offset=0, count=None):
        """sendfile(file[, offset[, count]]) -> sent

        Send a file until EOF is reached by using high-performance
        os.sendfile() and return the total number of bytes which
        were sent.
        *file* must be a regular file object opened in binary mode.
        If os.sendfile() is not available (e.g. Windows) or file is
        not a regular file socket.send() will be used instead.
        *offset* tells from where to start reading the file.
        If specified, *count* is the total number of bytes to transmit
        as opposed to sending the file until EOF is reached.
        File position is updated on return or also in case of error in
        which case file.tell() can be used to figure out the number of
        bytes which were sent.
        The socket must be of SOCK_STREAM type.
        Non-blocking sockets are not supported.

        .. versionadded:: 1.1rc4
           Added in Python 3.5, but available under all Python 3 versions in
           gevent.
        """
        return self._sendfile_use_send(file, offset, count)

    # get/set_inheritable new in 3.4 

def sendfile(self, file, offset=0, count=None):
        """sendfile(file[, offset[, count]]) -> sent

        Send a file until EOF is reached by using high-performance
        os.sendfile() and return the total number of bytes which
        were sent.
        *file* must be a regular file object opened in binary mode.
        If os.sendfile() is not available (e.g. Windows) or file is
        not a regular file socket.send() will be used instead.
        *offset* tells from where to start reading the file.
        If specified, *count* is the total number of bytes to transmit
        as opposed to sending the file until EOF is reached.
        File position is updated on return or also in case of error in
        which case file.tell() can be used to figure out the number of
        bytes which were sent.
        The socket must be of SOCK_STREAM type.
        Non-blocking sockets are not supported.
        """
        try:
            return self._sendfile_use_sendfile(file, offset, count)
        except _GiveupOnSendfile:
            return self._sendfile_use_send(file, offset, count) 

def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]):
        """A higher level wrapper representing how an application is
        supposed to use sendfile().
        """
        while 1:
            try:
                if self.SUPPORT_HEADERS_TRAILERS:
                    return os.sendfile(sock, file, offset, nbytes, headers,
                                       trailers)
                else:
                    return os.sendfile(sock, file, offset, nbytes)
            except OSError as err:
                if err.errno == errno.ECONNRESET:
                    # disconnected
                    raise
                elif err.errno in (errno.EAGAIN, errno.EBUSY):
                    # we have to retry send data
                    continue
                else:
                    raise 

def test_offset_overflow(self):
        # specify an offset > file size
        offset = len(self.DATA) + 4096
        try:
            sent = os.sendfile(self.sockno, self.fileno, offset, 4096)
        except OSError as e:
            # Solaris can raise EINVAL if offset >= file length, ignore.
            if e.errno != errno.EINVAL:
                raise
        else:
            self.assertEqual(sent, 0)
        self.client.shutdown(socket.SHUT_RDWR)
        self.client.close()
        self.server.wait()
        data = self.server.handler_instance.get_data()
        self.assertEqual(data, b'') 

def test_headers(self):
        total_sent = 0
        sent = os.sendfile(self.sockno, self.fileno, 0, 4096,
                            headers=[b"x" * 512])
        total_sent += sent
        offset = 4096
        nbytes = 4096
        while 1:
            sent = self.sendfile_wrapper(self.sockno, self.fileno,
                                                    offset, nbytes)
            if sent == 0:
                break
            total_sent += sent
            offset += sent

        expected_data = b"x" * 512 + self.DATA
        self.assertEqual(total_sent, len(expected_data))
        self.client.close()
        self.server.wait()
        data = self.server.handler_instance.get_data()
        self.assertEqual(hash(data), hash(expected_data)) 

def extract_file(self, start_cluster, file_size, rel_outfile):
        abs_outfile = self.scan_environment.unpack_path(rel_outfile)
        os.makedirs(abs_outfile.parent, exist_ok=True)
        outfile = open(abs_outfile, 'wb')
        size_read = 0
        cluster_size = self.data.boot_sector.bpb.ls_per_clus * \
                self.data.boot_sector.bpb.bytes_per_ls
        for cluster in self.cluster_chain(start_cluster):
            bytes_to_read = min(cluster_size, file_size - size_read)
            start = self.offset + self.pos_data + (cluster-2) * cluster_size
            check_condition(start+bytes_to_read <= self.fileresult.filesize,
                    "file data outside file")
            os.sendfile(outfile.fileno(), self.infile.fileno(), start, bytes_to_read)
            size_read += bytes_to_read
        outfile.close()
        outlabels = []
        return (rel_outfile, outlabels) 

def _sock_sendfile_native(self, sock, file, offset, count):
        try:
            os.sendfile
        except AttributeError as exc:
            raise events.SendfileNotAvailableError(
                "os.sendfile() is not available")
        try:
            fileno = file.fileno()
        except (AttributeError, io.UnsupportedOperation) as err:
            raise events.SendfileNotAvailableError("not a regular file")
        try:
            fsize = os.fstat(fileno).st_size
        except OSError as err:
            raise events.SendfileNotAvailableError("not a regular file")
        blocksize = count if count else fsize
        if not blocksize:
            return 0  # empty file

        fut = self.create_future()
        self._sock_sendfile_native_impl(fut, None, sock, fileno,
                                        offset, count, blocksize, 0)
        return await fut 

def _import_sendfile():
    # By default attempt to use os.sendfile introduced in Python 3.3:
    # http://bugs.python.org/issue10882
    # ...otherwise fallback on using third-party pysendfile module:
    # https://github.com/giampaolo/pysendfile/
    if os.name == 'posix':
        try:
            return os.sendfile  # py >= 3.3
        except AttributeError:
            try:
                import sendfile as sf
                # dirty hack to detect whether old 1.2.4 version is installed
                if hasattr(sf, 'has_sf_hdtr'):
                    raise ImportError
                return sf.sendfile
            except ImportError:
                pass
    return None 

def get_repr_info(self, as_str=False, extra_info={}):
        info = OrderedDict()
        info['id'] = id(self)
        info['addr'] = "%s:%s" % (self.remote_ip, self.remote_port)
        if _is_ssl_sock(self.socket):
            info['ssl'] = True
        if self.username:
            info['user'] = self.username
        # If threads are involved sometimes "self" may be None (?!?).
        dc = getattr(self, 'data_channel', None)
        if dc is not None:
            if _is_ssl_sock(dc.socket):
                info['ssl-data'] = True
            if dc.file_obj:
                if self.data_channel.receive:
                    info['sending-file'] = dc.file_obj
                    if dc.use_sendfile():
                        info['use-sendfile(2)'] = True
                else:
                    info['receiving-file'] = dc.file_obj
                info['bytes-trans'] = dc.get_transmitted_bytes()
        info.update(extra_info)
        if as_str:
            return ', '.join(['%s=%r' % (k, v) for (k, v) in info.items()])
        return info 

def test_sendfile_fails(self):
        # Makes sure that if sendfile() fails and no bytes were
        # transmitted yet the server falls back on using plain
        # send()
        data = b'abcde12345' * 100000
        self.dummy_sendfile.write(data)
        self.dummy_sendfile.seek(0)
        self.client.storbinary('stor ' + self.tempfile, self.dummy_sendfile)
        with mock.patch('pyftpdlib.handlers.sendfile',
                        side_effect=OSError(errno.EINVAL)) as fun:
            self.client.retrbinary(
                'retr ' + self.tempfile, self.dummy_recvfile.write)
            assert fun.called
            self.dummy_recvfile.seek(0)
            datafile = self.dummy_recvfile.read()
            self.assertEqual(len(data), len(datafile))
            self.assertEqual(hash(data), hash(datafile)) 

def test_headers(self):
        total_sent = 0
        sent = os.sendfile(self.sockno, self.fileno, 0, 4096,
                            headers=[b"x" * 512])
        total_sent += sent
        offset = 4096
        nbytes = 4096
        while 1:
            sent = self.sendfile_wrapper(self.sockno, self.fileno,
                                                    offset, nbytes)
            if sent == 0:
                break
            total_sent += sent
            offset += sent

        expected_data = b"x" * 512 + self.DATA
        self.assertEqual(total_sent, len(expected_data))
        self.client.close()
        self.server.wait()
        data = self.server.handler_instance.get_data()
        self.assertEqual(hash(data), hash(expected_data)) 

def initiate_sendfile(self):
        """A wrapper around sendfile."""
        try:
            sent = sendfile(self._fileno, self._filefd, self._offset,
                            self.ac_out_buffer_size)
        except OSError as err:
            if err.errno in _ERRNOS_RETRY or err.errno == errno.EBUSY:
                return
            elif err.errno in _ERRNOS_DISCONNECTED:
                self.handle_close()
            else:
                if self.tot_bytes_sent == 0:
                    logger.warning(
                        "sendfile() failed; falling back on using plain send")
                    raise _GiveUpOnSendfile
                else:
                    raise
        else:
            if sent == 0:
                # this signals the channel that the transfer is completed
                self.discard_buffers()
                self.handle_close()
            else:
                self._offset += sent
                self.tot_bytes_sent += sent

    # --- utility methods