Python trio.BrokenResourceError() Examples

def recv_neighbours_v4(self, remote: NodeAPI, payload: Sequence[Any], _: Hash32) -> None:
        # The neighbours payload should have 2 elements: nodes, expiration
        if len(payload) < 2:
            self.logger.warning('Ignoring NEIGHBOURS msg with invalid payload: %s', payload)
            return
        nodes, expiration = payload[:2]
        if self._is_msg_expired(expiration):
            return
        neighbours = _extract_nodes_from_payload(remote.address, nodes, self.logger)
        self.logger.debug2('<<< neighbours from %s: %s', remote, neighbours)
        try:
            channel = self.neighbours_channels.get_channel(remote)
        except KeyError:
            self.logger.debug(f'unexpected neighbours from {remote}, probably came too late')
            return

        try:
            await channel.send(neighbours)
        except trio.BrokenResourceError:
            # This means the receiver has already closed, probably because it timed out.
            pass 

def _send(self, event):
        '''
        Send an to the remote WebSocket.

        The reader task and one or more writers might try to send messages at
        the same time, so this method uses an internal lock to serialize
        requests to send data.

        :param wsproto.events.Event event:
        '''
        data = self._wsproto.send(event)
        async with self._stream_lock:
            logger.debug('%s sending %d bytes', self, len(data))
            try:
                await self._stream.send_all(data)
            except (trio.BrokenResourceError, trio.ClosedResourceError):
                await self._abort_web_socket()
                raise ConnectionClosed(self._close_reason) from None 

def _send_job_state_event(self, event):
        '''
        Send a job state event to all listeners.

        If a listener's channel is full, then the event is not sent to that
        listener. If the listener's channel is closed, then that channel is
        removed and will not be sent to in the future.

        :param JobStateEvent event:
        '''
        to_remove = list()
        for recv_channel, send_channel in self._job_state_channels.items():
            try:
                send_channel.send_nowait(event)
            except trio.WouldBlock:
                # The channel is full. Drop this message and move on.
                pass
            except trio.BrokenResourceError:
                # The consumer closed the channel. We can remove it from our
                # dict after the loop finishes.
                to_remove.append(recv_channel)
        for recv_channel in to_remove:
            del self._job_state_channels[recv_channel] 

def run(self):
        """
        Run the subscription.

        :returns: This function returns when the sync is complete.
        """
        logger.info("%r Starting", self)
        async with trio.open_nursery() as nursery:
            self._cancel_scope = nursery.cancel_scope
            await self._set_initial_job_status()
            nursery.start_soon(self._job_status_task)
            try:
                await self._run_sync()
            except (trio.BrokenResourceError, trio.ClosedResourceError):
                logger.info("%r Aborted", self)
            nursery.cancel_scope.cancel()
        try:
            await self._send_complete()
            logger.info("%r Finished", self)
        except (trio.BrokenResourceError, trio.ClosedResourceError):
            # If we can't send the completion message, then bail out.
            pass 

def download_file(self, url, headers=IPHONE_HEADER, timeout=DOWNLOAD_TIMEOUT, res_time=RETRIES_TIMES):
        if res_time <= 0: # 重试超过了次数
            return None
        try:
            _url = random.choice(url) if isinstance(url, list) else url
            res = await asks.get(_url, headers=headers, timeout=timeout, retries=3)
        except (socket.gaierror, trio.BrokenResourceError, trio.TooSlowError, asks.errors.RequestTimeout) as e:
            logging.error("download from %s fail]err=%s!" % (url, e))
            await trio.sleep(random.randint(1, 5)) # for scheduler
            return await self.download_file(url, res_time=res_time-1)

        if res.status_code not in [200, 202]:
            logging.warn(f"download from {url} fail]response={res}")
            await trio.sleep(random.randint(3, 10))
            return await self.download_file(url, res_time=res_time-1)
        return res.content 

def switch_offline(self, addr):
        netloc_suffix = addr_to_netloc(addr)
        if netloc_suffix in self._offlines:
            return

        for netloc, sock in self._socks:
            if not netloc.endswith(netloc_suffix):
                continue

            async def _broken_stream(*args, **kwargs):
                raise trio.BrokenResourceError()

            _broken_stream.old_send_all_hook = sock.send_stream.send_all_hook
            _broken_stream.old_receive_some_hook = sock.receive_stream.receive_some_hook

            sock.send_stream.send_all_hook = _broken_stream
            sock.receive_stream.receive_some_hook = _broken_stream

            # Unlike for send stream, patching `receive_some_hook` is not enough
            # because it is called by the stream before actually waiting for data
            # (so in case of a long receive call, we could be past the hook call
            # when patching, hence not blocking the next incoming frame)
            sock.receive_stream.put_eof()

        self._offlines.add(netloc_suffix) 

def protocol_send(self, event: Event) -> None:
        if isinstance(event, RawData):
            async with self.send_lock:
                try:
                    with trio.CancelScope() as cancel_scope:
                        cancel_scope.shield = True
                        await self.stream.send_all(event.data)
                except (trio.BrokenResourceError, trio.ClosedResourceError):
                    await self.protocol.handle(Closed())
        elif isinstance(event, Closed):
            await self._close()
            await self.protocol.handle(Closed())
        elif isinstance(event, Updated):
            pass  # Triggers the keep alive timeout update
        await self._update_keep_alive_timeout() 

def process_pong_v4(self, remote: NodeAPI, token: Hash32, enr_seq: int) -> None:
        # XXX: This hack is needed because there are lots of parity 1.10 nodes out there that send
        # the wrong token on pong msgs (https://github.com/paritytech/parity/issues/8038). We
        # should get rid of this once there are no longer too many parity 1.10 nodes out there.
        if token in self.parity_pong_tokens:
            # This is a pong from a buggy parity node, so need to lookup the actual token we're
            # expecting.
            token = self.parity_pong_tokens.pop(token)
        else:
            # This is a pong from a non-buggy node, so just cleanup self.parity_pong_tokens.
            self.parity_pong_tokens = eth_utils.toolz.valfilter(
                lambda val: val != token, self.parity_pong_tokens)

        try:
            channel = self.pong_channels.get_channel(remote)
        except KeyError:
            # This is probably a Node which changed its identity since it was added to the DHT,
            # causing us to expect a pong signed with a certain key when in fact it's using
            # a different one. Another possibility is that the pong came after we've given up
            # waiting.
            self.logger.debug(f'Unexpected pong from {remote} with token {encode_hex(token)}')
            return

        self.node_db.set_last_pong_time(remote.id, int(time.monotonic()))
        # Insert/update the Node in our DB and routing table as soon as we receive the pong, as
        # we want that to happen even if the original requestor (bond()) gives up waiting.
        self.update_routing_table(remote)

        try:
            await channel.send((token, enr_seq))
        except trio.BrokenResourceError:
            # This means the receiver has already closed, probably because it timed out.
            pass 

def shutdown_and_clean_up(self) -> None:
        # When this method is called, it's because we definitely want to kill
        # this connection, either as a clean shutdown or because of some kind
        # of error or loss-of-sync bug, and we no longer care if that violates
        # the protocol or not. So we ignore the state of self.conn, and just
        # go ahead and do the shutdown on the socket directly. (If you're
        # implementing a client you might prefer to send ConnectionClosed()
        # and let it raise an exception if that violates the protocol.)
        #
        try:
            await self.stream.send_eof()
        except trio.BrokenResourceError:
            # They're already gone, nothing to do
            return
        # Wait and read for a bit to give them a chance to see that we closed
        # things, but eventually give up and just close the socket.
        # XX FIXME: possibly we should set SO_LINGER to 0 here, so
        # that in the case where the client has ignored our shutdown and
        # declined to initiate the close themselves, we do a violent shutdown
        # (RST) and avoid the TIME_WAIT?
        # it looks like nginx never does this for keepalive timeouts, and only
        # does it for regular timeouts (slow clients I guess?) if explicitly
        # enabled ("Default: reset_timedout_connection off")
        with trio.move_on_after(TIMEOUT):
            try:
                while True:
                    # Attempt to read until EOF
                    got = await self.stream.receive_some(MAX_RECV)
                    if not got:
                        break
            except trio.BrokenResourceError:
                pass
            finally:
                await self.stream.aclose() 

def _handle_message_event(self, event):
        '''
        Handle a message event.

        :param event:
        :type event: wsproto.events.BytesMessage or wsproto.events.TextMessage
        '''
        self._message_size += len(event.data)
        self._message_parts.append(event.data)
        if self._message_size > self._max_message_size:
            err = 'Exceeded maximum message size: {} bytes'.format(
                self._max_message_size)
            self._message_size = 0
            self._message_parts = []
            self._close_reason = CloseReason(1009, err)
            await self._send(CloseConnection(code=1009, reason=err))
            await self._recv_channel.aclose()
            self._reader_running = False
        elif event.message_finished:
            msg = (b'' if isinstance(event, BytesMessage) else '') \
                .join(self._message_parts)
            self._message_size = 0
            self._message_parts = []
            try:
                await self._send_channel.send(msg)
            except trio.BrokenResourceError:
                # The receive channel is closed, probably because somebody
                # called ``aclose()``. We don't want to abort the reader task,
                # and there's no useful cleanup that we can do here.
                pass 

def _close_stream(self):
        ''' Close the TCP connection. '''
        self._reader_running = False
        try:
            await self._stream.aclose()
        except trio.BrokenResourceError:
            # This means the TCP connection is already dead.
            pass 

def read_message(self) -> Message:
        buffer = bytearray()

        while len(buffer) < 4:
            try:
                data = await self._socket.receive_some(4 - len(buffer))
            except (trio.ClosedResourceError, trio.BrokenResourceError) as err:
                raise RemoteDisconnected from err

            if data == b"":
                raise RemoteDisconnected()

            buffer.extend(data)

        t_size = 4 + int.from_bytes(buffer[:4], "little")

        while len(buffer) < t_size:
            try:
                data = await self._socket.receive_some(t_size - len(buffer))
            except (trio.ClosedResourceError, trio.BrokenResourceError) as err:
                raise RemoteDisconnected from err

            if data == b"":
                raise RemoteDisconnected()

            buffer.extend(data)

        msg = cast(Message, pickle.loads(buffer[4:t_size]))
        return msg 

def send_message(self, message: Msg) -> None:
        msg_data = pickle.dumps(message, protocol=pickle.HIGHEST_PROTOCOL)
        size = len(msg_data)
        try:
            async with self._write_lock:
                await self._socket.send_all(size.to_bytes(4, "little") + msg_data)
        except (trio.ClosedResourceError, trio.BrokenResourceError) as err:
            raise RemoteDisconnected from err 

def run(self):
        '''
        Run the resource monitor.

        :returns: Runs until cancelled.
        '''
        next_run = trio.current_time() + self._interval
        while True:
            measurement = self._measure()
            self._measurements.append(measurement)
            to_remove = set()
            for channel in self._channels:
                try:
                    channel.send_nowait(measurement)
                except trio.WouldBlock:
                    continue
                except trio.BrokenResourceError:
                    to_remove.add(channel)
            for channel in to_remove:
                logger.debug('Removing closed channel')
                self._channels.remove(channel)
            sleep_time = next_run - trio.current_time()
            while sleep_time < 0:
                sleep_time += self._interval
            await trio.sleep(sleep_time)
            next_run += self._interval 

def shutdown_and_clean_up(self):
        # When this method is called, it's because we definitely want to kill
        # this connection, either as a clean shutdown or because of some kind
        # of error or loss-of-sync bug, and we no longer care if that violates
        # the protocol or not. So we ignore the state of self.conn, and just
        # go ahead and do the shutdown on the socket directly. (If you're
        # implementing a client you might prefer to send ConnectionClosed()
        # and let it raise an exception if that violates the protocol.)
        #
        try:
            await self.stream.send_eof()
        except trio.BrokenResourceError:
            # They're already gone, nothing to do
            return
        # Wait and read for a bit to give them a chance to see that we closed
        # things, but eventually give up and just close the socket.
        # XX FIXME: possibly we should set SO_LINGER to 0 here, so
        # that in the case where the client has ignored our shutdown and
        # declined to initiate the close themselves, we do a violent shutdown
        # (RST) and avoid the TIME_WAIT?
        # it looks like nginx never does this for keepalive timeouts, and only
        # does it for regular timeouts (slow clients I guess?) if explicitly
        # enabled ("Default: reset_timedout_connection off")
        with trio.move_on_after(TIMEOUT):
            try:
                while True:
                    # Attempt to read until EOF
                    got = await self.stream.receive_some(MAX_RECV)
                    if not got:
                        break
            except trio.BrokenResourceError:
                pass
            finally:
                await self.stream.aclose() 

def test_send_http_request(running_backend):
    stream = await trio.open_tcp_stream(running_backend.addr.hostname, running_backend.addr.port)
    await stream.send_all(
        b"GET / HTTP/1.1\r\n"
        b"Host: parsec.example.com\r\n"
        b"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0\r\n"
        b"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n"
        b"Accept-Language: fr,fr-FR;q=0.8,en-US;q=0.5,en;q=0.3\r\n"
        b"Accept-Encoding: gzip, deflate\r\n"
        b"DNT: 1\r\n"
        b"Connection: keep-alive\r\n"
        b"Upgrade-Insecure-Requests: 1\r\n"
        b"Cache-Control: max-age=0\r\n"
        b"\r\n"
    )
    rep = await stream.receive_some(4096)
    assert rep == (
        b"HTTP/1.1 426 OK\r\n"
        b"Upgrade: WebSocket\r\n"
        b"Content-Length: 51\r\n"
        b"Connection: Upgrade\r\n"
        b"Content-Type: text/html; charset=UTF-8\r\n"
        b"\r\n"
        b"This service requires use of the WebSocket protocol"
    )
    # Connection has been closed by peer
    with pytest.raises(trio.BrokenResourceError):
        await stream.send_all(b"dummy") 

def _run_job(self, job, *args, sync=False):
        try:
            if sync:
                return trio.from_thread.run_sync(job, *args, trio_token=self._trio_token)
            else:
                return trio.from_thread.run(job, *args, trio_token=self._trio_token)

        except trio.BrokenResourceError:
            logger.info(f"The submitted job `{job}` won't run as the scheduler is stopped")
            raise JobSchedulerNotAvailable("The job scheduler is stopped")

        except trio.RunFinishedError:
            logger.info(f"The submitted job `{job}` won't run as the trio loop is not running")
            raise JobSchedulerNotAvailable("The trio loop is not running") 

def aclose(self) -> None:
        try:
            try:
                await self.stream.send_all(
                    self.ws.send(CloseConnection(code=CloseReason.NORMAL_CLOSURE))
                )
            except LocalProtocolError:
                # TODO: exception occurs when ws.state is already closed...
                pass
            await self.stream.aclose()

        except (BrokenResourceError, TransportError):
            pass 

def _net_send(self, wsmsg):
        try:
            await self.stream.send_all(self.ws.send(wsmsg))

        except BrokenResourceError as exc:
            raise TransportError(*exc.args) from exc

        except RemoteProtocolError as exc:
            raise TransportError(*exc.args) from exc 

def _net_recv(self):
        try:
            in_data = await self.stream.receive_some(self.RECEIVE_BYTES)

        except BrokenResourceError as exc:
            raise TransportError(*exc.args) from exc

        if not in_data:
            # A receive of zero bytes indicates the TCP socket has been closed. We
            # need to pass None to wsproto to update its internal state.
            self.ws.receive_data(None)
        else:
            self.ws.receive_data(in_data) 

def close(self, noreply_wait=False, token=None, exception=None):
        self._closing = True
        if exception is not None:
            err_message = "Connection is closed (%s)." % str(exception)
        else:
            err_message = "Connection is closed."

        # Cursors may remove themselves when errored, so copy a list of them
        for cursor in list(self._cursor_cache.values()):
            cursor._error(err_message)

        for _, future in self._user_queries.values():
            if not future.done():
                future.set_exception(ReqlDriverError(err_message))

        self._user_queries = {}
        self._cursor_cache = {}

        if noreply_wait:
            noreply = Query(P_QUERY.NOREPLY_WAIT, token, None, None)
            await self.run_query(noreply, False)

        try:
            await self._stream.aclose()
        except (trio.ClosedResourceError, trio.BrokenResourceError):
            pass
        # We must not wait for the _reader_task if we got an exception, because that
        # means that we were called from it. Waiting would lead to a deadlock.
        if self._reader_ended_event:
            await self._reader_ended_event.wait()

        return None 

def _read_exactly(self, num):
        data = b''
        try:
            while len(data) < num:
                data += await self._stream.receive_some(num - len(data))
            return data
        except (trio.BrokenResourceError, trio.ClosedResourceError):
            self._closed = True 

def _read_until(self, delimiter):
        """ Naive implementation of reading until a delimiter. """
        buffer = bytearray()

        try:
            while True:
                data = await self._stream.receive_some(1)
                buffer.append(data[0])
                if data == delimiter:
                    break
        except (trio.BrokenResourceError, trio.ClosedResourceError):
            self._closed = True

        return bytes(buffer) 

def _send(self, data):
        async with self._stream_lock:
            try:
                await self._stream.send_all(data)
            except (trio.BrokenResourceError, trio.ClosedResourceError):
                self._closed = True 

def test_http1_keep_alive_pre_request(
    client_stream: trio.testing._memory_streams.MemorySendStream,
) -> None:
    await client_stream.send_all(b"GET")
    await trio.sleep(2 * KEEP_ALIVE_TIMEOUT)
    # Only way to confirm closure is to invoke an error
    with pytest.raises(trio.BrokenResourceError):
        await client_stream.send_all(b"a") 

def _close(self) -> None:
        try:
            await self.stream.send_eof()
        except (
            trio.BrokenResourceError,
            AttributeError,
            trio.BusyResourceError,
            trio.ClosedResourceError,
        ):
            # They're already gone, nothing to do
            # Or it is a SSL stream
            pass
        await self.stream.aclose() 

def _read_data(self) -> None:
        while True:
            try:
                data = await self.stream.receive_some(MAX_RECV)
            except (trio.ClosedResourceError, trio.BrokenResourceError):
                await self.protocol.handle(Closed())
                break
            else:
                if data == b"":
                    await self._update_keep_alive_timeout()
                    break
                await self.protocol.handle(RawData(data))
                await self._update_keep_alive_timeout() 

def run(self) -> None:
        try:
            try:
                with trio.fail_after(self.config.ssl_handshake_timeout):
                    await self.stream.do_handshake()
            except (trio.BrokenResourceError, trio.TooSlowError):
                return  # Handshake failed
            alpn_protocol = self.stream.selected_alpn_protocol()
            socket = self.stream.transport_stream.socket
            ssl = True
        except AttributeError:  # Not SSL
            alpn_protocol = "http/1.1"
            socket = self.stream.socket
            ssl = False

        try:
            client = parse_socket_addr(socket.family, socket.getpeername())
            server = parse_socket_addr(socket.family, socket.getsockname())

            async with trio.open_nursery() as nursery:
                self.nursery = nursery
                context = Context(nursery)
                self.protocol = ProtocolWrapper(
                    self.app,
                    self.config,
                    context,
                    ssl,
                    client,
                    server,
                    self.protocol_send,
                    alpn_protocol,
                )
                await self.protocol.initiate()
                await self._update_keep_alive_timeout()
                await self._read_data()
        except (trio.MultiError, OSError):
            pass
        finally:
            await self._close() 

def _run_tcp_server(socket_file: Path, cmd_handler):
    async def _client_handler(stream):

        # General exception handling
        try:

            # Stream handling
            try:

                unpacker = Unpacker()
                async for raw in stream:
                    unpacker.feed(raw)
                    for cmd in unpacker:
                        cmd = cmd_req_serializer.load(cmd)
                        rep = await cmd_handler(cmd)
                        raw_rep = cmd_rep_serializer.dumps(rep)
                        logger.info("Command processed", cmd=cmd["cmd"], rep_status=rep["status"])
                        await stream.send_all(raw_rep)

            except SerdeError as exc:
                await stream.send_all(packb({"status": "invalid_format", "reason": str(exc)}))

            finally:
                await stream.aclose()

        except trio.BrokenResourceError:
            pass  # Peer has closed the connection while we were sending a response

        except Exception:
            logger.exception("Unexpected crash")

    try:
        async with trio.open_service_nursery() as nursery:
            listeners = await nursery.start(
                partial(trio.serve_tcp, _client_handler, 0, host="127.0.0.1")
            )
            port = listeners[0].socket.getsockname()[1]

            # Make sure the path exists and write the socket file
            socket_file.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
            socket_file.write_text(str(port))

            logger.info("IPC server ready", port=port)
            try:
                yield
            finally:
                nursery.cancel_scope.cancel()

    except OSError as exc:
        raise IPCServerError(f"Cannot start IPC server: {exc}") from exc 

def recv_ping_v4(
            self, remote: NodeAPI, payload: Sequence[Any], message_hash: Hash32) -> None:
        """Process a received ping packet.

        A ping packet may come any time, unrequested, or may be prompted by us bond()ing with a
        new node. In the former case we'll just reply with a pong, whereas in the latter we'll
        also send an empty msg on the appropriate channel from ping_channels, to notify any
        coroutine waiting for that ping.

        Also, if we have no valid bond with the given remote, we'll trigger one in the background.
        """
        if remote.id == self.this_node.id:
            self.logger.info('Invariant: received ping from this_node: %s', remote)
            return
        # The ping payload should have at least 4 elements: [version, from, to, expiration], with
        # an optional 5th element for the node's ENR sequence number.
        if len(payload) < 4:
            self.logger.warning('Ignoring PING msg with invalid payload: %s', payload)
            return
        elif len(payload) == 4:
            _, _, _, expiration = payload[:4]
            enr_seq = None
        else:
            _, _, _, expiration, enr_seq = payload[:5]
            enr_seq = big_endian_to_int(enr_seq)
        self.logger.debug2('<<< ping(v4) from %s, enr_seq=%s', remote, enr_seq)
        if self._is_msg_expired(expiration):
            return

        try:
            channel = self.ping_channels.get_channel(remote)
        except KeyError:
            pass
        else:
            try:
                await channel.send(None)
            except trio.BrokenResourceError:
                # This means the receiver has already closed, probably because it timed out.
                pass

        await self.send_pong_v4(remote, message_hash)

        # The spec says this about received pings:
        #   When a ping packet is received, the recipient should reply with a Pong packet.
        #   It may also consider the sender for addition into the local table
        # However, since we trigger a bond (see below) if we get a ping from a Node we haven't
        # heard about before, we let that happen when the bond is completed.

        # If no communication with the sender has occurred within the last 12h, a ping
        # should be sent in addition to pong in order to receive an endpoint proof.
        if not self.is_bond_valid_with(remote.id):
            self.manager.run_task(self.bond, remote.id)