Python twisted.internet.threads.deferToThread() Examples

def test_authorized_keys(self):
        """
        When the SSH connection is established, the ``~/.ssh/authorized_keys``
        file has the public part of the generated key pair appended to it.
        """
        configuring = deferToThread(
            self.configure_ssh, self.server.ip, self.server.port)

        def configured(ignored):
            id_rsa_pub = self.ssh_config.child(b"id_rsa_flocker.pub")
            keys = self.server.home.descendant([b".ssh", b"authorized_keys"])

            # Compare the contents ignoring comments for ease.
            self.assertEqual(goodlines(id_rsa_pub), goodlines(keys))

        configuring.addCallback(configured)
        return configuring 

def _read_handshake(self, data):
        """
        Read handshake message, parse the other peer's public key and
        schedule the key exchange for execution outside of the event loop.
        """

        log_prefix = "obfs3:_read_handshake()"
        if len(data) < PUBKEY_LEN:
            log.debug("%s: Not enough bytes for key (%d)." % (log_prefix, len(data)))
            return

        log.debug("%s: Got %d bytes of handshake data (waiting for key)." % (log_prefix, len(data)))

        # Get the public key from the handshake message, do the DH and
        # get the shared secret.
        other_pubkey = data.read(PUBKEY_LEN)

        # Do the UniformDH handshake asynchronously
        self.d = threads.deferToThread(self.dh.get_secret, other_pubkey)
        self.d.addCallback(self._read_handshake_post_dh, other_pubkey, data)
        self.d.addErrback(self._uniform_dh_errback, other_pubkey)

        self.state = ST_WAIT_FOR_HANDSHAKE 

def test_event_http_urlencoded(machinery, create_influxdb, reset_influxdb):
    """
    Submit event in ``x-www-form-urlencoded`` format to HTTP API
    and proof it is stored in the InfluxDB database.
    """

    # Submit a single event, without timestamp.
    yield threads.deferToThread(http_form_sensor, settings.channel_path_event, event_data)

    # Wait for some time to process the message.
    yield sleep(PROCESS_DELAY)

    # Proof that event arrived in InfluxDB.
    record = influx_events.get_first_record()
    del record['time']
    assert record == event_data
    yield record 

def test_airrohr_http_json(machinery, create_influxdb, reset_influxdb):
    """
    Submit single reading in Airrohr JSON format to HTTP API
    and proof it is stored in the InfluxDB database.
    """

    # Submit a single measurement, without timestamp.
    yield threads.deferToThread(http_json_sensor, settings.channel_path_airrohr, data_in)

    # Wait for some time to process the message.
    yield sleep(PROCESS_DELAY)

    # Proof that data arrived in InfluxDB.
    record = influx_sensors.get_first_record()
    del record['time']
    assert record == data_out
    yield record 

def test_mqtt_homie(machinery, create_influxdb, reset_influxdb):
    """
    Publish reading like a Homie device in JSON format to MQTT broker
    and proof it is stored in the InfluxDB database.
    """

    # Submit a single measurement, without timestamp.
    data = {
        'temperature': 42.84,
        'humidity': 83.1,
    }
    yield threads.deferToThread(mqtt_json_sensor, settings.mqtt_topic_homie, data)

    # Wait for some time to process the message.
    yield sleep(PROCESS_DELAY)

    # Proof that data arrived in InfluxDB.
    record = influx_sensors.get_first_record()
    del record['time']
    assert record == {u'humidity': 83.1, u'temperature': 42.84}
    yield record 

def test_http_urlencoded(machinery, create_influxdb, reset_influxdb):
    """
    Submit single reading in ``x-www-form-urlencoded`` format to HTTP API
    and proof it is stored in the InfluxDB database.
    """

    # Submit a single measurement, without timestamp.
    data = {
        'temperature': 25.26,
        'humidity': 51.8,
    }
    yield threads.deferToThread(http_form_sensor, settings.channel_path_data, data)

    # Wait for some time to process the message.
    yield sleep(PROCESS_DELAY)

    # Proof that data arrived in InfluxDB.
    record = influx_sensors.get_first_record()
    del record['time']
    assert record == {u'temperature': 25.26, u'humidity': 51.8}
    yield record 

def test_mqtt_to_influxdb_single(machinery, create_influxdb, reset_influxdb):
    """
    Publish discrete values to the MQTT broker and
    and proof they are stored in the InfluxDB database.
    """

    # Submit discrete measurement values, without timestamp.
    topic_temperature = settings.mqtt_topic_single + '/temperature'
    topic_humidity = settings.mqtt_topic_single + '/humidity'
    yield threads.deferToThread(mqtt_sensor, topic_temperature, 42.84)
    yield threads.deferToThread(mqtt_sensor, topic_humidity, 83.1)

    # Wait for some time to process the message.
    yield sleep(PROCESS_DELAY)

    # Proof that data arrived in InfluxDB.
    record = influx_sensors.get_first_record()
    assert 'temperature' in record or 'humidity' in record 

def test_mqtt_legacy(machinery, create_influxdb, reset_influxdb):
    """
    Publish single reading in JSON format to MQTT broker on legacy suffix
    and proof it is stored in the InfluxDB database.
    """

    # Submit a single measurement, without timestamp.
    data = {
        'temperature': 42.84,
        'humidity': 83.1,
    }
    yield threads.deferToThread(mqtt_json_sensor, settings.mqtt_topic_json_legacy, data)

    # Wait for some time to process the message.
    yield sleep(PROCESS_DELAY)

    # Proof that data arrived in InfluxDB.
    record = influx_sensors.get_first_record()
    assert 'temperature' in record or 'humidity' in record 

def test_weewx_mqtt(machinery, create_influxdb, reset_influxdb):
    """
    Publish single reading in JSON format to MQTT broker
    and proof it is stored in the InfluxDB database.
    """

    # Submit a single measurement, without timestamp.
    yield threads.deferToThread(mqtt_json_sensor, settings.mqtt_topic_json, data)

    # Wait for some time to process the message.
    yield sleep(PROCESS_DELAY)

    # Proof that data arrived in InfluxDB.
    record = influx_sensors.get_first_record()

    assert record["time"] == '2017-04-17T22:15:00Z'

    assert record["outTemp_C"] == 3.55555555556
    assert record["windSpeed10_kph"] == 5.78725803977
    assert record["cloudbase_meter"] == 773.082217509
    assert record["consBatteryVoltage_volt"] == 4.72

    yield record 

def test_http_csv(machinery, create_influxdb, reset_influxdb):
    """
    Submit single reading in CSV format to HTTP API
    and proof it is stored in the InfluxDB database.
    """

    # Submit a single measurement, without timestamp.
    data = {
        'temperature': 25.26,
        'humidity': 51.8,
    }
    yield threads.deferToThread(http_csv_sensor, settings.channel_path_data, data)

    # Wait for some time to process the message.
    yield sleep(PROCESS_DELAY)

    # Proof that data arrived in InfluxDB.
    record = influx_sensors.get_first_record()
    del record['time']
    assert record == {u'temperature': 25.26, u'humidity': 51.8}
    yield record 

def test_event_http_json(machinery, create_influxdb, reset_influxdb):
    """
    Submit event in JSON format to HTTP API
    and proof it is stored in the InfluxDB database.
    """

    # Submit a single event, without timestamp.
    yield threads.deferToThread(http_json_sensor, settings.channel_path_event, event_data)

    # Wait for some time to process the message.
    yield sleep(PROCESS_DELAY)

    # Proof that event arrived in InfluxDB.
    record = influx_events.get_first_record()
    del record['time']
    assert record == event_data
    yield record 

def test_deferredFailureAfterSuccess(self):
        """
        Check that a successful L{threads.deferToThread} followed by a one
        that raises an exception correctly result as a failure.
        """
        # set up a condition that causes cReactor to hang. These conditions
        # can also be set by other tests when the full test suite is run in
        # alphabetical order (test_flow.FlowTest.testThreaded followed by
        # test_internet.ReactorCoreTestCase.testStop, to be precise). By
        # setting them up explicitly here, we can reproduce the hang in a
        # single precise test case instead of depending upon side effects of
        # other tests.
        #
        # alas, this test appears to flunk the default reactor too

        d = threads.deferToThread(lambda: None)
        d.addCallback(lambda ign: threads.deferToThread(lambda: 1//0))
        return self.assertFailure(d, ZeroDivisionError) 

def test_runs_command(self):
        """
        ``run()`` on a SSH ``ProcessNode`` runs the command on the machine
        being ssh'd into.
        """
        node = make_sshnode(self)
        temp_file = FilePath(self.mktemp())

        def go():
            with node.run([b"python", b"-c",
                           b"file('%s', 'w').write(b'hello')"
                           % (temp_file.path,)]):
                pass
            return temp_file.getContent()
        d = deferToThread(go)

        def got_data(data):
            self.assertEqual(data, b"hello")
        d.addCallback(got_data)
        return d 

def test_run_stdin(self):
        """
        ``run()`` on a SSH ``ProcessNode`` writes to the remote command's
        stdin.
        """
        node = make_sshnode(self)
        temp_file = FilePath(self.mktemp())

        def go():
            with node.run([b"python", b"-c",
                           b"import sys; "
                           b"file('%s', 'wb').write(sys.stdin.read())"
                           % (temp_file.path,)]) as stdin:
                stdin.write(b"hello ")
                stdin.write(b"there")
            return temp_file.getContent()
        d = deferToThread(go)

        def got_data(data):
            self.assertEqual(data, b"hello there")
        d.addCallback(got_data)
        return d 

def test_get_output(self):
        """
        ``get_output()`` returns the command's output.
        """
        node = make_sshnode(self)
        temp_file = FilePath(self.mktemp())
        temp_file.setContent(b"hello!")

        def go():
            return node.get_output([b"python", b"-c",
                                    b"import sys; "
                                    b"sys.stdout.write(file('%s').read())"
                                    % (temp_file.path,)])
        d = deferToThread(go)

        def got_data(data):
            self.assertEqual(data, b"hello!")
        d.addCallback(got_data)
        return d 

def test_not_json(self):
        """
        Returns ``None`` if the metadata URL doesn't return JSON content.
        """
        listening = webserver_for_test(
            self,
            url_path="/" + "/".join(METADATA_RELATIVE_PATH),
            # Return non-json string.
            response_content=random_name(self)
        )
        connecting = listening.addCallback(
            lambda port: deferToThread(
                metadata_from_service,
                metadata_service_endpoint=next(
                    (a.host, a.port)
                    for a in [port.getHost()]
                ),
            )
        )

        def check(result):
            self.assertIs(None, result)

        checking = connecting.addCallback(check)
        return checking 

def test_existing_authorized_keys_preserved(self):
        """
        Any unrelated content in the ``~/.ssh/authorized_keys`` file is left in
        place by ``configure_ssh``.
        """
        existing_keys = (
            b"ssh-dss AAAAB3Nz1234567890 comment\n"
            b"ssh-dss AAAAB3Nz0987654321 comment\n"
        )
        ssh_path = self.server.home.child(b".ssh")
        ssh_path.makedirs()

        authorized_keys = ssh_path.child(b"authorized_keys")
        authorized_keys.setContent(existing_keys)
        configuring = deferToThread(
            self.configure_ssh, self.server.ip, self.server.port)

        def configured(ignored):
            self.assertIn(existing_keys, authorized_keys.getContent())
        configuring.addCallback(configured)
        return configuring 

def test_flocker_keypair_written(self):
        """
        ``configure_ssh`` writes the keypair to ``id_rsa_flocker`` and
        ``id_rsa_flocker.pub`` remotely.
        """
        configuring = deferToThread(
            self.configure_ssh, self.server.ip, self.server.port)

        def configured(ignored):
            expected = (
                self.ssh_config.child(b"id_rsa_flocker").getContent(),
                self.ssh_config.child(b"id_rsa_flocker.pub").getContent()
            )
            actual = (
                self.flocker_config.child(b"id_rsa_flocker").getContent(),
                self.flocker_config.child(b"id_rsa_flocker.pub").getContent()
            )
            self.assertEqual(expected, actual)
        configuring.addCallback(configured)
        return configuring 

def test_flocker_keypair_permissions(self):
        """
        ``configure_ssh`` writes the remote keypair with secure permissions.
        """
        configuring = deferToThread(
            self.configure_ssh, self.server.ip, self.server.port)

        expected_private_key_permissions = Permissions(0600)
        expected_public_key_permissions = Permissions(0644)

        def configured(ignored):
            expected = (
                expected_private_key_permissions,
                expected_public_key_permissions
            )
            actual = (
                self.flocker_config.child(b"id_rsa_flocker").getPermissions(),
                self.flocker_config.child(
                    b"id_rsa_flocker.pub").getPermissions()
            )
            self.assertEqual(expected, actual)
        configuring.addCallback(configured)
        return configuring 

def remove(self, unit_name):
        container_name = self._to_container_name(unit_name)

        def _remove():
            # Previously, this looped forever and didn't pause between loops.
            # We've arbitrarily chosen a wait interval of 0.001 seconds and
            # 1000 retries (i.e. a second of polling). These values may need
            # tuning.
            poll_until(
                partial(self._stop_container, container_name),
                repeat(0.001, 1000))

            # Previously, the container remove was only tried once. Again,
            # these parameters may need tuning.
            poll_until(
                partial(self._remove_container, container_name),
                repeat(0.001, 1000))

        d = deferToThread(_remove)
        return d 

def test_success(self):
        """
        The metadata is downloaded, decoded and returned.
        """
        expected_value = random_name(self)
        listening = webserver_for_test(
            self,
            url_path="/" + "/".join(METADATA_RELATIVE_PATH),
            response_content=json.dumps({"test_key": expected_value})
        )
        connecting = listening.addCallback(
            lambda port: deferToThread(
                metadata_from_service,
                metadata_service_endpoint=next(
                    (a.host, a.port)
                    for a in [port.getHost()]
                ),
            )
        )

        def check(result):
            self.assertEqual(expected_value, result["test_key"])

        checking = connecting.addCallback(check)
        return checking 

def test_not_found(self):
        """
        Returns ``None`` if the metadata URL is not found.
        """
        listening = webserver_for_test(
            self,
            # Force the webserver to return 404
            url_path="/some/other/path",
            response_content=json.dumps({})
        )
        connecting = listening.addCallback(
            lambda port: deferToThread(
                metadata_from_service,
                metadata_service_endpoint=next(
                    (a.host, a.port)
                    for a in [port.getHost()]
                ),
            )
        )

        def check(result):
            self.assertIs(None, result)

        checking = connecting.addCallback(check)
        return checking 

def _decorator(self, func):
        @wraps(func)
        def wrapper(request_handler, *args, **kwargs):
            start_time = time.time()
            # Wrap the handler in @cyclone.web.synchronous
            request_handler._auto_finish = False
            d = deferToThread(
                self._validate_request, request_handler, *args, **kwargs)
            d.addBoth(self._track_validation_timing, request_handler,
                      start_time)
            d.addCallback(self._call_func, func, request_handler)
            # Errbacks for _validate_request: handler functions should
            # explicitly manage their own Errbacks
            d.addErrback(request_handler._boto_err)
            d.addErrback(request_handler._validation_err)
            d.addErrback(request_handler._response_err)
        return wrapper 

def post(self, router_type, router_data):
        # type: (str, JSONDict) -> Deferred
        """HTTP POST

        Router type/data registration.

        """
        self.metrics.increment("ua.command.register", tags=self.base_tags())

        uaid = uuid.uuid4()

        # ALWAYS RETURN CHID AS .HEX, NO DASH
        chid = router_data.get("channelID", uuid.uuid4().hex)

        d = deferToThread(self._register_user_and_channel,
                          uaid, chid, router_type, router_data)
        d.addCallback(self._write_endpoint, uaid, chid, router_type,
                      router_data, new_uaid=True)
        d.addErrback(self._router_fail_err)
        d.addErrback(self._response_err)
        return d 

def force_retry(self, func, *args, **kwargs):
        # type: (Callable[..., Any], *Any, **Any) -> Deferred
        """Forcefully retry a function in a thread until it doesn't error

        Note that this does not use ``self.deferToThread``, so this will
        continue to retry even if the client drops.

        """
        def wrapper(result, *w_args, **w_kwargs):
            if isinstance(result, failure.Failure):
                # This is an exception, log it
                self.log_failure(result)

            d = deferToThread(func, *args, **kwargs)
            d.addErrback(wrapper)
            return d
        d = deferToThread(func, *args, **kwargs)
        d.addErrback(wrapper)
        return d 

def process_register(self, data):
        """Process a register message"""
        if "channelID" not in data:
            return self.bad_message("register")
        chid = data["channelID"]
        try:
            if str(uuid.UUID(chid)) != chid:
                return self.bad_message("register", "Bad UUID format, use"
                                        "lower case, dashed format")
        except (ValueError, TypeError):
            return self.bad_message("register", "Invalid UUID specified")
        self.transport.pauseProducing()

        d = self.deferToThread(self.conf.make_endpoint, self.ps.uaid, chid,
                               data.get("key"))
        d.addCallback(self.finish_register, chid)
        d.addErrback(self.trap_cancel)
        d.addErrback(self.error_register)
        return d 

def test_proxy_protocol(self):
        port = self.ep.conf.proxy_protocol_port
        ip = '198.51.100.22'
        req = """\
PROXY TCP4 {} 203.0.113.7 35646 80\r
GET /v1/err HTTP/1.1\r
Host: 127.0.0.1\r
\r\n""".format(ip)

        def proxy_request():
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.connect(("localhost", port))
            try:
                sock.sendall(req)
                return sock.recv(4096)
            finally:
                sock.close()

        response = yield deferToThread(proxy_request)
        assert response.startswith("HTTP/1.1 418 ")
        assert "Test Error" in response
        assert self.logs.logged_ci(lambda ci: ci.get('remote_ip') == ip) 

def connectionMade(self):
        self._sendSASLMessage(self.START, self.sasl.mechanism)
        initial_message = yield deferToThread(self.sasl.process)
        self._sendSASLMessage(self.OK, initial_message)

        while True:
            status, challenge = yield self._receiveSASLMessage()
            if status == self.OK:
                response = yield deferToThread(self.sasl.process, challenge)
                self._sendSASLMessage(self.OK, response)
            elif status == self.COMPLETE:
                if not self.sasl.complete:
                    msg = "The server erroneously indicated that SASL " \
                          "negotiation was complete"
                    raise TTransport.TTransportException(msg, message=msg)
                else:
                    break
            else:
                msg = "Bad SASL negotiation status: %d (%s)" % (status, challenge)
                raise TTransport.TTransportException(msg, message=msg)

        self._sasl_negotiation_deferred = None
        ThriftClientProtocol.connectionMade(self) 

def activate_filebot_license(self, data):
        """Uses data from a filebot license to activate FileBot"""
        log.info("Recieved license file data for filebot registration.")
        license_file = tempfile.NamedTemporaryFile(suffix='.psm', delete=False)
        license_file.write(data)
        license_file.close()
        try:
            result = yield threads.deferToThread(pyfilebot.license, license_file.name)
        except pyfilebot.FilebotLicenseError as error:
            log.error("Error during licensing", exc_info=True)
            result = "{0}: {1}".format(error.__class__.__name__, error.message)
        else:
            log.debug("Filebot Successufully licensed.")
        finally:
            os.unlink(license_file.name)
            del license_file
            defer.returnValue(result) 

def run_plugins_function(plugins, function, thread, *args, **kwargs):
    return_value = False

    for plugin in plugins:
        class_name = get_plugin_name(plugin).upper()
        try:
            func = getattr(plugin, function)
            if function != 'packet_logged':
                pass
                #log.msg(log.LCYAN, '[PLUGIN][' + class_name + ']', function.upper())

            if thread:
                threads.deferToThread(func, *copy.deepcopy(args), **copy.deepcopy(kwargs))
            else:
                return_value = func(*args, **kwargs)
                if not return_value:
                    return return_value
        except AttributeError:
            pass
        except Exception, ex:
            pass
            #log.msg(log.LRED, '[PLUGIN][' + class_name + '][ERR]', str(ex))