Python twisted.internet.defer.returnValue() Examples

def test_retry_with_twisted(mock_client, mock_response):
    from twisted.internet import defer

    @defer.inlineCallbacks
    def return_response():
        yield
        defer.returnValue(mock_response)

    # Setup
    mock_response.with_json({"id": 123, "name": "prkumar"})
    mock_client.with_side_effect([Exception, return_response()])
    mock_client.with_io(io.TwistedStrategy())
    github = GitHub(base_url=BASE_URL, client=mock_client)

    # Run
    response = yield github.get_user("prkumar")

    assert len(mock_client.history) == 2
    assert response.json() == {"id": 123, "name": "prkumar"} 

def getNotifications(self, start, end):
        """
        getNotifications(self, start, end)

        Read all planning notifications from sorted set {0}

        :param start: range start
        :type start: integer
        :param end: range end
        :type end: integer
        """

        pipeline = yield self.rc.pipeline()
        pipeline.zrange(NOTIFIER_NOTIFICATIONS, start=start, end=end)
        pipeline.zcard(NOTIFIER_NOTIFICATIONS)
        jsons, total = yield pipeline.execute_pipeline()
        defer.returnValue((jsons, total)) 

def render_GET(self, request):
        json, trigger = yield self.db.getTrigger(self.trigger_id)
        if json is None:
            defer.returnValue(bad_request(request, "Trigger not found"))
            raise StopIteration

        fromTime = request.args.get('from')[0]
        endTime = request.args.get('to')[0]
        context = createRequestContext(fromTime, endTime, allowRealTimeAlerting=True)
        result = {}
        for target in trigger.get("targets", [trigger.get("target")]):
            time_series = yield evaluateTarget(context, target)
            for time_serie in time_series:
                values = [(time_serie.start + time_serie.step * i, time_serie[i]) for i in range(0, len(time_serie))]
                result[time_serie.name] = [{"ts": ts, "value": value} for ts, value in values if value is not None]
        self.write_json(request, result) 

def getTriggersChecksPage(self, start, size):
        """
        getTriggersChecksPage(self, start, size)

        - Returns triggers range from sorted set {0}

        :param start: start position in range
        :type start: integer
        :param start: number of triggers
        :type start: integer
        :rtype: json
        """
        pipeline = yield self.rc.pipeline()
        pipeline.zrevrange(TRIGGERS_CHECKS, start=start, end=(start + size))
        pipeline.zcard(TRIGGERS_CHECKS)
        triggers_ids, total = yield pipeline.execute_pipeline()
        triggers = yield self._getTriggersChecks(triggers_ids)
        defer.returnValue((triggers, total)) 

def getAllContacts(self):
        """
        getAllContacts(self, login)

        Returns all contacts json

        :rtype: array of strings
        """
        result = []
        keys = yield self.rc.keys(CONTACT_PREFIX.format('*'))
        for key in keys:
            contact_id = key.split(':')[-1]
            contact = yield self.getContact(contact_id)
            if contact:
                result.append(contact)
        defer.returnValue(result) 

def getSubscription(self, sub_id):
        """
        getSubscription(self, sub_id)

        Returns subscription by given id from key {0}

        :param sub_id: subscription id
        :type sub_id: string
        :rtype: json dict
        """
        sub_json = yield self.rc.get(SUBSCRIPTION_PREFIX.format(sub_id))
        sub = None
        if sub_json is not None:
            sub = anyjson.loads(sub_json)
            sub["id"] = sub_id
        defer.returnValue(sub) 

def _getTriggersChecks(self, triggers_ids):
        triggers = []
        pipeline = yield self.rc.pipeline()
        for trigger_id in triggers_ids:
            pipeline.get(TRIGGER_PREFIX.format(trigger_id))
            pipeline.smembers(TRIGGER_TAGS_PREFIX.format(trigger_id))
            pipeline.get(LAST_CHECK_PREFIX.format(trigger_id))
            pipeline.get(TRIGGER_NEXT_PREFIX.format(trigger_id))
        results = yield pipeline.execute_pipeline()
        slices = [[triggers_ids[i / 4]] + results[i:i + 4] for i in range(0, len(results), 4)]
        for trigger_id, trigger_json, trigger_tags, last_check, throttling in slices:
            if trigger_json is None:
                continue
            trigger = anyjson.deserialize(trigger_json)
            trigger = trigger_reformat(trigger, trigger_id, trigger_tags)
            trigger["last_check"] = None if last_check is None else anyjson.deserialize(last_check)
            trigger["throttling"] = long(throttling) if throttling and time.time() < long(throttling) else 0
            triggers.append(trigger)
        defer.returnValue(triggers) 

def getTrigger(self, trigger_id):
        """
        getTrigger(self, trigger_id)

        - Read trigger by key {0}
        - Unpack trigger json

        :param tags: get with tags
        :type tags: boolean
        :param trigger_id: trigger identity
        :type trigger_id: string
        :rtype: tuple(json, trigger)
        """
        pipeline = yield self.rc.pipeline()
        pipeline.get(TRIGGER_PREFIX.format(trigger_id))
        pipeline.smembers(TRIGGER_TAGS_PREFIX.format(trigger_id))
        trigger = {}
        json, trigger_tags = yield pipeline.execute_pipeline()
        if json is not None:
            trigger = anyjson.deserialize(json)
            trigger = trigger_reformat(trigger, trigger_id, trigger_tags)
        defer.returnValue((json, trigger)) 

def getContact(self, contact_id):
        """
        getContact(self, contact_id)

        Returns contact by given id from key {0}

        :param contact_id: contact id
        :type contact_id: string
        :rtype: json dict
        """
        contact_json = yield self.rc.get(CONTACT_PREFIX.format(contact_id))
        contact = None
        if contact_json is not None:
            contact = anyjson.loads(contact_json)
            contact["id"] = contact_id
        defer.returnValue(contact) 

def getTagSubscriptions(self, tag):
        """
        getTagSubscriptions(self, tag)

        Returns all subscriptions by given tag from set {0}

        :type tag: string
        :rtype: list of strings
        """
        result = []
        subscriptions_ids = yield self.rc.smembers(TAG_SUBSCRIPTIONS_PREFIX.format(tag))
        for sub_id in subscriptions_ids:
            sub_json = yield self.rc.get(SUBSCRIPTION_PREFIX.format(sub_id))
            if sub_json is not None:
                sub = anyjson.loads(sub_json)
                sub["id"] = sub_id
                result.append(sub)
            else:
                yield self.rc.srem(TAG_SUBSCRIPTIONS_PREFIX.format(tag), sub_id)
        defer.returnValue(result) 

def saveUserContact(self, login, contact, existing=None):
        """
        saveUserContact(self, login, contact)

        Creates redis transaction for:
            - save *contact* json to key {0}
            - add *contact_id* to set {1}

        :param login: user login
        :type login: string
        :param contact: contact data
        :type contact: json dict
        :rtype: json dict
        """
        contact["user"] = login
        contact_id = contact.get("id", str(uuid4()))
        contact["id"] = contact_id
        t = yield self.rc.multi()
        yield t.set(CONTACT_PREFIX.format(contact_id), anyjson.serialize(contact))
        yield t.sadd(USER_CONTACTS_PREFIX.format(login), contact_id)
        yield t.commit()
        defer.returnValue(contact) 

def removeNotification(self, id):
        """
        removeNotification(self, id)

        Remove planning notification by id string from sorted set {0}

        :param id: notification id string
        :type id: string
        """
        notifications, total = yield self.getNotifications(0, -1)
        for json in notifications:
            notification = anyjson.loads(json)
            timestamp = str(notification.get('timestamp'))
            contact_id = notification.get('contact', {}).get('id')
            sub_id = notification.get('event', {}).get('sub_id')
            idstr = ''.join([timestamp, contact_id, sub_id])
            if idstr == id:
                result = yield self.rc.zrem(NOTIFIER_NOTIFICATIONS, json)
                defer.returnValue(result) 

def getMetricsValues(self, metrics, startTime, endTime='+inf'):
        """
        getMetricsValues(self, metrics, startTime, endTime)

        Read multiple metric values from sorted set {0} from startTime

        :param metrics: list of graphite metric path
        :type metrics: list of string
        :param startTime: unix epoch time
        :type startTime: long
        :param endTime: unix epoch time
        :type endTime: long
        :rtype: list of list of tuple ('value timestamp', long)
        """
        pipeline = yield self.rc.pipeline()
        for metric in metrics:
            pipeline.zrangebyscore(METRIC_PREFIX.format(metric), min=startTime, max=endTime, withscores=True)
        results = yield pipeline.execute_pipeline()
        defer.returnValue(results) 

def cache(f):
    @wraps(f)
    @defer.inlineCallbacks
    def wrapper(*args, **kwargs):
        if 'cache_key' in kwargs and 'cache_ttl' in kwargs:
            key = "%s%s" % (f, kwargs['cache_key'])
            ttl = kwargs['cache_ttl']
            del kwargs['cache_key']
            del kwargs['cache_ttl']
            now = reactor.seconds()

            @defer.inlineCallbacks
            def get_value():
                result = yield f(*args, **kwargs)
                defer.returnValue(result)
            timestamp, result = CACHE.get(key, (0, None))
            if timestamp + ttl < now:
                CACHE[key] = (now, result)
                result = yield get_value()
                CACHE[key] = (now, result)
        else:
            result = yield f(*args, **kwargs)
        defer.returnValue(result)
    return wrapper 

def get_timeseries(self, requestContext):
        targets = self.struct.get("targets", [])
        target_time_series = TargetTimeSeries()
        target_number = 1

        for target in targets:
            time_series = yield evaluateTarget(requestContext, target)

            if target_number > 1:
                if len(time_series) == 1:
                    target_time_series.other_targets_names["t%s" % target_number] = time_series[0].name
                elif not time_series:
                    raise Exception("Target #%s has no timeseries" % target_number)
                else:
                    raise Exception("Target #%s has more than one timeseries" % target_number)

            for time_serie in time_series:
                time_serie.last_state = self.last_check["metrics"].get(
                                        time_serie.name, {
                                            "state": state.NODATA,
                                            "timestamp": time_serie.start - 3600})
            target_time_series[target_number] = time_series
            target_number += 1

        defer.returnValue(target_time_series) 

def getEvents(self, repo_user, repo_name, until_id=None):
        """Get all repository events, following paging, until the end
        or until UNTIL_ID is seen.  Returns a Deferred."""
        done = False
        page = 0
        events = []
        while not done:
            new_events = yield self.api.makeRequest(
                    ['repos', repo_user, repo_name, 'events'],
                    page)

            # terminate if we find a matching ID
            if new_events:
                for event in new_events:
                    if event['id'] == until_id:
                        done = True
                        break
                    events.append(event)
            else:
                done = True

            page += 1
        defer.returnValue(events) 

def test_retry_fail_with_twisted(mock_client, mock_response):
    from twisted.internet import defer

    @defer.inlineCallbacks
    def return_response():
        yield
        defer.returnValue(mock_response)

    # Setup
    CustomException = type("CustomException", (Exception,), {})
    mock_response.with_json({"id": 123, "name": "prkumar"})
    mock_client.with_side_effect(
        [Exception, CustomException, return_response()]
    )
    mock_client.with_io(io.TwistedStrategy())
    github = GitHub(base_url=BASE_URL, client=mock_client)

    # Run
    with pytest.raises(CustomException):
        yield github.get_user("prkumar")

    assert len(mock_client.history) == 2 

def makeRequestAllPages(self, url_args):
        page = 0
        data = []
        while True:
            data.extend((yield self.makeRequest(url_args, page=page)))
            if 'link' not in self.last_response_headers:
                break
            link_hdr = self.last_response_headers['link'][0]
            for link in self.link_re.findall(link_hdr):
                if link[1] == 'next':
                    # note that we don't *use* the page -- why bother?
                    break
            else:
                break # no 'next' link, so we're done
            page += 1
        defer.returnValue(data) 

def _sendStartStatus(self, builderName, build):
        """
        Send start status to GitHub.
        """
        status = yield self._getGitHubRepoProperties(build)
        if not status:
            defer.returnValue(None)

        startTime, _ = build.getTimes()

        description = yield build.render(self._startDescription)

        status.update({
            'state': 'pending',
            'description': description,
            'builderName': builderName,
            'startDateTime': datetime.fromtimestamp(startTime).isoformat(' '),
            'endDateTime': 'In progress',
            'duration': 'In progress',
        })
        result = yield self._sendGitHubStatus(status)
        defer.returnValue(result) 

def _getGitHubRepoProperties(self, build):
        """
        Return a dictionary with GitHub related properties from `build`.
        """
        repoOwner, repoName, sha = yield defer.gatherResults([
            build.render(self._repoOwner),
            build.render(self._repoName),
            build.render(self._sha),
        ])

        if not repoOwner or not repoName:
            defer.returnValue({})

        if not sha:
            log.msg('GitHubStatus: No revision found.')
            defer.returnValue({})

        result = {
            'repoOwner': repoOwner,
            'repoName': repoName,
            'sha': sha,
            'targetURL': self._status.getURLForThing(build),
            'buildNumber': str(build.getNumber()),
        }
        defer.returnValue(result) 

def _fetch(self, arg):
        res = yield super(Gitea, self)._fetch(arg)
        if self.build.hasProperty("pr_id"):
            remote = yield self._dovccmd(
                ['config', 'remote.pr_source.url'], collectStdout=True)
            if remote is None or remote.strip() is '':
                yield self._dovccmd(
                    ['remote', 'add', 'pr_source',
                     self.build.getProperty("head_git_ssh_url", None)])
            else:
                yield self._dovccmd(
                    ['remote', 'set-url', 'pr_source',
                     self.build.getProperty("head_git_ssh_url", None)])
            yield self._dovccmd(['fetch', 'pr_source'])
            res = yield self._dovccmd(['merge', self.build.getProperty("head_sha", None)])
        defer.returnValue(res) 

def _get_inactive_private_keys(self):
        """
        Return all inactive private keys bound to address, that are
        stored locally.
        This can be used to attempt decryption from multiple keys.

        :return: A Deferred which fires the list of inactive keys sorted
                 according to their expiry dates.
        :rtype: Deferred
        """
        all_keys = yield self.get_all_keys(private=True)
        inactive_keys = filter(lambda _key: not _key.is_active(), all_keys)

        inactive_keys = \
            sorted(inactive_keys, key=lambda _key: _key.expiry_date)
        defer.returnValue(inactive_keys) 

def _start_vpn(self):
        try:
            self.log.debug('VPN: start')
            args = [self._vpnconfig, self._providerconfig,
                    self._host, self._port]
            kwargs = {'openvpn_verb': 4, 'remotes': self._remotes,
                      'restartfun': self._restart_vpn}
            vpnproc = VPNProcess(*args, **kwargs)
            self._vpnproc = vpnproc

            self.__start_pre_up(vpnproc)
            cmd = self.__start_get_cmd(vpnproc)
            running = yield self.__start_spawn_proc(vpnproc, cmd)
            vpnproc.pid = running.pid
            defer.returnValue(True)
        except Exception as exc:
            if self._vpnproc:
                self._vpnproc.failed = True
                self._vpnproc.errmsg = exc.message
            raise 

def _stop_vpn(self, restart=False):
        """
        Stops the openvpn subprocess.

        Attempts to send a SIGTERM first, and after a timeout
        it sends a SIGKILL.

        :param restart: whether this stop is part of a hard restart.
        :type restart: bool
        """
        # TODO how to return False if this fails
        # XXX maybe return a deferred

        if self._vpnproc is None:
            self.log.debug('Tried to stop VPN but no process found')
            defer.returnValue(False)

        self._vpnproc.restarting = restart
        self.__stop_pre_down(self._vpnproc)
        stopped = yield self._vpnproc.terminate_or_kill()
        defer.returnValue(stopped) 

def do_get_cert(self, username, anonymous=False):
        try:
            _, provider = username.split('@')
        except ValueError:
            if not username:
                raise ValueError('Need an username. are you logged in?')
            raise ValueError(username + ' is not a valid username, it should'
                             ' contain an @')

        # fetch vpn cert and store
        bonafide = self.parent.getServiceNamed("bonafide")
        _, cert_str = yield bonafide.do_get_vpn_cert(
            username, anonymous=anonymous)

        cert_path = get_vpn_cert_path(provider)
        cert_dir = os.path.dirname(cert_path)
        if not os.path.exists(cert_dir):
            os.makedirs(cert_dir, mode=0700)
        with open(cert_path, 'w') as outf:
            outf.write(cert_str)
        check_and_fix_urw_only(cert_path)
        defer.returnValue({'get_cert': 'ok'}) 

def do_list(self):
        bonafide = self.parent.getServiceNamed("bonafide")
        _providers = yield bonafide.do_provider_list()
        providers = [p['domain'] for p in _providers]
        provider_dict = {}
        for provider in providers:
            try:
                config = yield bonafide.do_provider_read(provider, 'eip')
            except ValueError:
                continue
            gateways = GatewaySelector(
                config.gateways, config.locations,
                preferred={'cc': self._cco, 'loc': self._loc}
            )
            provider_dict[provider] = gateways.get_sorted_gateways()
        defer.returnValue(provider_dict) 

def authenticate(self):
        uri = self._api.get_handshake_uri()
        met = self._api.get_handshake_method()
        self.log.debug('%s to %s' % (met, uri))
        params = self._srp_auth.get_handshake_params()

        handshake = yield self._request(self._agent, uri, values=params,
                                        method=met)

        self._srp_auth.process_handshake(handshake)
        uri = self._api.get_authenticate_uri(login=self.username)
        met = self._api.get_authenticate_method()

        self.log.debug('%s to %s' % (met, uri))
        params = self._srp_auth.get_authentication_params()

        auth = yield self._request(self._agent, uri, values=params,
                                   method=met)

        uuid, token = self._srp_auth.process_authentication(auth)
        self._srp_auth.verify_authentication()

        self._uuid = uuid
        self._token = token
        defer.returnValue(OK) 

def _get_or_create_mailbox(self, mailbox_name):
            """
            Avoid creating variations of the case.
            If there's already a 'Sent' folder, do not create 'SENT', just
            return that.
            """
            name = yield self._get_case_insensitive_mbox(mailbox_name)
            if name is None:
                name = mailbox_name
                yield self.account.add_mailbox(name)
            mailbox = yield self.account.get_collection_by_mailbox(
                name)

            # Pixelated expects the mailbox wrapper;
            # it should limit itself to the Mail API instead.
            # This is also a smell that the collection-mailbox-wrapper
            # distinction is not clearly cut.
            defer.returnValue(mailbox.mbox_wrapper) 

def _maybe_fetch_smtp_certificate(self, userid):
        # check if smtp cert exists
        username, provider = userid.split('@')
        cert_path = _get_smtp_client_cert_path(self._basedir, provider,
                                               username)
        if os.path.exists(cert_path):
            defer.returnValue(None)

        # fetch smtp cert and store
        bonafide = self.parent.getServiceNamed("bonafide")
        _, cert_str = yield bonafide.do_get_smtp_cert(userid)

        cert_dir = os.path.dirname(cert_path)
        if not os.path.exists(cert_dir):
            os.makedirs(cert_dir, mode=0700)
        with open(cert_path, 'w') as outf:
            outf.write(cert_str)
        check_and_fix_urw_only(cert_path) 

def do_msg_status(self, userid, mbox, msgid):
        account = self._get_account(userid)
        msg = yield account.get_message_by_msgid(mbox, msgid)
        if msg is None:
            raise Exception("Not found message id: " + msgid)

        headers = msg.get_headers()
        encryption = headers.get(IncomingMail.LEAP_ENCRYPTION_HEADER, '')
        signature = headers.get(IncomingMail.LEAP_SIGNATURE_HEADER, '')

        status = {}
        pubkey_re = re.compile(' pubkey="([0-9A-F]*)"')
        fingerprint = first(pubkey_re.findall(signature))
        status['signature'] = signature.split(';')[0]
        status['sign_fp'] = fingerprint
        status['encryption'] = encryption

        if ((IncomingMail.LEAP_ENCRYPTION_DECRYPTED == encryption) and
                (IncomingMail.LEAP_SIGNATURE_VALID == status['signature'])):
            status['secured'] = True
        else:
            status['secured'] = False

        defer.returnValue(status)