Python celery.signature() Examples

def await_blockchain_success_evil(task_uuid, timeout=None, poll_frequency=0.5):
    elapsed = 0
    print(f'Awaiting success for task uuid: {task_uuid}')
    while timeout is None or elapsed <= timeout:
        sig = signature(
            utils.eth_endpoint('get_task'),
            kwargs={'task_uuid': task_uuid}
        )

        task = utils.execute_synchronous_celery(sig)

        if task and task['status'] == 'SUCCESS':
            return task
        else:
            sleep(poll_frequency)
            elapsed += poll_frequency

    raise TimeoutError 

def topup_wallets(queue='low-priority'):
    wallets = persistence_module.get_all_wallets()

    for wallet in wallets:
        if (wallet.wei_topup_threshold or 0) > 0:

            last_topup_task_uuid = wallet.last_topup_task_uuid

            if last_topup_task_uuid:
                task = persistence_module.get_task_from_uuid(last_topup_task_uuid)

                if task and task.status in ['PENDING', 'UNSTARTED']:
                    return

            signature(utils.eth_endpoint('topup_wallet_if_required'),
                      kwargs={
                          'address': wallet.address
                      }).delay() 

def topup_if_required(address):
    balance = w3.eth.getBalance(address)

    wallet = persistence_module.get_wallet_by_address(address)
    wei_topup_threshold = wallet.wei_topup_threshold
    wei_target_balance = wallet.wei_target_balance or 0

    if balance <= wei_topup_threshold and wei_target_balance > balance:
        sig = signature(utils.eth_endpoint('send_eth'),
                        kwargs={
                            'signing_address': config.MASTER_WALLET_ADDRESS,
                            'amount_wei': wei_target_balance - balance,
                            'recipient_address': address,
                            'prior_tasks': []
                        })

        task_uuid = utils.execute_task(sig)

        persistence_module.set_wallet_last_topup_task_uuid(address, task_uuid)

        return task_uuid

    return None 

def topup_if_required(self, wallet, posterior_task_uuid):
        balance = self.w3.eth.getBalance(wallet.address)

        wei_topup_threshold = wallet.wei_topup_threshold
        wei_target_balance = wallet.wei_target_balance or 0

        if balance <= wei_topup_threshold and wei_target_balance > balance:
            sig = signature(utils.eth_endpoint('send_eth'),
                            kwargs={
                                'signing_address': config.MASTER_WALLET_ADDRESS,
                                'amount_wei': wei_target_balance - balance,
                                'recipient_address': wallet.address,
                                'prior_tasks': [],
                                'posterior_tasks': [posterior_task_uuid]
                            })

            task_uuid = utils.execute_task(sig)

            self.persistence_interface.set_wallet_last_topup_task_uuid(wallet.address, task_uuid)

            return task_uuid

        return None 

def new_transaction_attempt(self, task):
        number_of_attempts_this_round = abs(
            len(task.transactions) - self.task_max_retries * (task.previous_invocations or 0)
        )
        if number_of_attempts_this_round >= self.task_max_retries:
            print(f"Maximum retries exceeded for task {task.uuid}")

            if task.status_text != 'SUCCESS':
                self.persistence_interface.set_task_status_text(task, 'FAILED')

            raise TaskRetriesExceededError

        else:
            signature(utils.eth_endpoint('_attempt_transaction'),
                      args=(task.uuid,)).apply_async(
                countdown=RETRY_TRANSACTION_BASE_TIME * 4 ** number_of_attempts_this_round
            ) 

def barrier(self, data, merging=False, sequential=False, replace=None, root_docs=None):
    replacement = []
    if isinstance(data[0], basestring):
        data = [data]
    # XXX: ordering might be incorrect because of suffixes. fix by replacing
    # tuples with class
    data = sorted(data, key=lambda x: x[1])
    # merge output from same source documents
    if merging == 'doc':
        for docs, task in izip(cycle(_group_by_prefix(data, root_docs)), replace):
            if sequential:
                task[0]['args'] = [docs]
                replacement.append(chain(signature(t) for t in task))
            else:
                task['args'] = [docs]
                replacement.append(signature(task))
    # merge everything
    elif merging:
        for task in replace:
            if sequential:
                task[0]['args'] = [data]
                replacement.append(chain(signature(t) for t in task))
            else:
                task['args'] = [data]
                replacement.append(signature(task))
    else:
        for ret_val, task in izip(cycle(data), replace):
            if sequential:
                task[0]['args'] = [ret_val]
                replacement.append(chain(signature(t) for t in task))
            else:
                task['args'] = [ret_val]
                replacement.append(signature(task))
    raise self.replace(group(replacement)) 

def deploy_contract_task(signing_address, contract_name, args=None, prior_tasks=None):
    deploy_sig = signature(
        utils.eth_endpoint('deploy_contract'),
        kwargs={
            'signing_address': signing_address,
            'contract_name': contract_name,
            'args': args,
            'prior_tasks': prior_tasks
        })

    return utils.execute_task(deploy_sig) 

def run_task_finish_handler(task_id, name, title, metadata, priority):
    options = metadata['options']
    options['task_id'] = task_id
    options['parent_id'] = None
    options['main_task_id'] = None
    options['title'] = title
    options['run_after_sub_tasks_finished'] = False
    options['run_if_parent_task_failed'] = False
    priority = priority
    queue = get_queue_by_task_priority(priority)
    task = signature(name, args=metadata['args'], **options)
    task.apply_async(priority=priority, queue=queue) 

def generate_input_signature(self):
        loc_file = self.storage_link(self.portfolio.location_file)
        acc_file = self.storage_link(self.portfolio.accounts_file)
        info_file = self.storage_link(self.portfolio.reinsurance_info_file)
        scope_file = self.storage_link(self.portfolio.reinsurance_scope_file)
        settings_file = self.storage_link(self.settings_file)
        complex_data_files = self.create_complex_model_data_file_dicts()

        return signature(
            'generate_input',
            args=(self.pk, loc_file, acc_file, info_file, scope_file, settings_file, complex_data_files),
            queue=self.model.queue_name
        ) 

def generate_inputs(self, initiator):
        valid_choices = [
            self.status_choices.NEW,
            self.status_choices.INPUTS_GENERATION_ERROR,
            self.status_choices.INPUTS_GENERATION_CANCELLED,
            self.status_choices.READY,
            self.status_choices.RUN_COMPLETED,
            self.status_choices.RUN_CANCELLED,
            self.status_choices.RUN_ERROR,
        ]

        errors = {}
        if self.status not in valid_choices:
            errors['status'] = ['Analysis status must be one of [{}]'.format(', '.join(valid_choices))]

        if self.model.deleted:
            errors['model'] = ['Model pk "{}" has been deleted'.format(self.model.id)]

        if not self.portfolio.location_file:
            errors['portfolio'] = ['"location_file" must not be null']

        if errors:
            raise ValidationError(errors)

        self.status = self.status_choices.INPUTS_GENERATION_QUEUED
        self.lookup_errors_file = None
        self.lookup_success_file = None
        self.lookup_validation_file = None
        self.summary_levels_file = None
        self.input_generation_traceback_file_id = None

        generate_input_signature = self.generate_input_signature
        generate_input_signature.link(record_generate_input_result.s(self.pk, initiator.pk))
        generate_input_signature.link_error(
            signature('on_error', args=('record_generate_input_failure', self.pk, initiator.pk), queue=self.model.queue_name)
        )
        self.generate_inputs_task_id = generate_input_signature.delay().id
        self.task_started = timezone.now()
        self.task_finished = None
        self.save() 

def run_analysis_signature(self):
        complex_data_files = self.create_complex_model_data_file_dicts()
        input_file = self.storage_link(self.input_file)
        settings_file = self.storage_link(self.settings_file)

        return signature(
            'run_analysis',
            args=(self.pk, input_file, settings_file, complex_data_files),
            queue=self.model.queue_name,
        ) 

def run(self, initiator):
        self.validate_run()

        self.status = self.status_choices.RUN_QUEUED

        run_analysis_signature = self.run_analysis_signature
        run_analysis_signature.link(record_run_analysis_result.s(self.pk, initiator.pk))
        run_analysis_signature.link_error(
            signature('on_error', args=('record_run_analysis_failure', self.pk, initiator.pk), queue=self.model.queue_name)
        )
        dispatched_task = run_analysis_signature.delay()
        self.run_task_id = dispatched_task.id
        self.task_started = timezone.now()
        self.task_finished = None
        self.save() 

def _retry_task(self, task):
        self.persistence_interface.increment_task_invokations(task)
        signature(utils.eth_endpoint('_attempt_transaction'), args=(task.uuid,)).delay() 

def deploy_contract(
            self,
            uuid: UUID,
            contract_name: str,
            args: Optional[tuple] = None, kwargs: Optional[dict] = None,
            signing_address: Optional[str] = None, encrypted_private_key: Optional[str]=None,
            gas_limit: Optional[int] = None,
            prior_tasks: Optional[UUIDList] = None
    ):
        """
        The main deploy contract entrypoint for the processor.

        :param uuid: the celery generated uuid for the task
        :param contract_name: System will attempt to fetched abi and bytecode from this
        :param args: arguments for the constructor
        :param kwargs: keyword arguments for the constructor
        :param signing_address: address of the wallet signing the txn
        :param encrypted_private_key: private key of the wallet making the transaction, encrypted using key from settings
        :param gas_limit: limit on the amount of gas txn can use. Overrides system default
        :param prior_tasks: a list of task uuid that must succeed before this task will be attempted
        """

        signing_wallet_obj = self.get_signing_wallet_object(signing_address, encrypted_private_key)

        task = self.persistence_interface.create_deploy_contract_task(uuid,
                                                                      signing_wallet_obj,
                                                                      contract_name,
                                                                      args, kwargs,
                                                                      gas_limit,
                                                                      prior_tasks)

        # Attempt Create Async Transaction
        signature(utils.eth_endpoint('_attempt_transaction'), args=(task.uuid,)).delay() 

def send_eth(self,
                 uuid: UUID,
                 amount_wei: int,
                 recipient_address: str,
                 signing_address: Optional[str] = None, encrypted_private_key: Optional[str] = None,
                 prior_tasks: Optional[UUIDList] = None,
                 posterior_tasks: Optional[UUIDList] = None):
        """
        The main entrypoint sending eth.

        :param uuid: the celery generated uuid for the task
        :param amount_wei: the amount in WEI to send
        :param recipient_address: the recipient address
        :param signing_address: address of the wallet signing the txn
        :param encrypted_private_key: private key of the wallet making the transaction, encrypted using key from settings
        :param prior_tasks: a list of task uuids that must succeed before this task will be attempted
        :param posterior_tasks: a uuid list of tasks for which this task must succeed before they will be attempted
        :return: task_id
        """

        signing_wallet_obj = self.get_signing_wallet_object(signing_address, encrypted_private_key)

        task = self.persistence_interface.create_send_eth_task(uuid,
                                                               signing_wallet_obj,
                                                               recipient_address, amount_wei,
                                                               prior_tasks,
                                                               posterior_tasks)

        # Attempt Create Async Transaction
        signature(utils.eth_endpoint('_attempt_transaction'), args=(task.uuid,)).delay() 

def synchronous_call(contract_address, contract_type, func, args=None):
    call_sig = signature(
        utils.eth_endpoint('call_contract_function'),
        kwargs={
            'contract_address': contract_address,
            'abi_type': contract_type,
            'function': func,
            'args': args,
        })

    return utils.execute_synchronous_celery(call_sig) 

def send_eth_task(signing_address, amount_wei, recipient_address):
    sig = signature(
        utils.eth_endpoint('send_eth'),
        kwargs={
            'signing_address': signing_address,
            'amount_wei': amount_wei,
            'recipient_address': recipient_address
        })

    return utils.execute_task(sig) 

def transaction_task(signing_address,
                     contract_address, contract_type,
                     func, args=None,
                     gas_limit=None,
                     prior_tasks=None,
                     reverses_task=None):

    kwargs = {
        'signing_address': signing_address,
        'contract_address': contract_address,
        'abi_type': contract_type,
        'function': func,
        'args': args,
        'prior_tasks': prior_tasks,
        'reverses_task': reverses_task
    }

    if gas_limit:
        kwargs['gas_limit'] = gas_limit

    sig = signature(
        utils.eth_endpoint('transact_with_contract_function'),
        kwargs=kwargs)

    return utils.execute_task(sig) 

def check_transaction_response(self, celery_task, transaction_id):
        def transaction_response_countdown():
            t = lambda retries: ETH_CHECK_TRANSACTION_BASE_TIME * 2 ** retries

            # If the system has been longer than the max retry period
            # if previous_result:
            #     submitted_at = datetime.strptime(previous_result['submitted_date'], "%Y-%m-%d %H:%M:%S.%f")
            #     if (datetime.utcnow() - submitted_at).total_seconds() > ETH_CHECK_TRANSACTION_RETRIES_TIME_LIMIT:
            #         if self.request.retries != self.max_retries:
            #             self.request.retries = self.max_retries - 1
            #
            #         return 0

            return t(celery_task.request.retries)

        try:
            transaction_object = self.persistence_interface.get_transaction(transaction_id)

            task = transaction_object.task

            transaction_hash = transaction_object.hash

            result = self.check_transaction_hash(transaction_hash)

            self.persistence_interface.update_transaction_data(transaction_id, result)

            status = result.get('status')

            print(f'Status for transaction {transaction_object.id} of task UUID {task.uuid} is:'
            f'\n {status}')
            if status == 'SUCCESS':

                unstarted_posteriors = self.get_unstarted_posteriors(task)

                self.persistence_interface.set_task_status_text(task, 'SUCCESS')

                for dep_task in unstarted_posteriors:
                    print('Starting posterior task: {}'.format(dep_task.uuid))
                    signature(utils.eth_endpoint('_attempt_transaction'), args=(dep_task.uuid,)).delay()


            if status == 'PENDING':
                celery_task.request.retries = 0
                raise Exception("Need Retry")

            if status == 'FAILED':
                self.new_transaction_attempt(task)

        except TaskRetriesExceededError as e:
            pass

        except Exception as e:
            print(e)
            celery_task.retry(countdown=transaction_response_countdown())