Python rq.Queue() Examples

def get_status(task_id):
    with Connection(redis.from_url(current_app.config["REDIS_URL"])):
        q = Queue()
        task = q.fetch_job(task_id)
    if task:
        response_object = {
            "status": "success",
            "data": {
                "task_id": task.get_id(),
                "task_status": task.get_status(),
                "task_result": task.result,
            },
        }
    else:
        response_object = {"status": "error"}
    return jsonify(response_object) 

def grade_project(project_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    with Connection(redis.from_url(current_app.config['REDIS_URL'])):
        q = Queue()
        task = q.enqueue(
            create_task,
            project.url,
            current_app.config["OPENFAAS_URL"]
        )
    response_object = {
        'status': 'success',
        'data': {
            'task_id': task.get_id()
        }
    }
    return jsonify(response_object), 202 

def test_set_rq():

    """
    When Redis params are provided, set an RQ instance.
    """

    config = get_config('read/set-rq')

    # Should set an instance.
    assert isinstance(config.rq, Queue)

    args = config.rq.connection.connection_pool.connection_kwargs

    # Should use config args.
    assert args['host'] == 'host'
    assert args['port'] == 1337
    assert args['db']   == 1 

def get_tasks_on_queue(queue_name):
    q = Queue(queue_name, connection=Redis(host=envget('redis.host')))
    jobs = q.jobs
    tasks = []
    for job in jobs:
        task = {"date_enqueued": str(
            process_date(job.to_dict().get('enqueued_at')))}
        '''
        to_dict() returns something like this:
        {u'origin': u'task_no_vt', u'status': u'queued', u'description': u"Api.task.generic_task('N7UFZ56FQDITJ34F40TZB50XAWVNW575QGIL4YEC')", u'created_at': '2017-03-03T20:14:47Z', u'enqueued_at': '2017-03-03T20:14:47Z', u'timeout': 31536000, u'data': '\x80\x02(X\x15\x00\x00\x00Api.task.generic_taskq\x01NU(N7UFZ56FQDITJ34F40TZB50XAWVNW575QGIL4YECq\x02\x85q\x03}q\x04tq\x05.'}
        '''
        task_id = re.search('[A-Z0-9]{40}', job.to_dict().get('description'))
        if task_id is None:
            continue
        task['task_id'] = task_id.group(0)
        task['hashes'] = count_valid_hashes_in_task(task['task_id'])
        tasks.append(task)
    return tasks 

def request_poller(queue_1, queue_2, nb_to_do):
    requests = pollout_requests(queue_2, nb_to_do)
    try:
        with Connection(Redis()) as conn:
            q = Queue(queue_1, connection=conn)
    except:
        mod.display("ASYNC_HTTP",
                    "ERROR",
                    "Could not establish connection with Redis, check if you have redis_host, \
                    redis_port and maybe redis_password in /config/config.ini")

    loop = asyncio.get_event_loop()
    future = asyncio.ensure_future(run(requests))
    x = loop.run_until_complete(future)
    loop.close()
    for y in x:
        if y is not None:
            try:
                q.enqueue(module_worker_response, args=(y), result_ttl=0)
            except:
                mod.display("ASYNC_HTTP",
                            "ERROR",
                            "Could not enqueue job to Redis in func request_poller") 

def test_basic(sentry_init, capture_events):
    sentry_init(integrations=[RqIntegration()])
    events = capture_events()

    queue = rq.Queue(connection=FakeStrictRedis())
    worker = rq.SimpleWorker([queue], connection=queue.connection)

    queue.enqueue(crashing_job, foo=42)
    worker.work(burst=True)

    (event,) = events

    (exception,) = event["exception"]["values"]
    assert exception["type"] == "ZeroDivisionError"
    assert exception["mechanism"]["type"] == "rq"
    assert exception["stacktrace"]["frames"][-1]["vars"]["foo"] == "42"

    assert event["transaction"] == "tests.integrations.rq.test_rq.crashing_job"
    assert event["extra"]["rq-job"] == {
        "args": [],
        "description": "tests.integrations.rq.test_rq.crashing_job(foo=42)",
        "func": "tests.integrations.rq.test_rq.crashing_job",
        "job_id": event["extra"]["rq-job"]["job_id"],
        "kwargs": {"foo": 42},
    } 

def test_transport_shutdown(sentry_init, capture_events_forksafe):
    sentry_init(integrations=[RqIntegration()])

    events = capture_events_forksafe()

    queue = rq.Queue(connection=FakeStrictRedis())
    worker = rq.Worker([queue], connection=queue.connection)

    queue.enqueue(crashing_job, foo=42)
    worker.work(burst=True)

    event = events.read_event()
    events.read_flush()

    (exception,) = event["exception"]["values"]
    assert exception["type"] == "ZeroDivisionError" 

def get_worker(self, listen=()):
        if not listen:
            listen = self.config["queues"]

        def send_to_sentry(job, *exc_info):
            self.sentry.captureException(
                exc_info=exc_info,
                extra={
                    "job_id": job.id,
                    "func": job.__name__,
                    "args": job.args,
                    "kwargs": job.kwargs,
                    "description": job.description,
                },
            )

        exception_handlers = [send_to_sentry]

        return Worker(
            [QueueType(k, connection=self.connection) for k in listen],
            exception_handlers=exception_handlers,
            connection=self.connection,
        ) 

def add_task(requested):
    task_id = id_generator(40)
    if requested.get('document_name') is None:
        requested["document_name"] = ""

    response = {"requested": requested,
                "date_enqueued": datetime.datetime.now(),
                "task_id": task_id}
    save(response)
    if requested.get('vt_samples'):
        queue_name = "task_private_vt"  # task needs a private VT api
    elif requested.get('vt_av') and not requested.get('vt_samples'):
        queue_name = "task_public_vt"  # task needs a public VT api
    else:
        queue_name = "task_no_vt"  # task doesn't need VT
    q = Queue(queue_name, connection=Redis(host=envget('redis.host')))
    job = q.enqueue('Api.task.generic_task', args=(task_id,), timeout=31536000)
    return task_id 

def test_job_archiving_not_supported(self):
        """Check if it fails when archiving is not supported"""

        backend_args = {
            'uri': 'http://example.com/',
            'gitpath': os.path.join(self.dir, 'data/git_log.txt')
        }
        archive_args = {
            'archive_path': self.tmp_path,
            'fetch_from_archive': True
        }

        q = rq.Queue('queue', is_async=False)  # noqa: W606

        with self.assertRaises(AttributeError):
            _ = q.enqueue(execute_perceval_job,
                          backend='git', backend_args=backend_args,
                          category='commit',
                          qitems='items', task_id='mytask', job_number=8,
                          archive_args=archive_args) 

def _calc_rq_free() -> Dict[str, Any]:
    """Parses the output of `rq info` to return total number
    of workers and the count of workers currently idle."""

    conn = default_redis_conn()
    with rq.Connection(connection=conn):
        workers: List[rq.Worker] = [w for w in rq.Worker.all()]
    idle_workers = [w for w in workers if w.get_state() == 'idle']
    resp = {
        'workersTotal': len(workers),
        'workersIdle': len(idle_workers),
        'workersUnknown': len([w for w in workers if w.get_state() == '?'])
    }
    queues = 'default', 'build', 'publish'
    resp.update({f'queue{q.capitalize()}Size':
                 len(rq.Queue(f'gigantum-{q}-queue', connection=conn)) for q in queues})
    return resp 

def __init__(self, registry, conn, queues, polling=SCHED_POLLING, async_mode=True):
        super().__init__()
        self.registry = registry
        self.conn = conn
        self.polling = polling
        self.async_mode = async_mode

        self._rwlock = RWLock()
        self._delayer = sched.scheduler()
        self._queues = {
            queue_id: rq.Queue(queue_id,
                               connection=self.conn,
                               is_async=self.async_mode)  # noqa: W606
            for queue_id in queues
        }
        self._tasks_events = {}
        self._tasks_jobs = {} 

def app_with_scout(redis_conn, scout_config=None):
    """
    Context manager that configures a Huey app with Scout installed.
    """
    # Enable Scout by default in tests.
    if scout_config is None:
        scout_config = {}
    scout_config.setdefault("monitor", True)
    scout_config["core_agent_launch"] = False

    # Reset global state
    scout_apm.rq.install_attempted = False
    scout_apm.rq.installed = None

    # Setup according to https://docs.scoutapm.com/#rq
    # Using job_class argument to Queue
    Config.set(**scout_config)
    queue = Queue(name="myqueue", connection=redis_conn)
    worker = scout_apm.rq.SimpleWorker([queue], connection=queue.connection)

    App = namedtuple("App", ["queue", "worker"])
    try:
        yield App(queue=queue, worker=worker)
    finally:
        Config.reset_all() 

def _get_queue(self, method_name: str) -> rq.Queue:
        # Return the appropriate Queue instance for the given method.
        queue_name = JOB_QUEUE_MAP.get(method_name) or GigantumQueues.default_queue
        return rq.Queue(queue_name.value, connection=self._redis_conn) 

def run_worker():
    """Initializes a slim rq task queue."""
    listen = ['default']
    conn = Redis(
        host=app.config['RQ_DEFAULT_HOST'],
        port=app.config['RQ_DEFAULT_PORT'],
        db=0,
        password=app.config['RQ_DEFAULT_PASSWORD'])

    with Connection(conn):
        worker = Worker(map(Queue, listen))
        worker.work() 

def test_file_drop_GZHandler():
    """ Tests the GZZHandler for file drop """
    a = GZHandler()

    class Event:
        """ Creates a mock event object for tests """
        event_type = None
        src_path = None
        is_directory = None
        q = None
        r = None

        def __init__(self, event_type, src_path, is_directory):
            """ initializes necessary variables for the object """
            self.event_type = event_type
            self.src_path = src_path
            self.is_directory = is_directory
            self.q = Queue(connection=Redis(host='localhost'),
                           default_timeout=86400)
            self.r = StrictRedis(host='localhost', port=6379, db=0)

    b = Event('created', '/dev/null', False)
    c = Event('modified', '/etc/hosts', False)
    a.process(b)
    a.process(b)
    a.process(b)
    a.on_created(b)
    a.on_modified(c) 

def start_worker(queue_name):
    print "starting worker '{}'...".format(queue_name)

    with Connection(redis_rq_conn):
        worker = Worker(Queue(queue_name), exc_handler=failed_job_handler)
        worker.work() 

def test_ready_for_job(self):
        """tests existing and non-existing queues, and redis not running"""
        bad_queue = rq.Queue('a-totally-nonexistent-queue', connection=default_redis_conn())
        with patch.object(Dispatcher, '_get_queue', return_value=bad_queue):
            d = Dispatcher()
            # Here the method is ignored because of the patch above
            assert not d.ready_for_job(self.test_ready_for_job)
        time.sleep(1)
        d = Dispatcher()
        # Since this is an unknown method, it'll go to the default queue
        assert d.ready_for_job(self.test_ready_for_job) 

def run_worker():
    """Initializes a slim rq task queue."""
    listen = ['default']
    conn = Redis(
        host=app.config['RQ_DEFAULT_HOST'],
        port=app.config['RQ_DEFAULT_PORT'],
        db=0,
        password=app.config['RQ_DEFAULT_PASSWORD'])

    with Connection(conn):
        worker = Worker(map(Queue, listen))
        worker.work() 

def create_job_queues(host, port, num_of_queues):
    """Create the job queues for asynchronous processing

    TODO: The redis queue approach does not work and is deactivated

    Note:
        Make sure that the global configuration was updated
        before calling this function.

    Args:
        host: The hostname of the redis server
        port: The port of the redis server
        num_of_queues: The number fo queues that should be created

    """
    # Redis work queue and connection
    global job_queues, redis_conn, num_queues

    redis_conn = Redis(host=host, port=port)
    num_queues = num_of_queues

    for i in range(num_of_queues):
        name = "%s_%i" % (global_config.WORKER_QUEUE_NAME, i)

        string = "Create queue %s with server %s:%s" % (name, host, port)
        print(string)
        queue = rq.Queue(name, connection=redis_conn)
        job_queues.append(queue) 

def handle(self, *args, **options):
        redis_conn = django_rq.get_connection('default')
        
        q = Queue(settings.DJANGO_TEST_RQ_LOW_QUEUE, connection=redis_conn)
        worker = Worker([q], exc_handler=my_handler, connection=redis_conn)
        worker.work() 

def add_hash_to_process_queue(sha1):
    q = Queue('process', connection=Redis(host=envget('redis').get('host')))
    job = q.enqueue('process_hash.generic_process_hash',
                    args=(sha1,), timeout=70) 

def main():
    client_id = os.getenv('IMGUR_CLIENT_ID')
    if not client_id:
        raise Exception("Couldn't find IMGUR_CLIENT_ID environment variable!")
    download_dir = setup_download_dir()
    links = get_links(client_id)
    q = Queue(connection=Redis(host='localhost', port=6379))
    for link in links:
        q.enqueue(download_link, download_dir, link) 

def get_rq(self):

        """
        Get an RQ instance.

        Returns:
            rq.Queue
        """

        redis = self.get_redis()

        if redis:
            return Queue(connection=redis) 

def run_worker():
    """Initializes a slim rq task queue."""
    listen = ['default']
    conn = Redis(
        host=app.config['RQ_DEFAULT_HOST'],
        port=app.config['RQ_DEFAULT_PORT'],
        db=0,
        password=app.config['RQ_DEFAULT_PASSWORD'])

    with Connection(conn):
        worker = Worker(map(Queue, listen))
        worker.work() 

def start_worker(queue_name):
    print "starting worker '{}'...".format(queue_name)

    with Connection(redis_rq_conn):
        worker = Worker(Queue(queue_name), exc_handler=failed_job_handler)
        worker.work() 

def run(self, redis):
        redis_connection = Redis(redis[0], redis[1], password=redis[2])
        with Connection(redis_connection):
            qs = map(Queue, self.queues) or [Queue()]
            w = rq.Worker(qs)
            w.work()
            print('Items in queue \'{0}\': {1}'.format(self.queues[0], len(qs))) 

def _enqueue_jobs(self, configuration):
        redis_connection = Redis(host=configuration.REDIS.server, port=configuration.REDIS.port, password=configuration.REDIS.password)
        queue = Queue(self.job_queue_name, connection=redis_connection)
        for small_bbox in self.small_bboxes:
            queue.enqueue_call(func=worker_functions.detect, args=(small_bbox, self.configuration),
                               timeout=self.configuration.JOB.timeout)
        print('Number of enqueued jobs in queue \'{0}\': {1}'.format(self.job_queue_name, len(queue))) 

def enqueue_results(result_nodes, redis_connection):
    q = Queue('results', connection=redis_connection)
    q.enqueue_call(func=store, args=(result_nodes,), timeout=5400)
    for result_node in result_nodes:
        redis_connection.rpush('visualizing', result_node.to_geojson()) 

def time_based_poller(working_queue, request_queue):
        starttime = time.time()
        queue_1 = working_queue
        queue_2 = request_queue

        while True:
            redis_host, redis_port, redis_password = init_redis()
            try:
                r = redis.StrictRedis(host=redis_host, port=redis_port,
                                      password=redis_password)
            except:
                mod.display("POLLER",
                            message_type="ERROR",
                            string="Cannot establish connection with Redis in func time_based_poller")
            try:
                len = r.llen(queue_2)
            except:
                mod.display("POLLER",
                            message_type="ERROR",
                            string="Cannot ask queue: %s length to Redis" % (queue_2))

            if len <= 0:
                time.sleep(1.0 - ((time.time() - starttime) % 1.0))
                continue
            try:
                with Connection(Redis()) as conn:
                    q = Queue(queue_1, connection=conn)
                q.enqueue(async_http.request_poller,
                          args=(queue_1, queue_2, len), result_ttl=0)
            except:
                mod.display("POLLER",
                            message_type="ERROR",
                            string="Could not establish connection with Redis, check if you have redis_host, \
                            redis_port and maybe redis_password in /config/config.ini")
            time.sleep(1.0 - ((time.time() - starttime) % 1.0))