Python Examples of celery.exceptions.SoftTimeLimitExceeded

Source File: execute_command.py From scoringengine with MIT License

6 votes

def execute_command(job):
    output = ""
    # Disable duplicate celery log messages
    if logger.propagate:
        logger.propagate = False
    logger.info("Running cmd for " + str(job))
    try:
        cmd_result = subprocess.run(
            job['command'],
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT
        )
        output = cmd_result.stdout.decode("utf-8")
        job['errored_out'] = False
    except SoftTimeLimitExceeded:
        job['errored_out'] = True
    job['output'] = output
    return job

Source File: comment.py From weibospider with MIT License

6 votes

def crawl_comment_by_page(mid, page_num):
    try:
        cur_url = BASE_URL.format(mid, page_num)
        html = get_page(cur_url, auth_level=1, is_ajax=True)
        comment_datas = comment.get_comment_list(html, mid)
    except SoftTimeLimitExceeded:
        crawler.error(
            "comment SoftTimeLimitExceeded    mid={mid} page_num={page_num}".
            format(mid=mid, page_num=page_num))
        app.send_task(
            'tasks.comment.crawl_comment_by_page',
            args=(mid, page_num),
            queue='comment_page_crawler',
            routing_key='comment_page_info')
    CommentOper.add_all(comment_datas)
    if page_num == 1:
        WbDataOper.set_weibo_comment_crawled(mid)
    return html, comment_datas

Source File: retriever.py From http-observatory with Mozilla Public License 2.0

6 votes

def __get(session, relative_path='/', headers=None, cookies=None):
    if not headers:
        headers = {}

    if not cookies:
        cookies = {}

    try:
        # TODO: limit the maximum size of the response, to keep malicious site operators from killing us
        # TODO: Perhaps we can naively do it for now by simply setting a timeout?
        # TODO: catch TLS errors instead of just setting it to None?
        return session.get(session.url.scheme + '://' + session.url.netloc + relative_path,
                           headers=headers,
                           cookies=cookies,
                           timeout=TIMEOUT)
    # Let celery exceptions percolate upward
    except (SoftTimeLimitExceeded, TimeLimitExceeded):
        raise
    except (KeyboardInterrupt, SystemExit):
        raise
    except:
        return None

Source File: scan.py From seecode-scanner with GNU General Public License v3.0

6 votes

def start(**kwargs):
    """
    添加扫描任务
    :param kwargs: 任务配置
    :return:
    """
    pro = None
    try:
        init()
        result_file = '{0}.json'.format(kwargs.get("project_name"))
        kwargs['result_file'] = result_file
        pro = start_scan(**kwargs)
    except SoftTimeLimitExceeded as ex:
        logger.critical(ex)
        if pro:
            pro.update_scan_failed_status(ex)
    except Exception as ex:
        import traceback;traceback.print_exc()
        logger.critical(ex)
    except KeyboardInterrupt:
        logger.error("user aborted")
    except SystemExit:
        raise

    return False

Source File: test_execute_command.py From scoringengine with MIT License

5 votes

def test_timed_out(self):
        import subprocess
        subprocess.run = mock.Mock(side_effect=SoftTimeLimitExceeded)

        job = Job(environment_id="12345", command="echo 'HELLO'")
        task = execute_command.run(job)
        assert task['errored_out'] is True

Source File: user.py From weibospider with MIT License

5 votes

def crawl_person_infos(uid):
    """
    Crawl user info and their fans and followers
    For the limit of weibo's backend, we can only crawl 5 pages of the fans and followers.
    We also have no permissions to view enterprise's followers and fans info
    :param uid: current user id
    :return: None
    """
    if not uid:
        return

    try:
        user, is_crawled = get_profile(uid)
        # If it's enterprise user, just skip it
        if user and user.verify_type == 2:
            SeedidsOper.set_seed_other_crawled(uid)
            return

        # Crawl fans and followers
        if not is_crawled:
            app.send_task('tasks.user.crawl_follower_fans', args=(uid,), queue='fans_followers',
                          routing_key='for_fans_followers')

    # By adding '--soft-time-limit secs' when you start celery, this will resend task to broker
    # e.g. celery -A tasks.workers -Q user_crawler worker -l info -c 1 --soft-time-limit 10
    except SoftTimeLimitExceeded:
        crawler.error("user SoftTimeLimitExceeded    uid={uid}".format(uid=uid))
        app.send_task('tasks.user.crawl_person_infos', args=(uid, ), queue='user_crawler',
                      routing_key='for_user_info')