Python rq.get_current_job() Examples
The following are 28
code examples of rq.get_current_job().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
rq
, or try the search function
.
Example #1
Source File: tasks.py From recon-ng with GNU General Public License v3.0 | 6 votes |
def run_module(workspace, module): results = {} try: # instantiate important objects job = get_current_job() recon = base.Recon(check=False, analytics=False, marketplace=False) recon.start(base.Mode.JOB, workspace=workspace) tasks = Tasks(recon) # update the task's status tasks.update_task(job.get_id(), status=job.get_status()) # execute the task module = recon._loaded_modules.get(module) module.run() except Exception as e: results['error'] = { 'type': str(type(e)), 'message': str(e), 'traceback': traceback.format_exc(), } results['summary'] = module._summary_counts # update the task's status and results tasks.update_task(job.get_id(), status='finished', result=results) return results
Example #2
Source File: jobs.py From gigantum-client with MIT License | 6 votes |
def test_sleep(n): """Used only for testing -- example method with argument. """ logger = LMLogger.get_logger() logger.info("Starting test_sleep({}) in pid {}".format(n, os.getpid())) try: job = get_current_job() job.meta['sample'] = 'test_sleep metadata' job.meta['pid'] = int(os.getpid()) job.save_meta() time.sleep(n) logger.info("Completed test_sleep in pid {}".format(os.getpid())) return 0 except Exception as e: logger.error("Error on test_sleep in pid {}: {}".format(os.getpid(), e)) raise
Example #3
Source File: tasks.py From django-test-rq with The Unlicense | 6 votes |
def long_runnig_task(task): job = get_current_job() task.job_id = job.get_id() task.result = 'STARTED' duration_in_second_persentages = task.duration*1.0 / 100 for i in range(100): import time task.progress = i task.save() print task.progress time.sleep(duration_in_second_persentages) task.result = 'FINISHED' task.save() return task.result
Example #4
Source File: tasks.py From django-test-rq with The Unlicense | 6 votes |
def scheduled_get_url_words(url): """ This creates a ScheduledTask instance for each group of scheduled task - each time this scheduled task is run a new instance of ScheduledTaskInstance will be created """ job = get_current_job() task, created = ScheduledTask.objects.get_or_create( job_id=job.get_id(), name=url ) response = requests.get(url) response_len = len(response.text) ScheduledTaskInstance.objects.create( scheduled_task=task, result = response_len, ) return response_len
Example #5
Source File: __init__.py From ok with Apache License 2.0 | 5 votes |
def background_job(f): @functools.wraps(f) def job_handler(*args, **kwargs): job = get_current_job() job.status = 'running' db.session.commit() stream = io.StringIO() handler = JobLogHandler(stream, job) logger = get_job_logger() logger.setLevel(logging.INFO) logger.addHandler(handler) return_value = None try: return_value = f(*args, **kwargs) except Exception: job.failed = True logger.exception('Job failed') job.status = 'finished' job.result = _format_result(job, return_value) job.log = handler.contents stream.close() db.session.commit() return job_handler
Example #6
Source File: __init__.py From ok with Apache License 2.0 | 5 votes |
def get_job_logger(): return logging.getLogger('{}.job_{}'.format(__name__, get_current_job().id))
Example #7
Source File: __init__.py From ok with Apache License 2.0 | 5 votes |
def get_current_job(): rq_job = rq.get_current_job(connection=get_connection()) return Job.query.get(rq_job.id)
Example #8
Source File: jobs.py From upribox with GNU General Public License v3.0 | 5 votes |
def job_clear_messages(): job = get_current_job(connection=django_rq.get_connection()) job.meta['messages'] = deque() job.save_meta() job.save()
Example #9
Source File: jobs.py From upribox with GNU General Public License v3.0 | 5 votes |
def job_message(message, status="success"): job = get_current_job(connection=django_rq.get_connection()) if not job.meta.get('messages'): job.meta['messages'] = deque() job.meta['messages'].append({"message": message, "status": status}) job.save_meta() job.save()
Example #10
Source File: dataset_jobs.py From gigantum-client with MIT License | 5 votes |
def hash_dataset_files(logged_in_username: str, dataset_owner: str, dataset_name: str, file_list: List, config_file: str = None) -> None: """ Args: logged_in_username: username for the currently logged in user dataset_owner: Owner of the labbook if this dataset is linked dataset_name: Name of the labbook if this dataset is linked file_list: List of files to be hashed config_file: Optional config file to use Returns: None """ logger = LMLogger.get_logger() p = os.getpid() try: logger.info(f"(Job {p}) Starting hash_dataset_files(logged_in_username={logged_in_username}," f"dataset_owner={dataset_owner}, dataset_name={dataset_name}") ds = InventoryManager(config_file=config_file).load_dataset(logged_in_username, dataset_owner, dataset_name) manifest = Manifest(ds, logged_in_username) hash_result, fast_hash_result = manifest.hash_files(file_list) job = get_current_job() if job: job.meta['hash_result'] = ",".join(['None' if v is None else v for v in hash_result]) job.meta['fast_hash_result'] = ",".join(['None' if v is None else v for v in fast_hash_result]) job.save_meta() except Exception as err: logger.error(f"(Job {p}) Error in clean_dataset_file_cache job") logger.exception(err) raise
Example #11
Source File: jobs.py From gigantum-client with MIT License | 5 votes |
def import_dataset_from_zip(archive_path: str, username: str, owner: str, config_file: Optional[str] = None) -> str: """Method to import a dataset from a zip file Args: archive_path(str): Path to the uploaded zip username(str): Username owner(str): Owner username config_file(str): Optional path to a labmanager config file Returns: str: directory path of imported labbook """ def update_meta(msg): job = get_current_job() if not job: return job.meta['feedback'] = msg job.save_meta() p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting import_dataset_from_zip(archive_path={archive_path}," f"username={username}, owner={owner}, config_file={config_file})") try: lb = ZipExporter.import_dataset(archive_path, username, owner, config_file=config_file, update_meta=update_meta) return lb.root_dir except Exception as e: logger.exception(f"(Job {p}) Error on import_dataset_from_zip({archive_path}): {e}") raise finally: if os.path.exists(archive_path): os.remove(archive_path)
Example #12
Source File: jobs.py From gigantum-client with MIT License | 5 votes |
def import_labboook_from_zip(archive_path: str, username: str, owner: str, config_file: Optional[str] = None) -> str: """Method to import a labbook from a zip file Args: archive_path(str): Path to the uploaded zip username(str): Username owner(str): Owner username config_file(str): Optional path to a labmanager config file Returns: str: directory path of imported labbook """ def update_meta(msg): job = get_current_job() if not job: return job.meta['feedback'] = msg job.save_meta() p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting import_labbook_from_zip(archive_path={archive_path}," f"username={username}, owner={owner}, config_file={config_file})") try: lb = ZipExporter.import_labbook(archive_path, username, owner, config_file=config_file, update_meta=update_meta) return lb.root_dir except Exception as e: logger.exception(f"(Job {p}) Error on import_labbook_from_zip({archive_path}): {e}") raise finally: if os.path.exists(archive_path): os.remove(archive_path)
Example #13
Source File: jobs.py From gigantum-client with MIT License | 5 votes |
def import_labbook_from_remote(remote_url: str, username: str, config_file: str = None) -> str: """Return the root directory of the newly imported Project Args: remote_url: Canonical world-facing URI, like "https://repo.domain/owner/project". This will be converted to the actual network location for our repository, like "https://username@repo.domain/owner/project.git/", as robustly as we can manage. username: username for currently logged in user config_file: a copy of the parsed config file Returns: Path to project root directory """ p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting import_labbook_from_remote({remote_url}, {username})") def update_meta(msg): job = get_current_job() if not job: return if 'feedback' not in job.meta: job.meta['feedback'] = msg else: job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}' job.save_meta() remote = RepoLocation(remote_url, username) update_meta(f"Importing Project from {remote.owner_repo!r}...") try: wf = LabbookWorkflow.import_from_remote(remote, username, config_file) except Exception as e: update_meta(f"Could not import Project from {remote.remote_location}.") logger.exception(f"(Job {p}) Error on import_labbook_from_remote: {e}") raise update_meta(f"Imported Project {wf.labbook.name}!") return wf.labbook.root_dir
Example #14
Source File: jobs.py From gigantum-client with MIT License | 5 votes |
def sync_repository(repository: Repository, username: str, override: MergeOverride, remote: str = "origin", access_token: str = None, pull_only: bool = False, id_token: str = None) -> int: p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting sync_repository({str(repository)})") def update_feedback(msg: str, has_failures: Optional[bool] = None, failure_detail: Optional[str] = None, percent_complete: Optional[float] = None): """Method to update the job's metadata and provide feedback to the UI""" current_job = get_current_job() if not current_job: return if has_failures: current_job.meta['has_failures'] = has_failures if failure_detail: current_job.meta['failure_detail'] = failure_detail if percent_complete: current_job.meta['percent_complete'] = percent_complete current_job.meta['feedback'] = msg current_job.save_meta() try: update_feedback("Sync task in queue") with repository.lock(): if isinstance(repository, LabBook): wf = LabbookWorkflow(repository) else: wf = DatasetWorkflow(repository) # type: ignore cnt = wf.sync(username=username, remote=remote, override=override, feedback_callback=update_feedback, access_token=access_token, id_token=id_token, pull_only=pull_only) logger.info(f"(Job {p} Completed sync_repository with cnt={cnt}") return cnt except MergeConflict as me: logger.exception(f"(Job {p}) Merge conflict: {me}") raise
Example #15
Source File: jobs.py From gigantum-client with MIT License | 5 votes |
def publish_repository(repository: Repository, username: str, access_token: str, remote: Optional[str] = None, public: bool = False, id_token: str = None) -> None: p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting publish_repository({str(repository)})") def update_feedback(msg: str, has_failures: Optional[bool] = None, failure_detail: Optional[str] = None, percent_complete: Optional[float] = None): """Method to update the job's metadata and provide feedback to the UI""" current_job = get_current_job() if not current_job: return if has_failures: current_job.meta['has_failures'] = has_failures if failure_detail: current_job.meta['failure_detail'] = failure_detail if percent_complete: current_job.meta['percent_complete'] = percent_complete current_job.meta['feedback'] = msg current_job.save_meta() update_feedback("Publish task in queue") with repository.lock(): if isinstance(repository, LabBook): wf = LabbookWorkflow(repository) else: wf = DatasetWorkflow(repository) # type: ignore wf.publish(username=username, access_token=access_token, remote=remote or "origin", public=public, feedback_callback=update_feedback, id_token=id_token)
Example #16
Source File: rq_test.py From busy-beaver with MIT License | 5 votes |
def add(x, y): job = get_current_job() task = Task(job_id=job.id, name="Test Task", description="Trying things out") db.session.add(task) db.session.commit() set_task_progress(100) return x + y
Example #17
Source File: rq.py From busy-beaver with MIT License | 5 votes |
def set_task_progress(progress): job = get_current_job() if job: job.meta["progress"] = progress job.save_meta() if progress >= 100: task = Task.query.filter_by(job_id=job.get_id()).first() if task: task.complete = True db.session.commit()
Example #18
Source File: tasks.py From django-test-rq with The Unlicense | 5 votes |
def get_url_words(url): # This creates a Task instance to save the job instance and job result job = get_current_job() task = Task.objects.create( job_id=job.get_id(), name=url ) response = requests.get(url) task.result = len(response.text) task.save() return task.result
Example #19
Source File: treeitem.py From zing with GNU General Public License v3.0 | 5 votes |
def update_cache_job(instance): """RQ job""" job = get_current_job() job_wrapper = JobWrapper(job.id, job.connection) keys, decrement = job_wrapper.get_job_params() # close unusable and obsolete connections before and after the job # Note: setting CONN_MAX_AGE parameter can have negative side-effects # CONN_MAX_AGE value should be lower than DB wait_timeout connection.close_if_unusable_or_obsolete() instance._update_cache_job(keys, decrement) connection.close_if_unusable_or_obsolete() job_wrapper.clear_job_params()
Example #20
Source File: treeitem.py From zing with GNU General Public License v3.0 | 5 votes |
def unregister_dirty(self, decrement=1): """Unregister current TreeItem as dirty (should be called from RQ job procedure after cache is updated) """ r_con = get_connection() job = get_current_job() if job: logger.debug( "UNREGISTER %s (-%s) where job_id=%s", self.cache_key, decrement, job.id ) else: logger.debug("UNREGISTER %s (-%s)", self.cache_key, decrement) r_con.zincrby(KEY_DIRTY_TREEITEMS, 0 - decrement, self.cache_key)
Example #21
Source File: treeitem.py From zing with GNU General Public License v3.0 | 5 votes |
def unregister_all_dirty(self, decrement=1): """Unregister current TreeItem and all parent paths as dirty (should be called from RQ job procedure after cache is updated) """ r_con = get_connection() job = get_current_job() for p in self.all_pootle_paths(): if job: logger.debug( "UNREGISTER %s (-%s) where job_id=%s", p, decrement, job.id ) else: logger.debug("UNREGISTER %s (-%s)", p, decrement) r_con.zincrby(KEY_DIRTY_TREEITEMS, 0 - decrement, p)
Example #22
Source File: tasks.py From docsbox with MIT License | 5 votes |
def process_document(path, options, meta): current_task = get_current_job() with Office(app.config["LIBREOFFICE_PATH"]) as office: # acquire libreoffice lock with office.documentLoad(path) as original_document: # open original document with TemporaryDirectory() as tmp_dir: # create temp dir where output'll be stored for fmt in options["formats"]: # iterate over requested formats current_format = app.config["SUPPORTED_FORMATS"][fmt] output_path = os.path.join(tmp_dir, current_format["path"]) original_document.saveAs(output_path, fmt=current_format["fmt"]) if options.get("thumbnails", None): is_created = False if meta["mimetype"] == "application/pdf": pdf_path = path elif "pdf" in options["formats"]: pdf_path = os.path.join(tmp_dir, "pdf") else: pdf_tmp_file = NamedTemporaryFile() pdf_path = pdf_tmp_file.name original_document.saveAs(pdf_tmp_file.name, fmt="pdf") is_created = True image = Image(filename=pdf_path, resolution=app.config["THUMBNAILS_DPI"]) if is_created: pdf_tmp_file.close() thumbnails = make_thumbnails(image, tmp_dir, options["thumbnails"]["size"]) result_path, result_url = make_zip_archive(current_task.id, tmp_dir) remove_file.schedule( datetime.timedelta(seconds=app.config["RESULT_FILE_TTL"]), result_path ) return result_url
Example #23
Source File: jobs.py From gigantum-client with MIT License | 4 votes |
def update_unmanaged_dataset_from_remote(logged_in_username: str, access_token: str, id_token: str, dataset_owner: str, dataset_name: str) -> None: """Method to update/populate an unmanaged dataset from its remote automatically Args: logged_in_username: username for the currently logged in user access_token: bearer token id_token: identity token dataset_owner: Owner of the dataset containing the files to download dataset_name: Name of the dataset containing the files to download Returns: """ def update_meta(msg): job = get_current_job() if not job: return if 'feedback' not in job.meta: job.meta['feedback'] = msg else: job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}' job.save_meta() logger = LMLogger.get_logger() try: p = os.getpid() logger.info(f"(Job {p}) Starting update_unmanaged_dataset_from_remote(logged_in_username={logged_in_username}," f"dataset_owner={dataset_owner}, dataset_name={dataset_name}") im = InventoryManager() ds = im.load_dataset(logged_in_username, dataset_owner, dataset_name) ds.namespace = dataset_owner ds.backend.set_default_configuration(logged_in_username, access_token, id_token) if not isinstance(ds.backend, UnmanagedStorageBackend): raise ValueError("Can only auto-update unmanaged dataset types") if not ds.backend.can_update_from_remote: raise ValueError("Storage backend cannot update automatically from remote.") ds.backend.update_from_remote(ds, update_meta) except Exception as err: logger.exception(err) raise
Example #24
Source File: jobs.py From gigantum-client with MIT License | 4 votes |
def verify_dataset_contents(logged_in_username: str, access_token: str, id_token: str, dataset_owner: str, dataset_name: str, labbook_owner: Optional[str] = None, labbook_name: Optional[str] = None) -> None: """Method to update/populate an unmanaged dataset from it local state Args: logged_in_username: username for the currently logged in user access_token: bearer token id_token: identity token dataset_owner: Owner of the dataset containing the files to download dataset_name: Name of the dataset containing the files to download labbook_owner: Owner of the labbook if this dataset is linked labbook_name: Name of the labbook if this dataset is linked Returns: None """ job = get_current_job() def update_meta(msg): if not job: return if 'feedback' not in job.meta: job.meta['feedback'] = msg else: job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}' job.save_meta() logger = LMLogger.get_logger() try: p = os.getpid() logger.info(f"(Job {p}) Starting verify_dataset_contents(logged_in_username={logged_in_username}," f"dataset_owner={dataset_owner}, dataset_name={dataset_name}," f"labbook_owner={labbook_owner}, labbook_name={labbook_name}") im = InventoryManager() if labbook_owner is not None and labbook_name is not None: # This is a linked dataset, load repo from the Project lb = im.load_labbook(logged_in_username, labbook_owner, labbook_name) dataset_dir = os.path.join(lb.root_dir, '.gigantum', 'datasets', dataset_owner, dataset_name) ds = im.load_dataset_from_directory(dataset_dir) else: # this is a normal dataset. Load repo from working dir ds = im.load_dataset(logged_in_username, dataset_owner, dataset_name) ds.namespace = dataset_owner ds.backend.set_default_configuration(logged_in_username, access_token, id_token) result = ds.backend.verify_contents(ds, update_meta) job.meta['modified_keys'] = result except Exception as err: logger.exception(err) raise
Example #25
Source File: jobs.py From gigantum-client with MIT License | 4 votes |
def update_unmanaged_dataset_from_local(logged_in_username: str, access_token: str, id_token: str, dataset_owner: str, dataset_name: str) -> None: """Method to update/populate an unmanaged dataset from it local state Args: logged_in_username: username for the currently logged in user access_token: bearer token id_token: identity token dataset_owner: Owner of the dataset containing the files to download dataset_name: Name of the dataset containing the files to download Returns: """ def update_meta(msg): job = get_current_job() if not job: return if 'feedback' not in job.meta: job.meta['feedback'] = msg else: job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}' job.save_meta() logger = LMLogger.get_logger() try: p = os.getpid() logger.info(f"(Job {p}) Starting update_unmanaged_dataset_from_local(logged_in_username={logged_in_username}," f"dataset_owner={dataset_owner}, dataset_name={dataset_name}") im = InventoryManager() ds = im.load_dataset(logged_in_username, dataset_owner, dataset_name) ds.namespace = dataset_owner ds.backend.set_default_configuration(logged_in_username, access_token, id_token) if not isinstance(ds.backend, UnmanagedStorageBackend): raise ValueError("Can only auto-update unmanaged dataset types") ds.backend.update_from_local(ds, update_meta, verify_contents=True) except Exception as err: logger.exception(err) raise
Example #26
Source File: jobs.py From ckanext-xloader with GNU Affero General Public License v3.0 | 4 votes |
def xloader_data_into_datastore(input): '''This is the func that is queued. It is a wrapper for xloader_data_into_datastore_, and makes sure it finishes by calling xloader_hook to update the task_status with the result. Errors are stored in task_status and job log and this method returns 'error' to let RQ know too. Should task_status fails, then we also return 'error'. ''' # First flag that this task is running, to indicate the job is not # stillborn, for when xloader_submit is deciding whether another job would # be a duplicate or not job_dict = dict(metadata=input['metadata'], status='running') callback_xloader_hook(result_url=input['result_url'], api_key=input['api_key'], job_dict=job_dict) job_id = get_current_job().id errored = False try: xloader_data_into_datastore_(input, job_dict) job_dict['status'] = 'complete' db.mark_job_as_completed(job_id, job_dict) except JobError as e: db.mark_job_as_errored(job_id, str(e)) job_dict['status'] = 'error' job_dict['error'] = str(e) log = logging.getLogger(__name__) log.error('xloader error: {0}, {1}'.format(e, traceback.format_exc())) errored = True except Exception as e: db.mark_job_as_errored( job_id, traceback.format_tb(sys.exc_traceback)[-1] + repr(e)) job_dict['status'] = 'error' job_dict['error'] = str(e) log = logging.getLogger(__name__) log.error('xloader error: {0}, {1}'.format(e, traceback.format_exc())) errored = True finally: # job_dict is defined in xloader_hook's docstring is_saved_ok = callback_xloader_hook(result_url=input['result_url'], api_key=input['api_key'], job_dict=job_dict) errored = errored or not is_saved_ok return 'error' if errored else None
Example #27
Source File: jobs.py From grimoirelab-kingarthur with GNU General Public License v3.0 | 4 votes |
def execute_perceval_job(backend, backend_args, qitems, task_id, job_number, category, archive_args=None): """Execute a Perceval job on RQ. The items fetched during the process will be stored in a Redis queue named `queue`. Setting the parameter `archive_path`, raw data will be stored with the archive manager. The contents from the archive can be retrieved setting the parameter `fetch_from_archive` to `True`, too. Take into account this behaviour will be only available when the backend supports the use of the archive. If archiving is not supported, an `AttributeError` exception will be raised. :param backend: backend to execute :param backend_args: dict of arguments for running the backend :param qitems: name of the RQ queue used to store the items :param task_id: identifier of the task linked to this job :param job_number: human readable identifier for this job :param category: category of the items to retrieve :param archive_args: archive arguments :returns: a `JobResult` instance :raises NotFoundError: raised when the backend is not found :raises AttributeError: raised when archiving is not supported but any of the archive parameters were set """ rq_job = rq.get_current_job() job = PercevalJob(rq_job.id, job_number, task_id, backend, category, rq_job.connection, qitems) logger.debug("Running job #%s (task: %s) (%s) (cat:%s)", job.job_id, task_id, backend, category) if not job.has_archiving() and archive_args: raise AttributeError("archive attributes set but archive is not supported") try: job.run(backend_args, archive_args=archive_args) except AttributeError as e: raise e except Exception as e: rq_job = rq.get_current_job() rq_job.meta['result'] = job.result rq_job.save_meta() logger.debug("Error running job %s (%s) - %s", job.job_id, backend, str(e)) raise e result = job.result logger.debug("Job #%s (task: %s) completed (%s) - %s/%s items (%s) fetched", result.job_id, task_id, result.backend, str(result.summary.fetched), str(result.summary.skipped), result.category) return result
Example #28
Source File: rqtasks.py From xcessiv with Apache License 2.0 | 4 votes |
def start_automated_run(path, automated_run_id): """Starts automated run. This will automatically create base learners until the run finishes or errors out. Args: path (str): Path to Xcessiv notebook automated_run_id (str): Automated Run ID """ with functions.DBContextManager(path) as session: automated_run = session.query(models.AutomatedRun).filter_by(id=automated_run_id).first() if not automated_run: raise exceptions.UserError('Automated run {} ' 'does not exist'.format(automated_run_id)) automated_run.job_id = get_current_job().id automated_run.job_status = 'started' session.add(automated_run) session.commit() try: if automated_run.category == 'bayes': automatedruns.start_naive_bayes(automated_run, session, path) elif automated_run.category == 'tpot': automatedruns.start_tpot(automated_run, session, path) elif automated_run.category == 'greedy_ensemble_search': automatedruns.start_greedy_ensemble_search(automated_run, session, path) else: raise Exception('Something went wrong. Invalid category for automated run') automated_run.job_status = 'finished' session.add(automated_run) session.commit() except: session.rollback() automated_run.job_status = 'errored' automated_run.description['error_type'] = repr(sys.exc_info()[0]) automated_run.description['error_value'] = repr(sys.exc_info()[1]) automated_run.description['error_traceback'] = \ traceback.format_exception(*sys.exc_info()) session.add(automated_run) session.commit() raise