Python airflow.operators.python_operator.PythonOperator() Examples

def get_dated_main_runner_operator(
        dag,
        main_function,
        execution_timeout,
        day_shift=0,
        task_id='pull_image_data',
):
    args_str = f'{{{{ macros.ds_add(ds, -{day_shift}) }}}}'
    return PythonOperator(
        task_id=task_id,
        python_callable=main_function,
        op_args=[args_str],
        execution_timeout=execution_timeout,
        depends_on_past=False,
        dag=dag
    ) 

def get_action_xcom(self, task_id=dn.ACTION_XCOM):
        """Generate the action_xcom step

        Step responsible for getting the action information passed
        by the invocation of the dag, which includes any options.
        """
        def xcom_push(**kwargs):
            """xcom_push function

            Defines a push function to store the content of 'action' that is
            defined via 'dag_run' in XCOM so that it can be used by the
            Operators. Includes action-related information for later steps.
            """

            kwargs['ti'].xcom_push(key='action',
                                   value=kwargs['dag_run'].conf['action'])
            kwargs['ti'].xcom_push(key='action_type',
                                   value=self.action_type)

        return PythonOperator(task_id=task_id,
                              dag=self.dag,
                              python_callable=xcom_push) 

def get_file_deletion_operator(
        dag,
        output_dir,
        identifier=TIMESTAMP_TEMPLATE
):
    return PythonOperator(
        task_id='delete_staged_file',
        python_callable=paths.delete_staged_file,
        op_args=[output_dir, identifier],
        trigger_rule=TriggerRule.ALL_SUCCESS,
        dag=dag,
    ) 

def test_make_python_operator_missing_param():
    td = dagbuilder.DagBuilder("test_dag", DAG_CONFIG, DEFAULT_CONFIG)
    operator = "airflow.operators.python_operator.PythonOperator"
    task_params = {"task_id": "test_task", "python_callable_name": "print_test"}
    with pytest.raises(Exception):
        td.make_task(operator, task_params) 

def test_make_python_operator():
    td = dagbuilder.DagBuilder("test_dag", DAG_CONFIG, DEFAULT_CONFIG)
    operator = "airflow.operators.python_operator.PythonOperator"
    task_params = {
        "task_id": "test_task",
        "python_callable_name": "print_test",
        "python_callable_file": os.path.realpath(__file__),
    }
    actual = td.make_task(operator, task_params)
    assert actual.task_id == "test_task"
    assert callable(actual.python_callable)
    assert isinstance(actual, PythonOperator) 

def make_task(operator: str, task_params: Dict[str, Any]) -> BaseOperator:
        """
        Takes an operator and params and creates an instance of that operator.

        :returns: instance of operator object
        """
        try:
            # class is a Callable https://stackoverflow.com/a/34578836/3679900
            operator_obj: Callable[..., BaseOperator] = import_string(operator)
        except Exception as err:
            raise Exception(f"Failed to import operator: {operator}. err: {err}")
        try:
            if operator_obj == PythonOperator:
                if not task_params.get("python_callable_name") and not task_params.get(
                    "python_callable_file"
                ):
                    raise Exception(
                        "Failed to create task. PythonOperator requires `python_callable_name` \
                        and `python_callable_file` parameters."
                    )
                task_params["python_callable"]: Callable = utils.get_python_callable(
                    task_params["python_callable_name"],
                    task_params["python_callable_file"],
                )

            if utils.check_dict_key(task_params, "execution_timeout_secs"):
                task_params["execution_timeout"]: timedelta = timedelta(
                    seconds=task_params["execution_timeout_secs"]
                )
                del task_params["execution_timeout_secs"]

            task: BaseOperator = operator_obj(**task_params)
        except Exception as err:
            raise Exception(f"Failed to create {operator_obj} task. err: {err}")
        return task 

def python_task(self, function, instance):
    PythonOperator(
      task_id='%s_%d' % (function, instance),
      python_callable = getattr(import_module('starthinker.task.%s.run' % function), function),
      op_kwargs = {'recipe': self.recipe, 'instance':instance},
      dag=self.dag
    ) 

def get_runner_operator(dag):
    return PythonOperator(
        task_id='pull_phylopic_data',
        python_callable=phylopic.main,
        op_args=['{{ ds }}'],
        depends_on_past=False,
        dag=dag
    ) 

def get_main_runner_operator(dag, main_function):
    return PythonOperator(
        task_id='pull_image_data',
        python_callable=main_function,
        depends_on_past=False,
        dag=dag
    ) 

def get_flickr_sub_provider_update_operator(
        dag,
        postgres_conn_id,
):
    return PythonOperator(
        task_id='update_flickr_sub_providers',
        python_callable=sql.update_flickr_sub_providers,
        op_args=[postgres_conn_id],
        dag=dag
    ) 

def get_failure_moving_operator(
        dag,
        output_dir,
        identifier=TIMESTAMP_TEMPLATE
):
    return PythonOperator(
        task_id='move_staged_failures',
        python_callable=paths.move_staged_files_to_failure_directory,
        op_args=[output_dir, identifier],
        trigger_rule=TriggerRule.ONE_SUCCESS,
        dag=dag
    ) 

def generate_parallel_tasks(name_prefix, num_of_tasks, deps):
    """
    Generate a list of PythonOperator tasks. The generated tasks are set up to
    be dependent on the `deps` argument.
    """
    tasks = []
    for t_id in range(num_of_tasks):
        run_this = PythonOperator(
            task_id=f"{name_prefix}_{t_id}",
            python_callable=print_context,
        )
        run_this << deps
        tasks.append(run_this)
    return tasks 

def get_load_s3_data_operator(
        dag,
        bucket,
        aws_conn_id,
        postgres_conn_id,
        identifier=TIMESTAMP_TEMPLATE
):
    return PythonOperator(
        task_id='load_s3_data',
        python_callable=loader.load_s3_data,
        op_args=[bucket, aws_conn_id, postgres_conn_id, identifier],
        dag=dag
    ) 

def get_copy_to_s3_operator(
        dag,
        output_dir,
        storage_bucket,
        aws_conn_id,
        identifier=TIMESTAMP_TEMPLATE
):
    return PythonOperator(
        task_id='copy_to_s3',
        python_callable=loader.copy_to_s3,
        op_args=[output_dir, storage_bucket, identifier, aws_conn_id],
        dag=dag
    ) 

def get_load_local_data_operator(
        dag,
        output_dir,
        postgres_conn_id,
        identifier=TIMESTAMP_TEMPLATE
):
    return PythonOperator(
        task_id='load_local_data',
        python_callable=loader.load_local_data,
        op_args=[output_dir, postgres_conn_id, identifier],
        trigger_rule=TriggerRule.ALL_SUCCESS,
        dag=dag
    ) 

def get_table_creator_operator(
        dag,
        postgres_conn_id,
        identifier=TIMESTAMP_TEMPLATE
):
    return PythonOperator(
        task_id='create_loading_table',
        python_callable=sql.create_loading_table,
        op_args=[postgres_conn_id, identifier],
        dag=dag
    ) 

def get_runner_operator(dag):
    return PythonOperator(
        task_id="pull_cleveland_data",
        python_callable=cleveland_museum_of_art.main,
        depends_on_past=False,
        dag=dag
    ) 

def get_runner_operator(dag):
    return PythonOperator(
        task_id='pull_europeana_data',
        python_callable=europeana.main,
        op_args=['{{ ds }}'],
        depends_on_past=False,
        dag=dag
    ) 

def get_runner_operator(dag):
    return PythonOperator(
        task_id='pull_metropolitan_museum_data',
        python_callable=metropolitan_museum_of_art.main,
        op_args=['{{ ds }}'],
        depends_on_past=False,
        dag=dag
    ) 

def get_runner_operator(dag):
    return PythonOperator(
        task_id="pull_rawpixel_data",
        python_callable=raw_pixel.main,
        depends_on_past=False,
        dag=dag
    ) 

def get_runner_operator(dag):
    return PythonOperator(
        task_id='pull_wikimedia_commons_data',
        python_callable=wikimedia_commons.main,
        op_args=['{{ ds }}'],
        depends_on_past=False,
        dag=dag
    )