Python mlflow.log_artifacts() Examples

def log_project_artifacts_to_mlflow(function: Callable):
    """
    Log the artifact to mlflow

    Parameters
    ----------
    function
        function to wrap
    """

    @wraps(function)
    def wrapped(*args, **kwargs):
        if mlflow.active_run() is None:
            _warn_about_no_run()
            return function(*args, **kwargs)
        artifacts_path = project.get_active_artifacts_directory()
        artifacts_path_realpath = os.path.realpath(artifacts_path)
        mlflow.log_artifacts(artifacts_path_realpath)
        return function(*args, **kwargs)

    return wrapped


# pylint: disable=invalid-name
# this is method, not a constant, and is used inside of the patch 

def etl_data(ratings_csv, max_row_limit):
    with mlflow.start_run() as mlrun:
        tmpdir = tempfile.mkdtemp()
        ratings_parquet_dir = os.path.join(tmpdir, 'ratings-parquet')
        spark = pyspark.sql.SparkSession.builder.getOrCreate()
        print("Converting ratings CSV %s to Parquet %s" % (ratings_csv, ratings_parquet_dir))
        ratings_df = spark.read \
            .option("header", "true") \
            .option("inferSchema", "true") \
            .csv(ratings_csv) \
            .drop("timestamp")  # Drop unused column
        ratings_df.show()
        if max_row_limit != -1:
            ratings_df = ratings_df.limit(max_row_limit)
        ratings_df.write.parquet(ratings_parquet_dir)
        print("Uploading Parquet ratings: %s" % ratings_parquet_dir)
        mlflow.log_artifacts(ratings_parquet_dir, "ratings-parquet-dir") 

def log_artifacts(local_dir: str, artifact_path: str = None):
    mlflow.log_artifacts(local_dir, artifact_path) 

def log_artifacts(cls, self):
        raise NotImplementedError() 

def log_artifacts(cls, dir_path, artifact_path=None):
        raise NotImplementedError 

def log_artifacts(cls, dir_path, artifact_path=None):
        try:
            mlflow.log_artifacts(dir_path, artifact_path)
        except ConnectionError:
            logger.warning(f"ConnectionError in logging artifacts to MLFlow")
        except Exception as e:
            logger.warning(f"Failed to log artifacts: {e}") 

def log_artifacts(local_dir, artifact_path=None):
        return None 

def log_artifacts(local_dir, artifact_path=None):
        def send(dpath, e, path):
            mlflow.log_artifacts(dpath, artifact_path=path)
            e.set()

        event = threading.Event()
        t = threading.Thread(
            target=send, args=(local_dir, event, artifact_path), daemon=True
        )
        t.start()
        return event 

def _log_artifacts_with_warning(**kwargs):
    try_mlflow_log(mlflow.log_artifacts, **kwargs) 

def test_log_artifact():
    artifact_src_dir = tempfile.mkdtemp()
    # Create artifacts
    _, path0 = tempfile.mkstemp(dir=artifact_src_dir)
    _, path1 = tempfile.mkstemp(dir=artifact_src_dir)
    for i, path in enumerate([path0, path1]):
        with open(path, "w") as handle:
            handle.write("%s" % str(i))
    # Log an artifact, verify it exists in the directory returned by get_artifact_uri
    # after the run finishes
    artifact_parent_dirs = ["some_parent_dir", None]
    for parent_dir in artifact_parent_dirs:
        with start_run():
            artifact_uri = mlflow.get_artifact_uri()
            run_artifact_dir = local_file_uri_to_path(artifact_uri)
            mlflow.log_artifact(path0, parent_dir)
        expected_dir = os.path.join(run_artifact_dir, parent_dir) \
            if parent_dir is not None else run_artifact_dir
        assert os.listdir(expected_dir) == [os.path.basename(path0)]
        logged_artifact_path = os.path.join(expected_dir, path0)
        assert filecmp.cmp(logged_artifact_path, path0, shallow=False)
    # Log multiple artifacts, verify they exist in the directory returned by get_artifact_uri
    for parent_dir in artifact_parent_dirs:
        with start_run():
            artifact_uri = mlflow.get_artifact_uri()
            run_artifact_dir = local_file_uri_to_path(artifact_uri)

            mlflow.log_artifacts(artifact_src_dir, parent_dir)
        # Check that the logged artifacts match
        expected_artifact_output_dir = os.path.join(run_artifact_dir, parent_dir) \
            if parent_dir is not None else run_artifact_dir
        dir_comparison = filecmp.dircmp(artifact_src_dir, expected_artifact_output_dir)
        assert len(dir_comparison.left_only) == 0
        assert len(dir_comparison.right_only) == 0
        assert len(dir_comparison.diff_files) == 0
        assert len(dir_comparison.funny_files) == 0