Python joblib.effective_n_jobs() Examples

The following are 5 code examples of joblib.effective_n_jobs(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module joblib , or try the search function .
Example #1
Source File: SOMClustering.py    From susi with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
def _partition_bmus(self, X):
        """Private function used to partition bmus between jobs.

        Parameters
        ----------
        X : np.array
            List of datapoints

        Returns
        -------
        n_jobs : int
            Number of jobs
        list of int
            List of number of datapoints per job
        list of int
            List of start values for every job list

        """
        n_datapoints = len(X)
        n_jobs = min(effective_n_jobs(self.n_jobs), n_datapoints)

        n_datapoints_per_job = np.full(
            n_jobs, n_datapoints // n_jobs, dtype=np.int)

        n_datapoints_per_job[:n_datapoints % n_jobs] += 1
        starts = np.cumsum(n_datapoints_per_job)

        return n_jobs, n_datapoints_per_job.tolist(), [0] + starts.tolist()
Example #2
Source File: threaded.py    From pynndescent with BSD 2-Clause "Simplified" License 5 votes vote down vote up 
def effective_n_jobs_with_context(n_jobs=None):
    """Find the effective number of jobs, either specified directly, or from the joblib.parallel_backend context."""
    if n_jobs is None:
        _, n_jobs_from_context = joblib.parallel.get_active_backend()
        n_jobs = n_jobs_from_context
    return joblib.effective_n_jobs(n_jobs)
Example #3
Source File: demo_balanced_scheduling.py    From SUOD with BSD 2-Clause "Simplified" License 5 votes vote down vote up 
def _partition_estimators(n_estimators, n_jobs):
    """Private function used to partition estimators between jobs."""
    # Compute the number of jobs
    n_jobs = min(effective_n_jobs(n_jobs), n_estimators)

    # Partition estimators between jobs
    n_estimators_per_job = np.full(n_jobs, n_estimators // n_jobs,
                                   dtype=np.int)
    n_estimators_per_job[:n_estimators % n_jobs] += 1
    starts = np.cumsum(n_estimators_per_job)

    xdiff = [starts[n] - starts[n - 1] for n in range(1, len(starts))]

    print("Split among workers default:", starts, xdiff)
    return n_estimators_per_job.tolist(), [0] + starts.tolist(), n_jobs
Example #4
Source File: parallel_processes.py    From SUOD with BSD 2-Clause "Simplified" License 5 votes vote down vote up 
def _partition_estimators(n_estimators, n_jobs):
    """Private function used to partition estimators between jobs."""
    # Compute the number of jobs
    n_jobs = min(effective_n_jobs(n_jobs), n_estimators)

    # Partition estimators between jobs
    n_estimators_per_job = np.full(n_jobs, n_estimators // n_jobs,
                                   dtype=np.int)
    n_estimators_per_job[:n_estimators % n_jobs] += 1
    starts = np.cumsum(n_estimators_per_job)

    xdiff = [starts[n] - starts[n - 1] for n in range(1, len(starts))]

    print("Split among workers default:", starts, xdiff)
    return n_estimators_per_job.tolist(), [0] + starts.tolist(), n_jobs
Example #5
Source File: __init__.py    From sktime with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
def maybe_cythonize_extensions(top_path, config):
    """Tweaks for building extensions between release and development mode."""
    with_openmp = check_openmp_support()

    is_release = os.path.exists(os.path.join(top_path, 'PKG-INFO'))

    if is_release:
        build_from_c_and_cpp_files(config.ext_modules)
    else:
        message = ('Please install Cython with a version >= {0} in order '
                   'to build a sktime development version.').format(
            CYTHON_MIN_VERSION)
        try:
            import Cython
            if LooseVersion(Cython.__version__) < CYTHON_MIN_VERSION:
                message += ' Your version of Cython was {0}.'.format(
                    Cython.__version__)
                raise ValueError(message)
            from Cython.Build import cythonize
        except ImportError as exc:
            exc.args += (message,)
            raise

        n_jobs = 1
        with contextlib.suppress(ImportError):
            import joblib
            if LooseVersion(joblib.__version__) > LooseVersion("0.13.0"):
                # earlier joblib versions don't account for CPU affinity
                # constraints, and may over-estimate the number of available
                # CPU particularly in CI (cf loky#114)
                n_jobs = joblib.effective_n_jobs()

        config.ext_modules = cythonize(
            config.ext_modules,
            nthreads=n_jobs,
            compile_time_env={'SKTIME_OPENMP_SUPPORTED': with_openmp},
            compiler_directives={'language_level': 3}
        )