Python joblib.Memory() Examples

def test_memory_2():
    """Assert that the TPOT _setup_memory function runs normally with a valid path."""
    cachedir = mkdtemp()
    tpot_obj = TPOTClassifier(
        random_state=42,
        population_size=1,
        offspring_size=2,
        generations=1,
        config_dict='TPOT light',
        memory=cachedir,
        verbosity=0
    )
    tpot_obj._setup_memory()
    rmtree(cachedir)

    assert tpot_obj._cachedir == cachedir
    assert isinstance(tpot_obj._memory, Memory) 

def test_memory_3():
    """Assert that the TPOT fit function does not clean up caching directory when memory is a valid path."""
    cachedir = mkdtemp()
    tpot_obj = TPOTClassifier(
        random_state=42,
        population_size=1,
        offspring_size=2,
        generations=1,
        config_dict='TPOT light',
        memory=cachedir,
        verbosity=0
    )
    tpot_obj.fit(training_features, training_target)

    assert tpot_obj._cachedir == cachedir
    assert os.path.isdir(tpot_obj._cachedir)
    assert isinstance(tpot_obj._memory, Memory)
    # clean up
    rmtree(cachedir)
    tpot_obj._memory = None 

def test_memory_5():
    """Assert that the TPOT _setup_memory function runs normally with a Memory object."""
    cachedir = mkdtemp()
    memory = Memory(location=cachedir, verbose=0)
    tpot_obj = TPOTClassifier(
        random_state=42,
        population_size=1,
        offspring_size=2,
        generations=1,
        config_dict='TPOT light',
        memory=memory,
        verbosity=0
    )

    tpot_obj._setup_memory()
    rmtree(cachedir)
    assert tpot_obj.memory == memory
    assert tpot_obj._memory == memory
    # clean up
    tpot_obj._memory = None
    memory = None 

def _set_memory(self, cache_dir):
        """Create the joblib Memory instance."""

        # Try importing joblib.
        try:
            from joblib import Memory
            self._memory = Memory(
                location=self.cache_dir, mmap_mode=None, verbose=self.verbose,
                bytes_limit=self.cache_limit)
            logger.debug("Initialize joblib cache dir at `%s`.", self.cache_dir)
            logger.debug("Reducing the size of the cache if needed.")
            self._memory.reduce_size()
        except ImportError:  # pragma: no cover
            logger.warning(
                "Joblib is not installed. Install it with `conda install joblib`.")
            self._memory = None 

def set_cachedir(cachedir=None, verbose=0):
    """Set root directory for the joblib cache.

    :Parameters:
     cachedir
         the cache directory name; if ``None``, a temporary directory
         is created using `TemporaryDirectory`
     verbose
         an integer number, controls the verbosity of the cache
         (default is 0, i.e., not verbose)
    """

    global _cachedir
    global _cacheobj
    global _cached_methods
    global _memory

    if cachedir is None:
        _cacheobj = TemporaryDirectory(prefix='mdp-joblib-cache.')
        cachedir = _cacheobj.name

    # only reset if the directory changes
    if cachedir != _cachedir:
        _cachedir = cachedir
        _memory = joblib.Memory(cachedir, verbose=verbose)
        # reset cached methods
        _cached_methods.clear()

# initialize cache with temporary directory
#set_cachedir() 

def _setup_memory(self):
        """Setup Memory object for memory caching.
        """
        if self.memory:
            if isinstance(self.memory, str):
                if self.memory == "auto":
                    # Create a temporary folder to store the transformers of the pipeline
                    self._cachedir = mkdtemp()
                else:
                    if not os.path.isdir(self.memory):
                        try:
                            os.makedirs(self.memory)
                        except:
                            raise ValueError(
                                'Could not create directory for memory caching: {}'.format(self.memory)
                            )
                    self._cachedir = self.memory

                self._memory = Memory(location=self._cachedir, verbose=0)
            elif isinstance(self.memory, Memory):
                self._memory = self.memory
            else:
                raise ValueError(
                    'Could not recognize Memory object for pipeline caching. '
                    'Please provide an instance of joblib.Memory,'
                    ' a path to a directory on your system, or \"auto\".'
                ) 

def __setstate__(self, state):
        """Make sure that this class is picklable."""
        self.__dict__ = state
        # Recreate the joblib Memory instance.
        self._set_memory(state['cache_dir']) 

def cached(*args, **kargs):
    import joblib as jb
    from .. import CACHE
    memo = getattr(cached, 'memo', None)
    if memo is None:
        cached.memo = memo = jb.Memory(CACHE, verbose=0)
    return memo.cache(*args, **kargs) 

def init_data(data, client_options=None, plugin=None, cache_waveforms=False,
              get_waveforms=None):
    """Return appropriate get_waveforms function

    See example configuration file for a description of the options"""
    client_module = None
    if get_waveforms is None:
        if client_options is None:
            client_options = {}
        try:
            client_module = import_module('obspy.clients.%s' % data)
        except ImportError:
            pass
        if client_module:
            Client = getattr(client_module, 'Client')
            client = Client(**client_options)

            def get_waveforms(event=None, **args):
                return client.get_waveforms(**args)
        elif data == 'plugin':
            get_waveforms = _load_func(plugin)
        else:
            from obspy import read
            stream = read(data)

            def get_waveforms(network, station, location, channel,
                              starttime, endtime, event=None):
                st = stream.select(network=network, station=station,
                                   location=location, channel=channel)
                st = st.slice(starttime, endtime)
                return st

    def wrapper(**kwargs):
        try:
            return get_waveforms(**kwargs)
        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception as ex:
            seedid = '.'.join((kwargs['network'], kwargs['station'],
                               kwargs['location'], kwargs['channel']))
            msg = 'channel %s: error while retrieving data: %s'
            log.debug(msg, seedid, ex)

    use_cache = client_module is not None or data == 'plugin'
    use_cache = use_cache and cache_waveforms
    if use_cache:
        try:
            import joblib
        except ImportError:
            log.warning('install joblib to use cache_waveforms option')
        else:
            log.info('use waveform cache in %s', cache_waveforms)
            memory = joblib.Memory(cachedir=cache_waveforms, verbose=0)
            return memory.cache(wrapper)
    elif use_cache:
        log.warning('install joblib to use cache_waveforms option')
    return wrapper 

def check_constructor(Estimator):
    # Check that the constructor behaves correctly
    estimator = _construct_instance(Estimator)

    # Check that init does not construct object of other class than itself
    assert isinstance(estimator, Estimator)

    # Ensure that each parameter is set in init
    init_params = _get_args(type(estimator).__init__)
    invalid_attr = set(init_params) - set(vars(estimator)) - {"self"}
    assert not invalid_attr, (
            "Estimator %s should store all parameters"
            " as an attribute during init. Did not find "
            "attributes `%s`."
            % (estimator.__class__.__name__, sorted(invalid_attr)))

    # Ensure that init does nothing but set parameters
    # No logic/interaction with other parameters
    def param_filter(p):
        """Identify hyper parameters of an estimator"""
        return (p.name != 'self' and
                p.kind != p.VAR_KEYWORD and
                p.kind != p.VAR_POSITIONAL)

    init_params = [p for p in signature(estimator.__init__).parameters.values()
                   if param_filter(p)]

    params = estimator.get_params()

    # Filter out required parameters with no default value and parameters
    # set for running tests
    required_params = getattr(estimator, '_required_parameters', [])
    test_config_params = ESTIMATOR_TEST_PARAMS.get(Estimator, {}).keys()

    init_params = [param for param in init_params if
                   param.name not in required_params and
                   param.name not in test_config_params]

    for param in init_params:
        assert param.default != param.empty, (
                "parameter `%s` for %s has no default value and is not "
                "included in `_required_parameters`"
                % (param.name, estimator.__class__.__name__))
        if type(param.default) is type:
            assert param.default in [np.float64, np.int64]
        else:
            assert (type(param.default) in
                    [str, int, float, bool, tuple, type(None),
                     np.float64, types.FunctionType, joblib.Memory])

        param_value = params[param.name]
        if isinstance(param_value, np.ndarray):
            np.testing.assert_array_equal(param_value, param.default)
        else:
            if bool(isinstance(param_value, numbers.Real) and np.isnan(
                    param_value)):
                # Allows to set default parameters to np.nan
                assert param_value is param.default, param.name
            else:
                assert param_value == param.default, param.name