Python dask.array.from_array() Examples

def test_permutation(permutation_params):
    n_features, n_instances, n_permutations, mult = permutation_params
    xshape, yshape = (n_instances[0], n_features), (n_instances[1], n_features)
    np.random.seed(0)
    x = np.random.random(xshape).astype('float32')
    y = np.random.random(yshape).astype('float32') * mult
    xda = da.from_array(x, chunks=xshape)
    yda = da.from_array(y, chunks=yshape)

    kwargs = {'sigma': np.array([1.])}
    p_val = permutation_test(x, y, n_permutations=n_permutations,
                             metric=maximum_mean_discrepancy, **kwargs)
    p_val_da = permutation_test(xda, yda, n_permutations=n_permutations,
                                metric=maximum_mean_discrepancy, **kwargs)

    if mult == 1:
        assert p_val > .2 and p_val_da > .2
    elif mult > 1:
        assert p_val <= .2 and p_val_da <= .2 

def test_intensity_peaks_dask(self):
        numpy_array = np.zeros((10, 10, 50, 50))
        numpy_array[:, :, 27, 27] = 1

        peak_array = np.zeros(
            (numpy_array.shape[0], numpy_array.shape[1]), dtype=np.object
        )
        for index in np.ndindex(numpy_array.shape[:-2]):
            islice = np.s_[index]
            peak_array[islice] = np.asarray([(27, 27)])

        dask_array = da.from_array(numpy_array, chunks=(5, 5, 5, 5))
        dask_peak_array = da.from_array(peak_array, chunks=(5, 5))

        disk_r = 2
        intensity_array = dt._intensity_peaks_image(dask_array, dask_peak_array, disk_r)
        intensity_array_computed = intensity_array.compute()
        assert intensity_array_computed.shape == peak_array.shape 

def _get_dead_pixel_test_data_3d():
    """Get artifical 3D dataset with dead pixels.

    Values are 50, except [:, 14, 42] and [:, 2, 12]
    being 0 (to represent a "dead pixel").

    Examples
    --------
    >>> import pyxem.dummy_data.dask_test_data as dtd
    >>> data = dtd._get_dead_pixel_test_data_3d()

    """
    data = np.ones((5, 40, 50)) * 50
    data[:, 14, 42] = 0
    data[:, 2, 12] = 0
    dask_array = da.from_array(data, chunks=(5, 5, 5))
    return dask_array 

def _get_dead_pixel_test_data_2d():
    """Get artifical 2D dataset with dead pixels.

    Values are 50, except [14, 42] and [2, 12]
    being 0 (to represent a "dead pixel").

    Examples
    --------
    >>> import pyxem.dummy_data.dask_test_data as dtd
    >>> data = dtd._get_dead_pixel_test_data_2d()

    """
    data = np.ones((40, 50)) * 50
    data[14, 42] = 0
    data[2, 12] = 0
    dask_array = da.from_array(data, chunks=(5, 5))
    return dask_array 

def _get_hot_pixel_test_data_4d():
    """Get artifical 4D dataset with hot pixels.

    Values are 50, except [4, 2, 21, 11] and [6, 1, 5, 38]
    being 50000 (to represent a "hot pixel").

    Examples
    --------
    >>> import pyxem.dummy_data.dask_test_data as dtd
    >>> data = dtd._get_hot_pixel_test_data_4d()

    """
    data = np.ones((10, 5, 40, 50)) * 50
    data[4, 2, 21, 11] = 50000
    data[6, 1, 5, 38] = 50000
    dask_array = da.from_array(data, chunks=(5, 5, 5, 5))
    return dask_array 

def _get_hot_pixel_test_data_3d():
    """Get artifical 3D dataset with hot pixels.

    Values are 50, except [2, 21, 11] and [1, 5, 38]
    being 50000 (to represent a "hot pixel").

    Examples
    --------
    >>> import pyxem.dummy_data.dask_test_data as dtd
    >>> data = dtd._get_hot_pixel_test_data_3d()

    """
    data = np.ones((5, 40, 50)) * 50
    data[2, 21, 11] = 50000
    data[1, 5, 38] = 50000
    dask_array = da.from_array(data, chunks=(5, 5, 5))
    return dask_array 

def _get_hot_pixel_test_data_2d():
    """Get artifical 2D dataset with hot pixels.

    Values are 50, except [21, 11] and [5, 38]
    being 50000 (to represent a "hot pixel").

    Examples
    --------
    >>> import pyxem.dummy_data.dask_test_data as dtd
    >>> data = dtd._get_hot_pixel_test_data_2d()

    """
    data = np.ones((40, 50)) * 50
    data[21, 11] = 50000
    data[5, 38] = 50000
    dask_array = da.from_array(data, chunks=(5, 5))
    return dask_array 

def test_intensity_peaks_chunk(self):
        numpy_array = np.zeros((2, 2, 50, 50))
        numpy_array[:, :, 27, 27] = 1

        peak_array = np.zeros(
            (numpy_array.shape[0], numpy_array.shape[1]), dtype=np.object
        )
        for index in np.ndindex(numpy_array.shape[:-2]):
            islice = np.s_[index]
            peak_array[islice] = np.asarray([(27, 27)])

        dask_array = da.from_array(numpy_array, chunks=(1, 1, 25, 25))
        peak_array_dask = da.from_array(peak_array, chunks=(1, 1))
        disk_r = 2
        intensity_array = dt._intensity_peaks_image_chunk(
            dask_array, peak_array_dask, disk_r
        )

        assert intensity_array.shape == peak_array_dask.shape 

def test_mmd(mmd_params):
    n_features, n_instances = mmd_params
    xshape, yshape = (n_instances[0], n_features), (n_instances[1], n_features)
    np.random.seed(0)
    x = np.random.random(xshape).astype('float32')
    y = np.random.random(yshape).astype('float32')
    xda = da.from_array(x, chunks=xshape)
    yda = da.from_array(y, chunks=yshape)

    kwargs = {'sigma': np.array([1.])}
    mmd_xx = maximum_mean_discrepancy(x, x, **kwargs)
    mmd_xy = maximum_mean_discrepancy(x, y, **kwargs)
    mmd_xx_da = maximum_mean_discrepancy(xda, xda, **kwargs).compute()
    mmd_xy_da = maximum_mean_discrepancy(xda, yda, **kwargs).compute()

    assert mmd_xx == mmd_xx_da and mmd_xy == mmd_xy_da
    assert mmd_xy > mmd_xx 

def test_pairwise(pairwise_params):
    n_features, n_instances = pairwise_params
    xshape, yshape = (n_instances[0], n_features), (n_instances[1], n_features)
    np.random.seed(0)
    x = np.random.random(xshape).astype('float32')
    y = np.random.random(yshape).astype('float32')
    xda = da.from_array(x, chunks=xshape)
    yda = da.from_array(y, chunks=yshape)

    dist_xx = pairwise_distance(x, x)
    dist_xy = pairwise_distance(x, y)
    dist_xx_da = pairwise_distance(xda, xda).compute()
    dist_xy_da = pairwise_distance(xda, yda).compute()

    assert dist_xx.shape == dist_xx_da.shape == (xshape[0], xshape[0])
    assert dist_xy.shape == dist_xy_da.shape == n_instances
    assert (dist_xx == dist_xx_da).all() and (dist_xy == dist_xy_da).all()
    assert dist_xx.trace() == 0. 

def test_gaussian_kernel(gaussian_kernel_params):
    sigma, n_features, n_instances = gaussian_kernel_params
    xshape, yshape = (n_instances[0], n_features), (n_instances[1], n_features)
    x = np.random.random(xshape).astype('float32')
    y = np.random.random(yshape).astype('float32')
    xda = da.from_array(x, chunks=xshape)
    yda = da.from_array(y, chunks=yshape)

    gk_xy = gaussian_kernel(x, y, sigma=sigma)
    gk_xx = gaussian_kernel(x, x, sigma=sigma)

    gk_xy_da = gaussian_kernel(xda, yda, sigma=sigma).compute()
    gk_xx_da = gaussian_kernel(xda, xda, sigma=sigma).compute()

    assert gk_xy.shape == n_instances and gk_xx.shape == (xshape[0], xshape[0])
    assert (gk_xx == gk_xx_da).all() and (gk_xy == gk_xy_da).all()
    assert gk_xx.trace() == xshape[0] * len(sigma)
    assert (gk_xx > 0.).all() and (gk_xy > 0.).all() 

def test_intensity_peaks_image_disk_r(self):
        numpy_array = np.zeros((50, 50))
        numpy_array[27, 29] = 2
        numpy_array[11, 15] = 1
        image = da.from_array(numpy_array, chunks=(50, 50))
        peak = np.array([[27, 29], [11, 15]], np.int32)
        peak_dask = da.from_array(peak, chunks=(1, 1))
        disk_r0 = 1
        disk_r1 = 2
        intensity0 = dt._intensity_peaks_image_single_frame(image, peak_dask, disk_r0)
        intensity1 = dt._intensity_peaks_image_single_frame(image, peak_dask, disk_r1)

        assert intensity0[0].all() == np.array([27.0, 29.0, 2 / 9]).all()
        assert intensity0[1].all() == np.array([11.0, 15.0, 1 / 9]).all()
        assert intensity1[0].all() == np.array([27.0, 29.0, 2 / 25]).all()
        assert intensity1[1].all() == np.array([11.0, 15.0, 1 / 25]).all()
        assert intensity0.shape == intensity1.shape == (2, 3) 

def test_correct_radius_simple(self):
        x, y, r, px, py = 40, 51, 30, 4, 5
        s = mdtd.generate_4d_data(
            probe_size_x=px,
            probe_size_y=py,
            image_size_x=120,
            image_size_y=100,
            disk_I=0,
            ring_x=x,
            ring_y=y,
            ring_r=r,
            ring_I=5,
            blur=True,
            downscale=False,
        )
        dask_array = da.from_array(s.data, chunks=(4, 4, 50, 50))
        s = LazyDiffraction2D(dask_array)
        s.axes_manager.signal_axes[0].offset = -x
        s.axes_manager.signal_axes[1].offset = -y
        s_r = s.radial_average()
        assert s_r.axes_manager.navigation_shape == (px, py)
        assert (s_r.data.argmax(axis=-1) == 30).all() 

def ConstantArray(value, size, chunks=100000):
    """
    Return a dask array of the specified ``size`` holding a single value.

    This uses numpy's "stride tricks" to avoid replicating
    the data in memory for each element of the array.

    Parameters
    ----------
    value : float
        the scalar value to fill the array with
    size : int
        the length of the returned dask array
    chunks : int, optional
        the size of the dask array chunks
    """
    ele = numpy.array(value)
    toret = numpy.lib.stride_tricks.as_strided(ele, [size] + list(ele.shape), [0] + list(ele.strides))
    return da.from_array(toret, chunks=chunks, name=False) 

def test_correct_radius_random(self):
        x, y, px, py = 56, 48, 4, 5
        r = np.random.randint(20, 40, size=(py, px))
        s = mdtd.generate_4d_data(
            probe_size_x=px,
            probe_size_y=py,
            image_size_x=120,
            image_size_y=100,
            disk_I=0,
            ring_x=x,
            ring_y=y,
            ring_r=r,
            ring_I=5,
            blur=True,
            downscale=False,
        )
        dask_array = da.from_array(s.data, chunks=(4, 4, 50, 50))
        s = LazyDiffraction2D(dask_array)
        s.axes_manager.signal_axes[0].offset = -x
        s.axes_manager.signal_axes[1].offset = -y
        s_r = s.radial_average()
        assert (s_r.data.argmax(axis=-1) == r).all() 

def test_simple(self):
        array0 = da.ones(shape=(10, 10, 40, 40), chunks=(5, 5, 5, 5))
        s0 = LazyDiffraction2D(array0)
        s0_r = s0.radial_average()
        assert (s0_r.data[:, :, :-1] == 1).all()

        data_shape = 2, 2, 11, 11
        array1 = np.zeros(data_shape)
        array1[:, :, 5, 5] = 1
        dask_array = da.from_array(array1, chunks=(1, 1, 1, 1))
        s1 = LazyDiffraction2D(dask_array)
        s1.axes_manager.signal_axes[0].offset = -5
        s1.axes_manager.signal_axes[1].offset = -5
        s1_r = s1.radial_average()
        assert np.all(s1_r.data[:, :, 0] == 1)
        assert np.all(s1_r.data[:, :, 1:] == 0) 

def to_dask(self, chunks=None, name=None):
    """Return a dask array for this volume.

    Parameters
    ----------
    chunks: tuple of ints or tuples of ints
      Passed to ``da.from_array``, allows setting the chunks on
      initialisation, if the chunking scheme in the stored dataset is not
      optimal for the calculations to follow. Note that the chunking should
      be compatible with an underlying 4d array.
    name: str, optional
      An optional keyname for the array. Defaults to hashing the input

    Returns
    -------
    Dask array
    """
    import dask.array as da
    from dask.base import tokenize

    if chunks is None:
      chunks = tuple(self.chunk_size) + (self.num_channels, )
    if name is None:
      name = 'to-dask-' + tokenize(self, chunks)
    return da.from_array(self, chunks, name=name) 

def test_array_different_dimensions(self, nav_dims):
        shape = list(np.random.randint(2, 6, size=nav_dims))
        shape.extend([50, 50])
        chunks = [1] * nav_dims
        chunks.extend([25, 25])
        dask_array = da.random.random(size=shape, chunks=chunks)
        peak_array = np.zeros((dask_array.shape[:-2]), dtype=np.object)
        for index in np.ndindex(dask_array.shape[:-2]):
            islice = np.s_[index]
            peak_array[islice] = np.asarray([(27, 27)])
        square_size = 12
        peak_array_dask = da.from_array(peak_array, chunks=chunks[:-2])
        match_array_dask = dt._peak_refinement_centre_of_mass(
            dask_array, peak_array_dask, square_size
        )
        assert len(dask_array.shape) == nav_dims + 2
        match_array = match_array_dask.compute()
        assert peak_array_dask.shape == match_array.shape 

def test_dask_array(self):
        numpy_array = np.zeros((10, 10, 50, 50))
        numpy_array[:, :, 25, 25] = 1

        peak_array = np.zeros((numpy_array.shape[:-2]), dtype=np.object)
        real_array = np.zeros((numpy_array.shape[:-2]), dtype=np.object)
        for index in np.ndindex(numpy_array.shape[:-2]):
            islice = np.s_[index]
            peak_array[islice] = np.asarray([(27, 27)])
            real_array[islice] = np.asarray([(25, 25)])

        dask_array = da.from_array(numpy_array, chunks=(5, 5, 5, 5))
        dask_peak_array = da.from_array(peak_array, chunks=(5, 5))

        square_size = 12

        data = dt._peak_refinement_centre_of_mass(
            dask_array, dask_peak_array, square_size
        )
        data = data.compute()
        assert data.shape == (10, 10)
        assert np.sum(data - real_array).sum() == 0 

def test_dask_array_normalize_value(self):
        data = np.zeros((2, 3, 100, 100), dtype=np.uint16)
        data[:, :, 49:52, 49:52] = 100
        data[:, :, 19:22, 9:12] = 10
        dask_array = da.from_array(data, chunks=(1, 1, 100, 100))
        peak_array0 = dt._peak_find_dog(dask_array, normalize_value=100)
        peak_array1 = dt._peak_find_dog(dask_array, normalize_value=10)
        for ix, iy in np.ndindex(peak_array0.shape):
            assert (peak_array0[ix, iy] == [[50, 50]]).all()
            assert (peak_array1[ix, iy] == [[50, 50], [20, 10]]).all() 

def test_dask_min_sigma(self):
        min_sigma = 10
        numpy_array = np.ones((10, 10, 50, 50))
        numpy_array[:, :20:30, 20:30] = 5
        dask_array = da.from_array(numpy_array, chunks=(2, 2, 50, 50))

        data = dt._background_removal_dog(dask_array, min_sigma=min_sigma)
        data.compute()
        assert data.sum() != numpy_array.sum()
        assert data.shape == numpy_array.shape
        assert data[:, :, 0, :].all() == 0 

def test_array_different_dimensions(self, nav_dims):
        shape = list(np.random.randint(2, 6, size=nav_dims))
        shape.extend([50, 50])
        chunks = [1] * nav_dims
        chunks.extend([25, 25])
        dask_array = da.random.random(size=shape, chunks=chunks)
        peak_array = np.zeros((dask_array.shape[:-2]), dtype=np.object)
        for index in np.ndindex(dask_array.shape[:-2]):
            islice = np.s_[index]
            peak_array[islice] = np.asarray([(27, 27)])
        peak_array_dask = da.from_array(peak_array, chunks=chunks[:-2])
        match_array_dask = dt._intensity_peaks_image(dask_array, peak_array_dask, 5)
        assert len(dask_array.shape) == nav_dims + 2
        match_array = match_array_dask.compute()
        assert peak_array_dask.shape == match_array.shape 

def test_dask_array(self):
        data = np.zeros(shape=(2, 3, 200, 100), dtype=np.float64)
        data[0, 0, 50, 20] = 100
        data[0, 1, 51, 21] = 100
        data[0, 2, 52, 22] = 100
        data[1, 0, 53, 23] = 100
        data[1, 1, 54, 24] = 100
        data[1, 2, 55, 25] = 100
        dask_array = da.from_array(data, chunks=(1, 1, 200, 100))
        min_sigma, max_sigma, num_sigma = 0.08, 1, 10
        threshold, overlap = 0.06, 0.01
        peaks = dt._peak_find_log(
            dask_array,
            min_sigma=min_sigma,
            max_sigma=max_sigma,
            num_sigma=num_sigma,
            threshold=threshold,
            overlap=overlap,
        )
        peaks = peaks.compute()
        assert peaks[0, 0][0].tolist() == [50, 20]
        assert peaks[0, 1][0].tolist() == [51, 21]
        assert peaks[0, 2][0].tolist() == [52, 22]
        assert peaks[1, 0][0].tolist() == [53, 23]
        assert peaks[1, 1][0].tolist() == [54, 24]
        assert peaks[1, 2][0].tolist() == [55, 25] 

def test_dask_centre(self):
        centre_x = 25
        centre_y = 25
        numpy_array = np.ones((10, 10, 50, 50))
        numpy_array[:, :20:30, 20:30] = 5
        dask_array = da.from_array(numpy_array, chunks=(2, 2, 50, 50))

        data = dt._background_removal_radial_median(
            dask_array, centre_x=centre_x, centre_y=centre_y
        )
        data = data.compute()
        assert data.sum() != numpy_array.sum()
        assert data.shape == numpy_array.shape
        assert (data[:, :, 0, :]).all() == 0 

def test_2d(self):
        data = np.ones((100, 90))
        data[41, 21] = 0
        data[9, 81] = 50000
        dask_array = da.from_array(data, chunks=(10, 10))
        s = LazyDiffraction2D(dask_array)
        s_dead_pixels = s.find_dead_pixels(lazy_result=True)
        s_hot_pixels = s.find_hot_pixels(lazy_result=True)
        s_bad_pixels = s_dead_pixels + s_hot_pixels
        s_corr = s.correct_bad_pixels(s_bad_pixels)
        assert s_dead_pixels.data.shape == data.shape
        assert s_dead_pixels._lazy
        s_corr.compute()
        assert (s_corr.data == 1.0).all() 

def test_position(self):
        disk_r = 5
        data = np.zeros((2, 3, 90, 100))
        # Nav top left, sig x=5, y=5
        data[0, 0, :11, :11] = sm.disk(disk_r)
        # Nav top centre, sig x=94, y=84
        data[0, 1, -11:, -11:] = sm.disk(disk_r)
        # Nav top right, sig x=94, y=5
        data[0, 2, :11, -11:] = sm.disk(disk_r)
        # Nav bottom left, sig x=5, y=84
        data[1, 0, -11:, :11] = sm.disk(disk_r)
        # Nav bottom centre, sig x=75, y=25
        data[1, 1, 20:31, 70:81] = sm.disk(disk_r)
        # Nav bottom right, sig x=55, y=75
        data[1, 2, 70:81, 50:61] = sm.disk(disk_r)
        binary_image = sm.disk(disk_r)
        dask_array = da.from_array(data, chunks=(1, 1, 5, 5))
        out_dask = dt._template_match_with_binary_image(
            dask_array, binary_image=binary_image
        )
        out = out_dask.compute()
        match00 = np.unravel_index(np.argmax(out[0, 0]), out[0, 0].shape)
        assert (5, 5) == match00
        match01 = np.unravel_index(np.argmax(out[0, 1]), out[0, 1].shape)
        assert (84, 94) == match01
        match02 = np.unravel_index(np.argmax(out[0, 2]), out[0, 2].shape)
        assert (5, 94) == match02
        match10 = np.unravel_index(np.argmax(out[1, 0]), out[1, 0].shape)
        assert (84, 5) == match10
        match11 = np.unravel_index(np.argmax(out[1, 1]), out[1, 1].shape)
        assert (25, 75) == match11
        match12 = np.unravel_index(np.argmax(out[1, 2]), out[1, 2].shape)
        assert (75, 55) == match12 

def test_dask_max_sigma(self):
        max_sigma = 10
        numpy_array = np.ones((10, 10, 50, 50))
        numpy_array[:, :20:30, 20:30] = 5
        dask_array = da.from_array(numpy_array, chunks=(2, 2, 50, 50))

        data = dt._background_removal_dog(dask_array, max_sigma=max_sigma)
        data.compute()
        assert data.sum() != numpy_array.sum()
        assert data.shape == numpy_array.shape
        assert data[:, :, 0, :].all() == 0 

def test_dask_array_normalize_value(self):
        data = np.zeros((2, 3, 100, 100), dtype=np.uint16)
        data[:, :, 49:52, 49:52] = 100
        data[:, :, 19:22, 9:12] = 10
        dask_array = da.from_array(data, chunks=(1, 1, 100, 100))
        peak_array0 = dt._peak_find_log(dask_array, normalize_value=100)
        peak_array1 = dt._peak_find_log(dask_array, normalize_value=10)
        for ix, iy in np.ndindex(peak_array0.shape):
            assert (peak_array0[ix, iy] == [[50, 50]]).all()
            assert (peak_array1[ix, iy] == [[50, 50], [20, 10]]).all() 

def test_mask(self):
        numpy_array = np.zeros((10, 10, 30, 30))
        numpy_array[:, :, 0, 0] = 1000
        numpy_array[:, :, -1, -1] = 1
        dask_array = da.from_array(numpy_array, chunks=(5, 5, 5, 5))
        centre_x, centre_y = np.ones((2, 100)) * 15
        data = pst._radial_average_dask_array(
            dask_array,
            return_sig_size=22,
            centre_x=centre_x,
            centre_y=centre_y,
            normalize=False,
            show_progressbar=False,
        )
        assert data.shape == (10, 10, 22)
        assert (data != 0.0).any()
        mask = pst._make_circular_mask(15, 15, 30, 30, 15)
        data = pst._radial_average_dask_array(
            dask_array,
            return_sig_size=22,
            centre_x=centre_x,
            centre_y=centre_y,
            normalize=False,
            mask_array=mask,
            show_progressbar=False,
        )
        assert data.shape == (10, 10, 22)
        assert (data == 0.0).all() 

def test_lazy_output(self, methods):
        data = np.random.randint(100, size=(3, 2, 10, 20))
        s = LazyDiffraction2D(da.from_array(data, chunks=(1, 1, 5, 10)))
        peak_array = s.find_peaks_lazy(method=methods, lazy_result=False)
        assert s.data.shape[:2] == peak_array.shape
        assert not hasattr(peak_array, "compute")