Java Code Examples for org.nd4j.linalg.api.ndarray.INDArray#columns()

@Override
public INDArray getLFactor(INDArray A) {
    if (A.rows() > Integer.MAX_VALUE || A.columns() > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();

    int m = (int) A.rows();
    int n = (int) A.columns();

    INDArray L = Nd4j.create(m, n);
    for (int r = 0; r < m; r++) {
        for (int c = 0; c < n; c++) {
            if (r > c && r < m && c < n) {
                L.putScalar(r, c, A.getFloat(r, c));
            } else if (r < c) {
                L.putScalar(r, c, 0.f);
            } else {
                L.putScalar(r, c, 1.f);
            }
        }
    }
    return L;
}
 

/**
 * syr2 performs a rank-2 update of an n-by-n symmetric matrix a:
 * a := alpha*x*y' + alpha*y*x' + a.
 *
 * @param order
 * @param Uplo
 * @param TransA
 * @param Diag
 * @param A
 * @param X
 */
@Override
public void tbmv(char order, char Uplo, char TransA, char Diag, INDArray A, INDArray X) {
    if (Nd4j.getExecutioner().getProfilingMode() == OpExecutioner.ProfilingMode.ALL)
        OpProfiler.getInstance().processBlasCall(false, A, X);

    if (X.length() > Integer.MAX_VALUE || A.columns() > Integer.MAX_VALUE || A.size(0) > Integer.MAX_VALUE) {
        throw new ND4JArraySizeException();
    }

    if (X.data().dataType() == DataType.DOUBLE) {
        DefaultOpExecutioner.validateDataType(DataType.DOUBLE, A, X);
        dtbmv(order, Uplo, TransA, Diag, (int) X.length(), (int) A.columns(), A, (int) A.size(0), X, X.stride(-1));
    } else {
        DefaultOpExecutioner.validateDataType(DataType.FLOAT, A, X);
        stbmv(order, Uplo, TransA, Diag, (int) X.length(), (int) A.columns(), A, (int) A.size(0), X, X.stride(-1));
    }
}
 

/**
 * ?tbsv solves a system of linear equations whose coefficients are in a triangular band matrix.
 *
 * @param order
 * @param Uplo
 * @param TransA
 * @param Diag
 * @param A
 * @param X
 */
@Override
public void tbsv(char order, char Uplo, char TransA, char Diag, INDArray A, INDArray X) {
    if (Nd4j.getExecutioner().getProfilingMode() == OpExecutioner.ProfilingMode.ALL)
        OpProfiler.getInstance().processBlasCall(false, A, X);

    if (X.length() > Integer.MAX_VALUE || A.columns() > Integer.MAX_VALUE || A.size(0) > Integer.MAX_VALUE ) {
        throw new ND4JArraySizeException();
    }

    if (X.data().dataType() == DataType.DOUBLE) {
        DefaultOpExecutioner.validateDataType(DataType.DOUBLE, A, X);
        dtbsv(order, Uplo, TransA, Diag, (int) X.length(), (int) A.columns(), A, (int) A.size(0), X, X.stride(-1));
    } else {
        DefaultOpExecutioner.validateDataType(DataType.FLOAT, A, X);
        stbsv(order, Uplo, TransA, Diag, (int) X.length(), (int) A.columns(), A, (int) A.size(0), X, X.stride(-1));
    }

}
 

/**
 * Returns a column vector where each entry is the nth bilinear
 * product of the nth slices of the two tensors.
 */
@Override
public INDArray bilinearProducts(INDArray curr, INDArray in) {
    Preconditions.checkArgument(curr.rank() == 3, "Argument 'curr' must be rank 3. Got input with rank: %s", curr.rank());
    if (in.columns() != 1) {
        throw new AssertionError("Expected a column vector");
    }
    if (in.rows() != curr.size(curr.shape().length - 1)) {
        throw new AssertionError("Number of rows in the input does not match number of columns in tensor");
    }
    if (curr.size(curr.shape().length - 2) != curr.size(curr.shape().length - 1)) {
        throw new AssertionError("Can only perform this operation on a SimpleTensor with square slices");
    }

    INDArray ret = Nd4j.create(curr.slices(), 1);
    INDArray inT = in.transpose();
    for (int i = 0; i < curr.slices(); i++) {
        INDArray slice = curr.slice(i);
        INDArray inTTimesSlice = inT.mmul(slice);
        ret.putScalar(i, Nd4j.getBlasWrapper().dot(inTTimesSlice, in));
    }
    return ret;
}
 

@Override
public int syev(char jobz, char uplo, INDArray A, INDArray V) {

    if (A.rows() != A.columns()) {
        throw new Error("syev: A must be square.");
    }
    if (A.rows() != V.length()) {
        throw new Error("syev: V must be the length of the matrix dimension.");
    }

    if (A.rows() > Integer.MAX_VALUE || A.columns() > Integer.MAX_VALUE)
        throw new ND4JArraySizeException();

    int status = -1;
    if (A.data().dataType() == DataType.DOUBLE) {
        status = dsyev(jobz, uplo, (int) A.rows(), A, V);
    } else if (A.data().dataType() == DataType.FLOAT) {
        status = ssyev(jobz, uplo, (int) A.rows(), A, V);
    } else {
        throw new UnsupportedOperationException();
    }

    return status;
}
 

/**
 * sbmv computes a matrix-vector product using a symmetric band matrix:
 * y := alpha*a*x + beta*y.
 * Here a is an n-by-n symmetric band matrix with k superdiagonals, x and y are n-element vectors, alpha and beta are scalars.
 *
 * @param order
 * @param Uplo
 * @param alpha
 * @param A
 * @param X
 * @param beta
 * @param Y
 */
@Override
public void sbmv(char order, char Uplo, double alpha, INDArray A, INDArray X, double beta, INDArray Y) {
    if (Nd4j.getExecutioner().getProfilingMode() == OpExecutioner.ProfilingMode.ALL)
        OpProfiler.getInstance().processBlasCall(false, A, X, Y);

    if (X.length() > Integer.MAX_VALUE || A.columns() > Integer.MAX_VALUE || A.size(0) > Integer.MAX_VALUE) {
        throw new ND4JArraySizeException();
    }
    if (X.data().dataType() == DataType.DOUBLE) {
        DefaultOpExecutioner.validateDataType(DataType.DOUBLE, A, X, Y);
        dsbmv(order, Uplo, (int) X.length(), (int) A.columns(), alpha, A, (int) A.size(0), X, X.stride(-1), beta, Y,
                Y.stride(-1));
    } else {
        DefaultOpExecutioner.validateDataType(DataType.FLOAT, A, X, Y);
        ssbmv(order, Uplo, (int) X.length(), (int) A.columns(), (float) alpha, A, (int) A.size(0), X, X.stride(-1), (float) beta,
                        Y, Y.stride(-1));
    }

    OpExecutionerUtil.checkForAny(Y);
}
 

/**
 * This method performs a dimensionality reduction, including principal components
 * that cover a fraction of the total variance of the system.  It does all calculations
 * about the mean.
 * @param in A matrix of datapoints as rows, where column are features with fixed number N
 * @param variance The desired fraction of the total variance required
 * @return The reduced basis set
 */
public static INDArray pca2(INDArray in, double variance) {
    // let's calculate the covariance and the mean
    INDArray[] covmean = covarianceMatrix(in);
    // use the covariance matrix (inverse) to find "force constants" and then break into orthonormal
    // unit vector components
    INDArray[] pce = principalComponents(covmean[0]);
    // calculate the variance of each component
    INDArray vars = Transforms.pow(pce[1], -0.5, true);
    double res = vars.sumNumber().doubleValue();
    double total = 0.0;
    int ndims = 0;
    for (int i = 0; i < vars.columns(); i++) {
        ndims++;
        total += vars.getDouble(i);
        if (total / res > variance)
            break;
    }
    INDArray result = Nd4j.create(in.columns(), ndims);
    for (int i = 0; i < ndims; i++)
        result.putColumn(i, pce[0].getColumn(i));
    return result;
}
 

private static XYDataset createDataSetTest(INDArray features, INDArray labels, INDArray predicted ){
    int nRows = features.rows();

    int nClasses = labels.columns();

    XYSeries[] series = new XYSeries[nClasses*nClasses];    //new XYSeries("Data");
    for( int i=0; i<nClasses*nClasses; i++){
        int trueClass = i/nClasses;
        int predClass = i%nClasses;
        String label = "actual=" + trueClass + ", pred=" + predClass;
        series[i] = new XYSeries(label);
    }
    INDArray actualIdx = Nd4j.getExecutioner().exec(new IMax(labels), 1);
    INDArray predictedIdx = Nd4j.getExecutioner().exec(new IMax(predicted), 1);
    for( int i=0; i<nRows; i++ ){
        int classIdx = (int)actualIdx.getDouble(i);
        int predIdx = (int)predictedIdx.getDouble(i);
        int idx = classIdx * nClasses + predIdx;
        series[idx].add(features.getDouble(i, 0), features.getDouble(i, 1));
    }

    XYSeriesCollection c = new XYSeriesCollection();
    for( XYSeries s : series) c.addSeries(s);
    return c;
}
 

/**
 * Return a reduced basis set that covers a certain fraction of the variance of the data
 * @param variance The desired fractional variance (0 to 1), it will always be greater than the value.
 * @return The basis vectors as columns, size <i>N</i> rows by <i>ndims</i> columns, where <i>ndims</i> is less than or equal to <i>N</i>
 */
public INDArray reducedBasis(double variance) {
    INDArray vars = Transforms.pow(eigenvalues, -0.5, true);
    double res = vars.sumNumber().doubleValue();
    double total = 0.0;
    int ndims = 0;
    for (int i = 0; i < vars.columns(); i++) {
        ndims++;
        total += vars.getDouble(i);
        if (total / res > variance)
            break;
    }
    INDArray result = Nd4j.create(eigenvectors.rows(), ndims);
    for (int i = 0; i < ndims; i++)
        result.putColumn(i, eigenvectors.getColumn(i));
    return result;
}
 

/**
 * yr2k performs a rank-2k update of an n-by-n symmetric matrix c, that is, one of the following operations:
 * c := alpha*a*b' + alpha*b*a' + beta*c  for trans = 'N'or'n'
 * c := alpha*a'*b + alpha*b'*a + beta*c  for trans = 'T'or't',
 * where c is an n-by-n symmetric matrix;
 * a and b are n-by-k matrices, if trans = 'N'or'n',
 * a and b are k-by-n matrices, if trans = 'T'or't'.
 *  @param Order
 * @param Uplo
 * @param Trans
 * @param alpha
 * @param A
 * @param B
 * @param beta
 * @param C
 */
@Override
public void syr2k(char Order, char Uplo, char Trans, double alpha, INDArray A, INDArray B, double beta,
                INDArray C) {
    if (Nd4j.getExecutioner().getProfilingMode() == OpExecutioner.ProfilingMode.ALL)
        OpProfiler.getInstance().processBlasCall(false, A, B, C);

    if (A.rows() > Integer.MAX_VALUE || A.columns() > Integer.MAX_VALUE ||
        A.size(0) > Integer.MAX_VALUE || B.size(0) > Integer.MAX_VALUE || C.size(0) > Integer.MAX_VALUE) {
        throw new ND4JArraySizeException();
    }

    if (A.data().dataType() == DataType.DOUBLE) {
        DefaultOpExecutioner.validateDataType(DataType.DOUBLE, A, B, C);
        dsyr2k(Order, Uplo, Trans, (int) A.rows(), (int) A.columns(), alpha, A, (int) A.size(0), B, (int) B.size(0), beta, C, (int) C.size(0));
    } else {
        DefaultOpExecutioner.validateDataType(DataType.FLOAT, A, B, C);
        ssyr2k(Order, Uplo, Trans, (int) A.rows(), (int) A.columns(), (float) alpha, A, (int) A.size(0), B, (int) B.size(0), (float) beta, C, (int) C.size(0));
    }

    OpExecutionerUtil.checkForAny(C);
}
 

private String toCsv(DataSetIterator it, List<Integer> labels, int[] shape) {
    if (it.numExamples() != labels.size()) {
        throw new IllegalStateException(
                String.format("numExamples == %d != labels.size() == %d",
                        it.numExamples(), labels.size()));
    }

    StringBuffer sb = new StringBuffer();
    int l = 0;

    while (it.hasNext()) {
        INDArray features = it.next(1).getFeatures();

        if (!(Arrays.equals(features.shape(), shape))) {
            throw new IllegalStateException(String.format("wrong shape: got %s, expected",
                    Arrays.toString(features.shape()), Arrays.toString(shape)));
        }

        // Prepend the label
        sb.append(labels.get(l)).append(": ");
        l++;

        for (int i=0; i<features.columns(); i++) {
            sb.append(features.getColumn(i));

            if (i < features.columns()-1) {
                sb.append(", ");
            }
        }

        sb.append("\n");
    }

    return sb.toString();
}
 

@Override
public INDArray preOutput(boolean training, LayerWorkspaceMgr workspaceMgr) {
    assertInputSet(false);
    applyDropOutIfNecessary(training, workspaceMgr);
    INDArray W = getParamWithNoise(DefaultParamInitializer.WEIGHT_KEY, training, workspaceMgr);
    INDArray b = getParamWithNoise(DefaultParamInitializer.BIAS_KEY, training, workspaceMgr);

    INDArray ret = workspaceMgr.createUninitialized(ArrayType.ACTIVATIONS, input.size(0), W.size(1));
    ret.assign(0F);
    for (int row = 0; row < input.rows(); row++) {
        for (int column = 0; column < W.columns(); column++) {
            float value = 0F;
            int cursor = 0;
            for (int index = 0; index < input.columns(); index++) {
                value += W.getFloat(cursor + input.getInt(row, index), column);
                cursor += dimensionSizes[index];
            }
            ret.put(row, column, value);
        }
    }

    if (hasBias()) {
        ret.addiRowVector(b);
    }

    if (maskArray != null) {
        applyMask(ret);
    }

    return ret;
}
 

public double getSquaredFrobeniusNorm(final INDArray matrix) {
	double norm = 0;
	int m = matrix.rows();
	int n = matrix.columns();
	for (int i = 0; i < m; i++) {
		for (int j = 0; j < n; j++) {
			norm += Math.pow(matrix.getDouble(i, j), 2);
		}
	}
	return norm;
}
 

@Override
public List<Writable> call(INDArray arr) throws Exception {
    if (arr.rows() != 1)
        throw new UnsupportedOperationException("Only NDArray row vectors can be converted to list"
                                            + " of Writables (found " + arr.rows() + " rows)");
    List<Writable> record = new ArrayList<>();
    if (useNdarrayWritable) {
        record.add(new NDArrayWritable(arr));
    } else {
        for (int i = 0; i < arr.columns(); i++)
            record.add(new DoubleWritable(arr.getDouble(i)));
    }
    return record;
}
 

@Override
public void doBackward() {
    GlobalMatrix inputKeyMatrix = GlobalMatrix.class.cast(inputKeyValues[0].getKey());
    GlobalMatrix inputValueMatrix = GlobalMatrix.class.cast(inputKeyValues[0].getValue());
    Nd4jMatrix outputKeyMatrix = Nd4jMatrix.class.cast(outputKeyValue.getKey());
    Nd4jMatrix outputValueMatrix = Nd4jMatrix.class.cast(outputKeyValue.getValue());

    {
        INDArray innerError = outputValueMatrix.getArray();
        int cursor = 0;
        for (MathMatrix component : inputValueMatrix.getComponentMatrixes()) {
            // TODO 使用累计的方式计算
            // TODO 需要锁机制,否则并发计算会导致Bug
            Nd4jMatrix nd4j = Nd4jMatrix.class.cast(component);
            INDArray array = nd4j.getArray();
            synchronized (component) {
                if (orientation) {
                    array.addi(innerError.get(new INDArrayIndex[] { NDArrayIndex.all(), NDArrayIndex.interval(cursor, cursor + array.columns()) }));
                    cursor += array.columns();
                } else {
                    array.addi(innerError.get(new INDArrayIndex[] { NDArrayIndex.interval(cursor, cursor + array.rows()), NDArrayIndex.all() }));
                    cursor += array.rows();
                }
            }
        }
    }
}
 

/**
 * Convert a list of rows to a matrix
 * @param rows the list of rows to convert
 * @return the converted matrix
 */
public static INDArray toMatrix(List<Row> rows) {
    INDArray ret = Nd4j.create(rows.size(), rows.get(0).size());
    for (int i = 0; i < ret.rows(); i++) {
        for (int j = 0; j < ret.columns(); j++) {
            ret.putScalar(i, j, rows.get(i).getDouble(j));
        }
    }
    return ret;
}
 

@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
    super.initialize(conf, split);
    //train  a new one since it hasn't been specified
    if (tfidfVectorizer == null) {
        tfidfVectorizer = new TfidfVectorizer();
        tfidfVectorizer.initialize(conf);

        //clear out old strings
        records.clear();

        INDArray ret = tfidfVectorizer.fitTransform(this, new Vectorizer.RecordCallBack() {
            @Override
            public void onRecord(Record fullRecord) {
                records.add(fullRecord);
            }
        });

        //cache the number of features used for each document
        numFeatures = ret.columns();
        recordIter = records.iterator();
    } else {
        records = new ArrayList<>();

        //the record reader has 2 phases, we are skipping the
        //document frequency phase and just using the super() to get the file contents
        //and pass it to the already existing vectorizer.
        while (super.hasNext()) {
            Record fileContents = super.nextRecord();
            INDArray transform = tfidfVectorizer.transform(fileContents);

            org.datavec.api.records.impl.Record record = new org.datavec.api.records.impl.Record(
                            new ArrayList<>(Collections.<Writable>singletonList(new NDArrayWritable(transform))),
                            new RecordMetaDataURI(fileContents.getMetaData().getURI(), TfidfRecordReader.class));

            if (appendLabel)
                record.getRecord().add(fileContents.getRecord().get(fileContents.getRecord().size() - 1));

            records.add(record);
        }

        recordIter = records.iterator();
    }

    this.initialized = true;
}
 

private static double mse(INDArray output, INDArray target) {
    double mse_expect = Transforms.pow(output.sub(target), 2.0).sumNumber().doubleValue()
                    / (output.columns() * output.rows());
    return mse_expect;
}
 

/**
 * Calculates pca vectors of a matrix, for a given variance. A larger variance (99%)
 * will result in a higher order feature set.
 *
 * To use the returned factor: multiply feature(s) by the factor to get a reduced dimension
 *
 * INDArray Areduced = A.mmul( factor ) ;
 * 
 * The array Areduced is a projection of A onto principal components
 *
 * @see pca(INDArray, double, boolean)
 *
 * @param A the array of features, rows are results, columns are features - will be changed
 * @param variance the amount of variance to preserve as a float 0 - 1
 * @param normalize whether to normalize (set features to have zero mean)
 * @return the matrix to mulitiply a feature by to get a reduced feature set
 */
public static INDArray pca_factor(INDArray A, double variance, boolean normalize) {
    if (normalize) {
        // Normalize to mean 0 for each feature ( each column has 0 mean )
        INDArray mean = A.mean(0);
        A.subiRowVector(mean);
    }

    long m = A.rows();
    long n = A.columns();

    // The prepare SVD results, we'll decomp A to UxSxV'
    INDArray s = Nd4j.create(A.dataType(), m < n ? m : n);
    INDArray VT = Nd4j.create(A.dataType(), new long[]{n, n}, 'f');

    // Note - we don't care about U 
    Nd4j.getBlasWrapper().lapack().gesvd(A, s, null, VT);

    // Now convert the eigs of X into the eigs of the covariance matrix
    for (int i = 0; i < s.length(); i++) {
        s.putScalar(i, Math.sqrt(s.getDouble(i)) / (m - 1));
    }

    // Now find how many features we need to preserve the required variance
    // Which is the same percentage as a cumulative sum of the eigenvalues' percentages
    double totalEigSum = s.sumNumber().doubleValue() * variance;
    int k = -1; // we will reduce to k dimensions
    double runningTotal = 0;
    for (int i = 0; i < s.length(); i++) {
        runningTotal += s.getDouble(i);
        if (runningTotal >= totalEigSum) { // OK I know it's a float, but what else can we do ?
            k = i + 1; // we will keep this many features to preserve the reqd. variance
            break;
        }
    }
    if (k == -1) { // if we need everything
        throw new RuntimeException("No reduction possible for reqd. variance - use smaller variance");
    }
    // So now let's rip out the appropriate number of left singular vectors from
    // the V output (note we pulls rows since VT is a transpose of V)
    INDArray V = VT.transpose();
    INDArray factor = Nd4j.createUninitialized(A.dataType(), new long[]{n, k}, 'f');
    for (int i = 0; i < k; i++) {
        factor.putColumn(i, V.getColumn(i));
    }

    return factor;
}
 

@Override
public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException {
    super.initialize(conf, split);
    //train  a new one since it hasn't been specified
    if (tfidfVectorizer == null) {
        tfidfVectorizer = new TfidfVectorizer();
        tfidfVectorizer.initialize(conf);

        //clear out old strings
        records.clear();

        INDArray ret = tfidfVectorizer.fitTransform(this, new Vectorizer.RecordCallBack() {
            @Override
            public void onRecord(Record fullRecord) {
                records.add(fullRecord);
            }
        });

        //cache the number of features used for each document
        numFeatures = ret.columns();
        recordIter = records.iterator();
    } else {
        records = new ArrayList<>();

        //the record reader has 2 phases, we are skipping the
        //document frequency phase and just using the super() to get the file contents
        //and pass it to the already existing vectorizer.
        while (super.hasNext()) {
            Record fileContents = super.nextRecord();
            INDArray transform = tfidfVectorizer.transform(fileContents);

            org.datavec.api.records.impl.Record record = new org.datavec.api.records.impl.Record(
                            new ArrayList<>(Collections.<Writable>singletonList(new NDArrayWritable(transform))),
                            new RecordMetaDataURI(fileContents.getMetaData().getURI(), TfidfRecordReader.class));

            if (appendLabel)
                record.getRecord().add(fileContents.getRecord().get(fileContents.getRecord().size() - 1));

            records.add(record);
        }

        recordIter = records.iterator();
    }

    this.initialized = true;
}