Java Code Examples for org.nd4j.linalg.api.buffer.DataType#UTF8
The following examples show how to use
org.nd4j.linalg.api.buffer.DataType#UTF8 .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TensorflowConversion.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
private DataType typeFor(int tensorflowType) {
switch(tensorflowType) {
case DT_DOUBLE: return DataType.DOUBLE;
case DT_FLOAT: return DataType.FLOAT;
case DT_HALF: return DataType.HALF;
case DT_INT16: return DataType.SHORT;
case DT_INT32: return DataType.INT;
case DT_INT64: return DataType.LONG;
case DT_STRING: return DataType.UTF8;
case DT_INT8: return DataType.BYTE;
case DT_UINT8: return DataType.UBYTE;
case DT_UINT16: return DataType.UINT16;
case DT_UINT32: return DataType.UINT32;
case DT_UINT64: return DataType.UINT64;
case DT_BFLOAT16: return DataType.BFLOAT16;
case DT_BOOL: return DataType.BOOL;
default: throw new IllegalArgumentException("Illegal type " + tensorflowType);
}
}
Example 2
| Source File: ArrayOptionsHelper.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
public static DataType dataType(long opt) {
if (hasBitSet(opt, DTYPE_COMPRESSED_BIT))
return DataType.COMPRESSED;
else if (hasBitSet(opt, DTYPE_HALF_BIT))
return DataType.HALF;
else if (hasBitSet(opt, DTYPE_BFLOAT16_BIT))
return DataType.BFLOAT16;
else if (hasBitSet(opt, DTYPE_FLOAT_BIT))
return DataType.FLOAT;
else if (hasBitSet(opt, DTYPE_DOUBLE_BIT))
return DataType.DOUBLE;
else if (hasBitSet(opt, DTYPE_INT_BIT))
return hasBitSet(opt, DTYPE_UNSIGNED_BIT) ? DataType.UINT32 : DataType.INT;
else if (hasBitSet(opt, DTYPE_LONG_BIT))
return hasBitSet(opt, DTYPE_UNSIGNED_BIT) ? DataType.UINT64 : DataType.LONG;
else if (hasBitSet(opt, DTYPE_BOOL_BIT))
return DataType.BOOL;
else if (hasBitSet(opt, DTYPE_BYTE_BIT)) {
return hasBitSet(opt, DTYPE_UNSIGNED_BIT) ? DataType.UBYTE : DataType.BYTE; //Byte bit set for both UBYTE and BYTE
} else if (hasBitSet(opt, DTYPE_SHORT_BIT))
return hasBitSet(opt, DTYPE_UNSIGNED_BIT) ? DataType.UINT16 : DataType.SHORT;
else if (hasBitSet(opt, DTYPE_UTF8_BIT))
return DataType.UTF8;
else
throw new ND4JUnknownDataTypeException("Unknown extras set: [" + opt + "]");
}
Example 3
| Source File: NDValidation.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
/**
* Validate that the operation is being applied on numerical INDArrays (not boolean or utf8).
* Some operations (such as sum, norm2, add(Number) etc) don't make sense when applied to boolean/utf8 arrays
*
* @param opName Operation name to print in the exception
* @param v Variable to perform operation on
*/
public static void validateNumerical(String opName, String inputName, INDArray[] v) {
if (v == null)
return;
for (int i = 0; i < v.length; i++) {
if (v[i].dataType() == DataType.BOOL || v[i].dataType() == DataType.UTF8)
throw new IllegalStateException("Cannot apply operation \"" + opName + "\" to input \"" + inputName + "\" array " + i + " with non-numerical data type " + v[i].dataType());
}
}
Example 4
| Source File: ND4JUtil.java From konduit-serving with Apache License 2.0 | 5 votes |
|
public static DataType typeNDArrayTypeToNd4j(@NonNull NDArrayType type){
switch (type){
case DOUBLE:
return DataType.DOUBLE;
case FLOAT:
return DataType.FLOAT;
case FLOAT16:
return DataType.FLOAT16;
case BFLOAT16:
return DataType.BFLOAT16;
case INT64:
return DataType.INT64;
case INT32:
return DataType.INT32;
case INT16:
return DataType.INT16;
case INT8:
return DataType.INT8;
case UINT64:
return DataType.UINT64;
case UINT32:
return DataType.UINT32;
case UINT16:
return DataType.UINT16;
case UINT8:
return DataType.UINT8;
case BOOL:
return DataType.BOOL;
case UTF8:
return DataType.UTF8;
default:
throw new UnsupportedOperationException("Unable to convert datatype to ND4J datatype: " + type);
}
}
Example 5
| Source File: NDValidation.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
/**
* Validate that the operation is being applied on a numerical INDArray (not boolean or utf8).
* Some operations (such as sum, norm2, add(Number) etc) don't make sense when applied to boolean/utf8 arrays
*
* @param opName Operation name to print in the exception
* @param v Variable to validate datatype for (input to operation)
*/
public static void validateNumerical(String opName, String inputName, INDArray v) {
if (v == null)
return;
if (v.dataType() == DataType.BOOL || v.dataType() == DataType.UTF8)
throw new IllegalStateException("Input \"" + inputName + "\" for operation \"" + opName + "\" must be an numerical type type;" +
" got array with non-integer data type " + v.dataType());
}
Example 6
| Source File: NDValidation.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
/**
* Validate that the operation is being applied on numerical INDArrays (not boolean or utf8).
* Some operations (such as sum, norm2, add(Number) etc) don't make sense when applied to boolean/utf8 arrays
*
* @param opName Operation name to print in the exception
* @param v Variable to perform operation on
*/
public static void validateNumerical(String opName, INDArray[] v) {
if (v == null)
return;
for (int i = 0; i < v.length; i++) {
if (v[i].dataType() == DataType.BOOL || v[i].dataType() == DataType.UTF8)
throw new IllegalStateException("Cannot apply operation \"" + opName + "\" to input array " + i + " with non-numerical data type " + v[i].dataType());
}
}
Example 7
| Source File: ArrayOptionsHelper.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
public static DataType convertToDataType(org.tensorflow.framework.DataType dataType) {
switch (dataType) {
case DT_UINT16:
return DataType.UINT16;
case DT_UINT32:
return DataType.UINT32;
case DT_UINT64:
return DataType.UINT64;
case DT_BOOL:
return DataType.BOOL;
case DT_BFLOAT16:
return DataType.BFLOAT16;
case DT_FLOAT:
return DataType.FLOAT;
case DT_INT32:
return DataType.INT;
case DT_INT64:
return DataType.LONG;
case DT_INT8:
return DataType.BYTE;
case DT_INT16:
return DataType.SHORT;
case DT_DOUBLE:
return DataType.DOUBLE;
case DT_UINT8:
return DataType.UBYTE;
case DT_HALF:
return DataType.HALF;
case DT_STRING:
return DataType.UTF8;
default:
throw new UnsupportedOperationException("Unknown TF data type: [" + dataType.name() + "]");
}
}
Example 8
| Source File: SDValidation.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
protected static void validateNumerical(String opName, String inputName, SDVariable[] vars) {
for (SDVariable v : vars) {
if (v == null) continue;
if (v.dataType() == DataType.BOOL || v.dataType() == DataType.UTF8)
throw new IllegalStateException("Input \"" + inputName + "\" for operation \"" + opName + "\" must be an numerical type type; got variable \"" +
v.name() + "\" with non-integer data type " + v.dataType());
}
}
Example 9
| Source File: SDValidation.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
/**
* Validate that the operation is being applied on a numerical SDVariable (not boolean or utf8).
* Some operations (such as sum, norm2, add(Number) etc don't make sense when applied to boolean/utf8 arrays
*
* @param opName Operation name to print in the exception
* @param v Variable to validate datatype for (input to operation)
*/
protected static void validateNumerical(String opName, String inputName, SDVariable v) {
if (v == null)
return;
if (v.dataType() == DataType.BOOL || v.dataType() == DataType.UTF8)
throw new IllegalStateException("Input \"" + inputName + "\" for operation \"" + opName + "\" must be an numerical type type; got variable \"" +
v.name() + "\" with non-integer data type " + v.dataType());
}
Example 10
| Source File: JsonSerdeTests.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Test
public void testNDArrayTextSerializer() throws Exception {
for(char order : new char[]{'c', 'f'}) {
Nd4j.factory().setOrder(order);
for (DataType globalDT : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF}) {
Nd4j.setDefaultDataTypes(globalDT, globalDT);
Nd4j.getRandom().setSeed(12345);
INDArray in = Nd4j.rand(DataType.DOUBLE, 3, 4).muli(20).subi(10);
val om = new ObjectMapper();
for (DataType dt : new DataType[]{DataType.DOUBLE, DataType.FLOAT, DataType.HALF, DataType.LONG, DataType.INT, DataType.SHORT,
DataType.BYTE, DataType.UBYTE, DataType.BOOL, DataType.UTF8}) {
INDArray arr;
if(dt == DataType.UTF8){
arr = Nd4j.create("aaaaa", "bbbb", "ccc", "dd", "e", "f", "g", "h", "i", "j", "k", "l").reshape('c', 3, 4);
} else {
arr = in.castTo(dt);
}
TestClass tc = new TestClass(arr);
String s = om.writeValueAsString(tc);
// System.out.println(dt);
// System.out.println(s);
// System.out.println("\n\n\n");
TestClass deserialized = om.readValue(s, TestClass.class);
assertEquals(dt.toString(), tc, deserialized);
}
}
}
}
Example 11
| Source File: ArrayCacheMemoryMgr.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Override
public void release(@NonNull INDArray array) {
//Check for multiple releases of the array
long id = array.getId();
Preconditions.checkState(!lruCache.contains(id), "Array was released multiple times: id=%s, shape=%ndShape", id, array);
DataType dt = array.dataType();
long thisBytes = array.data().length() * dt.width();
if(array.dataType() == DataType.UTF8) {
//Don't cache string arrays due to variable length buffers
if(array.closeable())
array.close();
} else if (currentCacheSize + thisBytes > maxCacheBytes) {
if(thisBytes > maxCacheBytes){
//Can't store even if we clear everything - too large
if(array.closeable())
array.close();
return;
}
//Need to deallocate some arrays to stay under limit - do in "oldest first" order
Iterator<Long> iter = lruCache.iterator();
while(currentCacheSize + thisBytes > maxCacheBytes){
long next = iter.next();
iter.remove();
INDArray nextOldest = lruCacheValues.remove(next);
DataType ndt = nextOldest.dataType();
long nextBytes = ndt.width() * nextOldest.data().length();
arrayStores.get(ndt).removeObject(nextOldest);
currentCacheSize -= nextBytes;
if(nextOldest.closeable())
nextOldest.close();
}
//After clearing space - can now cache
cacheArray(array);
} else {
//OK to cache
cacheArray(array);
}
//Store in LRU cache for "last used" removal if we exceed cache size
lruCache.add(array.getId());
lruCacheValues.put(array.getId(), array);
}
Example 12
| Source File: BaseCpuDataBuffer.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
public void actualizePointerAndIndexer() {
val cptr = ptrDataBuffer.primaryBuffer();
// skip update if pointers are equal
if (cptr != null && pointer != null && cptr.address() == pointer.address())
return;
val t = dataType();
if (t == DataType.BOOL) {
pointer = new PagedPointer(cptr, length).asBoolPointer();
setIndexer(BooleanIndexer.create((BooleanPointer) pointer));
} else if (t == DataType.UBYTE) {
pointer = new PagedPointer(cptr, length).asBytePointer();
setIndexer(UByteIndexer.create((BytePointer) pointer));
} else if (t == DataType.BYTE) {
pointer = new PagedPointer(cptr, length).asBytePointer();
setIndexer(ByteIndexer.create((BytePointer) pointer));
} else if (t == DataType.UINT16) {
pointer = new PagedPointer(cptr, length).asShortPointer();
setIndexer(UShortIndexer.create((ShortPointer) pointer));
} else if (t == DataType.SHORT) {
pointer = new PagedPointer(cptr, length).asShortPointer();
setIndexer(ShortIndexer.create((ShortPointer) pointer));
} else if (t == DataType.UINT32) {
pointer = new PagedPointer(cptr, length).asIntPointer();
setIndexer(UIntIndexer.create((IntPointer) pointer));
} else if (t == DataType.INT) {
pointer = new PagedPointer(cptr, length).asIntPointer();
setIndexer(IntIndexer.create((IntPointer) pointer));
} else if (t == DataType.UINT64) {
pointer = new PagedPointer(cptr, length).asLongPointer();
setIndexer(LongIndexer.create((LongPointer) pointer));
} else if (t == DataType.LONG) {
pointer = new PagedPointer(cptr, length).asLongPointer();
setIndexer(LongIndexer.create((LongPointer) pointer));
} else if (t == DataType.BFLOAT16) {
pointer = new PagedPointer(cptr, length).asShortPointer();
setIndexer(Bfloat16Indexer.create((ShortPointer) pointer));
} else if (t == DataType.HALF) {
pointer = new PagedPointer(cptr, length).asShortPointer();
setIndexer(HalfIndexer.create((ShortPointer) pointer));
} else if (t == DataType.FLOAT) {
pointer = new PagedPointer(cptr, length).asFloatPointer();
setIndexer(FloatIndexer.create((FloatPointer) pointer));
} else if (t == DataType.DOUBLE) {
pointer = new PagedPointer(cptr, length).asDoublePointer();
setIndexer(DoubleIndexer.create((DoublePointer) pointer));
} else if (t == DataType.UTF8) {
pointer = new PagedPointer(cptr, length()).asBytePointer();
setIndexer(ByteIndexer.create((BytePointer) pointer));
} else
throw new IllegalArgumentException("Unknown datatype: " + dataType());
}
Example 13
| Source File: BaseCpuDataBuffer.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
/**
*
* @param length
* @param elementSize
*/
public BaseCpuDataBuffer(long length, int elementSize) {
if (length < 1)
throw new IllegalArgumentException("Length must be >= 1");
initTypeAndSize();
allocationMode = AllocUtil.getAllocationModeFromContext();
this.length = length;
this.underlyingLength = length;
this.elementSize = (byte) elementSize;
if (dataType() != DataType.UTF8)
ptrDataBuffer = OpaqueDataBuffer.allocateDataBuffer(length, dataType(), false);
if (dataType() == DataType.DOUBLE) {
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asDoublePointer();
indexer = DoubleIndexer.create((DoublePointer) pointer);
} else if (dataType() == DataType.FLOAT) {
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asFloatPointer();
setIndexer(FloatIndexer.create((FloatPointer) pointer));
} else if (dataType() == DataType.INT32) {
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asIntPointer();
setIndexer(IntIndexer.create((IntPointer) pointer));
} else if (dataType() == DataType.LONG) {
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asLongPointer();
setIndexer(LongIndexer.create((LongPointer) pointer));
} else if (dataType() == DataType.SHORT) {
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asShortPointer();
setIndexer(ShortIndexer.create((ShortPointer) pointer));
} else if (dataType() == DataType.BYTE) {
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asBytePointer();
setIndexer(ByteIndexer.create((BytePointer) pointer));
} else if (dataType() == DataType.UBYTE) {
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asBytePointer();
setIndexer(UByteIndexer.create((BytePointer) pointer));
} else if (dataType() == DataType.UTF8) {
ptrDataBuffer = OpaqueDataBuffer.allocateDataBuffer(length, INT8, false);
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asBytePointer();
setIndexer(ByteIndexer.create((BytePointer) pointer));
} else if(dataType() == DataType.FLOAT16){
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asShortPointer();
setIndexer(HalfIndexer.create((ShortPointer) pointer));
} else if(dataType() == DataType.BFLOAT16){
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asShortPointer();
setIndexer(Bfloat16Indexer.create((ShortPointer) pointer));
} else if(dataType() == DataType.BOOL){
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asBoolPointer();
setIndexer(BooleanIndexer.create((BooleanPointer) pointer));
} else if(dataType() == DataType.UINT16){
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asShortPointer();
setIndexer(UShortIndexer.create((ShortPointer) pointer));
} else if(dataType() == DataType.UINT32){
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asIntPointer();
setIndexer(UIntIndexer.create((IntPointer) pointer));
} else if (dataType() == DataType.UINT64) {
pointer = new PagedPointer(ptrDataBuffer.primaryBuffer(), length).asLongPointer();
setIndexer(LongIndexer.create((LongPointer) pointer));
}
Nd4j.getDeallocatorService().pickObject(this);
}
Example 14
| Source File: Shape.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
public static boolean isS(@NonNull DataType x) {
return x == DataType.UTF8;
}
Example 15
| Source File: Utf8Buffer.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
/**
* Initialize the opType of this buffer
*/
@Override
protected void initTypeAndSize() {
elementSize = 1;
type = DataType.UTF8;
}
Example 16
| Source File: BaseCudaDataBuffer.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
public void actualizePointerAndIndexer() {
val cptr = ptrDataBuffer.primaryBuffer();
// skip update if pointers are equal
if (cptr != null && pointer != null && cptr.address() == pointer.address())
return;
val t = dataType();
if (t == DataType.BOOL) {
pointer = new PagedPointer(cptr, length).asBoolPointer();
setIndexer(BooleanIndexer.create((BooleanPointer) pointer));
} else if (t == DataType.UBYTE) {
pointer = new PagedPointer(cptr, length).asBytePointer();
setIndexer(UByteIndexer.create((BytePointer) pointer));
} else if (t == DataType.BYTE) {
pointer = new PagedPointer(cptr, length).asBytePointer();
setIndexer(ByteIndexer.create((BytePointer) pointer));
} else if (t == DataType.UINT16) {
pointer = new PagedPointer(cptr, length).asShortPointer();
setIndexer(UShortIndexer.create((ShortPointer) pointer));
} else if (t == DataType.SHORT) {
pointer = new PagedPointer(cptr, length).asShortPointer();
setIndexer(ShortIndexer.create((ShortPointer) pointer));
} else if (t == DataType.UINT32) {
pointer = new PagedPointer(cptr, length).asIntPointer();
setIndexer(UIntIndexer.create((IntPointer) pointer));
} else if (t == DataType.INT) {
pointer = new PagedPointer(cptr, length).asIntPointer();
setIndexer(IntIndexer.create((IntPointer) pointer));
} else if (t == DataType.UINT64) {
pointer = new PagedPointer(cptr, length).asLongPointer();
setIndexer(LongIndexer.create((LongPointer) pointer));
} else if (t == DataType.LONG) {
pointer = new PagedPointer(cptr, length).asLongPointer();
setIndexer(LongIndexer.create((LongPointer) pointer));
} else if (t == DataType.BFLOAT16) {
pointer = new PagedPointer(cptr, length).asShortPointer();
setIndexer(Bfloat16Indexer.create((ShortPointer) pointer));
} else if (t == DataType.HALF) {
pointer = new PagedPointer(cptr, length).asShortPointer();
setIndexer(HalfIndexer.create((ShortPointer) pointer));
} else if (t == DataType.FLOAT) {
pointer = new PagedPointer(cptr, length).asFloatPointer();
setIndexer(FloatIndexer.create((FloatPointer) pointer));
} else if (t == DataType.DOUBLE) {
pointer = new PagedPointer(cptr, length).asDoublePointer();
setIndexer(DoubleIndexer.create((DoublePointer) pointer));
} else if (t == DataType.UTF8) {
pointer = new PagedPointer(cptr, length()).asBytePointer();
setIndexer(ByteIndexer.create((BytePointer) pointer));
} else
throw new IllegalArgumentException("Unknown datatype: " + dataType());
}
Example 17
| Source File: CudaUtf8Buffer.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
/**
* Initialize the opType of this buffer
*/
@Override
protected void initTypeAndSize() {
elementSize = 1;
type = DataType.UTF8;
}
Example 18
| Source File: SDValidation.java From deeplearning4j with Apache License 2.0 | 3 votes |
|
/**
* Validate that the operation is being applied on a numerical SDVariable (not boolean or utf8).
* Some operations (such as sum, norm2, add(Number) etc don't make sense when applied to boolean/utf8 arrays
*
* @param opName Operation name to print in the exception
* @param v Variable to perform operation on
*/
protected static void validateNumerical(String opName, SDVariable v) {
if (v == null)
return;
if (v.dataType() == DataType.BOOL || v.dataType() == DataType.UTF8)
throw new IllegalStateException("Cannot apply operation \"" + opName + "\" to variable \"" + v.name() + "\" with non-numerical data type " + v.dataType());
}
Example 19
| Source File: NDValidation.java From deeplearning4j with Apache License 2.0 | 3 votes |
|
/**
* Validate that the operation is being applied on a numerical INDArray (not boolean or utf8).
* Some operations (such as sum, norm2, add(Number) etc) don't make sense when applied to boolean/utf8 arrays
*
* @param opName Operation name to print in the exception
* @param v Variable to perform operation on
*/
public static void validateNumerical(String opName, INDArray v) {
if (v == null)
return;
if (v.dataType() == DataType.BOOL || v.dataType() == DataType.UTF8)
throw new IllegalStateException("Cannot apply operation \"" + opName + "\" to array with non-numerical data type " + v.dataType());
}
Example 20
| Source File: SDValidation.java From deeplearning4j with Apache License 2.0 | 2 votes |
|
/**
* Validate that the operation is being applied on numerical SDVariables (not boolean or utf8).
* Some operations (such as sum, norm2, add(Number) etc don't make sense when applied to boolean/utf8 arrays
*
* @param opName Operation name to print in the exception
* @param v1 Variable to validate datatype for (input to operation)
* @param v2 Variable to validate datatype for (input to operation)
*/
protected static void validateNumerical(String opName, SDVariable v1, SDVariable v2) {
if (v1.dataType() == DataType.BOOL || v1.dataType() == DataType.UTF8 || v2.dataType() == DataType.BOOL || v2.dataType() == DataType.UTF8)
throw new IllegalStateException("Cannot perform operation \"" + opName + "\" on variables \"" + v1.name() + "\" and \"" +
v2.name() + "\" if one or both variables are non-numerical: " + v1.dataType() + " and " + v2.dataType());
}
