Java Code Examples for org.datavec.api.transform.metadata.ColumnMetaData#getColumnType()
The following examples show how to use
org.datavec.api.transform.metadata.ColumnMetaData#getColumnType() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: CategoricalToIntegerTransform.java From DataVec with Apache License 2.0 | 6 votes |
|
@Override
public void setInputSchema(Schema inputSchema) {
super.setInputSchema(inputSchema);
columnIdx = inputSchema.getIndexOfColumn(columnName);
ColumnMetaData meta = inputSchema.getMetaData(columnName);
if (!(meta instanceof CategoricalMetaData))
throw new IllegalStateException("Cannot convert column \"" + columnName
+ "\" from categorical to one-hot: column is not categorical (is: " + meta.getColumnType()
+ ")");
this.stateNames = ((CategoricalMetaData) meta).getStateNames();
this.statesMap = new HashMap<>(stateNames.size());
for (int i = 0; i < stateNames.size(); i++) {
this.statesMap.put(stateNames.get(i), i);
}
}
Example 2
| Source File: CategoricalToOneHotTransform.java From DataVec with Apache License 2.0 | 6 votes |
|
@Override
public void setInputSchema(Schema inputSchema) {
super.setInputSchema(inputSchema);
columnIdx = inputSchema.getIndexOfColumn(columnName);
ColumnMetaData meta = inputSchema.getMetaData(columnName);
if (!(meta instanceof CategoricalMetaData))
throw new IllegalStateException("Cannot convert column \"" + columnName
+ "\" from categorical to one-hot: column is not categorical (is: " + meta.getColumnType()
+ ")");
this.stateNames = ((CategoricalMetaData) meta).getStateNames();
this.statesMap = new HashMap<>(stateNames.size());
for (int i = 0; i < stateNames.size(); i++) {
this.statesMap.put(stateNames.get(i), i);
}
}
Example 3
| Source File: CategoricalToIntegerTransform.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
@Override
public void setInputSchema(Schema inputSchema) {
super.setInputSchema(inputSchema);
columnIdx = inputSchema.getIndexOfColumn(columnName);
ColumnMetaData meta = inputSchema.getMetaData(columnName);
if (!(meta instanceof CategoricalMetaData))
throw new IllegalStateException("Cannot convert column \"" + columnName
+ "\" from categorical to one-hot: column is not categorical (is: " + meta.getColumnType()
+ ")");
this.stateNames = ((CategoricalMetaData) meta).getStateNames();
this.statesMap = new HashMap<>(stateNames.size());
for (int i = 0; i < stateNames.size(); i++) {
this.statesMap.put(stateNames.get(i), i);
}
}
Example 4
| Source File: CategoricalToOneHotTransform.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
@Override
public void setInputSchema(Schema inputSchema) {
super.setInputSchema(inputSchema);
columnIdx = inputSchema.getIndexOfColumn(columnName);
ColumnMetaData meta = inputSchema.getMetaData(columnName);
if (!(meta instanceof CategoricalMetaData))
throw new IllegalStateException("Cannot convert column \"" + columnName
+ "\" from categorical to one-hot: column is not categorical (is: " + meta.getColumnType()
+ ")");
this.stateNames = ((CategoricalMetaData) meta).getStateNames();
this.statesMap = new HashMap<>(stateNames.size());
for (int i = 0; i < stateNames.size(); i++) {
this.statesMap.put(stateNames.get(i), i);
}
}
Example 5
| Source File: IntegerToOneHotTransform.java From DataVec with Apache License 2.0 | 5 votes |
|
@Override
public void setInputSchema(Schema inputSchema) {
super.setInputSchema(inputSchema);
columnIdx = inputSchema.getIndexOfColumn(columnName);
ColumnMetaData meta = inputSchema.getMetaData(columnName);
if (!(meta instanceof IntegerMetaData))
throw new IllegalStateException("Cannot convert column \"" + columnName
+ "\" from integer to one-hot: column is not integer (is: " + meta.getColumnType() + ")");
}
Example 6
| Source File: StringListToCountsNDArrayTransform.java From DataVec with Apache License 2.0 | 5 votes |
|
@Override
public Schema transform(Schema inputSchema) {
int colIdx = inputSchema.getIndexOfColumn(columnName);
List<ColumnMetaData> oldMeta = inputSchema.getColumnMetaData();
List<ColumnMetaData> newMeta = new ArrayList<>();
List<String> oldNames = inputSchema.getColumnNames();
Iterator<ColumnMetaData> typesIter = oldMeta.iterator();
Iterator<String> namesIter = oldNames.iterator();
int i = 0;
while (typesIter.hasNext()) {
ColumnMetaData t = typesIter.next();
String name = namesIter.next();
if (i++ == colIdx) {
//Replace String column with a set of binary/integer columns
if (t.getColumnType() != ColumnType.String)
throw new IllegalStateException("Cannot convert non-string type");
ColumnMetaData meta = new NDArrayMetaData(newColumnName, new long[] {vocabulary.size()});
newMeta.add(meta);
} else {
newMeta.add(t);
}
}
return inputSchema.newSchema(newMeta);
}
Example 7
| Source File: StringListToCategoricalSetTransform.java From DataVec with Apache License 2.0 | 5 votes |
|
@Override
public Schema transform(Schema inputSchema) {
int colIdx = inputSchema.getIndexOfColumn(columnName);
List<ColumnMetaData> oldMeta = inputSchema.getColumnMetaData();
List<ColumnMetaData> newMeta = new ArrayList<>(oldMeta.size() + newColumnNames.size() - 1);
List<String> oldNames = inputSchema.getColumnNames();
Iterator<ColumnMetaData> typesIter = oldMeta.iterator();
Iterator<String> namesIter = oldNames.iterator();
int i = 0;
while (typesIter.hasNext()) {
ColumnMetaData t = typesIter.next();
String name = namesIter.next();
if (i++ == colIdx) {
//Replace String column with a set of binary/categorical columns
if (t.getColumnType() != ColumnType.String)
throw new IllegalStateException("Cannot convert non-string type");
for (int j = 0; j < newColumnNames.size(); j++) {
ColumnMetaData meta = new CategoricalMetaData(newColumnNames.get(j), "true", "false");
newMeta.add(meta);
}
} else {
newMeta.add(t);
}
}
return inputSchema.newSchema(newMeta);
}
Example 8
| Source File: IntegerToOneHotTransform.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Override
public void setInputSchema(Schema inputSchema) {
super.setInputSchema(inputSchema);
columnIdx = inputSchema.getIndexOfColumn(columnName);
ColumnMetaData meta = inputSchema.getMetaData(columnName);
if (!(meta instanceof IntegerMetaData))
throw new IllegalStateException("Cannot convert column \"" + columnName
+ "\" from integer to one-hot: column is not integer (is: " + meta.getColumnType() + ")");
}
Example 9
| Source File: StringListToCountsNDArrayTransform.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Override
public Schema transform(Schema inputSchema) {
int colIdx = inputSchema.getIndexOfColumn(columnName);
List<ColumnMetaData> oldMeta = inputSchema.getColumnMetaData();
List<ColumnMetaData> newMeta = new ArrayList<>();
List<String> oldNames = inputSchema.getColumnNames();
Iterator<ColumnMetaData> typesIter = oldMeta.iterator();
Iterator<String> namesIter = oldNames.iterator();
int i = 0;
while (typesIter.hasNext()) {
ColumnMetaData t = typesIter.next();
String name = namesIter.next();
if (i++ == colIdx) {
//Replace String column with a set of binary/integer columns
if (t.getColumnType() != ColumnType.String)
throw new IllegalStateException("Cannot convert non-string type");
ColumnMetaData meta = new NDArrayMetaData(newColumnName, new long[] {vocabulary.size()});
newMeta.add(meta);
} else {
newMeta.add(t);
}
}
return inputSchema.newSchema(newMeta);
}
Example 10
| Source File: StringListToCategoricalSetTransform.java From deeplearning4j with Apache License 2.0 | 5 votes |
|
@Override
public Schema transform(Schema inputSchema) {
int colIdx = inputSchema.getIndexOfColumn(columnName);
List<ColumnMetaData> oldMeta = inputSchema.getColumnMetaData();
List<ColumnMetaData> newMeta = new ArrayList<>(oldMeta.size() + newColumnNames.size() - 1);
List<String> oldNames = inputSchema.getColumnNames();
Iterator<ColumnMetaData> typesIter = oldMeta.iterator();
Iterator<String> namesIter = oldNames.iterator();
int i = 0;
while (typesIter.hasNext()) {
ColumnMetaData t = typesIter.next();
String name = namesIter.next();
if (i++ == colIdx) {
//Replace String column with a set of binary/categorical columns
if (t.getColumnType() != ColumnType.String)
throw new IllegalStateException("Cannot convert non-string type");
for (int j = 0; j < newColumnNames.size(); j++) {
ColumnMetaData meta = new CategoricalMetaData(newColumnNames.get(j), "true", "false");
newMeta.add(meta);
}
} else {
newMeta.add(t);
}
}
return inputSchema.newSchema(newMeta);
}
Example 11
| Source File: PivotTransform.java From DataVec with Apache License 2.0 | 4 votes |
|
@Override
public Schema transform(Schema inputSchema) {
if (!inputSchema.hasColumn(keyColumn) || !inputSchema.hasColumn(valueColumn)) {
throw new UnsupportedOperationException("Key or value column not found: " + keyColumn + ", " + valueColumn
+ " in " + inputSchema.getColumnNames());
}
List<String> origNames = inputSchema.getColumnNames();
List<ColumnMetaData> origMeta = inputSchema.getColumnMetaData();
int i = 0;
Iterator<String> namesIter = origNames.iterator();
Iterator<ColumnMetaData> typesIter = origMeta.iterator();
List<ColumnMetaData> newMeta = new ArrayList<>(inputSchema.numColumns());
int idxKey = inputSchema.getIndexOfColumn(keyColumn);
int idxValue = inputSchema.getIndexOfColumn(valueColumn);
ColumnMetaData valueMeta = inputSchema.getMetaData(idxValue);
while (namesIter.hasNext()) {
String s = namesIter.next();
ColumnMetaData t = typesIter.next();
if (i == idxKey) {
//Convert this to a set of separate columns
List<String> stateNames = ((CategoricalMetaData) inputSchema.getMetaData(idxKey)).getStateNames();
for (String stateName : stateNames) {
String newName = s + "[" + stateName + "]";
ColumnMetaData newValueMeta = valueMeta.clone();
newValueMeta.setName(newName);
newMeta.add(newValueMeta);
}
} else if (i == idxValue) {
i++;
continue; //Skip column
} else {
newMeta.add(t);
}
i++;
}
//Infer the default value if necessary
if (defaultValue == null) {
switch (valueMeta.getColumnType()) {
case String:
defaultValue = new Text("");
break;
case Integer:
defaultValue = new IntWritable(0);
break;
case Long:
defaultValue = new LongWritable(0);
break;
case Double:
defaultValue = new DoubleWritable(0.0);
break;
case Float:
defaultValue = new FloatWritable(0.0f);
break;
case Categorical:
defaultValue = new NullWritable();
break;
case Time:
defaultValue = new LongWritable(0);
break;
case Bytes:
throw new UnsupportedOperationException("Cannot infer default value for bytes");
case Boolean:
defaultValue = new Text("false");
break;
default:
throw new UnsupportedOperationException(
"Cannot infer default value for " + valueMeta.getColumnType());
}
}
return inputSchema.newSchema(newMeta);
}
Example 12
| Source File: PivotTransform.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
@Override
public Schema transform(Schema inputSchema) {
if (!inputSchema.hasColumn(keyColumn) || !inputSchema.hasColumn(valueColumn)) {
throw new UnsupportedOperationException("Key or value column not found: " + keyColumn + ", " + valueColumn
+ " in " + inputSchema.getColumnNames());
}
List<String> origNames = inputSchema.getColumnNames();
List<ColumnMetaData> origMeta = inputSchema.getColumnMetaData();
int i = 0;
Iterator<String> namesIter = origNames.iterator();
Iterator<ColumnMetaData> typesIter = origMeta.iterator();
List<ColumnMetaData> newMeta = new ArrayList<>(inputSchema.numColumns());
int idxKey = inputSchema.getIndexOfColumn(keyColumn);
int idxValue = inputSchema.getIndexOfColumn(valueColumn);
ColumnMetaData valueMeta = inputSchema.getMetaData(idxValue);
while (namesIter.hasNext()) {
String s = namesIter.next();
ColumnMetaData t = typesIter.next();
if (i == idxKey) {
//Convert this to a set of separate columns
List<String> stateNames = ((CategoricalMetaData) inputSchema.getMetaData(idxKey)).getStateNames();
for (String stateName : stateNames) {
String newName = s + "[" + stateName + "]";
ColumnMetaData newValueMeta = valueMeta.clone();
newValueMeta.setName(newName);
newMeta.add(newValueMeta);
}
} else if (i == idxValue) {
i++;
continue; //Skip column
} else {
newMeta.add(t);
}
i++;
}
//Infer the default value if necessary
if (defaultValue == null) {
switch (valueMeta.getColumnType()) {
case String:
defaultValue = new Text("");
break;
case Integer:
defaultValue = new IntWritable(0);
break;
case Long:
defaultValue = new LongWritable(0);
break;
case Double:
defaultValue = new DoubleWritable(0.0);
break;
case Float:
defaultValue = new FloatWritable(0.0f);
break;
case Categorical:
defaultValue = new NullWritable();
break;
case Time:
defaultValue = new LongWritable(0);
break;
case Bytes:
throw new UnsupportedOperationException("Cannot infer default value for bytes");
case Boolean:
defaultValue = new Text("false");
break;
default:
throw new UnsupportedOperationException(
"Cannot infer default value for " + valueMeta.getColumnType());
}
}
return inputSchema.newSchema(newMeta);
}
Example 13
| Source File: RecordConverter.java From deeplearning4j with Apache License 2.0 | 4 votes |
|
/**
* Convert a collection into a `List<Writable>`, i.e. a record that can be used with other datavec methods.
* Uses a schema to decide what kind of writable to use.
*
* @return a record
*/
public static List<Writable> toRecord(Schema schema, List<Object> source){
final List<Writable> record = new ArrayList<>(source.size());
final List<ColumnMetaData> columnMetaData = schema.getColumnMetaData();
if(columnMetaData.size() != source.size()){
throw new IllegalArgumentException("Schema and source list don't have the same length!");
}
for (int i = 0; i < columnMetaData.size(); i++) {
final ColumnMetaData metaData = columnMetaData.get(i);
final Object data = source.get(i);
if(!metaData.isValid(data)){
throw new IllegalArgumentException("Element "+i+": "+data+" is not valid for Column \""+metaData.getName()+"\" ("+metaData.getColumnType()+")");
}
try {
final Writable writable;
switch (metaData.getColumnType().getWritableType()){
case Float:
writable = new FloatWritable((Float) data);
break;
case Double:
writable = new DoubleWritable((Double) data);
break;
case Int:
writable = new IntWritable((Integer) data);
break;
case Byte:
writable = new ByteWritable((Byte) data);
break;
case Boolean:
writable = new BooleanWritable((Boolean) data);
break;
case Long:
writable = new LongWritable((Long) data);
break;
case Null:
writable = new NullWritable();
break;
case Bytes:
writable = new BytesWritable((byte[]) data);
break;
case NDArray:
writable = new NDArrayWritable((INDArray) data);
break;
case Text:
if(data instanceof String)
writable = new Text((String) data);
else if(data instanceof Text)
writable = new Text((Text) data);
else if(data instanceof byte[])
writable = new Text((byte[]) data);
else
throw new IllegalArgumentException("Element "+i+": "+data+" is not usable for Column \""+metaData.getName()+"\" ("+metaData.getColumnType()+")");
break;
default:
throw new IllegalArgumentException("Element "+i+": "+data+" is not usable for Column \""+metaData.getName()+"\" ("+metaData.getColumnType()+")");
}
record.add(writable);
} catch (ClassCastException e) {
throw new IllegalArgumentException("Element "+i+": "+data+" is not usable for Column \""+metaData.getName()+"\" ("+metaData.getColumnType()+")", e);
}
}
return record;
}
