Java Code Examples for org.apache.spark.sql.types.DataTypes#LongType
The following examples show how to use
org.apache.spark.sql.types.DataTypes#LongType .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: MLContextUtil.java From systemds with Apache License 2.0 | 7 votes |
|
/**
* Examine the DataFrame schema to determine whether the data appears to be
* a matrix.
*
* @param df
* the DataFrame
* @return {@code true} if the DataFrame appears to be a matrix,
* {@code false} otherwise
*/
public static boolean doesDataFrameLookLikeMatrix(Dataset<Row> df) {
StructType schema = df.schema();
StructField[] fields = schema.fields();
if (fields == null) {
return true;
}
for (StructField field : fields) {
DataType dataType = field.dataType();
if ((dataType != DataTypes.DoubleType) && (dataType != DataTypes.IntegerType)
&& (dataType != DataTypes.LongType) && (!(dataType instanceof org.apache.spark.ml.linalg.VectorUDT))
&& (!(dataType instanceof org.apache.spark.mllib.linalg.VectorUDT))) {
// uncomment if we support arrays of doubles for matrices
// if (dataType instanceof ArrayType) {
// ArrayType arrayType = (ArrayType) dataType;
// if (arrayType.elementType() == DataTypes.DoubleType) {
// continue;
// }
// }
return false;
}
}
return true;
}
Example 2
| Source File: LocalWithSparkSessionTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
private static DataType convertType(org.apache.kylin.metadata.datatype.DataType type) {
if (type.isTimeFamily())
return DataTypes.TimestampType;
if (type.isDateTimeFamily())
return DataTypes.DateType;
if (type.isIntegerFamily())
return DataTypes.LongType;
if (type.isNumberFamily())
return DataTypes.createDecimalType(19, 4);
if (type.isStringFamily())
return DataTypes.StringType;
if (type.isBoolean())
return DataTypes.BooleanType;
throw new IllegalArgumentException("KAP data type: " + type + " can not be converted to spark's type.");
}
Example 3
| Source File: CountDatasetRule.java From envelope with Apache License 2.0 | 6 votes |
|
@Override
public Dataset<Row> check(Dataset<Row> dataset, Map<String, Dataset<Row>> stepDependencies) {
if (isDependency()) {
Dataset<Row> expectedDependency = stepDependencies.get(dependency);
if (expectedDependency.count() == 1 && expectedDependency.schema().fields().length == 1
&& expectedDependency.schema().apply(0).dataType() == DataTypes.LongType) {
expected = expectedDependency.collectAsList().get(0).getLong(0);
} else {
throw new RuntimeException("Step dependency for count rule must have one row with a single field of long type");
}
}
if (expected < 0) {
throw new RuntimeException("Failed to determine expected count: must be specified either as literal or step dependency");
}
return dataset.groupBy().count().map(new CheckCount(expected, name), RowEncoder.apply(SCHEMA));
}
Example 4
| Source File: SchemaConverter.java From geowave with Apache License 2.0 | 6 votes |
|
private static AttributeDescriptor attrDescFromStructField(
final AttributeTypeBuilder attrBuilder,
final StructField field) {
if (field.name().equals("geom")) {
return attrBuilder.binding(Geometry.class).nillable(false).buildDescriptor("geom");
}
if (field.dataType() == DataTypes.StringType) {
return attrBuilder.binding(String.class).buildDescriptor(field.name());
} else if (field.dataType() == DataTypes.DoubleType) {
return attrBuilder.binding(Double.class).buildDescriptor(field.name());
} else if (field.dataType() == DataTypes.FloatType) {
return attrBuilder.binding(Float.class).buildDescriptor(field.name());
} else if (field.dataType() == DataTypes.LongType) {
return attrBuilder.binding(Long.class).buildDescriptor(field.name());
} else if (field.dataType() == DataTypes.IntegerType) {
return attrBuilder.binding(Integer.class).buildDescriptor(field.name());
} else if (field.dataType() == DataTypes.BooleanType) {
return attrBuilder.binding(Boolean.class).buildDescriptor(field.name());
} else if (field.dataType() == DataTypes.TimestampType) {
return attrBuilder.binding(Date.class).buildDescriptor(field.name());
}
return null;
}
Example 5
| Source File: DataFrames.java From DataVec with Apache License 2.0 | 6 votes |
|
/**
* Convert a datavec schema to a
* struct type in spark
*
* @param schema the schema to convert
* @return the datavec struct type
*/
public static StructType fromSchema(Schema schema) {
StructField[] structFields = new StructField[schema.numColumns()];
for (int i = 0; i < structFields.length; i++) {
switch (schema.getColumnTypes().get(i)) {
case Double:
structFields[i] = new StructField(schema.getName(i), DataTypes.DoubleType, false, Metadata.empty());
break;
case Integer:
structFields[i] =
new StructField(schema.getName(i), DataTypes.IntegerType, false, Metadata.empty());
break;
case Long:
structFields[i] = new StructField(schema.getName(i), DataTypes.LongType, false, Metadata.empty());
break;
case Float:
structFields[i] = new StructField(schema.getName(i), DataTypes.FloatType, false, Metadata.empty());
break;
default:
throw new IllegalStateException(
"This api should not be used with strings , binary data or ndarrays. This is only for columnar data");
}
}
return new StructType(structFields);
}
Example 6
| Source File: InstanceRelationWriter.java From rdf2x with Apache License 2.0 | 6 votes |
|
private DataType getDataType(int type) {
switch (type) {
case LiteralType.BOOLEAN:
return DataTypes.BooleanType;
case LiteralType.STRING:
return DataTypes.StringType;
case LiteralType.FLOAT:
return DataTypes.FloatType;
case LiteralType.DOUBLE:
return DataTypes.DoubleType;
case LiteralType.INTEGER:
return DataTypes.IntegerType;
case LiteralType.LONG:
return DataTypes.LongType;
case LiteralType.DATETIME:
// datetime not supported due to timezone issues with java.sql.Timestamp
// check the InstanceAggregator for more info
return DataTypes.StringType;
}
throw new NotImplementedException("Not able to write literal type " + type);
}
Example 7
| Source File: MLContextUtil.java From systemds with Apache License 2.0 | 6 votes |
|
/**
* Examine the DataFrame schema to determine whether the data appears to be
* a matrix.
*
* @param df
* the DataFrame
* @return {@code true} if the DataFrame appears to be a matrix,
* {@code false} otherwise
*/
public static boolean doesDataFrameLookLikeMatrix(Dataset<Row> df) {
StructType schema = df.schema();
StructField[] fields = schema.fields();
if (fields == null) {
return true;
}
for (StructField field : fields) {
DataType dataType = field.dataType();
if ((dataType != DataTypes.DoubleType) && (dataType != DataTypes.IntegerType)
&& (dataType != DataTypes.LongType) && (!(dataType instanceof org.apache.spark.ml.linalg.VectorUDT))
&& (!(dataType instanceof org.apache.spark.mllib.linalg.VectorUDT))) {
// uncomment if we support arrays of doubles for matrices
// if (dataType instanceof ArrayType) {
// ArrayType arrayType = (ArrayType) dataType;
// if (arrayType.elementType() == DataTypes.DoubleType) {
// continue;
// }
// }
return false;
}
}
return true;
}
Example 8
| Source File: TypeCastStep.java From bpmn.ai with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
private DataType mapDataType(List<StructField> datasetFields, String column, String typeConfig) {
DataType currentDatatype = getCurrentDataType(datasetFields, column);
// when typeConfig is null (no config for this column), return the current DataType
if(typeConfig == null) {
return currentDatatype;
}
switch (typeConfig) {
case "integer":
return DataTypes.IntegerType;
case "long":
return DataTypes.LongType;
case "double":
return DataTypes.DoubleType;
case "boolean":
return DataTypes.BooleanType;
case "date":
return DataTypes.DateType;
case "timestamp":
return DataTypes.TimestampType;
default:
return DataTypes.StringType;
}
}
Example 9
| Source File: DataFrames.java From deeplearning4j with Apache License 2.0 | 6 votes |
|
/**
* Convert a datavec schema to a
* struct type in spark
*
* @param schema the schema to convert
* @return the datavec struct type
*/
public static StructType fromSchema(Schema schema) {
StructField[] structFields = new StructField[schema.numColumns()];
for (int i = 0; i < structFields.length; i++) {
switch (schema.getColumnTypes().get(i)) {
case Double:
structFields[i] = new StructField(schema.getName(i), DataTypes.DoubleType, false, Metadata.empty());
break;
case Integer:
structFields[i] =
new StructField(schema.getName(i), DataTypes.IntegerType, false, Metadata.empty());
break;
case Long:
structFields[i] = new StructField(schema.getName(i), DataTypes.LongType, false, Metadata.empty());
break;
case Float:
structFields[i] = new StructField(schema.getName(i), DataTypes.FloatType, false, Metadata.empty());
break;
default:
throw new IllegalStateException(
"This api should not be used with strings , binary data or ndarrays. This is only for columnar data");
}
}
return new StructType(structFields);
}
Example 10
| Source File: DBClientWrapper.java From spark-data-sources with MIT License | 6 votes |
|
public static edb.common.Row sparkToDBRow(org.apache.spark.sql.Row row, StructType type) {
edb.common.Row dbRow = new edb.common.Row();
StructField[] fields = type.fields();
for (int i = 0; i < type.size(); i++) {
StructField sf = fields[i];
if (sf.dataType() == DataTypes.StringType) {
dbRow.addField(new edb.common.Row.StringField(sf.name(), row.getString(i)));
} else if (sf.dataType() == DataTypes.DoubleType) {
dbRow.addField(new edb.common.Row.DoubleField(sf.name(), row.getDouble(i)));
} else if (sf.dataType() == DataTypes.LongType) {
dbRow.addField(new edb.common.Row.Int64Field(sf.name(), row.getLong(i)));
} else {
// TODO: type leakage
}
}
return dbRow;
}
Example 11
| Source File: FrameRDDConverterUtils.java From systemds with Apache License 2.0 | 5 votes |
|
/**
* This function will convert Frame schema into DataFrame schema
*
* @param fschema frame schema
* @param containsID true if contains ID column
* @return Spark StructType of StructFields representing schema
*/
public static StructType convertFrameSchemaToDFSchema(ValueType[] fschema, boolean containsID)
{
// generate the schema based on the string of schema
List<StructField> fields = new ArrayList<>();
// add id column type
if( containsID )
fields.add(DataTypes.createStructField(RDDConverterUtils.DF_ID_COLUMN,
DataTypes.DoubleType, true));
// add remaining types
int col = 1;
for (ValueType schema : fschema) {
DataType dt = null;
switch(schema) {
case STRING: dt = DataTypes.StringType; break;
case FP64: dt = DataTypes.DoubleType; break;
case INT64: dt = DataTypes.LongType; break;
case BOOLEAN: dt = DataTypes.BooleanType; break;
default: dt = DataTypes.StringType;
LOG.warn("Using default type String for " + schema.toString());
}
fields.add(DataTypes.createStructField("C"+col++, dt, true));
}
return DataTypes.createStructType(fields);
}
Example 12
| Source File: DataFrames.java From DataVec with Apache License 2.0 | 5 votes |
|
/**
* Convert the DataVec sequence schema to a StructType for Spark, for example for use in
* {@link #toDataFrameSequence(Schema, JavaRDD)}}
* <b>Note</b>: as per {@link #toDataFrameSequence(Schema, JavaRDD)}}, the StructType has two additional columns added to it:<br>
* - Column 0: Sequence UUID (name: {@link #SEQUENCE_UUID_COLUMN}) - a UUID for the original sequence<br>
* - Column 1: Sequence index (name: {@link #SEQUENCE_INDEX_COLUMN} - an index (integer, starting at 0) for the position
* of this record in the original time series.<br>
* These two columns are required if the data is to be converted back into a sequence at a later point, for example
* using {@link #toRecordsSequence(DataRowsFacade)}
*
* @param schema Schema to convert
* @return StructType for the schema
*/
public static StructType fromSchemaSequence(Schema schema) {
StructField[] structFields = new StructField[schema.numColumns() + 2];
structFields[0] = new StructField(SEQUENCE_UUID_COLUMN, DataTypes.StringType, false, Metadata.empty());
structFields[1] = new StructField(SEQUENCE_INDEX_COLUMN, DataTypes.IntegerType, false, Metadata.empty());
for (int i = 0; i < schema.numColumns(); i++) {
switch (schema.getColumnTypes().get(i)) {
case Double:
structFields[i + 2] =
new StructField(schema.getName(i), DataTypes.DoubleType, false, Metadata.empty());
break;
case Integer:
structFields[i + 2] =
new StructField(schema.getName(i), DataTypes.IntegerType, false, Metadata.empty());
break;
case Long:
structFields[i + 2] =
new StructField(schema.getName(i), DataTypes.LongType, false, Metadata.empty());
break;
case Float:
structFields[i + 2] =
new StructField(schema.getName(i), DataTypes.FloatType, false, Metadata.empty());
break;
default:
throw new IllegalStateException(
"This api should not be used with strings , binary data or ndarrays. This is only for columnar data");
}
}
return new StructType(structFields);
}
Example 13
| Source File: SchemaConverter.java From geowave with Apache License 2.0 | 5 votes |
|
private static SimpleFeatureDataType attrDescToDataType(final AttributeDescriptor attrDesc) {
boolean isGeom = false;
DataType dataTypeOut = DataTypes.NullType;
if (attrDesc.getType().getBinding().equals(String.class)) {
dataTypeOut = DataTypes.StringType;
} else if (attrDesc.getType().getBinding().equals(Double.class)) {
dataTypeOut = DataTypes.DoubleType;
} else if (attrDesc.getType().getBinding().equals(Float.class)) {
dataTypeOut = DataTypes.FloatType;
} else if (attrDesc.getType().getBinding().equals(Long.class)) {
dataTypeOut = DataTypes.LongType;
} else if (attrDesc.getType().getBinding().equals(Integer.class)) {
dataTypeOut = DataTypes.IntegerType;
} else if (attrDesc.getType().getBinding().equals(Boolean.class)) {
dataTypeOut = DataTypes.BooleanType;
} else if (attrDesc.getType().getBinding().equals(Date.class)) {
dataTypeOut = DataTypes.TimestampType;
}
// Custom geometry types get WKB encoding
else if (Geometry.class.isAssignableFrom(attrDesc.getType().getBinding())) {
dataTypeOut = GeoWaveSpatialEncoders.geometryUDT;
isGeom = true;
}
return new SimpleFeatureDataType(dataTypeOut, isGeom);
}
Example 14
| Source File: IndexRUtil.java From indexr with Apache License 2.0 | 5 votes |
|
public static List<StructField> indexrSchemaToSparkSchema(SegmentSchema schema) {
List<StructField> fields = new ArrayList<>();
for (ColumnSchema cs : schema.getColumns()) {
DataType dataType;
switch (cs.getSqlType()) {
case INT:
dataType = DataTypes.IntegerType;
break;
case BIGINT:
dataType = DataTypes.LongType;
break;
case FLOAT:
dataType = DataTypes.FloatType;
break;
case DOUBLE:
dataType = DataTypes.DoubleType;
break;
case VARCHAR:
dataType = DataTypes.StringType;
break;
case DATE:
dataType = DataTypes.DateType;
break;
case DATETIME:
dataType = DataTypes.TimestampType;
break;
default:
throw new IllegalStateException("Unsupported type: " + cs.getSqlType());
}
fields.add(new StructField(cs.getName(), dataType, scala.Boolean.box(false), Metadata.empty()));
}
return fields;
}
Example 15
| Source File: DBClientWrapper.java From spark-data-sources with MIT License | 5 votes |
|
public static Schema sparkToDbSchema(StructType st) {
Schema schema = new Schema();
for (StructField sf: st.fields()) {
if (sf.dataType() == DataTypes.StringType) {
schema.addColumn(sf.name(), Schema.ColumnType.STRING);
} else if (sf.dataType() == DataTypes.DoubleType) {
schema.addColumn(sf.name(), Schema.ColumnType.DOUBLE);
} else if (sf.dataType() == DataTypes.LongType) {
schema.addColumn(sf.name(), Schema.ColumnType.INT64);
} else {
// TODO: type leakage
}
}
return schema;
}
Example 16
| Source File: FrameRDDConverterUtils.java From systemds with Apache License 2.0 | 5 votes |
|
/**
* This function will convert Frame schema into DataFrame schema
*
* @param fschema frame schema
* @param containsID true if contains ID column
* @return Spark StructType of StructFields representing schema
*/
public static StructType convertFrameSchemaToDFSchema(ValueType[] fschema, boolean containsID)
{
// generate the schema based on the string of schema
List<StructField> fields = new ArrayList<>();
// add id column type
if( containsID )
fields.add(DataTypes.createStructField(RDDConverterUtils.DF_ID_COLUMN,
DataTypes.DoubleType, true));
// add remaining types
int col = 1;
for (ValueType schema : fschema) {
DataType dt = null;
switch(schema) {
case STRING: dt = DataTypes.StringType; break;
case FP64: dt = DataTypes.DoubleType; break;
case INT64: dt = DataTypes.LongType; break;
case BOOLEAN: dt = DataTypes.BooleanType; break;
default: dt = DataTypes.StringType;
LOG.warn("Using default type String for " + schema.toString());
}
fields.add(DataTypes.createStructField("C"+col++, dt, true));
}
return DataTypes.createStructType(fields);
}
Example 17
| Source File: ConfigurationDataTypes.java From envelope with Apache License 2.0 | 4 votes |
|
public static DataType getSparkDataType(String typeString) {
DataType type;
String prec_scale_regex_groups = "\\s*(decimal)\\s*\\(\\s*(\\d+)\\s*,\\s*(\\d+)\\s*\\)\\s*";
Pattern prec_scale_regex_pattern = Pattern.compile(prec_scale_regex_groups);
Matcher prec_scale_regex_matcher = prec_scale_regex_pattern.matcher(typeString);
if (prec_scale_regex_matcher.matches()) {
int precision = Integer.parseInt(prec_scale_regex_matcher.group(2));
int scale = Integer.parseInt(prec_scale_regex_matcher.group(3));
type = DataTypes.createDecimalType(precision, scale);
}
else {
switch (typeString) {
case DECIMAL:
type = DataTypes.createDecimalType();
break;
case STRING:
type = DataTypes.StringType;
break;
case FLOAT:
type = DataTypes.FloatType;
break;
case DOUBLE:
type = DataTypes.DoubleType;
break;
case BYTE:
type = DataTypes.ByteType;
break;
case SHORT:
type = DataTypes.ShortType;
break;
case INT:
type = DataTypes.IntegerType;
break;
case LONG:
type = DataTypes.LongType;
break;
case BOOLEAN:
type = DataTypes.BooleanType;
break;
case BINARY:
type = DataTypes.BinaryType;
break;
case DATE:
type = DataTypes.DateType;
break;
case TIMESTAMP:
type = DataTypes.TimestampType;
break;
default:
throw new RuntimeException("Unsupported or unrecognized field type: " + typeString);
}
}
return type;
}
Example 18
| Source File: SQLHepler.java From sylph with Apache License 2.0 | 4 votes |
|
static DataType getSparkType(Type type)
{
if (type instanceof ParameterizedType && ((ParameterizedType) type).getRawType() == Map.class) {
Type[] arguments = ((ParameterizedType) type).getActualTypeArguments();
return DataTypes.createMapType(getSparkType(arguments[0]), getSparkType(arguments[1]));
}
else if (type instanceof ParameterizedType && ((ParameterizedType) type).getRawType() == List.class) {
DataType dataType = getSparkType(((ParameterizedType) type).getActualTypeArguments()[0]);
return DataTypes.createArrayType(dataType);
}
else {
if (type == String.class) {
return DataTypes.StringType;
}
else if (type == int.class || type == Integer.class) {
return DataTypes.IntegerType;
}
else if (type == long.class || type == Long.class) {
return DataTypes.LongType;
}
else if (type == boolean.class || type == Boolean.class) {
return DataTypes.BooleanType;
}
else if (type == double.class || type == Double.class) {
return DataTypes.DoubleType;
}
else if (type == float.class || type == Float.class) {
return DataTypes.FloatType;
}
else if (type == byte.class || type == Byte.class) {
return DataTypes.ByteType;
}
else if (type == Timestamp.class) {
return DataTypes.TimestampType;
}
else if (type == Date.class) {
return DataTypes.DateType;
}
else if (type == byte[].class || type == Byte[].class) {
return DataTypes.BinaryType;
}
else {
throw new IllegalArgumentException("this TYPE " + type + " have't support!");
}
}
}
Example 19
| Source File: SparkSturctTypeUtil.java From waterdrop with Apache License 2.0 | 4 votes |
|
private static DataType getType(String type) {
DataType dataType = DataTypes.NullType;
switch (type.toLowerCase()) {
case "string":
dataType = DataTypes.StringType;
break;
case "integer":
dataType = DataTypes.IntegerType;
break;
case "long":
dataType = DataTypes.LongType;
break;
case "double":
dataType = DataTypes.DoubleType;
break;
case "float":
dataType = DataTypes.FloatType;
break;
case "short":
dataType = DataTypes.ShortType;
break;
case "date":
dataType = DataTypes.DateType;
break;
case "timestamp":
dataType = DataTypes.TimestampType;
break;
case "boolean":
dataType = DataTypes.BooleanType;
break;
case "binary":
dataType = DataTypes.BinaryType;
break;
case "byte":
dataType = DataTypes.ByteType;
break;
default:
throw new ConfigRuntimeException("Throw data type exception, unknown type: " + type);
}
return dataType;
}
Example 20
| Source File: FlightDataSourceReader.java From flight-spark-source with Apache License 2.0 | 4 votes |
|
private DataType sparkFromArrow(FieldType fieldType) {
switch (fieldType.getType().getTypeID()) {
case Null:
return DataTypes.NullType;
case Struct:
throw new UnsupportedOperationException("have not implemented Struct type yet");
case List:
throw new UnsupportedOperationException("have not implemented List type yet");
case FixedSizeList:
throw new UnsupportedOperationException("have not implemented FixedSizeList type yet");
case Union:
throw new UnsupportedOperationException("have not implemented Union type yet");
case Int:
ArrowType.Int intType = (ArrowType.Int) fieldType.getType();
int bitWidth = intType.getBitWidth();
if (bitWidth == 8) {
return DataTypes.ByteType;
} else if (bitWidth == 16) {
return DataTypes.ShortType;
} else if (bitWidth == 32) {
return DataTypes.IntegerType;
} else if (bitWidth == 64) {
return DataTypes.LongType;
}
throw new UnsupportedOperationException("unknown int type with bitwidth " + bitWidth);
case FloatingPoint:
ArrowType.FloatingPoint floatType = (ArrowType.FloatingPoint) fieldType.getType();
FloatingPointPrecision precision = floatType.getPrecision();
switch (precision) {
case HALF:
case SINGLE:
return DataTypes.FloatType;
case DOUBLE:
return DataTypes.DoubleType;
}
case Utf8:
return DataTypes.StringType;
case Binary:
case FixedSizeBinary:
return DataTypes.BinaryType;
case Bool:
return DataTypes.BooleanType;
case Decimal:
throw new UnsupportedOperationException("have not implemented Decimal type yet");
case Date:
return DataTypes.DateType;
case Time:
return DataTypes.TimestampType; //note i don't know what this will do!
case Timestamp:
return DataTypes.TimestampType;
case Interval:
return DataTypes.CalendarIntervalType;
case NONE:
return DataTypes.NullType;
}
throw new IllegalStateException("Unexpected value: " + fieldType);
}
