com.fasterxml.jackson.databind.annotation.JsonDeserialize Scala Examples
The following examples show how to use com.fasterxml.jackson.databind.annotation.JsonDeserialize.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
Example 1
Source File: DeltaScan.scala From delta with Apache License 2.0 | 5 votes |
package org.apache.spark.sql.delta.stats import org.apache.spark.sql.delta.actions.AddFile import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.spark.sql.catalyst.expressions._ case class DeltaScan( version: Long, files: Seq[AddFile], total: DataSize, partition: DataSize, scanned: DataSize)( // Moved to separate argument list, to not be part of case class equals check - // expressions can differ by exprId or ordering, but as long as same files are scanned, the // PreparedDeltaFileIndex and HadoopFsRelation should be considered equal for reuse purposes. val partitionFilters: ExpressionSet, val dataFilters: ExpressionSet, val unusedFilters: ExpressionSet, val projection: AttributeSet) { def allFilters: ExpressionSet = partitionFilters ++ dataFilters ++ unusedFilters }
Example 2
Source File: Program.scala From fusion-data with Apache License 2.0 | 5 votes |
package mass.model.job import com.fasterxml.jackson.annotation.JsonValue import com.fasterxml.jackson.core.{ JsonGenerator, JsonParser } import com.fasterxml.jackson.databind.{ DeserializationContext, SerializerProvider } import com.fasterxml.jackson.databind.annotation.{ JsonDeserialize, JsonSerialize } import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.fasterxml.jackson.databind.ser.std.StdSerializer import helloscala.common.data.StringValueName @JsonSerialize(using = classOf[Program.EnumSer]) @JsonDeserialize(using = classOf[Program.EnumDeser]) sealed abstract class Program(@JsonValue val value: String, val name: String) { def toValueName: StringValueName = StringValueName(value, name) } object Program { case object SCALA extends Program("scala", "Scala") case object JAVA extends Program("java", "Java") case object PYTHON extends Program("python", "Python") case object SH extends Program("sh", "SH") case object SQL extends Program("sql", "SQL") case object JS extends Program("js", "Javascript") val values = Vector(SCALA, JAVA, PYTHON, SH, SQL, JS) def fromValue(value: String): Program = optionFromValue(value).getOrElse( throw new NoSuchElementException(s"Program.values by name not found, it is $value.")) def optionFromValue(value: String): Option[Program] = { val v = value.toLowerCase() values.find(_.value == v) } def fromName(name: String): Program = optionFromName(name).getOrElse(throw new NoSuchElementException(s"Program.values by name not found, it is $name.")) def optionFromName(name: String): Option[Program] = { val n = name.toLowerCase() values.find(_.name == n) } class EnumSer extends StdSerializer[Program](classOf[Program]) { override def serialize(value: Program, gen: JsonGenerator, provider: SerializerProvider): Unit = gen.writeString(value.value) } class EnumDeser extends StdDeserializer[Program](classOf[Program]) { override def deserialize(p: JsonParser, ctxt: DeserializationContext): Program = Program.fromValue(p.getValueAsString) } }
Example 3
Source File: TriggerType.scala From fusion-data with Apache License 2.0 | 5 votes |
package mass.model.job import com.fasterxml.jackson.core.{ JsonGenerator, JsonParser } import com.fasterxml.jackson.databind.annotation.{ JsonDeserialize, JsonSerialize } import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.fasterxml.jackson.databind.ser.std.StdSerializer import com.fasterxml.jackson.databind.{ DeserializationContext, SerializerProvider } import helloscala.common.util.{ IEnumTrait, IEnumTraitCompanion } @JsonSerialize(using = classOf[TriggerType.EnumSer]) @JsonDeserialize(using = classOf[TriggerType.EnumDeser]) sealed abstract class TriggerType extends IEnumTrait[String] { override val value: String = name } object TriggerType extends IEnumTraitCompanion[String] { self => override type Value = TriggerType case object SIMPLE extends TriggerType case object CRON extends TriggerType case object EVENT extends TriggerType override val values = Vector(CRON, EVENT, SIMPLE) override def optionFromValue(value: String): Option[TriggerType] = super.optionFromValue(value.toUpperCase()) class EnumSer extends StdSerializer[TriggerType](classOf[TriggerType]) { override def serialize(value: TriggerType, gen: JsonGenerator, provider: SerializerProvider): Unit = gen.writeString(value.value) } class EnumDeser extends StdDeserializer[TriggerType](classOf[TriggerType]) { override def deserialize(p: JsonParser, ctxt: DeserializationContext): TriggerType = TriggerType.fromValue(p.getValueAsString.toUpperCase()) } }
Example 4
Source File: FieldClassType.scala From NSDb with Apache License 2.0 | 5 votes |
package io.radicalbit.nsdb.common.protocol import com.fasterxml.jackson.core.{JsonGenerator, JsonParser} import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import com.fasterxml.jackson.databind.{DeserializationContext, SerializerProvider} import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.fasterxml.jackson.databind.ser.std.StdSerializer @JsonSerialize(using = classOf[FieldClassTypeJsonSerializer]) @JsonDeserialize(using = classOf[FieldClassTypeJsonDeserializer]) sealed trait FieldClassType case object TimestampFieldType extends FieldClassType case object ValueFieldType extends FieldClassType case object DimensionFieldType extends FieldClassType case object TagFieldType extends FieldClassType class FieldClassTypeJsonSerializer extends StdSerializer[FieldClassType](classOf[FieldClassType]) { override def serialize(value: FieldClassType, gen: JsonGenerator, provider: SerializerProvider): Unit = gen.writeString(value.toString) } class FieldClassTypeJsonDeserializer extends StdDeserializer[FieldClassType](classOf[FieldClassType]) { override def deserialize(p: JsonParser, ctxt: DeserializationContext): FieldClassType = { p.getText match { case "TimestampFieldType" => TimestampFieldType case "ValueFieldType" => ValueFieldType case "DimensionFieldType" => DimensionFieldType case "TagFieldType" => TagFieldType } } }
Example 5
Source File: SQLListener.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
package org.apache.spark.sql.execution.ui import com.fasterxml.jackson.databind.JavaType import com.fasterxml.jackson.databind.`type`.TypeFactory import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.databind.util.Converter import org.apache.spark.annotation.DeveloperApi import org.apache.spark.scheduler._ import org.apache.spark.sql.execution.SparkPlanInfo import org.apache.spark.sql.execution.metric._ @DeveloperApi case class SparkListenerSQLExecutionStart( executionId: Long, description: String, details: String, physicalPlanDescription: String, sparkPlanInfo: SparkPlanInfo, time: Long) extends SparkListenerEvent @DeveloperApi case class SparkListenerSQLExecutionEnd(executionId: Long, time: Long) extends SparkListenerEvent private class LongLongTupleConverter extends Converter[(Object, Object), (Long, Long)] { override def convert(in: (Object, Object)): (Long, Long) = { def toLong(a: Object): Long = a match { case i: java.lang.Integer => i.intValue() case l: java.lang.Long => l.longValue() } (toLong(in._1), toLong(in._2)) } override def getInputType(typeFactory: TypeFactory): JavaType = { val objectType = typeFactory.uncheckedSimpleType(classOf[Object]) typeFactory.constructSimpleType(classOf[(_, _)], classOf[(_, _)], Array(objectType, objectType)) } override def getOutputType(typeFactory: TypeFactory): JavaType = { val longType = typeFactory.uncheckedSimpleType(classOf[Long]) typeFactory.constructSimpleType(classOf[(_, _)], classOf[(_, _)], Array(longType, longType)) } }
Example 6
Source File: SQLAppStatusStore.scala From Spark-2.3.1 with Apache License 2.0 | 5 votes |
package org.apache.spark.sql.execution.ui import java.lang.{Long => JLong} import java.util.Date import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.spark.JobExecutionStatus import org.apache.spark.status.KVUtils.KVIndexParam import org.apache.spark.util.kvstore.{KVIndex, KVStore} class SparkPlanGraphNodeWrapper( val node: SparkPlanGraphNode, val cluster: SparkPlanGraphClusterWrapper) { def toSparkPlanGraphNode(): SparkPlanGraphNode = { assert(node == null ^ cluster == null, "One and only of of nore or cluster must be set.") if (node != null) node else cluster.toSparkPlanGraphCluster() } } case class SQLPlanMetric( name: String, accumulatorId: Long, metricType: String)
Example 7
Source File: WriteMode.scala From comet-data-pipeline with Apache License 2.0 | 5 votes |
package com.ebiznext.comet.schema.model import com.ebiznext.comet.schema.model.WriteMode.{APPEND, ERROR_IF_EXISTS, IGNORE, OVERWRITE} import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import com.fasterxml.jackson.databind.ser.std.ToStringSerializer import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer} import org.apache.spark.sql.SaveMode @JsonSerialize(using = classOf[ToStringSerializer]) @JsonDeserialize(using = classOf[WriteDeserializer]) sealed case class WriteMode(value: String) { override def toString: String = value def toSaveMode: SaveMode = { this match { case OVERWRITE => SaveMode.Overwrite case APPEND => SaveMode.Append case ERROR_IF_EXISTS => SaveMode.ErrorIfExists case IGNORE => SaveMode.Ignore case _ => throw new Exception("Should never happen") } } } object WriteMode { def fromString(value: String): WriteMode = { value.toUpperCase() match { case "OVERWRITE" => WriteMode.OVERWRITE case "APPEND" => WriteMode.APPEND case "ERROR_IF_EXISTS" => WriteMode.ERROR_IF_EXISTS case "IGNORE" => WriteMode.IGNORE case _ => throw new Exception(s"Invalid Write Mode try one of ${writes}") } } object OVERWRITE extends WriteMode("OVERWRITE") object APPEND extends WriteMode("APPEND") object ERROR_IF_EXISTS extends WriteMode("ERROR_IF_EXISTS") object IGNORE extends WriteMode("IGNORE") val writes: Set[WriteMode] = Set(OVERWRITE, APPEND, ERROR_IF_EXISTS, IGNORE) } class WriteDeserializer extends JsonDeserializer[WriteMode] { override def deserialize(jp: JsonParser, ctx: DeserializationContext): WriteMode = { val value = jp.readValueAs[String](classOf[String]) WriteMode.fromString(value) } }
Example 8
Source File: Trim.scala From comet-data-pipeline with Apache License 2.0 | 5 votes |
package com.ebiznext.comet.schema.model import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import com.fasterxml.jackson.databind.ser.std.ToStringSerializer import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer} @JsonSerialize(using = classOf[ToStringSerializer]) @JsonDeserialize(using = classOf[TrimDeserializer]) sealed case class Trim(value: String) { override def toString: String = value } object Trim { def fromString(value: String): Trim = { value.toUpperCase() match { case "LEFT" => Trim.LEFT case "RIGHT" => Trim.RIGHT case "BOTH" => Trim.BOTH case "NONE" => Trim.NONE } } object LEFT extends Trim("LEFT") object RIGHT extends Trim("RIGHT") object BOTH extends Trim("BOTH") object NONE extends Trim("NONE") val modes: Set[Trim] = Set(LEFT, RIGHT, BOTH, NONE) } class TrimDeserializer extends JsonDeserializer[Trim] { override def deserialize(jp: JsonParser, ctx: DeserializationContext): Trim = { val value = jp.readValueAs[String](classOf[String]) Trim.fromString(value) } }
Example 9
Source File: UserType.scala From comet-data-pipeline with Apache License 2.0 | 5 votes |
package com.ebiznext.comet.schema.model import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import com.fasterxml.jackson.databind.ser.std.ToStringSerializer import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer} @JsonSerialize(using = classOf[ToStringSerializer]) @JsonDeserialize(using = classOf[UserTypeDeserializer]) sealed case class UserType(value: String) { override def toString: String = value } object UserType { def fromString(value: String): UserType = { value.toUpperCase match { case "SA" => UserType.SA case "USER" => UserType.USER case "GROUP" => UserType.GROUP } } object SA extends UserType("SA") object USER extends UserType("USER") object GROUP extends UserType("GROUP") val formats: Set[UserType] = Set(SA, USER, GROUP) } class UserTypeDeserializer extends JsonDeserializer[UserType] { override def deserialize(jp: JsonParser, ctx: DeserializationContext): UserType = { val value = jp.readValueAs[String](classOf[String]) UserType.fromString(value) } }
Example 10
Source File: Partition.scala From comet-data-pipeline with Apache License 2.0 | 5 votes |
package com.ebiznext.comet.schema.model import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.databind.node.ArrayNode import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer, JsonNode} @JsonDeserialize(using = classOf[PartitionDeserializer]) case class Partition( sampling: Option[Double], attributes: Option[List[String]] ) { def getAttributes(): List[String] = attributes.getOrElse(Nil) def getSampling() = sampling.getOrElse(0.0) } class PartitionDeserializer extends JsonDeserializer[Partition] { override def deserialize(jp: JsonParser, ctx: DeserializationContext): Partition = { val node: JsonNode = jp.getCodec().readTree[JsonNode](jp) deserialize(node) } def deserialize(node: JsonNode): Partition = { def isNull(field: String): Boolean = node.get(field) == null || node.get(field).isNull val sampling = if (isNull("sampling")) 0.0 else node.get("sampling").asDouble() import scala.collection.JavaConverters._ val attributes = if (isNull("attributes")) None else Some( node .get("attributes") .asInstanceOf[ArrayNode] .elements .asScala .toList .map(_.asText()) ) Partition(Some(sampling), attributes) } }
Example 11
Source File: MetricType.scala From comet-data-pipeline with Apache License 2.0 | 5 votes |
package com.ebiznext.comet.schema.model import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import com.fasterxml.jackson.databind.ser.std.ToStringSerializer import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer} @JsonSerialize(using = classOf[ToStringSerializer]) @JsonDeserialize(using = classOf[MetricTypeDeserializer]) sealed case class MetricType(value: String) { override def toString: String = value } object MetricType { def fromString(value: String): MetricType = { value.toUpperCase() match { case "DISCRETE" => MetricType.DISCRETE case "CONTINUOUS" => MetricType.CONTINUOUS case "TEXT" => MetricType.TEXT case "NONE" => MetricType.NONE } } object DISCRETE extends MetricType("DISCRETE") object CONTINUOUS extends MetricType("CONTINUOUS") object TEXT extends MetricType("TEXT") object NONE extends MetricType("NONE") val metricTypes: Set[MetricType] = Set(NONE, DISCRETE, CONTINUOUS, TEXT) } class MetricTypeDeserializer extends JsonDeserializer[MetricType] { override def deserialize(jp: JsonParser, ctx: DeserializationContext): MetricType = { val value = jp.readValueAs[String](classOf[String]) MetricType.fromString(value) } }
Example 12
Source File: Mode.scala From comet-data-pipeline with Apache License 2.0 | 5 votes |
package com.ebiznext.comet.schema.model import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import com.fasterxml.jackson.databind.ser.std.ToStringSerializer import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer} @JsonSerialize(using = classOf[ToStringSerializer]) @JsonDeserialize(using = classOf[ModeDeserializer]) sealed case class Mode(value: String) { override def toString: String = value } object Mode { def fromString(value: String): Mode = { value.toUpperCase() match { case "FILE" => Mode.FILE case "STREAM" => Mode.STREAM case "FILE_AND_STREAM" => Mode.FILE_AND_STREAM } } object FILE extends Mode("FILE") object STREAM extends Mode("STREAM") object FILE_AND_STREAM extends Mode("FILE_AND_STREAM") val modes: Set[Mode] = Set(FILE, STREAM, FILE_AND_STREAM) } class ModeDeserializer extends JsonDeserializer[Mode] { override def deserialize(jp: JsonParser, ctx: DeserializationContext): Mode = { val value = jp.readValueAs[String](classOf[String]) Mode.fromString(value) } }
Example 13
Source File: IndexSink.scala From comet-data-pipeline with Apache License 2.0 | 5 votes |
package com.ebiznext.comet.schema.model import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import com.fasterxml.jackson.databind.ser.std.ToStringSerializer import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer} @JsonSerialize(using = classOf[ToStringSerializer]) @JsonDeserialize(using = classOf[IndexSinkDeserializer]) sealed case class IndexSink(value: String) { override def toString: String = value } object IndexSink { def fromString(value: String): IndexSink = { value.toUpperCase match { case "NONE" => IndexSink.None case "FS" => IndexSink.FS case "JDBC" => IndexSink.JDBC case "BQ" => IndexSink.BQ case "ES" => IndexSink.ES } } object None extends IndexSink("None") object FS extends IndexSink("FS") object BQ extends IndexSink("BQ") object ES extends IndexSink("ES") object JDBC extends IndexSink("JDBC") val sinks: Set[IndexSink] = Set(None, FS, BQ, ES, JDBC) } class IndexSinkDeserializer extends JsonDeserializer[IndexSink] { override def deserialize(jp: JsonParser, ctx: DeserializationContext): IndexSink = { val value = jp.readValueAs[String](classOf[String]) IndexSink.fromString(value) } }
Example 14
Source File: Format.scala From comet-data-pipeline with Apache License 2.0 | 5 votes |
package com.ebiznext.comet.schema.model import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import com.fasterxml.jackson.databind.ser.std.ToStringSerializer import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer} @JsonSerialize(using = classOf[ToStringSerializer]) @JsonDeserialize(using = classOf[FormatDeserializer]) sealed case class Format(value: String) { override def toString: String = value } object Format { def fromString(value: String): Format = { value.toUpperCase match { case "DSV" => Format.DSV case "POSITION" => Format.POSITION case "SIMPLE_JSON" => Format.SIMPLE_JSON case "JSON" | "ARRAY_JSON" => Format.JSON case "CHEW" => Format.CHEW } } object DSV extends Format("DSV") object POSITION extends Format("POSITION") object SIMPLE_JSON extends Format("SIMPLE_JSON") object JSON extends Format("JSON") object CHEW extends Format("CHEW") val formats: Set[Format] = Set(DSV, POSITION, SIMPLE_JSON, JSON, CHEW) } class FormatDeserializer extends JsonDeserializer[Format] { override def deserialize(jp: JsonParser, ctx: DeserializationContext): Format = { val value = jp.readValueAs[String](classOf[String]) Format.fromString(value) } }
Example 15
Source File: Stage.scala From comet-data-pipeline with Apache License 2.0 | 5 votes |
package com.ebiznext.comet.schema.model import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import com.fasterxml.jackson.databind.ser.std.ToStringSerializer import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer} @JsonSerialize(using = classOf[ToStringSerializer]) @JsonDeserialize(using = classOf[StageDeserializer]) sealed case class Stage(value: String) { override def toString: String = value } object Stage { def fromString(value: String): Stage = { value.toUpperCase() match { case "UNIT" => Stage.UNIT case "GLOBAL" => Stage.GLOBAL } } object UNIT extends Stage("UNIT") object GLOBAL extends Stage("GLOBAL") val stages: Set[Stage] = Set(UNIT, GLOBAL) } class StageDeserializer extends JsonDeserializer[Stage] { override def deserialize(jp: JsonParser, ctx: DeserializationContext): Stage = { val value = jp.readValueAs[String](classOf[String]) Stage.fromString(value) } }
Example 16
Source File: SQLAppStatusStore.scala From XSQL with Apache License 2.0 | 5 votes |
package org.apache.spark.sql.execution.ui import java.lang.{Long => JLong} import java.util.Date import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.databind.annotation.JsonDeserialize import org.apache.spark.JobExecutionStatus import org.apache.spark.status.KVUtils.KVIndexParam import org.apache.spark.util.kvstore.{KVIndex, KVStore} class SparkPlanGraphNodeWrapper( val node: SparkPlanGraphNode, val cluster: SparkPlanGraphClusterWrapper) { def toSparkPlanGraphNode(): SparkPlanGraphNode = { assert(node == null ^ cluster == null, "One and only of of nore or cluster must be set.") if (node != null) node else cluster.toSparkPlanGraphCluster() } } case class SQLPlanMetric( name: String, accumulatorId: Long, metricType: String)
Example 17
Source File: TestGenericTypedJsonDeserializer.scala From milan with Apache License 2.0 | 5 votes |
package com.amazon.milan.serialization import com.amazon.milan.typeutil.{TypeDescriptor, _} import com.fasterxml.jackson.annotation.JsonCreator import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import org.junit.Assert._ import org.junit.Test object TestGenericTypedJsonDeserializer { @JsonSerialize(using = classOf[Serializer]) @JsonDeserialize(using = classOf[Deserializer]) trait BaseClass[T] extends GenericTypeInfoProvider with SetGenericTypeInfo // Classes cannot have implicit parameters and still be deserializable by jackson, because any constructors will have a // second argument list - the implicit parameters - and jackson doesn't know how to handle that. @JsonSerialize @JsonDeserialize class DerivedClass[T](typeDesc: TypeDescriptor[T]) extends BaseClass[T] { @JsonCreator def this() { this(null) } private var genericArguments: List[TypeDescriptor[_]] = if (typeDesc == null) List() else List(typeDesc) override def getGenericArguments: List[TypeDescriptor[_]] = this.genericArguments override def setGenericArguments(genericArgs: List[TypeDescriptor[_]]): Unit = this.genericArguments = genericArgs override def equals(obj: Any): Boolean = this.genericArguments.equals(obj.asInstanceOf[DerivedClass[T]].genericArguments) } class Container(var x: BaseClass[_]) { override def equals(obj: Any): Boolean = this.x.equals(obj.asInstanceOf[Container].x) } class Serializer extends GenericTypedJsonSerializer[BaseClass[_]] class Deserializer extends GenericTypedJsonDeserializer[BaseClass[_]](name => s"com.amazon.milan.serialization.TestGenericTypedJsonDeserializer$$$name") case class GenericType[A, B, C](a: A, b: B, c: C) } import com.amazon.milan.serialization.TestGenericTypedJsonDeserializer._ @Test class TestGenericTypedJsonDeserializer { @Test def test_GenericTypedJsonDeserializer_WithGenericTypeAsTypeParameter(): Unit = { val original = new Container(new DerivedClass[GenericType[Int, Long, Double]](createTypeDescriptor[GenericType[Int, Long, Double]])) val json = MilanObjectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(original) val copy = MilanObjectMapper.readValue(json, classOf[Container]) assertEquals(original, copy) } }
Example 18
Source File: JsonDataOutputFormat.scala From milan with Apache License 2.0 | 5 votes |
package com.amazon.milan.dataformats import java.io.OutputStream import java.nio.charset.StandardCharsets import com.amazon.milan.HashUtil import com.amazon.milan.serialization.{JavaTypeFactory, MilanObjectMapper} import com.amazon.milan.typeutil.TypeDescriptor import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import scala.collection.JavaConverters._ @JsonSerialize @JsonDeserialize class JsonDataOutputFormat[T: TypeDescriptor] extends DataOutputFormat[T] { @transient private lazy val objectMapper = new MilanObjectMapper() @transient private lazy val javaType = new JavaTypeFactory(this.objectMapper.getTypeFactory).makeJavaType(this.recordTypeDescriptor) @transient private lazy val hashCodeValue = HashUtil.combineHashCodes(this.recordTypeDescriptor.hashCode()) @transient private lazy val writer = this.objectMapper.writerFor(this.javaType) @transient private lazy val newLine = "\n".getBytes(StandardCharsets.UTF_8) private var recordTypeDescriptor = implicitly[TypeDescriptor[T]] override def getGenericArguments: List[TypeDescriptor[_]] = List(implicitly[TypeDescriptor[T]]) override def setGenericArguments(genericArgs: List[TypeDescriptor[_]]): Unit = { this.recordTypeDescriptor = genericArgs.head.asInstanceOf[TypeDescriptor[T]] } override def writeValue(value: T, outputStream: OutputStream): Unit = { this.writer.writeValue(outputStream, value) outputStream.write(this.newLine) } override def writeValues(values: TraversableOnce[T], outputStream: OutputStream): Unit = { this.writer .withRootValueSeparator("\n") .writeValues(outputStream) .writeAll(values.toIterable.asJava) outputStream.write(this.newLine) } override def hashCode(): Int = this.hashCodeValue override def equals(obj: Any): Boolean = { obj match { case o: JsonDataOutputFormat[T] => this.recordTypeDescriptor.equals(o.recordTypeDescriptor) case _ => false } } }
Example 19
Source File: JsonDataInputFormat.scala From milan with Apache License 2.0 | 5 votes |
package com.amazon.milan.dataformats import java.io.InputStream import com.amazon.milan.HashUtil import com.amazon.milan.serialization.{DataFormatConfiguration, JavaTypeFactory, MilanObjectMapper} import com.amazon.milan.typeutil.TypeDescriptor import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import scala.collection.JavaConverters._ import scala.language.experimental.macros @JsonSerialize @JsonDeserialize class JsonDataInputFormat[T: TypeDescriptor](val config: DataFormatConfiguration) extends DataInputFormat[T] { @transient private lazy val objectMapper = new MilanObjectMapper(this.config) @transient private lazy val javaType = new JavaTypeFactory(this.objectMapper.getTypeFactory).makeJavaType(this.recordTypeDescriptor) @transient private lazy val hashCodeValue = HashUtil.combineHashCodes(this.recordTypeDescriptor.hashCode(), this.config.hashCode()) private var recordTypeDescriptor = implicitly[TypeDescriptor[T]] def this() { this(DataFormatConfiguration.default) } override def getGenericArguments: List[TypeDescriptor[_]] = List(implicitly[TypeDescriptor[T]]) override def setGenericArguments(genericArgs: List[TypeDescriptor[_]]): Unit = { this.recordTypeDescriptor = genericArgs.head.asInstanceOf[TypeDescriptor[T]] } override def readValue(bytes: Array[Byte], offset: Int, length: Int): Option[T] = { Some(this.objectMapper.readValue[T](bytes, offset, length, this.javaType)) } override def readValues(stream: InputStream): TraversableOnce[T] = { this.objectMapper.readerFor(this.javaType).readValues[T](stream).asScala } override def hashCode(): Int = this.hashCodeValue override def equals(obj: Any): Boolean = { obj match { case o: JsonDataInputFormat[T] => this.recordTypeDescriptor.equals(o.recordTypeDescriptor) && this.config.equals(o.config) case _ => false } } }
Example 20
Source File: KinesisDataSource.scala From milan with Apache License 2.0 | 5 votes |
package com.amazon.milan.application.sources import com.amazon.milan.application.DataSource import com.amazon.milan.dataformats.{DataInputFormat, JsonDataInputFormat} import com.amazon.milan.serialization.DataFormatConfiguration import com.amazon.milan.typeutil.TypeDescriptor import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} object KinesisDataSource { @JsonSerialize @JsonDeserialize class KinesisDataSource[T: TypeDescriptor](val streamName: String, val region: String, val dataFormat: DataInputFormat[T]) extends DataSource[T] { private var recordTypeDescriptor = implicitly[TypeDescriptor[T]] def this(streamName: String, region: String) { this(streamName, region, new JsonDataInputFormat[T](DataFormatConfiguration.default)) } override def getGenericArguments: List[TypeDescriptor[_]] = List(this.recordTypeDescriptor) override def setGenericArguments(genericArgs: List[TypeDescriptor[_]]): Unit = { this.recordTypeDescriptor = genericArgs.head.asInstanceOf[TypeDescriptor[T]] } }
Example 21
Source File: FileDataSource.scala From milan with Apache License 2.0 | 5 votes |
package com.amazon.milan.application.sources import com.amazon.milan.application.DataSource import com.amazon.milan.dataformats.{DataInputFormat, JsonDataInputFormat} import com.amazon.milan.serialization.DataFormatConfiguration import com.amazon.milan.typeutil.TypeDescriptor import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} case class Configuration(readMode: ReadMode.ReadMode) { def withReadMode(readMode: ReadMode.ReadMode): Configuration = Configuration(readMode) } object Configuration { val default: Configuration = { Configuration(ReadMode.Once) } } }
Example 22
Source File: S3DataSource.scala From milan with Apache License 2.0 | 5 votes |
package com.amazon.milan.application.sources import com.amazon.milan.dataformats.DataInputFormat import com.amazon.milan.typeutil.TypeDescriptor import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import org.apache.commons.lang.StringUtils @JsonSerialize @JsonDeserialize class S3DataSource[T: TypeDescriptor](bucket: String, key: String, dataFormat: DataInputFormat[T], configuration: FileDataSource.Configuration) extends FileDataSource[T](S3DataSource.getS3Url(bucket, key), dataFormat, configuration) { def this(bucket: String, key: String, dataFormat: DataInputFormat[T]) { this(bucket, key, dataFormat, FileDataSource.Configuration.default) } } object S3DataSource { def getS3Url(bucket: String, key: String): String = s"s3://${StringUtils.strip(bucket, "/")}/${StringUtils.strip(key, "/")}" }
Example 23
Source File: SingletonMemorySink.scala From milan with Apache License 2.0 | 5 votes |
package com.amazon.milan.application.sinks import java.time.{Duration, Instant} import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue} import java.util.function import com.amazon.milan.Id import com.amazon.milan.application.DataSink import com.amazon.milan.typeutil.TypeDescriptor import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize} import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.concurrent.TimeoutException object SingletonMemorySink { private val values = new ConcurrentHashMap[String, ArrayBuffer[MemorySinkRecord[_]]]() private val nextSeqNum = new mutable.HashMap[String, Int]() private val locks = new ConcurrentHashMap[String, Object]() private def makeCreateBufferFunction[T]: java.util.function.Function[String, ArrayBuffer[MemorySinkRecord[_]]] = new function.Function[String, ArrayBuffer[MemorySinkRecord[_]]] { override def apply(t: String): ArrayBuffer[MemorySinkRecord[_]] = (new ArrayBuffer[MemorySinkRecord[T]]()).asInstanceOf[ArrayBuffer[MemorySinkRecord[_]]] } private val createLocker = new java.util.function.Function[String, Object] { override def apply(t: String): AnyRef = new Object() } @JsonIgnore def getRecordCount: Int = SingletonMemorySink.getBuffer(this.sinkId).size @JsonIgnore def getValues: List[T] = { SingletonMemorySink.getBuffer[T](this.sinkId).map(_.value).toList } @JsonIgnore def getRecords: List[MemorySinkRecord[T]] = { SingletonMemorySink.getBuffer[T](this.sinkId).toList } def waitForItems(itemCount: Int, timeout: Duration = null): Unit = { val endTime = if (timeout == null) Instant.MAX else Instant.now().plus(timeout) while (SingletonMemorySink.getBuffer(this.sinkId).size < itemCount) { if (Instant.now().isAfter(endTime)) { throw new TimeoutException() } Thread.sleep(1) } } override def equals(obj: Any): Boolean = { obj match { case o: SingletonMemorySink[_] => this.sinkId.equals(o.sinkId) case _ => false } } } class MemorySinkRecord[T](val seqNum: String, val createdTime: Instant, val value: T) extends Serializable
Example 24
Source File: ParameterMixin.scala From swagger-check with MIT License | 5 votes |
package de.leanovate.swaggercheck.schema.jackson import com.fasterxml.jackson.annotation.{JsonCreator, JsonProperty} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import de.leanovate.swaggercheck.schema.model._ @JsonDeserialize(builder = classOf[ParameterBuilder]) trait ParameterMixin { } class ParameterBuilder @JsonCreator()( @JsonProperty("name") name: String, @JsonProperty("in") in: String, @JsonProperty("required") required: Option[Boolean] = None, @JsonProperty("type") schemaType: Option[String] = None, @JsonProperty("schema") schema: Option[Definition] = None, @JsonProperty("format") format: Option[String] = None, @JsonProperty("allowEmptyValue") allowEmptyValue: Option[Boolean] = None, @JsonProperty("items") items: Option[Definition] = None, @JsonProperty("maximum") maximum: Option[BigDecimal] = None, @JsonProperty("exclusiveMaximum") exclusiveMaximum: Option[Boolean] = None, @JsonProperty("minimum") minimum: Option[BigDecimal] = None, @JsonProperty("exclusiveMinimum") exclusiveMinimum: Option[Boolean] = None, @JsonProperty("maxLength") maxLength: Option[Int] = None, @JsonProperty("minLength") minLength: Option[Int] = None, @JsonProperty("pattern") pattern: Option[String] = None, @JsonProperty("maxItems") maxItems: Option[Int] = None, @JsonProperty("minItems") minItems: Option[Int] = None, @JsonProperty("uniqueItems") uniqueItems: Option[Boolean] = None, @JsonProperty("enum") enum: Option[List[String]] = None) { def build(): Parameter = { Parameter.build( name, in, required, schema, schemaType, format, allowEmptyValue, items, maximum, exclusiveMaximum, minimum, exclusiveMinimum, maxLength, minLength, pattern, maxItems, minItems, uniqueItems, enum ) } }
Example 25
Source File: DefinitionBuilder.scala From swagger-check with MIT License | 5 votes |
package de.leanovate.swaggercheck.schema.jackson import com.fasterxml.jackson.annotation.{JsonCreator, JsonProperty} import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.module.scala.DefaultScalaModule import de.leanovate.swaggercheck.schema.model._ @JsonDeserialize(builder = classOf[DefinitionBuilder]) trait DefinitionMixin { } class DefinitionBuilder @JsonCreator()( @JsonProperty("type") schemaType: Option[String] = None, @JsonProperty("allOf") allOf: Option[Seq[Definition]] = None, @JsonProperty("enum") enum: Option[List[String]] = None, @JsonProperty("exclusiveMinimum") exclusiveMinimum: Option[Boolean] = None, @JsonProperty("exclusiveMaximum") exclusiveMaximum: Option[Boolean] = None, @JsonProperty("format") format: Option[String] = None, @JsonProperty("items") items: Option[Definition] = None, @JsonProperty("minItems") minItems: Option[Int] = None, @JsonProperty("maxItems") maxItems: Option[Int] = None, @JsonProperty("minimum") minimum: Option[BigDecimal] = None, @JsonProperty("maximum") maximum: Option[BigDecimal] = None, @JsonProperty("minLength") minLength: Option[Int] = None, @JsonProperty("maxLength") maxLength: Option[Int] = None, @JsonProperty("oneOf") oneOf: Option[Seq[Definition]] = None, @JsonProperty("pattern") pattern: Option[String] = None, @JsonProperty("properties") properties: Option[Map[String, Definition]] = None, @JsonProperty("additionalProperties") additionalPropertiesNode: Option[JsonNode] = None, @JsonProperty("required") required: Option[Set[String]] = None, @JsonProperty("$ref") ref: Option[String] = None, @JsonProperty("uniqueItems") uniqueItems: Option[Boolean] = None) { val additionalProperties = additionalPropertiesNode.map { case node if node.isBoolean => Left(node.isBinary) case node => Right(DefinitionBuilder.mapper.treeToValue(node, classOf[Definition])) } def build(): Definition = { Definition.build( schemaType, allOf, enum, exclusiveMinimum, exclusiveMaximum, format, items, minItems, maxItems, minimum, maximum, minLength, maxLength, oneOf, pattern, properties, additionalProperties, required, ref, uniqueItems ) } } object DefinitionBuilder { val mapper = new ObjectMapper().registerModule(DefaultScalaModule).registerModule(JsonSchemaModule) }
Example 26
Source File: OperationParameter.scala From swagger-check with MIT License | 5 votes |
package de.leanovate.swaggercheck.schema import com.fasterxml.jackson.annotation.{JsonCreator, JsonProperty} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import de.leanovate.swaggercheck.SwaggerChecks import de.leanovate.swaggercheck.schema.Operation.RequestBuilder import de.leanovate.swaggercheck.schema.gen.GeneratableDefinition._ import de.leanovate.swaggercheck.schema.jackson.DefinitionBuilder import de.leanovate.swaggercheck.schema.model.Definition import org.scalacheck.Gen @JsonDeserialize(builder = classOf[OperationParameterBuilder]) case class OperationParameter( name: Option[String], in: String, required: Boolean, schema: Definition ) { def applyTo(context: SwaggerChecks, builder: RequestBuilder): RequestBuilder = (name, in) match { case (Some(paramName), "path") => builder.withPathParam(schema.generate(context) .map(value => Some(paramName -> value.asText(""))) .suchThat(_.get._2.length > 0)) case (Some(paramName), "query") if required => builder.withQueryParam(schema.generate(context).map(value => Some(paramName -> value.asText("")))) case (Some(paramName), "query") => builder.withQueryParam(Gen.option(schema.generate(context).map(value => paramName -> value.asText("")))) case (Some(headerName), "header") if required => builder.withHeader(schema.generate(context).map(value => Some(headerName -> value.asText("")))) case (Some(headerName), "header") => builder.withHeader(Gen.option(schema.generate(context).map(value => headerName -> value.asText("")))) case (_, "body") => builder.withBody(schema.generate(context)) case _ => builder } } class ReferenceOperationParameter(globalRef: String) extends OperationParameter(None, "", false, null) { require(globalRef.startsWith("#/parameters/"), "Global parameters references need to start with #/parameters/") val ref = globalRef.substring(13) override def applyTo(context: SwaggerChecks, builder: RequestBuilder): RequestBuilder = throw new IllegalStateException("Not resolved ReferenceParameter") } object ReferenceOperationParameter { def unapply(parameter: ReferenceOperationParameter): Option[String] = Some(parameter.ref) } class OperationParameterBuilder @JsonCreator()( @JsonProperty("name") name: Option[String], @JsonProperty("$ref") ref: Option[String], @JsonProperty("in") in: String, @JsonProperty("required") required: Option[Boolean], @JsonProperty("type") schemaType: Option[String], @JsonProperty("format") format: Option[String], @JsonProperty("schema") schema: Option[Definition], @JsonProperty("enum") enum: Option[List[String]] ) { def build(): OperationParameter = ref match { case Some(reference) => new ReferenceOperationParameter(reference) case None => OperationParameter( name, in, required.getOrElse(false), schema.getOrElse(new DefinitionBuilder(schemaType = schemaType, format = format, enum = enum).build())) } }
Example 27
Source File: SwaggerAPI.scala From swagger-check with MIT License | 5 votes |
package de.leanovate.swaggercheck.schema import java.io.InputStream import com.fasterxml.jackson.annotation.{JsonCreator, JsonProperty} import com.fasterxml.jackson.core.JsonFactory import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.databind.{DeserializationFeature, JsonNode, MappingJsonFactory, ObjectMapper} import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import com.fasterxml.jackson.module.scala.DefaultScalaModule import de.leanovate.swaggercheck.schema.jackson.JsonSchemaModule import de.leanovate.swaggercheck.schema.model.{Definition, Parameter} import scala.collection.JavaConverters._ import scala.io.Source @JsonDeserialize(builder = classOf[SwaggerAPIBuilder]) case class SwaggerAPI( basePath: Option[String], paths: Map[String, Map[String, Operation]], definitions: Map[String, Definition] ) object SwaggerAPI { val jsonMapper = objectMapper(new MappingJsonFactory()) val yamlMapper = objectMapper(new YAMLFactory()) def parse(jsonOrYaml: String): SwaggerAPI = { val mapper = if (jsonOrYaml.trim().startsWith("{")) jsonMapper else yamlMapper mapper.readValue(jsonOrYaml, classOf[SwaggerAPI]) } def parse(swaggerInput: InputStream): SwaggerAPI = { parse(Source.fromInputStream(swaggerInput).mkString) } def objectMapper(jsonFactory: JsonFactory): ObjectMapper = { val mapper = new ObjectMapper(jsonFactory) mapper.registerModule(DefaultScalaModule) mapper.registerModule(JsonSchemaModule) mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) mapper } } class SwaggerAPIBuilder @JsonCreator()( @JsonProperty("basePath") basePath: Option[String], @JsonProperty("consumes") consumes: Option[Seq[String]], @JsonProperty("produces") produces: Option[Seq[String]], @JsonProperty("paths") paths: Option[Map[String, JsonNode]], @JsonProperty("definitions") definitions: Option[Map[String, Definition]], @JsonProperty("parameters") globalParameters: Option[Map[String, Parameter]] ) { def build(): SwaggerAPI = { val defaultConsumes = consumes.map(_.toSet).getOrElse(Set.empty) val defaultProduces = produces.map(_.toSet).getOrElse(Set.empty) SwaggerAPI(basePath, paths.getOrElse(Map.empty).map { case (path, pathDefinition) => val defaultParameters = Option(pathDefinition.get("parameters")).map { node => node.iterator().asScala.map { element => SwaggerAPI.jsonMapper.treeToValue(element, classOf[OperationParameter]) }.toSeq }.getOrElse(Seq.empty) basePath.map(_ + path).getOrElse(path) -> pathDefinition.fields().asScala.filter(_.getKey != "parameters").map { entry => val operation = SwaggerAPI.jsonMapper.treeToValue(entry.getValue, classOf[Operation]) entry.getKey.toUpperCase -> operation.withDefaults(defaultParameters, defaultConsumes, defaultProduces).resolveGlobalParameters(globalParameters.getOrElse(Map())) }.toMap }, definitions.getOrElse(Map.empty)) } }
Example 28
Source File: OperationResponse.scala From swagger-check with MIT License | 5 votes |
package de.leanovate.swaggercheck.schema import com.fasterxml.jackson.annotation.{JsonCreator, JsonProperty} import com.fasterxml.jackson.databind.annotation.JsonDeserialize import de.leanovate.swaggercheck.SwaggerChecks import de.leanovate.swaggercheck.schema.model.{Definition, JsonPath, ValidationResult} import de.leanovate.swaggercheck.shrinkable.{CheckJsString, CheckJsValue} @JsonDeserialize(builder = classOf[OperationResponseBuilder]) case class OperationResponse( schema: Option[Definition], headers: Seq[(String, Definition)] ) { def verify(context: SwaggerChecks, requestHeader: Map[String, String], requestBody: String): ValidationResult = { val bodyVerify = schema.map { expected => expected.validate(context, JsonPath(), CheckJsValue.parse(requestBody)) }.getOrElse(ValidationResult.success) headers.foldLeft(bodyVerify) { case (result, (name, schema)) => result.combine( requestHeader.get(name.toLowerCase) .map(value => schema.validate[CheckJsValue](context, JsonPath(), CheckJsString.formatted(value))) .getOrElse(ValidationResult.success)) } } } class OperationResponseBuilder @JsonCreator()( @JsonProperty("schema") schema: Option[Definition], @JsonProperty("headers") headers: Option[Map[String, Definition]] ) { def build(): OperationResponse = OperationResponse(schema, headers.map(_.toSeq).getOrElse(Seq.empty)) }
Example 29
Source File: SQLListener.scala From XSQL with Apache License 2.0 | 5 votes |
package org.apache.spark.sql.execution.ui import com.fasterxml.jackson.databind.JavaType import com.fasterxml.jackson.databind.`type`.TypeFactory import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.databind.util.Converter import org.apache.spark.annotation.DeveloperApi import org.apache.spark.scheduler._ import org.apache.spark.sql.execution.SparkPlanInfo import org.apache.spark.sql.execution.metric._ @DeveloperApi case class SparkListenerSQLExecutionStart( executionId: Long, description: String, details: String, physicalPlanDescription: String, sparkPlanInfo: SparkPlanInfo, time: Long) extends SparkListenerEvent @DeveloperApi case class SparkListenerSQLExecutionEnd(executionId: Long, time: Long) extends SparkListenerEvent private class LongLongTupleConverter extends Converter[(Object, Object), (Long, Long)] { override def convert(in: (Object, Object)): (Long, Long) = { def toLong(a: Object): Long = a match { case i: java.lang.Integer => i.intValue() case l: java.lang.Long => l.longValue() } (toLong(in._1), toLong(in._2)) } override def getInputType(typeFactory: TypeFactory): JavaType = { val objectType = typeFactory.uncheckedSimpleType(classOf[Object]) typeFactory.constructSimpleType(classOf[(_, _)], classOf[(_, _)], Array(objectType, objectType)) } override def getOutputType(typeFactory: TypeFactory): JavaType = { val longType = typeFactory.uncheckedSimpleType(classOf[Long]) typeFactory.constructSimpleType(classOf[(_, _)], classOf[(_, _)], Array(longType, longType)) } }