spray.json.JsNull Scala Examples

package oharastream.ohara.agent

import oharastream.ohara.agent.ArgumentsBuilder.FileAppender
import oharastream.ohara.common.util.CommonUtils
import spray.json.{JsNull, JsNumber, JsString, JsValue}

import scala.collection.mutable


  def mainConfigFile(path: String): ArgumentsBuilder

  override def build: Seq[String]
}

object ArgumentsBuilder {
  trait FileAppender {
    private[this] val props                = mutable.Buffer[String]()
    def append(prop: Int): FileAppender    = append(prop.toString)
    def append(prop: String): FileAppender = append(Seq(prop))
    def append(props: Seq[String]): FileAppender = {
      this.props ++= props
      this
    }
    def append(key: String, value: Boolean): FileAppender = append(s"$key=$value")
    def append(key: String, value: Short): FileAppender   = append(s"$key=$value")
    def append(key: String, value: Int): FileAppender     = append(s"$key=$value")
    def append(key: String, value: String): FileAppender  = append(s"$key=$value")
    def append(key: String, value: JsValue): FileAppender = append(
      key,
      value match {
        case JsString(value) => value
        case JsNumber(value) => value.toString
        case JsNull          => throw new IllegalArgumentException(s"JsNull is not legal")
        case _               => value.toString()
      }
    )

    def done: ArgumentsBuilder = done(props.toSeq)

    protected def done(props: Seq[String]): ArgumentsBuilder
  }
  def apply(): ArgumentsBuilder = new ArgumentsBuilder {
    private[this] val files                  = mutable.Map[String, Seq[String]]()
    private[this] var mainConfigFile: String = _

    override def build: Seq[String] =
      if (CommonUtils.isEmpty(mainConfigFile))
        throw new IllegalArgumentException("you have to define the main configs")
      else
        // format: --file path=line0,line1 --file path1=line0,line1
        // NOTED: the path and props must be in different line. otherwise, k8s will merge them into single line and our
        // script will fail to parse the command-line arguments
        files.flatMap {
          case (path, props) => Seq("--file", s"$path=${props.mkString(",")}")
        }.toSeq ++ Seq("--config", mainConfigFile)

    override def file(path: String): FileAppender = (props: Seq[String]) => {
      this.files += (path -> props)
      this
    }

    override def mainConfigFile(path: String): ArgumentsBuilder = {
      this.mainConfigFile = CommonUtils.requireNonEmpty(path)
      this
    }
  }
} 

// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package com.daml.http
package util

import com.daml.ledger.api.v1.ledger_offset.LedgerOffset
import LedgerOffset.{LedgerBoundary, Value}
import Value.Boundary
import scalaz.Liskov.<~<
import spray.json.{JsNull, JsonWriter}

private[http] sealed abstract class BeginBookmark[+Off] extends Product with Serializable {
  def toLedgerApi(implicit ev: Off <~< domain.Offset): LedgerOffset =
    this match {
      case AbsoluteBookmark(offset) => domain.Offset.toLedgerApi(ev(offset))
      case LedgerBegin => LedgerOffset(Boundary(LedgerBoundary.LEDGER_BEGIN))
    }

  def toOption: Option[Off] = this match {
    case AbsoluteBookmark(offset) => Some(offset)
    case LedgerBegin => None
  }
}
private[http] final case class AbsoluteBookmark[+Off](offset: Off) extends BeginBookmark[Off]
private[http] case object LedgerBegin extends BeginBookmark[Nothing]

private[http] object BeginBookmark {
  implicit def jsonWriter[Off: JsonWriter]: JsonWriter[BeginBookmark[Off]] =
    (obj: BeginBookmark[Off]) => {
      val ev = implicitly[JsonWriter[Off]]
      obj match {
        case AbsoluteBookmark(offset) => ev.write(offset)
        case LedgerBegin => JsNull
      }
    }
} 

package sjsonnew
package support.spray

import spray.json.{ JsValue, JsNumber, JsString, JsNull, JsTrue, JsFalse, JsObject }
import org.specs2.mutable._
import scala.Right

class StandardFormatsSpec extends Specification with BasicJsonProtocol {
  case class Person(name: Option[String], value: Option[Int])
  implicit object PersonFormat extends JsonFormat[Person] {
    def write[J](x: Person, builder: Builder[J]): Unit = {
      builder.beginObject()
      builder.addField("name", x.name)
      builder.addField("value", x.value)
      builder.endObject()
    }
    def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Person =
      jsOpt match {
        case Some(js) =>
          unbuilder.beginObject(js)
          val name = unbuilder.readField[Option[String]]("name")
          val value = unbuilder.readField[Option[Int]]("value")
          unbuilder.endObject()
          Person(name, value)
        case None =>
          deserializationError("Expected JsObject but found None")
      }
  }

  "The optionFormat" should {
    "convert None to JsNull" in {
      Converter.toJsonUnsafe(None.asInstanceOf[Option[Int]]) mustEqual JsNull
    }
    "convert JsNull to None" in {
      Converter.fromJsonUnsafe[Option[Int]](JsNull) mustEqual None
    }
    "convert Some(Hello) to JsString(Hello)" in {
      Converter.toJsonUnsafe(Some("Hello").asInstanceOf[Option[String]]) mustEqual JsString("Hello")
    }
    "convert JsString(Hello) to Some(Hello)" in {
      Converter.fromJsonUnsafe[Option[String]](JsString("Hello")) mustEqual Some("Hello")
    }
    "omit None fields" in {
      Converter.toJsonUnsafe(Person(None, None)) mustEqual JsObject()
    }
  }

  "The eitherFormat" should {
    val a: Either[Int, String] = Left(42)
    val b: Either[Int, String] = Right("Hello")

    "convert the left side of an Either value to Json" in {
      Converter.toJsonUnsafe(a) mustEqual JsNumber(42)
    }
    "convert the right side of an Either value to Json" in {
      Converter.toJsonUnsafe(b) mustEqual JsString("Hello")
    }
    "convert the left side of an Either value from Json" in {
      Converter.fromJsonUnsafe[Either[Int, String]](JsNumber(42)) mustEqual Left(42)
    }
    "convert the right side of an Either value from Json" in {
      Converter.fromJsonUnsafe[Either[Int, String]](JsString("Hello")) mustEqual Right("Hello")
    }
  }
} 

package sjsonnew
package support.spray

import spray.json.{ JsValue, JsNumber, JsString, JsNull, JsTrue, JsFalse, JsObject }
import org.specs2.mutable._
import java.lang.{ Integer => JInteger, Long => JLong, Boolean => JBoolean,
  Float => JFloat, Double => JDouble, Byte => JByte, Short => JShort,
  Character => JCharacter }

class JavaPrimitiveFormatsSpec extends Specification with BasicJsonProtocol {
  "The JIntegerJsonFormat" should {
    "convert an JInteger to a JsNumber" in {
      Converter.toJsonUnsafe[JInteger](42: JInteger) mustEqual JsNumber(42)
    }
    "convert a JsNumber to an Int" in {
      Converter.fromJsonUnsafe[JInteger](JsNumber(42)) mustEqual (42: JInteger)
    }
  }

  "The JLongJsonFormat" should {
    "convert a JLong to a JsNumber" in {
      Converter.toJsonUnsafe[JLong](7563661897011259335L: JLong) mustEqual JsNumber(7563661897011259335L)
    }
    "convert a JsNumber to a JLong" in {
      Converter.fromJsonUnsafe[JLong](JsNumber(7563661897011259335L)) mustEqual (7563661897011259335L: JLong)
    }
  }

  "The JFloatJsonFormat" should {
    "convert a JFloat to a JsNumber" in {
      Converter.toJsonUnsafe[JFloat](4.2f: JFloat) mustEqual JsNumber(4.2f)
    }
    "convert a JsNumber to a JFloat" in {
      Converter.fromJsonUnsafe[JFloat](JsNumber(4.2f)) mustEqual (4.2f: JFloat)
    }
  }

  "The JDoubleJsonFormat" should {
    "convert a JDouble to a JsNumber" in {
      Converter.toJsonUnsafe[JDouble](4.2: JDouble) mustEqual JsNumber(4.2)
    }
    "convert a JsNumber to a JDouble" in {
      Converter.fromJsonUnsafe[JDouble](JsNumber(4.2)) mustEqual (4.2: JDouble)
    }
  }

  "The JByteJsonFormat" should {
    "convert a JByte to a JsNumber" in {
      Converter.toJsonUnsafe[JByte](42.toByte: JByte) mustEqual JsNumber(42)
    }
    "convert a JsNumber to a JByte" in {
      Converter.fromJsonUnsafe[JByte](JsNumber(42)) mustEqual (42.toByte: JByte)
    }
  }

  "The JShortJsonFormat" should {
    "convert a JShort to a JsNumber" in {
      Converter.toJsonUnsafe(42.toShort: JShort) mustEqual JsNumber(42)
    }
    "convert a JsNumber to a JShort" in {
      Converter.fromJsonUnsafe[JShort](JsNumber(42)) mustEqual (42.toShort: JShort)
    }
  }

  "The JBooleanJsonFormat" should {
    "convert true to a JsTrue" in { Converter.toJsonUnsafe[JBoolean](true: JBoolean) mustEqual JsTrue }
    "convert false to a JsFalse" in { Converter.toJsonUnsafe[JBoolean](false: JBoolean) mustEqual JsFalse }
    "convert a JsTrue to true" in { Converter.fromJsonUnsafe[JBoolean](JsTrue) mustEqual true }
    "convert a JsFalse to false" in { Converter.fromJsonUnsafe[JBoolean](JsFalse) mustEqual false }
  }

  "The JCharacterJsonFormat" should {
    "convert a JCharacter to a JsString" in {
      Converter.toJsonUnsafe[JCharacter]('c': JCharacter) mustEqual JsString("c")
    }
    "convert a JsString to a JCharacter" in {
      Converter.fromJsonUnsafe[JCharacter](JsString("c")) mustEqual ('c': JCharacter)
    }
  }
} 

package sjsonnew
package support.spray

import spray.json.{ JsValue, JsNumber, JsString, JsNull, JsTrue, JsFalse, JsObject }
import org.specs2.mutable._
import java.util.{ UUID, Optional }
import java.net.{ URI, URL }
import java.io.File

class JavaExtraFormatsSpec extends Specification with BasicJsonProtocol {
  case class Person(name: Optional[String], value: Optional[Int])
  implicit object PersonFormat extends JsonFormat[Person] {
    def write[J](x: Person, builder: Builder[J]): Unit = {
      builder.beginObject()
      builder.addField("name", x.name)
      builder.addField("value", x.value)
      builder.endObject()
    }
    def read[J](jsOpt: Option[J], unbuilder: Unbuilder[J]): Person =
      jsOpt match {
        case Some(js) =>
          unbuilder.beginObject(js)
          val name = unbuilder.readField[Optional[String]]("name")
          val value = unbuilder.readField[Optional[Int]]("value")
          unbuilder.endObject()
          Person(name, value)
        case None =>
          deserializationError("Expected JsObject but found None")
      }
  }

  "The uuidStringIso" should {
    val uuid = UUID.fromString("abc220ea-2a01-11e6-b67b-9e71128cae77")
    "convert a UUID to JsString" in {
      Converter.toJsonUnsafe(uuid) mustEqual JsString("abc220ea-2a01-11e6-b67b-9e71128cae77")
    }
    "convert the JsString back to the UUID" in {
      Converter.fromJsonUnsafe[UUID](JsString("abc220ea-2a01-11e6-b67b-9e71128cae77")) mustEqual uuid
    }
  }

  "The uriStringIso" should {
    val uri = new URI("http://localhost")
    "convert a URI to JsString" in {
      Converter.toJsonUnsafe(uri) mustEqual JsString("http://localhost")
    }
    "convert the JsString back to the URI" in {
      Converter.fromJsonUnsafe[URI](JsString("http://localhost")) mustEqual uri
    }
  }

  "The urlStringIso" should {
    val url = new URL("http://localhost")
    "convert a URL to JsString" in {
      Converter.toJsonUnsafe(url) mustEqual JsString("http://localhost")
    }
    "convert the JsString back to the URI" in {
      Converter.fromJsonUnsafe[URL](JsString("http://localhost")) mustEqual url
    }
  }

  "The fileStringIso" should {
    val f = new File("/tmp")
    val f2 = new File(new File("src"), "main")
    "convert a File to JsString" in {
      Converter.toJsonUnsafe(f) mustEqual JsString("file:///tmp/")
    }
    "convert a relative path to JsString" in {
      // https://tools.ietf.org/html/rfc3986#section-4.2
      Converter.toJsonUnsafe(f2) mustEqual JsString("src/main")
    }
    "convert the JsString back to the File" in {
      Converter.fromJsonUnsafe[File](JsString("file:///tmp/")) mustEqual f
    }
    "convert the JsString back to the relative path" in {
      Converter.fromJsonUnsafe[File](JsString("src/main")) mustEqual f2
    }
  }

  "The optionalFormat" should {
    "convert Optional.empty to JsNull" in {
      Converter.toJsonUnsafe(Optional.empty[Int]) mustEqual JsNull
    }
    "convert JsNull to None" in {
      Converter.fromJsonUnsafe[Optional[Int]](JsNull) mustEqual Optional.empty[Int]
    }
    "convert Some(Hello) to JsString(Hello)" in {
      Converter.toJsonUnsafe(Optional.of("Hello")) mustEqual JsString("Hello")
    }
    "convert JsString(Hello) to Some(Hello)" in {
      Converter.fromJsonUnsafe[Optional[String]](JsString("Hello")) mustEqual Optional.of("Hello")
    }
    "omit None fields" in {
      Converter.toJsonUnsafe(Person(Optional.empty[String], Optional.empty[Int])) mustEqual JsObject()
    }
  }
} 

package sjsonnew
package support.spray

import spray.json.{ JsValue, JsNumber, JsString, JsNull, JsTrue, JsFalse, JsArray }
import org.specs2.mutable._
import scala.Right

class TupleFormatsSpec extends Specification with BasicJsonProtocol {

  "The tuple1Format" should {
    "convert (42) to a JsNumber" in {
      Converter.toJsonUnsafe(Tuple1(42)) mustEqual JsArray(JsNumber(42))
    }
    "be able to convert a JsNumber to a Tuple1[Int]" in {
      Converter.fromJsonUnsafe[Tuple1[Int]](JsArray(JsNumber(42))) mustEqual Tuple1(42)
    }
  }

  "The tuple2Format" should {
    val json = JsArray(JsNumber(42), JsNumber(4.2))
    "convert (42, 4.2) to a JsArray" in {
      Converter.toJsonUnsafe((42, 4.2)) mustEqual json
    }
    "be able to convert a JsArray to a (Int, Double)]" in {
      Converter.fromJsonUnsafe[(Int, Double)](json) mustEqual (42, 4.2)
    }
  }

  "The tuple3Format" should {
    val json = JsArray(JsNumber(42), JsNumber(4.2), JsString("hello"))
    "convert (42, 4.2, \"hello\") to a JsArray" in {
      Converter.toJsonUnsafe((42, 4.2, "hello")) mustEqual json
    }
    "be able to convert a JsArray to a (Int, Double, Int)]" in {
      Converter.fromJsonUnsafe[(Int, Double, String)](json) mustEqual (42, 4.2, "hello")
    }
  }

} 

package ai.deepsense.workflowexecutor.communication.mq.json

import java.nio.charset.StandardCharsets

import org.scalatest.mockito.MockitoSugar
import spray.json.{JsArray, JsNull, JsObject, JsString}
import ai.deepsense.commons.StandardSpec
import ai.deepsense.models.workflows.Workflow
import ai.deepsense.workflowexecutor.communication.message.global._
import ai.deepsense.workflowexecutor.communication.mq.json.Global.GlobalMQDeserializer

class GlobalMQDeserializerSpec
  extends StandardSpec
  with MockitoSugar {

  "GlobalMQDeserializer" should {
    "deserialize Launch messages" in {
      val workflowId = Workflow.Id.randomId
      val nodesToExecute = Vector(Workflow.Id.randomId, Workflow.Id.randomId, Workflow.Id.randomId)
      val jsNodesToExecute = JsArray(nodesToExecute.map(id => JsString(id.toString)))

      val rawMessage = JsObject(
        "messageType" -> JsString("launch"),
        "messageBody" -> JsObject(
          "workflowId" -> JsString(workflowId.toString),
          "nodesToExecute" -> jsNodesToExecute
        )
      )

      val readMessage: Any = serializeAndRead(rawMessage)
      readMessage shouldBe Launch(workflowId, nodesToExecute.toSet)
    }

    "deserialize Heartbeat messages" in {
      val workflowId = "foo-workflow"
      val rawMessage = JsObject(
        "messageType" -> JsString("heartbeat"),
        "messageBody" -> JsObject(
          "workflowId" -> JsString(workflowId),
          "sparkUiAddress" -> JsNull))
      serializeAndRead(rawMessage) shouldBe Heartbeat(workflowId, None)
    }
    "deserialize PoisonPill messages" in {
      val rawMessage = JsObject(
        "messageType" -> JsString("poisonPill"),
        "messageBody" -> JsObject())
      serializeAndRead(rawMessage) shouldBe PoisonPill()
    }
    "deserialize Ready messages" in {
      val sessionId = "foo-session"
      val rawMessage = JsObject(
        "messageType" -> JsString("ready"),
        "messageBody" -> JsObject(
          "sessionId" -> JsString(sessionId)))
      serializeAndRead(rawMessage) shouldBe Ready(sessionId)
    }
  }

  private def serializeAndRead(
    rawMessage: JsObject): Any = {
    val bytes = rawMessage.compactPrint.getBytes(StandardCharsets.UTF_8)
    GlobalMQDeserializer.deserializeMessage(bytes)
  }
} 

package ai.deepsense.deeplang.doperations

import scala.reflect.runtime.universe.TypeTag

import spray.json.{JsNull, JsValue}

import ai.deepsense.commons.utils.Version
import ai.deepsense.deeplang.DOperation.Id
import ai.deepsense.deeplang.documentation.OperationDocumentation
import ai.deepsense.deeplang.doperables.dataframe.DataFrame
import ai.deepsense.deeplang.doperables.{Evaluator, MetricValue}
import ai.deepsense.deeplang.doperations.exceptions.TooManyPossibleTypesException
import ai.deepsense.deeplang.doperations.layout.SmallBlockLayout2To1
import ai.deepsense.deeplang.inference.{InferContext, InferenceWarnings}
import ai.deepsense.deeplang.params.DynamicParam
import ai.deepsense.deeplang.{DKnowledge, DOperation2To1, ExecutionContext}
import ai.deepsense.models.json.graph.GraphJsonProtocol.GraphReader

case class Evaluate()
  extends DOperation2To1[Evaluator, DataFrame, MetricValue]
    with SmallBlockLayout2To1
    with OperationDocumentation {

  override val id: Id = "a88eaf35-9061-4714-b042-ddd2049ce917"
  override val name: String = "Evaluate"
  override val description: String =
    "Evaluates a DataFrame using an Evaluator"

  override val since: Version = Version(1, 0, 0)

  val evaluatorParams = new DynamicParam(
    name = "Parameters of input Evaluator",
    description = Some("These parameters are rendered dynamically, depending on type of Evaluator."),
    inputPort = 0)
  setDefault(evaluatorParams, JsNull)

  def getEvaluatorParams: JsValue = $(evaluatorParams)
  def setEvaluatorParams(jsValue: JsValue): this.type = set(evaluatorParams, jsValue)

  override val specificParams: Array[ai.deepsense.deeplang.params.Param[_]] = Array(evaluatorParams)

  override lazy val tTagTI_0: TypeTag[Evaluator] = typeTag
  override lazy val tTagTI_1: TypeTag[DataFrame] = typeTag
  override lazy val tTagTO_0: TypeTag[MetricValue] = typeTag

  override protected def execute(evaluator: Evaluator, dataFrame: DataFrame)(context: ExecutionContext): MetricValue = {
    evaluatorWithParams(evaluator, context.inferContext.graphReader).evaluate(context)(())(dataFrame)
  }

  override protected def inferKnowledge(
      evaluatorKnowledge: DKnowledge[Evaluator],
      dataFrameKnowledge: DKnowledge[DataFrame])(
      context: InferContext): (DKnowledge[MetricValue], InferenceWarnings) = {

    if (evaluatorKnowledge.size > 1) {
      throw TooManyPossibleTypesException()
    }
    val evaluator = evaluatorKnowledge.single
    evaluatorWithParams(evaluator, context.graphReader).evaluate.infer(context)(())(dataFrameKnowledge)
  }

  private def evaluatorWithParams(evaluator: Evaluator, graphReader: GraphReader): Evaluator = {
    val evaluatorWithParams = evaluator.replicate()
      .setParamsFromJson(getEvaluatorParams, graphReader, ignoreNulls = true)
    validateDynamicParams(evaluatorWithParams)
    evaluatorWithParams
  }

} 

package ai.deepsense.deeplang.doperations

import scala.reflect.runtime.universe.TypeTag

import spray.json.{JsNull, JsValue}

import ai.deepsense.commons.utils.Version
import ai.deepsense.deeplang.DOperation.Id
import ai.deepsense.deeplang.documentation.OperationDocumentation
import ai.deepsense.deeplang.doperables.Transformer
import ai.deepsense.deeplang.doperables.dataframe.DataFrame
import ai.deepsense.deeplang.doperations.layout.SmallBlockLayout2To1
import ai.deepsense.deeplang.inference.{InferContext, InferenceWarnings}
import ai.deepsense.deeplang.params.DynamicParam
import ai.deepsense.deeplang.{DKnowledge, DOperation2To1, ExecutionContext}
import ai.deepsense.models.json.graph.GraphJsonProtocol.GraphReader

case class Transform()
  extends DOperation2To1[Transformer, DataFrame, DataFrame]
    with SmallBlockLayout2To1
    with OperationDocumentation {

  override val id: Id = "643d8706-24db-4674-b5b4-10b5129251fc"
  override val name: String = "Transform"
  override val description: String =
    "Transforms a DataFrame using a Transformer"

  override val since: Version = Version(1, 0, 0)

  val transformerParams = new DynamicParam(
    name = "Parameters of input Transformer",
    description = Some("These parameters are rendered dynamically, depending on type of Transformer."),
    inputPort = 0)
  setDefault(transformerParams, JsNull)

  def getTransformerParams: JsValue = $(transformerParams)
  def setTransformerParams(jsValue: JsValue): this.type = set(transformerParams, jsValue)

  override val specificParams: Array[ai.deepsense.deeplang.params.Param[_]] = Array(transformerParams)

  override lazy val tTagTI_0: TypeTag[Transformer] = typeTag
  override lazy val tTagTI_1: TypeTag[DataFrame] = typeTag
  override lazy val tTagTO_0: TypeTag[DataFrame] = typeTag

  override protected def execute(
      transformer: Transformer,
      dataFrame: DataFrame)(
      context: ExecutionContext): DataFrame = {
    transformerWithParams(transformer, context.inferContext.graphReader).transform(context)(())(dataFrame)
  }

  override protected def inferKnowledge(
      transformerKnowledge: DKnowledge[Transformer],
      dataFrameKnowledge: DKnowledge[DataFrame])(
      context: InferContext): (DKnowledge[DataFrame], InferenceWarnings) = {

    if (transformerKnowledge.size > 1) {
      (DKnowledge(DataFrame.forInference()), InferenceWarnings.empty)
    } else {
      val transformer = transformerKnowledge.single
      transformerWithParams(transformer, context.graphReader).transform.infer(context)(())(dataFrameKnowledge)
    }
  }

  private def transformerWithParams(transformer: Transformer, graphReader: GraphReader): Transformer = {
    val transformerWithParams = transformer.replicate()
      .setParamsFromJson(getTransformerParams, graphReader, ignoreNulls = true)
    validateDynamicParams(transformerWithParams)
    transformerWithParams
  }
} 

package ai.deepsense.deeplang.doperations

import scala.reflect.runtime.universe.TypeTag

import spray.json.{JsNull, JsValue}

import ai.deepsense.commons.utils.Version
import ai.deepsense.deeplang.DOperation.Id
import ai.deepsense.deeplang.documentation.OperationDocumentation
import ai.deepsense.deeplang.doperables.dataframe.DataFrame
import ai.deepsense.deeplang.doperables.{Estimator, Transformer}
import ai.deepsense.deeplang.doperations.exceptions.TooManyPossibleTypesException
import ai.deepsense.deeplang.doperations.layout.SmallBlockLayout2To1
import ai.deepsense.deeplang.inference.{InferContext, InferenceWarnings}
import ai.deepsense.deeplang.params.DynamicParam
import ai.deepsense.deeplang.{DKnowledge, DOperation2To1, ExecutionContext}
import ai.deepsense.models.json.graph.GraphJsonProtocol.GraphReader

case class Fit()
  extends DOperation2To1[Estimator[Transformer], DataFrame, Transformer]
    with SmallBlockLayout2To1
    with OperationDocumentation {

  override val id: Id = "0c2ff818-977b-11e5-8994-feff819cdc9f"
  override val name: String = "Fit"
  override val description: String =
    "Fits an Estimator on a DataFrame"

  override val since: Version = Version(1, 0, 0)

  val estimatorParams = new DynamicParam(
    name = "Parameters of input Estimator",
    description = Some("These parameters are rendered dynamically, depending on type of Estimator."),
    inputPort = 0)
  setDefault(estimatorParams -> JsNull)

  def setEstimatorParams(jsValue: JsValue): this.type = set(estimatorParams -> jsValue)

  override val specificParams: Array[ai.deepsense.deeplang.params.Param[_]] = Array(estimatorParams)

  override lazy val tTagTI_0: TypeTag[Estimator[Transformer]] = typeTag
  override lazy val tTagTI_1: TypeTag[DataFrame] = typeTag
  override lazy val tTagTO_0: TypeTag[Transformer] = typeTag

  override protected def execute(
      estimator: Estimator[Transformer],
      dataFrame: DataFrame)(
      ctx: ExecutionContext): Transformer = {
    estimatorWithParams(estimator, ctx.inferContext.graphReader).fit(ctx)(())(dataFrame)
  }

  override protected def inferKnowledge(
      estimatorKnowledge: DKnowledge[Estimator[Transformer]],
      dataFrameKnowledge: DKnowledge[DataFrame])(
      ctx: InferContext)
    : (DKnowledge[Transformer], InferenceWarnings) = {

    if (estimatorKnowledge.size > 1) {
      throw TooManyPossibleTypesException()
    }
    val estimator = estimatorKnowledge.single
    estimatorWithParams(estimator, ctx.graphReader).fit.infer(ctx)(())(dataFrameKnowledge)
  }

  
  private def estimatorWithParams(estimator: Estimator[Transformer], graphReader: GraphReader):
  Estimator[Transformer] = {
    val estimatorWithParams = estimator.replicate()
      .setParamsFromJson($(estimatorParams), graphReader, ignoreNulls = true)
    validateDynamicParams(estimatorWithParams)
    estimatorWithParams
  }
} 

package com.qvantel.jsonapi

import org.specs2.ScalaCheck
import org.specs2.mutable.Specification
import spray.json.{JsArray, JsNull, JsObject}
import _root_.spray.json._
import _root_.spray.json.DefaultJsonProtocol._

class RelatedResponseSpec extends Specification with ScalaCheck {
  implicit val apiRoot: com.qvantel.jsonapi.ApiRoot = ApiRoot(None)

  @jsonApiResource final case class Test(id: String, name: String, age: Int)
  @jsonApiResource final case class Test2(id: String, name: String, age: Int)

  val test: Option[Test]      = Some(Test("teståöä•Ωé®", "someName", 20)) // test UTF-8
  val emptyTest: Option[Test] = None
  val tests: List[Test]       = List(Test("test 1", "someName1", 20), Test("test 2", "someName2", 21))
  val emptyTests: List[Test]  = List.empty

  def transformToTest2(in: Test): Test2 = Test2(in.id + "-2", in.name, in.age)

  "correctly write to one none case" in {
    RelatedResponse(emptyTest).toResponse must be equalTo JsObject(
      "data" -> JsNull
    )

    RelatedResponse(emptyTest).map(transformToTest2).toResponse must be equalTo JsObject(
      "data" -> JsNull
    )
  }

  "correctly write to one some case" in {
    val answer = rawOne(test.get)

    RelatedResponse(test).toResponse must be equalTo answer
    RelatedResponse(test.get).toResponse must be equalTo answer

    val transformedAnswer = rawOne(transformToTest2(test.get))
    RelatedResponse(test).map(transformToTest2).toResponse must be equalTo transformedAnswer
    RelatedResponse(test.get).map(transformToTest2).toResponse must be equalTo transformedAnswer
  }

  "correctly write to many empty case" in {
    RelatedResponse(emptyTests).toResponse must be equalTo JsObject(
      "data" -> JsArray.empty
    )

    RelatedResponse(emptyTests).map(transformToTest2).toResponse must be equalTo JsObject(
      "data" -> JsArray.empty
    )
  }

  "correctly write to many non-empty case" in {
    val answer = rawCollection(tests)

    RelatedResponse(tests).toResponse must be equalTo answer
    RelatedResponse(tests.toSeq).toResponse must be equalTo answer
    RelatedResponse(tests.toIterable).toResponse must be equalTo answer
    RelatedResponse(tests.toSet).toResponse must be equalTo answer

    val transformedAnswer = rawCollection(tests.map(transformToTest2))

    RelatedResponse(tests).map(transformToTest2).toResponse must be equalTo transformedAnswer
    RelatedResponse(tests.toSeq).map(transformToTest2).toResponse must be equalTo transformedAnswer
    RelatedResponse(tests.toIterable).map(transformToTest2).toResponse must be equalTo transformedAnswer
    RelatedResponse(tests.toSet).map(transformToTest2).toResponse must be equalTo transformedAnswer
  }

  "correctly write sparse fieldsets" in {
    implicit val sparseFields: Map[String, List[String]] = Map("tests" -> List("age"), "test2s" -> List("age"))
    val answer                                           = rawOne(test.get)

    RelatedResponse(test).toResponse must be equalTo answer
    RelatedResponse(test.get).toResponse must be equalTo answer

    val transformedAnswer = rawOne(transformToTest2(test.get))

    RelatedResponse(test).map(transformToTest2).toResponse must be equalTo transformedAnswer
    RelatedResponse(test.get).map(transformToTest2).toResponse must be equalTo transformedAnswer
  }
} 

package io.deepsense.deeplang.doperations

import scala.reflect.runtime.universe.TypeTag

import spray.json.{JsNull, JsValue}

import io.deepsense.commons.utils.Version
import io.deepsense.deeplang.DOperation.Id
import io.deepsense.deeplang.documentation.OperationDocumentation
import io.deepsense.deeplang.doperables.dataframe.DataFrame
import io.deepsense.deeplang.doperables.{Evaluator, MetricValue}
import io.deepsense.deeplang.doperations.exceptions.TooManyPossibleTypesException
import io.deepsense.deeplang.doperations.layout.SmallBlockLayout2To1
import io.deepsense.deeplang.inference.{InferContext, InferenceWarnings}
import io.deepsense.deeplang.params.DynamicParam
import io.deepsense.deeplang.{DKnowledge, DOperation2To1, ExecutionContext}

case class Evaluate()
  extends DOperation2To1[Evaluator, DataFrame, MetricValue]
    with SmallBlockLayout2To1
    with OperationDocumentation {

  override val id: Id = "a88eaf35-9061-4714-b042-ddd2049ce917"
  override val name: String = "Evaluate"
  override val description: String =
    "Evaluates a DataFrame using an Evaluator"

  override val since: Version = Version(1, 0, 0)

  val evaluatorParams = new DynamicParam(
    name = "Parameters of input Evaluator",
    description = Some("These parameters are rendered dynamically, depending on type of Evaluator."),
    inputPort = 0)
  setDefault(evaluatorParams, JsNull)

  def getEvaluatorParams: JsValue = $(evaluatorParams)
  def setEvaluatorParams(jsValue: JsValue): this.type = set(evaluatorParams, jsValue)

  override val params: Array[io.deepsense.deeplang.params.Param[_]] = Array(evaluatorParams)

  override lazy val tTagTI_0: TypeTag[Evaluator] = typeTag
  override lazy val tTagTI_1: TypeTag[DataFrame] = typeTag
  override lazy val tTagTO_0: TypeTag[MetricValue] = typeTag

  override protected def execute(evaluator: Evaluator, dataFrame: DataFrame)(context: ExecutionContext): MetricValue = {
    evaluatorWithParams(evaluator).evaluate(context)(())(dataFrame)
  }

  override protected def inferKnowledge(
      evaluatorKnowledge: DKnowledge[Evaluator],
      dataFrameKnowledge: DKnowledge[DataFrame])(
      context: InferContext): (DKnowledge[MetricValue], InferenceWarnings) = {

    if (evaluatorKnowledge.size > 1) {
      throw TooManyPossibleTypesException()
    }
    val evaluator = evaluatorKnowledge.single
    evaluatorWithParams(evaluator).evaluate.infer(context)(())(dataFrameKnowledge)
  }

  private def evaluatorWithParams(evaluator: Evaluator): Evaluator = {
    val evaluatorWithParams = evaluator.replicate()
      .setParamsFromJson(getEvaluatorParams, ignoreNulls = true)
    validateDynamicParams(evaluatorWithParams)
    evaluatorWithParams
  }

} 

package io.deepsense.deeplang.doperations

import scala.reflect.runtime.universe.TypeTag

import spray.json.{JsNull, JsValue}

import io.deepsense.commons.utils.Version
import io.deepsense.deeplang.DOperation.Id
import io.deepsense.deeplang.documentation.OperationDocumentation
import io.deepsense.deeplang.doperables.Transformer
import io.deepsense.deeplang.doperables.dataframe.DataFrame
import io.deepsense.deeplang.doperations.layout.SmallBlockLayout2To1
import io.deepsense.deeplang.inference.{InferContext, InferenceWarnings}
import io.deepsense.deeplang.params.DynamicParam
import io.deepsense.deeplang.{DKnowledge, DOperation2To1, ExecutionContext}

case class Transform()
  extends DOperation2To1[Transformer, DataFrame, DataFrame]
    with SmallBlockLayout2To1
    with OperationDocumentation {

  override val id: Id = "643d8706-24db-4674-b5b4-10b5129251fc"
  override val name: String = "Transform"
  override val description: String =
    "Transforms a DataFrame using a Transformer"

  override val since: Version = Version(1, 0, 0)

  val transformerParams = new DynamicParam(
    name = "Parameters of input Transformer",
    description = Some("These parameters are rendered dynamically, depending on type of Transformer."),
    inputPort = 0)
  setDefault(transformerParams, JsNull)

  def getTransformerParams: JsValue = $(transformerParams)
  def setTransformerParams(jsValue: JsValue): this.type = set(transformerParams, jsValue)

  override val params: Array[io.deepsense.deeplang.params.Param[_]] = Array(transformerParams)

  override lazy val tTagTI_0: TypeTag[Transformer] = typeTag
  override lazy val tTagTI_1: TypeTag[DataFrame] = typeTag
  override lazy val tTagTO_0: TypeTag[DataFrame] = typeTag

  override protected def execute(
      transformer: Transformer,
      dataFrame: DataFrame)(
      context: ExecutionContext): DataFrame = {
    transformerWithParams(transformer).transform(context)(())(dataFrame)
  }

  override protected def inferKnowledge(
      transformerKnowledge: DKnowledge[Transformer],
      dataFrameKnowledge: DKnowledge[DataFrame])(
      context: InferContext): (DKnowledge[DataFrame], InferenceWarnings) = {

    if (transformerKnowledge.size > 1) {
      (DKnowledge(DataFrame.forInference()), InferenceWarnings.empty)
    } else {
      val transformer = transformerKnowledge.single
      transformerWithParams(transformer).transform.infer(context)(())(dataFrameKnowledge)
    }
  }

  private def transformerWithParams(transformer: Transformer): Transformer = {
    val transformerWithParams = transformer.replicate()
      .setParamsFromJson(getTransformerParams, ignoreNulls = true)
    validateDynamicParams(transformerWithParams)
    transformerWithParams
  }
} 

package io.deepsense.deeplang.doperations

import scala.reflect.runtime.universe.TypeTag

import spray.json.{JsNull, JsValue}

import io.deepsense.commons.utils.Version
import io.deepsense.deeplang.DOperation.Id
import io.deepsense.deeplang.documentation.OperationDocumentation
import io.deepsense.deeplang.doperables.dataframe.DataFrame
import io.deepsense.deeplang.doperables.{Estimator, Transformer}
import io.deepsense.deeplang.doperations.exceptions.TooManyPossibleTypesException
import io.deepsense.deeplang.doperations.layout.SmallBlockLayout2To1
import io.deepsense.deeplang.inference.{InferContext, InferenceWarnings}
import io.deepsense.deeplang.params.DynamicParam
import io.deepsense.deeplang.{DKnowledge, DOperation2To1, ExecutionContext}

case class Fit()
  extends DOperation2To1[Estimator[Transformer], DataFrame, Transformer]
    with SmallBlockLayout2To1
    with OperationDocumentation {

  override val id: Id = "0c2ff818-977b-11e5-8994-feff819cdc9f"
  override val name: String = "Fit"
  override val description: String =
    "Fits an Estimator on a DataFrame"

  override val since: Version = Version(1, 0, 0)

  val estimatorParams = new DynamicParam(
    name = "Parameters of input Estimator",
    description = Some("These parameters are rendered dynamically, depending on type of Estimator."),
    inputPort = 0)
  setDefault(estimatorParams -> JsNull)

  def setEstimatorParams(jsValue: JsValue): this.type = set(estimatorParams -> jsValue)

  override val params: Array[io.deepsense.deeplang.params.Param[_]] = Array(estimatorParams)

  override lazy val tTagTI_0: TypeTag[Estimator[Transformer]] = typeTag
  override lazy val tTagTI_1: TypeTag[DataFrame] = typeTag
  override lazy val tTagTO_0: TypeTag[Transformer] = typeTag

  override protected def execute(
      estimator: Estimator[Transformer],
      dataFrame: DataFrame)(
      ctx: ExecutionContext): Transformer = {
    estimatorWithParams(estimator).fit(ctx)(())(dataFrame)
  }

  override protected def inferKnowledge(
      estimatorKnowledge: DKnowledge[Estimator[Transformer]],
      dataFrameKnowledge: DKnowledge[DataFrame])(
      ctx: InferContext)
    : (DKnowledge[Transformer], InferenceWarnings) = {

    if (estimatorKnowledge.size > 1) {
      throw TooManyPossibleTypesException()
    }
    val estimator = estimatorKnowledge.single
    estimatorWithParams(estimator).fit.infer(ctx)(())(dataFrameKnowledge)
  }

  
  private def estimatorWithParams(estimator: Estimator[Transformer]): Estimator[Transformer] = {
    val estimatorWithParams = estimator.replicate()
      .setParamsFromJson($(estimatorParams), ignoreNulls = true)
    validateDynamicParams(estimatorWithParams)
    estimatorWithParams
  }
}