com.fasterxml.jackson.databind.ObjectMapper Scala Examples

package com.basho.riak.spark.streaming

import java.nio.ByteBuffer
import java.util.concurrent.{Callable, Executors, TimeUnit}

import com.basho.riak.spark._
import com.basho.riak.spark.rdd.RiakTSTests
import com.basho.riak.spark.rdd.timeseries.{AbstractTimeSeriesTest, TimeSeriesData}
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper, SerializationFeature}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.apache.spark.sql.Row
import org.junit.Assert._
import org.junit.experimental.categories.Category
import org.junit.{After, Before, Test}

@Category(Array(classOf[RiakTSTests]))
class TsStreamingTest extends AbstractTimeSeriesTest(false) with SparkStreamingFixture {

  protected final val executorService = Executors.newCachedThreadPool()
  private val dataSource = new SocketStreamingDataSource
  private var port = -1

  @Before
  def setUp(): Unit = {
    port = dataSource.start(client => {
      testData
        .map(tolerantMapper.writeValueAsString)
        .foreach(x => client.write(ByteBuffer.wrap(s"$x\n".getBytes)))
      logInfo(s"${testData.length} values were send to client")
    })
  }

  @After
  def tearDown(): Unit = {
    dataSource.stop()
  }

  @Test(timeout = 10 * 1000) // 10 seconds timeout
  def saveToRiak(): Unit = {
    executorService.submit(new Runnable {
      override def run(): Unit = {
        ssc.socketTextStream("localhost", port)
          .map(string => {
            val tsdata = new ObjectMapper()
              .configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, true)
              .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true)
              .configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)
              .configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
              .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false)
              .registerModule(DefaultScalaModule)
              .readValue(string, classOf[TimeSeriesData])
            Row(1, "f", tsdata.time, tsdata.user_id, tsdata.temperature_k)
          })
          .saveToRiakTS(bucketName)

        ssc.start()
        ssc.awaitTerminationOrTimeout(5 * 1000)
      }
    })

    val result = executorService.submit(new Callable[Array[Seq[Any]]] {
      override def call(): Array[Seq[Any]] = {
        var rdd = sc.riakTSTable[Row](bucketName)
          .sql(s"SELECT user_id, temperature_k FROM $bucketName $sqlWhereClause")
        var count = rdd.count()
        while (count < testData.length) {
          TimeUnit.SECONDS.sleep(2)

          rdd = sc.riakTSTable[Row](bucketName)
            .sql(s"SELECT user_id, temperature_k FROM $bucketName $sqlWhereClause")
          count = rdd.count()
        }
        rdd.collect().map(_.toSeq)
      }
    }).get()

    assertEquals(testData.length, result.length)
    assertEqualsUsingJSONIgnoreOrder(
      """
        |[
        |   ['bryce',305.37],
        |   ['bryce',300.12],
        |   ['bryce',295.95],
        |   ['ratman',362.121],
        |   ['ratman',3502.212]
        |]
      """.stripMargin, result)
  }
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.nio.charset.StandardCharsets
import java.text.SimpleDateFormat
import java.util.{Calendar, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes(StandardCharsets.UTF_8))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'")
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package org.apache.spark.metrics.sink

import java.util.Properties
import java.util.concurrent.TimeUnit
import javax.servlet.http.HttpServletRequest

import com.codahale.metrics.MetricRegistry
import com.codahale.metrics.json.MetricsModule
import com.fasterxml.jackson.databind.ObjectMapper
import org.eclipse.jetty.servlet.ServletContextHandler

import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.ui.JettyUtils._

private[spark] class MetricsServlet(
    val property: Properties,
    val registry: MetricRegistry,
    securityMgr: SecurityManager)
  extends Sink {

  val SERVLET_KEY_PATH = "path"
  val SERVLET_KEY_SAMPLE = "sample"

  val SERVLET_DEFAULT_SAMPLE = false

  val servletPath = property.getProperty(SERVLET_KEY_PATH)

  val servletShowSample = Option(property.getProperty(SERVLET_KEY_SAMPLE)).map(_.toBoolean)
    .getOrElse(SERVLET_DEFAULT_SAMPLE)

  val mapper = new ObjectMapper().registerModule(
    new MetricsModule(TimeUnit.SECONDS, TimeUnit.MILLISECONDS, servletShowSample))

  def getHandlers(conf: SparkConf): Array[ServletContextHandler] = {
    Array[ServletContextHandler](
      createServletHandler(servletPath,
        new ServletParams(request => getMetricsSnapshot(request), "text/json"), securityMgr, conf)
    )
  }

  def getMetricsSnapshot(request: HttpServletRequest): String = {
    mapper.writeValueAsString(registry)
  }

  override def start() { }

  override def stop() { }

  override def report() { }
} 

package com.wavesplatform.dex.api.http.json

import java.io.IOException

import akka.http.scaladsl.model.MediaType
import akka.http.scaladsl.model.MediaTypes.`application/json`
import com.fasterxml.jackson.core.io.SegmentedStringWriter
import com.fasterxml.jackson.core.util.BufferRecyclers
import com.fasterxml.jackson.core.{JsonGenerator, JsonProcessingException}
import com.fasterxml.jackson.databind.module.SimpleModule
import com.fasterxml.jackson.databind.{JsonMappingException, JsonSerializer, ObjectMapper, SerializerProvider}
import play.api.libs.json._

object NumberAsStringSerializer extends JsonSerializer[JsValue] {

  private val fieldNamesToTranslate = Set(
    "amount",
    "available",
    "balance",
    "buyMatcherFee",
    "currentReward",
    "desiredReward",
    "effective",
    "fee",
    "feeAmount",
    "generating",
    "in",
    "matcherFee",
    "minIncrement",
    "minSponsoredAssetFee",
    "out",
    "price",
    "quantity",
    "regular",
    "reward",
    "sellMatcherFee",
    "sponsorBalance",
    "totalAmount",
    "totalFee",
    "totalWavesAmount",
    "value"
  )

  override def serialize(value: JsValue, json: JsonGenerator, provider: SerializerProvider): Unit = {
    value match {
      case JsNumber(v)  => json.writeNumber(v.bigDecimal)
      case JsString(v)  => json.writeString(v)
      case JsBoolean(v) => json.writeBoolean(v)

      case JsArray(elements) =>
        json.writeStartArray()
        elements.foreach { t =>
          serialize(t, json, provider)
        }
        json.writeEndArray()

      case JsObject(values) =>
        json.writeStartObject()
        values.foreach {
          case (name, JsNumber(v)) if fieldNamesToTranslate(name) =>
            json.writeStringField(name, v.bigDecimal.toPlainString)
          case (name, jsv) =>
            json.writeFieldName(name)
            serialize(jsv, json, provider)
        }
        json.writeEndObject()

      case JsNull => json.writeNull()
    }
  }
}

object CustomJson {

  val jsonWithNumbersAsStrings: MediaType.WithFixedCharset = `application/json`.withParams(Map("large-significand-format" -> "string"))

  private lazy val mapper = (new ObjectMapper)
    .registerModule(new SimpleModule("WavesJson").addSerializer(classOf[JsValue], NumberAsStringSerializer))
    .configure(JsonGenerator.Feature.WRITE_BIGDECIMAL_AS_PLAIN, true)

  def writeValueAsString(value: JsValue): String = {
    val sw = new SegmentedStringWriter(BufferRecyclers.getBufferRecycler)
    try mapper.writeValue(sw, value)
    catch {
      case e: JsonProcessingException =>
        throw e
      case e: IOException =>
        // shouldn't really happen, but is declared as possibility so:
        throw JsonMappingException.fromUnexpectedIOE(e)
    }
    sw.getAndClear
  }
} 

package com.wavesplatform.dex.api.http.entities

import com.fasterxml.jackson.core.{JsonGenerator, JsonParser}
import com.fasterxml.jackson.databind.annotation.JsonSerialize
import com.fasterxml.jackson.databind.deser.std.StdDeserializer
import com.fasterxml.jackson.databind.module.SimpleModule
import com.fasterxml.jackson.databind.ser.std.StdSerializer
import com.fasterxml.jackson.databind.{DeserializationContext, JsonNode, ObjectMapper, SerializerProvider}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
import com.wavesplatform.dex.domain.asset.Asset.{IssuedAsset, Waves}
import com.wavesplatform.dex.domain.asset.{Asset, AssetPair}
import com.wavesplatform.dex.domain.bytes.ByteStr
import com.wavesplatform.dex.domain.model.Denormalization
import com.wavesplatform.dex.model.LevelAgg

@JsonSerialize(using = classOf[HttpOrderBook.Serializer])
case class HttpOrderBook(timestamp: Long, pair: AssetPair, bids: Seq[LevelAgg], asks: Seq[LevelAgg], assetPairDecimals: Option[(Int, Int)] = None)

object HttpOrderBook {

  private val coreTypeSerializers = new SimpleModule()
  coreTypeSerializers.addDeserializer(classOf[AssetPair], new AssetPairDeserializer)

  private val mapper = new ObjectMapper() with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)
  mapper.registerModule(coreTypeSerializers)

  private def serialize(value: Any): String = mapper.writeValueAsString(value)

  private class AssetPairDeserializer extends StdDeserializer[AssetPair](classOf[AssetPair]) {

    override def deserialize(p: JsonParser, ctxt: DeserializationContext): AssetPair = {
      val node = p.getCodec.readTree[JsonNode](p)

      def readAssetId(fieldName: String): Asset = {
        val x = node.get(fieldName).asText(Asset.WavesName)
        if (x == Asset.WavesName) Waves else IssuedAsset(ByteStr.decodeBase58(x).get)
      }

      AssetPair(readAssetId("amountAsset"), readAssetId("priceAsset"))
    }
  }

  private def formatValue(value: BigDecimal, decimals: Int): String = new java.text.DecimalFormat(s"0.${"0" * decimals}").format(value)

  private def denormalizeAndSerializeSide(side: Seq[LevelAgg], amountAssetDecimals: Int, priceAssetDecimals: Int, jg: JsonGenerator): Unit = {
    side.foreach { levelAgg =>
      val denormalizedPrice  = Denormalization.denormalizePrice(levelAgg.price, amountAssetDecimals, priceAssetDecimals)
      val denormalizedAmount = Denormalization.denormalizeAmountAndFee(levelAgg.amount, amountAssetDecimals)

      jg.writeStartArray(2)
      jg.writeString(formatValue(denormalizedPrice, priceAssetDecimals))
      jg.writeString(formatValue(denormalizedAmount, amountAssetDecimals))
      jg.writeEndArray()
    }
  }

  def toJson(x: HttpOrderBook): String = serialize(x)

  class Serializer extends StdSerializer[HttpOrderBook](classOf[HttpOrderBook]) {
    override def serialize(x: HttpOrderBook, j: JsonGenerator, serializerProvider: SerializerProvider): Unit = {
      j.writeStartObject()
      j.writeNumberField("timestamp", x.timestamp)

      x.assetPairDecimals.fold {

        j.writeFieldName("pair")
        j.writeStartObject()
        j.writeStringField("amountAsset", x.pair.amountAssetStr)
        j.writeStringField("priceAsset", x.pair.priceAssetStr)
        j.writeEndObject()

        j.writeArrayFieldStart("bids")
        x.bids.foreach(j.writeObject)
        j.writeEndArray()

        j.writeArrayFieldStart("asks")
        x.asks.foreach(j.writeObject)
        j.writeEndArray()

      } {
        case (amountAssetDecimals, priceAssetDecimals) =>
          j.writeArrayFieldStart("bids")
          denormalizeAndSerializeSide(x.bids, amountAssetDecimals, priceAssetDecimals, j)
          j.writeEndArray()

          j.writeArrayFieldStart("asks")
          denormalizeAndSerializeSide(x.asks, amountAssetDecimals, priceAssetDecimals, j)
          j.writeEndArray()
      }

      j.writeEndObject()
    }
  }
} 

package no.nextgentel.oss.akkatools.serializing

import akka.actor.ActorSystem
import akka.serialization.SerializationExtension
import com.fasterxml.jackson.annotation.JsonTypeInfo
import com.fasterxml.jackson.databind.{SerializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.typesafe.config.{ConfigFactory, Config}
import org.scalatest.{Matchers, FunSuite}

class JacksonJsonSerializerTest extends FunSuite with Matchers {

  val objectMapper = new ObjectMapper()
  objectMapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false)
  objectMapper.registerModule(new DefaultScalaModule)

  test("serializer") {
    JacksonJsonSerializer.setObjectMapper(objectMapper)
    val serializer = new JacksonJsonSerializer()
    val a = Animal("our cat", 12, Cat("black", true))
    val bytes = serializer.toBinary(a)
    val ar = serializer.fromBinary(bytes, classOf[Animal]).asInstanceOf[Animal]
    assert( a == ar)
  }

  test("Registering the serializer works") {
    JacksonJsonSerializer.setObjectMapper(objectMapper)
    val system = ActorSystem("JacksonJsonSerializerTest", ConfigFactory.load("akka-tools-json-serializing.conf"))

    val serialization = SerializationExtension.get(system)
    assert( classOf[JacksonJsonSerializer] ==  serialization.serializerFor(classOf[Animal]).getClass)

    system.terminate()
  }

  test("DepricatedTypeWithMigrationInfo") {
    JacksonJsonSerializer.setObjectMapper(objectMapper)
    val serializer = new JacksonJsonSerializer()
    val bytes = serializer.toBinary(OldType("12"))
    assert(NewType(12) == serializer.fromBinary(bytes, classOf[OldType]))
  }

  test("verifySerialization - no error") {
    JacksonJsonSerializer.setObjectMapper(objectMapper)
    JacksonJsonSerializer.setVerifySerialization(true)
    val serializer = new JacksonJsonSerializer()
    val a = Animal("our cat", 12, Cat("black", true))
    val ow = ObjectWrapperWithTypeInfo(a)
    serializer.toBinary(ow)
  }

  test("verifySerialization - with error") {
    JacksonJsonSerializer.setObjectMapper(objectMapper)
    JacksonJsonSerializer.setVerifySerialization(true)
    val serializer = new JacksonJsonSerializer()
    val a = Animal("our cat", 12, Cat("black", true))
    val ow = ObjectWrapperWithoutTypeInfo(a)
    intercept[JacksonJsonSerializerVerificationFailed] {
      serializer.toBinary(ow)
    }
  }

  test("verifySerialization - disabled") {
    JacksonJsonSerializer.setObjectMapper(objectMapper)
    JacksonJsonSerializer.setVerifySerialization(true)
    val serializer = new JacksonJsonSerializer()
    val a = Animal("our cat", 12, Cat("black", true))
    val ow = ObjectWrapperWithoutTypeInfoOverrided(a)
    serializer.toBinary(ow)
  }



}

case class Animal(name:String, age:Int, t:Cat) extends JacksonJsonSerializable

case class Cat(color:String, tail:Boolean)

case class OldType(s:String) extends DepricatedTypeWithMigrationInfo {
  override def convertToMigratedType(): AnyRef = NewType(s.toInt)
}
case class NewType(i:Int)


case class ObjectWrapperWithTypeInfo(@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@any_class") any:AnyRef)

case class ObjectWrapperWithoutTypeInfo(any:AnyRef)

case class ObjectWrapperWithoutTypeInfoOverrided(any:AnyRef) extends JacksonJsonSerializableButNotDeserializable 

package utilities

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.JsonNode
import com.typesafe.config.Config

object JSONHelper {

  private val json_mapper = new ObjectMapper
  private val json_writer = json_mapper.writerWithDefaultPrettyPrinter()
  private val json_reader = json_mapper.reader()

  def read(json: String): JsonNode = {
    val content = json.replaceAll("(,)\\s+]", "]") // hack for removing trailing commas (invalid JSON)
    json_reader.readTree(content)
  }

  def write(json_tree: JsonNode): String = {
    json_writer.writeValueAsString(json_tree)
  }

  def pretty(json: String): String = {
    val tree = read(json)
    write(tree)
  }

} 

package examples.nominatim

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Await
import scala.concurrent.duration.Duration
import play.api.libs.ws.ahc.AhcWSClient
import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.JsonNode
import clients.HTTPClient

// SEE: Prefix.cc Lookup - http://prefix.cc/foaf.file.json

class NominatimLookup {

  val http = HTTPClient

  def start() {
    http.start()
  }

  def stop() {
    http.stop()
  }

  def nominatim(address: String) = {

    val url = "http://nominatim.openstreetmap.org/search"

    val parameters = Map(
      "q" -> address,
      "addressdetails" -> "1",
      "format" -> "json",
      "limit" -> "4",
      "addressdetails" -> "1",
      "dedupe" -> "1",
      "extratags" -> "1",
      "namedetails" -> "1").toList

    val ret = http.ws.url(url)
      .withQueryString(parameters: _*)
      .get()
      .map { response =>
        response.status match {
          case 200 => response.body
          case _   => "{}"
        }

      }

    ret

  }

}

object MainNominatimLookup extends App {

  import scala.collection.JavaConversions._
  import scala.collection.JavaConverters._

  val nominatim = new NominatimLookup
  nominatim.start()

  val json_mapper = new ObjectMapper
  val json_reader = json_mapper.reader()

  val result = Await.ready(nominatim.nominatim("135 pilkington avenue, birmingham"), Duration.Inf)
    .value.get.get

  val json_list: List[JsonNode] = json_reader.readTree(result).elements().toList

  // simulazione di output...
  if (json_list.size > 0) {
    println(s"RESULTS [${json_list.size}]")
    json_list
      .zipWithIndex
      .foreach {
        case (node, i) =>
          println(s"result ${i + 1}")
          println(node.get("place_id"))
          println(node.get("address").get("road").asText() + ", " + node.get("address").get("house_number").asText())
      }
  } else {
    println("cannot find results...")
  }

  nominatim.stop()

} 

package it.almawave.kb.http.providers

import com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider
import com.fasterxml.jackson.databind.ObjectMapper
import javax.ws.rs.ext.Provider
import javax.ws.rs.Produces
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import javax.ws.rs.core.MediaType
import com.fasterxml.jackson.annotation.JsonInclude

import com.fasterxml.jackson.annotation.JsonAnyGetter

import com.fasterxml.jackson.databind.SerializationFeature
import com.fasterxml.jackson.databind.DeserializationFeature
import javax.ws.rs.ext.ContextResolver
import com.fasterxml.jackson.databind.JsonSerializer
import com.fasterxml.jackson.core.JsonGenerator
import com.fasterxml.jackson.databind.SerializerProvider
import java.lang.Double
import java.lang.Boolean

@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
class JacksonScalaProvider extends JacksonJaxbJsonProvider with ContextResolver[ObjectMapper] {

  println("\n\nregistered " + this.getClass)

  val mapper = new ObjectMapper()

  mapper
    .registerModule(DefaultScalaModule)
    .setSerializationInclusion(JsonInclude.Include.ALWAYS)

    .configure(SerializationFeature.INDENT_OUTPUT, true)
    .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)

    .configure(SerializationFeature.WRITE_NULL_MAP_VALUES, true)
    .configure(SerializationFeature.WRITE_SINGLE_ELEM_ARRAYS_UNWRAPPED, true)
    .configure(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS, true)

    .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
    .configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true)
    .configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, true)
    .configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true)
    //    .setVisibility(JsonMethod.FIELD, Visibility.ANY);

    .getSerializerProvider.setNullValueSerializer(new JsonSerializer[Object] {
      def serialize(obj: Object, gen: JsonGenerator, provider: SerializerProvider) {
        obj match {
          case bool: Boolean   => gen.writeBoolean(false)
          case number: Integer => gen.writeNumber(0)
          case number: Double  => gen.writeNumber(0.0D)
          case text: String    => gen.writeString("")
          case _               => gen.writeString("")
        }
      }
    })

  super.setMapper(mapper)

  override def getContext(klasses: Class[_]): ObjectMapper = mapper

} 

package specs

import org.junit.runner.RunWith
import org.specs2.mutable.Specification
import org.specs2.runner.JUnitRunner
import com.fasterxml.jackson.databind.ObjectMapper
import com.typesafe.config.ConfigFactory
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
import play.api.test.WithBrowser

@RunWith(classOf[JUnitRunner])
class IntegrationSpec extends Specification {

  val host = "localhost"

  "semantic_repository" should {

    "expose swagger specification" in new WithBrowser {
      browser.goTo(s"http://${host}:${port}/spec/semantic_repository.yaml")
      browser.pageSource must haveSize(greaterThan(0))
      browser.pageSource must contain("semantic repository")
    }

    "list all existing contexts" in new WithBrowser {
      browser.goTo(s"http://${host}:${port}/kb/v1/contexts")
      browser.pageSource must haveSize(greaterThan(0))

      // DISABLED
      //      val ctxs = JSONHelper.parseString(browser.pageSource).toList
      //      ctxs.find { el => el.get("context").equals("http://xmlns.com/foaf/0.1/") }
      //      ctxs.find { el => el.get("triples").equals(631) }
    }

  }
}

object JSONHelper {

  val json_mapper = new ObjectMapper
  val json_reader = json_mapper.reader()

  val cc = ConfigFactory.empty()

  def parseString(json: String) = json_reader.readTree(json)

} 

package com.basho.riak.spark.examples.streaming

import java.util.UUID

import kafka.serializer.StringDecoder
import org.apache.spark.sql.Row
import org.apache.spark.streaming.Durations
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.{SparkConf, SparkContext}
import com.basho.riak.spark.streaming._
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.joda.time.DateTime
import org.joda.time.format.DateTimeFormat


object StreamingTSExample {
  def main(args: Array[String]): Unit = {

    val sparkConf = new SparkConf(true)
      .setAppName("Simple Spark Streaming to Riak TS Demo")

    setSparkOpt(sparkConf, "spark.master", "local")
    setSparkOpt(sparkConf, "spark.riak.connection.host", "127.0.0.1:8087")
    setSparkOpt(sparkConf, "kafka.broker", "127.0.0.1:9092")

    val sc = new SparkContext(sparkConf)
    val streamCtx = new StreamingContext(sc, Durations.seconds(15))

    val kafkaProps = Map[String, String](
      "metadata.broker.list" -> sparkConf.get("kafka.broker"),
      "client.id" -> UUID.randomUUID().toString
    )

    KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](streamCtx, kafkaProps, Set[String]("ingest-ts")
    ) map { case (key, value) =>
      val mapper = new ObjectMapper()
      mapper.registerModule(DefaultScalaModule)
      val wr = mapper.readValue(value, classOf[Map[String,String]])
      Row(
        wr("weather"),
        wr("family"),
        DateTime.parse(wr("time"),DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS")).getMillis,
        wr("temperature"),
        wr("humidity"),
        wr("pressure"))
    } saveToRiakTS "ts_weather_demo"

    streamCtx.start()
    println("Spark streaming context started. Spark UI could be found at http://SPARK_MASTER_HOST:4040")
    println("NOTE: if you're running job on the 'local' master open http://localhost:4040")
    streamCtx.awaitTermination()
  }


  private def setSparkOpt(sparkConf: SparkConf, option: String, defaultOptVal: String): SparkConf = {
    val optval = sparkConf.getOption(option).getOrElse(defaultOptVal)
    sparkConf.set(option, optval)
  }
} 

package com.basho.riak.spark.util

import com.basho.riak.client.api.convert.{ConverterFactory, JSONConverter}
import com.basho.riak.client.core.query.{Location, RiakObject}
import com.basho.riak.client.core.util.BinaryValue
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule

import scala.reflect.ClassTag

object RiakObjectConversionUtil {

  DataMapper.ensureInitialized()

  private var mapper: Option[ObjectMapper] = None

  private def objectMapper(): ObjectMapper = {
    if (mapper.isEmpty) {
      mapper = Some(JSONConverter.getObjectMapper)
    }
    mapper.get
  }

  def to[T](value: T): RiakObject = {
    // TODO: we need to think about smarter approach to handle primitive types such as int, long, etc.
    value match {
      case s: String =>
        new RiakObject()
          .setContentType("text/plain")
          .setValue(BinaryValue.create(value.asInstanceOf[String]))
      case _ =>
        // value as a strict JSON
        val v = objectMapper().writeValueAsString(value)

        new RiakObject()
          .setContentType("application/json")
          .setValue(BinaryValue.create(v))
    }
  }
} 

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.annotation.JsonInclude.Include
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import de.zalando.play.controllers.WriterFactories
import it.gov.daf.catalogmanager.{ConversionField, MetaCatalog, StdSchema}
import play.api.libs.json._

object TestCaseClass extends App{

  //@JsonInclude(JsonInclude.Include.NON_ABSENT)
  case class Pippo (ogg1: Option[String], ogg2: String, ogg3: Option[Pluto])
  //case class Pippo (ogg1: Option[String], ogg2: String)
  //case class Pluto (prop1: String, prop2: String, prop3: List[String])
  case class Pluto (prop1: String, prop2: String)

  val pippo = Pippo(Some("ciao"), "", None)
  //val pippo = Pippo(Some("ciao"), "ciao2")

  def jacksonMapper(mimeType: String): ObjectMapper = {
    //noinspection ScalaStyle
    assert(mimeType != null)
    val factory = WriterFactories.factories(mimeType)
    val mapper = new ObjectMapper(factory)
    mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT)
    mapper.registerModule(DefaultScalaModule)

    mapper
  }
  val json = jacksonMapper("blabla").writeValueAsString(pippo)
  //val json = "{\"ogg1\":\"ciao\",\"ogg2\":\"ciao2\"}"
  println(json)
  //implicit val plutoReads = Json.reads[Pluto]
  //implicit val pippoReads = Json.reads[Pippo]
  //implicit val plutoReads = Json.reads[Option[Pluto]]
  




} 

package org.web3scala.json

import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper

class JacksonJsonMapper extends JsonMapper {
  override def writeAsBytes(value: AnyRef): Array[Byte] = {
    JacksonJsonMapper.mapper.writeValueAsBytes(value)
  }
}
object JacksonJsonMapper {
  private val mapperInstance = new ObjectMapper with ScalaObjectMapper
  mapperInstance.registerModule(DefaultScalaModule)
  mapperInstance.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
  def mapper: ObjectMapper with ScalaObjectMapper = mapperInstance
} 

class DWSHttpClient(clientConfig: DWSClientConfig, clientName: String)
  extends AbstractHttpClient(clientConfig, clientName) {

  override protected def createDiscovery(): Discovery = new DWSGatewayDiscovery


  override protected def prepareAction(requestAction: HttpAction): HttpAction = {
    requestAction match {
      case dwsAction: DWSHttpAction => dwsAction.setDWSVersion(clientConfig.getDWSVersion)
      case _ =>
    }
    requestAction
  }

  override protected def httpResponseToResult(response: HttpResponse, requestAction: HttpAction, responseBody: String): Option[Result] = {
    var entity = response.getEntity
    val statusCode: Int = response.getStatusLine.getStatusCode
    val url: String = requestAction.getURL
    val contentType: String = entity.getContentType.getValue
    DWSHttpMessageFactory.getDWSHttpMessageResult(url).map { case DWSHttpMessageResultInfo(_, clazz) =>
      clazz match {
        case c if ClassUtils.isAssignable(c, classOf[DWSResult]) =>
          val dwsResult = clazz.getConstructor().newInstance().asInstanceOf[DWSResult]
          dwsResult.set(responseBody, statusCode, url, contentType)
          BeanUtils.populate(dwsResult, dwsResult.getData)
          return Some(dwsResult)
        case _ =>
      }

      def transfer(value: Result, map: Map[String, Object]): Unit = {
        value match {
          case httpResult: HttpResult =>
            httpResult.set(responseBody, statusCode, url, contentType)
          case _ =>
        }
        val javaMap = mapAsJavaMap(map)
        BeanUtils.populate(value, javaMap)
        fillResultFields(javaMap, value)
      }
      deserializeResponseBody(response) match {
        case map: Map[String, Object] =>
          val value = clazz.getConstructor().newInstance().asInstanceOf[Result]
          transfer(value, map)
          value
        case list: List[Map[String, Object]] =>
          val results = list.map { map =>
            val value = clazz.getConstructor().newInstance().asInstanceOf[Result]
            transfer(value, map)
            value
          }.toArray
          new ListResult(responseBody, results)
      }
    }.orElse(nonDWSResponseToResult(response, requestAction))
  }

  protected def nonDWSResponseToResult(response: HttpResponse, requestAction: HttpAction): Option[Result] = None

  protected def fillResultFields(responseMap: util.Map[String, Object], value: Result): Unit = {}

  //TODO Consistent with workspace, plus expiration time(与workspace保持一致，加上过期时间)
  override protected def getFsByUser(user: String, path: FsPath): Fs = FSFactory.getFsByProxyUser(path, user)

}
object DWSHttpClient {
  val jacksonJson = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"))
} 

package io.eels.util

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper

object JacksonSupport {

  val mapper: ObjectMapper with ScalaObjectMapper = new ObjectMapper with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)

  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
  mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)
  mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true)
  mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
  mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)
} 

package io.eels.component.json

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
import io.eels.schema.StructType
import io.eels.{Row, Sink, SinkWriter}
import org.apache.hadoop.fs.{FileSystem, Path}

case class JsonSink(path: Path)(implicit fs: FileSystem) extends Sink {

  override def open(schema: StructType): SinkWriter = new SinkWriter {

    private val lock = new AnyRef()
    private val out = fs.create(path)

    val mapper = new ObjectMapper with ScalaObjectMapper
    mapper.registerModule(DefaultScalaModule)

    override def write(row: Row) {
      val map = schema.fieldNames.zip(row.values).toMap
      val json = mapper.writeValueAsString(map)
      lock.synchronized {
        out.writeBytes(json)
        out.writeBytes("\n")
      }
    }

    override def close() {
      out.close()
    }
  }
} 

package com.horizen.mainchain.api

import java.io.{BufferedReader, InputStreamReader}

import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}
import com.horizen.SidechainSettings
import com.horizen.serialization.ApplicationJsonSerializer
import com.horizen.utils.BytesUtils

class RpcMainchainNodeApi(val sidechainSettings: SidechainSettings)
  extends MainchainNodeApi
{

  private lazy val isOsWindows = {
    val osname = System.getProperty("os.name", "generic").toLowerCase()
    osname.contains("win")
  }

  private val clientPath = sidechainSettings.websocket.zencliCommandLine + " " +
    (sidechainSettings.genesisData.mcNetwork match {
      case "regtest" => "-regtest "
      case "testnet" => "-testnet "
      case _ => ""
    })

  private def callRpc(params: String) : String = {
    System.out.println(clientPath + " " + params)
    val process = Runtime.getRuntime.exec(clientPath + " " + params)

    val stdInput: BufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream))

    val stdError: BufferedReader = new BufferedReader(new InputStreamReader(process.getErrorStream))

    val error = stdError.readLine()
    if(error != null)
      throw new IllegalStateException("Error: " + error)

    stdInput.readLine
  }

  private def encloseJsonParameter(parameter: String): String = {
    if (isOsWindows)
      "\"" + parameter.replace("\"", "\\\"") + "\""
    else
      "'" + parameter + "'"
  }

  private def encloseStringParameter(parameter: String): String = {
    "\"" + parameter + "\""
  }

  override def getSidechainInfo: SidechainInfoResponse = {
    val objectMapper = new ObjectMapper()
    val response = callRpc("getscinfo")

    objectMapper.readValue(response, classOf[SidechainInfoResponse])
  }

  override def sendCertificate(certificateRequest: SendCertificateRequest): SendCertificateResponse = {
    val serializer = ApplicationJsonSerializer.getInstance() // TODO: maybe it's better to construct object mapper from scratch
    serializer.setDefaultConfiguration()
    val objectMapper = serializer.getObjectMapper
    objectMapper.disable(SerializationFeature.INDENT_OUTPUT)

    val response = callRpc("send_certificate "
      + encloseStringParameter(BytesUtils.toHexString(certificateRequest.sidechainId)) + " "
      + certificateRequest.epochNumber + " "
      + certificateRequest.quality + " "
      + encloseStringParameter(BytesUtils.toHexString(certificateRequest.endEpochBlockHash)) + " "
      + encloseStringParameter(BytesUtils.toHexString(certificateRequest.proofBytes)) + " "
      + encloseJsonParameter(objectMapper.writeValueAsString(certificateRequest.backwardTransfers)) + " "
      + certificateRequest.fee
      )

    SendCertificateResponse(BytesUtils.fromHexString(response))
  }
} 

package hydra.avro.util

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.node.ObjectNode
import org.scalatest.matchers.should.Matchers
import org.scalatest.funspec.AnyFunSpecLike

import scala.io.Source


class AvroSchemaGeneratorSpec extends Matchers with AnyFunSpecLike {

  val mapper = new ObjectMapper
  val converter = new AvroSchemaGenerator()

  val json = Source
    .fromFile(
      Thread.currentThread.getContextClassLoader
        .getResource("avro-test.json")
        .getFile
    )
    .getLines()
    .mkString

  describe("The json to avro schema converter") {
    it("Should include a valid namespace and a valid name") {
      val jsonNode = mapper.readTree(converter.convert(json, "hydra", "name"))
      jsonNode.at("/namespace").asText shouldBe "hydra"
      jsonNode.at("/name").asText shouldBe "name"
      jsonNode.at("/type").asText shouldBe "record"
    }
    it("Should have a valid record type") {
      val jsonNode = mapper.readTree(converter.convert(json, "hydra", "name"))
      val arrayNode = jsonNode.at("/fields")
      arrayNode.get(0).at("/type/type").asText shouldBe "record"
    }

    it("Should throw an exception with null values") {
      val jsonNode = mapper.readTree(json)
      jsonNode.asInstanceOf[ObjectNode].set("dummyString", null)
      intercept[IllegalArgumentException] {
        converter.convert(jsonNode.toString(), "hydra", "name")
      }
    }
    it("Should convert booleans") {
      val clipViewJson =
        """
          |{
          |	"clipId": "shawn-wildermuth|front-end-web-app-html5-javascript-css-m01|front-end-web-app-html5-javascript-css-m1-02",
          |	"clipModuleIndex": 1,
          |	"clipName": "front-end-web-app-html5-javascript-css-m1-02",
          |	"contentIndexPosition": 99999,
          |	"countsTowardTrialLimits": false,
          |	"courseName": "front-end-web-app-html5-javascript-css",
          |	"courseTitle": "Front-End Web Development Quick Start With HTML5, CSS, and JavaScript",
          |	"ipAddress": "127.0.0.1",
          |	"moduleAuthorHandle": "shawn-wildermuth",
          |	"moduleId": "shawn-wildermuth|front-end-web-app-html5-javascript-css-m01",
          |	"moduleName": "front-end-web-app-html5-javascript-css-m01",
          |	"online": true,
          |	"royaltiesPaid": true,
          |	"started": "2016-11-30T20:30:45.3136582Z",
          |	"userHandle": "44bbf444-ba44-444a-b444-b444bebb4b4b"
          |}
        """.stripMargin

      val jsonNode = mapper.readTree(clipViewJson)

      val schema = converter.convert(jsonNode.toString(), "hydra", "name")

      println(schema)
    }
  }
} 

package hydra.kafka.producer

import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import hydra.core.transport.AckStrategy
import org.apache.commons.lang3.StringUtils


case class JsonRecord(
    destination: String,
    key: String,
    payload: JsonNode,
    ackStrategy: AckStrategy
) extends KafkaRecord[String, JsonNode]

object JsonRecord {
  val mapper = new ObjectMapper()

  def apply(
      topic: String,
      key: Option[String],
      obj: Any,
      ackStrategy: AckStrategy
  ): JsonRecord = {
    val payload = mapper.convertValue[JsonNode](obj, classOf[JsonNode])
    new JsonRecord(topic, key.orNull, payload, ackStrategy)
  }
} 

package hydra.kafka.producer

import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import hydra.core.ingest.HydraRequest

import scala.concurrent.{ExecutionContext, Future}


object JsonRecordFactory extends KafkaRecordFactory[String, JsonNode] {

  val mapper = new ObjectMapper()

  override def build(
      request: HydraRequest
  )(implicit ec: ExecutionContext): Future[KafkaRecord[String, JsonNode]] = {
    for {
      topic <- Future.fromTry(getTopic(request))
      payload <- parseJson(request.payload)
    } yield JsonRecord(
      topic,
      getKey(request, payload),
      payload,
      request.ackStrategy
    )

  }

  private def parseJson(json: String)(implicit ec: ExecutionContext) =
    Future(mapper.reader().readTree(json))

} 

package hydra.kafka.producer

import com.fasterxml.jackson.databind.ObjectMapper
import hydra.core.ingest
import hydra.core.ingest.RequestParams
import hydra.kafka.producer.KafkaRecordFactory.RecordKeyExtractor.JsonRecordKeyExtractor
import org.scalatest.matchers.should.Matchers
import org.scalatest.flatspec.AnyFlatSpecLike

class JsonRecordKeyExtractorSpec extends Matchers with AnyFlatSpecLike {
  val mapper = new ObjectMapper()

  "The JsonRecordKeyExtractor" should "return none when no key is present" in {
    val json = """{"name":"hydra","rank":1}"""
    val node = mapper.reader().readTree(json)
    val request = ingest.HydraRequest("corr", node.asText())
    JsonRecordKeyExtractor.extractKeyValue(request, node) shouldBe None
  }

  it should "return a key" in {
    val json = """{"name":"hydra","rank":1}"""
    val node = mapper.reader().readTree(json)
    val request = ingest
      .HydraRequest("corr", node.asText())
      .withMetadata(RequestParams.HYDRA_RECORD_KEY_PARAM -> "{$.name}")
    JsonRecordKeyExtractor.extractKeyValue(request, node) shouldBe Some("hydra")
  }
} 

package hydra.kafka.producer

import com.fasterxml.jackson.core.JsonParseException
import com.fasterxml.jackson.databind.ObjectMapper
import hydra.core.ingest.HydraRequest
import hydra.core.ingest.RequestParams.{
  HYDRA_KAFKA_TOPIC_PARAM,
  HYDRA_RECORD_KEY_PARAM
}
import hydra.core.protocol.MissingMetadataException
import hydra.core.transport.AckStrategy
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.matchers.should.Matchers
import org.scalatest.funspec.AnyFunSpecLike

import scala.concurrent.ExecutionContext.Implicits.global


class JsonRecordFactorySpec
    extends Matchers
    with AnyFunSpecLike
    with ScalaFutures {

  describe("When using the JsonRecordFactory") {

    it("errors with no topic on the request") {
      val request = HydraRequest("123", """{"name":test"}""")
      val rec = JsonRecordFactory.build(request)
      whenReady(rec.failed)(_ shouldBe an[MissingMetadataException])
    }

    it("handles invalid json") {
      val request = HydraRequest("123", """{"name":test"}""")
        .withMetadata(HYDRA_KAFKA_TOPIC_PARAM -> "test-topic")
      val rec = JsonRecordFactory.build(request)
      whenReady(rec.failed)(_ shouldBe a[JsonParseException])
    }

    it("handles valid json") {
      val request = HydraRequest("123", """{"name":"test"}""")
        .withMetadata(HYDRA_KAFKA_TOPIC_PARAM -> "test-topic")
      val rec = JsonRecordFactory.build(request)
      val node = new ObjectMapper().reader().readTree("""{"name":"test"}""")
      whenReady(rec)(
        _ shouldBe JsonRecord("test-topic", None, node, AckStrategy.NoAck)
      )
    }

    it("builds") {
      val request = HydraRequest("123", """{"name":"test"}""")
        .withMetadata(HYDRA_RECORD_KEY_PARAM -> "{$.name}")
        .withMetadata(HYDRA_KAFKA_TOPIC_PARAM -> "test-topic")
      whenReady(JsonRecordFactory.build(request)) { msg =>
        msg.destination shouldBe "test-topic"
        msg.key shouldBe "test"
        msg.payload shouldBe new ObjectMapper()
          .reader()
          .readTree("""{"name":"test"}""")
      }
    }

    it("throws an error if no topic is in the request") {
      val request = HydraRequest("123", """{"name":"test"}""")
      whenReady(JsonRecordFactory.build(request).failed)(
        _ shouldBe a[MissingMetadataException]
      )
    }
  }
} 

package hydra.kafka.serializers

import com.fasterxml.jackson.databind.ObjectMapper

import scala.collection.JavaConverters._
import org.scalatest.matchers.should.Matchers
import org.scalatest.funspec.AnyFunSpecLike

class JsonSerializerSpec extends Matchers with AnyFunSpecLike {

  val mapper = new ObjectMapper()

  describe("The JSON serializer") {
    it("serializes values") {
      val jsonSerializer = new JsonSerializer()
      jsonSerializer.configure(Map.empty[String, Any].asJava, false)
      val node = mapper.readTree("""{"name":"hydra"}""")
      jsonSerializer.serialize("topic", node) shouldBe mapper.writeValueAsBytes(
        node
      )
      jsonSerializer.close()
    }
  }

  describe("The JSON deserializer") {
    it("de-serializes values") {
      val jd = new JsonDeserializer()
      jd.configure(Map.empty[String, Any].asJava, false)
      val node = mapper.readTree("""{"name":"hydra"}""")
      jd.deserialize("topic", mapper.writeValueAsBytes(node)) shouldBe node
      jd.close()
    }
  }
} 

package com.wixpress.build.codota

import java.net.SocketTimeoutException

import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.wixpress.build.codota.CodotaThinClient.{PathAttemptResult, RetriesLoop}
import org.slf4j.LoggerFactory

import scala.util.{Failure, Success, Try}
import scalaj.http.{Http, HttpOptions, HttpResponse}

class CodotaThinClient(token: String,
                       codePack: String,
                       baseURL: String = CodotaThinClient.DefaultBaseURL,
                       maxRetries: Int = 5) {

  private val logger = LoggerFactory.getLogger(getClass)
  private val timeout: Int = 1000
  private val mapper = new ObjectMapper()
    .registerModule(DefaultScalaModule)
    .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
    .configure(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, true)

  private val requestURLTemplate = s"$baseURL/api/codenav/artifact"

  def pathFor(artifactName: String): Option[String] = {
    retriesLoop(artifactName).collectFirst {
      case Success(p) => p
    }.flatten
  }

  private def retriesLoop(artifactName: String): RetriesLoop = {
    (1 to maxRetries).toStream.map(pathForArtifact(_, artifactName))
  }

  private def pathForArtifact(retry: Int, artifactName: String): PathAttemptResult = {
    pathForArtifact(artifactName) match {
      case Success(p) => Success(p)
      case Failure(e: SocketTimeoutException) if retry < maxRetries =>
        handleSocketTimeout(retry, artifactName, e)
      case Failure(e) => throw e
    }
  }

  private def handleSocketTimeout(retry: Int, artifactName: String, e: SocketTimeoutException) = {
    logger.warn(s"($retry/$maxRetries) Timeout when trying to get path for $artifactName")
    Thread.sleep(3000 / ((maxRetries - retry) + 1))
    Failure(e)
  }

  private def pathForArtifact(artifactName: String): PathAttemptResult = {
    for {
      response <- requestFor(artifactName)
      body <- responseBody(response)
      path <- extractPath(body, artifactName)
    } yield path
  }

  private def requestFor(artifactName: String) = {
    Try {
      Http(s"$requestURLTemplate/$artifactName/metadata")
        .param("codePack", codePack)
        .header("Authorization", s"bearer $token")
        .option(HttpOptions.readTimeout(timeout))
        .asString
    }
  }

  private def responseBody(resp: HttpResponse[String]) = {
    Try {
      val body = resp.body
      val code = resp.code

      code match {
        case 200 => body
        case 401 => throw NotAuthorizedException(body)
        case 404 if body.contains(codePack) => throw CodePackNotFoundException(body)
        case 403 => throw MissingCodePackException()
        case 404 => throw MetaDataOrArtifactNotFound(body)
      }
    }
  }

  private def extractPath(body: String, artifactName: String) = {
    Try {
      val metaData = mapper.readValue(body, classOf[ArtifactMetadata])
      Option(metaData.path)
    }
  }
}

object CodotaThinClient {
  val DefaultBaseURL = "https://gateway.codota.com"
  type PathAttemptResult = Try[Option[String]]
  type RetriesLoop = Stream[PathAttemptResult]
}

case class ArtifactMetadata(path: String) 

package com.wix.bazel.migrator.analyze

import java.nio.file.{Files, Path, Paths}
import java.util
import java.util.concurrent.atomic.AtomicInteger

import com.fasterxml.jackson.annotation.JsonTypeInfo
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.wix.bazel.migrator.model._
import com.wixpress.build.maven.MavenScope
import org.slf4j.LoggerFactory

import scala.collection.JavaConverters._
import scala.collection.parallel.ParMap

//this is needed since currently the transformer isn't thread safe but the dependency analyzer is
class CachingEagerEvaluatingDependencyAnalyzer(sourceModules: Set[SourceModule], dependencyAnalyzer: DependencyAnalyzer, performSourceAnalysis: Boolean) extends DependencyAnalyzer {
  private val log = LoggerFactory.getLogger(getClass)
  private val cachePath = Files.createDirectories(Paths.get("./cache"))
  private val objectMapper = new ObjectMapper()
    .registerModule(DefaultScalaModule)
    .registerModule(new RelativePathSupportingModule)
    .registerModule(new SourceModuleSupportingModule(sourceModules))
    .addMixIn(classOf[Target], classOf[TypeAddingMixin])
    .addMixIn(classOf[CodePurpose], classOf[TypeAddingMixin])
    .addMixIn(classOf[TestType], classOf[TypeAddingMixin])
    .addMixIn(classOf[MavenScope], classOf[TypeAddingMixin])

  private val collectionType = objectMapper.getTypeFactory.constructCollectionType(classOf[util.Collection[Code]], classOf[Code])
  private val clean = performSourceAnalysis

  private def cachePathForSourceModule(m: SourceModule) = {
    cachePath.resolve(m.relativePathFromMonoRepoRoot + ".cache")
  }

  private val size = sourceModules.size
  private val counter = new AtomicInteger()
  private val tenthSize = size / 10

  private def initCachePathForSourceModule(p: Path) = Files.createDirectories(p.getParent)

  private def maybeCodeFromCache(p: Path): Option[List[Code]] = {
    if (clean || !Files.exists(p)) return None
    try {
      val value: util.Collection[Code] = objectMapper.readValue(p.toFile, collectionType)
      val codeList = value.asScala.toList
      Some(codeList)
    } catch {
      case e: Exception =>
        log.warn(s"Error reading $p ,deleting cache file.")
        log.warn(e.getMessage)
        Files.deleteIfExists(p)
        None
    }
  }

  private def retrieveCodeAndCache(m: SourceModule, cachePath: Path): List[Code] = {
    val codeList = dependencyAnalyzer.allCodeForModule(m)
    Files.deleteIfExists(cachePath)
    initCachePathForSourceModule(cachePath)
    Files.createFile(cachePath)
    try {
      objectMapper.writeValue(cachePath.toFile, codeList)
    } catch {
      case e: InterruptedException =>
        log.warn(s"aborting write to file $cachePath")
        Files.deleteIfExists(cachePath)
        throw e
      case e: Exception =>
        log.warn(s"could not write to file $cachePath")
        log.warn(e.getMessage)
    }
    codeList
  }

  private def calculateMapEntryFor(sourceModule: SourceModule) = {
    printProgress()
    val cachePath = cachePathForSourceModule(sourceModule)
    (sourceModule, maybeCodeFromCache(cachePath).getOrElse(retrieveCodeAndCache(sourceModule, cachePath)))
  }

  private def printProgress(): Unit = {
    if (tenthSize > 0) {
      val currentCount = counter.incrementAndGet()
      if (currentCount % tenthSize == 0) {
        log.info(s"DependencyAnalyzer:allCodeForModule:\t ${currentCount / tenthSize * 10}% done")
      }
    }
  }

  private val allCode: ParMap[SourceModule, List[Code]] = sourceModules.par.map(calculateMapEntryFor).toMap

  override def allCodeForModule(module: SourceModule): List[Code] = allCode(module)
}

@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "__class")
trait TypeAddingMixin 

package com.wix.bazel.migrator.analyze

import com.fasterxml.jackson.annotation.{JsonIgnoreProperties, JsonTypeInfo}
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.wix.bazel.migrator.model.SourceModule
import com.wix.bazel.migrator.transform.failures.{AnalyzeException, AnalyzeFailure}

class ExceptionFormattingDependencyAnalyzer(dependencyAnalyzer: DependencyAnalyzer) extends DependencyAnalyzer {
  private val om = new ObjectMapper().registerModule(DefaultScalaModule)
    .addMixIn(classOf[AnalyzeFailure], classOf[AnalyzeFailureMixin])
    .addMixIn(classOf[Throwable], classOf[ThrowableMixin])
    .addMixIn(classOf[SourceModule], classOf[IgnoringMavenDependenciesMixin])

  override def allCodeForModule(module: SourceModule): List[Code] =
    try {
      dependencyAnalyzer.allCodeForModule(module)
    } catch {
      case e: AnalyzeException =>
        val message = om.writerWithDefaultPrettyPrinter().writeValueAsString(e.failure)
        throw new RuntimeException(message +
          """|***Detailed error is in a prettified json which starts above***
          |***Inner most AnalyzeFailure has root cause, look for it***
          |More info at https://github.com/wix-private/bazel-tooling/blob/master/migrator/docs
          |""".stripMargin)
    }
}
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "__class")
trait AnalyzeFailureMixin

@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "__class")
abstract class ThrowableMixin

@JsonIgnoreProperties(Array("dependencies"))
trait IgnoringMavenDependenciesMixin 

package com.wix.bazel.migrator.overrides

import java.nio.file.{Files, Path}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.wix.build.maven.analysis.SourceModulesOverrides

object SourceModulesOverridesReader {

  private val mapper = new ObjectMapper().registerModule(DefaultScalaModule)
  def from(repoRoot: Path): SourceModulesOverrides = {
    val overridesPath = repoRoot.resolve("bazel_migration").resolve("source_modules.overrides")
    if (Files.exists(overridesPath))
      mapper.readValue(
        Files.newBufferedReader(overridesPath),
        classOf[SourceModulesOverrides]
      )
    else
      SourceModulesOverrides.empty
  }

} 

package com.wix.bazel.migrator.overrides

import java.nio.file.{Files, Path}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule

object GeneratedCodeOverridesReader {
  private val mapper = new ObjectMapper()
    .registerModule(DefaultScalaModule)

  def from(repoRoot: Path): GeneratedCodeLinksOverrides = {
    val overridesPath = repoRoot.resolve("bazel_migration").resolve("code_paths.overrides")
    if (Files.exists(overridesPath))
      mapper.readValue(
        Files.newBufferedReader(overridesPath),
        classOf[GeneratedCodeLinksOverrides]
      )
    else
      GeneratedCodeLinksOverrides.empty
  }
} 

package com.wix.bazel.migrator.overrides

import java.nio.file.{Files, Path}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule

object MavenArchiveTargetsOverridesReader {
  def from(repoRoot: Path): MavenArchiveTargetsOverrides = {
    val overridesPath = repoRoot.resolve("bazel_migration").resolve("maven_archive_targets.overrides")
    if (Files.exists(overridesPath)) {
      val objectMapper = new ObjectMapper().registerModule(DefaultScalaModule)
      objectMapper.readValue(Files.readAllBytes(overridesPath), classOf[MavenArchiveTargetsOverrides])
    } else {
      MavenArchiveTargetsOverrides(Set.empty)
    }
  }

} 

package com.wix.bazel.migrator.overrides

import java.nio.file.{Files, Path}

import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.wix.bazel.migrator.model.TestType
import com.wix.bazel.migrator.utils.TypeAddingMixin

object InternalTargetOverridesReader {
  private val objectMapper = new ObjectMapper()
    .registerModule(DefaultScalaModule)
    .addMixIn(classOf[TestType], classOf[TypeAddingMixin])
    .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)

  def from(repoRootPath: Path): InternalTargetsOverrides = {
    val internalTargetsOverrides = repoRootPath.resolve("bazel_migration").resolve("internal_targets.overrides")

    if (Files.isReadable(internalTargetsOverrides)) {
      objectMapper.readValue(Files.newInputStream(internalTargetsOverrides), classOf[InternalTargetsOverrides])
    } else {
      InternalTargetsOverrides()
    }
  }
} 

package com.wix.bazel.migrator.overrides

import java.nio.file.{Files, Path}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule

import scala.util.{Failure, Success, Try}

object AdditionalDepsByMavenDepsOverridesReader {
  private val mapper = new ObjectMapper()
    .registerModule(DefaultScalaModule)

  def from(filepath: Path): AdditionalDepsByMavenDepsOverrides = {
    if (Files.exists(filepath))
      readContentIn(filepath)
    else
      AdditionalDepsByMavenDepsOverrides.empty
  }

  private def readContentIn(filepath: Path) = {
    Try(mapper.readValue(
      Files.newBufferedReader(filepath),
      classOf[AdditionalDepsByMavenDepsOverrides]
    )) match {
      case Success(overrides) => overrides
      case Failure(e) => throw OverrideParsingException(s"cannot parse $filepath", e)
    }
  }
} 

package com.wix.bazel.migrator

import java.io.File
import java.nio.file.attribute.BasicFileAttributes
import java.nio.file.{Files, Paths}
import java.time.Instant
import java.time.temporal.TemporalUnit
import java.util

import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.wix.bazel.migrator.model.{CodePurpose, Package, Target, TestType}
import com.wix.bazel.migrator.utils.{IgnoringIsArchiveDefMixin, IgnoringIsProtoArtifactDefMixin, IgnoringIsWarDefMixin, TypeAddingMixin}
import com.wix.build.maven.analysis.SourceModules
import com.wixpress.build.maven.{Coordinates, MavenScope, Packaging}

import scala.collection.JavaConverters._

object Persister {

  private val transformedFile = new File("dag.bazel")
  private val mavenCache = Paths.get("classpathModules.cache")
  val objectMapper = new ObjectMapper().registerModule(DefaultScalaModule)
    .addMixIn(classOf[Target], classOf[TypeAddingMixin])
    .addMixIn(classOf[CodePurpose], classOf[TypeAddingMixin])
    .addMixIn(classOf[TestType], classOf[TypeAddingMixin])
    .addMixIn(classOf[MavenScope], classOf[TypeAddingMixin])
    .addMixIn(classOf[Packaging], classOf[IgnoringIsArchiveDefMixin])
    .addMixIn(classOf[Packaging], classOf[IgnoringIsWarDefMixin])
    .addMixIn(classOf[Coordinates], classOf[IgnoringIsProtoArtifactDefMixin])
    .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)

  def persistTransformationResults(bazelPackages: Set[Package]): Unit = {
    println("Persisting transformation")
    objectMapper.writeValue(transformedFile, bazelPackages)
  }

  def readTransformationResults(): Set[Package] = {
    val collectionType = objectMapper.getTypeFactory.constructCollectionType(classOf[util.Collection[Package]], classOf[Package])
    val value: util.Collection[Package] = objectMapper.readValue(transformedFile, collectionType)
    val bazelPackages = value.asScala.toSet
    bazelPackages
  }

  def persistMavenClasspathResolution(sourceModules: SourceModules): Unit = {
    println("Persisting maven")
    objectMapper.writeValue(mavenCache.toFile, sourceModules)
  }

  def readTransMavenClasspathResolution(): SourceModules = {
    objectMapper.readValue[SourceModules](mavenCache.toFile, classOf[SourceModules])
  }

  def mavenClasspathResolutionIsUnavailableOrOlderThan(amount: Int, unit: TemporalUnit): Boolean =
    !Files.isReadable(mavenCache) ||
      lastModifiedMavenCache().toInstant.isBefore(Instant.now().minus(amount, unit))

  private def lastModifiedMavenCache() =
    Files.readAttributes(mavenCache, classOf[BasicFileAttributes]).lastModifiedTime()

} 

package jp.gihyo.spark.ch03.basic_transformation

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.apache.log4j.{Level, Logger}
import org.apache.spark.{SparkConf, SparkContext}

// scalastyle:off println

object MapPartitionsExample {
  def main(args: Array[String]) {
    Logger.getLogger("org").setLevel(Level.WARN)

    val conf = new SparkConf().setAppName("MapPartitionsExample")
    val sc = new SparkContext(conf)

    run(sc)
    sc.stop()
  }

  def run(sc: SparkContext) {
    val jsonLines = sc.parallelize(Array(
      """{"name": "Apple",  "num": 1}""",
      """{"name": "Orange", "num": 4}""",
      """{"name": "Apple",  "num": 2}""",
      """{"name": "Peach",  "num": 1}"""
    ))

    val parsed = jsonLines.mapPartitions { lines =>
      val mapper = new ObjectMapper()
      mapper.registerModule(DefaultScalaModule)
      lines.map { line =>
        val f = mapper.readValue(line, classOf[Map[String, String]])
        (f("name"), f("num"))
      }
    }

    println(s"""json:\n${jsonLines.collect().mkString("\n")}""")
    println()
    println(s"""parsed:\n${parsed.collect().mkString("\n")}""")
  }
}

// scalastyle:on println 

package com.twilio.open.odsc.realish.config

import java.io.File

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.scala.DefaultScalaModule

object AppConfig {
  private val mapper = new ObjectMapper(new YAMLFactory)
  mapper.registerModule(DefaultScalaModule)

  def parse(configPath: String): AppConfig = {
    mapper.readValue(new File(configPath), classOf[AppConfig])
  }

}

@SerialVersionUID(100L)
case class AppConfig(
  sparkAppConfig: SparkAppConfig,
  streamingQueryConfig: StreamingQueryConfig
) extends Serializable

@SerialVersionUID(100L)
case class SparkAppConfig(
  appName: String,
  core: Map[String, String]
) extends Serializable

trait KafkaConsumerConfig {
  val topic: String
  val subscriptionType: String
  val conf: Map[String, String]
}

@SerialVersionUID(100L)
case class ConsumerConfig(
  topic: String,
  subscriptionType: String,
  conf: Map[String, String]
) extends KafkaConsumerConfig with Serializable

@SerialVersionUID(100L)
case class StreamingQueryConfig(
  streamName: String,
  triggerInterval: String,
  triggerEnabled: Boolean,
  windowInterval: String,
  watermarkInterval: String
) extends Serializable 

package play.api.libs.ws

import akka.util.ByteString
import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.ObjectMapper
import play.api.libs.json.JsValue
import play.api.libs.json.Json

trait JsonBodyWritables {

  
  implicit val writeableOf_JsValue: BodyWritable[JsValue] = {
    BodyWritable(a => InMemoryBody(ByteString.fromArrayUnsafe(Json.toBytes(a))), "application/json")
  }

  def body(objectMapper: ObjectMapper): BodyWritable[JsonNode] =
    BodyWritable(
      json => InMemoryBody(ByteString.fromArrayUnsafe(objectMapper.writer.writeValueAsBytes(json))),
      "application/json"
    )
}

object JsonBodyWritables extends JsonBodyWritables 

package com.atomist.project.common.yaml

import java.util.regex.{Pattern, PatternSyntaxException}

import com.atomist.param._
import com.atomist.project.common.template.{InvalidTemplateException, TemplateBasedProjectOperationInfo}
import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
import org.apache.commons.lang3.builder.ReflectionToStringBuilder

import scala.util.{Failure, Success, Try}


object YamlProjectOperationInfoParser {

  private val mapper = new ObjectMapper(new YAMLFactory()) with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)

  @throws[InvalidYamlDescriptorException]
  def parse(yaml: String): TemplateBasedProjectOperationInfo = {
    if (yaml == null || "".equals(yaml))
      throw new InvalidYamlDescriptorException("YAML content required in template metadata file")

    Try(mapper.readValue(yaml, classOf[BoundProjectOperationInfo])) match {
      case s: Success[BoundProjectOperationInfo] =>
        val badPatterns = s.value.parameters.flatMap(p => patternError(p))
        if (badPatterns.nonEmpty)
          throw new InvalidYamlDescriptorException(s"Bad regexp patterns: ${badPatterns.mkString(",")}")
        s.value
      case f: Failure[BoundProjectOperationInfo] =>
        throw new InvalidYamlDescriptorException(s"Failed to parse YAML [$yaml]: ${f.exception.getMessage}", f.exception)
    }
  }

  private def patternError(p: Parameter): Option[String] = {
    try {
      Pattern.compile(p.getPattern)
      None
    } catch {
      case pse: PatternSyntaxException => Some(s"${p.getName}: Bad regular expression pattern: ${pse.getMessage}")
    }
  }
}

private class BoundProjectOperationInfo extends TemplateBasedProjectOperationInfo {

  @JsonProperty("name")
  var name: String = _

  @JsonProperty("description")
  var description: String = _

  @JsonProperty("template_name")
  var templateName: String = _

  @JsonProperty("type")
  var _templateType: String = _

  override def templateType: Option[String] =
    if (_templateType == null || "".equals(_templateType)) None
    else Some(_templateType)

  @JsonProperty("parameters")
  private var _params: Seq[Parameter] = Nil

  @JsonProperty("tags")
  private var _tags: Seq[TagHolder] = Nil

  override def parameters: Seq[Parameter] = _params

  override def tags: Seq[Tag] = _tags.map(tw => tw.toTag)

  override def toString = ReflectionToStringBuilder.toString(this)
}

private class TagHolder {

  @JsonProperty
  var name: String = _

  @JsonProperty
  var description: String = _

  def toTag = Tag(name, description)
}

class InvalidYamlDescriptorException(msg: String, ex: Throwable = null) extends InvalidTemplateException(msg, ex) 

package com.atomist.tree.marshal

import com.atomist.graph.GraphNode
import com.atomist.rug.ts.Cardinality
import com.atomist.tree.{SimpleTerminalTreeNode, TreeNode}
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
import com.typesafe.scalalogging.LazyLogging


  def fromJson(json: String): GraphNode = {
    val l = toListOfMaps(json)
    nodeify(l)
  }

  private def toListOfMaps(json: String): List[Map[String, Object]] = {
    mapper.readValue(json, classOf[List[Map[String, Object]]])
  }

  private def nodeify(l: List[Map[String, Object]]): GraphNode = {
    // Pass 1: Get all nodes individually and put them in a map
    var idToNode: Map[String, LinkableContainerTreeNode] = Map()
    val nodes: Seq[LinkableContainerTreeNode] =
      for {
        m <- l
        if !m.contains(StartNodeId)
      } yield {
        val nodeType: String = m.get(Type) match {
          case Some(l: Seq[_]) => l.last.toString
          case None => throw new IllegalArgumentException(s"Type is required")
          case _ => ???
        }
        val nodeTags: Set[String] = m.get(Type) match {
          case Some(l: Seq[_]) => l.map(_.toString).toSet
          case None => throw new IllegalArgumentException(s"Type is required")
          case _ => ???
        }
        val nodeName = nodeType
        val simpleFields =
          for {
            k <- m.keys
            if !SpecialProperties.contains(k)
          } yield {
            val nodeValue = m.get(k) match {
              case Some(s: String) => s
              case Some(ns) => ns.toString
              case None => null
            }
            SimpleTerminalTreeNode(k, nodeValue)
          }
        val ctn = new LinkableContainerTreeNode(nodeName, nodeTags + TreeNode.Dynamic, simpleFields.toSeq)
        val nodeId: String = requiredStringEntry(m, NodeId)
        idToNode += (nodeId -> ctn)
        ctn
      }

    // Create the linkages
    for {
      m <- l
      if m.contains(StartNodeId)
    } {
      val startNodeId: String = requiredStringEntry(m, StartNodeId)
      val endNodeId: String = requiredStringEntry(m, EndNodeId)
      val cardinality: Cardinality =
        Cardinality(defaultedStringEntry(m, CardinalityStr, Cardinality.One2One))
      val link: String = requiredStringEntry(m, Type)
      logger.debug(s"Creating link from $startNodeId to $endNodeId")
      idToNode.get(startNodeId) match {
        case Some(parent) => parent.link(
          idToNode.getOrElse(endNodeId,
            throw new IllegalArgumentException(s"Cannot link to end node $endNodeId: not found")),
          link,
          cardinality)
        case None =>
          throw new IllegalArgumentException(s"Cannot link to start node $startNodeId: not found")
      }
    }

    if (nodes.nonEmpty) nodes.head else new EmptyContainerGraphNode
  }

  private def requiredStringEntry(m: Map[String,Any], key: String): String =
    m.get(key) match {
      case None => throw new IllegalArgumentException(s"Property [$key] was required, but not found in map with keys [${m.keySet.mkString(",")}]")
      case Some(s: String) => s
      case Some(x) => x.toString
    }

  private def defaultedStringEntry(m: Map[String,Any], key: String, default: String): String =
    m.get(key) match {
      case None => default
      case Some(s: String) => s
      case Some(x) => x.toString
    }
} 

package com.atomist.rug.kind.docker

import _root_.java.util
import java.io.InputStreamReader
import javax.script.{Invocable, ScriptEngineManager}

import com.atomist.util.Utils.withCloseable
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.springframework.core.io.ClassPathResource

import scala.collection.JavaConverters._

object DockerfileParser {

  val mapper = new ObjectMapper().registerModule(DefaultScalaModule)
  val consoleJs =
    """
      |console = {
      |   log: print,
      |   warn: print,
      |   error: print
      |};
    """.stripMargin
  def parse(content: String): Dockerfile = {
    val param = Option(content).getOrElse("")
    val content1 = param.replace("\r\n", "\n").replace("\r", "\n")
    withCloseable(new ClassPathResource("docker/parser.js").getInputStream)(is => {
      withCloseable(new InputStreamReader(is))(reader => {
        try {
          val engine = new ScriptEngineManager(null).getEngineByName("nashorn")
          engine.eval(consoleJs)
          engine.eval(reader)
          val invocable = engine.asInstanceOf[Invocable]
          val result = invocable.invokeFunction("parse", content1, Map("includeComments" -> "true").asJava)
          val lines = result match {
            case map: util.Map[AnyRef @unchecked, AnyRef @unchecked] =>
              map.asScala.values.map(c => mapper.convertValue(c, classOf[DockerfileLine])).toSeq
            case _ => throw new IllegalArgumentException("Failed to parse content")
          }
          new Dockerfile(lines)
        } catch {
          case e: Exception =>
            throw DockerfileException("Failed to parse Dockerfile", e)
        }
      })
    })
  }
} 

package fr.davit.akka.http.metrics.dropwizard.marshalling

import java.io.StringWriter

import akka.http.scaladsl.marshalling.{Marshaller, ToEntityMarshaller}
import akka.http.scaladsl.model.{ContentTypes, HttpEntity}
import com.fasterxml.jackson.databind.ObjectMapper
import fr.davit.akka.http.metrics.dropwizard.DropwizardRegistry

trait DropwizardMarshallers {

  implicit val registryToEntityMarshaller: ToEntityMarshaller[DropwizardRegistry] = {

    val writer = new ObjectMapper().writer()

    Marshaller.opaque { registry =>
      val output = new StringWriter()
      try {
        writer.writeValue(output, registry.underlying)
        HttpEntity(output.toString).withContentType(ContentTypes.`application/json`)
      } finally {
        output.close()
      }
    }
  }
}

object DropwizardMarshallers extends DropwizardMarshallers 

package com.netflix.oss.tools.osstrackerscraper

import com.fasterxml.jackson.databind.introspect.VisibilityChecker.Std
import org.scalatest.FunSuite
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility._
import org.kohsuke.github.{GHIssue}

class GitHubAccessTest extends FunSuite {
  test("Should correctly count issues based on label") {
    // copied from org.kohsuke.github.Requestor
    //val github = GitHub.connect("fake", "fake")
    val mapper = new ObjectMapper
    mapper.setVisibilityChecker(new Std(NONE, NONE, NONE, NONE, ANY));
    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)

    val issuesJSON = scala.io.Source.fromResource("security_monkey-issues.json").mkString
    val issues = mapper.readValue(issuesJSON, classOf[Array[GHIssue]])

    val access = new GithubAccess("a", "a", false)

    val stats = access.getIssuesStats(new Array[GHIssue](0), issues)
    assert(stats.openCountTrulyOpen == 23)

    val issuesJSON2 = scala.io.Source.fromResource("hollow-issues.json").mkString
    val issues2 = mapper.readValue(issuesJSON2, classOf[Array[GHIssue]])

    val stats2 = access.getIssuesStats(new Array[GHIssue](0), issues2)
    assert(stats2.openCountTrulyOpen == 13)
  }
} 

package org.apache.spark.sql.execution.streaming.http

import java.nio.ByteBuffer
import org.apache.spark.serializer.SerializerInstance
import org.apache.spark.serializer.DeserializationStream
import org.apache.spark.serializer.SerializationStream
import java.io.OutputStream
import java.io.InputStream
import scala.reflect.ClassTag
import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.spark.SparkConf
import org.apache.spark.serializer.JavaSerializer
import org.apache.spark.serializer.KryoSerializer


object SerializerFactory {
	val DEFAULT = new SerializerFactory {
		override def getSerializerInstance(serializerName: String): SerializerInstance = {
			serializerName.toLowerCase() match {
				case "kryo" ⇒
					new KryoSerializer(new SparkConf()).newInstance();
				case "java" ⇒
					new JavaSerializer(new SparkConf()).newInstance();
				case _ ⇒ throw new InvalidSerializerNameException(serializerName);
			}
		}
	}
}

trait SerializerFactory {
	def getSerializerInstance(serializerName: String): SerializerInstance;
} 

package org.ebayopensource.regression.example

import java.net.URI

import org.ebayopensource.regression.internal.components.continuation.Continuation
import org.ebayopensource.regression.internal.http.{HTTPRequest, HTTPResponse}
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper

import scala.util.Try

endpoint/shopping/items/item[0-9]")) {
      val map = getMap(resp.body.get)
      Seq(HTTPRequest(new URI(map.get("seller").get.asInstanceOf[String]), request.headers, request.method, None))
    }
    else {
      Seq()
    }
  }

  def getMap(json: String) : Map[String, Object] = {
    val mapper = new ObjectMapper() with ScalaObjectMapper
    mapper.registerModule(DefaultScalaModule)
    mapper.readValue[Map[String, Object]](json)
  }
} 

package de.leanovate.swaggercheck.schema

import java.io.InputStream

import com.fasterxml.jackson.annotation.{JsonCreator, JsonProperty}
import com.fasterxml.jackson.core.JsonFactory
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
import com.fasterxml.jackson.databind.{DeserializationFeature, JsonNode, MappingJsonFactory, ObjectMapper}
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import de.leanovate.swaggercheck.schema.jackson.JsonSchemaModule
import de.leanovate.swaggercheck.schema.model.{Definition, Parameter}

import scala.collection.JavaConverters._
import scala.io.Source

@JsonDeserialize(builder = classOf[SwaggerAPIBuilder])
case class SwaggerAPI(
                       basePath: Option[String],
                       paths: Map[String, Map[String, Operation]],
                       definitions: Map[String, Definition]
                     )

object SwaggerAPI {
  val jsonMapper = objectMapper(new MappingJsonFactory())
  val yamlMapper = objectMapper(new YAMLFactory())

  def parse(jsonOrYaml: String): SwaggerAPI = {
    val mapper = if (jsonOrYaml.trim().startsWith("{")) jsonMapper else yamlMapper
    mapper.readValue(jsonOrYaml, classOf[SwaggerAPI])
  }

  def parse(swaggerInput: InputStream): SwaggerAPI = {
    parse(Source.fromInputStream(swaggerInput).mkString)
  }

  def objectMapper(jsonFactory: JsonFactory): ObjectMapper = {
    val mapper = new ObjectMapper(jsonFactory)
    mapper.registerModule(DefaultScalaModule)
    mapper.registerModule(JsonSchemaModule)
    mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
    mapper
  }
}

class SwaggerAPIBuilder @JsonCreator()(
                                        @JsonProperty("basePath") basePath: Option[String],
                                        @JsonProperty("consumes") consumes: Option[Seq[String]],
                                        @JsonProperty("produces") produces: Option[Seq[String]],
                                        @JsonProperty("paths") paths: Option[Map[String, JsonNode]],
                                        @JsonProperty("definitions") definitions: Option[Map[String, Definition]],
                                        @JsonProperty("parameters") globalParameters: Option[Map[String, Parameter]]
                                      ) {
  def build(): SwaggerAPI = {
    val defaultConsumes = consumes.map(_.toSet).getOrElse(Set.empty)
    val defaultProduces = produces.map(_.toSet).getOrElse(Set.empty)
    SwaggerAPI(basePath,
      paths.getOrElse(Map.empty).map {
        case (path, pathDefinition) =>
          val defaultParameters = Option(pathDefinition.get("parameters")).map {
             node =>
               node.iterator().asScala.map {
                 element => SwaggerAPI.jsonMapper.treeToValue(element, classOf[OperationParameter])
               }.toSeq
          }.getOrElse(Seq.empty)

          basePath.map(_ + path).getOrElse(path) -> pathDefinition.fields().asScala.filter(_.getKey != "parameters").map {
            entry =>
              val operation = SwaggerAPI.jsonMapper.treeToValue(entry.getValue, classOf[Operation])
              entry.getKey.toUpperCase -> operation.withDefaults(defaultParameters, defaultConsumes, defaultProduces).resolveGlobalParameters(globalParameters.getOrElse(Map()))
          }.toMap
      },
      definitions.getOrElse(Map.empty))
  }
} 

package de.leanovate.swaggercheck.schema.jackson

import com.fasterxml.jackson.annotation.{JsonCreator, JsonProperty}
import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import com.fasterxml.jackson.databind.annotation.JsonDeserialize
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import de.leanovate.swaggercheck.schema.model._

@JsonDeserialize(builder = classOf[DefinitionBuilder])
trait DefinitionMixin {

}

class DefinitionBuilder @JsonCreator()(
                                        @JsonProperty("type") schemaType: Option[String] = None,
                                        @JsonProperty("allOf") allOf: Option[Seq[Definition]] = None,
                                        @JsonProperty("enum") enum: Option[List[String]] = None,
                                        @JsonProperty("exclusiveMinimum") exclusiveMinimum: Option[Boolean] = None,
                                        @JsonProperty("exclusiveMaximum") exclusiveMaximum: Option[Boolean] = None,
                                        @JsonProperty("format") format: Option[String] = None,
                                        @JsonProperty("items") items: Option[Definition] = None,
                                        @JsonProperty("minItems") minItems: Option[Int] = None,
                                        @JsonProperty("maxItems") maxItems: Option[Int] = None,
                                        @JsonProperty("minimum") minimum: Option[BigDecimal] = None,
                                        @JsonProperty("maximum") maximum: Option[BigDecimal] = None,
                                        @JsonProperty("minLength") minLength: Option[Int] = None,
                                        @JsonProperty("maxLength") maxLength: Option[Int] = None,
                                        @JsonProperty("oneOf") oneOf: Option[Seq[Definition]] = None,
                                        @JsonProperty("pattern") pattern: Option[String] = None,
                                        @JsonProperty("properties") properties: Option[Map[String, Definition]] = None,
                                        @JsonProperty("additionalProperties") additionalPropertiesNode: Option[JsonNode] = None,
                                        @JsonProperty("required") required: Option[Set[String]] = None,
                                        @JsonProperty("$ref") ref: Option[String] = None,
                                        @JsonProperty("uniqueItems") uniqueItems: Option[Boolean] = None) {
  val additionalProperties = additionalPropertiesNode.map {
    case node if node.isBoolean => Left(node.isBinary)
    case node => Right(DefinitionBuilder.mapper.treeToValue(node, classOf[Definition]))
  }

  def build(): Definition = {
    Definition.build(
      schemaType,
      allOf,
      enum,
      exclusiveMinimum,
      exclusiveMaximum,
      format,
      items,
      minItems,
      maxItems,
      minimum,
      maximum,
      minLength,
      maxLength,
      oneOf,
      pattern,
      properties,
      additionalProperties,
      required,
      ref,
      uniqueItems
    )
  }
}

object DefinitionBuilder {
  val mapper = new ObjectMapper().registerModule(DefaultScalaModule).registerModule(JsonSchemaModule)
} 

package de.leanovate.swaggercheck.schema.jackson

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import de.leanovate.swaggercheck.schema.model.{StringDefinition, ObjectDefinition, Definition}
import org.scalatest.{MustMatchers, WordSpec}

class SchemaModuleSpec extends WordSpec with MustMatchers {
  val mapper = new ObjectMapper().registerModule(DefaultScalaModule).registerModule(JsonSchemaModule)

  "SchemaModule" should {
    "deserialize object_definition" in {
      val ObjectDefinition(required, properties, additionalProperties) = mapper.readValue(getClass.getClassLoader.getResource("object_definition.json"), classOf[Definition])

      required mustBe Some(Set("field1"))
      properties mustBe Some(Map("field1" -> StringDefinition(None, None, None, None, None)))
      additionalProperties mustBe Left(true)
    }
  }
} 

package tech.sourced.gemini

import java.io.{File, PrintWriter}

import scala.io.Source

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper



case class OrderedDocFreq(docs: Int, tokens: IndexedSeq[String], df: collection.Map[String, Int]) {
  def saveToJson(filename: String): Unit = {
    val mapper = new ObjectMapper() with ScalaObjectMapper
    mapper.registerModule(DefaultScalaModule)
    val out = new PrintWriter(filename)
    mapper.writeValue(out, Map(
      "docs" -> docs,
      "tokens" -> tokens,
      "df" -> df
    ))
    out.close()
  }
}

object OrderedDocFreq {
  def fromJson(file: File): OrderedDocFreq = {
    val docFreqMap = parseFile[Map[String, Any]](file)
    val docs = docFreqMap.get("docs") match {
      case Some(v) => v.asInstanceOf[Int]
      case None => throw new RuntimeException(s"Can not parse key 'docs' in docFreq:${file.getAbsolutePath}")
    }
    val df = docFreqMap.get("df") match {
      case Some(v) => v.asInstanceOf[Map[String, Int]]
      case None => throw new RuntimeException(s"Can not parse key 'df' in docFreq:${file.getAbsolutePath}")
    }
    val tokens = docFreqMap.get("tokens") match {
      case Some(v) => v.asInstanceOf[List[String]].toArray
      case None => throw new RuntimeException(s"Can not parse key 'tokens' in docFreq:${file.getAbsolutePath}")
    }
    OrderedDocFreq(docs, tokens, df)
  }

  def parseFile[T: Manifest](file: File): T = {
    val json = Source.fromFile(file)
    val mapper = new ObjectMapper with ScalaObjectMapper
    mapper.registerModule(DefaultScalaModule)
    mapper.readValue[T](json.reader)
  }
} 

package magellan

import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.spark.sql.types._
import org.scalatest.FunSuite

class LineSuite extends FunSuite with TestSparkContext {

  test("bounding box") {
    val line = Line(Point(0.0, 1.0), Point(1.0, 0.5))
    val BoundingBox(xmin, ymin, xmax, ymax) = line.boundingBox
    assert(xmin === 0.0)
    assert(ymin === 0.5)
    assert(xmax === 1.0)
    assert(ymax === 1.0)
  }

  test("touches") {
    val x = Line(Point(0.0, 0.0), Point(1.0, 1.0))
    val y = Line(Point(1.0, 1.0), Point(2.0, 2.0))
    val z = Line(Point(0.5, 0.5), Point(0.5, 0.0))
    val w = Line(Point(1.0, -1.0), Point(-1.0, -1.0))

    // test touches line
    assert(x.touches(y))
    assert(x.touches(z))
    assert(!x.touches(w))
  }

  test("intersects") {
    val x = Line(Point(0.0, 0.0), Point(1.0, 1.0))
    val y = Line(Point(1.0, 0.0), Point(0.0, 0.1))
    val z = Line(Point(0.5, 0.0), Point(1.0, 0.5))
    val w = Line(Point(1.0, -1.0), Point(-1.0, -1.0))
    val l = Line(Point(0.0, -1.0), Point(0.0, -1.0))
    val t = Line(Point(0.5, 0.5), Point(0.5, 0.0))
    val u = Line(Point(1.0, 1.0), Point(2.0, 2.0))
    val s = Line(Point(1.0, 1.0), Point(1.0, -1.0))
    assert(x.intersects(y))
    assert(!x.intersects(z))
    assert(w.intersects(l))
    assert(x.intersects(t))
    assert(u.intersects(s))
  }

  test("serialization") {
    val lineUDT = new LineUDT
    val line = Line(Point(0.0, 1.0), Point(1.0, 0.5))
    val BoundingBox(xmin, ymin, xmax, ymax) = line.boundingBox
    val row = lineUDT.serialize(line)
    assert(row.getInt(0) === line.getType())
    assert(row.getDouble(1) === xmin)
    assert(row.getDouble(2) === ymin)
    assert(row.getDouble(3) === xmax)
    assert(row.getDouble(4) === ymax)
    val serializedLine = lineUDT.deserialize(row)
    assert(line.equals(serializedLine))
  }

  test("contains line") {
    val x = Line(Point(0.0, 0.0), Point(1.0, 1.0))
    val y = Line(Point(0.0, 0.0), Point(0.5, 0.5))
    val z = Line(Point(0.0, 0.0), Point(1.5, 1.5))
    val w = Line(Point(0.5, 0.5), Point(0.0, 0.0))
    val u = Line(Point(0.0, 0.0), Point(0.0, 1.0))
    assert(x.contains(y))
    assert(!x.contains(z))
    assert(x.contains(w))
    assert(!x.contains(u))
  }

  test("contains point") {
    val y = Line(Point(0.5, 0.0), Point(0.0, 0.5))
    assert(y.contains(Point(0.5, 0.0)))
  }

  test("jackson serialization") {
    val s = new ObjectMapper().writeValueAsString(Line(Point(0.0, 0.0), Point(1.0, 1.0)))
    assert(s.contains("boundingBox"))
    assert(s.contains("start"))
    assert(s.contains("end"))
  }
} 

package magellan

import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.spark.sql.types._
import org.scalatest.FunSuite

class PointSuite extends FunSuite with TestSparkContext {

  test("bounding box") {
    val point = Point(1.0, 1.0)
    val BoundingBox(xmin, ymin, xmax, ymax) = point.boundingBox
    assert(xmin === 1.0)
    assert(ymin === 1.0)
    assert(xmax === 1.0)
    assert(ymax === 1.0)
  }

  test("serialization") {
    val point = Point(1.0, 1.0)
    val pointUDT = new PointUDT
    val BoundingBox(xmin, ymin, xmax, ymax) = point.boundingBox
    val row = pointUDT.serialize(point)
    assert(row.getInt(0) === point.getType())
    assert(row.getDouble(1) === xmin)
    assert(row.getDouble(2) === ymin)
    assert(row.getDouble(3) === xmax)
    assert(row.getDouble(4) === ymax)
    val serializedPoint = pointUDT.deserialize(row)
    assert(point.equals(serializedPoint))
  }

  test("point udf") {
    val sqlContext = this.sqlContext
    import sqlContext.implicits._
    val points = sc.parallelize(Seq((-1.0, -1.0), (-1.0, 1.0), (1.0, -1.0))).toDF("x", "y")
    import org.apache.spark.sql.functions.udf
    val toPointUDF = udf{(x:Double,y:Double) => Point(x,y) }
    val point = points.withColumn("point", toPointUDF('x, 'y))
      .select('point)
      .first()(0)
      .asInstanceOf[Point]

    assert(point.getX() === -1.0)
    assert(point.getY() === -1.0)
  }

  test("jackson serialization") {
    val s = new ObjectMapper().writeValueAsString(Point(1.0, 1.0))
    assert(s.contains("boundingBox"))
    assert(s.contains("x"))
    assert(s.contains("y"))
  }

  test("within circle") {
    assert(Point(0.0, 0.0) withinCircle (Point(0.5, 0.5), 0.75))
    assert(!(Point(0.0, 0.0) withinCircle (Point(0.5, 0.5), 0.5)))
  }

  test("buffer point") {
    val polygon = Point(0.0, 1.0).buffer(0.5)
    assert(polygon.getNumRings() === 1)
    // check that [0.0, 0.75] is within this polygon
    assert(polygon.contains(Point(0.0, 0.75)))
    // check that [0.4, 1.0] is within this polygon
    assert(polygon.contains(Point(0.4, 1.0)))
    // check that [0.6, 1.0] is outside this polygon
    assert(!polygon.contains(Point(0.6, 1.0)))
  }
} 

package net.manub.embeddedkafka.serializers

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.apache.kafka.common.errors.SerializationException
import org.apache.kafka.common.serialization.Serializer


class TestJsonSerializer[T] extends Serializer[T] {
  private val mapper = new ObjectMapper().registerModule(DefaultScalaModule)

  override def serialize(topic: String, data: T): Array[Byte] =
    Option(data).map { _ =>
      try mapper.writeValueAsBytes(data)
      catch {
        case e: Exception =>
          throw new SerializationException("Error serializing JSON message", e)
      }
    }.orNull
} 

package net.manub.embeddedkafka.serializers

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.apache.kafka.common.errors.SerializationException
import org.apache.kafka.common.serialization.Deserializer

import scala.reflect.ClassTag


class TestJsonDeserializer[T](implicit tag: ClassTag[T], ev: Null <:< T)
    extends Deserializer[T] {
  private val mapper = new ObjectMapper().registerModule(DefaultScalaModule)

  override def deserialize(topic: String, bytes: Array[Byte]): T =
    Option(bytes).map { _ =>
      try mapper.readValue(
        bytes,
        tag.runtimeClass.asInstanceOf[Class[T]]
      )
      catch {
        case e: Exception => throw new SerializationException(e)
      }
    }.orNull
} 

package com.wix.e2e.http.json

import java.lang.reflect.{ParameterizedType, Type}

import com.fasterxml.jackson.core.`type`.TypeReference
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module
import com.fasterxml.jackson.datatype.joda.JodaModule
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
import com.fasterxml.jackson.module.paramnames.ParameterNamesModule
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.wix.e2e.http.api.Marshaller

class JsonJacksonMarshaller extends Marshaller {

  def unmarshall[T : Manifest](jsonStr: String): T = objectMapper.readValue(jsonStr, typeReference[T])
  def marshall[T](t: T): String = objectMapper.writeValueAsString(t)

  def configure: ObjectMapper = objectMapper

  private val objectMapper = new ObjectMapper()
                                    .registerModule(new Jdk8Module().configureAbsentsAsNulls(true))
                                    .registerModules(new JodaModule, new ParameterNamesModule, new JavaTimeModule) // time modules
                                    .registerModule(new DefaultScalaModule)
                                    .disable( WRITE_DATES_AS_TIMESTAMPS )

  private def typeReference[T: Manifest] = new TypeReference[T] {
    override def getType = typeFromManifest(manifest[T])
  }

  private def typeFromManifest(m: Manifest[_]): Type = {
    if (m.typeArguments.isEmpty) { m.runtimeClass }
    else new ParameterizedType {
      def getRawType = m.runtimeClass
      def getActualTypeArguments = m.typeArguments.map(typeFromManifest).toArray
      def getOwnerType = null
    }
  }
} 

package com.wix.e2e.http.json

import java.time.LocalDateTime
import java.util.Optional

import com.fasterxml.jackson.databind.ObjectMapper
import com.wix.e2e.http.api.Marshaller
import com.wix.e2e.http.json.MarshallingTestObjects.SomeCaseClass
import com.wix.test.random._
import org.joda.time.DateTimeZone.UTC
import org.joda.time.{DateTime, DateTimeZone}
import org.specs2.mutable.Spec
import org.specs2.specification.Scope

class JsonJacksonMarshallerTest extends Spec {

  trait ctx extends Scope {
    val someStr = randomStr
    val javaDateTime = LocalDateTime.now()
    val someCaseClass = SomeCaseClass(randomStr, randomInt)
    val dateTime = new DateTime
    val dateTimeUTC = new DateTime(UTC)

    val marshaller: Marshaller = new JsonJacksonMarshaller
  }


  "JsonJacksonMarshaller" should {

    "marshall scala option properly" in new ctx {
      marshaller.unmarshall[Option[String]](
        marshaller.marshall( Some(someStr) )
      ) must beSome(someStr)
    }

    "marshall scala case classes properly" in new ctx {
      marshaller.unmarshall[SomeCaseClass](
        marshaller.marshall( someCaseClass )
      ) must_=== someCaseClass
    }

    "marshall datetime without zone" in new ctx {
      marshaller.unmarshall[DateTime](
        marshaller.marshall( dateTime.withZone(DateTimeZone.getDefault) )
      ) must_=== dateTime.withZone(UTC)
    }

    "marshall date time to textual format in UTC" in new ctx {
      marshaller.marshall( dateTime ) must contain(dateTime.withZone(UTC).toString)
    }


    "marshall java.time objects" in new ctx {
      marshaller.unmarshall[LocalDateTime](
        marshaller.marshall( javaDateTime )
      ) must_=== javaDateTime

    }

    "marshall java 8 Optional" in new ctx {
      marshaller.unmarshall[Optional[DateTime]](
        marshaller.marshall( dateTimeUTC )
      ) must_=== Optional.of(dateTimeUTC)

      marshaller.unmarshall[Optional[SomeCaseClass]](
        marshaller.marshall( someCaseClass )
      ) must_=== Optional.of(someCaseClass)
    }

    "expose jackson object mapper to allow external configuration" in new ctx {
      marshaller.asInstanceOf[JsonJacksonMarshaller].configure must beAnInstanceOf[ObjectMapper]
    }
  }
}

object MarshallingTestObjects {
  case class SomeCaseClass(s: String, i: Int)
} 

package com.github.aseigneurin.kafka.serialization.scala

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.typesafe.scalalogging.LazyLogging

import scala.reflect.{ClassTag, classTag}

class JsonSerde[T >: Null : ClassTag] extends BaseSerde[T] with LazyLogging {

  val mapper = new ObjectMapper
  mapper.registerModule(DefaultScalaModule)

  override def deserialize(topic: String, data: Array[Byte]): T = data match {
    case null => null
    case _ =>
      try {
        mapper.readValue(data, classTag[T].runtimeClass.asInstanceOf[Class[T]])
      } catch {
        case e: Exception =>
          val jsonStr = new String(data, "UTF-8")
          logger.warn(s"Failed parsing ${jsonStr}", e)
          null
      }
  }

  override def serialize(topic: String, obj: T): Array[Byte] = {
    mapper.writeValueAsBytes(obj)
  }

} 

package com.stratio.sparta.serving.core.models.dto

import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.node.ArrayNode

import scala.collection.JavaConverters._
import scala.util.Try


object LoggedUser{

  implicit def jsonToDto(stringJson: String): Option[LoggedUser] = {
    if (stringJson.trim.isEmpty) None
    else {
      implicit val json = new ObjectMapper().readTree(stringJson)
      Some(LoggedUser(getValue(LoggedUserConstant.infoIdTag), getValue(LoggedUserConstant.infoNameTag),
        getValue(LoggedUserConstant.infoMailTag, Some(LoggedUserConstant.dummyMail)),
        getValue(LoggedUserConstant.infoGroupIDTag), getArrayValues(LoggedUserConstant.infoGroupsTag),
        getArrayValues(LoggedUserConstant.infoRolesTag)))
    }
  }

  private def getValue(tag: String, defaultElse: Option[String]= None)(implicit json: JsonNode) : String = {
    Option(json.findValue(tag)) match {
      case Some(jsonValue) =>
        defaultElse match{
          case Some(value) => Try(jsonValue.asText()).getOrElse(value)
          case None => Try(jsonValue.asText()).get
        }
      case None =>
        defaultElse match {
          case Some(value) => value
          case None => ""
        }
    }
  }

  private def getArrayValues(tag:String)(implicit jsonNode: JsonNode): Seq[String] = {
    Option(jsonNode.findValue(tag)) match {
      case Some(roles: ArrayNode) => roles.asScala.map(x => x.asText()).toSeq
      case None => Seq.empty[String]
    }
  }
}

case class LoggedUser(id: String, name: String, email: String, gid: String,
                      groups:Seq[String], roles: Seq[String]){

  def isAuthorized(securityEnabled: Boolean, allowedRoles: Seq[String] = LoggedUserConstant.allowedRoles): Boolean = {
    if(securityEnabled){
      roles.intersect(allowedRoles).nonEmpty
    } else true
  }

} 

package org.apache.livy.test

import java.io.File
import java.net.URI
import java.util.concurrent.{TimeUnit, Future => JFuture}
import javax.servlet.http.HttpServletResponse

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.scalatest.BeforeAndAfterAll

import org.apache.http.client.methods.HttpGet

import org.apache.livy._
import org.apache.livy.client.common.HttpMessages._
import org.apache.livy.sessions.SessionKindModule
import org.apache.livy.test.framework.BaseIntegrationTestSuite
import org.apache.livy.test.jobs.spark2._

class Spark2JobApiIT extends BaseIntegrationTestSuite with BeforeAndAfterAll with Logging {

  private var client: LivyClient = _
  private var sessionId: Int = _
  private val mapper = new ObjectMapper()
    .registerModule(DefaultScalaModule)
    .registerModule(new SessionKindModule())

  override def afterAll(): Unit = {
    super.afterAll()

    if (client != null) {
      client.stop(true)
    }

    livyClient.connectSession(sessionId).stop()
  }

  test("create a new session and upload test jar") {
    val prevSessionCount = sessionList().total
    val tempClient = createClient(livyEndpoint)

    try {
      // Figure out the session ID by poking at the REST endpoint. We should probably expose this
      // in the Java API.
      val list = sessionList()
      assert(list.total === prevSessionCount + 1)
      val tempSessionId = list.sessions(0).id

      livyClient.connectSession(tempSessionId).verifySessionIdle()
      waitFor(tempClient.uploadJar(new File(testLib)))

      client = tempClient
      sessionId = tempSessionId
    } finally {
      if (client == null) {
        try {
          if (tempClient != null) {
            tempClient.stop(true)
          }
        } catch {
          case e: Exception => warn("Error stopping client.", e)
        }
      }
    }
  }

  test("run spark2 job") {
    assume(client != null, "Client not active.")
    val result = waitFor(client.submit(new SparkSessionTest()))
    assert(result === 3)
  }

  test("run spark2 dataset job") {
    assume(client != null, "Client not active.")
    val result = waitFor(client.submit(new DatasetTest()))
    assert(result === 2)
  }

  private def waitFor[T](future: JFuture[T]): T = {
    future.get(60, TimeUnit.SECONDS)
  }

  private def sessionList(): SessionList = {
    val httpGet = new HttpGet(s"$livyEndpoint/sessions/")
    val r = livyClient.httpClient.execute(httpGet)
    val statusCode = r.getStatusLine().getStatusCode()
    val responseBody = r.getEntity().getContent
    val sessionList = mapper.readValue(responseBody, classOf[SessionList])
    r.close()

    assert(statusCode ==  HttpServletResponse.SC_OK)
    sessionList
  }

  private def createClient(uri: String): LivyClient = {
    new LivyClientBuilder().setURI(new URI(uri)).build()
  }
} 

package org.apache.livy.server.recovery

import scala.reflect.{classTag, ClassTag}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule

import org.apache.livy.{LivyConf, Logging}
import org.apache.livy.server.recovery.ZooKeeperManager
import org.apache.livy.sessions.SessionKindModule
import org.apache.livy.sessions.SessionManager._

protected trait JsonMapper {
  protected val mapper = new ObjectMapper()
    .registerModule(DefaultScalaModule)
    .registerModule(new SessionKindModule())

  def serializeToBytes(value: Object): Array[Byte] = mapper.writeValueAsBytes(value)

  def deserialize[T: ClassTag](json: Array[Byte]): T =
    mapper.readValue(json, classTag[T].runtimeClass.asInstanceOf[Class[T]])
}


object StateStore extends Logging {
  private[this] var stateStore: Option[StateStore] = None

  def init(livyConf: LivyConf, zkManager: Option[ZooKeeperManager] = None): Unit = synchronized {
    if (stateStore.isEmpty) {
      val fileStateStoreClassTag = pickStateStore(livyConf)
      if (fileStateStoreClassTag == classOf[ZooKeeperStateStore]) {
        stateStore = Option(fileStateStoreClassTag.
          getDeclaredConstructor(classOf[LivyConf], classOf[ZooKeeperManager])
          .newInstance(livyConf, zkManager.get).asInstanceOf[StateStore])
      } else {
        stateStore = Option(fileStateStoreClassTag.getDeclaredConstructor(classOf[LivyConf])
          .newInstance(livyConf).asInstanceOf[StateStore])
      }
      info(s"Using ${stateStore.get.getClass.getSimpleName} for recovery.")
    }
  }

  def cleanup(): Unit = synchronized {
    stateStore = None
  }

  def get: StateStore = {
    assert(stateStore.isDefined, "StateStore hasn't been initialized.")
    stateStore.get
  }

  private[recovery] def pickStateStore(livyConf: LivyConf): Class[_] = {
    livyConf.get(LivyConf.RECOVERY_MODE) match {
      case SESSION_RECOVERY_MODE_OFF => classOf[BlackholeStateStore]
      case SESSION_RECOVERY_MODE_RECOVERY =>
        livyConf.get(LivyConf.RECOVERY_STATE_STORE) match {
          case "filesystem" => classOf[FileSystemStateStore]
          case "zookeeper" => classOf[ZooKeeperStateStore]
          case ss => throw new IllegalArgumentException(s"Unsupported state store $ss")
        }
      case rm => throw new IllegalArgumentException(s"Unsupported recovery mode $rm")
    }
  }
} 

package org.apache.livy.server.batch

import com.fasterxml.jackson.databind.{JsonMappingException, ObjectMapper}
import org.scalatest.FunSpec

import org.apache.livy.LivyBaseUnitTestSuite

class CreateBatchRequestSpec extends FunSpec with LivyBaseUnitTestSuite {

  private val mapper = new ObjectMapper()
    .registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)

  describe("CreateBatchRequest") {

    it("should have default values for fields after deserialization") {
      val json = """{ "file" : "foo" }"""
      val req = mapper.readValue(json, classOf[CreateBatchRequest])
      assert(req.file === "foo")
      assert(req.proxyUser === None)
      assert(req.args === List())
      assert(req.className === None)
      assert(req.jars === List())
      assert(req.pyFiles === List())
      assert(req.files === List())
      assert(req.driverMemory === None)
      assert(req.driverCores === None)
      assert(req.executorMemory === None)
      assert(req.executorCores === None)
      assert(req.numExecutors === None)
      assert(req.archives === List())
      assert(req.queue === None)
      assert(req.name === None)
      assert(req.conf === Map())
    }

  }

} 

package org.apache.livy.server.interactive

import com.fasterxml.jackson.databind.ObjectMapper
import org.scalatest.FunSpec

import org.apache.livy.LivyBaseUnitTestSuite
import org.apache.livy.sessions.{PySpark, SessionKindModule}

class CreateInteractiveRequestSpec extends FunSpec with LivyBaseUnitTestSuite {

  private val mapper = new ObjectMapper()
    .registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
    .registerModule(new SessionKindModule())

  describe("CreateInteractiveRequest") {

    it("should have default values for fields after deserialization") {
      val json = """{ "kind" : "pyspark" }"""
      val req = mapper.readValue(json, classOf[CreateInteractiveRequest])
      assert(req.kind === PySpark)
      assert(req.proxyUser === None)
      assert(req.jars === List())
      assert(req.pyFiles === List())
      assert(req.files === List())
      assert(req.driverMemory === None)
      assert(req.driverCores === None)
      assert(req.executorMemory === None)
      assert(req.executorCores === None)
      assert(req.numExecutors === None)
      assert(req.archives === List())
      assert(req.queue === None)
      assert(req.name === None)
      assert(req.conf === Map())
    }

  }

} 

package com.twilio.open.streaming.trend.discovery.config

import java.io.File

import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.scala.DefaultScalaModule

sealed trait Configuration extends Product with Serializable

@SerialVersionUID(101L)
case class AppConfiguration(
  @JsonProperty appName: String,
  @JsonProperty triggerInterval: String, // "30 seconds", "1 minute"
  @JsonProperty outputMode: String,
  @JsonProperty checkpointPath: String,
  @JsonProperty("windowInterval") windowIntervalMinutes: Long,
  @JsonProperty("watermarkInterval") watermarkIntervalMinutes: Long,
  @JsonProperty("core") sparkCoreConfig: Map[String, String],
  @JsonProperty callEventsTopic: KafkaReaderOrWriterConfig)
  extends Configuration with Serializable

trait KafkaConfig {
  val topic: String
  val subscriptionType: String
  val conf: Map[String, String]
}

case class KafkaReaderOrWriterConfig(
  override val topic: String,
  override val subscriptionType: String,
  override val conf: Map[String, String]) extends KafkaConfig with Serializable

object AppConfig {

  private val mapper = new ObjectMapper(new YAMLFactory)
  mapper.registerModule(DefaultScalaModule)

  @volatile
  private var config: AppConfiguration = _

  def apply(resourcePath: String): AppConfiguration = {
    if (config == null)
      synchronized {
        if (config == null)
          config = parse(resourcePath)
      }
    config
  }

  private def parse(resourcePath: String): AppConfiguration = {
    mapper.readValue(new File(resourcePath), classOf[AppConfiguration])
  }

} 

package com.twilio.open.streaming.trend.discovery

import java.io.{ByteArrayInputStream, InputStream}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.google.protobuf.Message
import com.googlecode.protobuf.format.JsonFormat
import com.holdenkarau.spark.testing.{LocalSparkContext, SparkContextProvider}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.SparkSession
import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers, Suite}
import org.slf4j.{Logger, LoggerFactory}

import scala.collection.Seq
import scala.io.Source
import scala.reflect.ClassTag
import scala.reflect.classTag

object TestHelper {
  val log: Logger = LoggerFactory.getLogger("com.twilio.open.streaming.trend.discovery.TestHelper")
  val mapper: ObjectMapper = {
    val m = new ObjectMapper()
    m.registerModule(DefaultScalaModule)
  }

  val jsonFormat: JsonFormat = new JsonFormat

  def loadScenario[T<: Message : ClassTag](file: String): Seq[T] = {
    val fileString = Source.fromFile(file).mkString
    val parsed = mapper.readValue(fileString, classOf[Sceanario])
    parsed.input.map { data =>
      val json = mapper.writeValueAsString(data)
      convert[T](json)
    }
  }

  def convert[T<: Message : ClassTag](json: String): T = {
    val clazz = classTag[T].runtimeClass
    val builder = clazz.getMethod("newBuilder").invoke(clazz).asInstanceOf[Message.Builder]
    try {
      val input: InputStream = new ByteArrayInputStream(json.getBytes())
      jsonFormat.merge(input, builder)
      builder.build().asInstanceOf[T]
    } catch {
      case e: Exception =>
        throw e
    }
  }

}

@SerialVersionUID(1L)
case class KafkaDataFrame(key: Array[Byte], topic: Array[Byte], value: Array[Byte]) extends Serializable

case class Sceanario(input: Seq[Any], expected: Option[Any] = None)

trait SparkSqlTest extends BeforeAndAfterAll with SparkContextProvider {
  self: Suite =>

  @transient var _sparkSql: SparkSession = _
  @transient private var _sc: SparkContext = _

  override def sc: SparkContext = _sc

  def conf: SparkConf

  def sparkSql: SparkSession = _sparkSql

  override def beforeAll() {
    _sparkSql = SparkSession.builder().config(conf).getOrCreate()

    _sc = _sparkSql.sparkContext
    setup(_sc)
    super.beforeAll()
  }

  override def afterAll() {
    try {
      _sparkSql.close()
      _sparkSql = null
      LocalSparkContext.stop(_sc)
      _sc = null
    } finally {
      super.afterAll()
    }
  }

} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.nio.charset.StandardCharsets
import java.text.SimpleDateFormat
import java.util.{Calendar, Locale, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes(StandardCharsets.UTF_8))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'", Locale.US)
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package org.apache.spark.metrics.sink

import java.util.Properties
import java.util.concurrent.TimeUnit
import javax.servlet.http.HttpServletRequest

import com.codahale.metrics.MetricRegistry
import com.codahale.metrics.json.MetricsModule
import com.fasterxml.jackson.databind.ObjectMapper
import org.eclipse.jetty.servlet.ServletContextHandler

import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.ui.JettyUtils._

private[spark] class MetricsServlet(
    val property: Properties,
    val registry: MetricRegistry,
    securityMgr: SecurityManager)
  extends Sink {

  val SERVLET_KEY_PATH = "path"
  val SERVLET_KEY_SAMPLE = "sample"

  val SERVLET_DEFAULT_SAMPLE = false

  val servletPath = property.getProperty(SERVLET_KEY_PATH)

  val servletShowSample = Option(property.getProperty(SERVLET_KEY_SAMPLE)).map(_.toBoolean)
    .getOrElse(SERVLET_DEFAULT_SAMPLE)

  val mapper = new ObjectMapper().registerModule(
    new MetricsModule(TimeUnit.SECONDS, TimeUnit.MILLISECONDS, servletShowSample))

  def getHandlers(conf: SparkConf): Array[ServletContextHandler] = {
    Array[ServletContextHandler](
      createServletHandler(servletPath,
        new ServletParams(request => getMetricsSnapshot(request), "text/json"), securityMgr, conf)
    )
  }

  def getMetricsSnapshot(request: HttpServletRequest): String = {
    mapper.writeValueAsString(registry)
  }

  override def start() { }

  override def stop() { }

  override def report() { }
} 

package instagram.api.yaml

import de.zalando.play.controllers._
import org.scalacheck._
import org.scalacheck.Arbitrary._
import org.scalacheck.Prop._
import org.scalacheck.Test._
import org.specs2.mutable._
import play.api.test.Helpers._
import play.api.test._
import play.api.mvc.MultipartFormData.FilePart
import play.api.mvc._

import org.junit.runner.RunWith
import org.specs2.runner.JUnitRunner
import java.net.URLEncoder
import com.fasterxml.jackson.databind.ObjectMapper

import play.api.http.Writeable
import play.api.libs.Files.TemporaryFile
import play.api.test.Helpers.{status => requestStatusCode_}
import play.api.test.Helpers.{contentAsString => requestContentAsString_}
import play.api.test.Helpers.{contentType => requestContentType_}

import scala.math.BigInt
import scala.math.BigDecimal

import Generators._

    @RunWith(classOf[JUnitRunner])
    class Instagram_api_yamlSpec extends Specification {
        def toPath[T](value: T)(implicit binder: PathBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")
        def toQuery[T](key: String, value: T)(implicit binder: QueryStringBindable[T]): String = Option(binder.unbind(key, value)).getOrElse("")
        def toHeader[T](value: T)(implicit binder: PathBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")

      def checkResult(props: Prop) =
        Test.check(Test.Parameters.default, props).status match {
          case Failed(args, labels) =>
            val failureMsg = labels.mkString("\n") + " given args: " + args.map(_.arg).mkString("'", "', '","'")
            failure(failureMsg)
          case Proved(_) | Exhausted | Passed => success
          case PropException(_, e, labels) =>
            val error = if (labels.isEmpty) e.getLocalizedMessage() else labels.mkString("\n")
            failure(error)
        }

      private def parserConstructor(mimeType: String) = PlayBodyParsing.jacksonMapper(mimeType)

      def parseResponseContent[T](mapper: ObjectMapper, content: String, mimeType: Option[String], expectedType: Class[T]) =
        mapper.readValue(content, expectedType)

} 

package security.api.yaml

import de.zalando.play.controllers._
import org.scalacheck._
import org.scalacheck.Arbitrary._
import org.scalacheck.Prop._
import org.scalacheck.Test._
import org.specs2.mutable._
import play.api.test.Helpers._
import play.api.test._
import play.api.mvc.MultipartFormData.FilePart
import play.api.mvc._

import org.junit.runner.RunWith
import org.specs2.runner.JUnitRunner
import java.net.URLEncoder
import com.fasterxml.jackson.databind.ObjectMapper

import play.api.http.Writeable
import play.api.libs.Files.TemporaryFile
import play.api.test.Helpers.{status => requestStatusCode_}
import play.api.test.Helpers.{contentAsString => requestContentAsString_}
import play.api.test.Helpers.{contentType => requestContentType_}

import de.zalando.play.controllers.ArrayWrapper

import Generators._

    @RunWith(classOf[JUnitRunner])
    class Security_api_yamlSpec extends Specification {
        def toPath[T](value: T)(implicit binder: PathBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")
        def toQuery[T](key: String, value: T)(implicit binder: QueryStringBindable[T]): String = Option(binder.unbind(key, value)).getOrElse("")
        def toHeader[T](value: T)(implicit binder: PathBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")

      def checkResult(props: Prop) =
        Test.check(Test.Parameters.default, props).status match {
          case Failed(args, labels) =>
            val failureMsg = labels.mkString("\n") + " given args: " + args.map(_.arg).mkString("'", "', '","'")
            failure(failureMsg)
          case Proved(_) | Exhausted | Passed => success
          case PropException(_, e, labels) =>
            val error = if (labels.isEmpty) e.getLocalizedMessage() else labels.mkString("\n")
            failure(error)
        }

      private def parserConstructor(mimeType: String) = PlayBodyParsing.jacksonMapper(mimeType)

      def parseResponseContent[T](mapper: ObjectMapper, content: String, mimeType: Option[String], expectedType: Class[T]) =
        mapper.readValue(content, expectedType)

} 

package shapeless.datatype.bigquery

import java.net.URI

import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}
import com.google.api.services.bigquery.model.TableRow
import com.google.common.io.BaseEncoding
import com.google.protobuf.ByteString
import org.joda.time.{Instant, LocalDate, LocalDateTime, LocalTime}
import org.scalacheck.Prop.forAll
import org.scalacheck.ScalacheckShapeless._
import org.scalacheck._
import shapeless._
import shapeless.datatype.record._

import scala.reflect.runtime.universe._

object BigQueryTypeSpec extends Properties("BigQueryType") {
  import shapeless.datatype.test.Records._
  import shapeless.datatype.test.SerializableUtils._

  val mapper = new ObjectMapper().disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)

  implicit def compareByteArrays(x: Array[Byte], y: Array[Byte]) = java.util.Arrays.equals(x, y)
  implicit def compareIntArrays(x: Array[Int], y: Array[Int]) = java.util.Arrays.equals(x, y)

  def roundTrip[A: TypeTag, L <: HList](m: A)(implicit
    gen: LabelledGeneric.Aux[A, L],
    fromL: FromTableRow[L],
    toL: ToTableRow[L],
    mr: MatchRecord[L]
  ): Boolean = {
    BigQuerySchema[A] // FIXME: verify the generated schema
    val t = ensureSerializable(BigQueryType[A])
    val f1: SerializableFunction[A, TableRow] =
      new SerializableFunction[A, TableRow] {
        override def apply(m: A): TableRow = t.toTableRow(m)
      }
    val f2: SerializableFunction[TableRow, Option[A]] =
      new SerializableFunction[TableRow, Option[A]] {
        override def apply(m: TableRow): Option[A] = t.fromTableRow(m)
      }
    val toFn = ensureSerializable(f1)
    val fromFn = ensureSerializable(f2)
    val copy = fromFn(mapper.readValue(mapper.writeValueAsString(toFn(m)), classOf[TableRow]))
    val rm = RecordMatcher[A]
    copy.exists(rm(_, m))
  }

  implicit val byteStringBigQueryMappableType = BigQueryType.at[ByteString]("BYTES")(
    x => ByteString.copyFrom(BaseEncoding.base64().decode(x.toString)),
    x => BaseEncoding.base64().encode(x.toByteArray)
  )
  property("required") = forAll { m: Required => roundTrip(m) }
  property("optional") = forAll { m: Optional => roundTrip(m) }
  property("repeated") = forAll { m: Repeated => roundTrip(m) }
  property("mixed") = forAll { m: Mixed => roundTrip(m) }
  property("nested") = forAll { m: Nested => roundTrip(m) }
  property("seqs") = forAll { m: Seqs => roundTrip(m) }

  implicit val arbDate = Arbitrary(arbInstant.arbitrary.map(i => new LocalDate(i.getMillis)))
  implicit val arbTime = Arbitrary(arbInstant.arbitrary.map(i => new LocalTime(i.getMillis)))
  implicit val arbDateTime = Arbitrary(
    arbInstant.arbitrary.map(i => new LocalDateTime(i.getMillis))
  )

  case class DateTimeTypes(
    instant: Instant,
    date: LocalDate,
    time: LocalTime,
    dateTime: LocalDateTime
  )
  property("date time types") = forAll { m: DateTimeTypes => roundTrip(m) }

  implicit val uriBigQueryType =
    BigQueryType.at[URI]("STRING")(v => URI.create(v.toString), _.toASCIIString)
  property("custom") = forAll { m: Custom => roundTrip(m) }
} 

package me.lyh.protobuf

import com.fasterxml.jackson.databind.ObjectMapper

package object generic {
  type GenericRecord = java.util.Map[String, Any]

  private val recordMapper = new ObjectMapper()

  implicit class JsonGenericRecord(val record: GenericRecord) {
    def toJson: String = recordMapper.writeValueAsString(record)
  }

  object GenericRecord {
    def fromJson(json: String): GenericRecord = recordMapper.readValue(json, classOf[GenericRecord])
  }

  implicit class JsonSchema(val schema: Schema) {
    def toJson: String = SchemaMapper.toJson(schema)
  }
} 

package io.bullet.borer.benchmarks

import com.fasterxml.jackson.core.`type`.TypeReference
import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import com.fasterxml.jackson.module.afterburner.AfterburnerModule
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.openjdk.jmh.annotations._

object JacksonCodecs {
  val mapper: ObjectMapper = new ObjectMapper().registerModule(DefaultScalaModule).registerModule(new AfterburnerModule)

  val foosTypeRef = new TypeReference[Map[String, Foo]] {}
  val intsTypeRef = new TypeReference[List[Int]] {}
}

import io.bullet.borer.benchmarks.JacksonCodecs._

class JacksonEncodingBenchmark extends EncodingBenchmark {

  @Benchmark
  def encodeFoos: Array[Byte] = mapper.writeValueAsBytes(foos)

  @Benchmark
  def encodeInts: Array[Byte] = mapper.writeValueAsBytes(ints)

  @Benchmark
  def encodeEmptyArray: Array[Byte] = mapper.writeValueAsBytes(List.empty[Int])
}

class JacksonDecodingBenchmark extends DecodingBenchmark {

  @Benchmark
  def decodeFoos: Map[String, Foo] = mapper.readValue(foosJson, foosTypeRef)

  @Benchmark
  def decodeInts: List[Int] = mapper.readValue(intsJson, intsTypeRef)

  @Benchmark
  def decodeEmptyArray: List[Int] = mapper.readValue(emptyArrayJson, intsTypeRef)
}

class JacksonDomBenchmark extends DomBenchmark {

  private var root: JsonNode = _
  def setup(): Unit          = root = mapper.readTree(fileBytes)

  @Benchmark
  def encodeDom: Array[Byte] = mapper.writeValueAsBytes(root)

  @Benchmark
  def decodeDom: JsonNode = mapper.readTree(fileBytes)
}

class JacksonModelBenchmark extends DomBenchmark {

  private var root: Product = _

  lazy val typeRef: TypeReference[Product] = {
    val c = fileName match {
      case "australia-abc.json"      => new TypeReference[Australia.RootInterface] {}
      case "bitcoin.json"            => new TypeReference[Bitcoin.RootInterface] {}
      case "doj-blog.json"           => new TypeReference[DojBlog.RootInterface] {}
      case "eu-lobby-country.json"   => new TypeReference[EuLobbyCountry.RootInterface] {}
      case "eu-lobby-financial.json" => new TypeReference[EuLobbyFinancial.RootInterface] {}
      case "eu-lobby-repr.json"      => new TypeReference[EuLobbyRepr.RootInterface] {}
      case "github-gists.json"       => new TypeReference[List[GithubGists.RootInterface]] {}
      case "json-generator.json"     => new TypeReference[List[JsonGenerator.RootInterface]] {}
      case "meteorites.json"         => new TypeReference[List[Meteorites.RootInterface]] {}
      case "movies.json"             => new TypeReference[List[Movies.RootInterface]] {}
      case "reddit-scala.json"       => new TypeReference[Reddit.RootInterface[Reddit.Data]] {}
      case "rick-morty.json"         => new TypeReference[RickMorty.RootInterface] {}
      case "temp-anomaly.json"       => new TypeReference[TempAnomaly.RootInterface] {}
      case "thai-cinemas.json"       => new TypeReference[ThaiCinemas.RootInterface] {}
      case "turkish.json"            => new TypeReference[Turkish.RootInterface] {}

      case "github-events.json" =>
        new TypeReference[
          List[GithubEvents.RootInterface[GithubEvents.Head[GithubEvents.Repo1], GithubEvents.Forkee]]] {}

      case "twitter_api_compact_response.json" | "twitter_api_response.json" =>
        new TypeReference[List[TwitterApiResponse.RootInterface]] {}
    }
    c.asInstanceOf[TypeReference[Product]]
  }

  def setup(): Unit =
    root =
      try mapper.readValue(fileBytes, typeRef)
      finally ()

  @Benchmark
  def encodeModel: Array[Byte] = mapper.writeValueAsBytes(root)

  @Benchmark
  def decodeModel: Product = mapper.readValue(fileBytes, typeRef)
} 

package com.github.plokhotnyuk.jsoniter_scala.benchmark

import java.lang.Character._
import java.nio.charset.StandardCharsets.UTF_8

import com.fasterxml.jackson.core.JsonGenerator
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.ScalaObjectMapper
import com.github.plokhotnyuk.jsoniter_scala.benchmark.JacksonSerDesers._
import org.openjdk.jmh.annotations.{Param, Setup}

class StringOfEscapedCharsBenchmark extends CommonParams {
  @Param(Array("1", "10", "100", "1000", "10000", "100000", "1000000"))
  var size: Int = 1000
  var obj: String = _
  var jsonString: String = _
  var jsonString2: String = _
  var jsonBytes: Array[Byte] = _
  var preallocatedBuf: Array[Byte] = _

  @Setup
  def setup(): Unit = {
    obj = {
      val cs = new Array[Char](size)
      var i = 0
      var j = 1
      while (i < cs.length) {
        cs(i) = {
          var ch: Char = 0
          while ({
            ch = (j * 1498724053).toChar
            j += 1
            ch < 128 || isSurrogate(ch)
          }) ()
          ch
        }
        i += 1
      }
      new String(cs)
    }
    jsonString = "\"" + obj.map(ch => f"\\u$ch%04x").mkString  + "\""
    jsonString2 = "\"" + obj.map(ch => f"\\u$ch%04X").mkString  + "\""
    jsonBytes = jsonString.getBytes(UTF_8)
    preallocatedBuf = new Array[Byte](jsonBytes.length + 100)
  }
} 

package com.github.plokhotnyuk.jsoniter_scala.benchmark

import java.nio.charset.StandardCharsets.UTF_8

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.module.SimpleModule
import com.fasterxml.jackson.module.scala.ScalaObjectMapper
import com.github.plokhotnyuk.jsoniter_scala.benchmark.JacksonSerDesers.createJacksonMapper
import org.openjdk.jmh.annotations.{Param, Setup}

abstract class ArrayOfBytesBenchmark extends CommonParams {
  @Param(Array("1", "10", "100", "1000", "10000", "100000", "1000000"))
  var size: Int = 1000
  var obj: Array[Byte] = _
  var jsonString: String = _
  var jsonBytes: Array[Byte] = _
  var preallocatedBuf: Array[Byte] = _
  @Setup
  def setup(): Unit = {
    obj = (1 to size).map(_.toByte).toArray
    jsonString = obj.mkString("[", ",", "]")
    jsonBytes = jsonString.getBytes(UTF_8)
    preallocatedBuf = new Array[Byte](jsonBytes.length + 100)
  }
} 

package ionroller

import java.text.SimpleDateFormat

import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
import play.api.libs.json.Json

object JsonUtil {

  object Implicits {
    implicit class Unmarshallable(unMarshallMe: String) {
      def toMap: Map[String, Any] = JsonUtil.toMap(unMarshallMe)

      def toMapOf[V]()(implicit m: Manifest[V]): Map[String, V] = JsonUtil.toMap[V](unMarshallMe)

      def fromJson[T]()(implicit m: Manifest[T]): T = JsonUtil.fromJson[T](unMarshallMe)
    }

    implicit class Marshallable[T](marshallMe: T) {
      def toJson: String = JsonUtil.toJson(marshallMe)

      def toJsonValue = Json.parse(JsonUtil.toJson(marshallMe))
    }
  }

  val mapper = new ObjectMapper() with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)
  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
  mapper.setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS"))

  def toJson(value: Map[Symbol, Any]): String = {
    toJson(value map { case (k, v) => k.name -> v })
  }

  def toJson(value: Any): String = {
    mapper.writeValueAsString(value)
  }

  def toMap[V](json: String)(implicit m: Manifest[V]) = fromJson[Map[String, V]](json)

  def fromJson[T](json: String)(implicit m: Manifest[T]): T = {
    mapper.readValue[T](json)
  }
} 

package com.github.johnynek.bazel_deps

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import io.circe.jackson.CirceJsonModule
import io.circe.{Decoder, Json, ParsingFailure, Parser}
import scala.util.control.NonFatal


object Yaml extends Parser {
  private[this] val mapper = new ObjectMapper(new YAMLFactory()).registerModule(CirceJsonModule)
  private[this] val factory = mapper.getFactory
  override def parse(input: String): Either[ParsingFailure, Json] =
    try {
      Right(mapper.readValue(factory.createParser(input), classOf[Json]))
    } catch {
      case NonFatal(error) => Left(ParsingFailure(error.getMessage, error))
    }
} 

package moe.pizza.auth.webapp

import java.time.Instant

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import moe.pizza.auth.interfaces.UserDatabase
import moe.pizza.auth.webapp.SessionManager._
import moe.pizza.auth.webapp.Types.{HydratedSession, Session, Session2}
import org.http4s.{HttpService, _}
import org.http4s.server._
import org.slf4j.LoggerFactory
import pdi.jwt.{JwtAlgorithm, JwtCirce, JwtClaim}
import io.circe.generic.auto._
import Utils._
import io.circe.Decoder.Result

import scala.util.Try

object SessionManager {
  val HYDRATEDSESSION = AttributeKey[HydratedSession]("HYDRATEDSESSION")
  val LOGOUT = AttributeKey[String]("LOGOUT")
  val COOKIESESSION = "authsession"
}

class SessionManager(secretKey: String, ud: UserDatabase)
    extends HttpMiddleware {
  val log = LoggerFactory.getLogger(getClass)
  val OM = new ObjectMapper()
  OM.registerModule(DefaultScalaModule)

  case class MyJwt(exp: Long, iat: Long, session: String)

  implicit def toOption[A](e: Result[A]): Option[A] = {
    e match {
      case Left(_) => None
      case Right(a) => Some(a)
    }
  }

  override def apply(s: HttpService): HttpService = Service.lift { req =>
    log.info(s"Intercepting request ${req}")
    // TODO: this used to be nice with toOption, what happened
    val sessions =
      req.headers.get(headers.Cookie).toList.flatMap(_.values.list).flatMap {
        header =>
          JwtCirce.decodeJson(header.content, secretKey, Seq(JwtAlgorithm.HS256))
            .toOption
            .flatMap { jwt =>
              jwt.as[MyJwt] match {
                case Right(x) => Some(x)
                case Left(_) => None
              }
            }
            .flatMap { myjwt =>
              Try { OM.readValue(myjwt.session, classOf[Session2]) }.toOption
            }
      }
    log.info(s"found sessions: ${sessions}")

    // if we didn't find a valid session, make them one
    val session =
      sessions.headOption.getOrElse(Session2(List.empty, None, None, None))

    // do the inner request
    val hydrated = session.hydrate(ud)
    log.info(s"running inner router with hydrated session ${hydrated}")
    val response =
      s(req.copy(attributes = req.attributes.put(HYDRATEDSESSION, hydrated)))

    response.map { resp =>
      // do all of this once the request has been created
      val sessionToSave = resp.attributes
        .get(HYDRATEDSESSION)
        .map(_.dehydrate())
        .getOrElse(session)
      val oldsessions = resp.headers
        .get(headers.Cookie)
        .toList
        .flatMap(_.values.list)
        .filter(_.name == COOKIESESSION)
      if (resp.attributes.get(LOGOUT).isEmpty) {
        log.info(s"saving the session as a cookie")
        val claim = JwtClaim(
            expiration = Some(
              Instant.now
                .plusSeconds(86400 * 30)
                .getEpochSecond), // lasts 30 days
            issuedAt = Some(Instant.now.getEpochSecond)
          ) + ("session", OM.writeValueAsString(sessionToSave))
        val token = JwtCirce.encode(claim, secretKey, JwtAlgorithm.HS256)
        resp.addCookie(
          new Cookie(COOKIESESSION, token, None, None, None, path = Some("/"))
        )
      } else {
        log.info(s"log out flag was set, not saving any cookies")
        resp.removeCookie(COOKIESESSION)
      }
    }
  }
} 

package moe.pizza.auth.webapp.rest

import org.http4s.server.HttpMiddleware
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import moe.pizza.auth.interfaces.UserDatabase
import moe.pizza.auth.webapp.SessionManager._
import moe.pizza.auth.webapp.Types.{HydratedSession, Session2, Session}
import org.http4s.{HttpService, _}
import org.http4s.server._
import org.slf4j.LoggerFactory
import pdi.jwt.{JwtAlgorithm, JwtCirce, JwtClaim}
import io.circe.generic.auto._
import scala.util.Try
import org.http4s.dsl.{Root, _}

import scalaz.concurrent.Task

class RestKeyMiddleware(apikeys: List[String]) extends HttpMiddleware {
  override def apply(s: HttpService): HttpService = Service.lift { req =>
    req.headers
      .get(headers.Authorization)
      .map(_.credentials.value.stripPrefix("Bearer "))
      .filter(apikeys.contains) match {
      case Some(k) =>
        s(req)
      case None =>
        Unauthorized(
          Challenge(scheme = "Bearer", realm = "Please enter a valid API key"))
    }
  }
} 

package moe.pizza.auth.webapp

import java.net.{Socket, InetSocketAddress, ServerSocket}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import moe.pizza.auth.config.ConfigFile.ConfigFile

import scala.concurrent.{Future, Await}
import scala.io.Source
import scala.util.Try
import scala.concurrent.duration._
import scala.concurrent.ExecutionContext.Implicits.global


object WebappTestSupports {
  val OM = new ObjectMapper(new YAMLFactory())
  OM.registerModule(DefaultScalaModule)

  def readTestConfig(): ConfigFile = {
    val config = Source
      .fromURL(getClass.getResource("/config.yml"))
      .getLines()
      .mkString("\n")
    val conf = OM.readValue[ConfigFile](config, classOf[ConfigFile])
    conf
  }

} 

package org.apache.spark.sql.delta.util

import com.fasterxml.jackson.annotation.JsonInclude.Include
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper


  val mapper = new ObjectMapper with ScalaObjectMapper
  mapper.setSerializationInclusion(Include.NON_ABSENT)
  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
  mapper.registerModule(DefaultScalaModule)

  def toJson[T: Manifest](obj: T): String = {
    mapper.writeValueAsString(obj)
  }

  def toPrettyJson[T: Manifest](obj: T): String = {
    mapper.writerWithDefaultPrettyPrinter().writeValueAsString(obj)
  }

  def fromJson[T: Manifest](json: String): T = {
    mapper.readValue[T](json)
  }
} 

package org.json4s.jackson.sparklinedata

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.smile.{SmileFactory, SmileGenerator}
import org.json4s.jackson.{Json4sScalaModule, JsonMethods}

object SmileJsonMethods extends JsonMethods {

  private[this] lazy val _defaultMapper = {

    val smileFactory = new SmileFactory
    smileFactory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, false)
    smileFactory.delegateToTextual(true)
    val m = new ObjectMapper(smileFactory)
    m.getFactory().setCodec(m)
    m.registerModule(SmileJson4sScalaModule)
    m
  }

  override def mapper = _defaultMapper

} 

package cmwell.analytics.util

import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import org.apache.http.HttpResponse
import org.apache.http.client.methods.{HttpGet, HttpPost}
import org.apache.http.client.{ClientProtocolException, ResponseHandler}
import org.apache.http.entity.{ContentType, StringEntity}
import org.apache.http.impl.client.{CloseableHttpClient, HttpClients}
import org.apache.http.util.EntityUtils


object HttpUtil {

  private val mapper = new ObjectMapper()

  private val responseHandler = new ResponseHandler[String]() {

    override def handleResponse(response: HttpResponse): String = {
      val status = response.getStatusLine.getStatusCode
      if (status >= 200 && status < 300) {
        val entity = response.getEntity

        if (entity != null)
          EntityUtils.toString(entity)
        else
          throw new ClientProtocolException("Unexpected null response")
      }
      else {
        throw new ClientProtocolException("Unexpected response status: " + status)
      }
    }
  }

  def get(url: String, client: CloseableHttpClient): String = {
    val httpGet = new HttpGet(url)
    client.execute(httpGet, responseHandler)
  }

  def get(url: String): String = {

    val client =  HttpClients.createDefault
    try {
      get(url, client)
    }
    finally {
      client.close()
    }
  }

  def getJson(url: String): JsonNode = {
    val client =  HttpClients.createDefault
    try {
      getJson(url, client)
    }
    finally {
      client.close()
    }
  }

  def getJson(url: String, client: CloseableHttpClient): JsonNode = {
    mapper.readTree(get(url, client))
  }

  def getJson(url: String, content: String, contentType: String): JsonNode = {
    val httpPost = new HttpPost(url)
    val entity = new StringEntity(content, ContentType.create(contentType, "UTF-8"))
    httpPost.setEntity(entity)

    val client = HttpClients.createDefault
    try {
      mapper.readTree(client.execute(httpPost, responseHandler))
    }
    finally {
      client.close()
    }
  }
} 

package cmwell.analytics.main

import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import cmwell.analytics.data.{DataWriterFactory, IndexWithCompleteDocument}
import cmwell.analytics.downloader.PartitionedDownloader
import cmwell.analytics.util.{DiscoverEsTopology, FindContactPoints}
import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.log4j.LogManager
import org.rogach.scallop.{ScallopConf, ScallopOption}

import scala.collection.JavaConverters._
import scala.concurrent.ExecutionContextExecutor

object CopyIndexesWithMapping {

  def main(args: Array[String]): Unit = {

    val logger = LogManager.getLogger(CopyIndexesWithMapping.getClass)

    // Since we expect this to be run on a CM-Well node, the default parallelism is to use half the processors
    // so as to avoid starving the CM-Well node from processor resources. A higher level of parallelism might
    // be possible (without interfering with CM-Well) since most of the work will actually be on the ES side.
    val defaultParallelism = 1 max (Runtime.getRuntime.availableProcessors / 2)

    implicit val system: ActorSystem = ActorSystem("copy-index-with-mapping")
    implicit val executionContext: ExecutionContextExecutor = system.dispatcher
    implicit val actorMaterializer: ActorMaterializer = ActorMaterializer()

    try {

      object Opts extends ScallopConf(args) {

        val indexMap: ScallopOption[String] = opt[String]("index-map", short = 'i', descr = "A map from source to target index names, in JSON format", required = true)

        val parallelism: ScallopOption[Int] = opt[Int]("parallelism", short = 'p', descr = "The parallelism level", default = Some(defaultParallelism))
        val url: ScallopOption[String] = trailArg[String]("url", descr = "A CM-Well URL", required = true)

        verify()
      }

      val esContactPoint = FindContactPoints.es(Opts.url())

      // Expect a map in the form: { "sourceIndex1": "targetIndex1", "sourceIndex2": "targetIndex2", ... }
      val indexMap: Map[String, String] = new ObjectMapper().readTree(Opts.indexMap()).fields.asScala.map { entry =>
        entry.getKey -> entry.getValue.asText
      }.toMap

      val esTopology = DiscoverEsTopology(esContactPoint = esContactPoint, aliases = indexMap.keys.toSeq)

      // Validate that the index-map parameter specified valid index names, and not aliases.
      for (indexName <- indexMap.keys)
        if (!esTopology.allIndexNames.contains(indexName))
          throw new RuntimeException(s"index-map parameter included $indexName as a source, which is not a valid index name.")

      for (indexName <- indexMap.values)
        if (!esTopology.allIndexNames.contains(indexName))
          throw new RuntimeException(s"index-map parameter included $indexName as a target, which is not a valid index name.")

      val dataWriterFactory = DataWriterFactory.index[IndexWithCompleteDocument](
        indexMap = indexMap,
        esEndpoint = esContactPoint)

      PartitionedDownloader.runDownload(
        esTopology = esTopology,
        parallelism = Opts.parallelism(),
        objectExtractor = IndexWithCompleteDocument,
        dataWriterFactory = dataWriterFactory,
        sourceFilter = false)
    }
    catch {
      case ex: Throwable =>
        logger.error(ex.getMessage, ex)
        System.exit(1)
    }
    finally {
      system.terminate()
    }
  }
} 

package cmwell.analytics.util

import akka.actor.ActorSystem
import akka.http.scaladsl.Http
import akka.http.scaladsl.model.RequestEntityAcceptance.Tolerated
import akka.http.scaladsl.model.{HttpMethod, HttpRequest, HttpResponse}
import akka.stream.ActorMaterializer
import akka.stream.scaladsl.Sink
import akka.util.ByteString
import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import com.typesafe.config.ConfigFactory

import scala.concurrent.duration.{MILLISECONDS, _}
import scala.concurrent.{Await, ExecutionContextExecutor, Future}

object HttpUtil {

  private val mapper = new ObjectMapper()

  private val config = ConfigFactory.load
  private val ReadTimeout = FiniteDuration(config.getDuration("extract-index-from-es.read-timeout").toMillis, MILLISECONDS)

  // Elasticsearch uses the POST verb in some places where the request is actually idempotent.
  // Requests that use POST, but are known to be idempotent can use this method.
  // The presence of any non-idempotent request in-flight causes Akka to not retry, and that will tend result in
  // entire downloads failing more often.
  val SAFE_POST = HttpMethod(
    value = "POST",
    isSafe = true,
    isIdempotent = true,
    requestEntityAcceptance = Tolerated)

  def resultAsync(request: HttpRequest,
                  action: String)
                 (implicit system: ActorSystem,
                  executionContext: ExecutionContextExecutor,
                  actorMaterializer: ActorMaterializer): Future[ByteString] =
    Http().singleRequest(request).map {

      case HttpResponse(status, _, entity, _) if status.isSuccess =>
        entity.dataBytes
          .fold(ByteString.empty)(_ ++ _)
          .runWith(Sink.head)

      case HttpResponse(status, _, entity, _) =>
        val message = Await.result(entity.toStrict(10.seconds).map(_.data), 10.seconds).utf8String
        throw new RuntimeException(s"HTTP request for $action failed. Status code: $status, message:$message")
    }
      .flatMap(identity)

  def result(request: HttpRequest,
             action: String,
             timeout: FiniteDuration = ReadTimeout)
            (implicit system: ActorSystem,
             executionContext: ExecutionContextExecutor,
             actorMaterializer: ActorMaterializer): ByteString =
    Await.result(resultAsync(request, action), timeout)

  def jsonResult(request: HttpRequest,
                 action: String,
                 timeout: FiniteDuration = ReadTimeout)
                (implicit system: ActorSystem,
                 executionContext: ExecutionContextExecutor,
                 actorMaterializer: ActorMaterializer): JsonNode =
    mapper.readTree(result(request, action, timeout).utf8String)

  def jsonResultAsync(request: HttpRequest,
                      action: String)
                     (implicit system: ActorSystem,
                      executionContext: ExecutionContextExecutor,
                      actorMaterializer: ActorMaterializer): Future[JsonNode] =
    resultAsync(request, action).map((bytes: ByteString) => mapper.readTree(bytes.utf8String))
} 

package com.ebiznext.comet.utils

import com.fasterxml.jackson.annotation.JsonIgnoreType
import com.fasterxml.jackson.core.JsonFactory
import com.fasterxml.jackson.databind.module.SimpleModule
import com.fasterxml.jackson.databind.{InjectableValues, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper

object CometObjectMapper {

  // https://github.com/FasterXML/jackson-databind/issues/962
  @JsonIgnoreType
  private class MixinsForObjectMapper

}

class CometObjectMapper(
  jf: JsonFactory = null,
  injectables: scala.collection.immutable.Seq[(Class[_], AnyRef)] = Nil
) extends ObjectMapper(jf)
    with ScalaObjectMapper {

  this.registerModule(DefaultScalaModule)
  this.registerModule(CometJacksonModule)

  this.registerModule(
    
    new SimpleModule()
      .setMixInAnnotation(classOf[ObjectMapper], classOf[CometObjectMapper.MixinsForObjectMapper])
  )

  if (injectables.nonEmpty) {
    val iv = new InjectableValues.Std()
    injectables.foreach { case (klass, value) => iv.addValue(klass, value) }
    this.setInjectableValues(iv: InjectableValues)
  }

} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.nio.charset.StandardCharsets
import java.text.SimpleDateFormat
import java.util.{Calendar, Locale, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes(StandardCharsets.UTF_8))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'", Locale.US)
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package org.apache.spark.metrics.sink

import java.util.Properties
import java.util.concurrent.TimeUnit
import javax.servlet.http.HttpServletRequest

import com.codahale.metrics.MetricRegistry
import com.codahale.metrics.json.MetricsModule
import com.fasterxml.jackson.databind.ObjectMapper
import org.eclipse.jetty.servlet.ServletContextHandler

import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.ui.JettyUtils._

private[spark] class MetricsServlet(
    val property: Properties,
    val registry: MetricRegistry,
    securityMgr: SecurityManager)
  extends Sink {

  val SERVLET_KEY_PATH = "path"
  val SERVLET_KEY_SAMPLE = "sample"

  val SERVLET_DEFAULT_SAMPLE = false

  val servletPath = property.getProperty(SERVLET_KEY_PATH)

  val servletShowSample = Option(property.getProperty(SERVLET_KEY_SAMPLE)).map(_.toBoolean)
    .getOrElse(SERVLET_DEFAULT_SAMPLE)

  val mapper = new ObjectMapper().registerModule(
    new MetricsModule(TimeUnit.SECONDS, TimeUnit.MILLISECONDS, servletShowSample))

  def getHandlers(conf: SparkConf): Array[ServletContextHandler] = {
    Array[ServletContextHandler](
      createServletHandler(servletPath,
        new ServletParams(request => getMetricsSnapshot(request), "text/json"), securityMgr, conf)
    )
  }

  def getMetricsSnapshot(request: HttpServletRequest): String = {
    mapper.writeValueAsString(registry)
  }

  override def start() { }

  override def stop() { }

  override def report() { }
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.text.SimpleDateFormat
import java.util.{Calendar, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes("utf-8"))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'")
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package org.apache.spark.metrics.sink

import java.util.Properties
import java.util.concurrent.TimeUnit

import javax.servlet.http.HttpServletRequest

import com.codahale.metrics.MetricRegistry
import com.codahale.metrics.json.MetricsModule
import com.fasterxml.jackson.databind.ObjectMapper
import org.eclipse.jetty.servlet.ServletContextHandler

import org.apache.spark.SecurityManager
import org.apache.spark.ui.JettyUtils._

private[spark] class MetricsServlet(
    val property: Properties,
    val registry: MetricRegistry,
    securityMgr: SecurityManager)
  extends Sink {

  val SERVLET_KEY_PATH = "path"
  val SERVLET_KEY_SAMPLE = "sample"

  val SERVLET_DEFAULT_SAMPLE = false

  val servletPath = property.getProperty(SERVLET_KEY_PATH)

  val servletShowSample = Option(property.getProperty(SERVLET_KEY_SAMPLE)).map(_.toBoolean)
    .getOrElse(SERVLET_DEFAULT_SAMPLE)

  val mapper = new ObjectMapper().registerModule(
    new MetricsModule(TimeUnit.SECONDS, TimeUnit.MILLISECONDS, servletShowSample))

  def getHandlers: Array[ServletContextHandler] = {
    Array[ServletContextHandler](
      createServletHandler(servletPath,
        new ServletParams(request => getMetricsSnapshot(request), "text/json"), securityMgr)
    )
  }

  def getMetricsSnapshot(request: HttpServletRequest): String = {
    mapper.writeValueAsString(registry)
  }

  override def start() { }

  override def stop() { }

  override def report() { }
} 

package helpers

import com.fasterxml.jackson.databind.ObjectMapper
import io.netty.handler.codec.http.EmptyHttpHeaders
import java.io.ByteArrayInputStream
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import java.util.Collections
import java.util.concurrent.CompletableFuture
import javax.ws.rs.container.AsyncResponse
import org.asynchttpclient.Response
import org.asynchttpclient.uri.Uri
import org.mockito.{ ArgumentMatchersSugar, MockitoSugar }
import org.scalatest.Assertions
import scala.reflect.ClassTag

object MockHelpers extends Assertions with MockitoSugar with ArgumentMatchersSugar {
  def mockAsyncResponse[T](future: CompletableFuture[T])(implicit cls: ClassTag[T]): AsyncResponse = {
    val asyncResponse = mock[AsyncResponse]

    when(asyncResponse.resume(any[T])) thenAnswer [AnyRef] { response =>
      response match {
        case t: Throwable => future.completeExceptionally(t)
        case other: T     => future.complete(other)
        case other        => fail(s"AsyncResponse.resume expected an object of type ${cls.runtimeClass.getName}, but got ${other.getClass.getName} instead")
      }
    }

    asyncResponse
  }

  def mockAHCResponse[T](uri: String, status: Int, maybeBody: Option[T] = None)(implicit mapper: ObjectMapper): Response = {
    val response = mock[Response]
    when(response.getUri) thenReturn Uri.create(uri)
    when(response.hasResponseStatus) thenReturn true
    when(response.getStatusCode) thenReturn status
    when(response.getStatusText) thenReturn "Some Status"
    when(response.hasResponseHeaders) thenReturn true
    when(response.getHeaders) thenReturn EmptyHttpHeaders.INSTANCE
    when(response.getHeader(any)) thenReturn null
    when(response.getHeaders(any)) thenReturn Collections.emptyList()
    maybeBody match {
      case None =>
        when(response.hasResponseBody) thenReturn true
      case Some(body) =>
        val responseBytes = mapper.writeValueAsBytes(body)
        val responseStr   = new String(responseBytes, StandardCharsets.UTF_8)
        when(response.hasResponseBody) thenReturn true
        when(response.getResponseBody(any)) thenReturn responseStr
        when(response.getResponseBody) thenReturn responseStr
        when(response.getResponseBodyAsStream) thenReturn new ByteArrayInputStream(responseBytes)
        when(response.getResponseBodyAsByteBuffer) thenReturn ByteBuffer.wrap(responseBytes)
        when(response.getResponseBodyAsBytes) thenReturn responseBytes
    }
    response
  }

} 

package core.Jackson

import com.fasterxml.jackson.databind.ObjectMapper
import org.scalatest.{ FreeSpec, Matchers }
import polymorphismMapped.client.dropwizard.definitions.{ A, B, Base, C, DiscrimEnum, EnumA, EnumB, EnumBase, EnumC }
import scala.reflect.ClassTag

class JacksonPolyMappingTest extends FreeSpec with Matchers {
  private val mapper = new ObjectMapper

  "Polymorphic definitions with discriminator mappings" - {
    "should have their discriminator initialized properly" in {
      val a = new A.Builder(42).build()
      a.getPolytype shouldBe "this_is_a"

      val b = new B.Builder("foo").build()
      b.getPolytype shouldBe "this_is_b"

      val c = new C.Builder(42.42).build()
      c.getPolytype shouldBe "C"
    }

    "should deserialize properly" in {
      def verify[T](json: String, discriminatorValue: String)(implicit cls: ClassTag[T]): Unit = {
        val pojo = mapper.readValue(json, classOf[Base])
        pojo shouldNot be(null)
        pojo.getClass shouldBe cls.runtimeClass
        pojo.getPolytype shouldBe discriminatorValue
      }

      verify[A]("""{"polytype": "this_is_a", "some_a": 42}""", "this_is_a")
      verify[B]("""{"polytype": "this_is_b", "some_b": "foo"}""", "this_is_b")
      verify[C]("""{"polytype": "C", "some_c": 42.42}""", "C")
    }
  }

  "Polymorphic definitions with enum discriminator mappings" - {
    "should have their discriminator initialized properly" in {
      val a = new EnumA.Builder(42).build()
      a.getPolytype shouldBe DiscrimEnum.SOME_VALUE_ONE

      val b = new EnumB.Builder("foo").build()
      b.getPolytype shouldBe DiscrimEnum.ANOTHER_VALUE

      val c = new EnumC.Builder(42.42).build()
      c.getPolytype shouldBe DiscrimEnum.YET_ANOTHER_VALUE
    }

    "should deserialize properly" in {
      def verify[T](json: String, discriminatorValue: DiscrimEnum)(implicit cls: ClassTag[T]): Unit = {
        val pojo = mapper.readValue(json, classOf[EnumBase])
        pojo shouldNot be(null)
        pojo.getClass shouldBe cls.runtimeClass
        pojo.getPolytype shouldBe discriminatorValue
      }

      verify[EnumA]("""{"polytype": "some-value-one", "some_a": 42}""", DiscrimEnum.SOME_VALUE_ONE)
      verify[EnumB]("""{"polytype": "another-value", "some_b": "foo"}""", DiscrimEnum.ANOTHER_VALUE)
      verify[EnumC]("""{"polytype": "yet-another-value", "some_c": 42.42}""", DiscrimEnum.YET_ANOTHER_VALUE)
    }
  }
} 

package core.Jackson

import com.fasterxml.jackson.databind.ObjectMapper
import issues.issue389.client.dropwizard.definitions.{ Bar, Foo }
import java.util
import org.scalatest.{ FreeSpec, Matchers }
import scala.collection.JavaConverters._
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module

class Issue389 extends FreeSpec with Matchers {
  "x-jvm-type in Array should use the correct type in the array" in {
    val mapper = new ObjectMapper().registerModule(new Jdk8Module())
    val foo = new Foo.Builder()
      .withCustomArray(
        (1 until 5)
          .map(new Bar.Builder().withA(_).build())
          .toList
          .asJava
      )
      .build()
    val expected = """{"customArray":[{"a":1},{"a":2},{"a":3},{"a":4}],"customMap":null}"""
    assertResult(expected)(mapper.writeValueAsString(foo))
  }

  "x-jvm-type in Map should use the correct type in the map" in {
    val mapper = new ObjectMapper().registerModule(new Jdk8Module())
    val foo = new Foo.Builder()
      .withCustomMap(
        (1 until 5)
          .map(x => x.toString() -> new Bar.Builder().withA(x).build())
          .toMap
          .asJava
      )
      .build()
    val expected = """{"customArray":null,"customMap":{"1":{"a":1},"2":{"a":2},"3":{"a":3},"4":{"a":4}}}"""
    assertResult(expected)(mapper.writeValueAsString(foo))
  }
} 

package core.Dropwizard

import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.databind.ObjectMapper
import invalidCharacters.client.dropwizard.invalidCharacters.InvalidCharactersClient
import invalidCharacters.server.dropwizard.definitions.{InvalidCharacters, InvalidCharactersEnum}
import io.netty.buffer.Unpooled
import java.net.{SocketAddress, URI, URLDecoder}
import java.util.concurrent.{CompletableFuture, CompletionStage}
import java.util.function
import org.asynchttpclient.Response.ResponseBuilder
import org.asynchttpclient.netty.EagerResponseBodyPart
import org.asynchttpclient.uri.Uri
import org.asynchttpclient.{HttpResponseStatus, Request, Response}
import org.scalatest.freespec.AnyFreeSpec
import org.scalatest.matchers.must.Matchers
import scala.collection.JavaConverters._

object JavaInvalidCharacterEscapingTest {
  private implicit class RichString(private val s: String) extends AnyVal {
    def dec: String = URLDecoder.decode(s, "UTF-8")
  }

  private object OkStatus extends HttpResponseStatus(Uri.create("http://localhost:1234/foo?foo^bar=query-param")) {
    override def getStatusCode = 200
    override def getStatusText = "OK"
    override def getProtocolName = "HTTP"
    override def getProtocolMajorVersion = 1
    override def getProtocolMinorVersion = 1
    override def getProtocolText = "HTTP/1.1"
    override def getRemoteAddress: SocketAddress = ???
    override def getLocalAddress: SocketAddress = ???
  }
}

class JavaInvalidCharacterEscapingTest extends AnyFreeSpec with Matchers {
  import JavaInvalidCharacterEscapingTest._

  "Invalid characters in Java enums should be escaped" in {
    InvalidCharactersEnum.NORMAL.getName mustBe "normal"
    InvalidCharactersEnum.BANG_MOO_COLON_COW_SEMICOLON.getName mustBe "!moo:cow;"
    InvalidCharactersEnum.POUND_YEAH.getName mustBe "#yeah"
    InvalidCharactersEnum.WEIRD_AT.getName mustBe "weird@"
  }

  "Invalid characters in Java POJO properties should be escaped" in {
    val invChar = new InvalidCharacters.Builder("stuff", InvalidCharactersEnum.POUND_YEAH).build()
    invChar.getCloseSquareBraceMoo mustBe "stuff"
    invChar.getSomeEnumAsteriskCaret mustBe InvalidCharactersEnum.POUND_YEAH

    classOf[InvalidCharacters].getDeclaredField("closeSquareBraceMoo").getAnnotation(classOf[JsonProperty]).value mustBe "]moo"
    classOf[InvalidCharacters].getDeclaredField("someEnumAsteriskCaret").getAnnotation(classOf[JsonProperty]).value mustBe "some-enum*^"
  }

  "Invalid characters in Java operation param names should be escaped" in {
    val httpClient = new function.Function[Request, CompletionStage[Response]] {
      override def apply(request: Request): CompletionStage[Response] = {
        println(request.getUri)
        println(request.getQueryParams.asScala.map(_.getName))
        val qps = request.getQueryParams.asScala.map(p => (p.getName.dec, p.getValue.dec))
        val fps = request.getFormParams.asScala.map(p => (p.getName.dec, p.getValue.dec))
        qps.find(_._1 == "foo^bar").map(_._2) mustBe Some("firstarg")
        fps.find(_._1 == "a*b").map(_._2) mustBe Some("secondarg")
        fps.find(_._1 == "bc?").map(_._2) mustBe Some("thirdarg")
        fps.find(_._1 == "d/c").map(_._2) mustBe Some("fourtharg")
        val response = new ResponseBuilder()
        response.accumulate(OkStatus)
        response.accumulate(new EagerResponseBodyPart(
          Unpooled.copiedBuffer(new ObjectMapper().writeValueAsBytes(new InvalidCharacters.Builder("foo", InvalidCharactersEnum.WEIRD_AT).build())),
          true
        ))
        CompletableFuture.completedFuture(response.build())
      }
    }

    val client = new InvalidCharactersClient.Builder(new URI("http://localhost:1234")).withHttpClient(httpClient).build()
    val response = client.getFoo("firstarg", "secondarg", "thirdarg", "fourtharg").call().toCompletableFuture.get()
    response.fold(
      { invChar =>
        invChar.getCloseSquareBraceMoo mustBe "foo"
        invChar.getSomeEnumAsteriskCaret mustBe invalidCharacters.client.dropwizard.definitions.InvalidCharactersEnum.WEIRD_AT
      }
    )
  }
} 

package core.Dropwizard

import com.fasterxml.jackson.databind.ObjectMapper
import examples.client.dropwizard.user.{ UserClient, GetUserByNameResponse => GetUserByNameClientResponse }
import examples.server.dropwizard.definitions.User
import examples.server.dropwizard.user.UserHandler._
import examples.server.dropwizard.user._
import helpers.MockHelpers._
import java.util
import java.util.Optional
import java.util.concurrent.{ CompletableFuture, CompletionStage }
import org.asynchttpclient.{ Request, Response }
import org.mockito.{ ArgumentMatchersSugar, MockitoSugar }
import org.scalatest.concurrent.Waiters
import org.scalatest.{ FreeSpec, Matchers }
import scala.compat.java8.FunctionConverters._

class DropwizardRoundTripTest extends FreeSpec with Matchers with Waiters with MockitoSugar with ArgumentMatchersSugar {
  private implicit val mapper = new ObjectMapper

  "Test server" in {
    val USERNAME = "foobar"

    val serverFuture  = new CompletableFuture[GetUserByNameResponse]
    val asyncResponse = mockAsyncResponse(serverFuture)

    val resource = new UserResource(new UserHandler {
      override def createUser(body: User): CompletionStage[CreateUserResponse]                                          = ???
      override def createUsersWithArrayInput(body: util.List[User]): CompletionStage[CreateUsersWithArrayInputResponse] = ???
      override def createUsersWithListInput(body: util.List[User]): CompletionStage[CreateUsersWithListInputResponse]   = ???
      override def loginUser(username: String, password: String): CompletionStage[LoginUserResponse]                    = ???
      override def logoutUser(): CompletionStage[LogoutUserResponse]                                                    = ???
      override def updateUser(username: String, body: User): CompletionStage[UpdateUserResponse]                        = ???
      override def deleteUser(username: String): CompletionStage[DeleteUserResponse]                                    = ???

      override def getUserByName(username: String): CompletionStage[GetUserByNameResponse] = {
        username match {
          case USERNAME =>
            serverFuture.complete(
              GetUserByNameResponse.Ok(
                new User.Builder()
                  .withEmail("[email protected]")
                  .withFirstName("Foo")
                  .withLastName("Bar")
                  .withId(1)
                  .withUsername(USERNAME)
                  .build()
              )
            )
          case "" =>
            serverFuture.complete(GetUserByNameResponse.BadRequest)
          case _ =>
            serverFuture.complete(GetUserByNameResponse.NotFound)
        }
        serverFuture
      }
    })

    val httpClient: Request => CompletionStage[Response] = { request =>
      val userPath = "^/v2/user/([^/]*)$".r
      request.getUri.getPath match {
        case userPath(username) =>
          resource.getUserByName(username, asyncResponse)
          serverFuture.thenApply({ response =>
            val entityBody = response match {
              case r: GetUserByNameResponse.Ok => Some(r.getEntityBody)
              case _                           => None
            }
            mockAHCResponse(request.getUrl, response.getStatusCode, entityBody)
          })
        case _ =>
          CompletableFuture.completedFuture(mockAHCResponse(request.getUrl, 404))
      }
    }

    val client = new UserClient.Builder()
      .withHttpClient(httpClient.asJava)
      .withObjectMapper(mapper)
      .build()

    val w = new Waiter
    client
      .getUserByName(USERNAME)
      .call()
      .whenComplete({ (response, t) =>
        w { t shouldBe null }
        response match {
          case r: GetUserByNameClientResponse.Ok =>
            w {
              r.getValue.getUsername.get shouldBe USERNAME
              r.getValue.getPassword shouldBe Optional.empty
            }
          case _: GetUserByNameClientResponse.BadRequest => w { fail("Got BadRequest") }
          case _: GetUserByNameClientResponse.NotFound   => w { fail("Got NotFound") }
        }
        w.dismiss()
      })
    w.await(dismissals(1))
  }
} 

package com.twilio.open.streaming.trend.discovery.config

import java.io.File

import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.scala.DefaultScalaModule

sealed trait Configuration extends Product with Serializable

@SerialVersionUID(101L)
case class AppConfiguration(
  @JsonProperty appName: String,
  @JsonProperty triggerInterval: String, // "30 seconds", "1 minute"
  @JsonProperty outputMode: String,
  @JsonProperty checkpointPath: String,
  @JsonProperty("windowInterval") windowIntervalMinutes: Long,
  @JsonProperty("watermarkInterval") watermarkIntervalMinutes: Long,
  @JsonProperty("core") sparkCoreConfig: Map[String, String],
  @JsonProperty callEventsTopic: KafkaReaderOrWriterConfig)
  extends Configuration with Serializable

trait KafkaConfig {
  val topic: String
  val subscriptionType: String
  val conf: Map[String, String]
}

case class KafkaReaderOrWriterConfig(
  override val topic: String,
  override val subscriptionType: String,
  override val conf: Map[String, String]) extends KafkaConfig with Serializable

object AppConfig {

  private val mapper = new ObjectMapper(new YAMLFactory)
  mapper.registerModule(DefaultScalaModule)

  @volatile
  private var config: AppConfiguration = _

  def apply(resourcePath: String): AppConfiguration = {
    if (config == null)
      synchronized {
        if (config == null)
          config = parse(resourcePath)
      }
    config
  }

  private def parse(resourcePath: String): AppConfiguration = {
    mapper.readValue(new File(resourcePath), classOf[AppConfiguration])
  }

} 

package com.twilio.open.streaming.trend.discovery

import java.io.{ByteArrayInputStream, InputStream}
import java.nio.charset.StandardCharsets

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.google.protobuf.Message
import com.googlecode.protobuf.format.JsonFormat
import com.holdenkarau.spark.testing.{LocalSparkContext, SparkContextProvider}
import com.twilio.open.protocol.Calls.CallEvent
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.SparkSession
import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers, Suite}
import org.slf4j.{Logger, LoggerFactory}

import scala.collection.Seq
import scala.io.Source
import scala.reflect.ClassTag
import scala.reflect.classTag

object TestHelper {
  val log: Logger = LoggerFactory.getLogger("com.twilio.open.streaming.trend.discovery.TestHelper")
  val mapper: ObjectMapper = {
    val m = new ObjectMapper()
    m.registerModule(DefaultScalaModule)
  }

  val jsonFormat: JsonFormat = new JsonFormat

  def loadScenario[T<: Message : ClassTag](file: String): Seq[T] = {
    val fileString = Source.fromFile(file).mkString
    val parsed = mapper.readValue(fileString, classOf[Sceanario])
    parsed.input.map { data =>
      val json = mapper.writeValueAsString(data)
      convert[T](json)
    }
  }

  def convert[T<: Message : ClassTag](json: String): T = {
    val clazz = classTag[T].runtimeClass
    val builder = clazz.getMethod("newBuilder").invoke(clazz).asInstanceOf[Message.Builder]
    try {
      val input: InputStream = new ByteArrayInputStream(json.getBytes())
      jsonFormat.merge(input, builder)
      builder.build().asInstanceOf[T]
    } catch {
      case e: Exception =>
        throw e
    }
  }

  def asMockKafkaDataFrame(event: CallEvent): MockKafkaDataFrame = {
    val key = event.getEventId.getBytes(StandardCharsets.UTF_8)
    val value = event.toByteArray
    MockKafkaDataFrame(key, value)
  }

}

case class MockKafkaDataFrame(key: Array[Byte], value: Array[Byte])


@SerialVersionUID(1L)
case class KafkaDataFrame(key: Array[Byte], topic: Array[Byte], value: Array[Byte]) extends Serializable

case class Sceanario(input: Seq[Any], expected: Option[Any] = None)

trait SparkSqlTest extends BeforeAndAfterAll with SparkContextProvider {
  self: Suite =>

  @transient var _sparkSql: SparkSession = _
  @transient private var _sc: SparkContext = _

  override def sc: SparkContext = _sc

  def conf: SparkConf

  def sparkSql: SparkSession = _sparkSql

  override def beforeAll() {
    _sparkSql = SparkSession.builder().config(conf).getOrCreate()

    _sc = _sparkSql.sparkContext
    setup(_sc)
    super.beforeAll()
  }

  override def afterAll() {
    try {
      _sparkSql.close()
      _sparkSql = null
      LocalSparkContext.stop(_sc)
      _sc = null
    } finally {
      super.afterAll()
    }
  }

} 

package shield.metrics

import java.util.concurrent.TimeUnit

import com.codahale.metrics.json.MetricsModule
import com.codahale.metrics.jvm.{ThreadStatesGaugeSet, MemoryUsageGaugeSet, GarbageCollectorMetricSet}
import com.codahale.metrics.{JvmAttributeGaugeSet, MetricRegistry}
import com.fasterxml.jackson.databind.ObjectMapper
import spray.http.{ContentTypes, HttpEntity, StatusCodes, HttpResponse}

// todo: Separate metrics per domain
object Registry {
  val metricRegistry = new MetricRegistry()
  metricRegistry.register("jvm.attribute", new JvmAttributeGaugeSet())
  metricRegistry.register("jvm.gc", new GarbageCollectorMetricSet())
  metricRegistry.register("jvm.memory", new MemoryUsageGaugeSet())
  metricRegistry.register("jvm.threads", new ThreadStatesGaugeSet())

  private val mapper = new ObjectMapper()
  mapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.MILLISECONDS, false))
  private val writer = mapper.writerWithDefaultPrettyPrinter()

  def metricsResponse : HttpResponse = HttpResponse(StatusCodes.OK, HttpEntity(ContentTypes.`application/json`,
    writer.writeValueAsBytes(metricRegistry)
  ))
} 

package yaooqinn.kyuubi.metrics

import java.io.{BufferedWriter, Closeable, IOException, OutputStreamWriter}
import java.util.{Timer, TimerTask}
import java.util.concurrent.TimeUnit

import scala.util.Try
import scala.util.control.NonFatal

import com.codahale.metrics.MetricRegistry
import com.codahale.metrics.json.MetricsModule
import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.fs.permission.FsPermission
import org.apache.kyuubi.Logging
import org.apache.spark.{KyuubiSparkUtil, SparkConf}
import org.apache.spark.KyuubiConf._

private[metrics] class JsonFileReporter(conf: SparkConf, registry: MetricRegistry)
  extends Closeable with Logging {

  private val jsonMapper = new ObjectMapper().registerModule(
    new MetricsModule(TimeUnit.MILLISECONDS, TimeUnit.MILLISECONDS, false))
  private val timer = new Timer(true)
  private val interval = KyuubiSparkUtil.timeStringAsMs(conf.get(METRICS_REPORT_INTERVAL))
  private val path = conf.get(METRICS_REPORT_LOCATION)
  private val hadoopConf = KyuubiSparkUtil.newConfiguration(conf)

  def start(): Unit = {
    timer.schedule(new TimerTask {
      var bw: BufferedWriter = _
      override def run(): Unit = try {
        val json = jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(registry)
        val tmpPath = new Path(path + ".tmp")
        val tmpPathUri = tmpPath.toUri
        val fs = if (tmpPathUri.getScheme == null && tmpPathUri.getAuthority == null) {
          FileSystem.getLocal(hadoopConf)
        } else {
          FileSystem.get(tmpPathUri, hadoopConf)
        }
        fs.delete(tmpPath, true)
        bw = new BufferedWriter(new OutputStreamWriter(fs.create(tmpPath, true)))
        bw.write(json)
        bw.close()
        fs.setPermission(tmpPath, FsPermission.createImmutable(Integer.parseInt("644", 8).toShort))
        val finalPath = new Path(path)
        fs.rename(tmpPath, finalPath)
        fs.setPermission(finalPath,
          FsPermission.createImmutable(Integer.parseInt("644", 8).toShort))
      } catch {
        case NonFatal(e) => error("Error writing metrics to json file" + path, e)
      } finally {
        if (bw != null) {
          Try(bw.close())
        }
      }
    }, 0, interval)
  }

  override def close(): Unit = {
    timer.cancel()
  }
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.text.SimpleDateFormat
import java.util.{Calendar, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes("utf-8"))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'")
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package org.apache.spark.metrics.sink

import java.util.Properties
import java.util.concurrent.TimeUnit

import javax.servlet.http.HttpServletRequest

import com.codahale.metrics.MetricRegistry
import com.codahale.metrics.json.MetricsModule
import com.fasterxml.jackson.databind.ObjectMapper
import org.eclipse.jetty.servlet.ServletContextHandler

import org.apache.spark.SecurityManager
import org.apache.spark.ui.JettyUtils._

private[spark] class MetricsServlet(
    val property: Properties,
    val registry: MetricRegistry,
    securityMgr: SecurityManager)
  extends Sink {

  val SERVLET_KEY_PATH = "path"
  val SERVLET_KEY_SAMPLE = "sample"

  val SERVLET_DEFAULT_SAMPLE = false

  val servletPath = property.getProperty(SERVLET_KEY_PATH)

  val servletShowSample = Option(property.getProperty(SERVLET_KEY_SAMPLE)).map(_.toBoolean)
    .getOrElse(SERVLET_DEFAULT_SAMPLE)

  val mapper = new ObjectMapper().registerModule(
    new MetricsModule(TimeUnit.SECONDS, TimeUnit.MILLISECONDS, servletShowSample))
  //最终生成处理/metrics/json请求的ServletContextHandler,而请求的真正处理由getMetricsSnapshot方法
  //利用fastJson解析,生成
  def getHandlers: Array[ServletContextHandler] = {
    Array[ServletContextHandler](
      createServletHandler(servletPath,
        new ServletParams(request => getMetricsSnapshot(request), "text/json"), securityMgr)
    )
  }

  def getMetricsSnapshot(request: HttpServletRequest): String = {
    mapper.writeValueAsString(registry)
  }

  override def start() { }

  override def stop() { }

  override def report() { }
} 

package com.expedia.www.haystack.trace.indexer.writers.es

import java.util

import com.expedia.www.haystack.trace.commons.config.entities.IndexFieldType.IndexFieldType
import com.expedia.www.haystack.trace.commons.config.entities.{IndexFieldType, WhitelistIndexFieldConfiguration}
import com.fasterxml.jackson.core.`type`.TypeReference
import com.fasterxml.jackson.databind.ObjectMapper
import io.searchbox.client.JestClient
import io.searchbox.indices.template.{GetTemplate, PutTemplate}
import org.slf4j.LoggerFactory

import scala.collection.JavaConverters._

class IndexTemplateHandler(client: JestClient,
                           applyTemplate: Option[String],
                           indexType: String,
                           whitelistFieldConfig: WhitelistIndexFieldConfiguration) {

  private val LOGGER = LoggerFactory.getLogger(classOf[IndexTemplateHandler])

  private val ES_TEMPLATE_NAME = "spans-index-template"
  private val mapper = new ObjectMapper()

  def run() {
    applyTemplate match {
      case Some(template) => updateESTemplate(template)
      case _ => 
    }

    whitelistFieldConfig.addOnChangeListener(() => {
      LOGGER.info("applying the new elastic template as whitelist fields have changed from query perspective like enableRangeQuery")
      readTemplate() match {
        case Some(template) => updateESTemplate(template)
        case _ =>
      }
    })
  }

  private def esDataType(`type`: IndexFieldType): String = {
    `type` match {
      case IndexFieldType.int => "integer"
      case IndexFieldType.string => "keyword"
      case _ => `type`.toString
    }
  }

  private def updateESTemplate(templateJson: String): Unit = {
    val esTemplate: util.HashMap[String, Object] = mapper.readValue(templateJson, new TypeReference[util.HashMap[String, Object]]() {})
    val mappings = esTemplate.get("mappings").asInstanceOf[util.HashMap[String, Object]]
    val propertyMap =
      mappings.get(indexType).asInstanceOf[util.HashMap[String, Object]]
        .get("properties").asInstanceOf[util.HashMap[String, Object]]
        .get(indexType).asInstanceOf[util.HashMap[String, Object]]
        .get("properties").asInstanceOf[util.HashMap[String, Object]]

    whitelistFieldConfig.whitelistIndexFields.foreach(wf => {
      val prop = propertyMap.get(wf.name)
      if (prop != null) {
        if (wf.enabled && wf.enableRangeQuery) {
          propertyMap.put(wf.name, Map("type" -> esDataType(wf.`type`), "doc_values" -> true, "norms" -> false).asJava)
        } else {
          prop.asInstanceOf[util.HashMap[String, Object]].put("doc_values", Boolean.box(wf.enableRangeQuery))
        }
      }
    })

    val newTemplateJson = mapper.writeValueAsString(esTemplate)

    LOGGER.info(s"setting the template with name $ES_TEMPLATE_NAME - $newTemplateJson")

    val putTemplateRequest = new PutTemplate.Builder(ES_TEMPLATE_NAME, newTemplateJson).build()
    val result = client.execute(putTemplateRequest)
    if (!result.isSucceeded) {
      throw new RuntimeException(s"Fail to apply the following template to elastic search with reason=${result.getErrorMessage}")
    }
  }

  private def readTemplate(): Option[String] = {
    val request = new GetTemplate.Builder(ES_TEMPLATE_NAME).build()
    val result = client.execute(request)
    if (result.isSucceeded) {
      Some(result.getJsonObject.get(ES_TEMPLATE_NAME).toString)
    } else {
      LOGGER.error(s"Fail to read the template with name $ES_TEMPLATE_NAME for reason ${result.getErrorMessage}")
      None
    }
  }
} 

package com.spotify.ratatool.diffy

import java.io.StringReader

import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}
import com.google.api.client.json.JsonObjectParser
import com.google.api.client.json.jackson2.JacksonFactory
import com.google.api.services.bigquery.model.{TableFieldSchema, TableRow, TableSchema}

import scala.jdk.CollectionConverters._
import scala.util.Try


class TableRowDiffy(tableSchema: TableSchema,
                    ignore: Set[String] = Set.empty,
                    unordered: Set[String] = Set.empty,
                    unorderedFieldKeys: Map[String, String] = Map())
  extends Diffy[TableRow](ignore, unordered, unorderedFieldKeys) {

  override def apply(x: TableRow, y: TableRow): Seq[Delta] =
    diff(Option(x), Option(y), schema.getFields.asScala.toList, "")

  private type Record = java.util.Map[String, AnyRef]

  // TableSchema is not serializable
  private val schemaString: String =
    new ObjectMapper().disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)
      .writeValueAsString(tableSchema)
  private lazy val schema: TableSchema =
    new JsonObjectParser(new JacksonFactory)
      .parseAndClose(new StringReader(schemaString), classOf[TableSchema])

  // scalastyle:off cyclomatic.complexity
  private def diff(x: Option[Record], y: Option[Record],
                   fields: Seq[TableFieldSchema], root: String): Seq[Delta] = {
    def getField(f: String)(x: Record): Option[AnyRef] = {
      Option(x.get(f))
    }

    fields.flatMap { f =>
      val name = f.getName
      val fullName = if (root.isEmpty) name else root + "." + name
      if (f.getType == "RECORD" && f.getMode != "REPEATED") {
        val a = x.flatMap(r => getField(name)(r).map(_.asInstanceOf[Record]))
        val b = y.flatMap(r => getField(name)(r).map(_.asInstanceOf[Record]))
        if (a.isEmpty && b.isEmpty) {
          Nil
        } else if (a.isEmpty || b.isEmpty) {
          Seq(Delta(fullName, a, b, UnknownDelta))
        } else {
          diff(a, b, f.getFields.asScala.toList, fullName)
        }
      } else if (f.getMode == "REPEATED" && unordered.contains(fullName)) {
        if (f.getType == "RECORD"
          && unorderedFieldKeys.contains(fullName)) {
          val l = x
            .flatMap(outer =>
              getField(name)(outer).map(_.asInstanceOf[java.util.List[Record]].asScala.toList))
            .getOrElse(List())
            .flatMap(inner =>
              Try(inner.get(unorderedFieldKeys(fullName))).toOption.map(k => (k, inner))).toMap
          val r = y
            .flatMap(outer =>
              getField(name)(outer).map(_.asInstanceOf[java.util.List[Record]].asScala.toList))
            .getOrElse(List())
            .flatMap(inner =>
              Try(inner.get(unorderedFieldKeys(fullName))).toOption.map(k => (k, inner))).toMap
          (l.keySet ++ r.keySet).flatMap(k =>
            diff(l.get(k), r.get(k), f.getFields.asScala.toList, fullName))
        } else {
          val a = x.flatMap(r => Option(r.get(name).asInstanceOf[java.util.List[AnyRef]]))
            .map(sortList)
          val b = y.flatMap(r => Option(r.get(name).asInstanceOf[java.util.List[AnyRef]]))
            .map(sortList)
          if (a == b) Nil else Seq(Delta(fullName, a, b, delta(a.orNull, b.orNull)))
        }
      } else {
        val a = x.flatMap(r => getField(name)(r))
        val b = y.flatMap(r => getField(name)(r))
        if (a == b) Nil else Seq(Delta(fullName, a, b, delta(a.orNull, b.orNull)))
      }
    }.filter(d => !ignore.contains(d.field))
  }
  // scalastyle:on cyclomatic.complexity
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.nio.charset.StandardCharsets
import java.text.SimpleDateFormat
import java.util.{Calendar, Locale, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes(StandardCharsets.UTF_8))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'", Locale.US)
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package org.apache.spark.metrics.sink

import java.util.Properties
import java.util.concurrent.TimeUnit
import javax.servlet.http.HttpServletRequest

import com.codahale.metrics.MetricRegistry
import com.codahale.metrics.json.MetricsModule
import com.fasterxml.jackson.databind.ObjectMapper
import org.eclipse.jetty.servlet.ServletContextHandler

import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.ui.JettyUtils._

private[spark] class MetricsServlet(
    val property: Properties,
    val registry: MetricRegistry,
    securityMgr: SecurityManager)
  extends Sink {

  val SERVLET_KEY_PATH = "path"
  val SERVLET_KEY_SAMPLE = "sample"

  val SERVLET_DEFAULT_SAMPLE = false

  val servletPath = property.getProperty(SERVLET_KEY_PATH)

  val servletShowSample = Option(property.getProperty(SERVLET_KEY_SAMPLE)).map(_.toBoolean)
    .getOrElse(SERVLET_DEFAULT_SAMPLE)

  val mapper = new ObjectMapper().registerModule(
    new MetricsModule(TimeUnit.SECONDS, TimeUnit.MILLISECONDS, servletShowSample))

  def getHandlers(conf: SparkConf): Array[ServletContextHandler] = {
    Array[ServletContextHandler](
      createServletHandler(servletPath,
        new ServletParams(request => getMetricsSnapshot(request), "text/json"), securityMgr, conf)
    )
  }

  def getMetricsSnapshot(request: HttpServletRequest): String = {
    mapper.writeValueAsString(registry)
  }

  override def start() { }

  override def stop() { }

  override def report() { }
} 

package com.sksamuel.pulsar4s.jackson

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.module.SimpleModule
import com.fasterxml.jackson.databind.ser.std.NumberSerializers
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper

object JacksonSupport {

  val mapper: ObjectMapper with ScalaObjectMapper = new ObjectMapper with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)

  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
  mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)
  mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true)
  mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
  mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)

  val module = new SimpleModule
  module.addSerializer(new NumberSerializers.DoubleSerializer(classOf[Double]))
} 

package com.github.simplesteph.ksm.source

import java.io.{Reader, StringReader}
import java.nio.charset.Charset
import java.util.Base64

import com.fasterxml.jackson.databind.ObjectMapper
import com.typesafe.config.Config
import org.slf4j.LoggerFactory
import skinny.http.{HTTP, HTTPException, Request, Response}

class GitLabSourceAcl extends SourceAcl {

  private val log = LoggerFactory.getLogger(classOf[GitLabSourceAcl])

  override val CONFIG_PREFIX: String = "gitlab"
  final val REPOID_CONFIG = "repoid"
  final val FILEPATH_CONFIG = "filepath"
  final val BRANCH_CONFIG = "branch"
  final val HOSTNAME_CONFIG = "hostname"
  final val ACCESSTOKEN_CONFIG = "accesstoken"

  var lastModified: Option[String] = None
  val objectMapper = new ObjectMapper()
  var repoid: String = _
  var filepath: String = _
  var branch: String = _
  var hostname: String = _
  var accessToken: String = _

  
  override def close(): Unit = {
    // HTTP
  }
} 

package com.github.simplesteph.ksm.source

import java.io.{Reader, StringReader}
import java.nio.charset.Charset
import java.util.Base64

import com.fasterxml.jackson.databind.ObjectMapper
import com.typesafe.config.Config
import org.slf4j.LoggerFactory
import skinny.http.{HTTP, HTTPException, Request, Response}

class BitbucketServerSourceAcl extends SourceAcl {

  private val log = LoggerFactory.getLogger(classOf[BitbucketServerSourceAcl])

  override val CONFIG_PREFIX: String = "bitbucket-server"

  final val HOSTNAME_CONFIG = "hostname"
  final val PORT_CONFIG = "port"
  final val PROTOCOL_CONFIG = "protocol"
  final val PROJECT_CONFIG = "project"
  final val REPO_CONFIG = "repo"
  final val FILEPATH_CONFIG = "filepath"
  final val AUTH_USERNAME_CONFIG = "auth.username"
  final val AUTH_PASSWORD_CONFIG = "auth.password"
  final val BRANCH_CONFIG = "branch"

  var lastCommit: Option[String] = None
  val objectMapper = new ObjectMapper()
  var http: HTTP = HTTP

  var hostname: String = _
  var port: String = _
  var protocol: String = _
  var project: String = _
  var repo: String = _
  var filePath: String = _
  var username: String = _
  var password: String = _
  var branch: Option[String] = _

  
  override def close(): Unit = {
    // HTTP
  }
} 

package com.github.simplesteph.ksm.source

import java.io.{Reader, StringReader}
import java.nio.charset.Charset
import java.util.Base64

import com.fasterxml.jackson.databind.ObjectMapper
import com.typesafe.config.Config
import org.slf4j.LoggerFactory
import skinny.http.{HTTP, HTTPException, Request, Response}

import scala.util.Try

class GitHubSourceAcl extends SourceAcl {

  private val log = LoggerFactory.getLogger(classOf[GitHubSourceAcl])

  override val CONFIG_PREFIX: String = "github"
  final val USER_CONFIG = "user"
  final val REPO_CONFIG = "repo"
  final val FILEPATH_CONFIG = "filepath"
  final val BRANCH_CONFIG = "branch"
  final val HOSTNAME_CONFIG = "hostname"
  final val AUTH_BASIC_CONFIG = "auth.basic"
  final val AUTH_TOKEN_CONFIG = "auth.token"

  var lastModified: Option[String] = None
  val objectMapper = new ObjectMapper()
  var user: String = _
  var repo: String = _
  var filepath: String = _
  var branch: String = _
  var hostname: String = _
  var basicOpt: Option[String] = _
  var tokenOpt: Option[String] = _

  
  override def close(): Unit = {
    // HTTP
  }
} 

package com.github.simplesteph.ksm.notification
import com.fasterxml.jackson.databind.ObjectMapper
import com.github.simplesteph.ksm.parser.CsvParserException
import com.typesafe.config.Config
import kafka.security.auth.{Acl, Resource}
import org.slf4j.LoggerFactory
import skinny.http.HTTP

import scala.util.{Failure, Success, Try}

class SlackNotification extends Notification {

  
  override def configure(config: Config): Unit = {
    webhook = config.getString(WEBHOOK_CONFIG)
    username = config.getString(USERNAME_CONFIG)
    icon = config.getString(ICON_CONFIG)
    channel = config.getString(CHANNEL_CONFIG)
  }

  override def notifyOne(action: String, acls: Set[(Resource, Acl)]): Unit = {
    if (acls.nonEmpty) {
      val messages = acls.map {
        case (resource, acl) =>
          val message = Notification.printAcl(acl, resource)
          s"$action $message"
      }.toList

      sendToSlack(messages)
    }
  }

  def sendToSlack(messages: List[String], retries: Int = 5): Unit = {
    if (retries > 0) {
      messages
        .grouped(50)
        .foreach(msgChunks => {
          val text =
            s"""```
             |${msgChunks.mkString("\n")}
             |```
           """.stripMargin

          val payload = objectMapper
            .createObjectNode()
            .put("text", text)
            .put("username", username)
            .put("icon_url", icon)
            .put("channel", channel)

          val response = HTTP.post(webhook, payload.toString)

          response.status match {
            case 200 => ()
            case _ =>
              log.warn(response.asString)
              if (retries > 1) log.warn("Retrying...")
              Thread.sleep(300)
              sendToSlack(msgChunks, retries - 1)
          }
        })
    } else {
      log.error("Can't send notification to Slack after retries")
    }
  }

  override def notifyErrors(errs: List[Try[Throwable]]): Unit = {

    val messages = errs.map {
      case Failure(cPE: CsvParserException) =>
        s"${cPE.getLocalizedMessage} | Row: ${cPE.printRow()}"
      case Success(t) => s"refresh exception: ${t.getLocalizedMessage}"
      case Failure(t) => s"refresh exception: ${t.getLocalizedMessage}"
    }

    sendToSlack(messages)
  }

  override def close(): Unit = {}
} 

package org.apache.spark.status.api.v1

import java.io.OutputStream
import java.lang.annotation.Annotation
import java.lang.reflect.Type
import java.text.SimpleDateFormat
import java.util.{Calendar, SimpleTimeZone}
import javax.ws.rs.Produces
import javax.ws.rs.core.{MediaType, MultivaluedMap}
import javax.ws.rs.ext.{MessageBodyWriter, Provider}

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}


@Provider
@Produces(Array(MediaType.APPLICATION_JSON))
private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{

  val mapper = new ObjectMapper() {
    override def writeValueAsString(t: Any): String = {
      super.writeValueAsString(t)
    }
  }
  mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
  mapper.enable(SerializationFeature.INDENT_OUTPUT)
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)

  override def isWriteable(
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Boolean = {
      true
  }

  override def writeTo(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType,
      multivaluedMap: MultivaluedMap[String, AnyRef],
      outputStream: OutputStream): Unit = {
    t match {
      case ErrorWrapper(err) => outputStream.write(err.getBytes("utf-8"))
      case _ => mapper.writeValue(outputStream, t)
    }
  }

  override def getSize(
      t: Object,
      aClass: Class[_],
      `type`: Type,
      annotations: Array[Annotation],
      mediaType: MediaType): Long = {
    -1L
  }
}

private[spark] object JacksonMessageWriter {
  def makeISODateFormat: SimpleDateFormat = {
    val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'")
    val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT"))
    iso8601.setCalendar(cal)
    iso8601
  }
} 

package org.apache.spark.metrics.sink

import java.util.Properties
import java.util.concurrent.TimeUnit

import javax.servlet.http.HttpServletRequest

import com.codahale.metrics.MetricRegistry
import com.codahale.metrics.json.MetricsModule
import com.fasterxml.jackson.databind.ObjectMapper
import org.eclipse.jetty.servlet.ServletContextHandler

import org.apache.spark.{SparkConf, SecurityManager}
import org.apache.spark.ui.JettyUtils._

private[spark] class MetricsServlet(
    val property: Properties,
    val registry: MetricRegistry,
    securityMgr: SecurityManager)
  extends Sink {

  val SERVLET_KEY_PATH = "path"
  val SERVLET_KEY_SAMPLE = "sample"

  val SERVLET_DEFAULT_SAMPLE = false

  val servletPath = property.getProperty(SERVLET_KEY_PATH)

  val servletShowSample = Option(property.getProperty(SERVLET_KEY_SAMPLE)).map(_.toBoolean)
    .getOrElse(SERVLET_DEFAULT_SAMPLE)

  val mapper = new ObjectMapper().registerModule(
    new MetricsModule(TimeUnit.SECONDS, TimeUnit.MILLISECONDS, servletShowSample))

  def getHandlers(conf: SparkConf): Array[ServletContextHandler] = {
    Array[ServletContextHandler](
      createServletHandler(servletPath,
        new ServletParams(request => getMetricsSnapshot(request), "text/json"), securityMgr, conf)
    )
  }

  def getMetricsSnapshot(request: HttpServletRequest): String = {
    mapper.writeValueAsString(registry)
  }

  override def start() { }

  override def stop() { }

  override def report() { }
} 

package asura.common.util

import java.text.SimpleDateFormat

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}

object JavaJsonUtils extends JsonUtils {

  val mapper: ObjectMapper = new ObjectMapper()
  mapper.setDateFormat(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"))
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true)
  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
  mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)
  mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true)
  mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
  mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)
} 

package asura.common.util

import java.io.InputStream
import java.text.SimpleDateFormat

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.core.`type`.TypeReference
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper

object JsonUtils extends JsonUtils {

  val mapper: ObjectMapper with ScalaObjectMapper = new ObjectMapper() with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)
  mapper.setDateFormat(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"))
  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.configure(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true)
  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
  mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)
  mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true)
  mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
  mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)

}

trait JsonUtils {
  val mapper: ObjectMapper

  def stringify(obj: AnyRef): String = {
    mapper.writeValueAsString(obj)
  }

  def parse[T <: AnyRef](content: String, c: Class[T]): T = {
    mapper.readValue(content, c)
  }

  def parse[T <: AnyRef](input: InputStream, c: Class[T]): T = {
    mapper.readValue(input, c)
  }

  def parse[T <: AnyRef](content: String, typeReference: TypeReference[T]): T = {
    mapper.readValue(content, typeReference)
  }
} 

package asura.core.redis

import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
import com.typesafe.scalalogging.Logger
import org.redisson.Redisson
import org.redisson.api.{RFuture, RedissonClient}
import org.redisson.codec.JsonJacksonCodec
import org.redisson.config.Config

import scala.compat.java8.FutureConverters.{toScala => javaFutureToScalaFuture}
import scala.concurrent.Future

object RedisClient {

  val logger = Logger("RedisClient")
  var redisson: RedissonClient = null
  private val mapper: ObjectMapper with ScalaObjectMapper = new ObjectMapper() with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)
  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
  mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)

  def init(servers: Seq[String] = Nil): Unit = {
    val config = new Config()
    config.setCodec(new JsonJacksonCodec(mapper))
    if (null == servers || servers.isEmpty) {
      config.useSingleServer().setAddress("redis://127.0.0.1:6379")
    } else if (servers.length == 1) {
      config.useSingleServer().setAddress(servers(0))
    } else {
      config.useClusterServers().setScanInterval(3000).addNodeAddress(servers: _*)
    }
    logger.info(s"init redis client with config: ${config.toJSON}")
    redisson = Redisson.create(config)
  }

  def shutdown(): Unit = {
    if (null != redisson) {
      logger.info("shutdown redis client")
      redisson.shutdown()
    }
  }

  implicit def toScala[T](rf: RFuture[T]): Future[T] = {
    javaFutureToScalaFuture(rf)
  }
} 

package org.squbs.marshallers.json

import akka.actor.ActorSystem
import akka.http.scaladsl.marshalling.Marshal
import akka.http.scaladsl.model.{HttpEntity, MediaTypes, MessageEntity}
import akka.http.scaladsl.unmarshalling.Unmarshal
import akka.stream.ActorMaterializer
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility
import com.fasterxml.jackson.annotation.PropertyAccessor
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.scalatest.{AsyncFlatSpec, BeforeAndAfterAll, Matchers}
import org.squbs.marshallers.json.TestData._

class JacksonMapperSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll {

  import JacksonMapperSupport._

  implicit val system = ActorSystem("JacksonMapperSpec")
  implicit val mat = ActorMaterializer()
  JacksonMapperSupport.setDefaultMapper(new ObjectMapper().registerModule(DefaultScalaModule))

  it should "marshal and unmarshal standard case classes" in {
    val entity = HttpEntity(MediaTypes.`application/json`, fullTeamJson)
    Marshal(fullTeam).to[MessageEntity] map { _ shouldBe entity }
    Unmarshal(entity).to[Team] map { _ shouldBe fullTeam }
  }

  it should "marshal and unmarshal Scala non-case classes" in {
    val entity = HttpEntity(MediaTypes.`application/json`, fullTeamJson)
    Marshal(fullTeamNonCaseClass).to[MessageEntity] map { _ shouldBe entity }
    Unmarshal(entity).to[TeamNonCaseClass] map { _ shouldBe fullTeamNonCaseClass }
  }

  it should "marshal and unmarshal Scala class with Java Bean members" in {
    val entity = HttpEntity(MediaTypes.`application/json`, fullTeamJson)
    Marshal(fullTeamWithBeanMember).to[MessageEntity] map { _ shouldBe entity }
    Unmarshal(entity).to[TeamWithBeanMember] map { _ shouldBe fullTeamWithBeanMember }
  }

  it should "marshal and unmarshal Java Bean with case class members" in {
    val entity = HttpEntity(MediaTypes.`application/json`, fullTeamJson)
    Marshal(fullTeamWithCaseClassMember).to[MessageEntity] map { _ shouldBe entity }
    Unmarshal(entity).to[TeamBeanWithCaseClassMember] map { _ shouldBe fullTeamWithCaseClassMember }
  }

  it should "marshal and unmarshal Java Bean" in {
    val fieldMapper = new ObjectMapper().setVisibility(PropertyAccessor.FIELD, Visibility.ANY)
      .registerModule(DefaultScalaModule)
    JacksonMapperSupport.register[TeamWithPrivateMembers](fieldMapper)
    val entity = HttpEntity(MediaTypes.`application/json`, fullTeamJson)
    Marshal(fullTeamWithPrivateMembers).to[MessageEntity] map { _ shouldBe entity }
    Unmarshal(entity).to[TeamWithPrivateMembers] map { _ shouldBe fullTeamWithPrivateMembers }
  }

  it should "Marshal and unmarshal Jackson annotated Java subclasses" in {
    JacksonMapperSupport.register[PageData](new ObjectMapper)
    val entity = HttpEntity(MediaTypes.`application/json`, pageTestJson)
    Marshal(pageTest).to[MessageEntity] map { _ shouldBe entity }
    Unmarshal(entity).to[PageData] map { _ shouldBe pageTest }
  }
} 

package org.marvin.util

import com.fasterxml.jackson.databind.{DeserializationFeature, JsonNode, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.github.fge.jsonschema.core.exceptions.ProcessingException
import com.github.fge.jsonschema.core.report.ProcessingMessage
import com.github.fge.jsonschema.main.JsonSchemaFactory
import grizzled.slf4j.Logging
import org.json4s.jackson.JsonMethods.{asJsonNode, parse}
import spray.json._

import scala.io.Source
import scala.reflect.{ClassTag, _}

object JsonUtil extends Logging {
  val jacksonMapper = new ObjectMapper()
  jacksonMapper.registerModule(DefaultScalaModule)
  jacksonMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)

  def toJson(value: Map[Symbol, Any]): String = {
    toJson(value map { case (k,v) => k.name -> v})
  }

  def toJson(value: Any): String = {
    jacksonMapper.writeValueAsString(value)
  }

  def toMap(jsonString: String): Map[String, Any] = {
    JsonUtil.fromJson[Map[String, List[Map[String, String]]]](jsonString)
  }

  def fromJson[T: ClassTag](jsonString: String, validate: Boolean = false): T = {

    if (validate) validateJson[T](jsonString)

    jacksonMapper.readValue[T](jsonString, classTag[T].runtimeClass.asInstanceOf[Class[T]])
  }

  def validateJson[T: ClassTag](jsonString: String): Unit = {
    val className = classTag[T].runtimeClass.getSimpleName
    val schemaName = className.toString + "Schema.json"

    var jsonSchema: String = null

    try{
      jsonSchema = Source.fromResource(schemaName).mkString
      validateJson(jsonString, jsonSchema)
    } catch {
      case e: NullPointerException => info(s"File ${schemaName} not found, check your schema file")
        throw e
    }
  }

  
  def validateJson(jsonString: String, jsonSchema: String): Unit = {

    val schema: JsonNode = asJsonNode(parse(jsonSchema))
    val jsonToValidate: JsonNode = asJsonNode(parse(jsonString))
    val validator = JsonSchemaFactory.byDefault().getValidator

    val processingReport = validator.validate(schema, jsonToValidate)
    if (!processingReport.isSuccess) {
      val sb = new StringBuilder()
      processingReport.forEach {
        message: ProcessingMessage => {
          warn(message.asJson())
          sb.append(message.getMessage)
        }
      }
      throw new ProcessingException(sb.toString)
    }
  }

  def format(jsonString: String): String ={
    return jsonString.parseJson.prettyPrint
  }

  def format(jsonMap: Map[String, Any]): String ={
    return this.format(this.toJson(jsonMap))
  }
} 

package org.crowdcrawler.proxycrawler

import java.nio.charset.StandardCharsets
import java.nio.file.{Paths, Files}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
import com.typesafe.scalalogging.Logger
import org.crowdcrawler.proxycrawler.checker.ProxyChecker
import org.slf4j.LoggerFactory


object Main {
  private val LOGGER = Logger(LoggerFactory.getLogger(Main.getClass))
  val OBJECT_MAPPER = new ObjectMapper() with ScalaObjectMapper
  OBJECT_MAPPER.registerModule(DefaultScalaModule)

  def main(args: Array[String]): Unit = {
    val usage = "Usage: \n\tcrawl [pluginClassName]* OutputFile\n" +
      "\tcheck proxies.json valid_proxies.json\n" +
      "\tfilter valid_proxies.json <HTTP|HTTPS|SOCKS> output.json\n" +
      "For example:\n" +
      "\t1. Crawl all supported websites and save proxies to proxies.json\n" +
      "\t\tcrawl proxies.json\n" +
      "\t2. Crawl www.cnproxy.com and save proxies to proxies.json:\n" +
      "\t\tcrawl CnProxyComPlugin proxies.json\n" +
      "\t3. Check the speed of proxies.\n" +
      "\t\tcheck proxies.json valid_proxies.json\n" +
      "\t4. Filter proxies by schema\n" +
      "\t\tfilter valid_proxies.json HTTP http.json\n"
    if (args.length < 2) {
      println(usage)
      return
    }

    val start = System.currentTimeMillis
    if (args(0) == "crawl") {
      val classNames = if (args.length == 2) {
        Array("CnProxyComPlugin", "CoolProxyNetPlugin", "GatherproxyComPlugin", "IpcnOrgPlugin",
          "ProxyListOrg", "SocksProxyNet", "USProxyOrgPlugin")
      } else {
        args.slice(1, args.length-1)
      }
      val crawler = ProxyCrawler(classNames: _*)

      val proxies = crawler.crawl()

      LOGGER.info("Writing to disk, " + proxies.size + " proxies")
      val json = OBJECT_MAPPER.writerWithDefaultPrettyPrinter.writeValueAsString(proxies)
      Files.write(Paths.get(args.last), json.getBytes(StandardCharsets.UTF_8))
    } else if (args(0) == "check") {
      val json = io.Source.fromFile(args(1), "utf-8").mkString
      val list = OBJECT_MAPPER.readValue[List[ProxyInfo]](json)

      // sort by speed desc
      val validProxies = ProxyChecker.check(list).filter(_.speed > 0)
        .sortWith((p1, p2) => p1.speed > p2.speed)
      LOGGER.info("Writing to disk, " + validProxies.size + " valid proxies out of " + list.size + " proxies")
      val newJson = OBJECT_MAPPER.writerWithDefaultPrettyPrinter
        .writeValueAsString(validProxies)
      Files.write(Paths.get(args(2)), newJson.getBytes(StandardCharsets.UTF_8))

    } else if (args(0) == "filter") {
      val json = io.Source.fromFile(args(1), "utf-8").mkString
      val list = OBJECT_MAPPER.readValue[List[ProxyInfo]](json)
      val filtered = if (args(2) == "SOCKS") {
        list.filter(p => p.schema == "SOCKS" | p.schema == "SOCKS4" || p.schema == "SOCKS5")
      } else {
        list.filter(p => p.schema == args(2))
      }

      val newJson = OBJECT_MAPPER.writerWithDefaultPrettyPrinter
        .writeValueAsString(filtered)
      Files.write(Paths.get(args(3)), newJson.getBytes(StandardCharsets.UTF_8))
    } else {
      println(usage)
      return
    }

    val end = System.currentTimeMillis
    LOGGER.info("Time elapsed " + (end - start) / 1000 + " seconds.")
  }
} 

package ai.diffy.lifter

import com.fasterxml.jackson.core.{JsonGenerator, JsonToken}
import com.fasterxml.jackson.databind.annotation.JsonSerialize
import com.fasterxml.jackson.databind.ser.std.StdSerializer
import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper, SerializerProvider}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
import com.twitter.util.Try

import scala.collection.JavaConversions._
import scala.language.postfixOps
import scala.reflect.runtime.universe.runtimeMirror
import scala.tools.reflect.ToolBox
import scala.util.control.NoStackTrace

object JsonLifter {
  @JsonSerialize(using = classOf[JsonNullSerializer])
  object JsonNull
  object JsonParseError extends Exception with NoStackTrace

  val toolbox = runtimeMirror(getClass.getClassLoader).mkToolBox()

  val Mapper = new ObjectMapper with ScalaObjectMapper
  Mapper.registerModule(DefaultScalaModule)
  def apply(obj: Any): JsonNode = Mapper.valueToTree(obj)

  def lift(node: JsonNode): Any = node.asToken match {
    case JsonToken.START_ARRAY        =>
      node.elements.toSeq.map {
        element => lift(element)
      }
    case JsonToken.START_OBJECT       => {
      val fields = node.fieldNames.toSet
      if (fields.exists{ field => Try(toolbox.parse(s"object ${field}123")).isThrow}) {
        node.fields map {field => (field.getKey -> lift(field.getValue))} toMap
      } else {
        FieldMap(
          node.fields map {field => (field.getKey -> lift(field.getValue))} toMap
        )
      }
    }
    case JsonToken.VALUE_FALSE        => false
    case JsonToken.VALUE_NULL         => JsonNull
    case JsonToken.VALUE_NUMBER_FLOAT => node.asDouble
    case JsonToken.VALUE_NUMBER_INT   => node.asLong
    case JsonToken.VALUE_TRUE         => true
    case JsonToken.VALUE_STRING       => node.textValue
    case _                            => throw JsonParseError
  }

  def decode(json: String): JsonNode = Mapper.readTree(json)
  def encode(item: Any): String = Mapper.writer.writeValueAsString(item)
}

class JsonNullSerializer(clazz: Class[Any]) extends StdSerializer[Any](clazz) {
  def this() {
    this(null)
  }

  override def serialize(t: Any, jsonGenerator: JsonGenerator, serializerProvider: SerializerProvider): Unit = {
    jsonGenerator.writeNull()
  }
} 

package io.github.azhur.kafkaserdejackson

import java.util

import scala.reflect.runtime.universe._

import com.fasterxml.jackson.core.FormatSchema
import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.kafka.common.errors.SerializationException
import org.apache.kafka.common.serialization.{ Deserializer, Serde, Serializer }
import Jackson.typeReference

import scala.language.implicitConversions
import scala.reflect.ClassTag
import scala.util.control.NonFatal

trait JacksonFormatSchemaSupport {
  implicit def toSerializer[T <: AnyRef](implicit mapper: ObjectMapper,
                                         schema: FormatSchema): Serializer[T] =
    new Serializer[T] {
      private val writer                                                         = mapper.writer(schema)
      override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {}
      override def close(): Unit                                                 = {}
      override def serialize(topic: String, data: T): Array[Byte] =
        if (data == null) null
        else
          try writer.writeValueAsBytes(data)
          catch {
            case NonFatal(e) => throw new SerializationException(e)
          }
    }

  implicit def toDeserializer[T >: Null <: AnyRef](
      implicit mapper: ObjectMapper,
      schema: FormatSchema,
      tt: TypeTag[T]
  ): Deserializer[T] =
    new Deserializer[T] {
      private val reader                                                         = mapper.readerFor(typeReference[T]).`with`(schema)
      override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {}
      override def close(): Unit                                                 = {}
      override def deserialize(topic: String, data: Array[Byte]): T =
        if (data == null) null
        else
          try reader.readValue[T](data)
          catch {
            case NonFatal(e) => throw new SerializationException(e)
          }
    }

  implicit def toSerde[T >: Null <: AnyRef](
      implicit mapper: ObjectMapper,
      schema: FormatSchema,
      ct: TypeTag[T]
  ): Serde[T] =
    new Serde[T] {
      override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {}
      override def close(): Unit                                                 = {}
      override def serializer(): Serializer[T]                                   = toSerializer[T]
      override def deserializer(): Deserializer[T]                               = toDeserializer[T]
    }
}

object JacksonFormatSchemaSupport extends JacksonFormatSchemaSupport 

package io.github.azhur.kafkaserdejackson

import java.util

import com.fasterxml.jackson.databind.ObjectMapper
import Jackson.typeReference
import org.apache.kafka.common.errors.SerializationException
import org.apache.kafka.common.serialization.{ Deserializer, Serde, Serializer }

import scala.language.implicitConversions
import scala.reflect.runtime.universe._
import scala.util.control.NonFatal

trait JacksonJsonSupport {
  implicit def toSerializer[T <: AnyRef](implicit mapper: ObjectMapper): Serializer[T] =
    new Serializer[T] {
      override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {}
      override def close(): Unit                                                 = {}
      override def serialize(topic: String, data: T): Array[Byte] =
        if (data == null) null
        else
          try mapper.writeValueAsBytes(data)
          catch {
            case NonFatal(e) => throw new SerializationException(e)
          }
    }

  implicit def toDeserializer[T >: Null <: AnyRef](
      implicit mapper: ObjectMapper,
      tt: TypeTag[T]
  ): Deserializer[T] =
    new Deserializer[T] {
      private val tr                                                             = typeReference[T]
      override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {}
      override def close(): Unit                                                 = {}
      override def deserialize(topic: String, data: Array[Byte]): T =
        if (data == null) null
        else
          try mapper.readValue[T](data, tr)
          catch {
            case NonFatal(e) => throw new SerializationException(e)
          }
    }

  implicit def toSerde[T >: Null <: AnyRef](
      implicit mapper: ObjectMapper,
      tt: TypeTag[T]
  ): Serde[T] =
    new Serde[T] {
      override def configure(configs: util.Map[String, _], isKey: Boolean): Unit = {}
      override def close(): Unit                                                 = {}
      override def serializer(): Serializer[T]                                   = toSerializer[T]
      override def deserializer(): Deserializer[T]                               = toDeserializer[T]
    }
}

object JacksonJsonSupport extends JacksonJsonSupport 

package mass.extension

import java.nio.file.{ Files, Path, Paths }

import akka.actor.ExtendedActorSystem
import akka.serialization.jackson.JacksonObjectMapperProvider
import com.fasterxml.jackson.databind.ObjectMapper
import com.typesafe.scalalogging.StrictLogging
import fusion.common.extension.{ FusionExtension, FusionExtensionId }
import fusion.core.extension.FusionCore
import mass.MassSettings
import mass.core.Constants


final class MassCore private (override val classicSystem: ExtendedActorSystem)
    extends FusionExtension
    with StrictLogging {
  FusionCore(classicSystem)
  val settings: MassSettings = MassSettings(classicSystem.settings.config)
  val jsonMapper: ObjectMapper = JacksonObjectMapperProvider(classicSystem).getOrCreate(Constants.JACKSON_JSON, None)
  val cborMapper: ObjectMapper = JacksonObjectMapperProvider(classicSystem).getOrCreate(Constants.JACKSON_CBOR, None)
  val tempDirectory: Path = {
    val _tempDirectory = Paths.get(
      configuration.getOrElse[String](s"${Constants.BASE_CONF}.core.temp-dir", System.getProperty("java.io.tmpdir")))
    if (!Files.isDirectory(_tempDirectory)) {
      Files.createDirectories(_tempDirectory)
    }
    _tempDirectory
  }

  logger.info(configuration.getConfig(Constants.BASE_CONF).toString)

  def name: String = configuration.getString(s"${Constants.BASE_CONF}.name")

  override def toString = s"MassCore($classicSystem)"
}

object MassCore extends FusionExtensionId[MassCore] {
  override def createExtension(system: ExtendedActorSystem): MassCore = new MassCore(system)
} 

package com.landoop.streamreactor.connect.hive

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper

object JacksonSupport {

  val mapper: ObjectMapper with ScalaObjectMapper = new ObjectMapper with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)

  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
  mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)
  mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true)
  mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
  mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)
} 

package com.landoop.streamreactor.hive.it

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper

object JacksonSupport {

  val mapper: ObjectMapper with ScalaObjectMapper = new ObjectMapper with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)

  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
  mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)
  mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true)
  mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
  mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)
} 

package com.datamountaineer.streamreactor.connect.rethink.source

import com.fasterxml.jackson.databind.ObjectMapper
import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.connect.data.{Schema, SchemaBuilder, Struct}



object ChangeFeedStructBuilder extends StrictLogging {

  val mapper = new ObjectMapper()
  val oldVal = "old_val"
  val newVal = "new_val"
  val state = "state"
  val `type` = "type"

  val schema: Schema = SchemaBuilder.struct.name("ReThinkChangeFeed")
    .version(1)
    .field(state, Schema.OPTIONAL_STRING_SCHEMA)
    .field(oldVal, Schema.OPTIONAL_STRING_SCHEMA)
    .field(newVal, Schema.OPTIONAL_STRING_SCHEMA)
    .field(`type`, Schema.OPTIONAL_STRING_SCHEMA)
    .build

  def apply(hm: Map[String, Object]): Struct = {
    val struct = new Struct(schema)
    hm.foreach({ case (k, v) => if (v != null) struct.put(k, v.toString) })
    struct
  }
} 

package com.datamountaineer.streamreactor.connect.redis.sink.writer

import com.datamountaineer.kcql.Kcql
import com.datamountaineer.streamreactor.connect.redis.sink.config.{RedisKCQLSetting, RedisSinkSettings}
import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.kafka.connect.errors.ConnectException
import org.apache.kafka.connect.sink.SinkRecord

import scala.collection.JavaConverters._
import scala.util.{Failure, Success, Try}


class RedisStreams(sinkSettings: RedisSinkSettings) extends RedisWriter with PubSubSupport {

  val configs: Set[Kcql] = sinkSettings.kcqlSettings.map(_.kcqlConfig)
  configs.foreach { c =>
//    assert(c.getTarget.length > 0, "Add to your KCQL syntax : INSERT INTO REDIS_KEY_NAME ")
    assert(c.getSource.trim.length > 0, "You need to define one (1) topic to source data. Add to your KCQL syntax: SELECT * FROM topicName")
    val allFields = if (c.getIgnoredFields.isEmpty) false else true
    assert(c.getStoredAs.equalsIgnoreCase("Stream"), "This mode requires the KCQL syntax: STOREAS Stream")
  }

  // Write a sequence of SinkRecords to Redis
  override def write(records: Seq[SinkRecord]): Unit = {
    if (records.isEmpty)
      logger.debug("No records received on 'STREAM' Redis writer")
    else {
      logger.debug(s"'STREAM' Redis writer received ${records.size} records")
      insert(records.groupBy(_.topic))
    }
  }

  // Insert a batch of sink records
  def insert(records: Map[String, Seq[SinkRecord]]): Unit = {
    records.foreach({
      case (topic, sinkRecords: Seq[SinkRecord]) => {
        val topicSettings: Set[RedisKCQLSetting] = sinkSettings.kcqlSettings.filter(_.kcqlConfig.getSource == topic)
        if (topicSettings.isEmpty)
          logger.warn(s"Received a batch for topic $topic - but no KCQL supports it")
        val t = Try {
          sinkRecords.foreach { record =>
            topicSettings.map { KCQL =>
              // Get a SinkRecord
              val recordToSink = convert(record, fields = KCQL.fieldsAndAliases, ignoreFields = KCQL.ignoredFields)
              val jsonPayload = convertValueToJson(recordToSink)
              val payload = Try(new ObjectMapper().convertValue(jsonPayload, classOf[java.util.HashMap[String, Any]])) match {
                case Success(value) =>
                  value.asScala.toMap.map{
                    case(k, v) =>
                      (k, v.toString)
                  }
                case Failure(exception) =>
                  throw new ConnectException(s"Failed to convert payload to key value pairs", exception)
              }
              jedis.xadd(KCQL.kcqlConfig.getTarget, null, payload.asJava)
            }
          }
        }
        handleTry(t)
      }
      logger.debug(s"Published ${sinkRecords.size} messages for topic $topic")
    })
  }
} 

package com.landoop.streamreactor.connect.hive

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper

object JacksonSupport {

  val mapper: ObjectMapper with ScalaObjectMapper = new ObjectMapper with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)

  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
  mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)
  mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true)
  mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
  mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)
} 

package com.landoop.streamreactor.hive.it

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper

object JacksonSupport {

  val mapper: ObjectMapper with ScalaObjectMapper = new ObjectMapper with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)

  mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
  mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false)
  mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true)
  mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true)
  mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true)
} 

package com.wavesplatform.http

import java.io.IOException

import akka.http.scaladsl.model.{MediaRange, MediaType}
import akka.http.scaladsl.model.MediaTypes.`application/json`
import com.fasterxml.jackson.core.io.SegmentedStringWriter
import com.fasterxml.jackson.core.util.BufferRecyclers
import com.fasterxml.jackson.core.{JsonGenerator, JsonProcessingException}
import com.fasterxml.jackson.databind.module.SimpleModule
import com.fasterxml.jackson.databind.{JsonMappingException, JsonSerializer, ObjectMapper, SerializerProvider}
import play.api.libs.json._

object NumberAsStringSerializer extends JsonSerializer[JsValue] {
  private val fieldNamesToTranslate = Set(
    "amount",
    "available",
    "balance",
    "buyMatcherFee",
    "currentReward",
    "desiredReward",
    "effective",
    "fee",
    "feeAmount",
    "generating",
    "in",
    "matcherFee",
    "minIncrement",
    "minSponsoredAssetFee",
    "out",
    "price",
    "quantity",
    "regular",
    "reward",
    "sellMatcherFee",
    "sponsorBalance",
    "totalAmount",
    "totalFee",
    "totalWavesAmount",
    "value"
  )

  override def serialize(value: JsValue, json: JsonGenerator, provider: SerializerProvider): Unit = {
    value match {
      case JsNumber(v)  => json.writeNumber(v.bigDecimal)
      case JsString(v)  => json.writeString(v)
      case JsBoolean(v) => json.writeBoolean(v)

      case JsArray(elements) =>
        json.writeStartArray()
        elements.foreach { t =>
          serialize(t, json, provider)
        }
        json.writeEndArray()

      case JsObject(values) =>
        json.writeStartObject()
        values.foreach {
          case (name, JsNumber(v)) if fieldNamesToTranslate(name) =>
            json.writeStringField(name, v.bigDecimal.toPlainString)
          case (name, jsv) =>
            json.writeFieldName(name)
            serialize(jsv, json, provider)
        }
        json.writeEndObject()

      case JsNull => json.writeNull()
    }
  }
}

object CustomJson {
  val jsonWithNumbersAsStrings: MediaType.WithFixedCharset = `application/json`.withParams(Map("large-significand-format" -> "string"))

  def acceptsNumbersAsStrings(mr: MediaRange): Boolean = mr match {
    case MediaRange.One(`jsonWithNumbersAsStrings`, _) => true
    case _                                             => false
  }

  private lazy val mapper = (new ObjectMapper)
    .registerModule(new SimpleModule("WavesJson").addSerializer(classOf[JsValue], NumberAsStringSerializer))
    .configure(JsonGenerator.Feature.WRITE_BIGDECIMAL_AS_PLAIN, true)

  def writeValueAsString(value: JsValue): String = {
    val sw = new SegmentedStringWriter(BufferRecyclers.getBufferRecycler)
    try mapper.writeValue(sw, value)
    catch {
      case e: JsonProcessingException =>
        throw e
      case e: IOException =>
        // shouldn't really happen, but is declared as possibility so:
        throw JsonMappingException.fromUnexpectedIOE(e)
    }
    sw.getAndClear
  }
} 

package instagram.api.yaml

import de.zalando.play.controllers._
import org.scalacheck._
import org.scalacheck.Arbitrary._
import org.scalacheck.Prop._
import org.scalacheck.Test._
import org.specs2.mutable._
import org.specs2.execute._
import play.api.test.Helpers._
import play.api.test._
import play.api.mvc.MultipartFormData.FilePart
import play.api.mvc._

import org.junit.runner.RunWith
import java.net.URLEncoder
import com.fasterxml.jackson.databind.ObjectMapper

import play.api.http.Writeable
import play.api.libs.Files.TemporaryFile
import play.api.test.Helpers.{status => requestStatusCode_}
import play.api.test.Helpers.{contentAsString => requestContentAsString_}
import play.api.test.Helpers.{contentType => requestContentType_}

import org.scalatest.{OptionValues, WordSpec}
import org.scalatestplus.play.{OneAppPerTest, WsScalaTestClient}

import Generators._

import scala.math.BigInt
import scala.math.BigDecimal

//noinspection ScalaStyle
class Instagram_api_yamlSpec extends WordSpec with OptionValues with WsScalaTestClient with OneAppPerTest  {
    def toPath[T](value: T)(implicit binder: PathBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")
    def toQuery[T](key: String, value: T)(implicit binder: QueryStringBindable[T]): String = Option(binder.unbind(key, value)).getOrElse("")
    def toHeader[T](value: T)(implicit binder: QueryStringBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")

  def checkResult(props: Prop): org.specs2.execute.Result =
    Test.check(Test.Parameters.default, props).status match {
      case Failed(args, labels) =>
        val failureMsg = labels.mkString("\n") + " given args: " + args.map(_.arg).mkString("'", "', '","'")
        org.specs2.execute.Failure(failureMsg)
      case Proved(_) | Exhausted | Passed => org.specs2.execute.Success()
      case PropException(_, e: IllegalStateException, _) => org.specs2.execute.Error(e.getMessage)
      case PropException(_, e, labels) =>
        val error = if (labels.isEmpty) e.getLocalizedMessage else labels.mkString("\n")
        org.specs2.execute.Failure(error)
    }

  private def parserConstructor(mimeType: String) = PlayBodyParsing.jacksonMapper(mimeType)

  def parseResponseContent[T](mapper: ObjectMapper, content: String, mimeType: Option[String], expectedType: Class[T]) =
    if (expectedType.getCanonicalName == "scala.runtime.Null$") null else mapper.readValue(content, expectedType)

} 

package security.api.yaml

import de.zalando.play.controllers._
import org.scalacheck._
import org.scalacheck.Arbitrary._
import org.scalacheck.Prop._
import org.scalacheck.Test._
import org.specs2.mutable._
import org.specs2.execute._
import play.api.test.Helpers._
import play.api.test._
import play.api.mvc.MultipartFormData.FilePart
import play.api.mvc._

import org.junit.runner.RunWith
import java.net.URLEncoder
import com.fasterxml.jackson.databind.ObjectMapper

import play.api.http.Writeable
import play.api.libs.Files.TemporaryFile
import play.api.test.Helpers.{status => requestStatusCode_}
import play.api.test.Helpers.{contentAsString => requestContentAsString_}
import play.api.test.Helpers.{contentType => requestContentType_}

import org.scalatest.{OptionValues, WordSpec}
import org.scalatestplus.play.{OneAppPerTest, WsScalaTestClient}

import Generators._

import de.zalando.play.controllers.ArrayWrapper

//noinspection ScalaStyle
class Security_api_yamlSpec extends WordSpec with OptionValues with WsScalaTestClient with OneAppPerTest  {
    def toPath[T](value: T)(implicit binder: PathBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")
    def toQuery[T](key: String, value: T)(implicit binder: QueryStringBindable[T]): String = Option(binder.unbind(key, value)).getOrElse("")
    def toHeader[T](value: T)(implicit binder: QueryStringBindable[T]): String = Option(binder.unbind("", value)).getOrElse("")

  def checkResult(props: Prop): org.specs2.execute.Result =
    Test.check(Test.Parameters.default, props).status match {
      case Failed(args, labels) =>
        val failureMsg = labels.mkString("\n") + " given args: " + args.map(_.arg).mkString("'", "', '","'")
        org.specs2.execute.Failure(failureMsg)
      case Proved(_) | Exhausted | Passed => org.specs2.execute.Success()
      case PropException(_, e: IllegalStateException, _) => org.specs2.execute.Error(e.getMessage)
      case PropException(_, e, labels) =>
        val error = if (labels.isEmpty) e.getLocalizedMessage else labels.mkString("\n")
        org.specs2.execute.Failure(error)
    }

  private def parserConstructor(mimeType: String) = PlayBodyParsing.jacksonMapper(mimeType)

  def parseResponseContent[T](mapper: ObjectMapper, content: String, mimeType: Option[String], expectedType: Class[T]) =
    if (expectedType.getCanonicalName == "scala.runtime.Null$") null else mapper.readValue(content, expectedType)

} 

package works.weave.socks.aws.orders

import com.fasterxml.jackson.annotation.JsonAutoDetect
import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.annotation.PropertyAccessor
import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.SerializationFeature
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
import com.fasterxml.jackson.module.scala.DefaultScalaModule

object ProjectDefaultJacksonMapper {
  def build() : ObjectMapper = {
    val mapper = new ObjectMapper()

    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
    mapper.setSerializationInclusion(JsonInclude.Include.ALWAYS)
    mapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY)
    mapper.enable(SerializationFeature.INDENT_OUTPUT)
    mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)

    val javaTime : JavaTimeModule = new JavaTimeModule
    mapper.registerModule(javaTime)

    val scalaModule = new DefaultScalaModule()
    mapper.registerModule(scalaModule)

    mapper
  }
} 

package io.projectglow

import java.util.ServiceLoader

import scala.collection.JavaConverters._

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.apache.spark.sql.{DataFrame, SQLUtils, SparkSession}

import io.projectglow.common.Named
import io.projectglow.sql.{GlowSQLExtensions, SqlExtensionProvider}
import io.projectglow.transformers.util.{SnakeCaseMap, StringUtils}


  def transform(operationName: String, df: DataFrame, options: Map[String, Any]): DataFrame = {
    val stringValuedMap = options.mapValues {
      case s: String => s
      case v => mapper.writeValueAsString(v)
    }.map(identity) // output of mapValues is not serializable: https://github.com/scala/bug/issues/7005
    lookupTransformer(operationName) match {
      case Some(transformer) => transformer.transform(df, new SnakeCaseMap(stringValuedMap))
      case None =>
        throw new IllegalArgumentException(s"No transformer with name $operationName")
    }
  }

  def transform(operationName: String, df: DataFrame, options: (String, Any)*): DataFrame = {
    transform(operationName, df, options.toMap)
  }

  def transform(
      operationName: String,
      df: DataFrame,
      options: java.util.Map[String, String]): DataFrame = {
    transform(operationName, df, options.asScala.toMap)
  }

  private def lookupTransformer(name: String): Option[DataFrameTransformer] = synchronized {
    transformerLoader.reload()
    transformerLoader
      .iterator()
      .asScala
      .find(n => StringUtils.toSnakeCase(n.name) == StringUtils.toSnakeCase(name))
  }

  private val transformerLoader = ServiceLoader
    .load(classOf[DataFrameTransformer])
}

object Glow extends GlowBase

trait DataFrameTransformer extends Named {
  def transform(df: DataFrame, options: Map[String, String]): DataFrame
} 

package io.circe.jackson

import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.{ DeserializationContext, JsonNode, ObjectMapper, ObjectWriter }
import com.fasterxml.jackson.databind.node.ObjectNode

private[jackson] trait JacksonCompat {
  protected def makeWriter(mapper: ObjectMapper): ObjectWriter = mapper.writerWithDefaultPrettyPrinter[ObjectWriter]()

  protected def handleUnexpectedToken(context: DeserializationContext)(
    klass: Class[_],
    parser: JsonParser
  ): Unit =
    throw context.mappingException(klass)

  protected def objectNodeSetAll(node: ObjectNode, fields: java.util.Map[String, JsonNode]): JsonNode =
    node.setAll(fields)
} 

package io.circe.jackson

import cats.data.Validated
import com.fasterxml.jackson.core.JsonToken
import com.fasterxml.jackson.databind.{ ObjectMapper, ObjectReader }
import io.circe.Json
import io.circe.testing.ParserTests
import java.io.{ ByteArrayInputStream, File }

import scala.io.Source

class JacksonParserSuite extends CirceSuite with JacksonInstances {
  checkAll("Parser", ParserTests(`package`).fromString(arbitraryCleanedJson, shrinkJson))
  checkAll(
    "Parser",
    ParserTests(`package`).fromFunction[Array[Byte]]("fromByteArray")(
      s => s.getBytes("UTF-8"),
      p => p.parseByteArray _,
      p => p.decodeByteArray[Json] _,
      p => p.decodeByteArrayAccumulating[Json] _
    )(arbitraryCleanedJson, shrinkJson)
  )

  "parse and decode(Accumulating)" should "fail on invalid input" in forAll { (s: String) =>
    assert(parse(s"Not JSON $s").isLeft)
    assert(decode[Json](s"Not JSON $s").isLeft)
    assert(decodeAccumulating[Json](s"Not JSON $s").isInvalid)
  }

  "parseFile and decodeFile(Accumulating)" should "parse a JSON file" in {
    val url = getClass.getResource("/io/circe/jackson/examples/glossary.json")
    val file = new File(url.toURI)

    assert(decodeFile[Json](file) === Right(glossary))
    assert(decodeFileAccumulating[Json](file) == Validated.valid(glossary))
    assert(parseFile(file) === Right(glossary))
  }

  "parseByteArray and decodeByteArray(Accumulating)" should "parse an array of elementAsBytes" in {
    val bytes = glossaryAsBytes

    assert(decodeByteArray[Json](bytes) === Right(glossary))
    assert(decodeByteArrayAccumulating[Json](bytes) === Validated.valid(glossary))
    assert(parseByteArray(bytes) === Right(glossary))
  }

  for (elementCount <- 1 to 4) {
    "CirceJsonDeserializer" should s"be useable with Jackson's MappingIterator " +
      s"with ${elementCount} elements in array" in {
      val input = new ByteArrayInputStream(createJsonArrayAsBytes(glossaryAsBytes, elementCount))
      val objectMapper = new ObjectMapper()
      objectMapper.registerModule(CirceJsonModule)
      val jsonParser = objectMapper.getFactory.createParser(input)

      assert(jsonParser.nextToken() == JsonToken.START_ARRAY)
      assert(jsonParser.nextToken() == JsonToken.START_OBJECT)

      val reader = createReader(objectMapper).forType(classOf[Json])
      val iterator = reader.readValues[Json](jsonParser)
      var counter = 0
      while (iterator.hasNext) {
        val glossaryFromIterator = iterator.next()
        assert(glossary == glossaryFromIterator)
        counter = counter + 1
      }
      assert(counter == elementCount)
    }
  }

  // workaround warnings from compiler with Jackson 2.5
  @unchecked
  private def createReader(objectMapper: ObjectMapper): ObjectReader =
    objectMapper.reader()

  private def createJsonArrayAsBytes(elementAsBytes: Array[Byte], elementCount: Int): Array[Byte] = {
    val byteArrayOutput = new java.io.ByteArrayOutputStream()
    byteArrayOutput.write('[')
    for (i <- 1 to elementCount) {
      if (i != 1) {
        byteArrayOutput.write(',')
      }
      byteArrayOutput.write(elementAsBytes)
    }
    byteArrayOutput.write(']')
    byteArrayOutput.toByteArray
  }

  private def glossaryAsBytes = {
    val stream = getClass.getResourceAsStream("/io/circe/jackson/examples/glossary.json")
    val source = Source.fromInputStream(stream)
    val bytes = source.map(_.toByte).toArray
    source.close()
    bytes
  }
} 

package io.circe.jackson

import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.{ DeserializationContext, JsonNode, ObjectMapper, ObjectWriter }
import com.fasterxml.jackson.databind.node.ObjectNode

private[jackson] trait JacksonCompat {
  protected def makeWriter(mapper: ObjectMapper): ObjectWriter = mapper.writerWithDefaultPrettyPrinter()

  protected def handleUnexpectedToken(context: DeserializationContext)(
    klass: Class[_],
    parser: JsonParser
  ): Unit =
    context.handleUnexpectedToken(klass, parser)

  protected def objectNodeSetAll(node: ObjectNode, fields: java.util.Map[String, JsonNode]): JsonNode =
    node.setAll(fields)
} 

package io.circe.jackson

import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.{ DeserializationContext, JsonNode, ObjectMapper, ObjectWriter }
import com.fasterxml.jackson.databind.node.ObjectNode

private[jackson] trait JacksonCompat {
  protected def makeWriter(mapper: ObjectMapper): ObjectWriter = mapper.writerWithDefaultPrettyPrinter()

  protected def handleUnexpectedToken(context: DeserializationContext)(
    klass: Class[_],
    parser: JsonParser
  ): Unit =
    throw context.mappingException(klass)

  protected def objectNodeSetAll(node: ObjectNode, fields: java.util.Map[String, JsonNode]): JsonNode =
    node.setAll(fields)
} 

package io.circe.jackson

import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.databind.{ DeserializationContext, JsonNode, ObjectMapper, ObjectWriter }
import com.fasterxml.jackson.databind.node.ObjectNode

private[jackson] trait JacksonCompat {
  protected def makeWriter(mapper: ObjectMapper): ObjectWriter = mapper.writerWithDefaultPrettyPrinter()

  protected def handleUnexpectedToken(context: DeserializationContext)(
    klass: Class[_],
    parser: JsonParser
  ): Unit =
    context.handleUnexpectedToken(klass, parser)

  protected def objectNodeSetAll(node: ObjectNode, fields: java.util.Map[String, JsonNode]): JsonNode =
    node.setAll[JsonNode](fields)
} 

package com.spotify.scio.util

import java.net.URI
import java.util.UUID

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.spotify.scio.ScioContext
import org.apache.beam.sdk.extensions.gcp.options.GcpOptions
import org.apache.beam.sdk.extensions.gcp.util.Transport
import org.apache.beam.sdk.{PipelineResult, PipelineRunner}
import org.slf4j.LoggerFactory

import scala.reflect.ClassTag
import scala.util.{Failure, Success, Try}

private[scio] object ScioUtil {
  @transient private lazy val log = LoggerFactory.getLogger(this.getClass)
  @transient lazy val jsonFactory = Transport.getJsonFactory

  def isLocalUri(uri: URI): Boolean =
    uri.getScheme == null || uri.getScheme == "file"

  def isRemoteUri(uri: URI): Boolean = !isLocalUri(uri)

  def isLocalRunner(runner: Class[_ <: PipelineRunner[_ <: PipelineResult]]): Boolean = {
    require(runner != null, "Pipeline runner not set!")
    // FIXME: cover Flink, Spark, etc. in local mode
    runner.getName == "org.apache.beam.runners.direct.DirectRunner"
  }

  def isRemoteRunner(runner: Class[_ <: PipelineRunner[_ <: PipelineResult]]): Boolean =
    !isLocalRunner(runner)

  def classOf[T: ClassTag]: Class[T] =
    implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]]

  def getScalaJsonMapper: ObjectMapper =
    new ObjectMapper().registerModule(DefaultScalaModule)

  def addPartSuffix(path: String, ext: String = ""): String =
    if (path.endsWith("/")) s"${path}part-*$ext" else s"$path/part-*$ext"

  def getTempFile(context: ScioContext, fileOrPath: String = null): String = {
    val fop = Option(fileOrPath).getOrElse("scio-materialize-" + UUID.randomUUID().toString)
    val uri = URI.create(fop)
    if ((ScioUtil.isLocalUri(uri) && uri.toString.startsWith("/")) || uri.isAbsolute) {
      fop
    } else {
      val filename = fop
      val tmpDir = if (context.options.getTempLocation != null) {
        context.options.getTempLocation
      } else {
        val m =
          "Specify a temporary location via --tempLocation or PipelineOptions.setTempLocation."
        Try(context.optionsAs[GcpOptions].getGcpTempLocation) match {
          case Success(l) =>
            log.warn(
              "Using GCP temporary location as a temporary location to materialize data. " + m
            )
            l
          case Failure(_) =>
            throw new IllegalArgumentException("No temporary location was specified. " + m)
        }
      }
      tmpDir + (if (tmpDir.endsWith("/")) "" else "/") + filename
    }
  }

  def pathWithShards(path: String): String =
    path.replaceAll("\\/+$", "") + "/part"
} 

package com.spotify.scio

import java.net.URI

import com.spotify.scio.testing.util.ItUtils
import com.spotify.scio.util.ScioUtil
import org.apache.beam.runners.core.construction.{PipelineTranslation, SdkComponents}
import org.apache.beam.runners.dataflow.{DataflowPipelineTranslator, DataflowRunner}
import org.apache.beam.sdk.extensions.gcp.options.GcpOptions
import org.apache.beam.sdk.io.FileSystems
import org.apache.beam.sdk.options.{PipelineOptions, PipelineOptionsFactory}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions
import scala.jdk.CollectionConverters._
import com.spotify.scio.runners.dataflow.DataflowContext
import org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions
import org.apache.beam.runners.dataflow.util.PackageUtil.StagedFile
import org.apache.beam.model.pipeline.v1.RunnerApi
import org.apache.beam.runners.core.construction.Environments
import org.apache.beam.sdk.options.PortablePipelineOptions

class ScioContextIT extends AnyFlatSpec with Matchers {
  "ScioContext" should "have temp location for DataflowRunner" in {
    val opts = PipelineOptionsFactory.create()
    opts.setRunner(classOf[DataflowRunner])
    opts.as(classOf[GcpOptions]).setProject(ItUtils.project)
    verify(opts)
  }

  it should "support user defined temp location for DataflowRunner" in {
    val opts = PipelineOptionsFactory.create()
    opts.setRunner(classOf[DataflowRunner])
    opts.as(classOf[GcpOptions]).setProject(ItUtils.project)
    opts.setTempLocation(ItUtils.gcpTempLocation("scio-context-it"))
    verify(opts)
  }

  private def verify(options: PipelineOptions): Unit = {
    val sc = ScioContext(options)
    val gcpTempLocation = sc.optionsAs[GcpOptions].getGcpTempLocation
    val tempLocation = sc.options.getTempLocation

    tempLocation should not be null
    gcpTempLocation should not be null
    tempLocation shouldBe gcpTempLocation
    ScioUtil.isRemoteUri(new URI(gcpTempLocation)) shouldBe true
    ()
  }

  it should "register remote file systems in the test context" in {
    val sc = ScioContext.forTest()
    noException shouldBe thrownBy {
      FileSystems.matchSingleFileSpec("gs://data-integration-test-eu/shakespeare.json")
    }
    sc.run()
  }

  it should "#1734: generate a reasonably sized job graph" in {
    val opts = PipelineOptionsFactory.create()
    opts.setRunner(classOf[DataflowRunner])
    opts.as(classOf[GcpOptions]).setProject(ItUtils.project)
    opts.as(classOf[DataflowPipelineOptions]).setRegion(ItUtils.Region)
    val sc = ScioContext(opts)
    sc.parallelize(1 to 100)
    val runner = DataflowRunner.fromOptions(sc.options)
    val sdkComponents = SdkComponents.create();
    sdkComponents.registerEnvironment(
      Environments
        .createOrGetDefaultEnvironment(opts.as(classOf[PortablePipelineOptions]))
        .toBuilder()
        .addAllCapabilities(Environments.getJavaCapabilities())
        .build()
    )
    val pipelineProto = PipelineTranslation.toProto(sc.pipeline, sdkComponents, true)
    val jobSpecification =
      runner.getTranslator.translate(
        sc.pipeline,
        pipelineProto,
        sdkComponents,
        runner,
        Nil.asJava
      )
    val newJob = jobSpecification.getJob()
    val graph = DataflowPipelineTranslator.jobToString(newJob)

    import com.fasterxml.jackson.databind.ObjectMapper
    val objectMapper = new ObjectMapper()
    val rootNode = objectMapper.readTree(graph)
    val path = "/steps/0/properties/output_info/0/encoding/component_encodings/0/@type"
    val coder = rootNode.at(path).asText
    coder should not(equal("org.apache.beam.sdk.coders.CustomCoder"))
  }
} 

package org.apache.amaterasu.leader.utilities

import java.io.{File, FileInputStream}
import java.nio.file.{Files, Paths}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.apache.amaterasu.common.configuration.ClusterConfig
import org.apache.amaterasu.common.dataobjects.{ActionData, ExecData, TaskData}
import org.apache.amaterasu.common.execution.dependencies.{Dependencies, PythonDependencies}
import org.apache.amaterasu.common.logging.Logging
import org.apache.amaterasu.common.runtime.Environment
import org.apache.mesos.protobuf.ByteString
import org.yaml.snakeyaml.Yaml

import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.io.Source


object DataLoader extends Logging {

  val mapper = new ObjectMapper()
  mapper.registerModule(DefaultScalaModule)

  val ymlMapper = new ObjectMapper(new YAMLFactory())
  ymlMapper.registerModule(DefaultScalaModule)

  def getTaskData(actionData: ActionData, env: String): ByteString = {

    val srcFile = actionData.src
    val src = Source.fromFile(s"repo/src/$srcFile").mkString
    val envValue = Source.fromFile(s"repo/env/$env/job.yml").mkString

    val envData = ymlMapper.readValue(envValue, classOf[Environment])

    val data = mapper.writeValueAsBytes(TaskData(src, envData, actionData.groupId, actionData.typeId, actionData.exports))
    ByteString.copyFrom(data)

  }

  def getExecutorData(env: String, clusterConf: ClusterConfig): ByteString = {

    // loading the job configuration
    val envValue = Source.fromFile(s"repo/env/$env/job.yml").mkString //TODO: change this to YAML
    val envData = ymlMapper.readValue(envValue, classOf[Environment])
    // loading all additional configurations
    val files = new File(s"repo/env/$env/").listFiles().filter(_.isFile).filter(_.getName != "job.yml")
    val config = files.map(yamlToMap).toMap
    // loading the job's dependencies
    var depsData: Dependencies = null
    var pyDepsData: PythonDependencies = null
    if (Files.exists(Paths.get("repo/deps/jars.yml"))) {
      val depsValue = Source.fromFile(s"repo/deps/jars.yml").mkString
      depsData = ymlMapper.readValue(depsValue, classOf[Dependencies])
    }
    if (Files.exists(Paths.get("repo/deps/python.yml"))) {
      val pyDepsValue = Source.fromFile(s"repo/deps/python.yml").mkString
      pyDepsData = ymlMapper.readValue(pyDepsValue, classOf[PythonDependencies])
    }
    val data = mapper.writeValueAsBytes(ExecData(envData, depsData, pyDepsData, config))
    ByteString.copyFrom(data)
  }

  def yamlToMap(file: File): (String, Map[String, Any]) = {

    val yaml = new Yaml()
    val conf = yaml.load(new FileInputStream(file)).asInstanceOf[java.util.Map[String, Any]].asScala.toMap

    (file.getName.replace(".yml",""), conf)
  }

}

class ConfMap[String,  T <: ConfMap[String, T]] extends mutable.ListMap[String, Either[String, T]] 

package com.datamountaineer.streamreactor.connect.converters.source

import com.datamountaineer.streamreactor.connect.schemas.ConverterUtil
import com.fasterxml.jackson.databind.ObjectMapper
import com.landoop.json.sql.JacksonJson
import org.apache.kafka.connect.data.Schema
import org.apache.kafka.connect.sink.SinkRecord
import org.json4s.jackson.JsonMethods._

import scala.util.Try


object SinkRecordToJson extends ConverterUtil {

  private val mapper = new ObjectMapper()

  def apply(record: SinkRecord,
            fields: Map[String, Map[String, String]],
            ignoreFields: Map[String, Set[String]]): String = {

    val schema = record.valueSchema()
    val value = record.value()

    if (schema == null) {
      if(value == null){
        throw new IllegalArgumentException(s"The sink record value is null.(topic=${record.topic()} partition=${record.kafkaPartition()} offset=${record.kafkaOffset()})".stripMargin)
      }
      //try to take it as string
      value match {
        case map: java.util.Map[_, _] =>
          val extracted = convertSchemalessJson(record,
            fields.getOrElse(record.topic(), Map.empty),
            ignoreFields.getOrElse(record.topic(), Set.empty))
            .asInstanceOf[java.util.Map[String, Any]]
          //not ideal; but the implementation is hashmap anyway
          mapper.writeValueAsString(extracted)

        case other => sys.error(
          s"""
             |For schemaless record only String and Map types are supported. Class =${Option(other).map(_.getClass.getCanonicalName).getOrElse("unknown(null value)}")}
             |Record info:
             |topic=${record.topic()} partition=${record.kafkaPartition()} offset=${record.kafkaOffset()}
             |${Try(JacksonJson.toJson(value)).getOrElse("")}""".stripMargin)
      }
    } else {
      schema.`type`() match {
        case Schema.Type.STRING =>
          val extracted = convertStringSchemaAndJson(record,
            fields.getOrElse(record.topic(), Map.empty),
            ignoreFields.getOrElse(record.topic(), Set.empty))
          compact(render(extracted))
        case Schema.Type.STRUCT =>
          val extracted = convert(record,
            fields.getOrElse(record.topic(), Map.empty),
            ignoreFields.getOrElse(record.topic(), Set.empty))

          simpleJsonConverter.fromConnectData(extracted.valueSchema(), extracted.value()).toString

        case other => sys.error(s"$other schema is not supported")
      }
    }
  }
} 

package coinyser

import java.sql.Timestamp
import java.text.SimpleDateFormat
import java.util.TimeZone

import cats.effect.IO
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.pusher.client.Client
import com.pusher.client.channel.SubscriptionEventListener
import com.typesafe.scalalogging.StrictLogging

object StreamingProducer extends StrictLogging {

  def subscribe(pusher: Client)(onTradeReceived: String => Unit): IO[Unit] =
    for {
      _ <- IO(pusher.connect())
      channel <- IO(pusher.subscribe("live_trades"))

      _ <- IO(channel.bind("trade", new SubscriptionEventListener() {
        override def onEvent(channel: String, event: String, data: String): Unit = {
          logger.info(s"Received event: $event with data: $data")
          onTradeReceived(data)
        }
      }))
    } yield ()


  val mapper: ObjectMapper = {
    val m = new ObjectMapper()
    m.registerModule(DefaultScalaModule)
    val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
    // Very important: the storage must be in UTC
    sdf.setTimeZone(TimeZone.getTimeZone("UTC"))
    m.setDateFormat(sdf)
  }

  def deserializeWebsocketTransaction(s: String): WebsocketTransaction =
    mapper.readValue(s, classOf[WebsocketTransaction])

  def convertWsTransaction(wsTx: WebsocketTransaction): Transaction =
    Transaction(
      timestamp = new Timestamp(wsTx.timestamp.toLong * 1000), tid = wsTx.id,
      price = wsTx.price, sell = wsTx.`type` == 1, amount = wsTx.amount)

  def serializeTransaction(tx: Transaction): String =
    mapper.writeValueAsString(tx)

} 

package uk.gov.hmrc.play.logging

import java.net.InetAddress
import java.nio.charset.StandardCharsets

import ch.qos.logback.classic.spi.{ILoggingEvent, ThrowableProxyUtil}
import ch.qos.logback.core.encoder.EncoderBase
import com.fasterxml.jackson.core.JsonGenerator.Feature
import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.commons.io.IOUtils._
import org.apache.commons.lang3.time.FastDateFormat
import com.typesafe.config.ConfigFactory

import scala.util.{Success, Try}
import scala.collection.JavaConverters._

class JsonEncoder extends EncoderBase[ILoggingEvent] {

  private val mapper = new ObjectMapper().configure(Feature.ESCAPE_NON_ASCII, true)

  lazy val appName: String = Try { ConfigFactory.load().getString("appName") } match {
    case Success(name) => name.toString
    case _             => "APP NAME NOT SET"
  }

  private val DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSSZZ"

  private lazy val dateFormat = {
    val dformat = Try { ConfigFactory.load().getString("logger.json.dateformat") } match {
      case Success(date) => date.toString
      case _             => DATE_FORMAT
    }
    FastDateFormat.getInstance(dformat)
  }

  override def encode(event: ILoggingEvent): Array[Byte] = {
    val eventNode = mapper.createObjectNode

    eventNode.put("app", appName)
    eventNode.put("hostname", InetAddress.getLocalHost.getHostName)
    eventNode.put("timestamp", dateFormat.format(event.getTimeStamp))
    eventNode.put("message", event.getFormattedMessage)

    Option(event.getThrowableProxy).map(p => eventNode.put("exception", ThrowableProxyUtil.asString(p)))

    eventNode.put("logger", event.getLoggerName)
    eventNode.put("thread", event.getThreadName)
    eventNode.put("level", event.getLevel.toString)

    Option(getContext).foreach(c =>
      c.getCopyOfPropertyMap.asScala foreach { case (k, v) => eventNode.put(k.toLowerCase, v) })
    event.getMDCPropertyMap.asScala foreach { case (k, v) => eventNode.put(k.toLowerCase, v) }

    s"${mapper.writeValueAsString(eventNode)}$LINE_SEPARATOR".getBytes(StandardCharsets.UTF_8)
  }

  override def footerBytes(): Array[Byte] =
    LINE_SEPARATOR.getBytes(StandardCharsets.UTF_8)

  override def headerBytes(): Array[Byte] =
    LINE_SEPARATOR.getBytes(StandardCharsets.UTF_8)

}