org.json4s.jackson.JsonMethods Scala Example

Source File: TSet.scala From hail with MIT License

5 votes

package is.hail.types.virtual

import is.hail.annotations.{Annotation, ExtendedOrdering}
import is.hail.check.Gen
import is.hail.types.physical.PSet
import is.hail.utils._
import org.json4s.jackson.JsonMethods

import scala.reflect.{ClassTag, classTag}

final case class TSet(elementType: Type) extends TContainer {
  override lazy val fundamentalType: TArray = TArray(elementType.fundamentalType)

  def _toPretty = s"Set[$elementType]"

  override def pyString(sb: StringBuilder): Unit = {
    sb.append("set<")
    elementType.pyString(sb)
    sb.append('>')
  }

  override def canCompare(other: Type): Boolean = other match {
    case TSet(otherType) => elementType.canCompare(otherType)
    case _ => false
  }

  override def unify(concrete: Type): Boolean = concrete match {
    case TSet(celementType) => elementType.unify(celementType)
    case _ => false
  }

  override def subst() = TSet(elementType.subst())

  def _typeCheck(a: Any): Boolean =
    a.isInstanceOf[Set[_]] && a.asInstanceOf[Set[_]].forall(elementType.typeCheck)

  override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false) {
    sb.append("Set[")
    elementType.pretty(sb, indent, compact)
    sb.append("]")
  }

  override lazy val ordering: ExtendedOrdering = mkOrdering()

  override def mkOrdering(missingEqual: Boolean): ExtendedOrdering =
    ExtendedOrdering.setOrdering(elementType.ordering, missingEqual)

  override def _showStr(a: Annotation): String =
    a.asInstanceOf[Set[Annotation]]
      .map { case elt => elementType.showStr(elt) }
      .mkString("{", ",", "}")


  override def str(a: Annotation): String = JsonMethods.compact(toJSON(a))

  override def genNonmissingValue: Gen[Annotation] = Gen.buildableOf[Set](elementType.genValue)

  override def scalaClassTag: ClassTag[Set[AnyRef]] = classTag[Set[AnyRef]]

  override def valueSubsetter(subtype: Type): Any => Any = {
    assert(elementType == subtype.asInstanceOf[TSet].elementType)
    identity
  }
}

Source File: Parser.scala From Scala-Design-Patterns-Second-Edition with MIT License

5 votes

package com.ivan.nikolov.behavioral.strategy

import java.io.InputStreamReader

import com.github.tototoshi.csv.CSVReader
import com.ivan.nikolov.behavioral.strategy.model.Person

import org.json4s._
import org.json4s.jackson.JsonMethods

trait Parser[T] {
  def parse(file: String): List[T]
}

class CSVParser extends Parser[Person] {
  override def parse(file: String): List[Person] =
    CSVReader.open(new InputStreamReader(this.getClass.getResourceAsStream(file))).all().map {
      case List(name, age, address) =>
        Person(name, age.toInt, address)
    }
}

class JsonParser extends Parser[Person] {
  implicit val formats = DefaultFormats
  
  override def parse(file: String): List[Person] =
    JsonMethods.parse(StreamInput(this.getClass.getResourceAsStream(file))).extract[List[Person]]
}

object Parser {
  def apply(filename: String): Parser[Person] =
    filename match {
      case f if f.endsWith(".json") => new JsonParser
      case f if f.endsWith(".csv") => new CSVParser
      case f => throw new RuntimeException(s"Unknown format: $f")
    }
}

class PersonApplication[T](parser: Parser[T]) {
  
  def write(file: String): Unit = {
    System.out.println(s"Got the following data ${parser.parse(file)}")
  }
}

object ParserExample {
  def main(args: Array[String]): Unit = {
    val csvPeople = Parser("people.csv")
    val jsonPeople = Parser("people.json")
    
    val applicationCsv = new PersonApplication(csvPeople)
    val applicationJson = new PersonApplication(jsonPeople)
    
    System.out.println("Using the csv: ")
    applicationCsv.write("people.csv")
    
    System.out.println("Using the json: ")
    applicationJson.write("people.json")
  }
}

Source File: ParsingStrategy.scala From Scala-Design-Patterns-Second-Edition with MIT License

5 votes

package com.ivan.nikolov.behavioral.strategy

import java.io.InputStreamReader

import com.github.tototoshi.csv.CSVReader
import com.ivan.nikolov.behavioral.strategy.model.Person
import org.json4s.{StreamInput, DefaultFormats}
import org.json4s.jackson.JsonMethods

class Application[T](strategy: (String) => List[T]) {
  def write(file: String): Unit = {
    System.out.println(s"Got the following data ${strategy(file)}")
  }
}

object StrategyFactory {
  implicit val formats = DefaultFormats
  
  def apply(filename: String): (String) => List[Person] =
    filename match {
      case f if f.endsWith(".json") => parseJson
      case f if f.endsWith(".csv") => parseCsv
      case f => throw new RuntimeException(s"Unknown format: $f")
    }
  
  def parseJson(file: String): List[Person] =
    JsonMethods.parse(StreamInput(this.getClass.getResourceAsStream(file))).extract[List[Person]]
  
  def parseCsv(file: String): List[Person] =
    CSVReader.open(new InputStreamReader(this.getClass.getResourceAsStream(file))).all().map {
      case List(name, age, address) =>
        Person(name, age.toInt, address)
    }
}

object StrategyExample {
  def main(args: Array[String]): Unit = {
    val applicationCsv = new Application[Person](StrategyFactory("people.csv"))
    val applicationJson = new Application[Person](StrategyFactory("people.json"))

    System.out.println("Using the csv: ")
    applicationCsv.write("people.csv")

    System.out.println("Using the json: ")
    applicationJson.write("people.json")
  }
}

Source File: DataFinder.scala From Scala-Design-Patterns-Second-Edition with MIT License

5 votes

package com.ivan.nikolov.behavioral.template

import java.io.{InputStreamReader, ByteArrayInputStream}

import com.github.tototoshi.csv.CSVReader
import com.ivan.nikolov.behavioral.template.model.Person
import org.json4s.{StringInput, DefaultFormats}
import org.json4s.jackson.JsonMethods

abstract class DataFinder[T, Y] {

  def find(f: T => Option[Y]): Option[Y] =
    try {
      val data = readData()
      val parsed = parse(data)
      f(parsed)
    } finally {
      cleanup()
    }

  def readData(): Array[Byte]

  def parse(data: Array[Byte]): T

  def cleanup()
}

class JsonDataFinder extends DataFinder[List[Person], Person] {
  implicit val formats = DefaultFormats

  override def readData(): Array[Byte] = {
    val stream = this.getClass.getResourceAsStream("people.json")
    Stream.continually(stream.read).takeWhile(_ != -1).map(_.toByte).toArray
  }

  override def cleanup(): Unit = {
    System.out.println("Reading json: nothing to do.")
  }

  override def parse(data: Array[Byte]): List[Person] =
    JsonMethods.parse(StringInput(new String(data, "UTF-8"))).extract[List[Person]]
}

class CSVDataFinder extends DataFinder[List[Person], Person] {
  override def readData(): Array[Byte] = {
    val stream = this.getClass.getResourceAsStream("people.csv")
    Stream.continually(stream.read).takeWhile(_ != -1).map(_.toByte).toArray
  }

  override def cleanup(): Unit = {
    System.out.println("Reading csv: nothing to do.")
  }

  override def parse(data: Array[Byte]): List[Person] =
    CSVReader.open(new InputStreamReader(new ByteArrayInputStream(data))).all().map {
      case List(name, age, address) =>
        Person(name, age.toInt, address)
    }
}


object DataFinderExample {
  def main(args: Array[String]): Unit = {
    val jsonDataFinder: DataFinder[List[Person], Person] = new JsonDataFinder
    val csvDataFinder: DataFinder[List[Person], Person] = new CSVDataFinder

    System.out.println(s"Find a person with name Ivan in the json: ${jsonDataFinder.find(_.find(_.name == "Ivan"))}")
    System.out.println(s"Find a person with name James in the json: ${jsonDataFinder.find(_.find(_.name == "James"))}")

    System.out.println(s"Find a person with name Maria in the csv: ${csvDataFinder.find(_.find(_.name == "Maria"))}")
    System.out.println(s"Find a person with name Alice in the csv: ${csvDataFinder.find(_.find(_.name == "Alice"))}")
  }
}

Source File: Parser.scala From Scala-Design-Patterns-Second-Edition with MIT License

5 votes

package com.ivan.nikolov.behavioral.strategy

import java.io.InputStreamReader

import com.github.tototoshi.csv.CSVReader
import com.ivan.nikolov.behavioral.strategy.model.Person

import org.json4s._
import org.json4s.jackson.JsonMethods

trait Parser[T] {
  def parse(file: String): List[T]
}

class CSVParser extends Parser[Person] {
  override def parse(file: String): List[Person] =
    CSVReader.open(new InputStreamReader(this.getClass.getResourceAsStream(file))).all().map {
      case List(name, age, address) =>
        Person(name, age.toInt, address)
    }
}

class JsonParser extends Parser[Person] {
  implicit val formats = DefaultFormats
  
  override def parse(file: String): List[Person] =
    JsonMethods.parse(StreamInput(this.getClass.getResourceAsStream(file))).extract[List[Person]]
}

object Parser {
  def apply(filename: String): Parser[Person] =
    filename match {
      case f if f.endsWith(".json") => new JsonParser
      case f if f.endsWith(".csv") => new CSVParser
      case f => throw new RuntimeException(s"Unknown format: $f")
    }
}

class PersonApplication[T](parser: Parser[T]) {
  
  def write(file: String): Unit = {
    System.out.println(s"Got the following data ${parser.parse(file)}")
  }
}

object ParserExample {
  def main(args: Array[String]): Unit = {
    val csvPeople = Parser("people.csv")
    val jsonPeople = Parser("people.json")
    
    val applicationCsv = new PersonApplication(csvPeople)
    val applicationJson = new PersonApplication(jsonPeople)
    
    System.out.println("Using the csv: ")
    applicationCsv.write("people.csv")
    
    System.out.println("Using the json: ")
    applicationJson.write("people.json")
  }
}

Source File: ParsingStrategy.scala From Scala-Design-Patterns-Second-Edition with MIT License

5 votes

package com.ivan.nikolov.behavioral.strategy

import java.io.InputStreamReader

import com.github.tototoshi.csv.CSVReader
import com.ivan.nikolov.behavioral.strategy.model.Person
import org.json4s.{StreamInput, DefaultFormats}
import org.json4s.jackson.JsonMethods

class Application[T](strategy: (String) => List[T]) {
  def write(file: String): Unit = {
    System.out.println(s"Got the following data ${strategy(file)}")
  }
}

object StrategyFactory {
  implicit val formats = DefaultFormats
  
  def apply(filename: String): (String) => List[Person] =
    filename match {
      case f if f.endsWith(".json") => parseJson
      case f if f.endsWith(".csv") => parseCsv
      case f => throw new RuntimeException(s"Unknown format: $f")
    }
  
  def parseJson(file: String): List[Person] =
    JsonMethods.parse(StreamInput(this.getClass.getResourceAsStream(file))).extract[List[Person]]
  
  def parseCsv(file: String): List[Person] =
    CSVReader.open(new InputStreamReader(this.getClass.getResourceAsStream(file))).all().map {
      case List(name, age, address) =>
        Person(name, age.toInt, address)
    }
}

object StrategyExample {
  def main(args: Array[String]): Unit = {
    val applicationCsv = new Application[Person](StrategyFactory("people.csv"))
    val applicationJson = new Application[Person](StrategyFactory("people.json"))

    System.out.println("Using the csv: ")
    applicationCsv.write("people.csv")

    System.out.println("Using the json: ")
    applicationJson.write("people.json")
  }
}

Source File: CheckPoint.scala From eclair with Apache License 2.0

5 votes

package fr.acinq.eclair.blockchain.electrum

import java.io.InputStream

import fr.acinq.bitcoin.{Block, ByteVector32, encodeCompact}
import fr.acinq.eclair.blockchain.electrum.db.HeaderDb
import org.json4s.JsonAST.{JArray, JInt, JString}
import org.json4s.jackson.JsonMethods


  def load(chainHash: ByteVector32, headerDb: HeaderDb): Vector[CheckPoint] = {
    val checkpoints = CheckPoint.load(chainHash)
    val checkpoints1 = headerDb.getTip match {
      case Some((height, header)) =>
        val newcheckpoints = for {h <- checkpoints.size * RETARGETING_PERIOD - 1 + RETARGETING_PERIOD to height - RETARGETING_PERIOD by RETARGETING_PERIOD} yield {
          // we * should * have these headers in our db
          val cpheader = headerDb.getHeader(h).get
          val nextDiff = headerDb.getHeader(h + 1).get.bits
          CheckPoint(cpheader.hash, nextDiff)
        }
        checkpoints ++ newcheckpoints
      case None => checkpoints
    }
    checkpoints1
  }
}

Source File: SmileJsonMethods.scala From spark-druid-olap with Apache License 2.0

5 votes

package org.json4s.jackson.sparklinedata

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.smile.{SmileFactory, SmileGenerator}
import org.json4s.jackson.{Json4sScalaModule, JsonMethods}

object SmileJsonMethods extends JsonMethods {

  private[this] lazy val _defaultMapper = {

    val smileFactory = new SmileFactory
    smileFactory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, false)
    smileFactory.delegateToTextual(true)
    val m = new ObjectMapper(smileFactory)
    m.getFactory().setCodec(m)
    m.registerModule(SmileJson4sScalaModule)
    m
  }

  override def mapper = _defaultMapper

}

Source File: TBaseStruct.scala From hail with MIT License

5 votes

package is.hail.types.virtual

import is.hail.annotations._
import is.hail.check.Gen
import is.hail.types.physical.PBaseStruct
import is.hail.utils._
import org.apache.spark.sql.Row
import org.json4s.jackson.JsonMethods

import scala.reflect.{ClassTag, classTag}

object TBaseStruct {
  
  def getOrdering(types: Array[Type], missingEqual: Boolean = true): ExtendedOrdering =
    ExtendedOrdering.rowOrdering(types.map(_.ordering), missingEqual)

  def getJoinOrdering(types: Array[Type]): ExtendedOrdering =
    ExtendedOrdering.rowOrdering(types.map(_.mkOrdering(missingEqual = false)), _missingEqual = false)
}

abstract class TBaseStruct extends Type {
  def types: Array[Type]

  def fields: IndexedSeq[Field]

  override def children: Seq[Type] = types

  def size: Int

  def _toPretty: String = {
    val sb = new StringBuilder
    _pretty(sb, 0, compact = true)
    sb.result()
  }

  override def _typeCheck(a: Any): Boolean = a match {
    case row: Row =>
      row.length == types.length &&
        isComparableAt(a)
    case _ => false
  }

  def relaxedTypeCheck(a: Any): Boolean = a match {
    case row: Row =>
      row.length <= types.length &&
        isComparableAt(a)
    case _ => false
  }

  def isComparableAt(a: Annotation): Boolean = a match {
    case row: Row =>
      row.toSeq.zip(types).forall {
        case (v, t) => t.typeCheck(v)
      }
    case _ => false
  }

  def isIsomorphicTo(other: TBaseStruct): Boolean =
    size == other.size && isCompatibleWith(other)

  def isPrefixOf(other: TBaseStruct): Boolean =
    size <= other.size && isCompatibleWith(other)

  def isCompatibleWith(other: TBaseStruct): Boolean =
    fields.zip(other.fields).forall{ case (l, r) => l.typ == r.typ }

  def truncate(newSize: Int): TBaseStruct

  override def _showStr(a: Annotation): String = {
    if (types.isEmpty)
      "()"
    else {
      Array.tabulate(size)(i => types(i).showStr(a.asInstanceOf[Row].get(i)))
        .mkString("(", ",", ")")
    }
  }

  override def str(a: Annotation): String = JsonMethods.compact(toJSON(a))

  override def genNonmissingValue: Gen[Annotation] = {
    if (types.isEmpty) {
      Gen.const(Annotation.empty)
    } else
      Gen.size.flatMap(fuel =>
        if (types.length > fuel)
          Gen.uniformSequence(types.map(t => Gen.const(null))).map(a => Annotation(a: _*))
        else
          Gen.uniformSequence(types.map(t => t.genValue)).map(a => Annotation(a: _*)))
  }

  override def valuesSimilar(a1: Annotation, a2: Annotation, tolerance: Double, absolute: Boolean): Boolean =
    a1 == a2 || (a1 != null && a2 != null
      && types.zip(a1.asInstanceOf[Row].toSeq).zip(a2.asInstanceOf[Row].toSeq)
      .forall {
        case ((t, x1), x2) =>
          t.valuesSimilar(x1, x2, tolerance, absolute)
      })

  override def scalaClassTag: ClassTag[Row] = classTag[Row]
}

Source File: TArray.scala From hail with MIT License

5 votes

package is.hail.types.virtual

import is.hail.annotations.{Annotation, ExtendedOrdering}
import is.hail.check.Gen
import org.json4s.jackson.JsonMethods

import scala.reflect.{ClassTag, classTag}

final case class TArray(elementType: Type) extends TContainer {
  override def pyString(sb: StringBuilder): Unit = {
    sb.append("array<")
    elementType.pyString(sb)
    sb.append('>')
  }
  override val fundamentalType: TArray = {
    if (elementType == elementType.fundamentalType)
      this
    else
      this.copy(elementType = elementType.fundamentalType)
  }

  def _toPretty = s"Array[$elementType]"

  override def canCompare(other: Type): Boolean = other match {
    case TArray(otherType) => elementType.canCompare(otherType)
    case _ => false
  }

  override def unify(concrete: Type): Boolean = concrete match {
    case TArray(celementType) => elementType.unify(celementType)
    case _ => false
  }

  override def subst() = TArray(elementType.subst())

  override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false) {
    sb.append("Array[")
    elementType.pretty(sb, indent, compact)
    sb.append("]")
  }

  def _typeCheck(a: Any): Boolean = a.isInstanceOf[IndexedSeq[_]] &&
    a.asInstanceOf[IndexedSeq[_]].forall(elementType.typeCheck)

  override def _showStr(a: Annotation): String =
    a.asInstanceOf[IndexedSeq[Annotation]]
      .map(elt => elementType.showStr(elt))
      .mkString("[", ",", "]")

  override def str(a: Annotation): String = JsonMethods.compact(toJSON(a))

  override def genNonmissingValue: Gen[IndexedSeq[Annotation]] =
    Gen.buildableOf[Array](elementType.genValue).map(x => x: IndexedSeq[Annotation])

  override val ordering: ExtendedOrdering = mkOrdering()

  def mkOrdering(missingEqual: Boolean): ExtendedOrdering =
    ExtendedOrdering.iterableOrdering(elementType.ordering, missingEqual)

  override def scalaClassTag: ClassTag[IndexedSeq[AnyRef]] = classTag[IndexedSeq[AnyRef]]

  override def valueSubsetter(subtype: Type): Any => Any = {
    if (this == subtype)
      return identity

    val subsetElem = elementType.valueSubsetter(subtype.asInstanceOf[TArray].elementType)
    (a: Any) => a.asInstanceOf[IndexedSeq[Any]].map(subsetElem)
  }
}

Source File: TDict.scala From hail with MIT License

5 votes

package is.hail.types.virtual

import is.hail.annotations.{Annotation, ExtendedOrdering}
import is.hail.check.Gen
import is.hail.types.physical.PDict
import is.hail.utils._
import org.json4s.jackson.JsonMethods

import scala.reflect.{ClassTag, classTag}

final case class TDict(keyType: Type, valueType: Type) extends TContainer {
  lazy val elementType: TBaseStruct = (TStruct("key" -> keyType, "value" -> valueType)).asInstanceOf[TBaseStruct]

  override val fundamentalType: TArray = TArray(elementType.fundamentalType)

  override def canCompare(other: Type): Boolean = other match {
    case TDict(okt, ovt) => keyType.canCompare(okt) && valueType.canCompare(ovt)
    case _ => false
  }

  override def children = FastSeq(keyType, valueType)

  override def unify(concrete: Type): Boolean = {
    concrete match {
      case TDict(kt, vt) => keyType.unify(kt) && valueType.unify(vt)
      case _ => false
    }
  }

  override def subst() = TDict(keyType.subst(), valueType.subst())

  def _toPretty = s"Dict[$keyType, $valueType]"

  override def pyString(sb: StringBuilder): Unit = {
    sb.append("dict<")
    keyType.pyString(sb)
    sb.append(", ")
    valueType.pyString(sb)
    sb.append('>')
  }

  override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false) {
    sb.append("Dict[")
    keyType.pretty(sb, indent, compact)
    if (compact)
      sb += ','
    else
      sb.append(", ")
    valueType.pretty(sb, indent, compact)
    sb.append("]")
  }

  def _typeCheck(a: Any): Boolean = a == null || (a.isInstanceOf[Map[_, _]] &&
    a.asInstanceOf[Map[_, _]].forall { case (k, v) => keyType.typeCheck(k) && valueType.typeCheck(v) })

  override def _showStr(a: Annotation): String =
    a.asInstanceOf[Map[Annotation, Annotation]]
      .map { case (k, v) => s"${keyType.showStr(k)}:${valueType.showStr(v)}}" }
      .mkString("{", ",", "}")

  override def str(a: Annotation): String = JsonMethods.compact(toJSON(a))

  override def genNonmissingValue: Gen[Annotation] =
    Gen.buildableOf2[Map](Gen.zip(keyType.genValue, valueType.genValue))

  override def valuesSimilar(a1: Annotation, a2: Annotation, tolerance: Double, absolute: Boolean): Boolean =
    a1 == a2 || (a1 != null && a2 != null &&
      a1.asInstanceOf[Map[Any, _]].outerJoin(a2.asInstanceOf[Map[Any, _]])
        .forall { case (_, (o1, o2)) =>
          o1.liftedZip(o2).exists { case (v1, v2) => valueType.valuesSimilar(v1, v2, tolerance, absolute) }
        })

  override def scalaClassTag: ClassTag[Map[_, _]] = classTag[Map[_, _]]

  override lazy val ordering: ExtendedOrdering = mkOrdering()

  override def mkOrdering(missingEqual: Boolean): ExtendedOrdering =
    ExtendedOrdering.mapOrdering(elementType.ordering, missingEqual)

  override def valueSubsetter(subtype: Type): Any => Any = {
    val subdict = subtype.asInstanceOf[TDict]
    assert(keyType == subdict.keyType)
    if (valueType == subdict.valueType)
      return identity

    val subsetValue = valueType.valueSubsetter(subdict.valueType)
    (a: Any) => a.asInstanceOf[Map[Any, Any]].mapValues(subsetValue)
  }
}

Source File: TStream.scala From hail with MIT License

5 votes

package is.hail.types.virtual

import is.hail.annotations.{Annotation, ExtendedOrdering}
import is.hail.check.Gen
import org.json4s.jackson.JsonMethods

import scala.reflect.{ClassTag, classTag}

final case class TStream(elementType: Type) extends TIterable {
  override def pyString(sb: StringBuilder): Unit = {
    sb.append("stream<")
    elementType.pyString(sb)
    sb.append('>')
  }
  override val fundamentalType: TStream = {
    if (elementType == elementType.fundamentalType)
      this
    else
      this.copy(elementType = elementType.fundamentalType)
  }

  def _toPretty = s"Stream[$elementType]"

  override def canCompare(other: Type): Boolean =
    throw new UnsupportedOperationException("Stream comparison is currently undefined.")

  override def unify(concrete: Type): Boolean = concrete match {
    case TStream(celementType) => elementType.unify(celementType)
    case _ => false
  }

  override def subst() = TStream(elementType.subst())

  override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean = false) {
    sb.append("Stream[")
    elementType.pretty(sb, indent, compact)
    sb.append("]")
  }

  def _typeCheck(a: Any): Boolean = a.isInstanceOf[IndexedSeq[_]] &&
    a.asInstanceOf[IndexedSeq[_]].forall(elementType.typeCheck)

  override def str(a: Annotation): String = JsonMethods.compact(toJSON(a))

  override def isRealizable = false

  override def genNonmissingValue: Gen[Annotation] =
    throw new UnsupportedOperationException("Streams don't have associated annotations.")

  override def mkOrdering(missingEqual: Boolean): ExtendedOrdering =
    throw new UnsupportedOperationException("Stream comparison is currently undefined.")

  override def scalaClassTag: ClassTag[Iterator[AnyRef]] = classTag[Iterator[AnyRef]]
}

Source File: DataFinder.scala From Scala-Design-Patterns-Second-Edition with MIT License

5 votes

package com.ivan.nikolov.behavioral.template

import java.io.{InputStreamReader, ByteArrayInputStream}

import com.github.tototoshi.csv.CSVReader
import com.ivan.nikolov.behavioral.template.model.Person
import org.json4s.{StringInput, DefaultFormats}
import org.json4s.jackson.JsonMethods

abstract class DataFinder[T, Y] {

  def find(f: T => Option[Y]): Option[Y] =
    try {
      val data = readData()
      val parsed = parse(data)
      f(parsed)
    } finally {
      cleanup()
    }

  def readData(): Array[Byte]

  def parse(data: Array[Byte]): T

  def cleanup()
}

class JsonDataFinder extends DataFinder[List[Person], Person] {
  implicit val formats = DefaultFormats

  override def readData(): Array[Byte] = {
    val stream = this.getClass.getResourceAsStream("people.json")
    Stream.continually(stream.read).takeWhile(_ != -1).map(_.toByte).toArray
  }

  override def cleanup(): Unit = {
    System.out.println("Reading json: nothing to do.")
  }

  override def parse(data: Array[Byte]): List[Person] =
    JsonMethods.parse(StringInput(new String(data, "UTF-8"))).extract[List[Person]]
}

class CSVDataFinder extends DataFinder[List[Person], Person] {
  override def readData(): Array[Byte] = {
    val stream = this.getClass.getResourceAsStream("people.csv")
    Stream.continually(stream.read).takeWhile(_ != -1).map(_.toByte).toArray
  }

  override def cleanup(): Unit = {
    System.out.println("Reading csv: nothing to do.")
  }

  override def parse(data: Array[Byte]): List[Person] =
    CSVReader.open(new InputStreamReader(new ByteArrayInputStream(data))).all().map {
      case List(name, age, address) =>
        Person(name, age.toInt, address)
    }
}


object DataFinderExample {
  def main(args: Array[String]): Unit = {
    val jsonDataFinder: DataFinder[List[Person], Person] = new JsonDataFinder
    val csvDataFinder: DataFinder[List[Person], Person] = new CSVDataFinder

    System.out.println(s"Find a person with name Ivan in the json: ${jsonDataFinder.find(_.find(_.name == "Ivan"))}")
    System.out.println(s"Find a person with name James in the json: ${jsonDataFinder.find(_.find(_.name == "James"))}")

    System.out.println(s"Find a person with name Maria in the csv: ${csvDataFinder.find(_.find(_.name == "Maria"))}")
    System.out.println(s"Find a person with name Alice in the csv: ${csvDataFinder.find(_.find(_.name == "Alice"))}")
  }
}

Source File: DeployConfig.scala From hail with MIT License

5 votes

package is.hail.services

import java.io.{File, FileInputStream}

import is.hail.utils._
import org.json4s.{DefaultFormats, Formats, JValue}
import org.json4s.jackson.JsonMethods

object DeployConfig {
  lazy val get: DeployConfig = fromConfigFile()

  def fromConfigFile(file0: String = null): DeployConfig = {
    var file = file0

    if (file == null)
      file = System.getenv("HAIL_DEPLOY_CONFIG_FILE")

    if (file == null) {
      val fromHome = s"${ System.getenv("HOME") }/.hail/deploy-config.json"
      if (new File(fromHome).exists())
        file = fromHome
    }

    if (file == null) {
      val f = "/deploy-config/deploy-config.json"
      if (new File(f).exists())
        file = f
    }

    if (file != null) {
      using(new FileInputStream(file)) { in =>
        fromConfig(JsonMethods.parse(in))
      }
    } else
      new DeployConfig(
        "external",
        "default",
        Map())
  }

  def fromConfig(config: JValue): DeployConfig = {
    implicit val formats: Formats = DefaultFormats
    new DeployConfig(
      (config \ "location").extract[String],
      (config \ "default_namespace").extract[String],
      (config \ "service_namespace").extract[Map[String, String]])
  }
}

class DeployConfig(
  val location: String,
  val defaultNamespace: String,
  val serviceNamespace: Map[String, String]) {

  def scheme(baseScheme: String = "http"): String = {
    if (location == "external" || location == "k8s")
      baseScheme + "s"
    else
      baseScheme
  }

  def getServiceNamespace(service: String): String = {
    serviceNamespace.getOrElse(service, defaultNamespace)
  }

  def domain(service: String): String = {
    val ns = getServiceNamespace(service)
    location match {
      case "k8s" =>
        s"$service.$ns"
      case "gce" =>
        if (ns == "default")
          s"$service.hail"
        else
          "internal.hail"
      case "external" =>
        if (ns == "default")
          s"$service.hail.is"
        else
          "internal.hail.is"
    }
  }

  def basePath(service: String): String = {
    val ns = getServiceNamespace(service)
    if (ns == "default")
      ""
    else
      s"/$ns/$service"
  }

  def baseUrl(service: String, baseScheme: String = "http"): String = {
    s"${ scheme(baseScheme) }://${ domain(service) }${ basePath(service) }"
  }
}

Source File: package.scala From hail with MIT License

5 votes

package is.hail.services

import is.hail.utils._
import org.json4s.{DefaultFormats, Formats}
import java.io.{File, FileInputStream}
import java.security.KeyStore

import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManagerFactory}
import org.apache.log4j.{LogManager, Logger}
import org.json4s.jackson.JsonMethods

class NoSSLConfigFound(
  message: String,
  cause: Throwable
) extends Exception(message, cause) {
  def this() = this(null, null)

  def this(message: String) = this(message, null)
}

case class SSLConfig(
  outgoing_trust: String,
  outgoing_trust_store: String,
  incoming_trust: String,
  incoming_trust_store: String,
  key: String,
  cert: String,
  key_store: String)

package object tls {
  lazy val log: Logger = LogManager.getLogger("is.hail.tls")

  private[this] lazy val _getSSLConfig: SSLConfig = {
    var configFile = System.getenv("HAIL_SSL_CONFIG_FILE")
    if (configFile == null)
      configFile = "/ssl-config/ssl-config.json"
    if (!new File(configFile).isFile)
      throw new NoSSLConfigFound(s"no ssl config file found at $configFile")

    log.info(s"ssl config file found at $configFile")

    using(new FileInputStream(configFile)) { is =>
      implicit val formats: Formats = DefaultFormats
      JsonMethods.parse(is).extract[SSLConfig]
    }
  }

  lazy val getSSLContext: SSLContext = {
    val sslConfig = _getSSLConfig

    val pw = "dummypw".toCharArray

    val ks = KeyStore.getInstance("PKCS12")
    using(new FileInputStream(sslConfig.key_store)) { is =>
      ks.load(is, pw)
    }
    val kmf = KeyManagerFactory.getInstance("SunX509")
    kmf.init(ks, pw)

    val ts = KeyStore.getInstance("JKS")
    using(new FileInputStream(sslConfig.outgoing_trust_store)) { is =>
      ts.load(is, pw)
    }
    val tmf = TrustManagerFactory.getInstance("SunX509")
    tmf.init(ts)

    val ctx = SSLContext.getInstance("TLS")
    ctx.init(kmf.getKeyManagers, tmf.getTrustManagers, null)

    ctx
  }
}

Source File: Tokens.scala From hail with MIT License

5 votes

package is.hail.services

import is.hail.utils._
import java.io.{File, FileInputStream}

import org.apache.http.client.methods.HttpUriRequest
import org.apache.log4j.{LogManager, Logger}
import org.json4s.{DefaultFormats, Formats}
import org.json4s.jackson.JsonMethods

object Tokens {
  lazy val log: Logger = LogManager.getLogger("Tokens")

  def get: Tokens = {
    val file = getTokensFile()
    if (new File(file).isFile) {
      using(new FileInputStream(file)) { is =>
        implicit val formats: Formats = DefaultFormats
        new Tokens(JsonMethods.parse(is).extract[Map[String, String]])
      }
    } else {
      log.info(s"tokens file not found: $file")
      new Tokens(Map())
    }
  }

  def getTokensFile(): String = {
    if (DeployConfig.get.location == "external")
      s"${ System.getenv("HOME") }/.hail/tokens.json"
    else
      "/user-tokens/tokens.json"
  }
}

class Tokens(
  tokens: Map[String, String]
) {
  def namespaceToken(ns: String): String = tokens(ns)

  def addNamespaceAuthHeaders(ns: String, req: HttpUriRequest): Unit = {
    val token = namespaceToken(ns)
    req.addHeader("Authorization", s"Bearer $token")
    val location = DeployConfig.get.location
    if (location == "external" && ns != "default")
      req.addHeader("X-Hail-Internal-Authorization", s"Bearer ${ namespaceToken("default") }")
  }

  def addServiceAuthHeaders(service: String, req: HttpUriRequest): Unit = {
    addNamespaceAuthHeaders(DeployConfig.get.getServiceNamespace(service), req)
  }
}

Source File: AbstractTableSpec.scala From hail with MIT License

5 votes

package is.hail.expr.ir

import java.io.OutputStreamWriter

import is.hail.utils._
import is.hail.types._
import is.hail.io.fs.FS
import is.hail.rvd._
import org.json4s.jackson.JsonMethods
import org.json4s.{DefaultFormats, Extraction, Formats, JValue, ShortTypeHints}

import scala.language.implicitConversions

object SortOrder {
  def deserialize(b: Byte): SortOrder =
    if (b == 0.toByte) Ascending
    else if (b == 1.toByte) Descending
    else throw new RuntimeException(s"invalid sort order: $b")
}

sealed abstract class SortOrder {
  def serialize: Byte
}

case object Ascending extends SortOrder {
  def serialize: Byte = 0.toByte
}

case object Descending extends SortOrder {
  def serialize: Byte = 1.toByte
}

case class SortField(field: String, sortOrder: SortOrder)

abstract class AbstractTableSpec extends RelationalSpec {
  def references_rel_path: String

  def table_type: TableType

  def rowsComponent: RVDComponentSpec = getComponent[RVDComponentSpec]("rows")

  def rowsSpec: AbstractRVDSpec

  def globalsSpec: AbstractRVDSpec

  def indexed: Boolean = rowsSpec.indexed
}

object TableSpec {
  def apply(fs: FS, path: String, params: TableSpecParameters): TableSpec = {
    val globalsComponent = params.components("globals").asInstanceOf[RVDComponentSpec]
    val globalsSpec = globalsComponent.rvdSpec(fs, path)

    val rowsComponent = params.components("rows").asInstanceOf[RVDComponentSpec]
    val rowsSpec = rowsComponent.rvdSpec(fs, path)

    new TableSpec(params, globalsSpec, rowsSpec)
  }

  def fromJValue(fs: FS, path: String, jv: JValue): TableSpec = {
    implicit val formats: Formats = RelationalSpec.formats
    val params = jv.extract[TableSpecParameters]
    TableSpec(fs, path, params)
  }
}

case class TableSpecParameters(
  file_version: Int,
  hail_version: String,
  references_rel_path: String,
  table_type: TableType,
  components: Map[String, ComponentSpec]) {

  def write(fs: FS, path: String) {
    using(new OutputStreamWriter(fs.create(path + "/metadata.json.gz"))) { out =>
      out.write(JsonMethods.compact(decomposeWithName(this, "TableSpec")(RelationalSpec.formats)))
    }
  }
}

class TableSpec(
  val params: TableSpecParameters,
  val globalsSpec: AbstractRVDSpec,
  val rowsSpec: AbstractRVDSpec) extends AbstractTableSpec {
  def file_version: Int = params.file_version

  def hail_version: String = params.hail_version

  def components: Map[String, ComponentSpec] = params.components

  def references_rel_path: String = params.references_rel_path

  def table_type: TableType = params.table_type

  def toJValue: JValue = {
    decomposeWithName(params, "TableSpec")(RelationalSpec.formats)
  }
}

Source File: JsonTestUtils.scala From multi-tenancy-spark with Apache License 2.0

5 votes

package org.apache.spark

import org.json4s._
import org.json4s.jackson.JsonMethods

trait JsonTestUtils {
  def assertValidDataInJson(validateJson: JValue, expectedJson: JValue) {
    val Diff(c, a, d) = validateJson.diff(expectedJson)
    val validatePretty = JsonMethods.pretty(validateJson)
    val expectedPretty = JsonMethods.pretty(expectedJson)
    val errorMessage = s"Expected:\n$expectedPretty\nFound:\n$validatePretty"
    import org.scalactic.TripleEquals._
    assert(c === JNothing, s"$errorMessage\nChanged:\n${JsonMethods.pretty(c)}")
    assert(a === JNothing, s"$errorMessage\nAdded:\n${JsonMethods.pretty(a)}")
    assert(d === JNothing, s"$errorMessage\nDeleted:\n${JsonMethods.pretty(d)}")
  }

}

Source File: JsonTestUtils.scala From iolap with Apache License 2.0

5 votes

package org.apache.spark

import org.json4s._
import org.json4s.jackson.JsonMethods

trait JsonTestUtils {
  def assertValidDataInJson(validateJson: JValue, expectedJson: JValue) {
    val Diff(c, a, d) = validateJson.diff(expectedJson)
    val validatePretty = JsonMethods.pretty(validateJson)
    val expectedPretty = JsonMethods.pretty(expectedJson)
    val errorMessage = s"Expected:\n$expectedPretty\nFound:\n$validatePretty"
    import org.scalactic.TripleEquals._
    assert(c === JNothing, s"$errorMessage\nChanged:\n${JsonMethods.pretty(c)}")
    assert(a === JNothing, s"$errorMessage\nAdded:\n${JsonMethods.pretty(a)}")
    assert(d === JNothing, s"$errorMessage\nDeleted:\n${JsonMethods.pretty(d)}")
  }

}

Source File: FeatureJsonHelper.scala From TransmogrifAI with BSD 3-Clause "New" or "Revised" License

5 votes

package com.salesforce.op.features

import com.salesforce.op.features.types._
import com.salesforce.op.stages.{OPStage, OpPipelineStage}
import org.json4s.JsonAST.{JObject, JValue}
import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods
import org.json4s.jackson.JsonMethods._
import org.json4s.{DefaultFormats, Formats}

import scala.reflect.runtime.universe.WeakTypeTag
import scala.util.Try



  def fromJson(
    json: JValue,
    stages: Map[String, OPStage],
    features: Map[String, OPFeature]
  ): Try[OPFeature] = Try {
    val typeName = (json \ "typeName").extract[String]
    val uid = (json \ "uid").extract[String]
    val name = (json \ "name").extract[String]
    val isResponse = (json \ "isResponse").extract[Boolean]
    val originStageUid = (json \ "originStage").extract[String]
    val parentUids = (json \ "parents").extract[Array[String]]

    val originStage: Option[OPStage] = stages.get(originStageUid)
    if (originStage.isEmpty) {
      throw new RuntimeException(s"Origin stage $originStageUid not found for feature $name ($uid)")
    }

    // Order is important and so are duplicates, eg f = f1 + f1 has 2 parents but both the same feature
    val parents: Seq[OPFeature] = parentUids.flatMap(id => features.get(id))
    if (parents.length != parentUids.length) {
      throw new RuntimeException(s"Not all the parent features were found for feature $name ($uid)")
    }

    val wtt = FeatureType.featureTypeTag(typeName).asInstanceOf[WeakTypeTag[FeatureType]]
    Feature[FeatureType](
      uid = uid,
      name = name,
      isResponse = isResponse,
      parents = parents,
      originStage = originStage.get.asInstanceOf[OpPipelineStage[FeatureType]]
    )(wtt = wtt)

  }

}

Source File: JsonTestUtils.scala From spark1.52 with Apache License 2.0

5 votes

package org.apache.spark

import org.json4s._
import org.json4s.jackson.JsonMethods

trait JsonTestUtils {
  def assertValidDataInJson(validateJson: JValue, expectedJson: JValue) {
    val Diff(c, a, d) = validateJson.diff(expectedJson)
    val validatePretty = JsonMethods.pretty(validateJson)
    val expectedPretty = JsonMethods.pretty(expectedJson)
    val errorMessage = s"Expected:\n$expectedPretty\nFound:\n$validatePretty"
    import org.scalactic.TripleEquals._
    assert(c === JNothing, s"$errorMessage\nChanged:\n${JsonMethods.pretty(c)}")
    assert(a === JNothing, s"$errorMessage\nAdded:\n${JsonMethods.pretty(a)}")
    assert(d === JNothing, s"$errorMessage\nDeleted:\n${JsonMethods.pretty(d)}")
  }

}

Source File: JsonTestUtils.scala From Spark-2.3.1 with Apache License 2.0

5 votes

package org.apache.spark

import org.json4s._
import org.json4s.jackson.JsonMethods

trait JsonTestUtils {
  def assertValidDataInJson(validateJson: JValue, expectedJson: JValue) {
    val Diff(c, a, d) = validateJson.diff(expectedJson)
    val validatePretty = JsonMethods.pretty(validateJson)
    val expectedPretty = JsonMethods.pretty(expectedJson)
    val errorMessage = s"Expected:\n$expectedPretty\nFound:\n$validatePretty"
    import org.scalactic.TripleEquals._
    assert(c === JNothing, s"$errorMessage\nChanged:\n${JsonMethods.pretty(c)}")
    assert(a === JNothing, s"$errorMessage\nAdded:\n${JsonMethods.pretty(a)}")
    assert(d === JNothing, s"$errorMessage\nDeleted:\n${JsonMethods.pretty(d)}")
  }

}

Source File: JsonTestUtils.scala From BigDatalog with Apache License 2.0

5 votes

package org.apache.spark

import org.json4s._
import org.json4s.jackson.JsonMethods

trait JsonTestUtils {
  def assertValidDataInJson(validateJson: JValue, expectedJson: JValue) {
    val Diff(c, a, d) = validateJson.diff(expectedJson)
    val validatePretty = JsonMethods.pretty(validateJson)
    val expectedPretty = JsonMethods.pretty(expectedJson)
    val errorMessage = s"Expected:\n$expectedPretty\nFound:\n$validatePretty"
    import org.scalactic.TripleEquals._
    assert(c === JNothing, s"$errorMessage\nChanged:\n${JsonMethods.pretty(c)}")
    assert(a === JNothing, s"$errorMessage\nAdded:\n${JsonMethods.pretty(a)}")
    assert(d === JNothing, s"$errorMessage\nDeleted:\n${JsonMethods.pretty(d)}")
  }

}

Source File: ForgeInstallSpec.scala From PackUpdate with Apache License 2.0

5 votes

package at.chaosfield.packupdate.json

import java.io.{File, FileNotFoundException}
import java.net.URI
import scala.xml.XML

import at.chaosfield.packupdate.common.{FileManager, Log, MavenPath}
import org.json4s.jackson.JsonMethods

case class ForgeInstallSpec(
                           install: InstallInformation,
                           versionInfo: VersionInformation,
                           spec: Int = 0
                           )

case class InstallInformation(
                               profileName: String,
                               target: String,
                               path: MavenPath,
                               version: String,
                               filePath: String,
                               welcome: String,
                               minecraft: String,
                               mirrorList: URI,
                               logo: String,
                               modList: String
                             )

case class VersionInformation(
                             id: String,
                             `type`: String,
                             minecraftArguments: String,
                             mainClass: String,
                             inheritsFrom: String,
                             jar: String,
                             libraries: Array[LibraryInformation]
                             )

case class LibraryInformation(
                             name: MavenPath,
                             url: Option[URI],
                             checksums: Array[String],
                             serverreq: Boolean = false,
                             clientreq: Boolean = false
                             ) {

  def getPom(mavenPath: MavenPath, log: Log): xml.Elem = {
    var lastException: Option[Exception] = None
    val tryUrls = url.toList ++ LibraryInformation.RepoList
    for (url <- tryUrls) {
      try {
        val pomUrl = url.resolve(mavenPath.getPom.getFilePath).toURL
        val data = FileManager.readStreamToString(FileManager.retrieveUrl(pomUrl, log)._1)

        return XML.loadString(data)
      } catch {
        case e: FileNotFoundException =>
          lastException = Some(e)
          log.debug(s"File not found at $url, trying next...")
      }
    }
    throw lastException.get
  }
}

object LibraryInformation {
  final val RepoList = List("https://repo.maven.apache.org/maven2/", "https://libraries.minecraft.net/").map(new URI(_))
}

Source File: FilterJsonLigatures.scala From eidos with Apache License 2.0

5 votes

package org.clulab.wm.eidos.apps

import java.io.File
import java.io.PrintWriter
import java.util.regex.Pattern

import org.clulab.wm.eidos.utils.Closer.AutoCloser
import org.clulab.wm.eidos.utils.FileUtils
import org.clulab.wm.eidos.utils.Sinker
import org.clulab.wm.eidos.utils.TsvWriter
import org.json4s.DefaultFormats
import org.json4s.JString
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJsonLigatures extends App {
  val pattern: Pattern = Pattern.compile("([A-Za-z]+(f([bhkl]|[ft]|[ij])|ij)) ([A-Za-z]+)")

  class Filter(tsvWriter: TsvWriter) {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    tsvWriter.println("file", "left", "right")

    def filter(jValue: JValue, inputFile: File): Unit = {
      val extractions: JValue = jValue \ "_source" \ "extracted_text"

      extractions match {
        case text: JString =>
          val matcher = pattern.matcher(text.extract[String])

          while (matcher.find)
            tsvWriter.println(inputFile.getName, matcher.group(1), matcher.group(4))
        case _ => throw new RuntimeException(s"Unexpected extractions value: $extractions")
      }
    }
  }

  val inputDir = args(0)
  val extension = args(1)
  val outputFile = args(2)

  new TsvWriter(Sinker.printWriterFromFile(outputFile)).autoClose { tsvWriter =>
    val filter = new Filter(tsvWriter)
    val inputFiles = FileUtils.findFiles(inputDir, extension)

    inputFiles.sortBy(_.getName).foreach { inputFile =>
      val text = FileUtils.getTextFromFile(inputFile)
      val json = JsonMethods.parse(text)

      filter.filter(json, inputFile)
    }
  }
}

Source File: ParameterTest.scala From maha with Apache License 2.0

5 votes

// Copyright 2017, Yahoo Holdings Inc.
// Licensed under the terms of the Apache License 2.0. Please see LICENSE file in project root for terms.
package com.yahoo.maha.core.request

import com.yahoo.maha.core.request.ReportFormatType.CSVFormat
import com.yahoo.maha.core.{Engine, HiveEngine}
import org.json4s._
import org.json4s.jackson.JsonMethods
import org.scalatest.{FunSuite, Matchers}

import scala.collection.mutable.HashMap


class ParameterTest extends FunSuite with Matchers {

  test("SerializeParameters should serialize a map of parameters into a List") {
    val map_parameters = new HashMap[Parameter, ParameterValue[_]]
    map_parameters.put(Parameter.ReportFormat, ReportFormatValue(ReportFormatType.CSVFormat))
    map_parameters.put(Parameter.DryRun, DryRunValue(false))
    map_parameters.put(Parameter.GeneratedQuery, GeneratedQueryValue("GeneratedQuery"))
    map_parameters.put(Parameter.QueryEngine, QueryEngineValue(Engine.from("hive").get))
    map_parameters.put(Parameter.Debug, DebugValue(true))
    map_parameters.put(Parameter.RequestId, RequestIdValue("RequestId"))
    map_parameters.put(Parameter.UserId, UserIdValue("UserId"))
    map_parameters.put(Parameter.TimeZone, TimeZoneValue("TimeZone"))
    map_parameters.put(Parameter.Schema, SchemaValue("Schema"))
    map_parameters.put(Parameter.Distinct, DistinctValue(true))
    map_parameters.put(Parameter.JobName, JobNameValue("tools_1"))
    map_parameters.put(Parameter.RegistryName, RegistryNameValue("mahaRegistry"))
    map_parameters.put(Parameter.HostName, HostNameValue("127.0.0.1"))

    val result = Parameter.serializeParameters(map_parameters.toMap)
    result.length shouldBe map_parameters.size

    val newMap = result.map(t=> t._1 -> t._2).toMap
    for((k,v) <- map_parameters) {
      newMap.get(k.entryName).get match{
        case JString(x) => v.value match {
          case CSVFormat => x shouldBe "csv"
          case HiveEngine => x shouldBe "Hive"
          case _ => x shouldBe v.value
        }
        case JBool(x) => x shouldBe v.value
        case _ => fail
      }
    }

  }

  test("DeserializeParameters should deserialize a JSON into a Map of parameter values") {
    val inputJson=
      """
        |{
        | "Report-Format": "csv",
        | "Dry-Run": false,
        | "Generated-Query": "Generated-Query",
        | "Query-Engine": "oracle",
        | "debug": true,
        | "Request-Id": "Request-Id",
        | "User-Id": "User-Id",
        | "TimeZone": "TimeZone",
        | "Schema": "Schema",
        | "Distinct": true,
        | "Job-Name": "Job-Name",
        | "RegistryName": "mahaRegistry",
        | "HostName": "127.0.0.1"
        |}
        |""".stripMargin
    val result = Parameter.deserializeParameters(JsonMethods.parse(inputJson))

    result.getOrElse() match{
      case m: Map[Parameter, ParameterValue[_]] => {
        m.size shouldBe 13
        m.get(Parameter.ReportFormat).get shouldBe ReportFormatValue(ReportFormatType.CSVFormat)
        m.get(Parameter.DryRun).get shouldBe DryRunValue(false)
        m.get(Parameter.GeneratedQuery).get shouldBe GeneratedQueryValue("Generated-Query")
        m.get(Parameter.QueryEngine).get shouldBe QueryEngineValue(Engine.from("oracle").get)
        m.get(Parameter.Debug).get shouldBe DebugValue(true)
        m.get(Parameter.RequestId).get shouldBe RequestIdValue("Request-Id")
        m.get(Parameter.UserId).get shouldBe UserIdValue("User-Id")
        m.get(Parameter.TimeZone).get shouldBe TimeZoneValue("TimeZone")
        m.get(Parameter.Schema).get shouldBe SchemaValue("Schema")
        m.get(Parameter.Distinct).get shouldBe DistinctValue(true)
        m.get(Parameter.JobName).get shouldBe JobNameValue("Job-Name")
        m.get(Parameter.RegistryName).get shouldBe RegistryNameValue("mahaRegistry")
        m.get(Parameter.HostName).get shouldBe HostNameValue("127.0.0.1")
      }
      case _ => fail
    }

  }

}

Source File: KinesisRDDWriter.scala From aws-kinesis-scala with Apache License 2.0

5 votes

package jp.co.bizreach.kinesis.spark

import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
import com.amazonaws.regions.Regions
import jp.co.bizreach.kinesis._
import org.apache.commons.codec.digest.DigestUtils
import org.apache.spark.TaskContext
import org.json4s.jackson.JsonMethods
import org.json4s.{DefaultFormats, Extraction, Formats}
import org.slf4j.LoggerFactory

class KinesisRDDWriter[A <: AnyRef](streamName: String, region: Regions,
                                    credentials: SparkAWSCredentials,
                                    chunk: Int, endpoint: Option[String]) extends Serializable {
  private val logger = LoggerFactory.getLogger(getClass)

  def write(task: TaskContext, data: Iterator[A]): Unit = {
    // send data, including retry
    def put(a: Seq[PutRecordsEntry]) = endpoint.map(e => KinesisRDDWriter.endpointClient(credentials)(e)(region))
      .getOrElse(KinesisRDDWriter.client(credentials)(region))
      .putRecordsWithRetry(PutRecordsRequest(streamName, a))
      .zipWithIndex.collect { case (Left(e), i) => a(i) -> s"${e.errorCode}: ${e.errorMessage}" }

    val errors = data.foldLeft(
      (Nil: Seq[PutRecordsEntry], Nil: Seq[(PutRecordsEntry, String)])
    ){ (z, x) =>
      val (records, failed) = z
      val payload = serialize(x)
      val entry   = PutRecordsEntry(DigestUtils.sha256Hex(payload), payload)

      // record exceeds max size
      if (entry.recordSize > recordMaxDataSize)
        records -> ((entry -> "per-record size limit") +: failed)

      // execute
      else if (records.size >= chunk || (records.map(_.recordSize).sum + entry.recordSize) >= recordsMaxDataSize)
        (entry +: Nil) -> (put(records) ++ failed)

      // buffering
      else
        (entry +: records) -> failed
    } match {
      case (Nil, e)  => e
      case (rest, e) => put(rest) ++ e
    }

    // failed records
    if (errors.nonEmpty) dump(errors)
  }

  protected def dump(errors: Seq[(PutRecordsEntry, String)]): Unit =
    logger.error(
      s"""Could not put record, count: ${errors.size}, following details:
         |${errors map { case (entry, message) => message + "\n" + new String(entry.data, "UTF-8") } mkString "\n"}
       """.stripMargin)

  protected def serialize(a: A)(implicit formats: Formats = DefaultFormats): Array[Byte] =
    JsonMethods.mapper.writeValueAsBytes(Extraction.decompose(a)(formats))

}

object KinesisRDDWriter {
  private val cache = collection.concurrent.TrieMap.empty[Regions, AmazonKinesis]


  private val client: SparkAWSCredentials => Regions => AmazonKinesis = {
    credentials => implicit region =>
      cache.getOrElseUpdate(region, AmazonKinesis(credentials.provider))
  }

  private val endpointClient: SparkAWSCredentials => String => Regions => AmazonKinesis = {
    credentials => endpoint => implicit region =>
      cache.getOrElseUpdate(region, AmazonKinesis(credentials.provider, new EndpointConfiguration(endpoint, region.getName)))
  }

}

Source File: LeapFrameScoringController.scala From mleap with Apache License 2.0

5 votes

package ml.combust.mleap.springboot

import java.util.concurrent.CompletionStage

import akka.actor.ActorSystem
import ml.combust.mleap.executor.{MleapExecutor, TransformFrameRequest}
import ml.combust.mleap.pb.ErrorTransformResponse
import ml.combust.mleap.pb.TransformStatus.STATUS_ERROR
import ml.combust.mleap.runtime.serialization.{BuiltinFormats, FrameReader, FrameWriter}
import ml.combust.mleap.springboot.TypeConverters._
import org.apache.commons.lang3.exception.ExceptionUtils
import org.json4s.jackson.JsonMethods
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.web.bind.annotation._

import scala.compat.java8.FutureConverters._
import scala.concurrent.Future
import scala.util.{Failure, Success}
import scalapb.json4s.{Parser, Printer}

@RestController
@RequestMapping
class LeapFrameScoringController(@Autowired val actorSystem : ActorSystem,
                                 @Autowired val mleapExecutor: MleapExecutor,
                                 @Autowired val jsonPrinter: Printer,
                                 @Autowired val jsonParser: Parser) {

  private val executor = actorSystem.dispatcher

  @PostMapping(path = Array("/models/{model_name}/transform"),
    consumes = Array("application/json; charset=UTF-8"),
    produces = Array("application/json; charset=UTF-8"))
  def transformJson(@RequestBody body: Array[Byte],
                @PathVariable("model_name") modelName: String,
                @RequestHeader(value = "timeout", defaultValue = "60000") timeout: Int) : CompletionStage[_] = {
    FrameReader(BuiltinFormats.json).fromBytes(body) match {
      case Success(frame) => mleapExecutor.transform(TransformFrameRequest(modelName, frame, None))(timeout)
        .mapAll {
          case Success(resp) => resp match {
            case Success(frame) => FrameWriter(frame, BuiltinFormats.json).toBytes().get
            case Failure(ex) => JsonMethods.compact(jsonPrinter.toJson(handleTransformFailure(ex)))
          }
          case Failure(ex) => JsonMethods.compact(jsonPrinter.toJson(handleTransformFailure(ex)))

        }(executor)
        .toJava
      case Failure(ex) => Future {
        JsonMethods.compact(jsonPrinter.toJson(handleTransformFailure(ex)))
      }(executor).toJava
    }
  }

  @PostMapping(path = Array("/models/{model_name}/transform"),
    consumes = Array("application/x-protobuf; charset=UTF-8"),
    produces = Array("application/x-protobuf; charset=UTF-8"))
  def transformProto(@RequestBody body: Array[Byte],
                @PathVariable("model_name") modelName: String,
                @RequestHeader(value = "timeout", defaultValue = "60000") timeout: Int) : CompletionStage[_] = {
    FrameReader(BuiltinFormats.binary).fromBytes(body) match {
      case Success(frame) =>
        mleapExecutor.transform(TransformFrameRequest(modelName, frame, None))(timeout)
          .mapAll {
            case Success(resp) => resp match {
              case Success(frame) => FrameWriter(frame, BuiltinFormats.binary).toBytes().get
              case Failure(ex) => ErrorTransformResponse.toJavaProto(handleTransformFailure(ex))
            }
            case Failure(ex) => ErrorTransformResponse.toJavaProto(handleTransformFailure(ex))
          }(executor).toJava
      case Failure(ex) => Future {
        ErrorTransformResponse.toJavaProto(handleTransformFailure(ex))
      }(executor).toJava
    }
  }

  private def handleTransformFailure(ex: Throwable): ErrorTransformResponse = {
    LeapFrameScoringController.logger.error("Transform error due to ", ex)
    ErrorTransformResponse(status = STATUS_ERROR,
      error = ExceptionUtils.getMessage(ex), backtrace = ExceptionUtils.getStackTrace(ex))
  }
}

object LeapFrameScoringController {
  val logger = LoggerFactory.getLogger(classOf[LeapFrameScoringController])
}

Source File: Json4sDefinitions.scala From circe-benchmarks with Apache License 2.0

5 votes

package io.circe.benchmarks

import org.json4s._
import org.json4s.jackson.JsonMethods
import org.openjdk.jmh.annotations._

trait Json4sData extends DefaultWriters { self: ExampleData =>

  implicit val formats = DefaultFormats

  @inline def encode4s[T](value: T): JValue = Extraction.decompose(value)

  val foos4s: JValue = encode4s(foos)
  val ints4s: JValue = encode4s(ints)
}

trait Json4sWriting { self: ExampleData =>
  @Benchmark
  def writeFoosJson4s: String = JsonMethods.compact(Extraction.decompose(foos))

  @Benchmark
  def writeIntsJson4s: String = JsonMethods.compact(Extraction.decompose(ints))
}

trait Json4sReading { self: ExampleData =>
  @Benchmark
  def readFoosJson4s: Map[String, Foo] = JsonMethods.parse(foosJson).extract[Map[String, Foo]]

  @Benchmark
  def readIntsJson4s: List[Int] = JsonMethods.parse(intsJson).extract[List[Int]]
}

trait Json4sEncoding { self: ExampleData =>
  @Benchmark
  def encodeFoosJson4s: JValue = encode4s(foos)

  @Benchmark
  def encodeIntsJson4s: JValue = encode4s(ints)
}

trait Json4sDecoding { self: ExampleData =>
  @Benchmark
  def decodeFoosJson4s: Map[String, Foo] = foos4s.extract[Map[String, Foo]]

  @Benchmark
  def decodeIntsJson4s: List[Int] = ints4s.extract[List[Int]]
}

trait Json4sPrinting { self: ExampleData =>
  @Benchmark
  def printFoosJson4s: String = JsonMethods.compact(foos4s)

  @Benchmark
  def printIntsJson4s: String = JsonMethods.compact(ints4s)
}

trait Json4sParsing { self: ExampleData =>
  @Benchmark
  def parseFoosJson4s: JValue = JsonMethods.parse(foosJson)

  @Benchmark
  def parseIntsJson4s: JValue = JsonMethods.parse(intsJson)
}

Source File: EncodingBenchmarkSpec.scala From circe-benchmarks with Apache License 2.0

5 votes

package io.circe.benchmarks

import argonaut.Parse
import argonaut.Argonaut._
import org.json4s.jackson.JsonMethods
import org.scalatest.flatspec.AnyFlatSpec
import play.api.libs.json.Json

class EncodingBenchmarkSpec extends AnyFlatSpec {
  val benchmark: EncodingBenchmark = new EncodingBenchmark

  import benchmark._

  def decodeInts(json: String): Option[List[Int]] =
    Parse.decodeOption[List[Int]](json)

  def decodeFoos(json: String): Option[Map[String, Foo]] =
    Parse.decodeOption[Map[String, Foo]](json)

  "The encoding benchmark" should "correctly encode integers using Circe" in {
    assert(decodeInts(encodeIntsCirce.noSpaces) === Some(ints))
  }

  it should "correctly encode integers using Argonaut" in {
    assert(decodeInts(encodeIntsArgonaut.nospaces) === Some(ints))
  }

  it should "correctly encode integers using Spray JSON" in {
    assert(decodeInts(encodeIntsSpray.compactPrint) === Some(ints))
  }

  it should "correctly encode integers using Json4s" in {
    assert(decodeInts(JsonMethods.compact(encodeIntsJson4s)) === Some(ints))
  }

  it should "correctly encode integers using Play JSON" in {
    assert(decodeInts(Json.prettyPrint(encodeIntsPlay)) === Some(ints))
  }

  it should "correctly encode case classes using Circe" in {
    assert(decodeFoos(encodeFoosCirce.noSpaces) === Some(foos))
  }

  it should "correctly encode case classes using Argonaut" in {
    assert(decodeFoos(encodeFoosArgonaut.nospaces) === Some(foos))
  }

  it should "correctly encode case classes using Spray JSON" in {
    assert(decodeFoos(encodeFoosSpray.compactPrint) === Some(foos))
  }

  it should "correctly encode case classes using Json4s" in {
    assert(decodeFoos(JsonMethods.compact(encodeFoosJson4s)) === Some(foos))
  }

  it should "correctly encode case classes using Play JSON" in {
    assert(decodeFoos(Json.prettyPrint(encodeFoosPlay)) === Some(foos))
  }
}

Source File: LookupTable.scala From jigg with Apache License 2.0

5 votes

package jigg.util



import java.io.Reader

import breeze.linalg.DenseMatrix
import org.json4s.{DefaultFormats, _}
import org.json4s.jackson.JsonMethods
import org.json4s.JsonAST.JValue

class LookupTable(rawTable: JValue) {

  implicit private val formats = DefaultFormats
  private val tables = rawTable.extract[Map[String, Map[String, Map[String, String]]]]

  private val key2id = tables("_lookup")("_key2id")
  private val id2key = tables("_lookup")("_id2key")

  // For raw text
  def encodeCharacter(str: String): DenseMatrix[Float] = {
    val strArray = str.map{x =>
      // Note: For skipping unknown character, this encoder returns dummy id.
      key2id.getOrElse(x.toString, "3").toFloat
    }.toArray
    new DenseMatrix[Float](1, str.length, strArray)
  }

  // For list of words
  def encodeWords(words: Array[String]): DenseMatrix[Float] = {
    val wordsArray = words.map{x =>
      // Note: For skipping unknown words, this encoder returns dummy id.
      key2id.getOrElse(x.toString, "3").toFloat
    }
    new DenseMatrix[Float](1, words.length, wordsArray)
  }

  def decode(data: DenseMatrix[Float]): Array[String] =
    data.map{x => id2key.getOrElse(x.toInt.toString, "NONE")}.toArray

  def getId(key: String): Int = key2id.getOrElse(key, "0").toInt
  def getId(key: Char): Int = getId(key.toString)

  def getKey(id: Int): String = id2key.getOrElse(id.toString, "UNKNOWN")
}


object LookupTable {

  // Load from a path on the file system
  def fromFile(path: String) = mkTable(IOUtil.openIn(path))

  // Load from class loader
  def fromResource(path: String) = mkTable(IOUtil.openResourceAsReader(path))

  private def mkTable(input: Reader) = {
    val j = try { JsonMethods.parse(input) } finally { input.close }
    new LookupTable(j)
  }
}

Source File: JsonTestUtils.scala From sparkoscope with Apache License 2.0

5 votes

package org.apache.spark

import org.json4s._
import org.json4s.jackson.JsonMethods

trait JsonTestUtils {
  def assertValidDataInJson(validateJson: JValue, expectedJson: JValue) {
    val Diff(c, a, d) = validateJson.diff(expectedJson)
    val validatePretty = JsonMethods.pretty(validateJson)
    val expectedPretty = JsonMethods.pretty(expectedJson)
    val errorMessage = s"Expected:\n$expectedPretty\nFound:\n$validatePretty"
    import org.scalactic.TripleEquals._
    assert(c === JNothing, s"$errorMessage\nChanged:\n${JsonMethods.pretty(c)}")
    assert(a === JNothing, s"$errorMessage\nAdded:\n${JsonMethods.pretty(a)}")
    assert(d === JNothing, s"$errorMessage\nDeleted:\n${JsonMethods.pretty(d)}")
  }

}

Source File: FilterJsonGeoAndTime.scala From eidos with Apache License 2.0

5 votes

package org.clulab.wm.eidos.apps

import java.io.File

import org.clulab.wm.eidos.utils.FileUtils
import org.json4s.DefaultFormats
import org.json4s.JArray
import org.json4s.JNothing
import org.json4s.JObject
import org.json4s.JString
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJsonGeoAndTime extends App {

  class Filter() {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    def filter(inputFile: File, jValue: JValue): Unit = {
      println(s"Extracting from ${inputFile.getName}")

      def filterGeo(): Unit = {
        val geoLocations: JValue = (jValue \\ "geolocs" \ "text")

        geoLocations match {
          case JArray(geoLocations: List[_]) => // Type erasure removes the [JString]
            geoLocations.foreach { geoLocation =>
              val text = geoLocation.extract[String]
              val oneLiner = text
                  .replace("\n", "\\n")
                  .replace("\t", "\\t")

              println("\tGeo\t" + oneLiner)
            }
          case JNothing =>
          case _ => throw new RuntimeException(s"Unexpected geoLocations value: $geoLocations")
        }
      }

      def filterTime(): Unit = {
        val timexes: JValue = (jValue \\ "timexes" \ "text")

        timexes match {
          case JArray(timexes: List[_]) => // Type erasure removes the [JString]
            timexes.foreach { timex =>
              val text = timex.extract[String]
              val oneLiner = text
                  .replace("\n", "\\n")
                  .replace("\t", "\\t")

              println("\tTime\t" + oneLiner)
            }
          case JNothing =>
          case _ => throw new RuntimeException(s"Unexpected geoLocations value: $timexes")
        }
      }

      filterGeo()
      filterTime()
    }
  }

  val inputDir = args(0)
  val extension = args(1)
  val inputFiles = FileUtils.findFiles(inputDir, extension)
  val filter = new Filter()

  inputFiles.foreach { inputFile =>
    val text = FileUtils.getTextFromFile(inputFile)
    val json = JsonMethods.parse(text)
    filter.filter(inputFile, json)
  }
}

Source File: ExtractDocumentFromDirectory.scala From eidos with Apache License 2.0

5 votes

package org.clulab.wm.eidos.apps

import org.clulab.processors.Document
import org.clulab.serialization.DocumentSerializer
import org.clulab.serialization.json.{JSONSerializer, stringify}
import org.clulab.wm.eidos.EidosSystem
import org.clulab.wm.eidos.document.AnnotatedDocument
import org.clulab.wm.eidos.serialization.jsonld.JLDCorpus
import org.clulab.wm.eidos.utils.Closer.AutoCloser
import org.clulab.wm.eidos.utils.FileUtils
import org.json4s.jackson.JsonMethods

object ExtractDocumentFromDirectory extends App {

  def annotateTxt(reader: EidosSystem, text: String): AnnotatedDocument = {
    reader.extractFromText(text)
  }

  protected def annotateDocument(reader: EidosSystem, document: Document): AnnotatedDocument = {
    val eidosDoc = reader.annotateDoc(document)

    reader.extractFromDoc(eidosDoc)
  }

  def annotateJson(reader: EidosSystem, text: String): AnnotatedDocument = {
    val doc: Document = JSONSerializer.toDocument(JsonMethods.parse(text))

    annotateDocument(reader, doc)
  }

  def annotateDoc(reader: EidosSystem, text: String): AnnotatedDocument = {
    val doc: Document = (new DocumentSerializer).load(text)

    annotateDocument(reader, doc)
  }

  val inputDir = args(0)
  val outputDir = args(1)
  val format = if (args.length > 2) args(2) else "1"
  val extension = if (args.length > 3) args(3) else "txt"
  val annotator = format match {
    case "1" => annotateTxt _
    case "2" => annotateJson _
    case "3" => annotateDoc _
    case _ => throw new Exception(s"Unknown format '$format'")
  }
  val files = FileUtils.findFiles(inputDir, extension)
  val reader = new EidosSystem()

  // For each file in the input directory:
  files.par.foreach { file =>
    // 1. Open corresponding output file
    println(s"Extracting from ${file.getName}")
    FileUtils.printWriterFromFile(s"$outputDir/${file.getName}.jsonld").autoClose { pw =>
      // 2. Get the input file contents
      val text = FileUtils.getTextFromFile(file)
      // 3. Extract causal mentions from the text
      val annotatedDocument = annotator(reader, text)
      // 4. Convert to JSON
      val corpus = new JLDCorpus(annotatedDocument)
      val mentionsJSONLD = corpus.serialize()
      // 5. Write to output file
      pw.println(stringify(mentionsJSONLD, pretty = true))
    }
  }
}

Source File: FilterJson.scala From eidos with Apache License 2.0

5 votes

package org.clulab.wm.eidos.apps

import java.io.File

import org.clulab.wm.eidos.utils.FileUtils
import org.json4s.DefaultFormats
import org.json4s.JArray
import org.json4s.JNothing
import org.json4s.JString
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJson extends App {

  class Filter(val outputDir: String) {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    def filter(inputFile: File, jValue: JValue): Unit = {
      // println(s"Extracting from ${inputFile.getName}")
      val geoLocations: JValue = (jValue \\ "geolocs" \ "text")

      geoLocations match {
        case JArray(geoLocations: List[_]) => // Type erasure removes the [JString]
          geoLocations.foreach { geoLocation =>
            println(geoLocation.extract[String])
          }
        case JNothing =>
        case _ => throw new RuntimeException(s"Unexpected geoLocations value: $geoLocations")
      }
    }
  }

  val inputDir = args(0)
  val extension = args(1)
  val outputDir = args(2)
  val inputFiles = FileUtils.findFiles(inputDir, extension)
  val filter = new Filter(outputDir)

  inputFiles.foreach { inputFile =>
    val text = FileUtils.getTextFromFile(inputFile)
    val json = JsonMethods.parse(text)
    filter.filter(inputFile, json)
  }
}

Source File: FilterJsonCanonicalNames.scala From eidos with Apache License 2.0

5 votes

package org.clulab.wm.eidos.apps

import java.io.File
import java.io.PrintWriter

import org.clulab.wm.eidos.utils.FileUtils
import org.clulab.wm.eidos.utils.Sinker
import org.clulab.wm.eidos.utils.Closer.AutoCloser
import org.clulab.wm.eidos.utils.TsvWriter
import org.json4s.DefaultFormats
import org.json4s.JArray
import org.json4s.JObject
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJsonCanonicalNames extends App {

  class Filter(tsvWriter: TsvWriter) {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    tsvWriter.println("file", "id", "text", "canonicalName")

    def filter(jValue: JValue, inputFile: File): Unit = {
      val extractions: JValue = jValue \\ "extractions"

      extractions match {
        case JArray(extractions: List[_]) => // Type erasure removes the [JObject]
          extractions.foreach { extraction =>
            val id = (extraction \ "@id").extract[String]
            val text = (extraction \ "text").extract[String]
            val canonicalName = (extraction \ "canonicalName").extract[String]

            tsvWriter.println(inputFile.getName, id, text, canonicalName)
          }
        case JObject(_) =>
        case _ => throw new RuntimeException(s"Unexpected extractions value: $extractions")
      }
    }
  }

  val inputDir = args(0)
  val extension = args(1)
  val outputFile = args(2)

  new TsvWriter(Sinker.printWriterFromFile(outputFile)).autoClose { tsvWriter =>
    val filter = new Filter(tsvWriter)
    val inputFiles = FileUtils.findFiles(inputDir, extension)

    inputFiles.sortBy(_.getName).foreach { inputFile =>
      val text = FileUtils.getTextFromFile(inputFile)
      val json = JsonMethods.parse(text)

      filter.filter(json, inputFile)
    }
  }
}

Source File: JsonTestUtils.scala From drizzle-spark with Apache License 2.0

5 votes

package org.apache.spark

import org.json4s._
import org.json4s.jackson.JsonMethods

trait JsonTestUtils {
  def assertValidDataInJson(validateJson: JValue, expectedJson: JValue) {
    val Diff(c, a, d) = validateJson.diff(expectedJson)
    val validatePretty = JsonMethods.pretty(validateJson)
    val expectedPretty = JsonMethods.pretty(expectedJson)
    val errorMessage = s"Expected:\n$expectedPretty\nFound:\n$validatePretty"
    import org.scalactic.TripleEquals._
    assert(c === JNothing, s"$errorMessage\nChanged:\n${JsonMethods.pretty(c)}")
    assert(a === JNothing, s"$errorMessage\nAdded:\n${JsonMethods.pretty(a)}")
    assert(d === JNothing, s"$errorMessage\nDeleted:\n${JsonMethods.pretty(d)}")
  }

}

Source File: FilterJsonPretty.scala From eidos with Apache License 2.0

5 votes

package org.clulab.wm.eidos.apps

import java.io.File

import org.clulab.serialization.json.stringify
import org.clulab.wm.eidos.utils.Closer.AutoCloser
import org.clulab.wm.eidos.utils.FileEditor
import org.clulab.wm.eidos.utils.FileUtils
import org.json4s.DefaultFormats
import org.json4s.JObject
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJsonPretty extends App {

  class Filter(outputDir: String) {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    def filter(jValue: JValue, inputFile: File): Unit = {
      val extractions: JValue = jValue

      extractions match {
        case jObject: JObject =>
          val json = stringify(jObject, pretty = true)
          val path = FileEditor(inputFile).setDir(outputDir).get

          FileUtils.printWriterFromFile(path).autoClose { pw =>
            pw.println(json)
          }

        case _ => throw new RuntimeException(s"Unexpected extractions value: $extractions")
      }
    }
  }

  val inputDir = args(0)
  val outputDir = args(1)
  val filter = new Filter(outputDir)
  val inputFiles = FileUtils.findFiles(inputDir, "json")

  inputFiles.sortBy(_.getName).foreach { inputFile =>
    val text = FileUtils.getTextFromFile(inputFile)
    val json = JsonMethods.parse(text)

    filter.filter(json, inputFile)
  }
}

Source File: FilterJsonText.scala From eidos with Apache License 2.0

5 votes

package org.clulab.wm.eidos.apps

import java.io.File

import org.clulab.wm.eidos.utils.Closer.AutoCloser
import org.clulab.wm.eidos.utils.FileEditor
import org.clulab.wm.eidos.utils.FileUtils
import org.clulab.wm.eidos.utils.StringUtils
import org.json4s.DefaultFormats
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJsonText extends App {

  class Filter(outputDir: String) {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    def filter(jValue: JValue, inputFile: File): Unit = {
      val jString: JValue = jValue \ "extracted_text"
      val text: String = jString.extract[String]
      val path = FileEditor(inputFile).setDir(outputDir).setExt("txt").get

      FileUtils.printWriterFromFile(path).autoClose { pw =>
        pw.println(text)
      }
    }
  }

  val inputDir = args(0)
  val outputDir = args(1)
  val filter = new Filter(outputDir)
  val inputFiles = FileUtils.findFiles(inputDir, "json")

  inputFiles.sortBy(_.getName).foreach { inputFile =>
    val text = FileUtils.getTextFromFile(inputFile)
    val json = JsonMethods.parse(text)

    filter.filter(json, inputFile)
  }
}

Source File: SeparateCdrTextFromDirectory.scala From eidos with Apache License 2.0

5 votes

package org.clulab.wm.eidos.apps.batch

import java.io.File

import org.clulab.wm.eidos.utils.Closer.AutoCloser
import org.clulab.wm.eidos.utils.FileEditor
import org.clulab.wm.eidos.utils.FileUtils
import org.clulab.wm.eidos.utils.StringUtils
import org.clulab.wm.eidos.utils.meta.CluText
import org.json4s.DefaultFormats
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object SeparateCdrTextFromDirectory extends App {

  class Filter(outputDir: String) {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    def filter(inputFile: File, jValue: JValue): Unit = {
      println(s"Extracting from ${inputFile.getName}")
      val jString: JValue = jValue \ "extracted_text"
      val text: String = jString.extract[String]
      val outputFile = FileEditor(inputFile).setDir(outputDir).setExt("txt").get

      FileUtils.printWriterFromFile(outputFile).autoClose { printWriter =>
        printWriter.print(text)
      }
    }
  }

  val inputDir = args(0)
  val outputDir = args(1)
  val inputFiles = FileUtils.findFiles(inputDir, "json")
  val filter = new Filter(outputDir)

  inputFiles.foreach { inputFile =>
    val json = CluText.getJValue(inputFile)

    filter.filter(inputFile, json)
  }
}

Source File: FilterJsonSource.scala From eidos with Apache License 2.0

5 votes

package org.clulab.wm.eidos.apps

import java.io.File

import org.clulab.serialization.json.stringify
import org.clulab.wm.eidos.utils.Closer.AutoCloser
import org.clulab.wm.eidos.utils.FileEditor
import org.clulab.wm.eidos.utils.FileUtils
import org.clulab.wm.eidos.utils.StringUtils
import org.json4s.DefaultFormats
import org.json4s.JObject
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJsonSource extends App {

  class Filter(outputDir: String) {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    def filter(jValue: JValue, inputFile: File): Unit = {
      val extractions: JValue = jValue \ "_source"

      extractions match {
        case jObject: JObject =>
          val json = stringify(jObject, pretty = true)
          val path = FileEditor(inputFile).setDir(outputDir).get

          FileUtils.printWriterFromFile(path).autoClose { pw =>
            pw.println(json)
          }

        case _ => throw new RuntimeException(s"Unexpected extractions value: $extractions")
      }
    }
  }

  val inputDir = args(0)
  val outputDir = args(1)
  val filter = new Filter(outputDir)
  val inputFiles = FileUtils.findFiles(inputDir, "json")

  inputFiles.sortBy(_.getName).foreach { inputFile =>
    val text = FileUtils.getTextFromFile(inputFile)
    val json = JsonMethods.parse(text)

    filter.filter(json, inputFile)
  }
}

Source File: FilterJsonExtractions.scala From eidos with Apache License 2.0

5 votes

package org.clulab.wm.eidos.apps

import java.io.File

import org.clulab.wm.eidos.utils.FileUtils
import org.json4s.DefaultFormats
import org.json4s.JArray
import org.json4s.JNothing
import org.json4s.JObject
import org.json4s.JString
import org.json4s.JValue
import org.json4s.jackson.JsonMethods

object FilterJsonExtractions extends App {

  class Filter() {
    implicit val formats: DefaultFormats.type = org.json4s.DefaultFormats

    def filter(inputFile: File, jValue: JValue): Unit = {
      println(s"Extracting from ${inputFile.getName}")
      val extractions: JValue = (jValue \\ "extractions")

      extractions match {
        case JArray(extractions: List[_]) => // Type erasure removes the [JObject]
          extractions.foreach { extraction =>
            val jString = (extraction \ "text")
            val text = jString.extract[String]
            val oneLiner = text
                .replace("\n", "\\n")
                .replace("\t", "\\t")

            println("\t" + oneLiner)
          }
        case JObject(_) =>
        case _ => throw new RuntimeException(s"Unexpected extractions value: $extractions")
      }
    }
  }

  val inputDir = args(0)
  val extension = args(1)
  val inputFiles = FileUtils.findFiles(inputDir, extension)
  val filter = new Filter()

  inputFiles.foreach { inputFile =>
    val text = FileUtils.getTextFromFile(inputFile)
    val json = JsonMethods.parse(text)
    filter.filter(inputFile, json)
  }
}

Source File: DataFinder.scala From Scala-Design-Patterns-Second-Edition with MIT License

5 votes

package com.ivan.nikolov.behavioral.template

import java.io.{InputStreamReader, ByteArrayInputStream}

import com.github.tototoshi.csv.CSVReader
import com.ivan.nikolov.behavioral.template.model.Person
import org.json4s.{StringInput, DefaultFormats}
import org.json4s.jackson.JsonMethods

abstract class DataFinder[T, Y] {

  def find(f: T => Option[Y]): Option[Y] =
    try {
      val data = readData()
      val parsed = parse(data)
      f(parsed)
    } finally {
      cleanup()
    }

  def readData(): Array[Byte]

  def parse(data: Array[Byte]): T

  def cleanup()
}

class JsonDataFinder extends DataFinder[List[Person], Person] {
  implicit val formats = DefaultFormats

  override def readData(): Array[Byte] = {
    val stream = this.getClass.getResourceAsStream("people.json")
    Stream.continually(stream.read).takeWhile(_ != -1).map(_.toByte).toArray
  }

  override def cleanup(): Unit = {
    System.out.println("Reading json: nothing to do.")
  }

  override def parse(data: Array[Byte]): List[Person] =
    JsonMethods.parse(StringInput(new String(data, "UTF-8"))).extract[List[Person]]
}

class CSVDataFinder extends DataFinder[List[Person], Person] {
  override def readData(): Array[Byte] = {
    val stream = this.getClass.getResourceAsStream("people.csv")
    Stream.continually(stream.read).takeWhile(_ != -1).map(_.toByte).toArray
  }

  override def cleanup(): Unit = {
    System.out.println("Reading csv: nothing to do.")
  }

  override def parse(data: Array[Byte]): List[Person] =
    CSVReader.open(new InputStreamReader(new ByteArrayInputStream(data))).all().map {
      case List(name, age, address) =>
        Person(name, age.toInt, address)
    }
}


object DataFinderExample {
  def main(args: Array[String]): Unit = {
    val jsonDataFinder: DataFinder[List[Person], Person] = new JsonDataFinder
    val csvDataFinder: DataFinder[List[Person], Person] = new CSVDataFinder

    System.out.println(s"Find a person with name Ivan in the json: ${jsonDataFinder.find(_.find(_.name == "Ivan"))}")
    System.out.println(s"Find a person with name James in the json: ${jsonDataFinder.find(_.find(_.name == "James"))}")

    System.out.println(s"Find a person with name Maria in the csv: ${csvDataFinder.find(_.find(_.name == "Maria"))}")
    System.out.println(s"Find a person with name Alice in the csv: ${csvDataFinder.find(_.find(_.name == "Alice"))}")
  }
}

Source File: Parser.scala From Scala-Design-Patterns-Second-Edition with MIT License

5 votes

package com.ivan.nikolov.behavioral.strategy

import java.io.InputStreamReader

import com.github.tototoshi.csv.CSVReader
import com.ivan.nikolov.behavioral.strategy.model.Person

import org.json4s._
import org.json4s.jackson.JsonMethods

trait Parser[T] {
  def parse(file: String): List[T]
}

class CSVParser extends Parser[Person] {
  override def parse(file: String): List[Person] =
    CSVReader.open(new InputStreamReader(this.getClass.getResourceAsStream(file))).all().map {
      case List(name, age, address) =>
        Person(name, age.toInt, address)
    }
}

class JsonParser extends Parser[Person] {
  implicit val formats = DefaultFormats
  
  override def parse(file: String): List[Person] =
    JsonMethods.parse(StreamInput(this.getClass.getResourceAsStream(file))).extract[List[Person]]
}

object Parser {
  def apply(filename: String): Parser[Person] =
    filename match {
      case f if f.endsWith(".json") => new JsonParser
      case f if f.endsWith(".csv") => new CSVParser
      case f => throw new RuntimeException(s"Unknown format: $f")
    }
}

class PersonApplication[T](parser: Parser[T]) {
  
  def write(file: String): Unit = {
    System.out.println(s"Got the following data ${parser.parse(file)}")
  }
}

object ParserExample {
  def main(args: Array[String]): Unit = {
    val csvPeople = Parser("people.csv")
    val jsonPeople = Parser("people.json")
    
    val applicationCsv = new PersonApplication(csvPeople)
    val applicationJson = new PersonApplication(jsonPeople)
    
    System.out.println("Using the csv: ")
    applicationCsv.write("people.csv")
    
    System.out.println("Using the json: ")
    applicationJson.write("people.json")
  }
}

Source File: ParsingStrategy.scala From Scala-Design-Patterns-Second-Edition with MIT License

5 votes

package com.ivan.nikolov.behavioral.strategy

import java.io.InputStreamReader

import com.github.tototoshi.csv.CSVReader
import com.ivan.nikolov.behavioral.strategy.model.Person
import org.json4s.{StreamInput, DefaultFormats}
import org.json4s.jackson.JsonMethods

class Application[T](strategy: (String) => List[T]) {
  def write(file: String): Unit = {
    System.out.println(s"Got the following data ${strategy(file)}")
  }
}

object StrategyFactory {
  implicit val formats = DefaultFormats
  
  def apply(filename: String): (String) => List[Person] =
    filename match {
      case f if f.endsWith(".json") => parseJson
      case f if f.endsWith(".csv") => parseCsv
      case f => throw new RuntimeException(s"Unknown format: $f")
    }
  
  def parseJson(file: String): List[Person] =
    JsonMethods.parse(StreamInput(this.getClass.getResourceAsStream(file))).extract[List[Person]]
  
  def parseCsv(file: String): List[Person] =
    CSVReader.open(new InputStreamReader(this.getClass.getResourceAsStream(file))).all().map {
      case List(name, age, address) =>
        Person(name, age.toInt, address)
    }
}

object StrategyExample {
  def main(args: Array[String]): Unit = {
    val applicationCsv = new Application[Person](StrategyFactory("people.csv"))
    val applicationJson = new Application[Person](StrategyFactory("people.json"))

    System.out.println("Using the csv: ")
    applicationCsv.write("people.csv")

    System.out.println("Using the json: ")
    applicationJson.write("people.json")
  }
}

Source File: DataFinder.scala From Scala-Design-Patterns-Second-Edition with MIT License

5 votes

package com.ivan.nikolov.behavioral.template

import java.io.{InputStreamReader, ByteArrayInputStream}

import com.github.tototoshi.csv.CSVReader
import com.ivan.nikolov.behavioral.template.model.Person
import org.json4s.{StringInput, DefaultFormats}
import org.json4s.jackson.JsonMethods

abstract class DataFinder[T, Y] {

  def find(f: T => Option[Y]): Option[Y] =
    try {
      val data = readData()
      val parsed = parse(data)
      f(parsed)
    } finally {
      cleanup()
    }

  def readData(): Array[Byte]

  def parse(data: Array[Byte]): T

  def cleanup()
}

class JsonDataFinder extends DataFinder[List[Person], Person] {
  implicit val formats = DefaultFormats

  override def readData(): Array[Byte] = {
    val stream = this.getClass.getResourceAsStream("people.json")
    Stream.continually(stream.read).takeWhile(_ != -1).map(_.toByte).toArray
  }

  override def cleanup(): Unit = {
    System.out.println("Reading json: nothing to do.")
  }

  override def parse(data: Array[Byte]): List[Person] =
    JsonMethods.parse(StringInput(new String(data, "UTF-8"))).extract[List[Person]]
}

class CSVDataFinder extends DataFinder[List[Person], Person] {
  override def readData(): Array[Byte] = {
    val stream = this.getClass.getResourceAsStream("people.csv")
    Stream.continually(stream.read).takeWhile(_ != -1).map(_.toByte).toArray
  }

  override def cleanup(): Unit = {
    System.out.println("Reading csv: nothing to do.")
  }

  override def parse(data: Array[Byte]): List[Person] =
    CSVReader.open(new InputStreamReader(new ByteArrayInputStream(data))).all().map {
      case List(name, age, address) =>
        Person(name, age.toInt, address)
    }
}


object DataFinderExample {
  def main(args: Array[String]): Unit = {
    val jsonDataFinder: DataFinder[List[Person], Person] = new JsonDataFinder
    val csvDataFinder: DataFinder[List[Person], Person] = new CSVDataFinder

    System.out.println(s"Find a person with name Ivan in the json: ${jsonDataFinder.find(_.find(_.name == "Ivan"))}")
    System.out.println(s"Find a person with name James in the json: ${jsonDataFinder.find(_.find(_.name == "James"))}")

    System.out.println(s"Find a person with name Maria in the csv: ${csvDataFinder.find(_.find(_.name == "Maria"))}")
    System.out.println(s"Find a person with name Alice in the csv: ${csvDataFinder.find(_.find(_.name == "Alice"))}")
  }
}

Source File: Parser.scala From Scala-Design-Patterns-Second-Edition with MIT License

5 votes

package com.ivan.nikolov.behavioral.strategy

import java.io.InputStreamReader

import com.github.tototoshi.csv.CSVReader
import com.ivan.nikolov.behavioral.strategy.model.Person

import org.json4s._
import org.json4s.jackson.JsonMethods

trait Parser[T] {
  def parse(file: String): List[T]
}

class CSVParser extends Parser[Person] {
  override def parse(file: String): List[Person] =
    CSVReader.open(new InputStreamReader(this.getClass.getResourceAsStream(file))).all().map {
      case List(name, age, address) =>
        Person(name, age.toInt, address)
    }
}

class JsonParser extends Parser[Person] {
  implicit val formats = DefaultFormats
  
  override def parse(file: String): List[Person] =
    JsonMethods.parse(StreamInput(this.getClass.getResourceAsStream(file))).extract[List[Person]]
}

object Parser {
  def apply(filename: String): Parser[Person] =
    filename match {
      case f if f.endsWith(".json") => new JsonParser
      case f if f.endsWith(".csv") => new CSVParser
      case f => throw new RuntimeException(s"Unknown format: $f")
    }
}

class PersonApplication[T](parser: Parser[T]) {
  
  def write(file: String): Unit = {
    System.out.println(s"Got the following data ${parser.parse(file)}")
  }
}

object ParserExample {
  def main(args: Array[String]): Unit = {
    val csvPeople = Parser("people.csv")
    val jsonPeople = Parser("people.json")
    
    val applicationCsv = new PersonApplication(csvPeople)
    val applicationJson = new PersonApplication(jsonPeople)
    
    System.out.println("Using the csv: ")
    applicationCsv.write("people.csv")
    
    System.out.println("Using the json: ")
    applicationJson.write("people.json")
  }
}

Source File: ParsingStrategy.scala From Scala-Design-Patterns-Second-Edition with MIT License

5 votes

package com.ivan.nikolov.behavioral.strategy

import java.io.InputStreamReader

import com.github.tototoshi.csv.CSVReader
import com.ivan.nikolov.behavioral.strategy.model.Person
import org.json4s.{StreamInput, DefaultFormats}
import org.json4s.jackson.JsonMethods

class Application[T](strategy: (String) => List[T]) {
  def write(file: String): Unit = {
    System.out.println(s"Got the following data ${strategy(file)}")
  }
}

object StrategyFactory {
  implicit val formats = DefaultFormats
  
  def apply(filename: String): (String) => List[Person] =
    filename match {
      case f if f.endsWith(".json") => parseJson
      case f if f.endsWith(".csv") => parseCsv
      case f => throw new RuntimeException(s"Unknown format: $f")
    }
  
  def parseJson(file: String): List[Person] =
    JsonMethods.parse(StreamInput(this.getClass.getResourceAsStream(file))).extract[List[Person]]
  
  def parseCsv(file: String): List[Person] =
    CSVReader.open(new InputStreamReader(this.getClass.getResourceAsStream(file))).all().map {
      case List(name, age, address) =>
        Person(name, age.toInt, address)
    }
}

object StrategyExample {
  def main(args: Array[String]): Unit = {
    val applicationCsv = new Application[Person](StrategyFactory("people.csv"))
    val applicationJson = new Application[Person](StrategyFactory("people.json"))

    System.out.println("Using the csv: ")
    applicationCsv.write("people.csv")

    System.out.println("Using the json: ")
    applicationJson.write("people.json")
  }
}

org.json4s.jackson.JsonMethods Scala Examples