com.google.protobuf.Message Scala Examples

// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package com.daml.platform.apiserver

import java.io.IOException
import java.net.{BindException, InetAddress, InetSocketAddress}
import java.util.concurrent.TimeUnit.SECONDS

import com.daml.metrics.Metrics
import com.daml.platform.apiserver.GrpcServerOwner._
import com.daml.ports.Port
import com.daml.resources.{Resource, ResourceOwner}
import com.google.protobuf.Message
import io.grpc.netty.NettyServerBuilder
import io.grpc._
import io.netty.channel.socket.nio.NioServerSocketChannel
import io.netty.handler.ssl.SslContext

import scala.concurrent.{ExecutionContext, Future}
import scala.util.control.NoStackTrace

final class GrpcServerOwner(
    address: Option[String],
    desiredPort: Port,
    maxInboundMessageSize: Int,
    sslContext: Option[SslContext] = None,
    interceptors: List[ServerInterceptor] = List.empty,
    metrics: Metrics,
    eventLoopGroups: ServerEventLoopGroups,
    services: Iterable[BindableService],
) extends ResourceOwner[Server] {
  override def acquire()(implicit executionContext: ExecutionContext): Resource[Server] = {
    val host = address.map(InetAddress.getByName).getOrElse(InetAddress.getLoopbackAddress)
    Resource(Future {
      val builder = NettyServerBuilder.forAddress(new InetSocketAddress(host, desiredPort.value))
      builder.sslContext(sslContext.orNull)
      builder.channelType(classOf[NioServerSocketChannel])
      builder.permitKeepAliveTime(10, SECONDS)
      builder.permitKeepAliveWithoutCalls(true)
      builder.directExecutor()
      builder.maxInboundMessageSize(maxInboundMessageSize)
      interceptors.foreach(builder.intercept)
      builder.intercept(new MetricsInterceptor(metrics))
      eventLoopGroups.populate(builder)
      services.foreach { service =>
        builder.addService(service)
        toLegacyService(service).foreach(builder.addService)
      }
      val server = builder.build()
      try {
        server.start()
      } catch {
        case e: IOException if e.getCause != null && e.getCause.isInstanceOf[BindException] =>
          throw new UnableToBind(desiredPort, e.getCause)
      }
      server
    })(server => Future(server.shutdown().awaitTermination()))
  }

  // This exposes the existing services under com.daml also under com.digitalasset.
  // This is necessary to allow applications built with an earlier version of the SDK
  // to still work.
  // The "proxy" services will not show up on the reflection service, because of the way it
  // processes service definitions via protobuf file descriptors.
  private def toLegacyService(service: BindableService): Option[ServerServiceDefinition] = {
    val `com.daml` = "com.daml"
    val `com.digitalasset` = "com.digitalasset"

    val damlDef = service.bindService()
    val damlDesc = damlDef.getServiceDescriptor
    // Only add "proxy" services if it actually contains com.daml in the service name.
    // There are other services registered like the reflection service, that doesn't need the special treatment.
    if (damlDesc.getName.contains(`com.daml`)) {
      val digitalassetName = damlDesc.getName.replace(`com.daml`, `com.digitalasset`)
      val digitalassetDef = ServerServiceDefinition.builder(digitalassetName)
      damlDef.getMethods.forEach { methodDef =>
        val damlMethodDesc = methodDef.getMethodDescriptor
        val digitalassetMethodName =
          damlMethodDesc.getFullMethodName.replace(`com.daml`, `com.digitalasset`)
        val digitalassetMethodDesc =
          damlMethodDesc.toBuilder.setFullMethodName(digitalassetMethodName).build()
        val _ = digitalassetDef.addMethod(
          digitalassetMethodDesc.asInstanceOf[MethodDescriptor[Message, Message]],
          methodDef.getServerCallHandler.asInstanceOf[ServerCallHandler[Message, Message]]
        )
      }
      Option(digitalassetDef.build())
    } else None
  }
}

object GrpcServerOwner {

  final class UnableToBind(port: Port, cause: Throwable)
      extends RuntimeException(
        s"The API server was unable to bind to port $port. Terminate the process occupying the port, or choose a different one.",
        cause)
      with NoStackTrace

} 

package com.twitter.finagle.protobuf.rpc.impl

import com.twitter.finagle.protobuf.rpc.RpcFactory
import com.twitter.finagle.protobuf.rpc.RpcControllerWithOnFailureCallback
import com.twitter.finagle.protobuf.rpc.RpcServer
import com.twitter.finagle.builder.ServerBuilder
import com.twitter.finagle.builder.ClientBuilder
import com.twitter.util.Duration
import com.google.protobuf.RpcController
import com.google.protobuf.RpcChannel
import com.google.protobuf.Message
import com.google.protobuf.Service
import java.util.concurrent.ExecutorService
import com.twitter.finagle.protobuf.rpc.ServiceExceptionHandler
import com.twitter.finagle.protobuf.rpc.ExceptionResponseHandler

class RpcFactoryImpl extends RpcFactory {

  def createServer(sb: ServerBuilder[(String, Message), (String, Message), Any, Any, Any], port: Int, service: Service, handler: ServiceExceptionHandler[Message], executorService: ExecutorService): RpcServer = new RpcServerImpl(sb, port, service, handler, executorService)

  def createStub[T <: Service](cb: ClientBuilder[(String, Message), (String, Message), Any, Any, Any], service: {def newStub(c: RpcChannel): T}, handler: ExceptionResponseHandler[Message], executorService: ExecutorService): T = {
    service.newStub(new RpcChannelImpl(cb, service.asInstanceOf[T], handler, executorService))
  }

  def createController(): RpcController = {
    new RpcControllerWithOnFailureCallback()
  }

  def release(stub: {def getChannel(): RpcChannel}) {
    stub.getChannel().asInstanceOf[RpcChannelImpl].release()
  }
} 

package scalapb.json4s

import com.google.protobuf.{InvalidProtocolBufferException, Message}
import jsontest.test.{EnumTest, MyEnum}
import jsontest.test3.EnumTest3
import jsontest.test3.MyTest3.MyEnum3
import org.scalatest.Assertion
import scalapb.GeneratedMessageCompanion
import scalapb.JavaProtoSupport
import com.google.protobuf.util.JsonFormat.{Parser => JavaParser}
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.must.Matchers

class EnumFormatSpec extends AnyFlatSpec with Matchers with JavaAssertions {
  // not ignoring unknown fields:

  "default parser" should "match Java behavior for string enums" in new DefaultParserContext {
    assertFails("""{"enum":"ZAZA"}""", EnumTest)
    assertFails("""{"enum":"ZAZA"}""", EnumTest3)
    assertFails("""{"enum":""}""", EnumTest)
    assertFails("""{"enum":""}""", EnumTest3)
    assertParse("""{"enum":"V1"}""", EnumTest(Some(MyEnum.V1)))
    assertParse("""{"enum":"V1"}""", EnumTest3(MyEnum3.V1))
    assertParse("""{"enum":"0"}""", EnumTest(Some(MyEnum.UNKNOWN)))
    assertParse("""{"enum":"0"}""", EnumTest3())
    assertParse("""{"enum":"1.0"}""", EnumTest(Some(MyEnum.V1)))
    assertParse("""{"enum":"1.0"}""", EnumTest3(MyEnum3.V1))
    assertFails("""{"enum":"1.4"}""", EnumTest)
    assertFails("""{"enum":"1.4"}""", EnumTest3)
    assertFails("""{"enum":"10"}""", EnumTest)
    assertParse("""{"enum":"10"}""", EnumTest3(MyEnum3.Unrecognized(10)))
  }

  "default parser" should "match Java behavior for int enums" in new DefaultParserContext {
    assertFails("""{"enum":10}""", EnumTest)
    assertParse("""{"enum":10}""", EnumTest3(MyEnum3.Unrecognized(10)))
    assertParse("""{"enum":0}""", EnumTest(Some(MyEnum.UNKNOWN)))
    assertParse("""{"enum":0}""", EnumTest3(MyEnum3.UNKNOWN))
    assertParse("""{"enum":0.0}""", EnumTest(Some(MyEnum.UNKNOWN)))
    assertParse("""{"enum":0.0}""", EnumTest3(MyEnum3.UNKNOWN))
    assertFails("""{"enum":0.4}""", EnumTest)
    assertFails("""{"enum":0.4}""", EnumTest3)
    assertParse("""{"enum":1}""", EnumTest(Some(MyEnum.V1)))
    assertParse("""{"enum":1}""", EnumTest3(MyEnum3.V1))
    assertParse("""{"enum":1.0}""", EnumTest(Some(MyEnum.V1)))
    assertParse("""{"enum":1.0}""", EnumTest3(MyEnum3.V1))
    assertFails("""{"enum":-1}""", EnumTest)
    assertParse("""{"enum":-1}""", EnumTest3(MyEnum3.Unrecognized(-1)))
  }

  "ignoring unknown fields parser" should "match Java behavior for strings enums" in new IgnoringUnknownParserContext {
    assertParse("""{"enum":"ZAZA"}""", EnumTest())
    assertParse("""{"enum":"ZAZA"}""", EnumTest3())
    assertParse("""{"enum":""}""", EnumTest())
    assertParse("""{"enum":""}""", EnumTest3())
    assertParse("""{"enum":"V1"}""", EnumTest(Some(MyEnum.V1)))
    assertParse("""{"enum":"V1"}""", EnumTest3(MyEnum3.V1))
    assertParse("""{"enum":"0"}""", EnumTest(Some(MyEnum.UNKNOWN)))
    assertParse("""{"enum":"0"}""", EnumTest3())
    assertParse("""{"enum":"1.0"}""", EnumTest(Some(MyEnum.V1)))
    assertParse("""{"enum":"1.0"}""", EnumTest3(MyEnum3.V1))
    assertParse("""{"enum":"1.4"}""", EnumTest())
    assertParse("""{"enum":"1.4"}""", EnumTest3())
    assertParse("""{"enum":"10"}""", EnumTest())
    assertParse("""{"enum":"10"}""", EnumTest3(MyEnum3.Unrecognized(10)))
  }

  "ignoring unknown fields parser" should "match Java behavior for int enums" in new IgnoringUnknownParserContext {
    assertParse("""{"enum":10}""", EnumTest())
    assertParse("""{"enum":10}""", EnumTest3(MyEnum3.Unrecognized(10)))
    assertParse("""{"enum":0}""", EnumTest(Some(MyEnum.UNKNOWN)))
    assertParse("""{"enum":0}""", EnumTest3(MyEnum3.UNKNOWN))
    assertParse("""{"enum":0.0}""", EnumTest(Some(MyEnum.UNKNOWN)))
    assertParse("""{"enum":0.0}""", EnumTest3(MyEnum3.UNKNOWN))
    assertParse("""{"enum":0.4}""", EnumTest())
    assertParse("""{"enum":0.4}""", EnumTest())
    assertParse("""{"enum":1}""", EnumTest(Some(MyEnum.V1)))
    assertParse("""{"enum":1}""", EnumTest3(MyEnum3.V1))
    assertParse("""{"enum":1.0}""", EnumTest(Some(MyEnum.V1)))
    assertParse("""{"enum":1.0}""", EnumTest3(MyEnum3.V1))
    assertParse("""{"enum":-1}""", EnumTest())
    assertParse("""{"enum":-1}""", EnumTest3(MyEnum3.Unrecognized(-1)))
  }

  "Enum" should "be serialized the same way as java" in {
    assertJsonIsSameAsJava(jsontest.test.EnumTest())
    assertJsonIsSameAsJava(jsontest.test.EnumTest(Some(MyEnum.V1)))
  }
} 

package com.twilio.open.streaming.trend.discovery

import java.io.{ByteArrayInputStream, InputStream}
import java.nio.charset.StandardCharsets

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.google.protobuf.Message
import com.googlecode.protobuf.format.JsonFormat
import com.holdenkarau.spark.testing.{LocalSparkContext, SparkContextProvider}
import com.twilio.open.protocol.Calls.CallEvent
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.SparkSession
import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers, Suite}
import org.slf4j.{Logger, LoggerFactory}

import scala.collection.Seq
import scala.io.Source
import scala.reflect.ClassTag
import scala.reflect.classTag

object TestHelper {
  val log: Logger = LoggerFactory.getLogger("com.twilio.open.streaming.trend.discovery.TestHelper")
  val mapper: ObjectMapper = {
    val m = new ObjectMapper()
    m.registerModule(DefaultScalaModule)
  }

  val jsonFormat: JsonFormat = new JsonFormat

  def loadScenario[T<: Message : ClassTag](file: String): Seq[T] = {
    val fileString = Source.fromFile(file).mkString
    val parsed = mapper.readValue(fileString, classOf[Sceanario])
    parsed.input.map { data =>
      val json = mapper.writeValueAsString(data)
      convert[T](json)
    }
  }

  def convert[T<: Message : ClassTag](json: String): T = {
    val clazz = classTag[T].runtimeClass
    val builder = clazz.getMethod("newBuilder").invoke(clazz).asInstanceOf[Message.Builder]
    try {
      val input: InputStream = new ByteArrayInputStream(json.getBytes())
      jsonFormat.merge(input, builder)
      builder.build().asInstanceOf[T]
    } catch {
      case e: Exception =>
        throw e
    }
  }

  def asMockKafkaDataFrame(event: CallEvent): MockKafkaDataFrame = {
    val key = event.getEventId.getBytes(StandardCharsets.UTF_8)
    val value = event.toByteArray
    MockKafkaDataFrame(key, value)
  }

}

case class MockKafkaDataFrame(key: Array[Byte], value: Array[Byte])


@SerialVersionUID(1L)
case class KafkaDataFrame(key: Array[Byte], topic: Array[Byte], value: Array[Byte]) extends Serializable

case class Sceanario(input: Seq[Any], expected: Option[Any] = None)

trait SparkSqlTest extends BeforeAndAfterAll with SparkContextProvider {
  self: Suite =>

  @transient var _sparkSql: SparkSession = _
  @transient private var _sc: SparkContext = _

  override def sc: SparkContext = _sc

  def conf: SparkConf

  def sparkSql: SparkSession = _sparkSql

  override def beforeAll() {
    _sparkSql = SparkSession.builder().config(conf).getOrCreate()

    _sc = _sparkSql.sparkContext
    setup(_sc)
    super.beforeAll()
  }

  override def afterAll() {
    try {
      _sparkSql.close()
      _sparkSql = null
      LocalSparkContext.stop(_sc)
      _sc = null
    } finally {
      super.afterAll()
    }
  }

} 

package com.github.phisgr.gatling

import com.google.protobuf.Message
import io.gatling.commons.validation.{Failure, Success, Validation}
import io.gatling.core.Predef.value2Expression
import io.gatling.core.session.{Expression, Session}

package object javapb {
  private type BuilderMutation[B] = (B, Session) => Failure // nullable

  private case class MutationWithExpression[B <: Message.Builder, F](setter: B => F => Message.Builder, e: Expression[F])
    extends BuilderMutation[B] {
    override def apply(builder: B, session: Session): Failure = {
      e(session) match {
        case Success(value) =>
          setter(builder)(value)
          null
        case f@Failure(_) =>
          f
      }
    }
  }

  private[javapb] class MessageExpressionUpdater[M <: Message, B <: Message.Builder](
    private[javapb] val original: Expression[M],
    private[javapb] val reversedMutations: List[BuilderMutation[B]]
  ) {
    def update[Field](setter: B => Field => Message.Builder)(value: Expression[Field]) =
      new MessageExpressionUpdater[M, B](original, MutationWithExpression(setter, value) :: reversedMutations)
  }

  implicit def toExpression[M <: Message, B <: Message.Builder](u: MessageExpressionUpdater[M, B]): Expression[M] = {
    val mutations = u.reversedMutations.reverse.toArray
    val size = mutations.length
    val original = u.original

    s =>
      original(s) match {
        case Success(message) =>
          val builder = message.toBuilder.asInstanceOf[B]
          var ret: Validation[M] = null
          var i = 0
          do {
            if (i < size) {
              ret = mutations(i)(builder, s)
              i += 1
            } else {
              ret = Success(builder.build().asInstanceOf[M])
            }
          } while (ret eq null)
          ret
        case f@Failure(_) => f
      }
  }

  type BuilderOf[M <: Message] = {def build(): M}

  implicit def message2ExprUpdater[M <: Message, B <: Message.Builder with BuilderOf[M]](e: M)(implicit evidence: BuilderEvidence[M, B]): MessageExpressionUpdater[M, B] =
    expr2exprUpdater(value2Expression(e))

  implicit def expr2exprUpdater[M <: Message, B <: Message.Builder with BuilderOf[M]](e: Expression[M])(implicit evidence: BuilderEvidence[M, B]): MessageExpressionUpdater[M, B] =
    new MessageExpressionUpdater[M, B](e, Nil)

  implicit def builderEvidence[M <: Message, B]: BuilderEvidence[M, B] = macro BuilderEvidence.impl[M, B]

} 

package me.lyh.protobuf.generic.test

import java.io.ByteArrayInputStream
import java.nio.ByteBuffer

import com.google.protobuf.{ByteString, Message}
import me.lyh.protobuf.generic._
import me.lyh.protobuf.generic.proto2.Schemas._

import scala.reflect.ClassTag
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class ProtobufGenericSpec extends AnyFlatSpec with Matchers {
  def roundTrip[T <: Message: ClassTag](record: T): Unit = {
    val schema = SerializableUtils.ensureSerializable(Schema.of[T])
    val schemaCopy = Schema.fromJson(schema.toJson)
    schemaCopy shouldBe schema

    val reader = SerializableUtils.ensureSerializable(GenericReader.of(schema))
    val writer = SerializableUtils.ensureSerializable(GenericWriter.of(schema))
    val jsonRecord = reader.read(record.toByteArray).toJson
    jsonRecord shouldBe reader.read(ByteBuffer.wrap(record.toByteArray)).toJson
    jsonRecord shouldBe reader.read(new ByteArrayInputStream(record.toByteArray)).toJson
    val bytes = writer.write(GenericRecord.fromJson(jsonRecord))

    val recordCopy = ProtobufType[T].parseFrom(bytes)
    recordCopy shouldBe record
  }

  "ProtobufGeneric" should "round trip required" in {
    roundTrip[Required](Records.required)
  }

  it should "round trip optional" in {
    roundTrip[Optional](Records.optional)
    roundTrip[Optional](Records.optionalEmpty)
  }

  it should "round trip repeated" in {
    roundTrip[Repeated](Records.repeated)
    roundTrip[Repeated](Records.repeatedEmpty)
    roundTrip[RepeatedPacked](Records.repeatedPacked)
    roundTrip[RepeatedUnpacked](Records.repeatedUnpacked)
  }

  it should "round trip oneofs" in {
    Records.oneOfs.foreach(roundTrip[OneOf])
  }

  it should "round trip mixed" in {
    roundTrip[Mixed](Records.mixed)
    roundTrip[Mixed](Records.mixedEmpty)
  }

  it should "round trip nested" in {
    roundTrip[Nested](Records.nested)
    roundTrip[Nested](Records.nestedEmpty)
  }

  it should "round trip with custom options" in {
    roundTrip[CustomOptionMessage](Records.customOptionMessage)
    roundTrip[CustomOptionMessage](Records.customOptionMessageEmpty)
  }

  it should "round trip with custom defaults" in {
    roundTrip[CustomDefaults](CustomDefaults.getDefaultInstance)
  }

  it should "populate default values" in {
    val schema = Schema.of[CustomDefaults]
    val record = GenericReader.of(schema).read(CustomDefaults.getDefaultInstance.toByteArray)
    record.get("double_field") shouldBe 101.0
    record.get("float_field") shouldBe 102.0f
    record.get("int32_field") shouldBe 103
    record.get("int64_field") shouldBe 104L
    record.get("uint32_field") shouldBe 105
    record.get("uint64_field") shouldBe 106L
    record.get("sint32_field") shouldBe 107
    record.get("sint64_field") shouldBe 108L
    record.get("fixed32_field") shouldBe 109
    record.get("fixed64_field") shouldBe 110L
    record.get("sfixed32_field") shouldBe 111
    record.get("sfixed64_field") shouldBe 112L
    record.get("bool_field") shouldBe true
    record.get("string_field") shouldBe "hello"
    record.get("bytes_field") shouldBe
      Base64.encode(ByteString.copyFromUtf8("world").toByteArray)
    record.get("color_field") shouldBe "GREEN"
  }
} 

package me.lyh.protobuf.generic.test

import com.google.protobuf.Message
import com.google.protobuf.util.JsonFormat
import me.lyh.protobuf.generic._
import me.lyh.protobuf.generic.proto3.Schemas._

import scala.reflect.ClassTag
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

class ProtobufGenericSpec extends AnyFlatSpec with Matchers {
  private val printer = JsonFormat.printer().preservingProtoFieldNames()
  private val parser = JsonFormat.parser()

  def roundTrip[T <: Message: ClassTag](record: T): Unit = {
    val schema = SerializableUtils.ensureSerializable(Schema.of[T])
    val schemaCopy = Schema.fromJson(schema.toJson)
    schemaCopy shouldBe schema

    val reader = SerializableUtils.ensureSerializable(GenericReader.of(schema))
    val writer = SerializableUtils.ensureSerializable(GenericWriter.of(schema))
    val jsonRecord = reader.read(record.toByteArray).toJson
    val bytes = writer.write(GenericRecord.fromJson(jsonRecord))
    val recordCopy = ProtobufType[T].parseFrom(bytes)
    recordCopy shouldBe record

    compatibleWithJsonFormat(record)
  }

  def compatibleWithJsonFormat[T <: Message: ClassTag](record: T): Unit = {
    val protoType = ProtobufType[T]
    val schema = Schema.of[T]
    val reader = GenericReader.of(schema)
    val writer = GenericWriter.of(schema)

    val json1 = reader.read(record.toByteArray).toJson
    val record1 = {
      val builder = protoType.newBuilder()
      parser.merge(json1, builder)
      builder.build().asInstanceOf[T]
    }

    val json2 = printer.print(record)
    val record2 = protoType.parseFrom(writer.write(GenericRecord.fromJson(json2)))

    record1 shouldBe record2
  }

  def test[T <: Message: ClassTag](record: T): Unit = {
    roundTrip(record)
    compatibleWithJsonFormat(record)
  }

  "ProtobufGeneric" should "round trip optional" in {
    test[Optional](Records.optional)
    test[Optional](Records.optionalEmpty)
  }

  it should "round trip repeated" in {
    test[Repeated](Records.repeated)
    test[Repeated](Records.repeatedEmpty)
    test[RepeatedPacked](Records.repeatedPacked)
    test[RepeatedUnpacked](Records.repeatedUnpacked)
  }

  it should "round trip oneofs" in {
    Records.oneOfs.foreach(test[OneOf])
  }

  it should "round trip mixed" in {
    test[Mixed](Records.mixed)
    test[Mixed](Records.mixedEmpty)
  }

  it should "round trip nested" in {
    test[Nested](Records.nested)
    test[Nested](Records.nestedEmpty)
  }

  it should "round trip with custom options" in {
    test[CustomOptionMessage](Records.customOptionMessage)
    test[CustomOptionMessage](Records.customOptionMessageEmpty)
  }
} 

package me.lyh.protobuf.generic.test

import com.google.protobuf.{ByteString, Message}
import me.lyh.protobuf.generic._
import me.lyh.protobuf.generic.proto3.Schemas._
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

import scala.reflect.ClassTag

class FieldReaderSpec extends AnyFlatSpec with Matchers {
  def read[T <: Message: ClassTag](record: T, fields: List[String], expected: List[Any]): Unit = {
    val schema = SerializableUtils.ensureSerializable(Schema.of[T])
    val reader = SerializableUtils.ensureSerializable(FieldReader.of(schema, fields))
    val actual = reader.read(record.toByteArray)
    actual.toList shouldBe expected
  }

  val fields: List[String] = List(
    "double_field",
    "float_field",
    "int32_field",
    "int64_field",
    "uint32_field",
    "uint64_field",
    "sint32_field",
    "sint64_field",
    "fixed32_field",
    "fixed64_field",
    "sfixed32_field",
    "sfixed64_field",
    "bool_field",
    "string_field",
    "bytes_field",
    "color_field"
  )

  val expected: List[Any] = List(
    math.Pi,
    math.E.toFloat,
    10,
    15,
    20,
    25,
    30,
    35,
    40,
    45,
    50,
    55,
    true,
    "hello",
    ByteString.copyFromUtf8("world"),
    "WHITE"
  )

  "FieldReader" should "read optional" in {
    read[Optional](Records.optional, fields, expected)
    read[Optional](
      Records.optionalEmpty,
      fields,
      List(0.0, 0.0f, 0, 0L, 0, 0L, 0, 0L, 0, 0L, 0, 0L, false, "", ByteString.EMPTY, "BLACK")
    )
  }

  it should "read oneofs" in {
    (Records.oneOfs.drop(1) zip (fields zip expected)).foreach {
      case (r, (f, e)) =>
        read[OneOf](r, List(f), List(e))
    }
  }

  it should "read nested" in {
    val fields = List(
      "mixed_field_o.double_field_o",
      "mixed_field_o.string_field_o",
      "mixed_field_o.bytes_field_o",
      "mixed_field_o.color_field_o"
    )
    val expected = List(math.Pi, "hello", ByteString.copyFromUtf8("world"), "WHITE")
    read[Nested](Records.nested, fields, expected)

    val expectedEmpty = List(0.0, "", ByteString.EMPTY, "BLACK")
    read[Nested](Records.nestedEmpty, fields, expectedEmpty)
  }
} 

package com.twilio.open.streaming.trend.discovery

import java.io.{ByteArrayInputStream, InputStream}

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.google.protobuf.Message
import com.googlecode.protobuf.format.JsonFormat
import com.holdenkarau.spark.testing.{LocalSparkContext, SparkContextProvider}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.SparkSession
import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers, Suite}
import org.slf4j.{Logger, LoggerFactory}

import scala.collection.Seq
import scala.io.Source
import scala.reflect.ClassTag
import scala.reflect.classTag

object TestHelper {
  val log: Logger = LoggerFactory.getLogger("com.twilio.open.streaming.trend.discovery.TestHelper")
  val mapper: ObjectMapper = {
    val m = new ObjectMapper()
    m.registerModule(DefaultScalaModule)
  }

  val jsonFormat: JsonFormat = new JsonFormat

  def loadScenario[T<: Message : ClassTag](file: String): Seq[T] = {
    val fileString = Source.fromFile(file).mkString
    val parsed = mapper.readValue(fileString, classOf[Sceanario])
    parsed.input.map { data =>
      val json = mapper.writeValueAsString(data)
      convert[T](json)
    }
  }

  def convert[T<: Message : ClassTag](json: String): T = {
    val clazz = classTag[T].runtimeClass
    val builder = clazz.getMethod("newBuilder").invoke(clazz).asInstanceOf[Message.Builder]
    try {
      val input: InputStream = new ByteArrayInputStream(json.getBytes())
      jsonFormat.merge(input, builder)
      builder.build().asInstanceOf[T]
    } catch {
      case e: Exception =>
        throw e
    }
  }

}

@SerialVersionUID(1L)
case class KafkaDataFrame(key: Array[Byte], topic: Array[Byte], value: Array[Byte]) extends Serializable

case class Sceanario(input: Seq[Any], expected: Option[Any] = None)

trait SparkSqlTest extends BeforeAndAfterAll with SparkContextProvider {
  self: Suite =>

  @transient var _sparkSql: SparkSession = _
  @transient private var _sc: SparkContext = _

  override def sc: SparkContext = _sc

  def conf: SparkConf

  def sparkSql: SparkSession = _sparkSql

  override def beforeAll() {
    _sparkSql = SparkSession.builder().config(conf).getOrCreate()

    _sc = _sparkSql.sparkContext
    setup(_sc)
    super.beforeAll()
  }

  override def afterAll() {
    try {
      _sparkSql.close()
      _sparkSql = null
      LocalSparkContext.stop(_sc)
      _sc = null
    } finally {
      super.afterAll()
    }
  }

} 

package com.twitter.finagle.protobuf.rpc.impl

import java.net.InetSocketAddress
import com.google.protobuf.Descriptors.MethodDescriptor
import com.google.protobuf.RpcCallback
import com.google.protobuf.Message
import com.google.protobuf.RpcChannel
import com.google.protobuf.RpcController
import com.google.protobuf.Service
import java.util.logging.Logger
import com.twitter.util.Duration
import com.twitter.util.FuturePool
import com.twitter.finagle.builder.ClientBuilder
import java.util.concurrent.ExecutorService
import com.twitter.finagle.protobuf.rpc.RpcControllerWithOnFailureCallback
import com.twitter.finagle.protobuf.rpc.channel.ProtoBufCodec
import com.twitter.finagle.ChannelClosedException
import com.twitter.finagle.protobuf.rpc.Util
import com.twitter.finagle.protobuf.rpc.ExceptionResponseHandler

class RpcChannelImpl(cb: ClientBuilder[(String, Message), (String, Message), Any, Any, Any], s: Service, handler: ExceptionResponseHandler[Message], executorService: ExecutorService) extends RpcChannel {

  private val log = Logger.getLogger(getClass.toString)

  private val futurePool = FuturePool(executorService)

  private val client: com.twitter.finagle.Service[(String, Message), (String, Message)] = cb
    .codec(new ProtoBufCodec(s))
    .unsafeBuild()

  def callMethod(m: MethodDescriptor, controller: RpcController,
                 request: Message, responsePrototype: Message,
                 done: RpcCallback[Message]): Unit = {
    // retries is a workaround for ChannelClosedException raised when servers shut down.
    val retries = 3

    callMethod(m, controller, request, responsePrototype, done, retries)
  }

  def callMethod(m: MethodDescriptor, controller: RpcController,
                 request: Message, responsePrototype: Message,
                 done: RpcCallback[Message], retries: Int): Unit = {

    Util.log("Request", m.getName(), request)
    val req = (m.getName(), request)

    client(req) onSuccess {
      result =>
        Util.log("Response", m.getName(), result._2)
        futurePool({
          handle(done, controller, result._2)
        })
    } onFailure {
      e =>
        log.warning("#callMethod# Failed. "+ e.getMessage)
        e match {
          case cc: ChannelClosedException => if (retries > 1) {
            log.warning("#callMethod# Retrying.")
            callMethod(m, controller, request, responsePrototype, done, retries - 1);
          } else {
            controller.asInstanceOf[RpcControllerWithOnFailureCallback].setFailed(e)
          }
          case _ => controller.asInstanceOf[RpcControllerWithOnFailureCallback].setFailed(e)
        }
    }
  }

  def handle(done: RpcCallback[Message], controller: RpcController, m: Message) {
    if (handler.canHandle(m)) {
      controller.asInstanceOf[RpcControllerWithOnFailureCallback].setFailed(handler.handle(m))
    } else {
      done.run(m)
    }
  }

  def release() {
     client.close()
  }
} 

package com.twitter.finagle.protobuf.rpc

import java.util.List

import com.google.common.base.Function
import com.google.common.collect.Lists
import com.google.protobuf.Descriptors.MethodDescriptor
import com.google.protobuf.Service
import com.google.protobuf.Message

import scala.collection.JavaConversions._
import scala.collection.mutable._

import org.slf4j.LoggerFactory

object Util {

  private val log = LoggerFactory.getLogger(getClass)

  def extractMethodNames(s: Service): List[String] = {
    return Lists.transform(s.getDescriptorForType().getMethods(),
      new Function[MethodDescriptor, String]() {

        @Override
        def apply(d: MethodDescriptor): String = {
          return d.getName()
        }
      })
  }

  def log(reqOrResp: String, method: String, m: Message) {
    if (log.isDebugEnabled()) {
      log.debug("#log# {} {}: {}", Array[AnyRef](reqOrResp, method, m.toString()))
    }
  }
} 

package com.twitter.finagle.protobuf.rpc.channel

import org.jboss.netty.buffer.ChannelBuffer
import org.jboss.netty.buffer.ChannelBufferInputStream
import org.jboss.netty.channel.Channel
import org.jboss.netty.channel.ChannelHandlerContext
import org.jboss.netty.handler.codec.frame.FrameDecoder

import com.google.protobuf.Message
import com.google.protobuf.Service


class ClientSideDecoder(val repo: MethodLookup, val service: Service) extends FrameDecoder with ProtobufDecoder {

  @throws(classOf[Exception])
  def decode(ctx: ChannelHandlerContext, channel: Channel, buf: ChannelBuffer): Object = {
    decode(ctx, channel, buf, repo)
  }

  def getPrototype(methodName: String): Message = {
    val m = service.getDescriptorForType().findMethodByName(methodName)
    service.getResponsePrototype(m)
  }
} 

package com.twitter.finagle.protobuf.rpc.channel

import org.jboss.netty.channel.ChannelPipelineFactory
import org.jboss.netty.channel.Channels
import com.google.protobuf.Message
import com.google.protobuf.Service
import com.twitter.conversions.storage.intToStorageUnitableWholeNumber
import com.twitter.finagle.Codec
import com.twitter.finagle.CodecFactory

class ProtoBufCodec(val service: Service) extends CodecFactory[(String, Message), (String, Message)] {

  val maxFrameSize = 1.megabytes.inBytes.intValue

  val repo = SimpleMethodLookup(service)

  def server = Function.const {
    new Codec[(String, Message), (String, Message)] {
      def pipelineFactory = new ChannelPipelineFactory {
        def getPipeline = {
          val pipeline = Channels.pipeline()
          pipeline.addLast("decoder", new ServerSideDecoder(repo, service))
          pipeline.addLast("encoder", new CustomProtobufEncoder(repo));
          pipeline
        }
      }
    }
  }

  def client = Function.const {
    new Codec[(String, Message), (String, Message)] {
      def pipelineFactory = new ChannelPipelineFactory {
        def getPipeline = {
          val pipeline = Channels.pipeline()
          pipeline.addLast("encoder", new CustomProtobufEncoder(repo))
          pipeline.addLast("decoder",
            new ClientSideDecoder(repo, service))
          pipeline
        }
      }
    }
  }

} 

package com.twitter.finagle.protobuf.rpc.channel

import org.jboss.netty.buffer.ChannelBuffer
import org.jboss.netty.channel.Channel
import org.jboss.netty.channel.ChannelHandlerContext
import org.jboss.netty.handler.codec.frame.FrameDecoder

import com.google.protobuf.Message
import com.google.protobuf.Service

class ServerSideDecoder(val repo: MethodLookup, val service: Service) extends FrameDecoder with ProtobufDecoder {

  @throws(classOf[Exception])
  def decode(ctx: ChannelHandlerContext, channel: Channel, buf: ChannelBuffer): Object = {
    decode(ctx, channel, buf, repo)
  }

  def getPrototype(methodName: String): Message = {
    val m = service.getDescriptorForType().findMethodByName(methodName)
    service.getRequestPrototype(m)
  }
} 

package akka.stream.integration

import akka.camel.CamelMessage
import com.google.protobuf.Message

import scalaz.Semigroup

trait ProtobufCamelMessageExtractor {
  implicit def protobufCamelMessageExtractor[Out: ProtobufReader](implicit headersExtractor: HeadersExtractor[Out] = null, semigroup: Semigroup[Out] = null) = new CamelMessageExtractor[Out] {
    override def extract(in: CamelMessage): Out = {
      val msg = in.body.asInstanceOf[Message]
      val out = implicitly[ProtobufReader[Out]].read(msg)
      (for {
        extractor <- Option(headersExtractor)
        semigroup <- Option(semigroup)
      } yield semigroup.append(out, extractor.extract(in.headers))).getOrElse(out)
    }
  }
}

object ProtobufCamelMessageExtractor extends ProtobufCamelMessageExtractor 

package com.github.dnvriend.persistence

import com.github.dnvriend.domain.Person.{ NameChangedEvent, NameRegisteredEvent, SurnameChangedEvent }
import com.google.protobuf.Message
import proto.person.Command
import proto.person.command.{ NameChangedMessage, NameRegisteredMessage, SurnameChangedMessage }

object ProtobufFormats {
  implicit def strToOptionString(str: String): Option[String] = Option(str)
  implicit def optionStrToString(opt: Option[String]): String = opt.getOrElse("")

  implicit val surnameChangedProtobufProtocol = new ProtobufFormat[SurnameChangedEvent] {
    override def write(event: SurnameChangedEvent): Message =
      SurnameChangedMessage.toJavaProto(SurnameChangedMessage(event.surname))

    override def read(proto: Message): SurnameChangedEvent = proto match {
      case p: Command.SurnameChangedMessage ⇒
        val msg = SurnameChangedMessage.fromJavaProto(p)
        SurnameChangedEvent(msg.surname)
    }
  }

  implicit val nameChangedProtobufProtocol = new ProtobufFormat[NameChangedEvent] {
    override def write(event: NameChangedEvent): Message =
      NameChangedMessage.toJavaProto(NameChangedMessage(event.name))

    override def read(proto: Message): NameChangedEvent = proto match {
      case p: Command.NameChangedMessage ⇒
        val msg = NameChangedMessage.fromJavaProto(p)
        NameChangedEvent(msg.name)
    }
  }

  implicit val nameRegisteredProtobufProtocol = new ProtobufFormat[NameRegisteredEvent] {
    override def write(event: NameRegisteredEvent): Message = {
      NameRegisteredMessage.toJavaProto(NameRegisteredMessage(event.name, event.surname))
    }

    override def read(proto: Message): NameRegisteredEvent = proto match {
      case p: Command.NameRegisteredMessage ⇒
        val msg = NameRegisteredMessage.fromJavaProto(p)
        NameRegisteredEvent(msg.name, msg.surname)
    }
  }
} 

package com.github.dnvriend.persistence

import akka.persistence.journal.{ EventSeq, ReadEventAdapter }
import com.github.dnvriend.domain.Person.{ NameChangedEvent, NameRegisteredEvent, SurnameChangedEvent }
import com.google.protobuf.Message

class ProtobufReadEventAdapter extends ReadEventAdapter {
  import ProtobufFormats._

  protected def deserialize[A: ProtobufReader](proto: Message): EventSeq =
    EventSeq.single(implicitly[ProtobufReader[A]].read(proto))

  override def fromJournal(event: Any, manifest: String): EventSeq = event match {
    case proto: Message if manifest == classOf[NameRegisteredEvent].getSimpleName ⇒ deserialize[NameRegisteredEvent](proto)
    case proto: Message if manifest == classOf[NameChangedEvent].getSimpleName    ⇒ deserialize[NameChangedEvent](proto)
    case proto: Message if manifest == classOf[SurnameChangedEvent].getSimpleName ⇒ deserialize[SurnameChangedEvent](proto)
    case _                                                                        ⇒ throw new RuntimeException(s"[${this.getClass.getName}]Cannot deserialize '${event.getClass.getName}' with manifest: '$manifest' from protobuf ")
  }
}