java.io.DataInput Scala Examples

package com.stripe.bonsai
package layout

import java.io.{ DataOutput, DataInput }

class ProductLayout[A, B, C](
  leftLayout: Layout[A],
  rightLayout: Layout[B],
  unpack: C => (A, B),
  pack: (A, B) => C
) extends Layout[C] {
  def newBuilder: ProductBuilder[A, B, C] =
    new ProductBuilder[A, B, C](
      leftLayout.newBuilder,
      rightLayout.newBuilder,
      unpack, pack
    )

  def isSafeToCast(vec: Vec[_]): Boolean = vec match {
    case ProductVec(left, right, _) =>
      leftLayout.isSafeToCast(left) && rightLayout.isSafeToCast(right)
    case _ =>
      false
  }

  def write(vec: Vec[C], out: DataOutput): Unit = {
    val ProductVec(left, right, _) = recast(vec)
    out.writeByte(ProductLayout.SplitEncoding)
    leftLayout.write(left, out)
    rightLayout.write(right, out)
  }

  def read(in: DataInput): Vec[C] = {
    in.readByte() match {
      case ProductLayout.SplitEncoding =>
        val left = leftLayout.read(in)
        val right = rightLayout.read(in)
        ProductVec(left, right, pack)

      case _ =>
        throw new java.io.IOException("unsupported encoding for product2 layout")
    }
  }

  private def recast(vec: Vec[C]): ProductVec[A, B, C] = {
    if (isSafeToCast(vec)) {
      vec.asInstanceOf[ProductVec[A, B, C]]
    } else {
      (newBuilder ++= vec).result()
    }
  }
}

object ProductLayout {
  final val SplitEncoding = 1.toByte
}

class ProductBuilder[A, B, C](
  leftBldr: VecBuilder[A],
  rightBldr: VecBuilder[B],
  unpack: C => (A, B),
  pack: (A, B) => C
) extends VecBuilder[C] {
  def +=(that: C) = {
    val (a, b) = unpack(that)
    leftBldr += a
    rightBldr += b
    this
  }

  def clear(): Unit = {
    leftBldr.clear()
    rightBldr.clear()
  }

  def result(): ProductVec[A, B, C] =
    new ProductVec[A, B, C](leftBldr.result(), rightBldr.result(), pack)
}

case class ProductVec[A, B, C](
  left: Vec[A],
  right: Vec[B],
  pack: (A, B) => C
) extends Vec[C] {
  def size: Int = left.size
  def apply(index: Int): C = pack(left(index), right(index))
} 

package com.stripe.bonsai
package layout

import java.io.{ DataOutput, DataInput }
import scala.collection.IterableLike
import scala.collection.generic.{ CanBuildFrom, IsTraversableLike }

case class ColLayout[A, Repr <: IterableLike[A, Repr]](
  layout: Layout[A],
  offsetsLayout: Layout[Int]
)(implicit
  cbf: CanBuildFrom[Nothing, A, Repr]
) extends Layout[Repr] {
  def newBuilder: VecBuilder[Repr] =
    new DenseBuilder(Vector.newBuilder[Repr], Vec.fromVector)

  def isSafeToCast(vec: Vec[_]): Boolean = false

  def write(vec: Vec[Repr], out: DataOutput): Unit = {
    out.writeByte(ColLayout.OffsetEncoding)
    val offsetsBldr = offsetsLayout.newBuilder
    val bldr = layout.newBuilder
    var offset = 0
    vec.foreach { values =>
      offsetsBldr += offset
      offset += values.size
      bldr ++= values
    }
    offsetsLayout.write(offsetsBldr.result(), out)
    layout.write(bldr.result(), out)
  }

  def read(in: DataInput): Vec[Repr] =
    in.readByte() match {
      case ColLayout.OffsetEncoding =>
        val offsets = offsetsLayout.read(in)
        val values = layout.read(in)
        ColVec(offsets, values)(cbf)

      case e =>
        throw new java.io.IOException(s"unsupported encoding for ColLayout: $e")
    }
}

object ColLayout {
  final val OffsetEncoding = 1.toByte
} 

package com.stripe.bonsai
package layout

import java.io.{ DataOutput, DataInput }

class Product3Layout[A, B, C, D](
  layoutA: Layout[A],
  layoutB: Layout[B],
  layoutC: Layout[C],
  unpack: D => (A, B, C),
  pack: (A, B, C) => D
) extends Layout[D] {
  def newBuilder: Product3Builder[A, B, C, D] =
    new Product3Builder[A, B, C, D](
      layoutA.newBuilder,
      layoutB.newBuilder,
      layoutC.newBuilder,
      unpack, pack
    )

  def isSafeToCast(vec: Vec[_]): Boolean = vec match {
    case Product3Vec(a, b, c, _) =>
      layoutA.isSafeToCast(a) && layoutB.isSafeToCast(b) && layoutC.isSafeToCast(c)
    case _ =>
      false
  }

  def write(vec: Vec[D], out: DataOutput): Unit = {
    val Product3Vec(as, bs, cs, _) = recast(vec)
    out.writeByte(Product3Layout.SplitEncoding)
    layoutA.write(as, out)
    layoutB.write(bs, out)
    layoutC.write(cs, out)
  }

  def read(in: DataInput): Vec[D] = {
    in.readByte() match {
      case Product3Layout.SplitEncoding =>
        val as = layoutA.read(in)
        val bs = layoutB.read(in)
        val cs = layoutC.read(in)
        Product3Vec(as, bs, cs, pack)

      case _ =>
        throw new java.io.IOException("unsupported encoding for product3 layout")
    }
  }

  private def recast(vec: Vec[D]): Product3Vec[A, B, C, D] = {
    if (isSafeToCast(vec)) {
      vec.asInstanceOf[Product3Vec[A, B, C, D]]
    } else {
      (newBuilder ++= vec).result()
    }
  }
}

object Product3Layout {
  final val SplitEncoding = 1.toByte
}

class Product3Builder[A, B, C, D](
  aBldr: VecBuilder[A],
  bBldr: VecBuilder[B],
  cBldr: VecBuilder[C],
  unpack: D => (A, B, C),
  pack: (A, B, C) => D
) extends VecBuilder[D] {
  def +=(that: D) = {
    val (a, b, c) = unpack(that)
    aBldr += a
    bBldr += b
    cBldr += c
    this
  }

  def clear(): Unit = {
    aBldr.clear()
    bBldr.clear()
    cBldr.clear()
  }

  def result(): Product3Vec[A, B, C, D] =
    new Product3Vec[A, B, C, D](aBldr.result(), bBldr.result(), cBldr.result(), pack)
}

case class Product3Vec[A, B, C, D](
  as: Vec[A],
  bs: Vec[B],
  cs: Vec[C],
  pack: (A, B, C) => D
) extends Vec[D] {
  def size: Int = as.size
  def apply(index: Int): D = pack(as(index), bs(index), cs(index))
} 

package com.stripe.bonsai
package layout

import java.io.{ DataOutput, DataInput }

case class OptionalLayout[A](layout: Layout[A]) extends Layout[Option[A]] {
  def newBuilder: OptionalBuilder[A] =
    new OptionalBuilder[A](layout.newBuilder)

  def write(vec: Vec[Option[A]], out: DataOutput): Unit = {
    val OptionalVec(bitset, underlying) = recast(vec)
    out.writeByte(OptionalLayout.BitSetEncoding)
    layout.write(underlying, out)
    out.writeInt(bitset.length)
    IndexedBitSet.write(bitset, out)
  }

  def read(in: DataInput): Vec[Option[A]] = {
    in.readByte() match {
      case DisjunctionLayout.SplitEncoding =>
        val underlying = layout.read(in)
        val length = in.readInt()
        val bitset = IndexedBitSet.read(in)
        OptionalVec(bitset, underlying)

      case _ =>
        throw new java.io.IOException("unsupported encoding for optional layout")
    }
  }

  def isSafeToCast(vec: Vec[_]): Boolean = vec match {
    case OptionalVec(_, underlying) =>
      layout.isSafeToCast(underlying)
    case _ =>
      false
  }

  private def recast(vec: Vec[Option[A]]): OptionalVec[A] = {
    if (isSafeToCast(vec)) {
      vec.asInstanceOf[OptionalVec[A]]
    } else {
      (newBuilder ++= vec).result()
    }
  }
}

object OptionalLayout {
  final val BitSetEncoding = 1.toByte
}

class OptionalBuilder[A](bldr: VecBuilder[A]) extends VecBuilder[Option[A]] {
  val bitsetBldr = IndexedBitSet.newBuilder

  def +=(opt: Option[A]) = {
    opt match {
      case Some(a) =>
        bldr += a
        bitsetBldr += true

      case None => 
        bitsetBldr += false
    }
    this
  }

  def clear(): Unit = {
    bitsetBldr.clear()
    bldr.clear()
  }

  def result(): OptionalVec[A] =
    new OptionalVec[A](bitsetBldr.result(), bldr.result())
}

case class OptionalVec[A](bitset: IndexedBitSet, vec: Vec[A]) extends Vec[Option[A]] {
  def size: Int = bitset.length
  def apply(index: Int): Option[A] =
    if (bitset(index)) Some(vec(bitset.rank(index) - 1))
    else None
} 

package com.stripe.bonsai
package layout

import java.io.{ DataOutput, DataInput }

class TransformedLayout[A, B](
  layout: Layout[A],
  prepare: B => A,
  present: A => B
) extends Layout[B] {
  def newBuilder: TransformedBuilder[A, B] =
    new TransformedBuilder[A, B](
      layout.newBuilder,
      prepare, present
    )

  def write(vec: Vec[B], out: DataOutput): Unit = {
    val MappedVec(underlying, _) = recast(vec)
    layout.write(underlying, out)
  }

  def read(in: DataInput): Vec[B] = {
    val underlying = layout.read(in)
    MappedVec(underlying, present)
  }

  def isSafeToCast(vec: Vec[_]): Boolean = vec match {
    case MappedVec(underlying, _) =>
      layout.isSafeToCast(underlying)
    case _ =>
      false
  }

  private def recast(vec: Vec[B]): MappedVec[A, B] = {
    if (isSafeToCast(vec)) {
      vec.asInstanceOf[MappedVec[A, B]]
    } else {
      (newBuilder ++= vec).result()
    }
  }
}

class TransformedBuilder[A, B](
  bldr: VecBuilder[A],
  prepare: B => A,
  present: A => B
) extends VecBuilder[B] {
  def +=(that: B) = {
    bldr += prepare(that)
    this
  }

  def clear(): Unit = bldr.clear()

  def result(): MappedVec[A, B] =
    MappedVec(bldr.result(), present)
} 

package io.delta.hive

import java.io.{DataInput, DataOutput}

import org.apache.hadoop.io.Writable


case class PartitionColumnInfo(
    var index: Int,
    var tpe: String,
    var value: String) extends Writable {

  def this() {
    this(0, null, null)
  }

  override def write(out: DataOutput): Unit = {
    out.writeInt(index)
    out.writeUTF(tpe)
    out.writeUTF(value)
  }

  override def readFields(in: DataInput): Unit = {
    index = in.readInt()
    tpe = in.readUTF()
    value = in.readUTF()
  }
} 

package magellan.io

import java.io.{DataInput, DataOutput}

import magellan.Shape
import org.apache.commons.io.EndianUtils
import org.apache.hadoop.io.Writable

private[magellan] class ShapeWritable extends Writable {

  var shape: Shape = _

  override def write(dataOutput: DataOutput): Unit = {
    ???
  }

  override def readFields(dataInput: DataInput): Unit = {
    val shapeType = EndianUtils.swapInteger(dataInput.readInt())
    val h = shapeType match {
      case 0 => new NullShapeReader()
      case 1 => new PointReader()
      case 3 => new PolyLineReader()
      case 5 => new PolygonReader()
      case 13 => new PolyLineZReader()
      case _ => ???
    }
    shape = h.readFields(dataInput)
  }

} 

package magellan.io

import org.apache.spark.SerializableWritable
import java.io.{DataInput, DataOutput, ByteArrayOutputStream}
import org.apache.hadoop.io.{Writable, Text, FloatWritable, MapWritable, ArrayWritable}
import magellan.{Shape, Point}
import collection.JavaConversions._

case class OsmKey(val shapeType: String, val id: String) extends Serializable { }

abstract class OsmShape(val id: String, val tags: Map[String, String]) extends Serializable { }

case class OsmNode(
    override val id: String,
    val lat: Double,
    val lon: Double,
    override val tags: Map[String, String])
  extends OsmShape(id, tags) {
  
  def point: Point = Point(lon, lat)
}

case class OsmWay(
    override val id: String,
    val nodeIds: Seq[String],
    override val tags: Map[String, String])
  extends OsmShape(id, tags) { }

case class OsmRelation(
    override val id: String,
    val wayIds: Seq[String],
    override val tags: Map[String, String])
  extends OsmShape(id, tags) { } 

package org.apache.gearpump.streaming

import java.io.{DataInput, DataOutput}

import org.apache.gearpump.streaming.task._

class TaskIdSerializer extends TaskMessageSerializer[TaskId] {
  override def getLength(obj: TaskId): Int = 8

  override def write(dataOutput: DataOutput, obj: TaskId): Unit = {
    dataOutput.writeInt(obj.processorId)
    dataOutput.writeInt(obj.index)
  }

  override def read(dataInput: DataInput): TaskId = {
    val processorId = dataInput.readInt()
    val index = dataInput.readInt()
    new TaskId(processorId, index)
  }
}

class AckSerializer extends TaskMessageSerializer[Ack] {
  val taskIdSerializer = new TaskIdSerializer

  override def getLength(obj: Ack): Int = taskIdSerializer.getLength(obj.taskId) + 16

  override def write(dataOutput: DataOutput, obj: Ack): Unit = {
    taskIdSerializer.write(dataOutput, obj.taskId)
    dataOutput.writeShort(obj.seq)
    dataOutput.writeShort(obj.actualReceivedNum)
    dataOutput.writeInt(obj.sessionId)
    dataOutput.writeLong(obj.watermark)
  }

  override def read(dataInput: DataInput): Ack = {
    val taskId = taskIdSerializer.read(dataInput)
    val seq = dataInput.readShort()
    val actualReceivedNum = dataInput.readShort()
    val sessionId = dataInput.readInt()
    val watermark = dataInput.readLong()
    Ack(taskId, seq, actualReceivedNum, sessionId, watermark)
  }
}

class InitialAckRequestSerializer extends TaskMessageSerializer[InitialAckRequest] {
  val taskIdSerialzer = new TaskIdSerializer()

  override def getLength(obj: InitialAckRequest): Int = taskIdSerialzer.getLength(obj.taskId) + 4

  override def write(dataOutput: DataOutput, obj: InitialAckRequest): Unit = {
    taskIdSerialzer.write(dataOutput, obj.taskId)
    dataOutput.writeInt(obj.sessionId)
  }

  override def read(dataInput: DataInput): InitialAckRequest = {
    val taskId = taskIdSerialzer.read(dataInput)
    val sessionId = dataInput.readInt()
    InitialAckRequest(taskId, sessionId)
  }
}

class AckRequestSerializer extends TaskMessageSerializer[AckRequest] {
  val taskIdSerializer = new TaskIdSerializer

  override def getLength(obj: AckRequest): Int = taskIdSerializer.getLength(obj.taskId) + 14

  override def write(dataOutput: DataOutput, obj: AckRequest): Unit = {
    taskIdSerializer.write(dataOutput, obj.taskId)
    dataOutput.writeShort(obj.seq)
    dataOutput.writeInt(obj.sessionId)
    dataOutput.writeLong(obj.watermark)
  }

  override def read(dataInput: DataInput): AckRequest = {
    val taskId = taskIdSerializer.read(dataInput)
    val seq = dataInput.readShort()
    val sessionId = dataInput.readInt()
    val watermark = dataInput.readLong()
    AckRequest(taskId, seq, sessionId, watermark)
  }
}

class LatencyProbeSerializer extends TaskMessageSerializer[LatencyProbe] {
  override def getLength(obj: LatencyProbe): Int = 8

  override def write(dataOutput: DataOutput, obj: LatencyProbe): Unit = {
    dataOutput.writeLong(obj.timestamp)
  }

  override def read(dataInput: DataInput): LatencyProbe = {
    val timestamp = dataInput.readLong()
    LatencyProbe(timestamp)
  }
} 

package org.apache.gearpump.streaming.task

import java.io.{DataInput, DataOutput}

import org.apache.gearpump.Time.MilliSeconds

case class SerializedMessage(timeStamp: MilliSeconds, bytes: Array[Byte])

class SerializedMessageSerializer extends TaskMessageSerializer[SerializedMessage] {
  override def getLength(obj: SerializedMessage): Int = 12 + obj.bytes.length

  override def write(dataOutput: DataOutput, obj: SerializedMessage): Unit = {
    dataOutput.writeLong(obj.timeStamp)
    dataOutput.writeInt(obj.bytes.length)
    dataOutput.write(obj.bytes)
  }

  override def read(dataInput: DataInput): SerializedMessage = {
    val timestamp = dataInput.readLong()
    val length = dataInput.readInt()
    val bytes = new Array[Byte](length)
    dataInput.readFully(bytes)
    SerializedMessage(timestamp, bytes)
  }
} 

package org.apache.gearpump.streaming.task

import java.io.{DataInput, DataOutput}

import org.slf4j.Logger

import org.apache.gearpump.streaming.{AckRequestSerializer, AckSerializer, InitialAckRequestSerializer, LatencyProbeSerializer}
import org.apache.gearpump.transport.netty.ITransportMessageSerializer
import org.apache.gearpump.util.LogUtil

class StreamingTransportSerializer extends ITransportMessageSerializer {
  private val log: Logger = LogUtil.getLogger(getClass)
  private val serializers = new SerializerResolver

  serializers.register(classOf[Ack], new AckSerializer)
  serializers.register(classOf[AckRequest], new AckRequestSerializer)
  serializers.register(classOf[InitialAckRequest], new InitialAckRequestSerializer)
  serializers.register(classOf[LatencyProbe], new LatencyProbeSerializer)
  serializers.register(classOf[SerializedMessage], new SerializedMessageSerializer)

  override def serialize(dataOutput: DataOutput, obj: Object): Unit = {
    val registration = serializers.getRegistration(obj.getClass)
    if (registration != null) {
      dataOutput.writeInt(registration.id)
      registration.serializer.asInstanceOf[TaskMessageSerializer[AnyRef]].write(dataOutput, obj)
    } else {
      log.error(s"Can not find serializer for class type ${obj.getClass}")
    }
  }

  override def deserialize(dataInput: DataInput, length: Int): Object = {
    val classID = dataInput.readInt()
    val registration = serializers.getRegistration(classID)
    if (registration != null) {
      registration.serializer.asInstanceOf[TaskMessageSerializer[AnyRef]].read(dataInput)
    } else {
      log.error(s"Can not find serializer for class id $classID")
      null
    }
  }

  override def getLength(obj: Object): Int = {
    val registration = serializers.getRegistration(obj.getClass)
    if (registration != null) {
      registration.serializer.asInstanceOf[TaskMessageSerializer[AnyRef]].getLength(obj) + 4
    } else {
      log.error(s"Can not find serializer for class type ${obj.getClass}")
      0
    }
  }
} 

package org.apache.gearpump.transport

import java.io.{DataInput, DataOutput}
import org.apache.gearpump.transport.MockTransportSerializer.NettyMessage
import org.apache.gearpump.transport.netty.ITransportMessageSerializer

class MockTransportSerializer extends ITransportMessageSerializer {
  override def getLength(obj: scala.Any): Int = 4

  override def serialize(dataOutput: DataOutput, transportMessage: scala.Any): Unit = {
    transportMessage match {
      case msg: NettyMessage =>
        dataOutput.writeInt(msg.num)
    }
  }

  override def deserialize(dataInput: DataInput, length: Int): AnyRef = {
    NettyMessage(dataInput.readInt())
  }
}

object MockTransportSerializer {
  case class NettyMessage(num: Int)
} 

package com.esri.gdb

import java.io.{DataInput, File}
import java.nio.{ByteBuffer, ByteOrder}

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FSDataInputStream, Path}
import org.apache.spark.Logging

object GDBIndex {
  def apply(path: String, name: String, conf: Configuration = new Configuration()) = {
    val filename = StringBuilder.newBuilder.append(path).append(File.separator).append(name).append(".gdbtablx").toString()
    val hdfsPath = new Path(filename)
    val dataInput = hdfsPath.getFileSystem(conf).open(hdfsPath)

    val bytes = new Array[Byte](16)
    dataInput.readFully(bytes)
    val byteBuffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)

    val signature = byteBuffer.getInt
    val n1024Blocks = byteBuffer.getInt
    val numRows = byteBuffer.getInt
    val indexSize = byteBuffer.getInt

    new GDBIndex(dataInput, numRows, indexSize)
  }
}

private[gdb] class GDBIndex(dataInput: FSDataInputStream,
                            val numRows: Int,
                            indexSize: Int
                           ) extends Logging with AutoCloseable with Serializable {

  def readSeekForRowNum(rowNum: Int) = {
    val bytes = new Array[Byte](indexSize)
    dataInput.seek(16 + rowNum * indexSize)
    dataInput.readFully(bytes)
    ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).getInt
  }

  def iterator(startAtRow: Int = 0, numRowsToRead: Int = -1) = {
    dataInput.seek(16 + startAtRow * indexSize)
    val maxRows = if (numRowsToRead == -1) numRows else numRowsToRead
    // log.info(s"iterator::startAtRow=$startAtRow maxRows=$maxRows")
    new GDBIndexIterator(dataInput, startAtRow, maxRows, indexSize).withFilter(_.isSeekable)
  }

  def close() {
    dataInput.close()
  }
}

private[gdb] class GDBIndexIterator(dataInput: DataInput,
                                    startID: Int,
                                    maxRows: Int,
                                    indexSize: Int
                                   ) extends Iterator[IndexInfo] with Logging with Serializable {

  private val indexInfo = IndexInfo(0, 0)
  private val bytes = new Array[Byte](indexSize)
  private val byteBuffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)

  private var objectID = startID
  private var nextRow = 0

  def hasNext() = nextRow < maxRows

  def next() = {
    // log.info(s"next::nextRow=$nextRow maxRows=$maxRows")
    nextRow += 1

    objectID += 1
    indexInfo.objectID = objectID

    byteBuffer.clear
    dataInput.readFully(bytes)
    indexInfo.seek = byteBuffer.getInt

    indexInfo
  }
}